diff --git a/.github/workflows/update-zstd.yml b/.github/workflows/update-zstd.yml deleted file mode 100644 index c19fc9092a51..000000000000 --- a/.github/workflows/update-zstd.yml +++ /dev/null @@ -1,106 +0,0 @@ -name: Update zstd - -permissions: - contents: read - -on: - schedule: - - cron: "0 1 * * 0" - workflow_dispatch: - -jobs: - check-update: - if: github.repository == 'oven-sh/bun' - runs-on: ubuntu-latest - permissions: - contents: write - pull-requests: write - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Check zstd version - id: check-version - run: | - set -euo pipefail - - DEP_FILE=scripts/build/deps/zstd.ts - CURRENT_VERSION=$(sed -nE 's/^const ZSTD_COMMIT = "([0-9a-f]{40})";$/\1/p' "$DEP_FILE") - - if [ -z "$CURRENT_VERSION" ]; then - echo "Error: Could not find ZSTD_COMMIT in $DEP_FILE" - exit 1 - fi - - # Validate that it looks like a git hash - if ! [[ $CURRENT_VERSION =~ ^[0-9a-f]{40}$ ]]; then - echo "Error: Invalid git hash format in $DEP_FILE" - echo "Found: $CURRENT_VERSION" - echo "Expected: 40 character hexadecimal string" - exit 1 - fi - - echo "current=$CURRENT_VERSION" >> $GITHUB_OUTPUT - - LATEST_RELEASE=$(curl -sL https://api.github.com/repos/facebook/zstd/releases/latest) - if [ -z "$LATEST_RELEASE" ]; then - echo "Error: Failed to fetch latest release from GitHub API" - exit 1 - fi - - LATEST_TAG=$(echo "$LATEST_RELEASE" | jq -r '.tag_name') - if [ -z "$LATEST_TAG" ] || [ "$LATEST_TAG" = "null" ]; then - echo "Error: Could not extract tag name from GitHub API response" - exit 1 - fi - - LATEST_TAG_SHA=$(curl -sL "https://api.github.com/repos/facebook/zstd/git/refs/tags/$LATEST_TAG" | jq -r '.object.sha') - if [ -z "$LATEST_TAG_SHA" ] || [ "$LATEST_TAG_SHA" = "null" ]; then - echo "Error: Could not fetch SHA for tag $LATEST_TAG" - exit 1 - fi - LATEST_SHA=$(curl -sL "https://api.github.com/repos/facebook/zstd/git/tags/$LATEST_TAG_SHA" | jq -r '.object.sha') - if [ -z "$LATEST_SHA" ] || [ "$LATEST_SHA" = "null" ]; then - echo "Error: Could not fetch SHA for tag $LATEST_TAG @ $LATEST_TAG_SHA" - exit 1 - fi - - if ! [[ $LATEST_SHA =~ ^[0-9a-f]{40}$ ]]; then - echo "Error: Invalid SHA format received from GitHub" - echo "Found: $LATEST_SHA" - echo "Expected: 40 character hexadecimal string" - exit 1 - fi - - echo "latest=$LATEST_SHA" >> $GITHUB_OUTPUT - echo "tag=$LATEST_TAG" >> $GITHUB_OUTPUT - - - name: Update version if needed - if: success() && steps.check-version.outputs.current != steps.check-version.outputs.latest - env: - LATEST: ${{ steps.check-version.outputs.latest }} - run: | - set -euo pipefail - sed -i -E 's/^(const ZSTD_COMMIT = ")[0-9a-f]{40}(";)$/\1'"$LATEST"'\2/' scripts/build/deps/zstd.ts - - - name: Create Pull Request - if: success() && steps.check-version.outputs.current != steps.check-version.outputs.latest - uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - add-paths: | - scripts/build/deps/zstd.ts - commit-message: "deps: update zstd to ${{ steps.check-version.outputs.tag }} (${{ steps.check-version.outputs.latest }})" - title: "deps: update zstd to ${{ steps.check-version.outputs.tag }}" - delete-branch: true - branch: deps/update-zstd - body: | - ## What does this PR do? - - Updates zstd to version ${{ steps.check-version.outputs.tag }} - - Compare: https://github.com/facebook/zstd/compare/${{ steps.check-version.outputs.current }}...${{ steps.check-version.outputs.latest }} - - Auto-updated by [this workflow](https://github.com/oven-sh/bun/actions/workflows/update-zstd.yml) diff --git a/.gitignore b/.gitignore index 18a643d7dc9d..44a9908999ef 100644 --- a/.gitignore +++ b/.gitignore @@ -169,7 +169,6 @@ codegen-for-zig-team.tar.gz /src/deps/WebKit /src/deps/zig /src/deps/zlib -/src/deps/zstd # Generated files diff --git a/CLAUDE.md b/CLAUDE.md index 8feccff501bc..81a19f9c2c31 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -147,7 +147,7 @@ The Rust side is a Cargo workspace of ~200 crates rooted at `Cargo.toml`. The ke #### Vendored Dependencies (`vendor/`) -Third-party native libraries are vendored locally and can be read from disk (not git submodules): boringssl (TLS/crypto), brotli, cares (async DNS), highway (SIMD), libdeflate, libuv (Windows event loop), lolhtml (HTML rewriter), lshpack (HTTP/2 HPACK), lsqpack + lsquic (HTTP/3), mimalloc (allocator), nodejs (headers), picohttpparser, rust-argon2 (argon2 for `Bun.password`, Rust), tinycc (FFI JIT, fork: oven-sh/tinycc), WebKit (JavaScriptCore), zlib (zlib-ng), zstd. Build configuration for these is in `scripts/build/deps/*.ts`. +Third-party native libraries are vendored locally and can be read from disk (not git submodules): boringssl (TLS/crypto), brotli, cares (async DNS), highway (SIMD), libdeflate, libuv (Windows event loop), lolhtml (HTML rewriter), lshpack (HTTP/2 HPACK), lsqpack + lsquic (HTTP/3), mimalloc (allocator), nodejs (headers), picohttpparser, rust-argon2 (argon2 for `Bun.password`, Rust), tinycc (FFI JIT, fork: oven-sh/tinycc), WebKit (JavaScriptCore), zlib (zlib-ng). Build configuration for these is in `scripts/build/deps/*.ts`. ### JavaScript Class Implementation (C++) diff --git a/Cargo.lock b/Cargo.lock index 93d0ec1d51cc..d87438fef4cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1808,7 +1808,6 @@ dependencies = [ "bun_sys", "bun_threading", "bun_zstd", - "cc", "libc", ] @@ -2257,9 +2256,9 @@ dependencies = [ name = "bun_zstd" version = "0.0.0" dependencies = [ - "bun_core", - "bun_opaque", + "bun_pod", "strum", + "thiserror", ] [[package]] @@ -2274,16 +2273,6 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" -[[package]] -name = "cc" -version = "1.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3eb0f42d6c360dc3f8a821f6bf2fdea7f72bfd36b3076eb0e6d1e9e0752fff4" -dependencies = [ - "find-msvc-tools", - "shlex", -] - [[package]] name = "cfg-if" version = "1.0.4" @@ -2647,12 +2636,6 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" -[[package]] -name = "find-msvc-tools" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" - [[package]] name = "fnv" version = "1.0.7" @@ -3286,12 +3269,6 @@ dependencies = [ "keccak", ] -[[package]] -name = "shlex" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" - [[package]] name = "siphasher" version = "1.0.3" diff --git a/Cargo.toml b/Cargo.toml index 700ef89cb5d0..5effadc2b2ad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -514,6 +514,11 @@ bun_simdutf_sys = { path = "src/simdutf_sys" } bun_which = { path = "src/which" } bun_transpiler = { path = "src/transpiler" } +# At opt-level 0 a codec is slow enough for tests to time out. The C library that this replaces was optimized in debug +# builds too. +[profile.dev.package.bun_zstd] +opt-level = 3 + # Loops over every byte of an image: unoptimized, a debug build takes seconds per megapixel. Assertions and overflow # checks stay on. scripts/build/flags.ts does the same for highway_png.cpp. [profile.dev.package.bun_png] diff --git a/LICENSE.md b/LICENSE.md index 6abe6776e993..4ac35773f2e1 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -29,7 +29,7 @@ Bun statically links these libraries: | [`lsquic`](https://github.com/litespeedtech/lsquic) | MIT (portions derived from [Chromium proto-quic](https://github.com/litespeedtech/lsquic/blob/master/LICENSE.chrome), BSD 3-Clause) | | [`mimalloc`](https://github.com/microsoft/mimalloc) | MIT | | [`picohttp`](https://github.com/h2o/picohttpparser) | dual-licensed under the Perl License or the MIT License | -| [`zstd`](https://github.com/facebook/zstd) | dual-licensed under the BSD License or GPLv2 license | +| [`zstd`](https://github.com/facebook/zstd) (not linked: `src/zstd` is a port of it) | dual-licensed under the BSD License or GPLv2 license | | [`simdutf`](https://github.com/simdutf/simdutf) | Apache 2.0 | | [`tinycc`](https://github.com/tinycc/tinycc) | LGPL v2.1 | | [`uSockets`](https://github.com/uNetworking/uSockets) | Apache 2.0 | diff --git a/patches/zstd/bmi2-probe-once.patch b/patches/zstd/bmi2-probe-once.patch deleted file mode 100644 index 603ccfe44678..000000000000 --- a/patches/zstd/bmi2-probe-once.patch +++ /dev/null @@ -1,37 +0,0 @@ -Probe BMI2 support once instead of once per context. - -ZSTD_cpuSupportsBmi2() runs on every CCtx and DCtx init when DYNAMIC_BMI2 is -on, which it is for Bun's x64 builds (they target nehalem, so __BMI2__ is not -defined). It executes CPUID twice. Under a hypervisor each CPUID is a VM exit -of about 2 us, so a create + free of a DCtx costs 3.6 us instead of 30 ns, and -a one-shot ZSTD_decompress() of 1 KB costs 3.9 us instead of 0.4 us. - -facebook/zstd#4767 removed the per-context probe for Linux kernel builds only. -Userspace builds still run it. - ---- a/lib/common/zstd_internal.h -+++ b/lib/common/zstd_internal.h -@@ -314,11 +314,21 @@ - - /** - * @returns true iff the CPU supports dynamic BMI2 dispatch. -+ * -+ * Bun: every CCtx and DCtx init calls this, and CPUID is a VM exit under a -+ * hypervisor (about 2 us each, two per call). The answer cannot change, so -+ * probe once per translation unit. 0: not probed yet, 1: no BMI2, 2: BMI2. - */ - MEM_STATIC int ZSTD_cpuSupportsBmi2(void) - { -- ZSTD_cpuid_t cpuid = ZSTD_cpuid(); -- return ZSTD_cpuid_bmi1(cpuid) && ZSTD_cpuid_bmi2(cpuid); -+ static int cached = 0; -+ int state = __atomic_load_n(&cached, __ATOMIC_SEQ_CST); -+ if (state == 0) { -+ ZSTD_cpuid_t const cpuid = ZSTD_cpuid(); -+ state = 1 + (ZSTD_cpuid_bmi1(cpuid) && ZSTD_cpuid_bmi2(cpuid)); -+ __atomic_store_n(&cached, state, __ATOMIC_SEQ_CST); -+ } -+ return state - 1; - } - - #endif /* ZSTD_CCOMMON_H_MODULE */ diff --git a/patches/zstd/free-workers-before-dictionaries.patch b/patches/zstd/free-workers-before-dictionaries.patch deleted file mode 100644 index c954c98e7d61..000000000000 --- a/patches/zstd/free-workers-before-dictionaries.patch +++ /dev/null @@ -1,28 +0,0 @@ -Free the worker threads of a context before the dictionaries they read. - -ZSTD_freeCCtxContent() calls ZSTD_clearAllDicts() and then ZSTDMT_freeCCtx(). -With nbWorkers >= 1, ZSTD_compressStream2() posts a job to a worker thread and -returns. The first job of a frame reads the dictionary of the context through -a pointer that it does not own (job->cdict). A job stops only in -ZSTDMT_freeCCtx(), which joins the threads. So ZSTD_freeCCtx() of a context that -has a dictionary and a running job frees the dictionary under that job. - -ZSTD_CCtx_reset() with parameters, ZSTD_CCtx_loadDictionary(), ZSTD_CCtx_refCDict() -and ZSTD_CCtx_refPrefix() free the dictionaries too. Bun does not call them on a -context whose jobs still run. - ---- a/lib/compress/zstd_compress.c -+++ b/lib/compress/zstd_compress.c -@@ -171,10 +171,11 @@ - { - assert(cctx != NULL); - assert(cctx->staticSize == 0); -- ZSTD_clearAllDicts(cctx); - #ifdef ZSTD_MULTITHREAD - ZSTDMT_freeCCtx(cctx->mtctx); cctx->mtctx = NULL; - #endif -+ /* Bun: after the join above. A job that still runs reads these dictionaries. */ -+ ZSTD_clearAllDicts(cctx); - ZSTD_cwksp_free(&cctx->workspace, cctx->customMem); - } - diff --git a/scripts/build/CLAUDE.md b/scripts/build/CLAUDE.md index 1b2eca70ddc9..8481d0163352 100644 --- a/scripts/build/CLAUDE.md +++ b/scripts/build/CLAUDE.md @@ -22,7 +22,7 @@ This directory generates `build.ninja`. The scripts **describe** the build; ninj **Deps in our graph.** `BuildSpec` variants: -- `direct` — list the dep's sources explicitly; each becomes a first-class `cc`/`cxx` edge in our graph and the `.o`s go straight into bun's link. Every C/C++ dep that is compiled (zlib, zstd, boringssl, mimalloc, … and a local WebKit checkout). Skips a sub-process configure entirely and lets LTO see across the dep boundary. `groups` are further flag sets (a cmake "target" each), `steps` are generators and target executables. +- `direct` — list the dep's sources explicitly; each becomes a first-class `cc`/`cxx` edge in our graph and the `.o`s go straight into bun's link. Every C/C++ dep that is compiled (zlib, boringssl, mimalloc, … and a local WebKit checkout). Skips a sub-process configure entirely and lets LTO see across the dep boundary. `groups` are further flag sets (a cmake "target" each), `steps` are generators and target executables. - `cargo` — (rule kept for out-of-workspace cargo deps; none today — lolhtml and rust-argon2 are path dependencies compiled as units of bun's own Rust graph). - `prebuilt` — skip build entirely, download compiled `.a`/`.lib` (WebKit, nodejs-headers). @@ -48,7 +48,7 @@ rule cc depfile = $out.d deps = gcc -build obj/src/foo.c.o: cc ../../src/foo.c | deps/zstd/libzstd.a || codegen/generated.h +build obj/src/foo.c.o: cc ../../src/foo.c | deps/cares/libcares.a || codegen/generated.h cflags = -O2 -I... ``` @@ -60,9 +60,9 @@ rule dep_fetch restat = 1 pool = dep -build ../../vendor/zstd/.ref: dep_fetch | ../../scripts/build/fetch-cli.ts - name = zstd - repo = facebook/zstd +build ../../vendor/zlib/.ref: dep_fetch | ../../scripts/build/fetch-cli.ts + name = zlib + repo = zlib-ng/zlib-ng commit = abc123... ``` @@ -174,7 +174,7 @@ For `mode: "full"` (the normal case): 1. **Codegen** — `emitCodegen(n, cfg, sources, engine)` emits ~20 generation steps (bindgen, bundled modules, LUTs). Returns grouped outputs. 2. **Rust** — `emitRust(n, cfg, {...})` emits one rustc edge per crate and returns the crates' rlibs, `bun_runtime`'s and std's included, which the link takes beside the C/C++ objects (after resolving the vendored path deps, lolhtml and rust-argon2). cargo plans, ninja executes: the `rust_plan` edge runs `cargo build … --unit-graph` + `cargo metadata` for exactly the arguments `cargoBuildInvocation()` computes and writes `rust/plan.json`; `build.ninja` depends on that file, so a new plan (lockfile/manifest/toolchain change) reconfigures and ninja restarts. `rust/units.ts` turns each unit into a rustc argv/env (cargo's rules, transcribed and diffed against `cargo -vv`), written to `rust/units/-.json`; `rust/emit.ts` emits the edges; `rust/run.ts` executes them. Every edge is one process from start to exit, and a crate is one edge: one rustc with the `.rlib` and the `.rmeta` as outputs. Dependent libraries name only the `.rmeta`, which rustc writes long before it has generated code; the edge carries `early_output_prefix`, which asks ninja to release an output when the running command announces it. oven-sh/ninja (what the driver runs, see `ninja-release.ts`) exports the prefix to the command as `NINJA_EARLY_OUTPUT_PREFIX`, `rust/run.ts` prints it with the `.rmeta`'s path when rustc reports the file written, and dependents start while rustc goes on — cargo's pipelining. A stock ninja ignores the binding and releases both outputs at exit: the same graph, built correctly, without the overlap. Build scripts are a compile edge plus a `rust_build_script` run edge whose parsed directives (`output.json`, restat) feed the package's rustc edges. A Windows target has a second graph built the same way, under `rust/shim/`: the `.bin/` launcher (`src/install/windows-shim`), planned with `--profile shim` and `-Zbuild-std`, with a `bin` root that `run.ts` also writes to `/bun-shim-impl.exe`, where `bun_install` embeds it from (every edge of that package waits for it). Codegen and Rust are emitted before the deps on purpose: with no `.ninja_log` (every CI build) ninja weighs each edge as 1 and runs the longest remaining chain first, ties in emission order. -3. **Deps** — loop `allDeps`, call `resolveDep(n, cfg, dep)`. Each emits fetch → direct cc/cxx edges, or fetch → cargo, or prebuilt download. Collects objects, lib paths, include dirs, outputs. +3. **Deps** — loop `allDeps`, call `resolveDep(n, cfg, dep)`. Each emits fetch → direct cc/cxx edges, or fetch → cargo, or prebuilt download. Collects objects, lib paths, include dirs, outputs. `programDeps` are resolved the same way, but nothing of them is collected for bun: only the programs beside it that name them take them. 4. **Flags** — `computeFlags(cfg)` evaluates flag tables → cflags/cxxflags/defines/ldflags/stripflags. 5. **PCH** — compile `root-pch.h` → PCH. 6. **Compile** — loop sources, `cxx()`/`cc()` per file. @@ -259,7 +259,7 @@ It is configured by `configureCodegen()`, not `configure()`, and resolves a `Cod | `glob-sources.ts` (parent dir) | Source glob patterns + CLI to print them | | `ci-images/spec.ts` | CI's machines in one file: images, version pins, locations, every tool, and the generator of the bake scripts; `bun run ci:images` | | `deps/*.ts` | One `Dependency` object per vendored dep | -| `deps/index.ts` | `allDeps` array — fetch order + link order | +| `deps/index.ts` | `allDeps` array — fetch order + link order; `programDeps` | | `shims/*.c` | Platform workaround sources | | `trim/*.ts` | The trimmable link: section graph, partition by feature, layout, the trim table (`link.ts` is the entry) | diff --git a/scripts/build/bun.ts b/scripts/build/bun.ts index cd3061801d5d..532fc67b7a85 100644 --- a/scripts/build/bun.ts +++ b/scripts/build/bun.ts @@ -19,7 +19,7 @@ import { emitCodegen, type CodegenOutputs } from "./codegen.ts"; import { cc, cxx, link, pch, type PrecompiledHeader } from "./compile.ts"; import { bunExeName, shouldStrip, type Config } from "./config.ts"; import { generateDepVersionsHeader } from "./depVersionsHeader.ts"; -import { allDeps } from "./deps/index.ts"; +import { allDeps, programDeps } from "./deps/index.ts"; import { lolhtml } from "./deps/lolhtml.ts"; import { rustArgon2 } from "./deps/rust-argon2.ts"; import { jscPrograms, linuxIcuRoot, webkitEngineSources } from "./deps/webkit.ts"; @@ -197,6 +197,10 @@ export function emitBun(n: Ninja, cfg: Config, sources: Sources): BunOutput { depsByName.set(dep.name, resolved); } } + for (const dep of programDeps) { + const resolved = resolveDep(n, cfg, dep, depsByName); + if (resolved !== null) depsByName.set(dep.name, resolved); + } // Collect all dep lib paths, include dirs, output stamps, and directly- // compiled source files (deps like picohttpparser that provide .c files @@ -1138,7 +1142,7 @@ export function validateBunConfig(cfg: Config): void { // --local-deps names must match a dep — a typo would otherwise silently // build the pinned tarball while the banner claims `local:`. // Keyed by string: these names are the user's. - const depsByName = new Map(allDeps.map(d => [d.name, d])); + const depsByName = new Map([...allDeps, ...programDeps].map(d => [d.name, d])); for (const [name, path] of Object.entries(cfg.localDeps)) { const dep = depsByName.get(name); assert(dep !== undefined, `--local-deps: unknown dep '${name}'`, { diff --git a/scripts/build/clean.ts b/scripts/build/clean.ts index 5d31524245bc..325f2ca5511c 100644 --- a/scripts/build/clean.ts +++ b/scripts/build/clean.ts @@ -10,7 +10,7 @@ import { rm } from "node:fs/promises"; import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { sharedCacheDir as resolveSharedCacheDir } from "./config.ts"; -import { allDeps } from "./deps/index.ts"; +import { allDeps, programDeps } from "./deps/index.ts"; const cwd = resolve(dirname(fileURLToPath(import.meta.url)), "..", ".."); // Machine-shared cache (ccache/cargo/tarballs/webkit): resolveConfig()'s @@ -56,8 +56,8 @@ function buildProfiles(): string[] { const profile = (name: string) => () => [resolve(cwd, "build", name)]; // Deps whose vendor// dir is user-managed (manual clone, not fetched -// by the build system). `deep` skips these; everything else in allDeps gets -// its vendor dir nuked. +// by the build system). `deep` skips these; every other dep gets its vendor +// dir nuked. const userManagedDeps = new Set(["WebKit"]); const presets: Record string[]> = { @@ -81,7 +81,7 @@ const presets: Record string[]> = { deep: () => [ resolve(cwd, "build"), resolve(cwd, "target"), - ...allDeps.filter(d => !userManagedDeps.has(d.name)).map(d => resolve(cwd, "vendor", d.name)), + ...[...allDeps, ...programDeps].filter(d => !userManagedDeps.has(d.name)).map(d => resolve(cwd, "vendor", d.name)), ], }; diff --git a/scripts/build/compile.ts b/scripts/build/compile.ts index a36bc9385f91..4f254210f4d7 100644 --- a/scripts/build/compile.ts +++ b/scripts/build/compile.ts @@ -298,7 +298,7 @@ export function cxx(n: Ninja, cfg: Config, src: string, opts: CompileOpts): stri * * `.S` (preprocessed assembly) is also accepted — clang dispatches on the * extension and runs cpp + as. Used by deps that ship hand-tuned kernels - * (e.g. zstd's huf_decompress_amd64.S). + * (e.g. boringssl's aesni-x86_64-linux.S). */ export function cc(n: Ninja, cfg: Config, src: string, opts: Omit): string { const ext = extname(src); diff --git a/scripts/build/config.ts b/scripts/build/config.ts index c1199ec49b4f..f17a60580cca 100644 --- a/scripts/build/config.ts +++ b/scripts/build/config.ts @@ -1752,7 +1752,7 @@ export function findRepoRoot(): string { /** * Parse `--local-deps=name=path[,name=path...]` into name → absolute path. - * Names are checked against `allDeps` later (bun.ts) where the dep list is + * Names are checked against the dep lists later (bun.ts) where they are * in scope; here we only validate shape and resolve paths. */ function parseLocalDeps(spec: string | undefined, cwd: string): Record { diff --git a/scripts/build/deps/README.md b/scripts/build/deps/README.md index b5e33cc258cc..123dd2b89daf 100644 --- a/scripts/build/deps/README.md +++ b/scripts/build/deps/README.md @@ -8,7 +8,7 @@ libraries/headers it provides. 1. Copy `libdeflate.ts` (the simplest direct dep) to `.ts` 2. Fill in `name`, `repo`, `commit`, `sources`, `includes`, `provides.includes` -3. Add `import { } from "./.ts"` + entry in `allDeps` array in `index.ts` +3. Add `import { } from "./.ts"` + entry in `allDeps` array in `index.ts` (`programDeps` if only a program beside bun takes it, which then names it in its `JSCProgram.deps`) 4. Add the name to `DepName` in `../source.ts` 5. `bun run scripts/build/phase3-test.ts` to verify it builds @@ -28,7 +28,7 @@ Case-sensitive filesystems enforce this. ## Removing a dependency 1. Delete `.ts` -2. Remove from `allDeps` in `index.ts`, and from `DepName` in `../source.ts` +2. Remove from `allDeps` (or `programDeps`) in `index.ts`, and from `DepName` in `../source.ts` 3. If any other dep has `fetchDeps: [""]`, remove that reference ## Updating a commit diff --git a/scripts/build/deps/index.ts b/scripts/build/deps/index.ts index b2a63ca571fe..e7968111de33 100644 --- a/scripts/build/deps/index.ts +++ b/scripts/build/deps/index.ts @@ -34,7 +34,7 @@ import { zlib } from "./zlib.ts"; import { zstd } from "./zstd.ts"; /** - * All deps in dependency-resolution + link order. + * All deps of bun in dependency-resolution + link order. * * zlib FIRST — lsquic's fetchDeps reference it. * brotli libs in internal dep order (common last on link line). @@ -46,7 +46,6 @@ export const allDeps: readonly Dependency[] = [ nodejsHeaders, zlib, - zstd, brotli, libdeflate, // Image codecs. @@ -72,6 +71,13 @@ export const allDeps: readonly Dependency[] = [ webkit, ]; +/** + * What only programs beside bun take (`JSCProgram.sourcesFrom`). Fetched like + * the others, but no part of bun: not linked into it, no include dir of its + * sources, no entry of its versions. + */ +export const programDeps: readonly Dependency[] = [zstd]; + // Re-export individuals for direct import when needed. export { boringssl, diff --git a/scripts/build/deps/zstd.ts b/scripts/build/deps/zstd.ts index 351bb62c9fc9..09b8d2d09151 100644 --- a/scripts/build/deps/zstd.ts +++ b/scripts/build/deps/zstd.ts @@ -1,39 +1,23 @@ /** - * Zstandard — fast compression with a good ratio/speed tradeoff. Backs - * bun's install cache and the `zstd` Content-Encoding in fetch. + * The sources of Zstandard in C, for programs that are not bun: `jsc` and + * `testaot` where the ICU that they are linked with is that of a prebuilt + * WebKit. Its data has a zstd frame an item, and these programs compile zstd's + * decoder for themselves to read it (webkit.ts withZstdDecoder). * - * DirectBuild: globbed common/compress/decompress/dictBuilder. Legacy-format - * decoders (zstd v0.5-v0.7, pre-1.0) are NOT built — Bun never reads - * pre-1.0 frames (install cache, fetch Content-Encoding, and Bun.zstd all - * use the current format). The amd64 - * Huffman kernel ships as a .S file that clang assembles directly; on other - * targets ZSTD_DISABLE_ASM falls through to the C implementation. + * No other build fetches it, nothing is built here, and it is one of + * `programDeps`, not of `allDeps`: nothing of it reaches bun, whose zstd is + * src/zstd. */ -import type { Dependency, DirectBuild } from "../source.ts"; +import type { Dependency } from "../source.ts"; +import { jscPrograms } from "./webkit.ts"; const ZSTD_COMMIT = "f8745da6ff1ad1e7bab384bd1f9d742439278e99"; -// prettier-ignore -const SOURCES = [ - "common/debug", "common/entropy_common", "common/error_private", - "common/fse_decompress", "common/pool", "common/threading", "common/xxhash", - "common/zstd_common", - "compress/fse_compress", "compress/hist", "compress/huf_compress", - "compress/zstd_compress", "compress/zstd_compress_literals", - "compress/zstd_compress_sequences", "compress/zstd_compress_superblock", - "compress/zstd_double_fast", "compress/zstd_fast", "compress/zstd_lazy", - "compress/zstd_ldm", "compress/zstd_opt", "compress/zstd_preSplit", - "compress/zstdmt_compress", - "decompress/huf_decompress", "decompress/zstd_ddict", - "decompress/zstd_decompress", "decompress/zstd_decompress_block", - "dictBuilder/cover", "dictBuilder/divsufsort", "dictBuilder/fastcover", - "dictBuilder/zdict", -].map(s => `lib/${s}.c`); - export const zstd: Dependency = { name: "zstd", - versionMacro: "ZSTD_HASH", + + enabled: cfg => jscPrograms(cfg).some(program => program.sourcesFrom?.includes("zstd")), source: () => ({ kind: "github-archive", @@ -41,56 +25,10 @@ export const zstd: Dependency = { commit: ZSTD_COMMIT, }), - patches: [ - // x64 targets nehalem, so zstd picks its BMI2 kernels at run time and - // probes CPUID in every CCtx/DCtx init. CPUID is a VM exit under a - // hypervisor (about 2 us each, two per init). Probe once instead. - "patches/zstd/bmi2-probe-once.patch", - // ZSTD_freeCCtx frees a context's dictionaries before joining the worker - // threads, so a job that's still running can read freed memory. - "patches/zstd/free-workers-before-dictionaries.patch", - ], - - build: cfg => { - const sources = [...SOURCES]; - const defines: Record = { - ZSTD_MULTITHREAD: true, - ZSTD_LEGACY_SUPPORT: 0, - }; - - // Upstream's if(MSVC) block sets these for the static target. - // ZSTD_HEAPMODE=0 makes the one-shot ZSTD_decompress() (used by - // src/zstd/lib.rs) stack-allocate its DCtx instead of malloc/free - // per call; the source default is 1. - if (cfg.windows) { - defines.ZSTD_HEAPMODE = 0; - defines._CRT_SECURE_NO_WARNINGS = true; - } - - // huf_decompress_amd64.S is GNU-as syntax. clang assembles it on - // posix x64; clang-cl can't, so Windows takes the C path. - if (cfg.x64 && !cfg.windows) { - sources.push("lib/decompress/huf_decompress_amd64.S"); - } else { - defines.ZSTD_DISABLE_ASM = 1; - } - - const spec: DirectBuild = { - kind: "direct", - sources, - defines, - includes: ["lib", "lib/common"], - pic: true, - // XXH_NAMESPACE must be a bare token prefix (xxhash pastes it onto - // symbol names), not a string literal — DirectBuild.defines would - // wrap it in quotes. Avoids clashes with lshpack's copy. - cflags: ["-DXXH_NAMESPACE=ZSTD_"], - }; - return spec; - }, + build: () => ({ kind: "none" }), provides: () => ({ libs: [], - includes: ["lib"], + includes: [], }), }; diff --git a/scripts/build/flags.ts b/scripts/build/flags.ts index 36af7c7601f6..ca5e5aa7ba73 100644 --- a/scripts/build/flags.ts +++ b/scripts/build/flags.ts @@ -26,7 +26,6 @@ export const safeStackDependencies: readonly DepName[] = [ "libwebp", "lshpack", "zlib", - "zstd", ]; export function usesSafeStack(c: Pick): boolean { @@ -301,7 +300,7 @@ export const globalFlags: Flag[] = [ { // cmake's Release/RelWithDebInfo build types append this to // CMAKE__FLAGS_ automatically. Deps only see globalFlags, so - // it must be here — otherwise every assert() in zstd/boringssl/mimalloc/etc. + // it must be here — otherwise every assert() in zlib/boringssl/mimalloc/etc. // stays live in release. (bun's own NDEBUG in `defines` below is // redundant after this, but harmless.) flag: "-DNDEBUG", diff --git a/scripts/build/stream.ts b/scripts/build/stream.ts index a7e8faa3b7cb..a827ac00fefa 100644 --- a/scripts/build/stream.ts +++ b/scripts/build/stream.ts @@ -72,7 +72,7 @@ export const STREAM_FD = 3; /** * Produce a colored `[name] ` prefix. Color derived from a hash of the - * name — deterministic, so zstd is always the same shade across runs. + * name — deterministic, so zlib is always the same shade across runs. * * Palette: 12 ANSI 256-color codes from gold → green → cyan → blue → * purple. No reds/pinks (error connotation), no dark blues (illegible diff --git a/scripts/rust-miri.ts b/scripts/rust-miri.ts index af54e8b974ae..2785a1364f9d 100644 --- a/scripts/rust-miri.ts +++ b/scripts/rust-miri.ts @@ -42,6 +42,7 @@ const MIRI_CRATES = [ "bun_sort_by_number", "bun_ast", "bun_paths", + "bun_zstd", "bun_hash", "bun_base64", "bun_bundler", diff --git a/scripts/rust-test.ts b/scripts/rust-test.ts index 764415ec0a16..73850cb9c1d6 100755 --- a/scripts/rust-test.ts +++ b/scripts/rust-test.ts @@ -61,6 +61,7 @@ const crates = [ "bun_threading", "bun_trim", "bun_wyhash", + "bun_zstd", ]; // Not `spawnSync`: what runs beside it writes into pipes that this process empties. @@ -120,8 +121,8 @@ const extraArgs = process.argv.slice(2); const programs: Promise[] = []; const running = new Set(); -// The crates `include!()` generated sources, cargo cannot resolve the workspace until the vendored path dependencies -// exist, and the build script of bun_sema_standalone compiles zstd's decoder. Of WebKit the code generators are enough. +// The crates `include!()` generated sources, and cargo cannot resolve the workspace until the vendored path dependencies +// exist. Of WebKit the code generators are enough. await run("bun", ["install", "--frozen-lockfile"]); // They need nothing of what follows, which leaves most of the processors idle until cargo starts. if (bunUnderTest && !extraArgs.length) { @@ -133,10 +134,7 @@ if (bunUnderTest && !extraArgs.length) { await run( // A CI machine runs the build with the node of its image, and the build checks the version of what runs it. isBuildkite ? "node" : "bun", - [ - "scripts/build.ts", - ...["codegen", "clone-zstd", "clone-lolhtml", "clone-rust-argon2"].map(name => `--target=${name}`), - ], + ["scripts/build.ts", ...["codegen", "clone-lolhtml", "clone-rust-argon2"].map(name => `--target=${name}`)], { BUN_WEBKIT_GENERATORS_ONLY: "1", ...process.env }, ); diff --git a/scripts/verify-baseline-static/CLAUDE.md b/scripts/verify-baseline-static/CLAUDE.md index bb911b65fd15..b1e27a88c25e 100644 --- a/scripts/verify-baseline-static/CLAUDE.md +++ b/scripts/verify-baseline-static/CLAUDE.md @@ -159,7 +159,6 @@ table, an HWCAP test. Known patterns: | simdutf | `set_best()` — CPUID first call, cached atomic ptr | `vendor/` or WebKit's bundled copy | | Highway (Bun) | `BUN_HWY_DISPATCH` → `hwy::SupportedTargets()` | `src/jsc/bindings/highway_dispatch.h` | | BoringSSL | `OPENSSL_ia32cap_P` global, set at init | `vendor/boringssl/crypto/cpu_intel.c` | -| zstd | `ZSTD_cpuid()` | `vendor/zstd/lib/common/cpu.h` | | libdeflate | `libdeflate_init_x86_cpu_features()` / `HWCAP_ASIMDDP` | `vendor/libdeflate/lib/x86/cpu_features.c` | | Rust `memchr` | `is_x86_feature_detected!()` | (via lolhtml dep) | | compiler-rt outline-atomics (aarch64) | `__aarch64_have_lse_atomics` (= `AT_HWCAP & HWCAP_ATOMICS`) | compiler-rt builtin | diff --git a/scripts/verify-baseline-static/allowlist-x64.txt b/scripts/verify-baseline-static/allowlist-x64.txt index 509ee2685964..eed3e2834fb5 100644 --- a/scripts/verify-baseline-static/allowlist-x64.txt +++ b/scripts/verify-baseline-static/allowlist-x64.txt @@ -936,31 +936,6 @@ CRYPTO_rdrand [RDRAND] CRYPTO_rdrand_multiple8_buf [RDRAND] -# ---------------------------------------------------------------------------- -# zstd Huffman/FSE BMI2. Gate: ZSTD_cpuid(). -# (19 symbols) -# ---------------------------------------------------------------------------- -FSE_decompress_wksp_body_bmi2 [BMI2, LZCNT] -FSE_readNCount_body_bmi2 [BMI2, LZCNT] -HUF_compress1X_usingCTable_internal_bmi2 [BMI2] -HUF_decompress1X1_usingDTable_internal_bmi2 [BMI1, BMI2, LZCNT] -HUF_decompress1X2_usingDTable_internal_bmi2 [BMI1, BMI2, LZCNT] -HUF_decompress4X1_usingDTable_internal_bmi2 [BMI1, BMI2, LZCNT] -HUF_decompress4X1_usingDTable_internal_fast [BMI1, BMI2] -HUF_decompress4X1_usingDTable_internal_fast_asm_loop [BMI2] -HUF_decompress4X1_usingDTable_internal_fast_c_loop [BMI2] -HUF_decompress4X2_usingDTable_internal_bmi2 [BMI1, BMI2, LZCNT] -HUF_decompress4X2_usingDTable_internal_fast [BMI1, BMI2] -HUF_decompress4X2_usingDTable_internal_fast_asm_loop [BMI2] -HUF_decompress4X2_usingDTable_internal_fast_c_loop [BMI2] -HUF_readStats_body_bmi2 [BMI2, LZCNT] -ZSTD_buildFSETable_body_bmi2 [BMI2] -ZSTD_decompressSequencesLong_bmi2 [BMI1, BMI2, LZCNT] -ZSTD_decompressSequencesSplitLitBuffer_bmi2 [BMI1, BMI2, LZCNT] -ZSTD_decompressSequences_bmi2 [BMI1, BMI2, LZCNT] -ZSTD_encodeSequences_bmi2 [BMI2] - - # ---------------------------------------------------------------------------- # libdeflate. Gate: libdeflate_init_x86_cpu_features. # (10 symbols) diff --git a/src/bun_core/lib.rs b/src/bun_core/lib.rs index 412272fcf591..240556b08fd4 100644 --- a/src/bun_core/lib.rs +++ b/src/bun_core/lib.rs @@ -27,7 +27,7 @@ pub mod util; pub use atomic_cell::{Atom, AtomicCell, ThreadCell}; /// Shared state-machine tag for the streaming (de)compressors in -/// `bun_brotli` / `bun_zlib` / `bun_zstd`. +/// `bun_brotli` / `bun_zlib`. pub mod compress { #[derive(Clone, Copy, PartialEq, Eq)] pub enum State { diff --git a/src/hash/xxhash.rs b/src/hash/xxhash.rs index 9d6e00a8a76d..bcdcd775fbea 100644 --- a/src/hash/xxhash.rs +++ b/src/hash/xxhash.rs @@ -32,8 +32,9 @@ impl XxHash64 { /// Streaming XxHash64 — used by the bundler's `ContentHasher` /// (length-prefixed chunk hashing across many `update()` calls before a single /// `digest()`), plus the dev-server source-map hash and the resolver stat hash. -/// Wraps `bun_highway::XxHash64State` so the workspace has exactly one xxhash -/// implementation; output is bit-identical to the xxHash reference. +/// Wraps `bun_highway::XxHash64State`; output is bit-identical to the xxHash +/// reference. The one other XXH64 in the workspace is `bun_zstd`'s, for frame +/// checksums: that crate is safe Rust with no foreign code under it. pub struct XxHash64Streaming(bun_highway::XxHash64State); impl XxHash64Streaming { diff --git a/src/http/Decompressor.rs b/src/http/Decompressor.rs index dc7ea22724c3..346f96e5e3f5 100644 --- a/src/http/Decompressor.rs +++ b/src/http/Decompressor.rs @@ -1,7 +1,7 @@ use bun_core::MutableString; use bun_http_types::Encoding::Encoding; -// The streaming decoders below own only their C-side state and take +// The streaming decoders below own only their codec's state and take // `(input, output)` per call to [`Decompressor::decompress_chunk`], so no // borrow of the request's `compressed_body` / `body_out_str` escapes the // call. @@ -22,7 +22,7 @@ pub(crate) fn has_zlib_header(buffer: &[u8]) -> bool { } impl Decompressor { - // Note: each variant's `Drop` releases the underlying C state, so an + // Note: each variant's `Drop` releases the codec's state, so an // explicit `Drop` is unnecessary. Callers that want a mid-lifecycle reset // assign `*self = Decompressor::None`. @@ -66,7 +66,7 @@ impl Decompressor { )?)); } Encoding::Zstd => { - *self = Decompressor::Zstd(Box::new(bun_zstd::StreamingDecoder::new()?)); + *self = Decompressor::Zstd(Box::new(bun_zstd::StreamingDecoder::new())); } _ => unreachable!("Invalid encoding. This code should not be reachable"), } diff --git a/src/http/compress_body.rs b/src/http/compress_body.rs index c0db8458fea8..78a66817738f 100644 --- a/src/http/compress_body.rs +++ b/src/http/compress_body.rs @@ -226,14 +226,12 @@ fn compress_zstd( level: Option, spill: &mut Vec, ) -> crate::Result { + let level = level.unwrap_or(bun_zstd::ZSTD_CLEVEL_DEFAULT); let bound = bun_zstd::compress_bound(input.len()); - if bun_zstd::is_error(bound) { - return Err(crate::Error::CompressionFailed); - } if bound <= state.shared_buffer.len() { return match bun_zstd::compress(&mut state.shared_buffer, input, level) { - bun_zstd::Result::Success(n) => Ok(CompressOutput::Shared(n)), - bun_zstd::Result::Err(_) => Err(crate::Error::CompressionFailed), + Ok(n) => Ok(CompressOutput::Shared(n)), + Err(_) => Err(crate::Error::CompressionFailed), }; } @@ -241,7 +239,7 @@ fn compress_zstd( .try_reserve_exact(bound) .map_err(|_| bun_alloc::AllocError)?; match bun_zstd::compress_append(spill, input, level) { - bun_zstd::Result::Success(_) => Ok(CompressOutput::Spilled), - bun_zstd::Result::Err(_) => Err(crate::Error::CompressionFailed), + Ok(_) => Ok(CompressOutput::Spilled), + Err(_) => Err(crate::Error::CompressionFailed), } } diff --git a/src/jsc/bindings/BunAOTCompile.cpp b/src/jsc/bindings/BunAOTCompile.cpp index e740613c221d..60009961629e 100644 --- a/src/jsc/bindings/BunAOTCompile.cpp +++ b/src/jsc/bindings/BunAOTCompile.cpp @@ -33,7 +33,6 @@ #include #include #include -#include #if OS(WINDOWS) #include @@ -97,16 +96,14 @@ extern "C" size_t Bun__AOT__applyOptionWord(Wire::CompileOptions* options, BunAO return utf8.length(); } -static size_t maxCompressedSize(size_t sourceSize) -{ - return ZSTD_compressBound(sourceSize); -} +// src/runtime/aot_source_compression.rs +extern "C" size_t Bun__AOT__maxCompressedSourceSize(size_t textSize); +extern "C" size_t Bun__AOT__compressSource(uint8_t* compressed, size_t compressedCapacity, const uint8_t* text, size_t textSize, int level); template static size_t compressSource(std::span compressed, std::span text) { - size_t written = ZSTD_compress(compressed.data(), compressed.size(), text.data(), text.size(), level); - return ZSTD_isError(written) ? 0 : written; + return Bun__AOT__compressSource(compressed.data(), compressed.size(), text.data(), text.size(), level); } extern "C" void Bun__encoding__encodeLatin1(const uint8_t* input, size_t length, uint8_t encoding, void* sink, void (*append)(void* sink, const uint8_t* bytes, size_t size)); @@ -157,7 +154,7 @@ extern "C" BunAOTResult* Bun__AOT__compile(BunAOTBytes program, const JSC::AOT:: bool isTestRun = JSC::AOT::hasFlag(options->admittedCode, Wire::AdmittedCode::ReplacedBuiltins); auto compress = isTestRun ? compressSource<1> : compressSource<3>; auto* result = new BunAOTResult; - result->output = JSC::callRemovable(std::span { program.ptr, program.len }, *options, threadPool, JSC::AOT::ProgramOwner { releaseProgram, programOwner, externalBodies, { programOwner, functionSourceSize, copyFunctionSource, maxCompressedSize, compress, viewFunctionSource, releaseFunctionSources }, releaseBytesOfProgram, { encodeForBuffer } }); + result->output = JSC::callRemovable(std::span { program.ptr, program.len }, *options, threadPool, JSC::AOT::ProgramOwner { releaseProgram, programOwner, externalBodies, { programOwner, functionSourceSize, copyFunctionSource, Bun__AOT__maxCompressedSourceSize, compress, viewFunctionSource, releaseFunctionSources }, releaseBytesOfProgram, { encodeForBuffer } }); return result; } diff --git a/src/jsc/bindings/BunCompiledModules.cpp b/src/jsc/bindings/BunCompiledModules.cpp index e330d5fc3a9a..27140fa38f3d 100644 --- a/src/jsc/bindings/BunCompiledModules.cpp +++ b/src/jsc/bindings/BunCompiledModules.cpp @@ -22,7 +22,6 @@ #include "ZigGlobalObject.h" #if ENABLE(AOT) #include "RemovableFeatures.h" -#include #include #include @@ -2218,12 +2217,12 @@ static void crashWithReason(std::span reason) Bun__panic(reason.data(), reason.size()); } +// src/runtime/aot_source_compression.rs +extern "C" bool Bun__AOT__decompressSource(uint8_t* text, size_t textSize, const uint8_t* compressed, size_t compressedSize); + bool decompressSource(std::span text, std::span compressed) { - ZSTD_DCtx* context = ZSTD_createDCtx(); - bool isOpened = context && ZSTD_decompressDCtx(context, text.data(), text.size(), compressed.data(), compressed.size()) == text.size(); - ZSTD_freeDCtx(context); - return isOpened; + return Bun__AOT__decompressSource(text.data(), text.size(), compressed.data(), compressed.size()); } static bool loadsSynchronously(JSGlobalObject* globalObject, AOT::ModuleID module) diff --git a/src/jsc/bindings/BunProcess.cpp b/src/jsc/bindings/BunProcess.cpp index b9e9085150bb..24fefe0c0389 100644 --- a/src/jsc/bindings/BunProcess.cpp +++ b/src/jsc/bindings/BunProcess.cpp @@ -310,8 +310,8 @@ static JSValue constructVersions(VM& vm, JSObject* processObject) { "libdeflate", BUN_VERSION_LIBDEFLATE_HASH }, { "usockets", BUN_VERSION_USOCKETS }, { "lshpack", BUN_VERSION_LSHPACK }, - // Use commit hash for zstd (semantic version extraction not working yet) - { "zstd", BUN_VERSION_ZSTD_HASH }, + // The version of zstd that src/zstd gives the same output as. + { "zstd", "1.5.7" }, { "v8", REPORTED_NODEJS_V8_VERSION }, #if !OS(WINDOWS) { "uv", "1.48.0" }, diff --git a/src/jsc/bindings/ProcessBindingConstants.cpp b/src/jsc/bindings/ProcessBindingConstants.cpp index 1fd7d80098ba..9864121c7a6d 100644 --- a/src/jsc/bindings/ProcessBindingConstants.cpp +++ b/src/jsc/bindings/ProcessBindingConstants.cpp @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include @@ -55,6 +54,9 @@ struct NumericConstant { double value; }; +// src/runtime/node/zlib/NativeZstd.rs +extern "C" const char* Bun__zstdConstant(size_t index, double* value); + // One loop instead of a `putDirect` call sequence per constant; these objects // are each built once, lazily. A long table goes into a dictionary: no other // object ever has these names, so a Structure per name would serve nobody. @@ -1161,70 +1163,12 @@ static JSValue processBindingConstantsGetZlib(VM& vm, JSObject* bindingObject) { "BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_2", static_cast(BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_2) }, { "BROTLI_DECODER_ERROR_ALLOC_BLOCK_TYPE_TREES", static_cast(BROTLI_DECODER_ERROR_ALLOC_BLOCK_TYPE_TREES) }, { "BROTLI_DECODER_ERROR_UNREACHABLE", static_cast(BROTLI_DECODER_ERROR_UNREACHABLE) }, - { "ZSTD_e_continue", static_cast(ZSTD_e_continue) }, - { "ZSTD_e_flush", static_cast(ZSTD_e_flush) }, - { "ZSTD_e_end", static_cast(ZSTD_e_end) }, - { "ZSTD_fast", static_cast(ZSTD_fast) }, - { "ZSTD_dfast", static_cast(ZSTD_dfast) }, - { "ZSTD_greedy", static_cast(ZSTD_greedy) }, - { "ZSTD_lazy", static_cast(ZSTD_lazy) }, - { "ZSTD_lazy2", static_cast(ZSTD_lazy2) }, - { "ZSTD_btlazy2", static_cast(ZSTD_btlazy2) }, - { "ZSTD_btopt", static_cast(ZSTD_btopt) }, - { "ZSTD_btultra", static_cast(ZSTD_btultra) }, - { "ZSTD_btultra2", static_cast(ZSTD_btultra2) }, - { "ZSTD_c_compressionLevel", static_cast(ZSTD_c_compressionLevel) }, - { "ZSTD_c_windowLog", static_cast(ZSTD_c_windowLog) }, - { "ZSTD_c_hashLog", static_cast(ZSTD_c_hashLog) }, - { "ZSTD_c_chainLog", static_cast(ZSTD_c_chainLog) }, - { "ZSTD_c_searchLog", static_cast(ZSTD_c_searchLog) }, - { "ZSTD_c_minMatch", static_cast(ZSTD_c_minMatch) }, - { "ZSTD_c_targetLength", static_cast(ZSTD_c_targetLength) }, - { "ZSTD_c_strategy", static_cast(ZSTD_c_strategy) }, - { "ZSTD_c_enableLongDistanceMatching", static_cast(ZSTD_c_enableLongDistanceMatching) }, - { "ZSTD_c_ldmHashLog", static_cast(ZSTD_c_ldmHashLog) }, - { "ZSTD_c_ldmMinMatch", static_cast(ZSTD_c_ldmMinMatch) }, - { "ZSTD_c_ldmBucketSizeLog", static_cast(ZSTD_c_ldmBucketSizeLog) }, - { "ZSTD_c_ldmHashRateLog", static_cast(ZSTD_c_ldmHashRateLog) }, - { "ZSTD_c_contentSizeFlag", static_cast(ZSTD_c_contentSizeFlag) }, - { "ZSTD_c_checksumFlag", static_cast(ZSTD_c_checksumFlag) }, - { "ZSTD_c_dictIDFlag", static_cast(ZSTD_c_dictIDFlag) }, - { "ZSTD_c_nbWorkers", static_cast(ZSTD_c_nbWorkers) }, - { "ZSTD_c_jobSize", static_cast(ZSTD_c_jobSize) }, - { "ZSTD_c_overlapLog", static_cast(ZSTD_c_overlapLog) }, - { "ZSTD_d_windowLogMax", static_cast(ZSTD_d_windowLogMax) }, - { "ZSTD_CLEVEL_DEFAULT", static_cast(ZSTD_CLEVEL_DEFAULT) }, - { "ZSTD_error_no_error", static_cast(ZSTD_error_no_error) }, - { "ZSTD_error_GENERIC", static_cast(ZSTD_error_GENERIC) }, - { "ZSTD_error_prefix_unknown", static_cast(ZSTD_error_prefix_unknown) }, - { "ZSTD_error_version_unsupported", static_cast(ZSTD_error_version_unsupported) }, - { "ZSTD_error_frameParameter_unsupported", static_cast(ZSTD_error_frameParameter_unsupported) }, - { "ZSTD_error_frameParameter_windowTooLarge", static_cast(ZSTD_error_frameParameter_windowTooLarge) }, - { "ZSTD_error_corruption_detected", static_cast(ZSTD_error_corruption_detected) }, - { "ZSTD_error_checksum_wrong", static_cast(ZSTD_error_checksum_wrong) }, - { "ZSTD_error_literals_headerWrong", static_cast(ZSTD_error_literals_headerWrong) }, - { "ZSTD_error_dictionary_corrupted", static_cast(ZSTD_error_dictionary_corrupted) }, - { "ZSTD_error_dictionary_wrong", static_cast(ZSTD_error_dictionary_wrong) }, - { "ZSTD_error_dictionaryCreation_failed", static_cast(ZSTD_error_dictionaryCreation_failed) }, - { "ZSTD_error_parameter_unsupported", static_cast(ZSTD_error_parameter_unsupported) }, - { "ZSTD_error_parameter_combination_unsupported", static_cast(ZSTD_error_parameter_combination_unsupported) }, - { "ZSTD_error_parameter_outOfBound", static_cast(ZSTD_error_parameter_outOfBound) }, - { "ZSTD_error_tableLog_tooLarge", static_cast(ZSTD_error_tableLog_tooLarge) }, - { "ZSTD_error_maxSymbolValue_tooLarge", static_cast(ZSTD_error_maxSymbolValue_tooLarge) }, - { "ZSTD_error_maxSymbolValue_tooSmall", static_cast(ZSTD_error_maxSymbolValue_tooSmall) }, - { "ZSTD_error_stabilityCondition_notRespected", static_cast(ZSTD_error_stabilityCondition_notRespected) }, - { "ZSTD_error_stage_wrong", static_cast(ZSTD_error_stage_wrong) }, - { "ZSTD_error_init_missing", static_cast(ZSTD_error_init_missing) }, - { "ZSTD_error_memory_allocation", static_cast(ZSTD_error_memory_allocation) }, - { "ZSTD_error_workSpace_tooSmall", static_cast(ZSTD_error_workSpace_tooSmall) }, - { "ZSTD_error_dstSize_tooSmall", static_cast(ZSTD_error_dstSize_tooSmall) }, - { "ZSTD_error_srcSize_wrong", static_cast(ZSTD_error_srcSize_wrong) }, - { "ZSTD_error_dstBuffer_null", static_cast(ZSTD_error_dstBuffer_null) }, - { "ZSTD_error_noForwardProgress_destFull", static_cast(ZSTD_error_noForwardProgress_destFull) }, - { "ZSTD_error_noForwardProgress_inputEmpty", static_cast(ZSTD_error_noForwardProgress_inputEmpty) }, { nullptr, 0 }, }; putNumericConstants(vm, object, kConstants12); + double value; + for (size_t i = 0; const char* name = Bun__zstdConstant(i, &value); ++i) + object->putDirect(vm, Identifier::fromString(vm, ASCIILiteral::fromLiteralUnsafe(name)), jsNumber(value)); return object; } diff --git a/src/jsc/bindings/bun_icu_decompress.cpp b/src/jsc/bindings/bun_icu_decompress.cpp index 241de79c554e..e878a5337a55 100644 --- a/src/jsc/bindings/bun_icu_decompress.cpp +++ b/src/jsc/bindings/bun_icu_decompress.cpp @@ -34,16 +34,19 @@ #include #include -#define ZSTD_STATIC_LINKING_ONLY -#include - -static_assert(ZSTD_MAGICNUMBER == 0xFD2FB528); +static constexpr uint32_t ZSTD_MAGICNUMBER = 0xFD2FB528; // Raw ICU items have bytes[2..3] == {0xda, 0x27} (ucmndata.h MAGIC1/MAGIC2), // so their first u32 is 0x27da'hhhh — cannot collide with zstd's magic. extern "C" __attribute__((weak)) const unsigned char bun_icu_zstd_dict[]; extern "C" __attribute__((weak)) const unsigned int bun_icu_zstd_dict_size; +// src/runtime/icu_decompress.rs +struct IcuDecompressor; +extern "C" IcuDecompressor* Bun__IcuDecompressor__create(const unsigned char* dict, size_t dictSize); +extern "C" bool Bun__IcuDecompressor__frameSizes(const void* src, size_t bound, size_t* compressedSize, uint64_t* contentSize); +extern "C" bool Bun__IcuDecompressor__decompress(IcuDecompressor*, void* dst, size_t dstSize, const void* src, size_t srcSize); + namespace Bun { class ICUDecompressor { @@ -61,15 +64,13 @@ class ICUDecompressor { Locker locker { m_lock }; if (auto it = m_cache.find(p); it != m_cache.end()) { - *length = static_cast(ZSTD_getFrameContentSize(p, frameBound(*length))); - return it->value; + *length = it->value.second; + return it->value.first; } - size_t clen = ZSTD_findFrameCompressedSize(p, frameBound(*length)); - if (ZSTD_isError(clen)) - return p; - auto dlen = ZSTD_getFrameContentSize(p, clen); - if (dlen == ZSTD_CONTENTSIZE_UNKNOWN || dlen == ZSTD_CONTENTSIZE_ERROR) + size_t clen; + uint64_t dlen; + if (!Bun__IcuDecompressor__frameSizes(p, frameBound(*length), &clen, &dlen)) return p; // tryAlignedMalloc asserts size is a multiple of alignment in debug @@ -78,25 +79,23 @@ class ICUDecompressor { void* buf = MimallocMalloc::tryAlignedMalloc(alloc, 16); if (!buf) return p; - size_t r = m_ddict - ? ZSTD_decompress_usingDDict(m_dctx, buf, static_cast(dlen), p, clen, m_ddict) - : ZSTD_decompressDCtx(m_dctx, buf, static_cast(dlen), p, clen); - if (ZSTD_isError(r)) { + // The decoder takes initialized memory. + std::memset(buf, 0, alloc); + if (!Bun__IcuDecompressor__decompress(m_decompressor, buf, static_cast(dlen), p, clen)) { MimallocMalloc::free(buf); return p; } - m_cache.add(p, buf); *length = static_cast(dlen); + m_cache.add(p, std::pair { buf, *length }); return buf; } private: ICUDecompressor() - : m_dctx(ZSTD_createDCtx()) - , m_ddict(&bun_icu_zstd_dict_size && bun_icu_zstd_dict_size - ? ZSTD_createDDict_byReference(bun_icu_zstd_dict, bun_icu_zstd_dict_size) - : nullptr) + : m_decompressor(&bun_icu_zstd_dict_size && bun_icu_zstd_dict_size + ? Bun__IcuDecompressor__create(bun_icu_zstd_dict, bun_icu_zstd_dict_size) + : Bun__IcuDecompressor__create(nullptr, 0)) { } @@ -105,9 +104,8 @@ class ICUDecompressor { friend class WTF::LazyNeverDestroyed; WTF::Lock m_lock; - WTF::HashMap m_cache WTF_GUARDED_BY_LOCK(m_lock); - ZSTD_DCtx* const m_dctx; - ZSTD_DDict* const m_ddict; + WTF::HashMap> m_cache WTF_GUARDED_BY_LOCK(m_lock); + IcuDecompressor* const m_decompressor; }; } // namespace Bun diff --git a/src/jsc/bindings/xxhash3.cpp b/src/jsc/bindings/xxhash3.cpp index f088916b24a8..490186bbc470 100644 --- a/src/jsc/bindings/xxhash3.cpp +++ b/src/jsc/bindings/xxhash3.cpp @@ -15,7 +15,7 @@ // verbatim; they do not depend on vector width. Verified against the xxHash // reference test vectors and SMHasher constants in test/js/bun/util/hash.test.js. // -// References (byte-identical constants): vendor/zstd/lib/common/xxhash.h +// References (byte-identical constants): xxhash.h of https://github.com/Cyan4973/xxHash // (XXH3_kSecret, PRIME*), and the twox-hash crate this replaces. // No "root.h" — like the sibling Highway TU image_resize.cpp, this file uses no diff --git a/src/pod/lib.rs b/src/pod/lib.rs index 8101df9337bf..5e9afd2b32ec 100644 --- a/src/pod/lib.rs +++ b/src/pod/lib.rs @@ -327,5 +327,24 @@ pub fn vec_from_bytes(bytes: &[u8]) -> Option(len: usize) -> Option> { + let layout = core::alloc::Layout::array::(len).ok()?; + if layout.size() == 0 { + // Nothing to allocate: `len` is 0, or `T` has no bytes. + return Some(core::iter::repeat_with(zeroed::).take(len).collect()); + } + // SAFETY: the size of `layout` is not zero. + let ptr = unsafe { alloc::alloc::alloc_zeroed(layout) }.cast::(); + if ptr.is_null() { + return None; + } + // SAFETY: `ptr` comes from the global allocator with the layout of `[T; len]`, and all-zero bytes are `len` valid + // `T`s because `T: Zeroable`. + Some(unsafe { alloc::vec::Vec::from_raw_parts(ptr, len, len) }) +} + #[cfg(test)] mod tests; diff --git a/src/pod/tests.rs b/src/pod/tests.rs index 4602d307f5ec..b6182416e206 100644 --- a/src/pod/tests.rs +++ b/src/pod/tests.rs @@ -122,6 +122,19 @@ fn vec_from_bytes_copies_from_any_address() { assert_eq!(vec_from_bytes::(&[]), Some(Vec::new())); } +#[test] +fn try_zeroed_vec_is_zeros_or_none() { + assert_eq!(try_zeroed_vec::(5).unwrap(), [0; 5]); + assert_eq!(try_zeroed_vec::<[u16; 3]>(2).unwrap(), [[0; 3]; 2]); + assert_eq!(try_zeroed_vec::(0).unwrap().capacity(), 0); + assert_eq!(try_zeroed_vec::<()>(7).unwrap().len(), 7); + assert!(try_zeroed_vec::(usize::MAX / 8).is_none()); + assert!(try_zeroed_vec::(usize::MAX).is_none()); + let mut grows = try_zeroed_vec::(3).unwrap(); + grows.extend_from_slice(&[1; 100]); + assert_eq!(grows.len(), 103); +} + #[test] fn read_prefix_takes_the_front() { let bytes = &BYTES.0; diff --git a/src/runtime/aot_source_compression.rs b/src/runtime/aot_source_compression.rs new file mode 100644 index 000000000000..66882227f58d --- /dev/null +++ b/src/runtime/aot_source_compression.rs @@ -0,0 +1,58 @@ +//! The zstd side of the source text in a compiled program: `BunAOTCompile.cpp` compresses it and +//! `BunCompiledModules.cpp` decompresses it. + +/// # Safety +/// `ptr` is valid for writes of `len` bytes that nothing else uses for `'a`. They need not be initialized. +unsafe fn zeroed<'a>(ptr: *mut u8, len: usize) -> &'a mut [u8] { + // SAFETY: the caller's contract, and then the bytes are initialized. + unsafe { + ptr.write_bytes(0, len); + bun_core::ffi::slice_mut(ptr, len) + } +} + +#[unsafe(no_mangle)] +pub(crate) extern "C" fn Bun__AOT__maxCompressedSourceSize(text_size: usize) -> usize { + bun_zstd::compress_bound(text_size) +} + +/// Compresses the `text_size` bytes at `text` to `compressed`, which is `compressed_capacity` bytes that only this call +/// uses and that need not be initialized. Returns the size of the frame, or 0 if it fails. +#[unsafe(no_mangle)] +pub(crate) extern "C" fn Bun__AOT__compressSource( + compressed: *mut u8, + compressed_capacity: usize, + text: *const u8, + text_size: usize, + level: core::ffi::c_int, +) -> usize { + // More room than the bound changes nothing in the frame. + let room = compressed_capacity.min(bun_zstd::compress_bound(text_size)); + // SAFETY: what the caller says of the two ranges. + let (compressed, text) = unsafe { + ( + zeroed(compressed, room), + bun_core::ffi::slice(text, text_size), + ) + }; + bun_zstd::compress(compressed, text, level).unwrap_or(0) +} + +/// Decompresses the `compressed_size` bytes at `compressed` to `text`, which is `text_size` bytes that only this call +/// uses and that need not be initialized. False unless it fills `text`. +#[unsafe(no_mangle)] +pub(crate) extern "C" fn Bun__AOT__decompressSource( + text: *mut u8, + text_size: usize, + compressed: *const u8, + compressed_size: usize, +) -> bool { + // SAFETY: what the caller says of the two ranges. + let (text, compressed) = unsafe { + ( + zeroed(text, text_size), + bun_core::ffi::slice(compressed, compressed_size), + ) + }; + bun_zstd::DCtx::new().decompress(text, compressed) == Ok(text_size) +} diff --git a/src/runtime/api/BunObject.rs b/src/runtime/api/BunObject.rs index 094842fd754c..69147c75f8d2 100644 --- a/src/runtime/api/BunObject.rs +++ b/src/runtime/api/BunObject.rs @@ -1963,25 +1963,11 @@ pub(crate) mod JSZstd { /// Boxed (trimmed to the bytes produced) so an empty result owns no memory, as `create_buffer_from_box` requires. fn compress_to_box(input: &[u8], level: i32) -> Result, Failure> { - let max_size = bun_zstd::compress_bound(input.len()); - // `ZSTD_compressBound` returns an error code for inputs over `ZSTD_MAX_INPUT_SIZE`. - if bun_zstd::is_error(max_size) { - return Err(Failure::Compression(b"Input is too large to compress")); + match bun_zstd::compress_alloc(input, level) { + Ok(output) => Ok(output.into_boxed_slice()), + Err(bun_zstd::Error::MemoryAllocation) => Err(Failure::OutOfMemory), + Err(err) => Err(Failure::Compression(err.message().to_bytes())), } - - // Reserved, not zero-filled: zstd initializes exactly the bytes it reports. - let mut output: Vec = Vec::new(); - output - .try_reserve_exact(max_size) - .map_err(|_| Failure::OutOfMemory)?; - - if let bun_zstd::Result::Err(err) = - bun_zstd::compress_append(&mut output, input, Some(level)) - { - return Err(Failure::Compression(err.as_bytes())); - } - - Ok(output.into_boxed_slice()) } fn decompress_to_box(input: &[u8]) -> Result, Failure> { diff --git a/src/runtime/icu_decompress.rs b/src/runtime/icu_decompress.rs new file mode 100644 index 000000000000..1abababddfa3 --- /dev/null +++ b/src/runtime/icu_decompress.rs @@ -0,0 +1,77 @@ +//! The zstd side of `bun_icu_decompress.cpp`. + +use core::ffi::c_void; + +use bun_zstd::{DCtx, DDict, FrameExtent}; + +pub(crate) struct IcuDecompressor { + dctx: DCtx, + ddict: Option>, +} + +/// `dict` is the `dict_size` bytes of a dictionary that lives as long as the process, or null. +#[unsafe(no_mangle)] +pub(crate) extern "C" fn Bun__IcuDecompressor__create( + dict: *const u8, + dict_size: usize, +) -> *mut IcuDecompressor { + let ddict = if dict.is_null() || dict_size == 0 { + None + } else { + // SAFETY: the caller's contract. + DDict::by_reference(unsafe { core::slice::from_raw_parts(dict, dict_size) }).ok() + }; + bun_core::heap::into_raw(Box::new(IcuDecompressor { + dctx: DCtx::new(), + ddict, + })) +} + +/// The sizes of the frame that starts at `src`, compressed and decompressed. False if it is not a frame that says its +/// size, or if it is more than `bound` bytes. +#[unsafe(no_mangle)] +pub(crate) extern "C" fn Bun__IcuDecompressor__frameSizes( + src: *const u8, + bound: usize, + compressed_size: &mut usize, + content_size: &mut u64, +) -> bool { + // `bound` can be past the end of the data, so this only looks at the bytes that the frame says it has. + let mut known = 0; + loop { + // SAFETY: `src` starts a frame, which is at least `known` bytes. + let frame = unsafe { core::slice::from_raw_parts(src, known) }; + match bun_zstd::frame_extent(frame) { + Ok(FrameExtent::AtLeast(size)) if size > known && size <= bound => known = size, + Ok(FrameExtent::Whole(size)) => { + *compressed_size = size; + *content_size = bun_zstd::get_frame_content_size(frame); + return *content_size < bun_zstd::ZSTD_CONTENTSIZE_ERROR; + } + _ => return false, + } + } +} + +/// Decompresses the frame `src` to `dst`, which is initialized and of the size that the frame says. +#[unsafe(no_mangle)] +pub(crate) extern "C" fn Bun__IcuDecompressor__decompress( + this: &mut IcuDecompressor, + dst: *mut c_void, + dst_size: usize, + src: *const u8, + src_size: usize, +) -> bool { + // SAFETY: the caller's contract. + let (dst, src) = unsafe { + ( + core::slice::from_raw_parts_mut(dst.cast::(), dst_size), + core::slice::from_raw_parts(src, src_size), + ) + }; + match &this.ddict { + Some(ddict) => this.dctx.decompress_using_ddict(dst, src, ddict), + None => this.dctx.decompress(dst, src), + } + .is_ok() +} diff --git a/src/runtime/lib.rs b/src/runtime/lib.rs index bca6e1039e1d..0a48ed749f8a 100644 --- a/src/runtime/lib.rs +++ b/src/runtime/lib.rs @@ -43,6 +43,7 @@ pub(crate) mod socket; #[path = "webcore.rs"] pub(crate) mod webcore; +pub(crate) mod aot_source_compression; pub(crate) mod bake; pub(crate) mod cli; pub(crate) mod shell; @@ -57,6 +58,13 @@ pub(crate) mod checker_queries_testing; pub(crate) mod code_resolver_testing; pub(crate) mod dispatch; pub(crate) mod hw_exports; +#[cfg(any( + target_os = "linux", + target_os = "android", + target_os = "freebsd", + windows +))] +pub(crate) mod icu_decompress; #[cfg(debug_assertions)] pub(crate) mod imported_class_test; pub(crate) mod ipc; diff --git a/src/runtime/node/node_zlib_binding.rs b/src/runtime/node/node_zlib_binding.rs index 9f68d9304d39..1ec11e378d8f 100644 --- a/src/runtime/node/node_zlib_binding.rs +++ b/src/runtime/node/node_zlib_binding.rs @@ -33,7 +33,7 @@ impl Drop for CountedKeepAlive { /// `msg` / `code` are nullable NUL-terminated C strings. /// Kept as raw `*const c_char` (not `&'static str`) because zlib (`z_stream.msg`) -/// and zstd (`ZSTD_getErrorString`) hand back runtime C pointers. +/// hands back a runtime C pointer. #[derive(Clone, Copy)] pub(crate) struct Error { pub(crate) msg: *const c_char, diff --git a/src/runtime/node/zlib/NativeZstd.rs b/src/runtime/node/zlib/NativeZstd.rs index 186e1b4648f7..e38b1e3ca5b6 100644 --- a/src/runtime/node/zlib/NativeZstd.rs +++ b/src/runtime/node/zlib/NativeZstd.rs @@ -1,14 +1,113 @@ pub(crate) use _impl::NativeZstd; +/// The `ZSTD_*` members of `zlib.constants`, in Node's order, for `ProcessBindingConstants.cpp`. +mod constants { + use core::ffi::{CStr, c_char}; + + use bun_zstd::{ + EndDirective, Error, Strategy, ZSTD_CLEVEL_DEFAULT, ZSTD_D_WINDOW_LOG_MAX, c_param, + }; + + struct Constant { + name: &'static CStr, + value: i32, + } + + const fn named(name: &'static CStr, value: i32) -> Constant { + Constant { name, value } + } + + const fn error(code: Error) -> Constant { + named(code.name(), code as i32) + } + + static CONSTANTS: [Constant; 61] = [ + named(c"ZSTD_e_continue", EndDirective::Continue as i32), + named(c"ZSTD_e_flush", EndDirective::Flush as i32), + named(c"ZSTD_e_end", EndDirective::End as i32), + named(c"ZSTD_fast", Strategy::Fast as i32), + named(c"ZSTD_dfast", Strategy::DFast as i32), + named(c"ZSTD_greedy", Strategy::Greedy as i32), + named(c"ZSTD_lazy", Strategy::Lazy as i32), + named(c"ZSTD_lazy2", Strategy::Lazy2 as i32), + named(c"ZSTD_btlazy2", Strategy::BtLazy2 as i32), + named(c"ZSTD_btopt", Strategy::BtOpt as i32), + named(c"ZSTD_btultra", Strategy::BtUltra as i32), + named(c"ZSTD_btultra2", Strategy::BtUltra2 as i32), + named(c"ZSTD_c_compressionLevel", c_param::COMPRESSION_LEVEL), + named(c"ZSTD_c_windowLog", c_param::WINDOW_LOG), + named(c"ZSTD_c_hashLog", c_param::HASH_LOG), + named(c"ZSTD_c_chainLog", c_param::CHAIN_LOG), + named(c"ZSTD_c_searchLog", c_param::SEARCH_LOG), + named(c"ZSTD_c_minMatch", c_param::MIN_MATCH), + named(c"ZSTD_c_targetLength", c_param::TARGET_LENGTH), + named(c"ZSTD_c_strategy", c_param::STRATEGY), + named( + c"ZSTD_c_enableLongDistanceMatching", + c_param::ENABLE_LONG_DISTANCE_MATCHING, + ), + named(c"ZSTD_c_ldmHashLog", c_param::LDM_HASH_LOG), + named(c"ZSTD_c_ldmMinMatch", c_param::LDM_MIN_MATCH), + named(c"ZSTD_c_ldmBucketSizeLog", c_param::LDM_BUCKET_SIZE_LOG), + named(c"ZSTD_c_ldmHashRateLog", c_param::LDM_HASH_RATE_LOG), + named(c"ZSTD_c_contentSizeFlag", c_param::CONTENT_SIZE_FLAG), + named(c"ZSTD_c_checksumFlag", c_param::CHECKSUM_FLAG), + named(c"ZSTD_c_dictIDFlag", c_param::DICT_ID_FLAG), + named(c"ZSTD_c_nbWorkers", c_param::NB_WORKERS), + named(c"ZSTD_c_jobSize", c_param::JOB_SIZE), + named(c"ZSTD_c_overlapLog", c_param::OVERLAP_LOG), + named(c"ZSTD_d_windowLogMax", ZSTD_D_WINDOW_LOG_MAX), + named(c"ZSTD_CLEVEL_DEFAULT", ZSTD_CLEVEL_DEFAULT), + named(c"ZSTD_error_no_error", 0), + error(Error::Generic), + error(Error::PrefixUnknown), + error(Error::VersionUnsupported), + error(Error::FrameParameterUnsupported), + error(Error::FrameParameterWindowTooLarge), + error(Error::CorruptionDetected), + error(Error::ChecksumWrong), + error(Error::LiteralsHeaderWrong), + error(Error::DictionaryCorrupted), + error(Error::DictionaryWrong), + error(Error::DictionaryCreationFailed), + error(Error::ParameterUnsupported), + error(Error::ParameterCombinationUnsupported), + error(Error::ParameterOutOfBound), + error(Error::TableLogTooLarge), + error(Error::MaxSymbolValueTooLarge), + error(Error::MaxSymbolValueTooSmall), + error(Error::StabilityConditionNotRespected), + error(Error::StageWrong), + error(Error::InitMissing), + error(Error::MemoryAllocation), + error(Error::WorkSpaceTooSmall), + error(Error::DstSizeTooSmall), + error(Error::SrcSizeWrong), + error(Error::DstBufferNull), + error(Error::NoForwardProgressDestFull), + error(Error::NoForwardProgressInputEmpty), + ]; + + /// The name of the constant at `index`, whose value goes to `value`, or null past the last one. + #[unsafe(no_mangle)] + extern "C" fn Bun__zstdConstant(index: usize, value: &mut f64) -> *const c_char { + let Some(constant) = CONSTANTS.get(index) else { + return core::ptr::null(); + }; + *value = f64::from(constant.value); + constant.name.as_ptr() + } +} + mod _impl { use core::cell::Cell; - use core::ffi::{c_int, c_uint, c_void}; + use core::ffi::{c_int, c_uint}; use core::ptr; use bun_jsc::bindgen::{JsLocal, JsScope, JsWrapper}; use bun_jsc::{self as jsc, JSGlobalObject, JsCell, JsResult, StrongOptional, WorkPoolTask}; use bun_ptr::RefPtr; - use bun_zstd::c; // `bun.c` translated-c-headers (ZSTD_* fns/consts live here) + use bun_zstd::{CCtx, DCtx, EndDirective, InBuffer, OutBuffer}; use crate::node::node_zlib_binding::{CompressionStreamImpl as _, CountedKeepAlive, Error}; use crate::node::util::validators; @@ -92,8 +191,8 @@ mod _impl { /// Per-mode external-allocation footprint, fixed at construction. fn external_size_for(mode: NodeMode) -> usize { match mode { - NodeMode::ZSTD_COMPRESS => 5272, // estimate of ZSTD_sizeof_CCtx - NodeMode::ZSTD_DECOMPRESS => 95968, // estimate of ZSTD_sizeof_DCtx + NodeMode::ZSTD_COMPRESS => CCtx::SIZE, + NodeMode::ZSTD_DECOMPRESS => DCtx::SIZE, _ => 0, } } @@ -194,8 +293,15 @@ mod _impl { .stream .with_mut(|s| s.init(pledged_src_size, dictionary)); if err.is_error() { + // SAFETY: is_error() ⇔ msg is non-null; it points at a NUL-terminated C string. + let msg = unsafe { bun_core::ffi::cstr(err.msg) }.to_bytes(); self.emit_error(global, this_value, err); - return Ok(false); + return Err(global + .err( + jsc::ErrorCode::ZLIB_INITIALIZATION_FAILED, + format_args!("{}", bstr::BStr::new(msg)), + ) + .throw()); } let Some(mut params_) = init_params_array_value.as_array_buffer(global) else { @@ -243,25 +349,22 @@ mod _impl { } } - // `poll_ref` and `this_value` (Strong) clean up via their own Drop impls. - impl Drop for NativeZstd { - fn drop(&mut self) { - self.stream.with_mut(|s| match s.mode { - NodeMode::ZSTD_COMPRESS | NodeMode::ZSTD_DECOMPRESS => s.close(), - _ => {} - }); - } + enum State { + Compress(Box), + Decompress(Box), } pub(crate) struct Context { pub(crate) mode: NodeMode, - // LIFETIMES.tsv: FFI → Option<*mut c_void> (ZSTD_createCCtx/DCtx; freed in deinit_state) - pub(crate) state: Option<*mut c_void>, - pub flush: c_int, - pub(crate) input: c::ZSTD_inBuffer, - pub(crate) output: c::ZSTD_outBuffer, - pub(crate) pledged_src_size: u64, - pub(crate) remaining: u64, + state: Option, + flush: EndDirective, + // What `set_buffers` was given, which the caller keeps alive until the write is over. + input: *const [u8], + input_pos: usize, + output: *mut [u8], + output_pos: usize, + pledged_src_size: u64, + error: Option, } impl Default for Context { @@ -269,108 +372,55 @@ mod _impl { Self { mode: NodeMode::NONE, state: None, - flush: c::ZSTD_e_continue as c_int, - input: c::ZSTD_inBuffer { - src: ptr::null(), - size: 0, - pos: 0, - }, - output: c::ZSTD_outBuffer { - dst: ptr::null_mut(), - size: 0, - pos: 0, - }, + flush: EndDirective::Continue, + input: ptr::slice_from_raw_parts(ptr::dangling(), 0), + input_pos: 0, + output: ptr::slice_from_raw_parts_mut(ptr::dangling_mut(), 0), + output_pos: 0, pledged_src_size: u64::MAX, - remaining: 0, + error: None, } } } impl Context { - /// zstd copies the dictionary into the context, so `dictionary` does not - /// need to outlive this call (unlike brotli's attach APIs). - fn load_dictionary( - dictionary: Option<&[u8]>, - load: impl FnOnce(*const c_void, usize) -> usize, - ) -> Error { - let Some(dict) = dictionary.filter(|d| !d.is_empty()) else { - return Error::OK; - }; - let result = load(dict.as_ptr().cast(), dict.len()); - if c::ZSTD_isError(result) > 0 { - return Error::init( - c"Failed to load zstd dictionary".as_ptr(), - -1, - c"ERR_ZLIB_DICTIONARY_LOAD_FAILED".as_ptr(), - ); - } - Error::OK - } + const DICTIONARY_LOAD_FAILED: Error = Error::init( + c"Failed to load zstd dictionary".as_ptr(), + -1, + c"ERR_ZLIB_DICTIONARY_LOAD_FAILED".as_ptr(), + ); + /// The dictionary is copied into the context, so `dictionary` does not + /// need to outlive this call (unlike brotli's attach APIs). pub(crate) fn init(&mut self, pledged_src_size: u64, dictionary: Option<&[u8]>) -> Error { // Mirrors node's `state_.reset(ZSTD_createCCtx())`: free the previous // context first. `init` is JS-reachable twice on one handle. - if self.state.is_some() { - self.deinit_state(); - } + self.state = None; + let dictionary = dictionary.unwrap_or_default(); match self.mode { NodeMode::ZSTD_COMPRESS => { self.pledged_src_size = pledged_src_size; - let state = c::ZSTD_createCCtx(); - if state.is_null() { - return Error::init( - c"Could not initialize zstd instance".as_ptr(), - -1, - c"ERR_ZLIB_INITIALIZATION_FAILED".as_ptr(), - ); - } - self.state = Some(state.cast()); + let mut state = Box::new(CCtx::new()); // Node loads the dictionary before setting the pledged size. - // SAFETY: FFI — state is a valid CCtx allocated above. - let err = Self::load_dictionary(dictionary, |d, n| unsafe { - c::ZSTD_CCtx_loadDictionary(state, d, n) - }); - if err.is_error() { - // SAFETY: state is a valid CCtx allocated above. - let _ = unsafe { c::ZSTD_freeCCtx(state) }; - self.state = None; - return err; + if state.load_dictionary(dictionary).is_err() { + return Self::DICTIONARY_LOAD_FAILED; } - // SAFETY: state is non-null (checked above). - let result = - unsafe { c::ZSTD_CCtx_setPledgedSrcSize(state, pledged_src_size as _) }; - if c::ZSTD_isError(result) > 0 { - // SAFETY: state is a valid CCtx allocated above. - let _ = unsafe { c::ZSTD_freeCCtx(state) }; - self.state = None; + if state.set_pledged_src_size(pledged_src_size).is_err() { return Error::init( c"Could not set pledged src size".as_ptr(), -1, c"ERR_ZLIB_INITIALIZATION_FAILED".as_ptr(), ); } + self.state = Some(State::Compress(state)); Error::OK } NodeMode::ZSTD_DECOMPRESS => { - let state = c::ZSTD_createDCtx(); - if state.is_null() { - return Error::init( - c"Could not initialize zstd instance".as_ptr(), - -1, - c"ERR_ZLIB_INITIALIZATION_FAILED".as_ptr(), - ); - } - self.state = Some(state.cast()); - // SAFETY: FFI — state is a valid DCtx allocated above. - let err = Self::load_dictionary(dictionary, |d, n| unsafe { - c::ZSTD_DCtx_loadDictionary(state, d, n) - }); - if err.is_error() { - // SAFETY: state is a valid DCtx allocated above. - let _ = unsafe { c::ZSTD_freeDCtx(state) }; - self.state = None; - return err; + let mut state = Box::new(DCtx::new()); + if state.load_dictionary(dictionary).is_err() { + return Self::DICTIONARY_LOAD_FAILED; } + self.state = Some(State::Decompress(state)); Error::OK } _ => unreachable!(), @@ -378,37 +428,20 @@ mod _impl { } pub(crate) fn set_params(&mut self, key: c_uint, value: u32) -> Error { - match self.mode { - NodeMode::ZSTD_COMPRESS => { - // SAFETY: state is a valid CCtx set by init(); @bitCast u32→c_int is a same-size reinterpret. - let result = unsafe { - c::ZSTD_CCtx_setParameter(self.state_ptr().cast(), key, value as c_int) - }; - if c::ZSTD_isError(result) > 0 { - return Error::init( - c"Setting parameter failed".as_ptr(), - -1, - c"ERR_ZSTD_PARAM_SET_FAILED".as_ptr(), - ); - } - Error::OK - } - NodeMode::ZSTD_DECOMPRESS => { - // SAFETY: state is a valid DCtx set by init(). - let result = unsafe { - c::ZSTD_DCtx_setParameter(self.state_ptr().cast(), key, value as c_int) - }; - if c::ZSTD_isError(result) > 0 { - return Error::init( - c"Setting parameter failed".as_ptr(), - -1, - c"ERR_ZSTD_PARAM_SET_FAILED".as_ptr(), - ); - } - Error::OK - } - _ => unreachable!(), + let (key, value) = (key as i32, value as i32); + let result = match &mut self.state { + Some(State::Compress(state)) => state.set_parameter(key, value).map(drop), + Some(State::Decompress(state)) => state.set_parameter(key, value), + None => unreachable!(), + }; + if result.is_err() { + return Error::init( + c"Setting parameter failed".as_ptr(), + -1, + c"ERR_ZSTD_PARAM_SET_FAILED".as_ptr(), + ); } + Error::OK } pub(crate) fn reset(&mut self) -> Error { @@ -418,34 +451,11 @@ mod _impl { self.init(self.pledged_src_size, None) } - /// Frees the Zstd encoder/decoder state without changing mode. - /// Use close() for full cleanup that also sets mode to NONE. - fn deinit_state(&mut self) { - let _ = match self.mode { - // SAFETY: state was allocated by ZSTD_create{C,D}Ctx and not yet freed. - NodeMode::ZSTD_COMPRESS => unsafe { c::ZSTD_freeCCtx(self.state_ptr().cast()) }, - // SAFETY: state was allocated by ZSTD_createDCtx and not yet freed. - NodeMode::ZSTD_DECOMPRESS => unsafe { c::ZSTD_freeDCtx(self.state_ptr().cast()) }, - _ => unreachable!(), - }; - self.state = None; - } - pub(crate) fn set_buffers(&mut self, in_: Option<&[u8]>, out: Option<&mut [u8]>) { - self.input.src = in_.map_or(ptr::null(), |p| p.as_ptr().cast()); - self.input.size = in_.map_or(0, |p| p.len()); - self.input.pos = 0; - match out { - Some(p) => { - self.output.size = p.len(); - self.output.dst = p.as_mut_ptr().cast(); - } - None => { - self.output.size = 0; - self.output.dst = ptr::null_mut(); - } - } - self.output.pos = 0; + self.input = in_.unwrap_or_default(); + self.input_pos = 0; + self.output = out.unwrap_or_default(); + self.output_pos = 0; } pub(crate) fn flush_value_is_valid(flush: u32) -> bool { @@ -453,165 +463,98 @@ mod _impl { } pub(crate) fn set_flush(&mut self, flush: c_int) { - self.flush = flush; + self.flush = match flush { + 0 => EndDirective::Continue, + 1 => EndDirective::Flush, + 2 => EndDirective::End, + n => unreachable!("invalid ZSTD_EndDirective {n}"), + }; } const ZSTD_MAGICNUMBER: [u8; 4] = 0xFD2FB528u32.to_le_bytes(); const ZSTD_MAGIC_SKIPPABLE: [u8; 4] = 0x184D2A50u32.to_le_bytes(); - /// True when the unconsumed input begins a zstd/skippable frame magic (or a prefix of one). - fn next_input_is_frame(&self) -> bool { - let n = (self.input.size - self.input.pos).min(4); + /// True when `input` begins a zstd/skippable frame magic (or a prefix of one). + fn is_frame_start(input: &[u8]) -> bool { + let n = input.len().min(4); if n == 0 { return false; } - let mut head = [0u8; 4]; - // SAFETY: input.src[pos..pos+n] lies within the slice installed by set_buffers. - unsafe { - ptr::copy_nonoverlapping( - self.input.src.cast::().add(self.input.pos), - head.as_mut_ptr(), - n, - ); - } - head[..n] == Self::ZSTD_MAGICNUMBER[..n] + let head = &input[..n]; + *head == Self::ZSTD_MAGICNUMBER[..n] || (head[0] & 0xF0 == Self::ZSTD_MAGIC_SKIPPABLE[0] - && head[1..n] == Self::ZSTD_MAGIC_SKIPPABLE[1..n]) + && head[1..] == Self::ZSTD_MAGIC_SKIPPABLE[1..n]) } pub(crate) fn do_work(&mut self) { - // A handle driven before `init()` has no CCtx/DCtx; zstd - // dereferences the context pointer unconditionally. - if self.state.is_none() { + // A handle driven before `init()` has no context. + let Some(state) = &mut self.state else { return; - } - self.remaining = match self.mode { - // SAFETY: state is a valid CCtx; input/output point to caller-kept-alive buffers (set_buffers). - NodeMode::ZSTD_COMPRESS => unsafe { - c::ZSTD_compressStream2( - self.state_ptr().cast(), - &raw mut self.output, - &raw mut self.input, - // cast c_int → ZSTD_EndDirective (c_uint) - self.flush as c_uint, - ) - }, - NodeMode::ZSTD_DECOMPRESS => { - // SAFETY: state is a valid DCtx. - let mut ret = unsafe { - c::ZSTD_decompressStream( - self.state_ptr().cast(), - &raw mut self.output, - &raw mut self.input, - ) - }; - // ret == 0 is frame-complete; mirrors NativeZlib::do_work_inflate's GUNZIP loop. - while ret == 0 - && self.output.pos < self.output.size - && self.next_input_is_frame() + }; + let mut input = InBuffer { + // SAFETY: `set_buffers` got it from a slice that the caller keeps alive until the write is over. + src: unsafe { &*self.input }, + pos: self.input_pos, + }; + let mut output = OutBuffer { + // SAFETY: as above, and nothing else writes to it until then. + dst: unsafe { &mut *self.output }, + pos: self.output_pos, + }; + let result = match state { + State::Compress(state) => { + let mut ret = state.compress_stream2(&mut output, &mut input, self.flush); + // With workers, a call returns after any progress. `zlib.ts` takes room left in + // the output to mean that all the input is consumed. + while ret.is_ok() + && self.flush == EndDirective::Continue + && input.pos < input.src.len() + && output.pos < output.dst.len() { - // SAFETY: state is a valid DCtx; input/output point to caller-kept-alive buffers. - ret = unsafe { - c::ZSTD_decompressStream( - self.state_ptr().cast(), - &raw mut self.output, - &raw mut self.input, - ) - }; + ret = state.compress_stream2(&mut output, &mut input, self.flush); } ret } - _ => unreachable!(), - } as u64; + State::Decompress(state) => { + let mut ret = state.decompress_stream(&mut output, &mut input); + // 0 is frame-complete; mirrors NativeZlib::do_work_inflate's GUNZIP loop. + while ret == Ok(0) + && output.pos < output.dst.len() + && Self::is_frame_start(&input.src[input.pos..]) + { + ret = state.decompress_stream(&mut output, &mut input); + } + ret + } + }; + self.input_pos = input.pos; + self.output_pos = output.pos; + self.error = result.err(); } pub(crate) fn write_result(&self) -> [u32; 2] { [ - self.output.size - self.output.pos, - self.input.size - self.input.pos, + self.output.len() - self.output_pos, + self.input.len() - self.input_pos, ] .map(|n| u32::try_from(n).expect("int cast")) } pub(crate) fn get_error_info(&mut self) -> Error { - // Compute result, then clear `remaining`, then return. - let err = c::ZSTD_getErrorCode(self.remaining as usize); - let result = if err == 0 { - Error::OK - } else { - Error { + match self.error.take() { + None => Error::OK, + Some(err) => Error { err: err as c_int, - msg: c::ZSTD_getErrorString(err), - code: match err { - c::ZSTD_error_no_error => c"ZSTD_error_no_error", - c::ZSTD_error_GENERIC => c"ZSTD_error_GENERIC", - c::ZSTD_error_prefix_unknown => c"ZSTD_error_prefix_unknown", - c::ZSTD_error_version_unsupported => c"ZSTD_error_version_unsupported", - c::ZSTD_error_frameParameter_unsupported => { - c"ZSTD_error_frameParameter_unsupported" - } - c::ZSTD_error_frameParameter_windowTooLarge => { - c"ZSTD_error_frameParameter_windowTooLarge" - } - c::ZSTD_error_corruption_detected => c"ZSTD_error_corruption_detected", - c::ZSTD_error_checksum_wrong => c"ZSTD_error_checksum_wrong", - c::ZSTD_error_literals_headerWrong => c"ZSTD_error_literals_headerWrong", - c::ZSTD_error_dictionary_corrupted => c"ZSTD_error_dictionary_corrupted", - c::ZSTD_error_dictionary_wrong => c"ZSTD_error_dictionary_wrong", - c::ZSTD_error_dictionaryCreation_failed => { - c"ZSTD_error_dictionaryCreation_failed" - } - c::ZSTD_error_parameter_unsupported => c"ZSTD_error_parameter_unsupported", - c::ZSTD_error_parameter_combination_unsupported => { - c"ZSTD_error_parameter_combination_unsupported" - } - c::ZSTD_error_parameter_outOfBound => c"ZSTD_error_parameter_outOfBound", - c::ZSTD_error_tableLog_tooLarge => c"ZSTD_error_tableLog_tooLarge", - c::ZSTD_error_maxSymbolValue_tooLarge => { - c"ZSTD_error_maxSymbolValue_tooLarge" - } - c::ZSTD_error_maxSymbolValue_tooSmall => { - c"ZSTD_error_maxSymbolValue_tooSmall" - } - c::ZSTD_error_stabilityCondition_notRespected => { - c"ZSTD_error_stabilityCondition_notRespected" - } - c::ZSTD_error_stage_wrong => c"ZSTD_error_stage_wrong", - c::ZSTD_error_init_missing => c"ZSTD_error_init_missing", - c::ZSTD_error_memory_allocation => c"ZSTD_error_memory_allocation", - c::ZSTD_error_workSpace_tooSmall => c"ZSTD_error_workSpace_tooSmall", - c::ZSTD_error_dstSize_tooSmall => c"ZSTD_error_dstSize_tooSmall", - c::ZSTD_error_srcSize_wrong => c"ZSTD_error_srcSize_wrong", - c::ZSTD_error_dstBuffer_null => c"ZSTD_error_dstBuffer_null", - c::ZSTD_error_noForwardProgress_destFull => { - c"ZSTD_error_noForwardProgress_destFull" - } - c::ZSTD_error_noForwardProgress_inputEmpty => { - c"ZSTD_error_noForwardProgress_inputEmpty" - } - _ => c"ZSTD_error_GENERIC", - } - .as_ptr(), - } - }; - self.remaining = 0; - result + msg: err.message().as_ptr(), + code: err.name().as_ptr(), + }, + } } pub(crate) fn close(&mut self) { - // Idempotent: a handle that was never (successfully) initialized, - // or that was already closed, has no CCtx/DCtx to free. - if self.state.is_some() { - self.deinit_state(); - } + self.state = None; self.mode = NodeMode::NONE; } - - #[inline] - fn state_ptr(&self) -> *mut c_void { - // Passes null through if unset. - self.state.unwrap_or(ptr::null_mut()) - } } crate::__impl_compression_stream!(NativeZstd, Context, js_NativeZstd); diff --git a/src/runtime/webcore/CompressionStreamCoder.rs b/src/runtime/webcore/CompressionStreamCoder.rs index 5d85d3224e35..ea86325ccccd 100644 --- a/src/runtime/webcore/CompressionStreamCoder.rs +++ b/src/runtime/webcore/CompressionStreamCoder.rs @@ -25,7 +25,7 @@ use bun_jsc::{ErrorCode, JSGlobalObject, JSUint8Array, JSValue, PinnedArrayBuffe use bun_brotli::c as brotli; use bun_zlib as zlib; -use bun_zstd::c as zstd; +use bun_zstd as zstd; /// Matches `Bun::WebStreams::CompressionFormat` in `StreamsForward.h`. #[repr(u8)] @@ -74,9 +74,8 @@ fn spare(out: &mut Vec, cap: usize) -> Result<&mut [core::mem::MaybeUninit CodecError { - if zstd::ZSTD_getErrorCode(rc) == zstd::ZSTD_error_memory_allocation { +fn zstd_error(code: zstd::Error, message: &'static str) -> CodecError { + if code == zstd::Error::MemoryAllocation { CodecError::OutOfMemory } else { CodecError::Message(message) @@ -113,8 +112,8 @@ enum Backend { }, BrotliEncode(NonNull), BrotliDecode(NonNull), - ZstdEncode(NonNull), - ZstdDecode(NonNull), + ZstdEncode(Box), + ZstdDecode(Box), } #[derive(bun_ptr::ThreadSafeRefCounted)] @@ -161,12 +160,7 @@ impl Drop for CompressionStreamCoder { } Backend::BrotliEncode(p) => brotli::BrotliEncoderDestroyInstance(p.as_ptr()), Backend::BrotliDecode(p) => brotli::BrotliDecoderDestroyInstance(p.as_ptr()), - Backend::ZstdEncode(p) => { - zstd::ZSTD_freeCCtx(p.as_ptr()); - } - Backend::ZstdDecode(p) => { - zstd::ZSTD_freeDCtx(p.as_ptr()); - } + Backend::ZstdEncode(_) | Backend::ZstdDecode(_) => {} } } } @@ -261,26 +255,15 @@ impl CompressionStreamCoder { Backend::BrotliDecode(p) } (Format::Zstd, false) => { - let p = NonNull::new(zstd::ZSTD_createCCtx()) - .ok_or(CodecError::Message("failed to initialize zstd encoder"))?; - if let Some(lvl) = level { - // SAFETY: `p` was just created and is exclusively owned here. - let rc = unsafe { - zstd::ZSTD_CCtx_setParameter(p.as_ptr(), zstd::ZSTD_c_compressionLevel, lvl) - }; - if zstd::ZSTD_isError(rc) != 0 { - // SAFETY: `p` was created above and not stored anywhere. - unsafe { zstd::ZSTD_freeCCtx(p.as_ptr()) }; - return Err(CodecError::Message("failed to set zstd level")); - } + let mut cctx = Box::new(zstd::CCtx::new()); + if let Some(lvl) = level + && (cctx.set_parameter(zstd::c_param::COMPRESSION_LEVEL, lvl)).is_err() + { + return Err(CodecError::Message("failed to set zstd level")); } - Backend::ZstdEncode(p) - } - (Format::Zstd, true) => { - let p = NonNull::new(zstd::ZSTD_createDCtx()) - .ok_or(CodecError::Message("failed to initialize zstd decoder"))?; - Backend::ZstdDecode(p) + Backend::ZstdEncode(cctx) } + (Format::Zstd, true) => Backend::ZstdDecode(Box::new(zstd::DCtx::new())), }; Ok(Box::new(Self { backend, @@ -605,55 +588,42 @@ impl CompressionStreamCoder { } } } - Backend::ZstdEncode(p) => { - // ZSTD_EndDirective: 0 = ZSTD_e_continue, 2 = ZSTD_e_end. - let end: core::ffi::c_uint = if finish { 2 } else { 0 }; - let mut input_buf = zstd::ZSTD_inBuffer { - src: input.as_ptr().cast(), - size: input.len(), - pos: 0, + Backend::ZstdEncode(cctx) => { + let end = if finish { + zstd::EndDirective::End + } else { + zstd::EndDirective::Continue }; + let mut input_buf = zstd::InBuffer { src: input, pos: 0 }; + let mut out = zstd::VecOutput::new(out); loop { - if out.len() >= cap { + if out.filled() >= cap { return Ok(Progress::More { consumed: input_buf.pos, }); } - let spare = spare(out, cap)?; - let mut output_buf = zstd::ZSTD_outBuffer { - dst: spare.as_mut_ptr().cast(), - size: spare.len(), - pos: 0, - }; - // SAFETY: `p` is a live CCtx; the buffers borrow locals / - // spare for this one call. - let remaining = unsafe { - zstd::ZSTD_compressStream2( - p.as_ptr(), - &raw mut output_buf, - &raw mut input_buf, - end, - ) - }; - // SAFETY: compressStream2 wrote exactly `output_buf.pos` - // bytes. - unsafe { out.set_len(out.len() + output_buf.pos) }; - if zstd::ZSTD_isError(remaining) != 0 { - return Err(zstd_error(remaining, "zstd encode failed")); - } - if input_buf.pos == input_buf.size && (!finish || remaining == 0) { + let mut output_buf = out + .buffer(CHUNK, cap) + .map_err(|_| CodecError::OutOfMemory)?; + let remaining = cctx.compress_stream2(&mut output_buf, &mut input_buf, end); + let written = output_buf.pos; + out.advance(written); + let remaining = + remaining.map_err(|code| zstd_error(code, "zstd encode failed"))?; + if input_buf.pos == input.len() && (!finish || remaining == 0) { return Ok(Progress::Done); } } } - Backend::ZstdDecode(p) => { + Backend::ZstdDecode(dctx) => { // After a frame completes, whatever follows must be another frame // magic (see `zstd_head`); `step` has already re-attached a split one. - let mut input_buf = zstd::ZSTD_inBuffer { - src: input.as_ptr().cast(), - size: input.len(), - pos: 0, - }; + if input.is_empty() && !finish && !continuing { + // The decoder gives up after 16 calls in a row that get it nowhere. + return Ok(Progress::Done); + } + let mut input_buf = zstd::InBuffer { src: input, pos: 0 }; + let mut out = zstd::VecOutput::new(out); loop { if self.ended { let rest = &input[input_buf.pos..]; @@ -672,46 +642,27 @@ impl CompressionStreamCoder { self.zstd_head_len = head.len() as u8; return Ok(Progress::Done); } - // SAFETY: `p` is a live DCtx. - unsafe { - zstd::ZSTD_DCtx_reset( - p.as_ptr(), - zstd::ZSTD_reset_session_and_parameters, - ) - }; + dctx.init_stream(); self.ended = false; } - if out.len() >= cap { + if out.filled() >= cap { return Ok(Progress::More { consumed: input_buf.pos, }); } - let spare = spare(out, cap)?; - let mut output_buf = zstd::ZSTD_outBuffer { - dst: spare.as_mut_ptr().cast(), - size: spare.len(), - pos: 0, - }; - // SAFETY: `p` is a live DCtx; buffers borrow locals / spare - // for this one call. - let remaining = unsafe { - zstd::ZSTD_decompressStream( - p.as_ptr(), - &raw mut output_buf, - &raw mut input_buf, - ) - }; - // SAFETY: decompressStream wrote exactly `output_buf.pos` - // bytes. - unsafe { out.set_len(out.len() + output_buf.pos) }; - if zstd::ZSTD_isError(remaining) != 0 { - return Err(zstd_error(remaining, "zstd decode failed")); - } + let mut output_buf = out + .buffer(CHUNK, cap) + .map_err(|_| CodecError::OutOfMemory)?; + let remaining = dctx.decompress_stream(&mut output_buf, &mut input_buf); + let (written, room) = (output_buf.pos, output_buf.dst.len()); + out.advance(written); + let remaining = + remaining.map_err(|code| zstd_error(code, "zstd decode failed"))?; if remaining == 0 { self.ended = true; continue; } - if input_buf.pos == input_buf.size && output_buf.pos < output_buf.size { + if input_buf.pos == input.len() && written < room { if finish { return Err(CodecError::Message("unexpected end of file")); } diff --git a/src/sema/standalone/Cargo.toml b/src/sema/standalone/Cargo.toml index 202260e91218..0f7212e6f015 100644 --- a/src/sema/standalone/Cargo.toml +++ b/src/sema/standalone/Cargo.toml @@ -2,7 +2,6 @@ name = "bun_sema_standalone" version.workspace = true edition.workspace = true -build = "build.rs" [lib] path = "lib.rs" @@ -30,8 +29,5 @@ bun_boringssl_sys.workspace = true libc.workspace = true bstr.workspace = true -[build-dependencies] -cc = "1" - [lints] workspace = true diff --git a/src/sema/standalone/build.rs b/src/sema/standalone/build.rs deleted file mode 100644 index a70c85069578..000000000000 --- a/src/sema/standalone/build.rs +++ /dev/null @@ -1,73 +0,0 @@ -#![allow( - clippy::disallowed_methods, - clippy::disallowed_types, - clippy::disallowed_macros -)] - -use std::env; -use std::path::PathBuf; - -const SOURCES: [&str; 26] = [ - "common/debug.c", - "common/entropy_common.c", - "common/error_private.c", - "common/fse_decompress.c", - "common/pool.c", - "common/threading.c", - "common/xxhash.c", - "common/zstd_common.c", - "compress/fse_compress.c", - "compress/hist.c", - "compress/huf_compress.c", - "compress/zstd_compress.c", - "compress/zstd_compress_literals.c", - "compress/zstd_compress_sequences.c", - "compress/zstd_compress_superblock.c", - "compress/zstd_double_fast.c", - "compress/zstd_fast.c", - "compress/zstd_lazy.c", - "compress/zstd_ldm.c", - "compress/zstd_opt.c", - "compress/zstd_preSplit.c", - "compress/zstdmt_compress.c", - "decompress/huf_decompress.c", - "decompress/zstd_ddict.c", - "decompress/zstd_decompress.c", - "decompress/zstd_decompress_block.c", -]; - -fn main() { - let manifest = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); - let repository = manifest - .ancestors() - .nth(3) - .expect("repository root from CARGO_MANIFEST_DIR"); - let library = repository.join("vendor/zstd/lib"); - let header = library.join("zstd.h"); - println!("cargo:rerun-if-changed={}", header.display()); - - if env::var("TARGET").unwrap() != env::var("HOST").unwrap() { - return; - } - if !header.exists() { - println!( - "cargo:warning=vendor/zstd is missing: a program that reads Bun's own modules will not link. Run bun scripts/build.ts --target=clone-zstd" - ); - return; - } - - let mut decoder = cc::Build::new(); - decoder - .files(SOURCES.map(|source| library.join(source))) - .include(&library) - .include(library.join("common")) - .define("ZSTD_DISABLE_ASM", "1") - .define("ZSTD_LEGACY_SUPPORT", "0") - .define("XXH_NAMESPACE", "ZSTD_") - .opt_level(2) - .warnings(false); - if env::var_os("CARGO_CFG_WINDOWS").is_some() { - decoder.compiler("clang-cl").define("ZSTD_HEAPMODE", "0"); - } - decoder.compile("bun_sema_standalone_zstd"); -} diff --git a/src/standalone_graph/StandaloneModuleGraph.rs b/src/standalone_graph/StandaloneModuleGraph.rs index 82c5f5051e10..ddd3906d806d 100644 --- a/src/standalone_graph/StandaloneModuleGraph.rs +++ b/src/standalone_graph/StandaloneModuleGraph.rs @@ -4018,21 +4018,11 @@ pub(crate) fn serialize_json_source_map_for_standalone( let offset = string_payload.len(); - let bound = bun_zstd::compress_bound(utf8.len()); - // `ZSTD_compressBound` returns an *error code* (a value near - // `usize::MAX`) when the input size exceeds `ZSTD_MAX_INPUT_SIZE`; - // feeding that to `Vec::reserve` below would abort with a capacity - // overflow instead of failing the build. - if bun_zstd::is_error(bound) { - return Err(crate::Error::SourceMapTooLarge); - } - string_payload.reserve(bound); - if let bun_zstd::Result::Err(err_msg) = - bun_zstd::compress_append(string_payload, utf8, Some(1)) - { + string_payload.reserve(bun_zstd::compress_bound(utf8.len())); + if let Err(err) = bun_zstd::compress_append(string_payload, utf8, 1) { Output::panic(format_args!( "Unexpected error compressing sourcemap: {}", - bstr::BStr::new(err_msg.as_bytes()) + bstr::BStr::new(err.message().to_bytes()) )); } diff --git a/src/trim/features/bun.def b/src/trim/features/bun.def index b89f2fbfdfbf..c0aee84c105a 100644 --- a/src/trim/features/bun.def +++ b/src/trim/features/bun.def @@ -109,8 +109,7 @@ feature BrotliEncoder "Brotli compression" c-function BrotliEncoderPrepareDictionary BrotliEncoderDestroyPreparedDictionary BrotliEncoderAttachPreparedDictionary feature ZstdCompressor "zstd compression" - c-function ZSTD_compress ZSTD_compressBound ZSTD_compressStream2 ZSTD_createCCtx ZSTD_freeCCtx - c-function ZSTD_CCtx_loadDictionary ZSTD_CCtx_reset ZSTD_CCtx_setParameter ZSTD_CCtx_setPledgedSrcSize + cxx-function bun_zstd ^(); +const CONTAINER_BITS: u32 = u64::BITS; + +/// `BIT_CStream_t` +pub(crate) struct BitCStream<'a> { + dst: &'a mut [u8], + container: u64, + bit_pos: u32, + pos: usize, + /// The last position a whole container can be written to. + end: usize, +} + +impl<'a> BitCStream<'a> { + /// `BIT_initCStream` + #[inline] + pub(crate) fn new(dst: &'a mut [u8]) -> Result { + if dst.len() <= CONTAINER_BYTES { + return Err(Error::DstSizeTooSmall); + } + let end = dst.len() - CONTAINER_BYTES; + Ok(Self { + dst, + container: 0, + bit_pos: 0, + pos: 0, + end, + }) + } + + /// `BIT_addBits`: `nb_bits` is less than 32, and there is room for them in the container. + #[inline(always)] + pub(crate) fn add_bits(&mut self, value: u64, nb_bits: u32) { + self.add_bits_fast(value & ((1u64 << (nb_bits & 31)) - 1), nb_bits); + } + + /// `BIT_addBitsFast`: the bits of `value` above `nb_bits` are 0. + #[inline(always)] + pub(crate) fn add_bits_fast(&mut self, value: u64, nb_bits: u32) { + debug_assert!(nb_bits + self.bit_pos < CONTAINER_BITS); + self.container |= value << (self.bit_pos & (CONTAINER_BITS - 1)); + self.bit_pos += nb_bits; + } + + #[inline(always)] + fn flush(&mut self) { + let nb_bytes = (self.bit_pos >> 3) as usize; + // At the end of the output the stream stops advancing, which `close` reports. + let pos = self.pos.min(self.end); + if let Ok(out) = self.dst.array_at_mut(pos) { + *out = self.container.to_le_bytes(); + } + self.pos = pos + nb_bytes; + self.bit_pos &= 7; + self.container >>= (nb_bytes as u32 * 8) & (CONTAINER_BITS - 1); + } + + /// `BIT_flushBitsFast`: the caller knows that the output is large enough. + #[inline(always)] + pub(crate) fn flush_bits_fast(&mut self) { + self.flush(); + } + + /// `BIT_flushBits` + #[inline(always)] + pub(crate) fn flush_bits(&mut self) { + self.flush(); + } + + /// `BIT_closeCStream`: the size of the stream, or 0 if it did not fit. + #[inline] + pub(crate) fn close(mut self) -> usize { + self.add_bits_fast(1, 1); + self.flush_bits(); + if self.pos >= self.end { + return 0; + } + self.pos + usize::from(self.bit_pos > 0) + } +} + +/// `BIT_DStream_status` +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) enum DStreamStatus { + Unfinished, + EndOfBuffer, + Completed, + Overflow, +} + +/// `BIT_DStream_t` +#[derive(Clone, Copy)] +pub(crate) struct BitDStream<'a> { + /// The stream up to the end of what `container` was read from: its last 8 bytes, or all of it if it is shorter. + src: &'a [u8], + pub(crate) container: u64, + pub(crate) bits_consumed: u32, +} + +impl<'a> BitDStream<'a> { + /// `BIT_initDStream` + #[inline] + pub(crate) fn new(src: &'a [u8]) -> Result { + let Some(&last_byte) = src.last() else { + return Err(Error::SrcSizeWrong); + }; + let padding = 8 - highbit32(u32::from(last_byte)); + if let Some(tail) = src.last_chunk::() { + if last_byte == 0 { + return Err(Error::Generic); + } + return Ok(Self { + src, + container: u64::from_le_bytes(*tail), + bits_consumed: padding, + }); + } + if last_byte == 0 { + return Err(Error::CorruptionDetected); + } + let mut bytes = [0u8; CONTAINER_BYTES]; + bytes.write_at(0, src)?; + Ok(Self { + src, + container: u64::from_le_bytes(bytes), + bits_consumed: padding + (CONTAINER_BYTES - src.len()) as u32 * 8, + }) + } + + /// A stream in the state that the 4-stream Huffman loops leave it in. `src` has at least 8 bytes from `pos` on. + #[inline] + pub(crate) fn resume(src: &'a [u8], pos: usize, bits_consumed: u32) -> Result { + Ok(Self { + src: src.sub(..pos + CONTAINER_BYTES)?, + container: read_le64(src, pos)?, + bits_consumed, + }) + } + + /// The stream up to the end of what `container` was read from. + #[inline(always)] + pub(crate) fn bytes(&self) -> &'a [u8] { + self.src + } + + /// `BIT_lookBits`: `nb_bits` is less than 32. + #[inline(always)] + pub(crate) fn look_bits(&self, nb_bits: u32) -> u64 { + let start = CONTAINER_BITS + .wrapping_sub(self.bits_consumed) + .wrapping_sub(nb_bits); + (self.container >> (start & (CONTAINER_BITS - 1))) & ((1u64 << (nb_bits & 31)) - 1) + } + + /// `BIT_lookBitsFast`: `nb_bits` is at least 1. + #[inline(always)] + pub(crate) fn look_bits_fast(&self, nb_bits: u32) -> u64 { + const MASK: u32 = CONTAINER_BITS - 1; + (self.container << (self.bits_consumed & MASK)) + >> (CONTAINER_BITS.wrapping_sub(nb_bits) & MASK) + } + + /// `BIT_skipBits` + #[inline(always)] + pub(crate) fn skip_bits(&mut self, nb_bits: u32) { + self.bits_consumed = self.bits_consumed.wrapping_add(nb_bits); + } + + /// `BIT_readBits` + #[inline(always)] + pub(crate) fn read_bits(&mut self, nb_bits: u32) -> u64 { + let value = self.look_bits(nb_bits); + self.skip_bits(nb_bits); + value + } + + /// `BIT_readBitsFast` + #[inline(always)] + pub(crate) fn read_bits_fast(&mut self, nb_bits: u32) -> u64 { + let value = self.look_bits_fast(nb_bits); + self.skip_bits(nb_bits); + value + } + + /// `ptr - start` + #[inline(always)] + fn pos(&self) -> usize { + self.src.len().saturating_sub(CONTAINER_BYTES) + } + + /// Goes `nb_bytes` back, which is no more than `pos`. + #[inline(always)] + fn load(&mut self, nb_bytes: usize) { + if let Some(src) = self.src.get(..self.src.len().wrapping_sub(nb_bytes)) + && let Some(tail) = src.last_chunk() + { + self.src = src; + self.container = u64::from_le_bytes(*tail); + } + } + + /// `BIT_reloadDStreamFast` + #[inline(always)] + pub(crate) fn reload_fast(&mut self) -> DStreamStatus { + if self.src.len() < 2 * CONTAINER_BYTES { + return DStreamStatus::Overflow; + } + self.reload_internal() + } + + /// `BIT_reloadDStream_internal`: `pos` is at least 8 and at most 64 bits are consumed. + #[inline(always)] + fn reload_internal(&mut self) -> DStreamStatus { + let nb_bytes = (self.bits_consumed >> 3) as usize; + self.bits_consumed &= 7; + self.load(nb_bytes); + DStreamStatus::Unfinished + } + + /// `BIT_reloadDStream` + #[inline(always)] + pub(crate) fn reload(&mut self) -> DStreamStatus { + if self.bits_consumed > CONTAINER_BITS { + return DStreamStatus::Overflow; + } + if self.src.len() >= 2 * CONTAINER_BYTES { + return self.reload_internal(); + } + let pos = self.pos(); + if pos == 0 { + if self.bits_consumed < CONTAINER_BITS { + return DStreamStatus::EndOfBuffer; + } + return DStreamStatus::Completed; + } + let mut nb_bytes = (self.bits_consumed >> 3) as usize; + let mut result = DStreamStatus::Unfinished; + if pos < nb_bytes { + nb_bytes = pos; + result = DStreamStatus::EndOfBuffer; + } + self.bits_consumed -= nb_bytes as u32 * 8; + self.load(nb_bytes); + result + } + + /// `BIT_endOfDStream` + #[inline(always)] + pub(crate) fn is_at_end(&self) -> bool { + self.pos() == 0 && self.bits_consumed == CONTAINER_BITS + } +} diff --git a/src/zstd/common/entropy.rs b/src/zstd/common/entropy.rs new file mode 100644 index 000000000000..85079c2a654a --- /dev/null +++ b/src/zstd/common/entropy.rs @@ -0,0 +1,439 @@ +//! `entropy_common.c`: the headers of FSE and Huffman tables. + +use super::fse_decompress::{Spread, fse_decompress_weights}; +use super::{FSE_MIN_TABLELOG, FSE_TABLELOG_ABSOLUTE_MAX, HUF_TABLELOG_MAX}; +use crate::error::{Error, Result}; +use crate::util::{BUG, SliceExt, highbit32, read_le32}; + +/// What [`read_ncount`] found besides the counts. +pub(crate) struct NCount { + /// How many bytes of the input the header takes. + pub(crate) size: usize, + pub(crate) max_symbol_value: u32, + pub(crate) table_log: u32, +} + +/// `FSE_readNCount`: `normalized_counter` has room for the symbols up to `max_symbol_value`. +pub(crate) fn read_ncount( + normalized_counter: &mut [i16], + max_symbol_value: u32, + header: &[u8], +) -> Result { + match read_valid_ncount(normalized_counter, max_symbol_value, header) { + Some(ncount) => Ok(ncount), + None => read_any_ncount(normalized_counter, max_symbol_value, header), + } +} + +/// The same for a header that has nothing wrong with it and is not at the very end of `header`. +fn read_valid_ncount( + normalized_counter: &mut [i16], + max_symbol_value: u32, + header: &[u8], +) -> Option { + /// The bits from `position` on, from the lowest on: `MIN_LOADED` of them or more, then zeroes. + #[inline(always)] + fn load(header: &[u8], position: usize) -> Option { + let bytes = header.get(position / 8..position / 8 + 8)?; + Some(u64::from_le_bytes(bytes.try_into().ok()?) >> (position % 8)) + } + const MIN_LOADED: u32 = 57; + /// After how many bits more are loaded. The 17 that are left are enough for a count. + const MAX_USED: u32 = 40; + + // As few as there usually are, and as many as can be cleared without a call. + let counts = normalized_counter.first_chunk_mut::<64>()?; + *counts = [0; 64]; + let counts = counts.get_mut(..(max_symbol_value as usize + 1).min(64))?; + // `bits` is what comes `used` bits after `position`. + let mut position = 0usize; + let mut bits = load(header, 0)?; + let table_log = (bits & 0xF) as u32 + FSE_MIN_TABLELOG; + if table_log > FSE_TABLELOG_ABSOLUTE_MAX { + return None; + } + bits >>= 4; + let mut used = 4; + let mut remaining = (1i32 << table_log) + 1; + let mut threshold = 1i32 << table_log; + let mut nb_bits = table_log + 1; + let mut symbol = 0usize; + + loop { + if used > MAX_USED { + position += used as usize; + used = 0; + bits = load(header, position)?; + } + + // With `max` for `2 * threshold - 1 - remaining`: a value below `max` is stored with a bit less, and one from + // `threshold + max` on stands for `max` less. Written so that little has to wait for `remaining`. + let low = bits as i32 & (threshold - 1); + let all = bits as i32 & (2 * threshold - 1); + let count_bits = nb_bits - u32::from(low + remaining < 2 * threshold - 1); + bits >>= count_bits; + used += count_bits; + // The stored value is the count plus one, and a count of -1 takes one slot. + let count; + if all + remaining >= 3 * threshold - 1 { + count = all + remaining - 2 * threshold; + remaining = 2 * threshold - all; + } else { + count = low - 1; + remaining -= count.abs(); + } + *counts.get_mut(symbol)? = count as i16; + symbol += 1; + if remaining < threshold { + if remaining <= 1 { + break; + } + nb_bits = remaining.ilog2() + 1; + threshold = 1 << (nb_bits - 1); + } + + if count == 0 { + // Each pair of set bits stands for three more symbols of count 0. + let mut repeats = (!bits).trailing_zeros() / 2; + if 2 * repeats + 2 > MIN_LOADED - used { + position += used as usize; + used = 0; + bits = load(header, position)?; + repeats = (!bits).trailing_zeros() / 2; + if 3 * repeats as usize > counts.len() { + return None; + } + } + symbol += 3 * repeats as usize + (bits >> (2 * repeats) & 3) as usize; + bits >>= 2 * repeats + 2; + used += 2 * repeats + 2; + } + } + if remaining != 1 { + return None; + } + Some(NCount { + size: (position + used as usize).div_ceil(8), + max_symbol_value: symbol as u32 - 1, + table_log, + }) +} + +/// The same as the C code does it. +fn read_any_ncount( + normalized_counter: &mut [i16], + max_symbol_value: u32, + header: &[u8], +) -> Result { + if header.len() < 8 { + let mut buffer = [0u8; 8]; + buffer.write_at(0, header)?; + let ncount = read_any_ncount(normalized_counter, max_symbol_value, &buffer)?; + if ncount.size > header.len() { + return Err(Error::CorruptionDetected); + } + return Ok(ncount); + } + + let iend = header.len() as isize; + let mut ip: isize = 0; + let load = |ip: isize, bit_count: i32| -> Result { + Ok(read_le32(header, ip as usize)? >> (bit_count & 31)) + }; + let max_sv1 = max_symbol_value + 1; + let mut charnum: u32 = 0; + let mut previous0 = false; + + normalized_counter.sub_mut(..max_sv1 as usize)?.fill(0); + let mut bit_stream = read_le32(header, 0)?; + let mut nb_bits = ((bit_stream & 0xF) + FSE_MIN_TABLELOG) as i32; + if nb_bits > FSE_TABLELOG_ABSOLUTE_MAX as i32 { + return Err(Error::TableLogTooLarge); + } + bit_stream >>= 4; + let mut bit_count: i32 = 4; + let table_log = nb_bits as u32; + let mut remaining: i32 = (1 << nb_bits) + 1; + let mut threshold: i32 = 1 << nb_bits; + nb_bits += 1; + + loop { + if previous0 { + // Each pair of set bits stands for three more symbols of count 0. + let mut repeats = ((!bit_stream | 0x8000_0000).trailing_zeros() >> 1) as i32; + while repeats >= 12 { + charnum += 3 * 12; + if ip <= iend - 7 { + ip += 3; + } else { + bit_count -= (8 * (iend - 7 - ip)) as i32; + bit_count &= 31; + ip = iend - 4; + } + bit_stream = load(ip, bit_count)?; + repeats = ((!bit_stream | 0x8000_0000).trailing_zeros() >> 1) as i32; + } + charnum += 3 * repeats as u32; + bit_stream >>= 2 * repeats; + bit_count += 2 * repeats; + + charnum += bit_stream & 3; + bit_count += 2; + + if charnum >= max_sv1 { + break; + } + + if ip <= iend - 7 || ip + (bit_count >> 3) as isize <= iend - 4 { + ip += (bit_count >> 3) as isize; + bit_count &= 7; + } else { + bit_count -= (8 * (iend - 4 - ip)) as i32; + bit_count &= 31; + ip = iend - 4; + } + bit_stream = load(ip, bit_count)?; + } + { + let max = (2 * threshold - 1) - remaining; + let mut count: i32; + + if (bit_stream & (threshold - 1) as u32) < max as u32 { + count = (bit_stream & (threshold - 1) as u32) as i32; + bit_count += nb_bits - 1; + } else { + count = (bit_stream & (2 * threshold - 1) as u32) as i32; + if count >= threshold { + count -= max; + } + bit_count += nb_bits; + } + + // The stored value is the count plus one, and a count of -1 takes one slot. + count -= 1; + if count >= 0 { + remaining -= count; + } else { + remaining += count; + } + normalized_counter.put(charnum as usize, count as i16)?; + charnum += 1; + previous0 = count == 0; + + if remaining < threshold { + if remaining <= 1 { + break; + } + nb_bits = highbit32(remaining as u32) as i32 + 1; + threshold = 1 << (nb_bits - 1); + } + if charnum >= max_sv1 { + break; + } + + if ip <= iend - 7 || ip + (bit_count >> 3) as isize <= iend - 4 { + ip += (bit_count >> 3) as isize; + bit_count &= 7; + } else { + bit_count -= (8 * (iend - 4 - ip)) as i32; + bit_count &= 31; + ip = iend - 4; + } + bit_stream = load(ip, bit_count)?; + } + } + if remaining != 1 { + return Err(Error::CorruptionDetected); + } + if charnum > max_sv1 { + return Err(Error::MaxSymbolValueTooSmall); + } + if bit_count > 32 { + return Err(Error::CorruptionDetected); + } + ip += ((bit_count + 7) >> 3) as isize; + Ok(NCount { + size: ip as usize, + max_symbol_value: charnum - 1, + table_log, + }) +} + +/// What [`huf_read_stats`] found besides the weights and how many symbols have each. +pub(crate) struct HufStats { + /// How many bytes of the input the header takes. + pub(crate) size: usize, + pub(crate) nb_symbols: u32, + pub(crate) table_log: u32, +} + +/// `weights`, which are fewer than 4, then zeroes. +fn pad(weights: &[u8]) -> [u8; 4] { + let mut padded = [0; 4]; + for (to, &weight) in padded.iter_mut().zip(weights) { + *to = weight; + } + padded +} + +/// `HUF_readStats` +pub(crate) fn huf_read_stats( + huff_weight: &mut [u8; 256], + rank_stats: &mut [u32; HUF_TABLELOG_MAX as usize + 1], + src: &[u8], +) -> Result { + huf_read_stats_wksp(huff_weight, rank_stats, src, &mut Spread::new()) +} + +/// `HUF_readStats_wksp` +pub(crate) fn huf_read_stats_wksp( + huff_weight: &mut [u8; 256], + rank_stats: &mut [u32; HUF_TABLELOG_MAX as usize + 1], + src: &[u8], + spread: &mut Spread, +) -> Result { + let Some((&first, rest)) = src.split_first() else { + return Err(Error::SrcSizeWrong); + }; + let mut i_size = usize::from(first); + let o_size; + + if i_size >= 128 { + // Not compressed: two weights to a byte. + o_size = i_size - 127; + i_size = o_size.div_ceil(2); + if i_size + 1 > src.len() { + return Err(Error::SrcSizeWrong); + } + if o_size >= huff_weight.len() { + return Err(Error::CorruptionDetected); + } + let (pairs, _) = huff_weight.as_chunks_mut::<2>(); + for (pair, &byte) in pairs.iter_mut().zip(rest.sub(..i_size)?) { + *pair = [byte >> 4, byte & 15]; + } + } else { + if i_size + 1 > src.len() { + return Err(Error::SrcSizeWrong); + } + o_size = fse_decompress_weights(huff_weight.sub_mut(..255)?, rest.sub(..i_size)?, spread)?; + } + + // Four counts of each weight: one that is incremented again and again has to be waited for. + let mut counts = [[0u32; 16]; 4]; + let mut all_bits = 0; + let (fours, others) = huff_weight.sub(..o_size)?.as_chunks::<4>(); + for weights in fours.iter().chain(&[pad(others)]) { + all_bits |= u32::from_le_bytes(*weights); + for (&weight, counts) in weights.iter().zip(&mut counts) { + if let Some(count) = counts.get_mut(usize::from(weight) % 16) { + *count += 1; + } + } + } + let [counts0, counts1, counts2, counts3] = &counts; + let mut ranks = [0u32; 16]; + for (weight, rank) in ranks.iter_mut().enumerate() { + *rank = + counts0.at(weight)? + counts1.at(weight)? + counts2.at(weight)? + counts3.at(weight)?; + } + let Some((valid, too_large)) = ranks.split_first_chunk() else { + return Err(BUG); + }; + if all_bits & 0xF0F0_F0F0 != 0 || too_large.iter().any(|&rank| rank != 0) { + return Err(Error::CorruptionDetected); + } + *rank_stats = *valid; + // Those that complete the last four. + if let Some(rank) = rank_stats.first_mut() { + *rank -= (4 - others.len()) as u32; + } + let weight_total = (rank_stats.iter().enumerate().skip(1)) + .map(|(weight, &rank)| rank << (weight - 1)) + .sum::(); + if weight_total == 0 { + return Err(Error::CorruptionDetected); + } + + // The last symbol's weight is not stored: it completes the total to a power of 2. + let table_log = highbit32(weight_total) + 1; + if table_log > HUF_TABLELOG_MAX { + return Err(Error::CorruptionDetected); + } + let total = 1u32 << table_log; + let rest = total - weight_total; + let verif = 1u32 << highbit32(rest); + let last_weight = highbit32(rest) + 1; + if verif != rest { + return Err(Error::CorruptionDetected); + } + huff_weight.put(o_size, last_weight as u8)?; + *rank_stats.at_mut(last_weight as usize)? += 1; + + let [_, rank1, ..] = *rank_stats; + if rank1 < 2 || rank1 & 1 != 0 { + return Err(Error::CorruptionDetected); + } + + Ok(HufStats { + size: i_size + 1, + nb_symbols: o_size as u32 + 1, + table_log, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::compress::fse::write_ncount; + + #[test] + fn the_quick_way_reads_what_the_careful_way_reads() -> Result<()> { + let mut state = 1u64; + let mut random = move || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state + }; + let nb_headers = if cfg!(miri) { 300 } else { 100_000 }; + let mut nb_quick = 0; + for index in 0..nb_headers { + let max_symbol_value = [35, 52, 31, 255, 7].at(index % 5)?; + let mut header = [0u8; 48]; + for byte in &mut header { + *byte = random() as u8; + } + if index % 2 == 0 { + header[0] = header[0] & 0xF0 | (random() % 8) as u8; + } else { + // A few symbols, far from each other. + let table_log = 5 + random() as u32 % 5; + let mut counts = [0i16; 64]; + let mut left = 1i16 << table_log; + while left > 0 { + let count = 1 + (random() % left as u64) as i16; + *counts.at_mut(random() as usize % (max_symbol_value.min(63) + 1))? += count; + left -= count; + } + let last = counts.iter().rposition(|&count| count != 0).unwrap_or(0); + write_ncount(&mut header, &counts, last as u32, table_log)?; + } + let max_symbol_value = max_symbol_value as u32; + let (mut quick, mut careful) = ([1i16; 256], [1i16; 256]); + let Some(read) = read_valid_ncount(&mut quick, max_symbol_value, &header) else { + continue; + }; + nb_quick += 1; + let expected = read_any_ncount(&mut careful, max_symbol_value, &header)?; + assert_eq!( + (read.size, read.max_symbol_value, read.table_log), + (expected.size, expected.max_symbol_value, expected.table_log) + ); + let nb_symbols = read.max_symbol_value as usize + 1; + assert_eq!(quick.sub(..nb_symbols)?, careful.sub(..nb_symbols)?); + } + assert!(nb_quick > nb_headers / 4); + Ok(()) + } +} diff --git a/src/zstd/common/fse_decompress.rs b/src/zstd/common/fse_decompress.rs new file mode 100644 index 000000000000..9c7c0212826a --- /dev/null +++ b/src/zstd/common/fse_decompress.rs @@ -0,0 +1,424 @@ +//! `fse_decompress.c`. zstd only decodes the weights of a Huffman table this way: sequences have their own decoder. + +use super::bitstream::BitDStream; +use super::entropy::read_ncount; +use super::{FSE_MAX_SYMBOL_VALUE, FSE_MAX_TABLELOG, fse_table_step}; +use crate::error::{Error, Result}; +use crate::util::{SliceExt, write_le64}; + +/// The largest table log of the weights of a Huffman table. +const MAX_LOG: u32 = 6; + +/// `FSE_decode_t` +#[derive(Clone, Copy)] +struct FseDecode { + new_state: u8, + symbol: u8, + nb_bits: u8, + /// As many bits set as `nb_bits`. + mask: u8, +} + +impl FseDecode { + const EMPTY: Self = Self { + new_state: 0, + symbol: 0, + nb_bits: 0, + mask: 0, + }; +} + +/// `FSE_DTable` +struct FseDTable { + table_log: u32, + table: [FseDecode; 1 << MAX_LOG], +} + +/// Where a state leads: to `state` and what the next `nb_bits` are. +pub(crate) struct NextState { + pub(crate) state: u16, + pub(crate) nb_bits: u8, +} + +impl NextState { + /// That of the `n`th state of a symbol that has `count` of them, in a table of `size`, for `2 * size + count + n`. + pub(crate) const fn at(index: u32) -> Self { + let mut next = Self { + state: 0, + nb_bits: 0, + }; + if index >= 4 { + let table_log = index.ilog2() - 1; + let next_state = index - (2 << table_log); + if next_state != 0 { + let nb_bits = table_log - next_state.ilog2(); + next.state = ((next_state << nb_bits) - (1 << table_log)) as u16; + next.nb_bits = nb_bits as u8; + } + } + next + } +} + +/// [`NextState::at`] as an [`FseDecode`] of symbol 0. +static NEXT_STATES: [FseDecode; 4 << MAX_LOG] = { + let mut all = [FseDecode::EMPTY; 4 << MAX_LOG]; + let mut left: &mut [FseDecode] = &mut all; + let mut index = 0; + while let [decode, others @ ..] = left { + let NextState { state, nb_bits } = NextState::at(index); + *decode = FseDecode { + new_state: state as u8, + symbol: 0, + nb_bits, + mask: ((1u32 << nb_bits) - 1) as u8, + }; + index += 1; + left = others; + } + all +}; + +/// The largest table that [`Spread`] is for. +pub(crate) const MAX_SPREAD_SIZE: usize = 1 << 9; + +/// Which symbol each state of a table decodes: what tables are built from. +#[derive(Clone)] +pub(crate) struct Spread { + /// The symbols, each as many times as it has states. + in_order: [u8; MAX_SPREAD_SIZE + 8], + /// The symbol of each state. + by_state: [u8; MAX_SPREAD_SIZE + 8], + /// For each symbol, what to give [`NextState::at`] for its next state. + symbol_next: [u16; FSE_MAX_SYMBOL_VALUE as usize + 1], + /// The symbols of low probability. + rare: [u8; FSE_MAX_SYMBOL_VALUE as usize + 1], + /// Room for the counts that all that is made from. + pub(crate) counts: [i16; FSE_MAX_SYMBOL_VALUE as usize + 1], +} + +impl Spread { + pub(crate) const fn new() -> Self { + Self { + in_order: [0; _], + by_state: [0; _], + symbol_next: [0; _], + rare: [0; _], + counts: [0; _], + } + } + + /// The first two passes of `FSE_buildDTable` and `ZSTD_buildFSETable`, with few branches to mispredict, for the + /// first `nb_symbols` of `counts`. They add up to the size of the table, which is at most [`MAX_SPREAD_SIZE`]. + pub(crate) fn states(&mut self, nb_symbols: usize, table_log: u32) -> Result> { + let table_size = 1usize << table_log; + let table_mask = table_size - 1; + let step = fse_table_step(table_size); + // What `step` times this is 1 more than a multiple of the size of the table. + let mut stride = step; + for _ in 0..2 { + stride = stride.wrapping_mul(2usize.wrapping_sub(step.wrapping_mul(stride))); + } + let mut nb_rare = 0usize; + let mut nb_in_order = 0usize; + let mut eight_times = 0u64; + for ((symbol, &count), next) in self + .counts + .sub(..nb_symbols)? + .iter() + .enumerate() + .zip(&mut self.symbol_next) + { + let is_rare = count < 0; + if let Some(rare) = self.rare.get_mut(nb_rare) { + *rare = symbol as u8; + } + nb_rare += usize::from(is_rare); + *next = (2 * table_size) as u16 + if is_rare { 1 } else { count as u16 }; + let count = if is_rare { 0 } else { count as usize }; + write_le64(&mut self.in_order, nb_in_order, eight_times)?; + if count > 8 { + for done in (8..count).step_by(8) { + write_le64(&mut self.in_order, nb_in_order + done, eight_times)?; + } + } + nb_in_order += count; + eight_times += 0x0101_0101_0101_0101; + } + + if nb_rare == 0 { + // One symbol after the other goes `step` states further, so the symbol of the next state is `stride` symbols + // further. + return Ok(States { + symbols: &self.in_order, + index: 0, + stride: stride & table_mask, + mask: table_mask, + symbol_next: &mut self.symbol_next, + }); + } + + // The states from here on are those of the rare symbols. Others that would go there go to the next place. + let first_rare = table_size - nb_rare.min(table_size); + let by_state = self.by_state.sub_mut(..table_size)?; + let mut position = 0usize; + let mut from = 0usize; + for _ in 0..table_size / 2 { + for position in [position, (position + step) & table_mask] { + let symbol = self.in_order.get(from % MAX_SPREAD_SIZE).copied(); + by_state.put(position, symbol.unwrap_or(0))?; + from += usize::from(position < first_rare); + } + position = (position + 2 * step) & table_mask; + } + for (symbol, &rare) in by_state.iter_mut().rev().zip(self.rare.sub(..nb_rare)?) { + *symbol = rare; + } + Ok(States { + symbols: &self.by_state, + index: 0, + stride: 1, + mask: table_mask, + symbol_next: &mut self.symbol_next, + }) + } +} + +/// The states of a table, from the first on. +pub(crate) struct States<'a> { + symbols: &'a [u8; MAX_SPREAD_SIZE + 8], + /// Where in `symbols` the symbol of the next state is. + index: usize, + /// How much further the one after it is, around `mask`. + stride: usize, + mask: usize, + symbol_next: &'a mut [u16; FSE_MAX_SYMBOL_VALUE as usize + 1], +} + +/// A state of a table. +pub(crate) struct State { + pub(crate) symbol: u8, + /// What to give [`NextState::at`]. + pub(crate) next: u16, +} + +impl States<'_> { + #[inline(always)] + pub(crate) fn next(&mut self) -> State { + let symbol = (self.symbols.get(self.index % MAX_SPREAD_SIZE)).map_or(0, |&symbol| symbol); + self.index = (self.index + self.stride) & self.mask; + let mut next = 0; + if let Some(symbol_next) = self.symbol_next.get_mut(usize::from(symbol)) { + next = *symbol_next; + *symbol_next = next.wrapping_add(1); + } + State { symbol, next } + } +} + +// The C code carves its tables out of a workspace of `HUF_READ_STATS_WORKSPACE_SIZE` bytes, and refuses the headers +// whose tables do not fit. +const fn dtable_size_u32(table_log: u32) -> usize { + 1 + (1 << table_log) +} +const fn build_dtable_wksp_size(table_log: u32, max_symbol_value: u32) -> usize { + size_of::() * (max_symbol_value as usize + 1) + (1 << table_log) + 8 +} +const fn decompress_wksp_size(table_log: u32, max_symbol_value: u32) -> usize { + (dtable_size_u32(table_log) + + 1 + + build_dtable_wksp_size(table_log, max_symbol_value).div_ceil(size_of::()) + + NCOUNT_SIZE / size_of::() + + 1) + * size_of::() +} +const WKSP_SIZE: usize = decompress_wksp_size(MAX_LOG, super::HUF_TABLELOG_MAX - 1); +const NCOUNT_SIZE: usize = size_of::() * (FSE_MAX_SYMBOL_VALUE as usize + 1); + +/// `FSE_buildDTable_internal`, from `spread.counts`. +fn build_dtable( + dt: &mut FseDTable, + spread: &mut Spread, + max_symbol_value: u32, + table_log: u32, + wksp_size: usize, +) -> Result<()> { + if build_dtable_wksp_size(table_log, max_symbol_value) > wksp_size { + return Err(Error::MaxSymbolValueTooLarge); + } + if max_symbol_value > FSE_MAX_SYMBOL_VALUE { + return Err(Error::MaxSymbolValueTooLarge); + } + if table_log > FSE_MAX_TABLELOG { + return Err(Error::TableLogTooLarge); + } + dt.table_log = table_log; + let mut states = spread.states(max_symbol_value as usize + 1, table_log)?; + for decode in dt.table.sub_mut(..1 << table_log)? { + let State { symbol, next } = states.next(); + let next_state = NEXT_STATES.get(usize::from(next) % NEXT_STATES.len()); + *decode = FseDecode { + symbol, + ..*next_state.unwrap_or(&FseDecode::EMPTY) + }; + } + Ok(()) +} + +/// A stream that two states take turns to read. +struct Decoder<'a> { + table: &'a [FseDecode; 1 << MAX_LOG], + /// The state that decodes the next symbol, then the other one. + states: [u32; 2], + /// Zeroes, then the stream. + src: &'a [u8; PADDING + MAX_SRC_SIZE], + /// How many bits are left to read of `src`. + position: u32, +} + +/// No less than is read without a look at what is left, and 8 bytes. +const PADDING: usize = 16; +/// `Decoder::position` at the start of the stream. +const START: u32 = 8 * PADDING as u32; +/// What the size of a stream of weights is below. +const MAX_SRC_SIZE: usize = 128; + +impl Decoder<'_> { + /// The next 57 bits or more, from the highest on. + #[inline(always)] + fn bits(&self) -> u64 { + let end = (self.position as usize).div_ceil(8); + let container = (self.src.get(end.wrapping_sub(8)..end)) + .and_then(|bytes| bytes.try_into().ok()) + .map_or(0, u64::from_le_bytes); + container.rotate_left(self.position.wrapping_neg() & 7) + } + + /// `FSE_initDState` + fn read_state(&mut self, table_log: u32) -> u32 { + let bits = self.bits().rotate_left(table_log); + self.position -= table_log; + bits as u32 & ((1 << table_log) - 1) + } + + /// Decodes as many symbols as `out` takes, which are few enough to take no more than 57 bits. `positions` gets + /// `position` after each. + #[inline(always)] + fn decode(&mut self, out: &mut [u8; N], positions: &mut [u32; N]) { + // The bits that are read go to the other end. + let mut bits = self.bits(); + let [mut state, mut other_state] = self.states; + for (symbol, position) in out.iter_mut().zip(positions) { + let decode = + (self.table.get(state as usize % (1 << MAX_LOG))).unwrap_or(&FseDecode::EMPTY); + *symbol = decode.symbol; + bits = bits.rotate_left(u32::from(decode.nb_bits)); + self.position = self.position.wrapping_sub(u32::from(decode.nb_bits)); + *position = self.position; + state = other_state; + other_state = u32::from(decode.new_state) + (bits as u32 & u32::from(decode.mask)); + } + self.states = [state, other_state]; + } + + fn decode_one(&mut self) -> u8 { + let mut symbol = [0]; + self.decode(&mut symbol, &mut [0]); + let [symbol] = symbol; + symbol + } + + /// Whether more was read than the stream has. + fn is_past_start(&self) -> bool { + self.position < START + } +} + +/// `FSE_decompress_usingDTable_generic` +fn decompress_using_dtable(dst: &mut [u8], src: &[u8], dt: &FseDTable) -> Result { + /// 8 symbols take at most 48 bits. + const GROUP: usize = 8; + + let end_mark = + BitDStream::new(src)?.bits_consumed - 8 * 8usize.saturating_sub(src.len()) as u32; + let mut padded = [0; PADDING + MAX_SRC_SIZE]; + padded.write_at(PADDING, src)?; + let mut decoder = Decoder { + table: &dt.table, + states: [0; 2], + src: &padded, + position: 8 * (PADDING + src.len()) as u32 - end_mark, + }; + decoder.states = [ + decoder.read_state(dt.table_log), + decoder.read_state(dt.table_log), + ]; + if decoder.is_past_start() { + return Err(Error::CorruptionDetected); + } + + // All but the last symbol leave room for one more. + let omax = dst.len(); + let mut op = 0usize; + while let Some(group) = dst.get_mut(op..op + GROUP) + && let Some(group) = group.as_mut_array::() + && op + GROUP < omax + { + let mut positions = [0; GROUP]; + decoder.decode(group, &mut positions); + op += GROUP; + // The symbol after the one that takes more bits than there are is the last. + if let Some(extra) = (positions.iter().rev()).position(|&position| position >= START) { + if extra == 0 { + continue; + } + if extra == 1 { + dst.put(op, decoder.decode_one())?; + } + return Ok(op + 2 - extra); + } + return Ok(op + 2 - GROUP); + } + loop { + if op + 2 > omax { + return Err(Error::DstSizeTooSmall); + } + dst.put(op, decoder.decode_one())?; + op += 1; + if decoder.is_past_start() { + dst.put(op, decoder.decode_one())?; + return Ok(op + 1); + } + } +} + +/// `FSE_decompress_wksp_bmi2`, as `HUF_readStats` calls it. Returns how many weights there are. +pub(crate) fn fse_decompress_weights( + dst: &mut [u8], + src: &[u8], + spread: &mut Spread, +) -> Result { + let header = read_ncount(&mut spread.counts, FSE_MAX_SYMBOL_VALUE, src)?; + if header.table_log > MAX_LOG { + return Err(Error::TableLogTooLarge); + } + let body = src.sub(header.size..)?; + + if decompress_wksp_size(header.table_log, header.max_symbol_value) > WKSP_SIZE { + return Err(Error::TableLogTooLarge); + } + let wksp_size = WKSP_SIZE - NCOUNT_SIZE - dtable_size_u32(header.table_log) * size_of::(); + + let mut dtable = FseDTable { + table_log: 0, + table: [FseDecode::EMPTY; 1 << MAX_LOG], + }; + build_dtable( + &mut dtable, + spread, + header.max_symbol_value, + header.table_log, + wksp_size, + )?; + decompress_using_dtable(dst, body, &dtable) +} diff --git a/src/zstd/common/mod.rs b/src/zstd/common/mod.rs new file mode 100644 index 000000000000..75a19054e2a4 --- /dev/null +++ b/src/zstd/common/mod.rs @@ -0,0 +1,151 @@ +//! What the compressor and the decompressor share: `zstd_internal.h` and `lib/common`. + +pub(crate) mod bitstream; +pub(crate) mod entropy; +pub(crate) mod fse_decompress; +pub(crate) mod xxhash; + +pub(crate) const ZSTD_MAGICNUMBER: u32 = 0xFD2F_B528; +pub(crate) const ZSTD_MAGIC_DICTIONARY: u32 = 0xEC30_A437; +pub(crate) const ZSTD_MAGIC_SKIPPABLE_START: u32 = 0x184D_2A50; +pub(crate) const ZSTD_MAGIC_SKIPPABLE_MASK: u32 = 0xFFFF_FFF0; +pub(crate) const ZSTD_SKIPPABLEHEADERSIZE: usize = 8; + +pub(crate) const ZSTD_BLOCKSIZELOG_MAX: u32 = 17; +pub(crate) const ZSTD_BLOCKSIZE_MAX: usize = 1 << ZSTD_BLOCKSIZELOG_MAX; + +pub(crate) const ZSTD_WINDOWLOG_MAX: u32 = 31; +pub(crate) const ZSTD_WINDOWLOG_ABSOLUTEMIN: u32 = 10; +pub(crate) const ZSTD_WINDOWLOG_LIMIT_DEFAULT: u32 = 27; + +pub(crate) const ZSTD_CONTENTSIZE_UNKNOWN: u64 = u64::MAX; +pub const ZSTD_CONTENTSIZE_ERROR: u64 = u64::MAX - 1; + +pub(crate) const ZSTD_REP_NUM: usize = 3; +pub(crate) const REP_START_VALUE: [u32; ZSTD_REP_NUM] = [1, 4, 8]; + +pub(crate) const ZSTD_FCS_FIELD_SIZE: [usize; 4] = [0, 2, 4, 8]; +pub(crate) const ZSTD_DID_FIELD_SIZE: [usize; 4] = [0, 1, 2, 4]; + +pub(crate) const ZSTD_FRAMEIDSIZE: usize = 4; +pub(crate) const ZSTD_BLOCKHEADERSIZE: usize = 3; +pub(crate) const ZSTD_FRAMEHEADERSIZE_MAX: usize = 18; + +pub(crate) const ZSTD_WORKSPACETOOLARGE_FACTOR: usize = 3; +pub(crate) const ZSTD_WORKSPACETOOLARGE_MAXDURATION: usize = 128; + +/// `blockType_e` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum BlockType { + Raw, + Rle, + Compressed, + Reserved, +} + +/// `SymbolEncodingType_e` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum SymbolEncodingType { + Basic, + Rle, + Compressed, + Repeat, +} + +impl SymbolEncodingType { + #[inline] + pub(crate) const fn from_bits(bits: u8) -> Self { + match bits & 3 { + 0 => Self::Basic, + 1 => Self::Rle, + 2 => Self::Compressed, + _ => Self::Repeat, + } + } +} + +pub(crate) const MIN_SEQUENCES_SIZE: usize = 1; +pub(crate) const MIN_CBLOCK_SIZE: usize = 1 + 1; +pub(crate) const MIN_LITERALS_FOR_4_STREAMS: usize = 6; + +pub(crate) const LONGNBSEQ: usize = 0x7F00; + +pub(crate) const MINMATCH: u32 = 3; + +pub(crate) const LITBITS: u32 = 8; +pub(crate) const LIT_HUF_LOG: u32 = 11; +pub(crate) const MAX_ML: u32 = 52; +pub(crate) const MAX_LL: u32 = 35; +pub(crate) const DEFAULT_MAX_OFF: u32 = 28; +pub(crate) const MAX_OFF: u32 = 31; +pub(crate) const MAX_SEQ: u32 = MAX_ML; +pub(crate) const ML_FSE_LOG: u32 = 9; +pub(crate) const LL_FSE_LOG: u32 = 9; +pub(crate) const OFF_FSE_LOG: u32 = 8; + +#[rustfmt::skip] +pub(crate) const LL_BITS: [u8; MAX_LL as usize + 1] = [ + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 2, 2, 3, 3, + 4, 6, 7, 8, 9,10,11,12, + 13,14,15,16, +]; + +#[rustfmt::skip] +pub(crate) const LL_DEFAULT_NORM: [i16; MAX_LL as usize + 1] = [ + 4, 3, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 1, 1, 1, + 2, 2, 2, 2, 2, 2, 2, 2, + 2, 3, 2, 1, 1, 1, 1, 1, + -1,-1,-1,-1, +]; +pub(crate) const LL_DEFAULT_NORM_LOG: u32 = 6; + +#[rustfmt::skip] +pub(crate) const ML_BITS: [u8; MAX_ML as usize + 1] = [ + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 2, 2, 3, 3, + 4, 4, 5, 7, 8, 9,10,11, + 12,13,14,15,16, +]; + +#[rustfmt::skip] +pub(crate) const ML_DEFAULT_NORM: [i16; MAX_ML as usize + 1] = [ + 1, 4, 3, 2, 2, 2, 2, 2, + 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1,-1,-1, + -1,-1,-1,-1,-1, +]; +pub(crate) const ML_DEFAULT_NORM_LOG: u32 = 6; + +#[rustfmt::skip] +pub(crate) const OF_DEFAULT_NORM: [i16; DEFAULT_MAX_OFF as usize + 1] = [ + 1, 1, 1, 1, 1, 1, 2, 2, + 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + -1,-1,-1,-1,-1, +]; +pub(crate) const OF_DEFAULT_NORM_LOG: u32 = 5; + +pub(crate) const WILDCOPY_OVERLENGTH: usize = 32; + +pub(crate) const FSE_MAX_SYMBOL_VALUE: u32 = 255; +pub(crate) const FSE_MAX_TABLELOG: u32 = 12; +pub(crate) const FSE_MIN_TABLELOG: u32 = 5; +pub(crate) const FSE_TABLELOG_ABSOLUTE_MAX: u32 = 15; + +/// `FSE_TABLESTEP` +#[inline(always)] +pub(crate) const fn fse_table_step(table_size: usize) -> usize { + (table_size >> 1) + (table_size >> 3) + 3 +} + +pub(crate) const HUF_TABLELOG_MAX: u32 = 12; +pub(crate) const HUF_SYMBOLVALUE_MAX: u32 = 255; diff --git a/src/zstd/common/xxhash.rs b/src/zstd/common/xxhash.rs new file mode 100644 index 000000000000..c45636fd457d --- /dev/null +++ b/src/zstd/common/xxhash.rs @@ -0,0 +1,139 @@ +//! XXH64, which is the checksum of a frame. + +const PRIME64_1: u64 = 0x9E37_79B1_85EB_CA87; +const PRIME64_2: u64 = 0xC2B2_AE3D_27D4_EB4F; +const PRIME64_3: u64 = 0x1656_67B1_9E37_79F9; +const PRIME64_4: u64 = 0x85EB_CA77_C2B2_AE63; +const PRIME64_5: u64 = 0x27D4_EB2F_1656_67C5; + +const STRIPE: usize = 32; + +#[inline(always)] +fn round(acc: u64, input: u64) -> u64 { + acc.wrapping_add(input.wrapping_mul(PRIME64_2)) + .rotate_left(31) + .wrapping_mul(PRIME64_1) +} + +#[inline(always)] +fn merge_round(acc: u64, val: u64) -> u64 { + (acc ^ round(0, val)) + .wrapping_mul(PRIME64_1) + .wrapping_add(PRIME64_4) +} + +/// `XXH64_state_t`, with a seed of 0. +#[derive(Clone)] +pub(crate) struct Xxh64 { + total_len: u64, + acc: [u64; 4], + buffer: [u8; STRIPE], + buffered: usize, +} + +impl Xxh64 { + /// `XXH64_reset` + pub(crate) const fn new() -> Self { + Self { + total_len: 0, + acc: [ + PRIME64_1.wrapping_add(PRIME64_2), + PRIME64_2, + 0, + 0u64.wrapping_sub(PRIME64_1), + ], + buffer: [0; STRIPE], + buffered: 0, + } + } + + #[inline(always)] + fn consume(acc: &mut [u64; 4], stripe: &[u8; STRIPE]) { + let (lanes, _) = stripe.as_chunks::<8>(); + for (acc, lane) in acc.iter_mut().zip(lanes) { + *acc = round(*acc, u64::from_le_bytes(*lane)); + } + } + + /// `XXH64_update` + pub(crate) fn update(&mut self, mut input: &[u8]) { + self.total_len = self.total_len.wrapping_add(input.len() as u64); + + if self.buffered > 0 { + let Some(free) = self.buffer.get_mut(self.buffered..) else { + return; + }; + let Some((head, tail)) = input.split_at_checked(free.len()) else { + if let Some(free) = free.get_mut(..input.len()) { + free.copy_from_slice(input); + } + self.buffered += input.len(); + return; + }; + free.copy_from_slice(head); + Self::consume(&mut self.acc, &self.buffer); + self.buffered = 0; + input = tail; + } + + let (stripes, rest) = input.as_chunks::(); + let mut acc = self.acc; + for stripe in stripes { + Self::consume(&mut acc, stripe); + } + self.acc = acc; + + if let Some(buffer) = self.buffer.get_mut(..rest.len()) { + buffer.copy_from_slice(rest); + } + self.buffered = rest.len(); + } + + /// `XXH64_digest` + pub(crate) fn digest(&self) -> u64 { + let [v1, v2, v3, v4] = self.acc; + let mut hash = if self.total_len >= STRIPE as u64 { + let mut hash = v1 + .rotate_left(1) + .wrapping_add(v2.rotate_left(7)) + .wrapping_add(v3.rotate_left(12)) + .wrapping_add(v4.rotate_left(18)); + for v in self.acc { + hash = merge_round(hash, v); + } + hash + } else { + v3.wrapping_add(PRIME64_5) + }; + hash = hash.wrapping_add(self.total_len); + + let tail = self.buffer.get(..self.buffered).unwrap_or_default(); + let (lanes, tail) = tail.as_chunks::<8>(); + for lane in lanes { + hash ^= round(0, u64::from_le_bytes(*lane)); + hash = hash + .rotate_left(27) + .wrapping_mul(PRIME64_1) + .wrapping_add(PRIME64_4); + } + let (words, tail) = tail.as_chunks::<4>(); + for word in words { + hash ^= u64::from(u32::from_le_bytes(*word)).wrapping_mul(PRIME64_1); + hash = hash + .rotate_left(23) + .wrapping_mul(PRIME64_2) + .wrapping_add(PRIME64_3); + } + for &byte in tail { + hash ^= u64::from(byte).wrapping_mul(PRIME64_5); + hash = hash.rotate_left(11).wrapping_mul(PRIME64_1); + } + + hash ^= hash >> 33; + hash = hash.wrapping_mul(PRIME64_2); + hash ^= hash >> 29; + hash = hash.wrapping_mul(PRIME64_3); + hash ^= hash >> 32; + hash + } +} diff --git a/src/zstd/compress/block_entropy.rs b/src/zstd/compress/block_entropy.rs new file mode 100644 index 000000000000..a71a7c6cd989 --- /dev/null +++ b/src/zstd/compress/block_entropy.rs @@ -0,0 +1,677 @@ +//! The part of `zstd_compress.c` that turns the sequences and the literals of a block into its content. + +use super::fse::{FseCTable, FseRepeat}; +use super::huf::{self, HUF_OPTIMAL_DEPTH_THRESHOLD, HufFlags, HufRepeat}; +use super::internal::{ + EntropyCTables, EntropyCTablesMetadata, FseCTables, FseCTablesMetadata, HufCTables, + HufCTablesMetadata, LongLengthType, SeqChunk, SeqStore, Strategy, min_gain, +}; +use super::literals::compress_literals; +use super::params::CCtxParams; +use super::sequences::{ + self, CodeCounts, CodeKind, LL_KIND, ML_KIND, OF_KIND, SeqCodes, SeqHistograms, build_ctable, + cross_entropy_cost, encode_sequences, fse_bit_cost, select_encoding_type, summarize, +}; +use super::{hist, huf::HufCTable}; +use crate::common::{ + HUF_SYMBOLVALUE_MAX, LIT_HUF_LOG, LL_BITS, LL_DEFAULT_NORM, LL_DEFAULT_NORM_LOG, LONGNBSEQ, + MINMATCH, ML_BITS, ML_DEFAULT_NORM, ML_DEFAULT_NORM_LOG, OF_DEFAULT_NORM, OF_DEFAULT_NORM_LOG, + SymbolEncodingType, ZSTD_BLOCKHEADERSIZE, +}; +use crate::error::{Error, Result}; +use crate::util::{SliceExt, write_le16}; + +/// `ZSTD_literalsCompressionIsDisabled` +pub(crate) fn literals_compression_is_disabled(cctx_params: &CCtxParams) -> bool { + cctx_params.c_params.strat() == Strategy::Fast && cctx_params.c_params.target_length > 0 +} + +/// `ZSTD_seqToCodes`, and the distributions of the codes. +fn seq_to_codes(store: &mut SeqStore, chunk: &SeqChunk) -> Result { + let range = chunk.seq_start..chunk.seq_end; + store.make_room_for_codes()?; + Ok(sequences::seq_to_codes( + store.codes.sub_mut(range.clone())?, + store.sequences.sub(range)?, + chunk.long_length_type, + chunk.long_length_pos as usize, + )) +} + +/// `ZSTD_symbolEncodingTypeStats_t` +struct SymbolEncodingTypeStats { + ll_type: SymbolEncodingType, + off_type: SymbolEncodingType, + ml_type: SymbolEncodingType, + /// Of the headers of the tables. + size: usize, + /// The size of the last header written for a table, 0 if there is none. + last_count_size: usize, +} + +/// The table of one kind of code in the previous block. +#[derive(Clone, Copy)] +struct PrevTable<'a> { + table: &'a FseCTable, + repeat_mode: FseRepeat, +} + +/// The header of a table. +struct TableHeader { + ty: SymbolEncodingType, + size: usize, +} + +/// Chooses how to encode `nb_seq` codes, the last of which is `last_code` and of which `count` is the distribution, +/// makes the table and writes its header. +fn build_table( + kind: &CodeKind, + prev: PrevTable<'_>, + next_table: &mut FseCTable, + next_repeat_mode: &mut FseRepeat, + count: &mut CodeCounts, + nb_seq: usize, + last_code: u8, + dst: &mut [u8], + strategy: Strategy, +) -> Result { + let summary = summarize(count); + let max = summary.max_symbol_value; + *next_repeat_mode = prev.repeat_mode; + let ty = select_encoding_type( + next_repeat_mode, + count, + max, + summary.largest, + nb_seq, + kind.fse_log, + prev.table, + kind.default_norm, + kind.default_norm_log, + // The predefined table has no code for the largest offsets. + max <= kind.default_max, + strategy, + )?; + let size = build_ctable( + dst, next_table, kind, ty, count, max, nb_seq, last_code, prev.table, + )?; + Ok(TableHeader { ty, size }) +} + +/// `ZSTD_buildSequencesStatistics`: makes the tables for `nb_seq` sequences, which is not 0, the last of which has the +/// codes `last_codes`, and writes their headers to `dst`. +fn build_sequences_statistics( + histograms: &mut SeqHistograms, + nb_seq: usize, + last_codes: SeqCodes, + prev_entropy: &FseCTables, + next_entropy: &mut FseCTables, + dst: &mut [u8], + strategy: Strategy, +) -> Result { + let mut op = 0usize; + let mut last_count_size = 0; + let mut written = |header: &TableHeader| { + if header.ty == SymbolEncodingType::Compressed { + last_count_size = header.size; + } + op += header.size; + op + }; + + let ll = build_table( + &LL_KIND, + PrevTable { + table: &prev_entropy.litlength_c_table, + repeat_mode: prev_entropy.litlength_repeat_mode, + }, + &mut next_entropy.litlength_c_table, + &mut next_entropy.litlength_repeat_mode, + &mut histograms.ll, + nb_seq, + last_codes.ll as u8, + dst, + strategy, + )?; + let op_of = written(&ll); + let of = build_table( + &OF_KIND, + PrevTable { + table: &prev_entropy.offcode_c_table, + repeat_mode: prev_entropy.offcode_repeat_mode, + }, + &mut next_entropy.offcode_c_table, + &mut next_entropy.offcode_repeat_mode, + &mut histograms.of, + nb_seq, + last_codes.of as u8, + dst.sub_mut(op_of..)?, + strategy, + )?; + let op_ml = written(&of); + let ml = build_table( + &ML_KIND, + PrevTable { + table: &prev_entropy.matchlength_c_table, + repeat_mode: prev_entropy.matchlength_repeat_mode, + }, + &mut next_entropy.matchlength_c_table, + &mut next_entropy.matchlength_repeat_mode, + &mut histograms.ml, + nb_seq, + last_codes.ml as u8, + dst.sub_mut(op_ml..)?, + strategy, + )?; + let size = written(&ml); + Ok(SymbolEncodingTypeStats { + ll_type: ll.ty, + off_type: of.ty, + ml_type: ml.ty, + size, + last_count_size, + }) +} + +/// Writes how many sequences a block has. `dst` has room for 3 bytes. Returns how many it takes. +pub(crate) fn write_nb_seq(dst: &mut [u8], nb_seq: usize) -> Result { + if nb_seq < 128 { + dst.put(0, nb_seq as u8)?; + Ok(1) + } else if nb_seq < LONGNBSEQ { + dst.put(0, ((nb_seq >> 8) + 0x80) as u8)?; + dst.put(1, nb_seq as u8)?; + Ok(2) + } else { + dst.put(0, 0xFF)?; + write_le16(dst, 1, (nb_seq - LONGNBSEQ) as u16)?; + Ok(3) + } +} + +/// `ZSTD_entropyCompressSeqStore_internal`: returns 0 if the block is better left uncompressed. +fn entropy_compress_seq_store_internal( + dst: &mut [u8], + store: &mut SeqStore, + chunk: &SeqChunk, + prev_entropy: &EntropyCTables, + next_entropy: &mut EntropyCTables, + cctx_params: &CCtxParams, + sampled_literals: Option<&[u8]>, +) -> Result { + const SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO: usize = 20; + let strategy = cctx_params.c_params.strat(); + let nb_seq = chunk.nb_seq(); + let lit_size = chunk.lit_size(); + let mut op = 0usize; + debug_assert!(sampled_literals.is_none() || nb_seq == 0); + + { + let suspect_uncompressible = sampled_literals.is_none() + && (nb_seq == 0 || lit_size / nb_seq >= SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO); + op += compress_literals( + dst, + match sampled_literals { + Some(literals) => literals, + None => store.lit_buffer.sub(chunk.lit_start..chunk.lit_end)?, + }, + &prev_entropy.huf, + &mut next_entropy.huf, + strategy, + literals_compression_is_disabled(cctx_params), + suspect_uncompressible, + )?; + } + + if dst.len() - op < 3 + 1 { + return Err(Error::DstSizeTooSmall); + } + op += write_nb_seq(dst.sub_mut(op..)?, nb_seq)?; + if nb_seq == 0 { + next_entropy.fse.copy_from(&prev_entropy.fse); + return Ok(op); + } + let (last_count_size, many_bits) = { + let seq_head = op; + op += 1; + let mut histograms = seq_to_codes(store, chunk)?; + let stats = build_sequences_statistics( + &mut histograms, + nb_seq, + store.codes.at(chunk.seq_end - 1)?, + &prev_entropy.fse, + &mut next_entropy.fse, + dst.sub_mut(op..)?, + strategy, + )?; + dst.put( + seq_head, + (((stats.ll_type as u32) << 6) + + ((stats.off_type as u32) << 4) + + ((stats.ml_type as u32) << 2)) as u8, + )?; + op += stats.size; + (stats.last_count_size, histograms.many_bits) + }; + + let range = chunk.seq_start..chunk.seq_end; + let bitstream_size = encode_sequences( + dst.sub_mut(op..)?, + &next_entropy.fse, + store.codes.sub(range.clone())?, + store.sequences.sub(range)?, + many_bits, + )?; + op += bitstream_size; + // Decoders up to 1.3.4 fail on a table header with fewer than 4 bytes after it. + if last_count_size != 0 && last_count_size + bitstream_size < 4 { + return Ok(0); + } + Ok(op) +} + +/// `ZSTD_entropyCompressSeqStore`: the content of a compressed block for `chunk`, which stands for `block_size` bytes. +/// Returns 0 if the block is better left uncompressed. A block without sequences can have its literals in +/// `sampled_literals` instead of the store, if [`huf::samples_are_incompressible`] has been asked and is false for them. +pub(crate) fn entropy_compress_seq_store( + store: &mut SeqStore, + chunk: &SeqChunk, + prev_entropy: &EntropyCTables, + next_entropy: &mut EntropyCTables, + cctx_params: &CCtxParams, + dst: &mut [u8], + block_size: usize, + sampled_literals: Option<&[u8]>, +) -> Result { + let c_size = match entropy_compress_seq_store_internal( + dst, + store, + chunk, + prev_entropy, + next_entropy, + cctx_params, + sampled_literals, + ) { + // There is room for the block as it is. + Err(Error::DstSizeTooSmall) if block_size <= dst.len() => return Ok(0), + c_size => c_size?, + }; + let max_c_size = block_size.wrapping_sub(min_gain(block_size, cctx_params.c_params.strat())); + if c_size >= max_c_size { + return Ok(0); + } + Ok(c_size) +} + +/// What the C code has for the size of something that failed: an error code, which is more than any size. +fn size_or_error(size: Result) -> usize { + size.unwrap_or_else(|code| 0usize.wrapping_sub(code as usize)) +} + +/// `ZSTD_buildBlockEntropyStats_literals`: makes the table for the literals `src`, and its description in +/// `huf_metadata`. Returns the size of the description. +fn build_block_entropy_stats_literals( + src: &[u8], + prev_huf: &HufCTables, + next_huf: &mut HufCTables, + huf_metadata: &mut HufCTablesMetadata, + literals_compression_is_disabled: bool, + huf_flags: HufFlags, +) -> Result { + const COMPRESS_LITERALS_SIZE_MIN: usize = 63; + let mut count = [0u32; HUF_SYMBOLVALUE_MAX as usize + 1]; + let src_size = src.len(); + let mut max_symbol_value = HUF_SYMBOLVALUE_MAX; + let mut repeat = prev_huf.repeat_mode; + + next_huf.copy_from(prev_huf); + + if literals_compression_is_disabled { + huf_metadata.h_type = SymbolEncodingType::Basic; + return Ok(0); + } + + let min_lit_size = if prev_huf.repeat_mode == HufRepeat::Valid { + 6 + } else { + COMPRESS_LITERALS_SIZE_MIN + }; + if src_size <= min_lit_size { + huf_metadata.h_type = SymbolEncodingType::Basic; + return Ok(0); + } + + let largest = hist::count_wksp(&mut count, &mut max_symbol_value, src)?; + if largest == src_size { + huf_metadata.h_type = SymbolEncodingType::Rle; + return Ok(0); + } + if largest <= (src_size >> 7) + 4 { + huf_metadata.h_type = SymbolEncodingType::Basic; + return Ok(0); + } + + if repeat == HufRepeat::Check + && !huf::validate_ctable(&prev_huf.c_table, &count, max_symbol_value) + { + repeat = HufRepeat::None; + } + + next_huf.c_table = HufCTable::new(); + let huff_log = huf::optimal_table_log( + LIT_HUF_LOG, + src_size, + max_symbol_value, + &mut next_huf.c_table, + &count, + huf_flags, + ); + let huff_log = + huf::build_ctable(&mut next_huf.c_table, &count, max_symbol_value, huff_log)? as u32; + + let new_c_size = huf::estimate_compressed_size(&next_huf.c_table, &count, max_symbol_value); + let h_size_result = huf::write_ctable( + &mut huf_metadata.huf_des_buffer, + &next_huf.c_table, + max_symbol_value, + huff_log, + ); + let h_size = size_or_error(h_size_result); + if repeat != HufRepeat::None { + let old_c_size = huf::estimate_compressed_size(&prev_huf.c_table, &count, max_symbol_value); + if old_c_size < src_size + && (old_c_size <= h_size.wrapping_add(new_c_size) + || h_size.wrapping_add(12) >= src_size) + { + next_huf.copy_from(prev_huf); + huf_metadata.h_type = SymbolEncodingType::Repeat; + return Ok(0); + } + } + if new_c_size.wrapping_add(h_size) >= src_size { + next_huf.copy_from(prev_huf); + huf_metadata.h_type = SymbolEncodingType::Basic; + return Ok(0); + } + huf_metadata.h_type = SymbolEncodingType::Compressed; + next_huf.repeat_mode = HufRepeat::Check; + h_size_result +} + +/// `ZSTD_buildBlockEntropyStats_sequences`: makes the tables for the sequences of `chunk`, and their description in +/// `fse_metadata`. Returns the distributions of the codes. +fn build_block_entropy_stats_sequences( + store: &mut SeqStore, + chunk: &SeqChunk, + prev_entropy: &FseCTables, + next_entropy: &mut FseCTables, + cctx_params: &CCtxParams, + fse_metadata: &mut FseCTablesMetadata, +) -> Result { + let mut histograms = seq_to_codes(store, chunk)?; + let stats = if chunk.nb_seq() != 0 { + build_sequences_statistics( + &mut histograms, + chunk.nb_seq(), + store.codes.at(chunk.seq_end - 1)?, + prev_entropy, + next_entropy, + &mut fse_metadata.fse_tables_buffer, + cctx_params.c_params.strat(), + )? + } else { + next_entropy.litlength_repeat_mode = FseRepeat::None; + next_entropy.offcode_repeat_mode = FseRepeat::None; + next_entropy.matchlength_repeat_mode = FseRepeat::None; + SymbolEncodingTypeStats { + ll_type: SymbolEncodingType::Basic, + off_type: SymbolEncodingType::Basic, + ml_type: SymbolEncodingType::Basic, + size: 0, + last_count_size: 0, + } + }; + fse_metadata.ll_type = stats.ll_type; + fse_metadata.of_type = stats.off_type; + fse_metadata.ml_type = stats.ml_type; + fse_metadata.last_count_size = stats.last_count_size; + fse_metadata.fse_tables_size = stats.size; + Ok(histograms) +} + +/// `ZSTD_buildBlockEntropyStats`: makes the tables for `chunk` in `next_entropy`, and what it takes to write them in +/// `entropy_metadata`. Returns the distributions of the codes of the sequences. +pub(crate) fn build_block_entropy_stats( + store: &mut SeqStore, + chunk: &SeqChunk, + prev_entropy: &EntropyCTables, + next_entropy: &mut EntropyCTables, + cctx_params: &CCtxParams, + entropy_metadata: &mut EntropyCTablesMetadata, +) -> Result { + let huf_flags = HufFlags { + optimal_depth: cctx_params.c_params.strat() >= HUF_OPTIMAL_DEPTH_THRESHOLD, + ..HufFlags::default() + }; + entropy_metadata.huf_metadata.huf_des_size = build_block_entropy_stats_literals( + store.lit_buffer.sub(chunk.lit_start..chunk.lit_end)?, + &prev_entropy.huf, + &mut next_entropy.huf, + &mut entropy_metadata.huf_metadata, + literals_compression_is_disabled(cctx_params), + huf_flags, + )?; + build_block_entropy_stats_sequences( + store, + chunk, + &prev_entropy.fse, + &mut next_entropy.fse, + cctx_params, + &mut entropy_metadata.fse_metadata, + ) +} + +/// `ZSTD_estimateBlockSize_literal` +fn estimate_block_size_literal( + literals: &[u8], + huf: &HufCTables, + huf_metadata: &HufCTablesMetadata, + write_entropy: bool, +) -> usize { + let mut count = [0u32; HUF_SYMBOLVALUE_MAX as usize + 1]; + let mut max_symbol_value = HUF_SYMBOLVALUE_MAX; + let lit_size = literals.len(); + let literal_section_header_size = + 3 + usize::from(lit_size >= 1 << 10) + usize::from(lit_size >= 16 << 10); + let single_stream = lit_size < 256; + + match huf_metadata.h_type { + SymbolEncodingType::Basic => lit_size, + SymbolEncodingType::Rle => 1, + SymbolEncodingType::Compressed | SymbolEncodingType::Repeat => { + if hist::count_wksp(&mut count, &mut max_symbol_value, literals).is_err() { + return lit_size; + } + let mut c_lit_size_estimate = + huf::estimate_compressed_size(&huf.c_table, &count, max_symbol_value); + if write_entropy { + c_lit_size_estimate += huf_metadata.huf_des_size; + } + if !single_stream { + // The jump table. + c_lit_size_estimate += 6; + } + c_lit_size_estimate + literal_section_header_size + } + } +} + +/// `ZSTD_estimateBlockSize_symbolType`: `count` is the distribution of `nb_seq` codes. +fn estimate_block_size_symbol_type( + ty: SymbolEncodingType, + count: &CodeCounts, + nb_seq: usize, + fse_c_table: &FseCTable, + additional_bits: Option<&[u8]>, + default_norm: &[i16], + default_norm_log: u32, +) -> Result { + let max = summarize(count).max_symbol_value; + let estimate_in_bits = match ty { + SymbolEncodingType::Basic => cross_entropy_cost(default_norm, default_norm_log, count, max), + SymbolEncodingType::Rle => Ok(0), + SymbolEncodingType::Compressed | SymbolEncodingType::Repeat => { + fse_bit_cost(fse_c_table, count, max) + } + }; + let Ok(estimate_in_bits) = estimate_in_bits else { + return Ok(nb_seq * 10); + }; + Ok((estimate_in_bits + additional_bits_of(count, additional_bits)) >> 3) +} + +/// How many bits follow the codes counted in `count`: as many as the code itself says, or as `additional_bits` does. +pub(crate) fn additional_bits_of(count: &CodeCounts, additional_bits: Option<&[u8]>) -> usize { + match additional_bits { + Some(bits) => (count.iter().zip(bits)) + .map(|(&count, &bits)| count as usize * usize::from(bits)) + .sum(), + None => (count.iter().enumerate()) + .map(|(code, &count)| count as usize * code) + .sum(), + } +} + +/// `ZSTD_estimateBlockSize_sequences` +fn estimate_block_size_sequences( + histograms: &SeqHistograms, + nb_seq: usize, + fse_tables: &FseCTables, + fse_metadata: &FseCTablesMetadata, + write_entropy: bool, +) -> Result { + let sequences_section_header_size = + 1 + 1 + usize::from(nb_seq >= 128) + usize::from(nb_seq >= LONGNBSEQ); + let mut c_seq_size_estimate = estimate_block_size_symbol_type( + fse_metadata.of_type, + &histograms.of, + nb_seq, + &fse_tables.offcode_c_table, + None, + &OF_DEFAULT_NORM, + OF_DEFAULT_NORM_LOG, + )?; + c_seq_size_estimate += estimate_block_size_symbol_type( + fse_metadata.ll_type, + &histograms.ll, + nb_seq, + &fse_tables.litlength_c_table, + Some(&LL_BITS), + &LL_DEFAULT_NORM, + LL_DEFAULT_NORM_LOG, + )?; + c_seq_size_estimate += estimate_block_size_symbol_type( + fse_metadata.ml_type, + &histograms.ml, + nb_seq, + &fse_tables.matchlength_c_table, + Some(&ML_BITS), + &ML_DEFAULT_NORM, + ML_DEFAULT_NORM_LOG, + )?; + if write_entropy { + c_seq_size_estimate += fse_metadata.fse_tables_size; + } + Ok(c_seq_size_estimate + sequences_section_header_size) +} + +/// `ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize`: about how large a block for `chunk` would be. Overwrites +/// `next_entropy`. +pub(crate) fn build_entropy_statistics_and_estimate_sub_block_size( + store: &mut SeqStore, + chunk: &SeqChunk, + prev_entropy: &EntropyCTables, + next_entropy: &mut EntropyCTables, + cctx_params: &CCtxParams, + entropy_metadata: &mut EntropyCTablesMetadata, +) -> Result { + let histograms = build_block_entropy_stats( + store, + chunk, + prev_entropy, + next_entropy, + cctx_params, + entropy_metadata, + )?; + let literals_size = estimate_block_size_literal( + store.lit_buffer.sub(chunk.lit_start..chunk.lit_end)?, + &next_entropy.huf, + &entropy_metadata.huf_metadata, + entropy_metadata.huf_metadata.h_type == SymbolEncodingType::Compressed, + ); + let seq_size = estimate_block_size_sequences( + &histograms, + chunk.nb_seq(), + &next_entropy.fse, + &entropy_metadata.fse_metadata, + true, + )?; + Ok(seq_size + literals_size + ZSTD_BLOCKHEADERSIZE) +} + +/// `ZSTD_countSeqStoreLiteralsBytes` +pub(crate) fn count_seq_store_literals_bytes(store: &SeqStore, chunk: &SeqChunk) -> Result { + let sequences = store.sequences.sub(chunk.seq_start..chunk.seq_end)?; + let mut literals_bytes: usize = sequences + .iter() + .map(|seq| usize::from(seq.lit_length)) + .sum(); + if chunk.long_length_type == LongLengthType::LiteralLength + && (chunk.long_length_pos as usize) < sequences.len() + { + literals_bytes += 0x10000; + } + Ok(literals_bytes) +} + +/// `ZSTD_countSeqStoreMatchBytes` +pub(crate) fn count_seq_store_match_bytes(store: &SeqStore, chunk: &SeqChunk) -> Result { + let sequences = store.sequences.sub(chunk.seq_start..chunk.seq_end)?; + let mut match_bytes: usize = (sequences.iter()) + .map(|seq| usize::from(seq.ml_base) + MINMATCH as usize) + .sum(); + if chunk.long_length_type == LongLengthType::MatchLength + && (chunk.long_length_pos as usize) < sequences.len() + { + match_bytes += 0x10000; + } + Ok(match_bytes) +} + +/// `ZSTD_deriveSeqStoreChunk`: the sequences of `store` from `start_idx` to `end_idx`. +pub(crate) fn derive_seq_store_chunk( + store: &SeqStore, + start_idx: usize, + end_idx: usize, +) -> Result { + let mut result = store.whole(); + if start_idx > 0 { + result.seq_end = start_idx; + result.lit_start += count_seq_store_literals_bytes(store, &result)?; + } + if store.long_length_type != LongLengthType::None { + let long_length_pos = store.long_length_pos as usize; + if long_length_pos < start_idx || long_length_pos > end_idx { + result.long_length_type = LongLengthType::None; + } else { + result.long_length_pos -= start_idx as u32; + } + } + result.seq_start = start_idx; + result.seq_end = end_idx; + if end_idx != store.sequences.len() { + // Only the last chunk has the literals that follow the last sequence. + result.lit_end = result.lit_start + count_seq_store_literals_bytes(store, &result)?; + } + Ok(result) +} diff --git a/src/zstd/compress/cctx.rs b/src/zstd/compress/cctx.rs new file mode 100644 index 000000000000..f2bc9bd24a74 --- /dev/null +++ b/src/zstd/compress/cctx.rs @@ -0,0 +1,2004 @@ +//! The part of `zstd_compress.c` that writes frames: `ZSTD_compressBegin`, `ZSTD_compressContinue`, `ZSTD_compressEnd` +//! and what they need. + +use std::sync::Arc; +use std::sync::atomic::AtomicBool; + +use super::block_entropy::{ + build_entropy_statistics_and_estimate_sub_block_size, count_seq_store_literals_bytes, + count_seq_store_match_bytes, derive_seq_store_chunk, entropy_compress_seq_store, + literals_compression_is_disabled, +}; +use super::fse::FseRepeat; +use super::huf::{self, HufRepeat}; +use super::internal::{ + BlockSrc, BufId, Buffers, CompressedBlockState, CompressionParameters, DictMatchState, + EntropyCTablesMetadata, HASH_READ_SIZE, LdmState, LongLengthType, MatchState, OptState, + ParamSwitch, Prefix, PriceTable, RawSeq, RawSeqStore, RowTags, SeqChunk, SeqStore, Strategy, + Table, TableFillPurpose, Window, ZSTD_CHUNKSIZE_MAX, ZSTD_CURRENT_MAX, ZSTD_DUBT_UNSORTED_MARK, + ZSTD_OPT_SIZE, ZSTD_SHORT_CACHE_TAG_BITS, ZSTD_WINDOW_START_INDEX, block_header, min_gain, + no_compress_block, offbase_is_repcode, offset_to_offbase, rle_compress_block, update_rep, +}; +use super::literals::min_literals_to_compress; +use super::params::{ + CCtxParams, CParamMode, Format, MatchStateSizes, Workspace, WorkspaceNeeds, + adjust_c_params_internal, allocate_chain_table, cdict_indices_are_tagged, compress_bound, + cycle_log, get_c_params_from_cctx_params, max_nb_seq, resolve_row_match_finder_mode, + row_match_finder_used, +}; +use super::superblock::compress_super_block; +use super::{compress_block, double_fast, fast, lazy, ldm, opt, presplit, sequences}; +use crate::common::entropy::read_ncount; +use crate::common::xxhash::Xxh64; +use crate::common::{ + BlockType, LITBITS, LL_FSE_LOG, MAX_LL, MAX_ML, MAX_OFF, MIN_CBLOCK_SIZE, ML_FSE_LOG, + OFF_FSE_LOG, ZSTD_BLOCKHEADERSIZE, ZSTD_BLOCKSIZE_MAX, ZSTD_CONTENTSIZE_UNKNOWN, + ZSTD_FRAMEHEADERSIZE_MAX, ZSTD_MAGIC_DICTIONARY, ZSTD_MAGICNUMBER, ZSTD_REP_NUM, + ZSTD_WINDOWLOG_ABSOLUTEMIN, +}; +use crate::error::{Error, Result}; +use crate::util::{ + SliceExt, append, append_at, highbit32, read_le32, reset_table, write_le16, write_le24, + write_le32, write_le64, +}; + +/// `ZSTD_compressionStage_e` +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum CompressionStage { + Created, + Init, + Ongoing, + Ending, +} + +/// `ZSTD_compResetPolicy_e` +#[derive(Clone, Copy, PartialEq, Eq)] +enum CompResetPolicy { + MakeClean, + /// The caller overwrites the tables. + LeaveDirty, +} + +/// `ZSTD_indexResetPolicy_e` +#[derive(Clone, Copy, PartialEq, Eq)] +enum IndexResetPolicy { + /// Indices go on from where they are, which makes the ones in the tables invalid without erasing them. + Continue, + Reset, +} + +/// `ZSTD_resetTarget_e` +#[derive(Clone, Copy, PartialEq, Eq)] +enum ResetTarget { + CDict, + CCtx, +} + +/// Some bytes of one of the [`Buffers`]. +#[derive(Clone, Copy)] +pub(crate) struct Span { + pub(crate) buf: BufId, + pub(crate) start: usize, + pub(crate) end: usize, +} + +impl Span { + #[inline] + const fn len(&self) -> usize { + self.end - self.start + } + + #[inline] + const fn start_addr(&self) -> usize { + self.buf.start() + self.start + } + + #[inline] + const fn end_addr(&self) -> usize { + self.buf.start() + self.end + } + + #[inline] + fn bytes<'a>(&self, bufs: &Buffers<'a>) -> Result<&'a [u8]> { + bufs.get(self.buf).sub(self.start..self.end) + } +} + +/// What the pointers of a `ZSTD_CCtx` lead to, that it does not own. +#[derive(Clone, Copy)] +pub(crate) struct Input<'a> { + pub(crate) bufs: Buffers<'a>, + pub(crate) cdict: Option<&'a CDict>, +} + +/// Where a frame goes. +pub(crate) enum Output<'a> { + Slice(&'a mut [u8]), + /// What is from `start` to `end` in `vec`, which has the capacity for it. Only what is asked for becomes part of its + /// length, as zeros, and what is appended: most of the room for a frame is never written to. + Spare { + vec: &'a mut Vec, + start: usize, + end: usize, + }, +} + +/// The header of a block, those of its literals and its sequences, the description of a Huffman table, and far more than +/// any encoder wants beyond what it writes. +const ROOM_TO_SPARE: usize = 1024; + +impl Output<'_> { + /// How many bytes there are from `pos` on. + fn room_from(&self, pos: usize) -> usize { + match self { + Output::Slice(slice) => slice.len().saturating_sub(pos), + Output::Spare { start, end, .. } => end.saturating_sub(*start + pos), + } + } + + /// [`no_compress_block`] at `pos`, up to which the output is written. + fn store_block(&mut self, pos: usize, src: &[u8], last_block: bool) -> Result { + match self { + Output::Slice(slice) => no_compress_block(slice.sub_mut(pos..)?, src, last_block), + Output::Spare { vec, start, end } => { + let pos = *start + pos; + let size = ZSTD_BLOCKHEADERSIZE + src.len(); + if pos + size > *end { + return Err(Error::DstSizeTooSmall); + } + // Appended, which takes no zeros first. + let header = block_header(BlockType::Raw, src.len(), last_block); + let [header @ .., _] = header.to_le_bytes(); + append_at(vec, pos, &header)?; + append(vec, src)?; + Ok(size) + } + } + } + + /// What there is from `pos` on, or `needed` bytes of it at least. + fn from(&mut self, pos: usize, needed: usize) -> Result<&mut [u8]> { + match self { + Output::Slice(slice) => slice.sub_mut(pos..), + Output::Spare { vec, start, end } => { + let pos = *start + pos; + let len = pos.saturating_add(needed).min(*end); + if vec.len() < len { + vec.resize(len, 0); + } + vec.sub_mut(pos..) + } + } + } +} + +fn window_prefix<'a>(window: &Window, bufs: &Buffers<'a>) -> Prefix<'a> { + Prefix { + buf: bufs.get(window.base_buf), + base: window.base.wrapping_sub(window.base_buf.start()), + } +} + +/// `ZSTD_bitmix` +const fn bitmix(mut val: u64, len: u64) -> u64 { + val ^= val.rotate_right(49) ^ val.rotate_right(24); + val = val.wrapping_mul(0x9FB2_1C65_1E98_DF25); + val ^= (val >> 35).wrapping_add(len); + val = val.wrapping_mul(0x9FB2_1C65_1E98_DF25); + val ^ (val >> 28) +} + +/// `ZSTD_invalidateMatchState` +fn invalidate_match_state(ms: &mut MatchState) { + ms.window.clear(); + ms.next_to_update = ms.window.dict_limit; + ms.loaded_dict_end = 0; + ms.opt.lit_length_sum = 0; + ms.has_dict_match_state = false; +} + +/// `ZSTD_cwksp_mark_tables_dirty` and then `ZSTD_cwksp_mark_tables_clean` +fn forget_unused_indices(ms: &mut MatchState) { + ms.hash_table.forget_rest(); + ms.chain_table.forget_rest(); + ms.hash_table3.forget_rest(); +} + +/// `ZSTD_reset_matchState` +fn reset_match_state( + ms: &mut MatchState, + c_params: &CompressionParameters, + use_row_match_finder: ParamSwitch, + crp: CompResetPolicy, + force_reset_index: IndexResetPolicy, + for_who: ResetTarget, + src_size: Option, +) -> Result<()> { + let sizes = MatchStateSizes::new(c_params, use_row_match_finder, for_who == ResetTarget::CCtx); + + if force_reset_index == IndexResetPolicy::Reset { + ms.window = Window::new(); + } + ms.hash_log3 = sizes.hash_log3; + invalidate_match_state(ms); + + // As long as indices go on, the ones in the tables are too low to be valid. + let clean = force_reset_index == IndexResetPolicy::Reset && crp != CompResetPolicy::LeaveDirty; + let use_rows = row_match_finder_used(c_params.strat(), use_row_match_finder); + ms.hash_table.reset(sizes.h_size, clean)?; + ms.chain_table.reset(sizes.chain_size, clean)?; + ms.hash_table3.reset(sizes.h3_size, clean)?; + if force_reset_index == IndexResetPolicy::Reset { + forget_unused_indices(ms); + } + + if use_rows { + ms.row_tags = match for_who { + ResetTarget::CCtx => lazy::row_tags_at_start(c_params, src_size), + ResetTarget::CDict => RowTags::Flat, + }; + let tag_table_size = lazy::row_tag_table_size(ms.row_tags, sizes.h_size); + // In a context, a new salt instead of erasing the tags. + let is_new = (ms.tag_table).reset(tag_table_size, for_who == ResetTarget::CDict)?; + // The table suits the size of the input if that is known. Most of it is then read before it is written, for + // which the system would provide each page twice. Else the input may be too short to get to most of it. + if is_new && (src_size.is_some() || for_who == ResetTarget::CDict) { + ms.tag_table.fill(0); + } + ms.hash_salt = match for_who { + ResetTarget::CCtx => { + bitmix(ms.hash_salt, 8) ^ bitmix(u64::from(ms.hash_salt_entropy), 4) + } + ResetTarget::CDict => 0, + }; + let row_log = c_params.search_log.clamp(4, 6); + ms.row_hash_log = c_params.hash_log - row_log; + } + + ms.c_params = *c_params; + Ok(()) +} + +/// The part of `ZSTD_reset_matchState` that is for the optimal parser. +fn reset_opt_state(opt: &mut OptState, block_size: usize) -> Result<()> { + reset_table(&mut opt.lit_freq, 1 << LITBITS, false)?; + reset_table(&mut opt.lit_length_freq, MAX_LL as usize + 1, false)?; + reset_table(&mut opt.match_length_freq, MAX_ML as usize + 1, false)?; + reset_table(&mut opt.off_code_freq, MAX_OFF as usize + 1, false)?; + let nb_positions = nb_opt_positions(block_size); + reset_table(&mut opt.match_table, nb_positions, false)?; + opt.price_table.resize(nb_positions) +} + +/// How many positions the tables of the optimal parser have: `ZSTD_OPT_SIZE`, or fewer if no block is that long. One +/// before each byte, one after the last, and one more that is only given a price. +fn nb_opt_positions(block_size: usize) -> usize { + ZSTD_OPT_SIZE.min(block_size + 2) +} + +/// `ZSTD_indexTooCloseToMax`: starting a frame this close to the limit would mean bringing indices down during it. +fn index_too_close_to_max(w: &Window) -> bool { + const ZSTD_INDEXOVERFLOW_MARGIN: usize = 16 << 20; + w.next_src.wrapping_sub(w.base) > ZSTD_CURRENT_MAX as usize - ZSTD_INDEXOVERFLOW_MARGIN +} + +/// `ZSTD_reduceTable_internal` +fn reduce_table(table: &mut [u32], reducer_value: u32, preserve_mark: bool) { + let reducer_threshold = reducer_value.wrapping_add(ZSTD_WINDOW_START_INDEX); + for cell in table { + *cell = if preserve_mark && *cell == ZSTD_DUBT_UNSORTED_MARK { + ZSTD_DUBT_UNSORTED_MARK + } else if *cell < reducer_threshold { + 0 + } else { + *cell - reducer_value + }; + } +} + +/// `ZSTD_reduceIndex` +fn reduce_index(ms: &mut MatchState, params: &CCtxParams, reducer_value: u32) { + forget_unused_indices(ms); + reduce_table(&mut ms.hash_table, reducer_value, false); + if allocate_chain_table(params.c_params.strat(), params.use_row_match_finder) { + let preserve_mark = params.c_params.strat() == Strategy::BtLazy2; + reduce_table(&mut ms.chain_table, reducer_value, preserve_mark); + } + if ms.hash_log3 != 0 { + reduce_table(&mut ms.hash_table3, reducer_value, false); + } +} + +/// `ZSTD_overflowCorrectIfNeeded` +fn overflow_correct_if_needed(ms: &mut MatchState, params: &CCtxParams, ip: usize, iend: usize) { + let cycle_log = cycle_log(params.c_params.chain_log, params.c_params.strat()); + let max_dist = 1u32 << params.c_params.window_log; + if ms.window.need_overflow_correction(iend) { + let correction = ms.window.correct_overflow(cycle_log, max_dist, ip); + reduce_index(ms, params, correction); + ms.next_to_update = ms.next_to_update.saturating_sub(correction); + // Even if the dictionary is still within the window. + ms.loaded_dict_end = 0; + ms.has_dict_match_state = false; + } +} + +/// `ZSTD_loadDictionaryContent`: makes `src` the start of the window, and indexes it. +/// +/// The C code also indexes it for long distance matching if it is a prefix from the user, which there is none of here. +fn load_dictionary_content( + ms: &mut MatchState, + params: &CCtxParams, + bufs: &Buffers<'_>, + mut src: Span, + tfp: TableFillPurpose, +) -> Result<()> { + { + // Indices have to fit, along with a tag in the tables that have one. + let mut max_dict_size = ZSTD_CURRENT_MAX - ZSTD_WINDOW_START_INDEX; + if cdict_indices_are_tagged(¶ms.c_params) && tfp == TableFillPurpose::ForCDict { + let short_cache_max_dict_size = + (1u32 << (32 - ZSTD_SHORT_CACHE_TAG_BITS)) - ZSTD_WINDOW_START_INDEX; + max_dict_size = max_dict_size.min(short_cache_max_dict_size); + } + if src.len() > max_dict_size as usize { + src.start = src.end - max_dict_size as usize; + } + } + + ms.window + .update(src.start_addr(), src.buf, src.len(), false); + + { + // More than the tables can hold is not worth indexing. + let log = (params.c_params.hash_log + 3) + .max(params.c_params.chain_log + 1) + .min(31); + let max_dict_size = 1usize << log; + if src.len() > max_dict_size { + src.start = src.end - max_dict_size; + } + } + + ms.next_to_update = ms.window.index_of(src.start_addr()); + ms.loaded_dict_end = if params.force_window { + 0 + } else { + ms.window.index_of(src.end_addr()) + }; + ms.force_non_contiguous = false; + + if src.len() <= HASH_READ_SIZE { + return Ok(()); + } + + overflow_correct_if_needed(ms, params, src.start_addr(), src.end_addr()); + + let prefix = window_prefix(&ms.window, bufs); + match params.c_params.strat() { + Strategy::Fast => fast::fill_hash_table(ms, prefix, src.end, tfp), + Strategy::DFast => double_fast::fill_double_hash_table(ms, prefix, src.end, tfp), + Strategy::Greedy | Strategy::Lazy | Strategy::Lazy2 => { + if params.use_row_match_finder == ParamSwitch::Enable { + ms.tag_table.fill(0); + lazy::row_update(ms, prefix, src.end - HASH_READ_SIZE)?; + } else { + lazy::insert_and_find_first_index(ms, prefix, src.end - HASH_READ_SIZE)?; + } + } + Strategy::BtLazy2 | Strategy::BtOpt | Strategy::BtUltra | Strategy::BtUltra2 => { + opt::update_tree(ms, prefix, src.end - HASH_READ_SIZE, src.end)?; + } + } + + ms.next_to_update = ms.window.index_of(src.end_addr()); + Ok(()) +} + +/// `ZSTD_dictNCountRepeat`: a table that has a code for every symbol can be used without checking. +fn dict_ncount_repeat( + normalized_counter: &[i16], + dict_max_symbol_value: u32, + max_symbol_value: u32, +) -> FseRepeat { + if dict_max_symbol_value < max_symbol_value { + return FseRepeat::Check; + } + let counts = normalized_counter + .iter() + .take(max_symbol_value as usize + 1); + if counts.into_iter().any(|&count| count == 0) { + return FseRepeat::Check; + } + FseRepeat::Valid +} + +/// `ZSTD_loadCEntropy`: reads the tables of the dictionary `dict` into `bs`. Returns where its content starts. +fn load_c_entropy(bs: &mut CompressedBlockState, dict: &[u8]) -> Result { + const CORRUPTED: Error = Error::DictionaryCorrupted; + let mut offcode_ncount = [0i16; MAX_OFF as usize + 1]; + let mut dict_ptr = 8usize; + bs.entropy.huf.repeat_mode = HufRepeat::Check; + + { + let stats = huf::read_ctable(&mut bs.entropy.huf.c_table, 255, dict.sub(dict_ptr..)?) + .map_err(|_| CORRUPTED)?; + // A table that has a code for every byte can be used without checking. + if !stats.has_zero_weights && stats.max_symbol_value == 255 { + bs.entropy.huf.repeat_mode = HufRepeat::Valid; + } + dict_ptr += stats.size; + } + + let offcode_max_value = { + let header = read_ncount(&mut offcode_ncount, MAX_OFF, dict.sub(dict_ptr..)?) + .map_err(|_| CORRUPTED)?; + if header.table_log > OFF_FSE_LOG { + return Err(CORRUPTED); + } + (bs.entropy.fse.offcode_c_table) + .build(&offcode_ncount, MAX_OFF, header.table_log) + .map_err(|_| CORRUPTED)?; + dict_ptr += header.size; + header.max_symbol_value + }; + + { + let mut matchlength_ncount = [0i16; MAX_ML as usize + 1]; + let header = read_ncount(&mut matchlength_ncount, MAX_ML, dict.sub(dict_ptr..)?) + .map_err(|_| CORRUPTED)?; + if header.table_log > ML_FSE_LOG { + return Err(CORRUPTED); + } + (bs.entropy.fse.matchlength_c_table) + .build( + &matchlength_ncount, + header.max_symbol_value, + header.table_log, + ) + .map_err(|_| CORRUPTED)?; + bs.entropy.fse.matchlength_repeat_mode = + dict_ncount_repeat(&matchlength_ncount, header.max_symbol_value, MAX_ML); + dict_ptr += header.size; + } + + { + let mut litlength_ncount = [0i16; MAX_LL as usize + 1]; + let header = read_ncount(&mut litlength_ncount, MAX_LL, dict.sub(dict_ptr..)?) + .map_err(|_| CORRUPTED)?; + if header.table_log > LL_FSE_LOG { + return Err(CORRUPTED); + } + (bs.entropy.fse.litlength_c_table) + .build(&litlength_ncount, header.max_symbol_value, header.table_log) + .map_err(|_| CORRUPTED)?; + bs.entropy.fse.litlength_repeat_mode = + dict_ncount_repeat(&litlength_ncount, header.max_symbol_value, MAX_LL); + dict_ptr += header.size; + } + + if dict_ptr + 12 > dict.len() { + return Err(CORRUPTED); + } + bs.rep = [ + read_le32(dict, dict_ptr)?, + read_le32(dict, dict_ptr + 4)?, + read_le32(dict, dict_ptr + 8)?, + ]; + dict_ptr += 12; + + { + let dict_content_size = dict.len() - dict_ptr; + let mut offcode_max = MAX_OFF; + if dict_content_size <= u32::MAX as usize - (128 << 10) { + // The largest offset that a first block can have. + let max_offset = dict_content_size as u32 + (128 << 10); + offcode_max = highbit32(max_offset); + } + bs.entropy.fse.offcode_repeat_mode = + dict_ncount_repeat(&offcode_ncount, offcode_max_value, offcode_max.min(MAX_OFF)); + + for rep in bs.rep { + if rep == 0 || rep as usize > dict_content_size { + return Err(CORRUPTED); + } + } + } + + Ok(dict_ptr) +} + +/// `ZSTD_compress_insertDictionary`: returns the ID of the dictionary, which is 0 if it has none. +fn compress_insert_dictionary( + bs: &mut CompressedBlockState, + ms: &mut MatchState, + params: &CCtxParams, + bufs: &Buffers<'_>, + dict: Option, + tfp: TableFillPurpose, +) -> Result { + let Some(dict) = dict.filter(|dict| dict.len() >= 8) else { + return Ok(0); + }; + + bs.reset(); + + let bytes = dict.bytes(bufs)?; + if tfp == TableFillPurpose::ForCCtx || read_le32(bytes, 0)? != ZSTD_MAGIC_DICTIONARY { + load_dictionary_content(ms, params, bufs, dict, tfp)?; + return Ok(0); + } + + // `ZSTD_loadZstdDictionary` + let dict_id = if params.f_params.no_dict_id_flag { + 0 + } else { + read_le32(bytes, 4)? + }; + let e_size = load_c_entropy(bs, bytes)?; + let content = Span { + start: dict.start + e_size, + ..dict + }; + load_dictionary_content(ms, params, bufs, content, tfp)?; + Ok(dict_id) +} + +/// `ZSTD_CDict`: a dictionary that is indexed, ready to start frames with. +pub(crate) struct CDict { + dict_content: Arc>, + match_state: MatchState, + c_block_state: Box, + dict_id: u32, + use_row_match_finder: ParamSwitch, +} + +impl CDict { + /// `ZSTD_createCDict_advanced2` + pub(crate) fn new(dict: Arc>, original_cctx_params: &CCtxParams) -> Result { + let mut params = *original_cctx_params; + params.c_params = get_c_params_from_cctx_params( + ¶ms, + ZSTD_CONTENTSIZE_UNKNOWN, + dict.len(), + CParamMode::CreateCDict, + ); + params.use_row_match_finder = + resolve_row_match_finder_mode(params.use_row_match_finder, ¶ms.c_params); + + let mut cdict = Self { + dict_content: dict, + match_state: MatchState::new(), + c_block_state: Box::new(CompressedBlockState::new()), + dict_id: 0, + use_row_match_finder: params.use_row_match_finder, + }; + reset_match_state( + &mut cdict.match_state, + ¶ms.c_params, + params.use_row_match_finder, + CompResetPolicy::MakeClean, + IndexResetPolicy::Reset, + ResetTarget::CDict, + None, + )?; + let bufs = Buffers { + in_buff: &[], + user: &[], + dict: &cdict.dict_content, + }; + cdict.dict_id = compress_insert_dictionary( + &mut cdict.c_block_state, + &mut cdict.match_state, + ¶ms, + &bufs, + Some(Span { + buf: BufId::Dict, + start: 0, + end: cdict.dict_content.len(), + }), + TableFillPurpose::ForCDict, + )?; + Ok(cdict) + } + + pub(crate) fn dict_content(&self) -> &[u8] { + &self.dict_content + } + + pub(crate) const fn dict_id(&self) -> u32 { + self.dict_id + } + + fn dict_match_state(&self) -> DictMatchState<'_> { + let window = &self.match_state.window; + let start = BufId::Dict.start(); + DictMatchState { + ms: &self.match_state, + buf: &self.dict_content, + base: window.base.wrapping_sub(start), + end: window.next_src.wrapping_sub(start), + } + } + + /// `ZSTD_shouldAttachDict`: whether to search the tables of the dictionary, rather than start from a copy of them. + pub(crate) fn should_attach(&self, params: &CCtxParams, pledged_src_size: u64) -> bool { + let cutoff: u64 = match self.match_state.c_params.strat() { + Strategy::Fast | Strategy::BtUltra | Strategy::BtUltra2 => 8 << 10, + Strategy::DFast => 16 << 10, + _ => 32 << 10, + }; + (pledged_src_size <= cutoff || pledged_src_size == ZSTD_CONTENTSIZE_UNKNOWN) + && !params.force_window + } +} + +/// `ZSTD_isRLE`: `src` is not empty. +fn is_rle(src: &[u8]) -> bool { + let Some((&value, rest)) = src.split_first() else { + return false; + }; + let (chunks, tail) = rest.as_chunks::<32>(); + chunks.iter().all(|chunk| *chunk == [value; 32]) && tail.iter().all(|&b| b == value) +} + +/// `ZSTD_maybeRLE` +fn maybe_rle(seq_store: &SeqStore) -> bool { + seq_store.sequences.len() < 4 && seq_store.lit_len < 10 +} + +/// `writeBlockHeader` +fn write_block_header( + op: &mut [u8], + c_size: usize, + block_size: usize, + last_block: bool, +) -> Result<()> { + let c_block_header = if c_size == 1 { + block_header(BlockType::Rle, block_size, last_block) + } else { + block_header(BlockType::Compressed, c_size, last_block) + }; + write_le24(op, 0, c_block_header) +} + +/// `ZSTD_resolveRepcodeToRawOffset` +fn resolve_repcode_to_raw_offset(rep: &[u32; ZSTD_REP_NUM], off_base: u32, ll0: bool) -> u32 { + let [rep0, ..] = *rep; + let adjusted_rep_code = off_base - 1 + u32::from(ll0); + match rep.get(adjusted_rep_code as usize) { + Some(&offset) => offset, + None => rep0.wrapping_sub(1), + } +} + +/// `ZSTD_seqStore_resolveOffCodes`: a block that ends up uncompressed leaves the repeat offsets of the decoder, +/// `d_repcodes`, behind those that the sequences were found with, `c_repcodes`. Where they disagree, the repeat codes +/// of `chunk` are replaced by the offsets they stand for. +fn seq_store_resolve_off_codes( + d_repcodes: &mut [u32; ZSTD_REP_NUM], + c_repcodes: &mut [u32; ZSTD_REP_NUM], + store: &mut SeqStore, + chunk: &SeqChunk, +) -> Result<()> { + let sequences = store.sequences.sub_mut(chunk.seq_start..chunk.seq_end)?; + let long_lit_len_idx = if chunk.long_length_type == LongLengthType::LiteralLength { + chunk.long_length_pos as usize + } else { + sequences.len() + }; + for (idx, seq) in sequences.iter_mut().enumerate() { + let ll0 = seq.lit_length == 0 && idx != long_lit_len_idx; + let off_base = seq.off_base; + if offbase_is_repcode(off_base) { + let d_raw_offset = resolve_repcode_to_raw_offset(d_repcodes, off_base, ll0); + let c_raw_offset = resolve_repcode_to_raw_offset(c_repcodes, off_base, ll0); + if d_raw_offset != c_raw_offset { + seq.off_base = offset_to_offbase(c_raw_offset); + } + } + update_rep(d_repcodes, seq.off_base, ll0); + update_rep(c_repcodes, off_base, ll0); + } + Ok(()) +} + +/// `ZSTD_writeFrameHeader` +fn write_frame_header( + dst: &mut [u8], + params: &CCtxParams, + pledged_src_size: u64, + dict_id: u32, +) -> Result { + let dict_id_size_code_length = + u32::from(dict_id > 0) + u32::from(dict_id >= 256) + u32::from(dict_id >= 65536); + let dict_id_size_code = if params.f_params.no_dict_id_flag { + 0 + } else { + dict_id_size_code_length + }; + let checksum_flag = u32::from(params.f_params.checksum_flag); + let window_size = 1u32 << params.c_params.window_log; + let single_segment = + params.f_params.content_size_flag && u64::from(window_size) >= pledged_src_size; + let window_log_byte = ((params.c_params.window_log - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3) as u8; + let fcs_code = if params.f_params.content_size_flag { + u32::from(pledged_src_size >= 256) + + u32::from(pledged_src_size >= 65536 + 256) + + u32::from(pledged_src_size >= 0xFFFF_FFFF) + } else { + 0 + }; + let frame_header_description_byte = (dict_id_size_code + + (checksum_flag << 2) + + (u32::from(single_segment) << 5) + + (fcs_code << 6)) as u8; + let mut pos = 0usize; + + if dst.len() < ZSTD_FRAMEHEADERSIZE_MAX { + return Err(Error::DstSizeTooSmall); + } + if params.format == Format::Zstd1 { + write_le32(dst, 0, ZSTD_MAGICNUMBER)?; + pos = 4; + } + dst.put(pos, frame_header_description_byte)?; + pos += 1; + if !single_segment { + dst.put(pos, window_log_byte)?; + pos += 1; + } + match dict_id_size_code { + 0 => {} + 1 => { + dst.put(pos, dict_id as u8)?; + pos += 1; + } + 2 => { + write_le16(dst, pos, dict_id as u16)?; + pos += 2; + } + _ => { + write_le32(dst, pos, dict_id)?; + pos += 4; + } + } + match fcs_code { + 0 => { + if single_segment { + dst.put(pos, pledged_src_size as u8)?; + pos += 1; + } + } + 1 => { + write_le16(dst, pos, (pledged_src_size - 256) as u16)?; + pos += 2; + } + 2 => { + write_le32(dst, pos, pledged_src_size as u32)?; + pos += 4; + } + _ => { + write_le64(dst, pos, pledged_src_size)?; + pos += 8; + } + } + Ok(pos) +} + +/// `ZSTD_writeLastEmptyBlock` +pub(crate) fn write_last_empty_block(dst: &mut [u8]) -> Result { + if dst.len() < ZSTD_BLOCKHEADERSIZE { + return Err(Error::DstSizeTooSmall); + } + write_le24(dst, 0, block_header(BlockType::Raw, 0, true))?; + Ok(ZSTD_BLOCKHEADERSIZE) +} + +/// `ZSTD_BuildSeqStore_e` +#[derive(Clone, Copy, PartialEq, Eq)] +enum BuildSeqStore { + Compress, + NoCompress, +} + +/// The C code stops splitting at 196 parts, but the calls under way at that time each still add one, past the end of +/// its array, in memory that happens to be free. +const ZSTD_MAX_NB_BLOCK_SPLITS: usize = 196; +const PARTITIONS_LEN: usize = ZSTD_MAX_NB_BLOCK_SPLITS + 20; + +/// `seqStoreSplits` +struct SeqStoreSplits { + split_locations: [u32; PARTITIONS_LEN], + idx: usize, +} + +/// The sizes of the buffers that streaming takes. +pub(crate) struct BufferSizes { + pub(crate) buff_in_size: usize, + pub(crate) buff_out_size: usize, +} + +/// The part of a `ZSTD_CCtx` that writes frames from blocks of input. +pub(crate) struct FrameCompressor { + pub(crate) stage: CompressionStage, + pub(crate) applied_params: CCtxParams, + pub(crate) dict_id: u32, + pub(crate) dict_content_size: usize, + workspace: Workspace, + pub(crate) block_size_max: usize, + pub(crate) pledged_src_size_plus_one: u64, + pub(crate) consumed_src_size: u64, + pub(crate) produced_c_size: u64, + xxh_state: Xxh64, + is_first_block: bool, + initialized: bool, + seq_store: SeqStore, + ldm_state: LdmState, + ldm_sequences: RawSeqStore, + extern_seq_store: RawSeqStore, + /// `None` stands for [`INITIAL_BLOCK_STATE`]. + prev_c_block: Option>, + /// Made when the first block needs it. + next_c_block: Option>, + match_state: MatchState, +} + +static INITIAL_BLOCK_STATE: CompressedBlockState = CompressedBlockState::new(); + +fn block_state(state: &Option>) -> &CompressedBlockState { + state.as_deref().unwrap_or(&INITIAL_BLOCK_STATE) +} + +fn block_state_mut(state: &mut Option>) -> &mut CompressedBlockState { + state.get_or_insert_with(|| Box::new(CompressedBlockState::new())) +} + +const fn empty_raw_seq_store() -> RawSeqStore { + RawSeqStore { + seq: Vec::new(), + pos: 0, + pos_in_sequence: 0, + size: 0, + capacity: 0, + } +} + +impl FrameCompressor { + pub(crate) fn new() -> Self { + Self { + stage: CompressionStage::Created, + applied_params: CCtxParams::new(0), + dict_id: 0, + dict_content_size: 0, + workspace: Workspace::new(), + block_size_max: 0, + pledged_src_size_plus_one: 0, + consumed_src_size: 0, + produced_c_size: 0, + xxh_state: Xxh64::new(), + is_first_block: true, + initialized: false, + seq_store: SeqStore::new(), + ldm_state: LdmState::new(), + ldm_sequences: empty_raw_seq_store(), + extern_seq_store: empty_raw_seq_store(), + prev_c_block: None, + next_c_block: None, + match_state: MatchState::new(), + } + } + + /// For jobs of workers, which stop if `cancelled` is set. + pub(crate) fn for_worker(cancelled: Arc) -> Self { + let mut worker = Self::new(); + worker.match_state.cancelled = Some(cancelled); + worker + } + + /// The next frame starts from nothing, as after a reset that failed. + pub(crate) fn forget(&mut self) { + self.initialized = false; + } + + /// `ZSTD_resetCCtx_internal` + fn reset_internal( + &mut self, + params: &CCtxParams, + pledged_src_size: u64, + loaded_dict_size: usize, + crp: CompResetPolicy, + buffered: bool, + ) -> Result { + // Until this is through: if it fails, the next frame starts from nothing. + let initialized = core::mem::take(&mut self.initialized); + self.is_first_block = true; + self.applied_params = *params; + if self.applied_params.ldm_params.enable_ldm == ParamSwitch::Enable { + ldm::adjust_parameters(&mut self.applied_params.ldm_params, ¶ms.c_params); + } + let params = self.applied_params; + + let window_size = (1u64 << params.c_params.window_log) + .min(pledged_src_size) + .max(1) as usize; + let block_size = ZSTD_BLOCKSIZE_MAX.min(window_size); + let max_nb_seq = max_nb_seq(block_size, params.c_params.min_match); + let sizes = BufferSizes { + buff_out_size: if buffered { + compress_bound(block_size) + 1 + } else { + 0 + }, + buff_in_size: if buffered { + window_size + block_size + } else { + 0 + }, + }; + let max_nb_ldm_seq = ldm::get_max_nb_seq(params.ldm_params, block_size); + + let index_too_close = index_too_close_to_max(&self.match_state.window); + let dict_too_big = loaded_dict_size > ZSTD_CHUNKSIZE_MAX as usize; + // Where indices start shows in where they get to the limit, which is where a dictionary is let go of. + if !initialized { + self.workspace = Workspace::new(); + } + let new_workspace = self.workspace.reset(&WorkspaceNeeds { + c_params: ¶ms.c_params, + ldm_params: ¶ms.ldm_params, + use_row_match_finder: params.use_row_match_finder, + buff_in_size: sizes.buff_in_size, + buff_out_size: sizes.buff_out_size, + block_size, + }); + let needs_index_reset = if index_too_close || dict_too_big || !initialized || new_workspace + { + IndexResetPolicy::Reset + } else { + IndexResetPolicy::Continue + }; + + // A context does not hold on for long to much more than it uses. + if new_workspace { + let ms = &mut self.match_state; + ms.hash_table = Table::new(); + ms.chain_table = Table::new(); + ms.hash_table3 = Table::new(); + ms.tag_table = Table::new(); + ms.opt.match_table = Vec::new(); + ms.opt.price_table = PriceTable::default(); + } + + self.match_state.c_params = params.c_params; + self.pledged_src_size_plus_one = pledged_src_size.wrapping_add(1); + self.consumed_src_size = 0; + self.produced_c_size = 0; + if pledged_src_size == ZSTD_CONTENTSIZE_UNKNOWN { + self.applied_params.f_params.content_size_flag = false; + } + self.block_size_max = block_size; + + self.xxh_state = Xxh64::new(); + self.stage = CompressionStage::Init; + self.dict_id = 0; + self.dict_content_size = 0; + + if let Some(prev_c_block) = &mut self.prev_c_block { + prev_c_block.reset(); + } + + reset_match_state( + &mut self.match_state, + ¶ms.c_params, + params.use_row_match_finder, + crp, + needs_index_reset, + ResetTarget::CCtx, + (pledged_src_size != ZSTD_CONTENTSIZE_UNKNOWN).then_some(pledged_src_size), + )?; + if params.c_params.strat() >= Strategy::BtOpt { + reset_opt_state(&mut self.match_state.opt, block_size)?; + } + + if params.ldm_params.enable_ldm == ParamSwitch::Enable { + let ldm_h_size = 1usize << params.ldm_params.hash_log; + reset_table(&mut self.ldm_state.hash_table, ldm_h_size, true)?; + reset_table(&mut self.ldm_sequences.seq, max_nb_ldm_seq, false)?; + self.ldm_sequences.capacity = max_nb_ldm_seq; + self.ldm_state.window = Window::new(); + self.ldm_state.loaded_dict_end = 0; + let num_buckets = + 1usize << (params.ldm_params.hash_log - params.ldm_params.bucket_size_log); + reset_table(&mut self.ldm_state.bucket_offsets, num_buckets, true)?; + } else { + self.ldm_state.hash_table = Vec::new(); + self.ldm_state.bucket_offsets = Vec::new(); + self.ldm_sequences = empty_raw_seq_store(); + } + + self.seq_store.set_limits(max_nb_seq, block_size); + self.reference_external_sequences(Vec::new()); + + self.initialized = true; + Ok(sizes) + } + + /// `ZSTD_invalidateRepCodes` + pub(crate) fn invalidate_rep_codes(&mut self) { + block_state_mut(&mut self.prev_c_block).rep = [0; ZSTD_REP_NUM]; + } + + /// `ZSTD_resetCCtx_byAttachingCDict` + fn reset_by_attaching_cdict( + &mut self, + cdict: &CDict, + mut params: CCtxParams, + pledged_src_size: u64, + buffered: bool, + ) -> Result { + let window_log = params.c_params.window_log; + // The tables only have to hold the input. + params.c_params = adjust_c_params_internal( + cdict.match_state.c_params, + pledged_src_size, + cdict.dict_content.len(), + CParamMode::AttachDict, + params.use_row_match_finder, + ); + params.c_params.window_log = window_log; + params.use_row_match_finder = cdict.use_row_match_finder; + let sizes = self.reset_internal( + ¶ms, + pledged_src_size, + 0, + CompResetPolicy::MakeClean, + buffered, + )?; + + let cdict_window = &cdict.match_state.window; + let cdict_end = cdict_window.index_of(cdict_window.next_src); + let cdict_len = cdict_end.wrapping_sub(cdict_window.dict_limit); + if cdict_len != 0 { + let ms = &mut self.match_state; + ms.has_dict_match_state = true; + // The indices of the input have to be above those of the dictionary. + if ms.window.dict_limit < cdict_end { + ms.window.next_src = ms.window.base.wrapping_add(cdict_end as usize); + ms.window.clear(); + } + ms.loaded_dict_end = ms.window.dict_limit; + } + + self.dict_id = cdict.dict_id; + self.dict_content_size = cdict.dict_content.len(); + block_state_mut(&mut self.prev_c_block).clone_from(&cdict.c_block_state); + Ok(sizes) + } + + /// `ZSTD_resetCCtx_byCopyingCDict` + fn reset_by_copying_cdict( + &mut self, + cdict: &CDict, + mut params: CCtxParams, + pledged_src_size: u64, + buffered: bool, + ) -> Result { + /// `ZSTD_copyCDictTableIntoCCtx` + fn copy_table(dst: &mut [u32], src: &[u32], tagged: bool) -> Result<()> { + if tagged { + for (dst, &tagged_index) in dst.iter_mut().zip(src) { + *dst = tagged_index >> ZSTD_SHORT_CACHE_TAG_BITS; + } + Ok(()) + } else { + dst.write_at(0, src) + } + } + + let cdict_c_params = cdict.match_state.c_params; + let window_log = params.c_params.window_log; + params.c_params = cdict_c_params; + params.c_params.window_log = window_log; + params.use_row_match_finder = cdict.use_row_match_finder; + let sizes = self.reset_internal( + ¶ms, + pledged_src_size, + 0, + CompResetPolicy::LeaveDirty, + buffered, + )?; + + let src = &cdict.match_state; + let dst = &mut self.match_state; + forget_unused_indices(dst); + let tagged = cdict_indices_are_tagged(&cdict_c_params); + copy_table(&mut dst.hash_table, &src.hash_table, tagged)?; + copy_table(&mut dst.chain_table, &src.chain_table, tagged)?; + if row_match_finder_used(cdict_c_params.strat(), cdict.use_row_match_finder) { + dst.tag_table.write_at(0, &src.tag_table)?; + dst.row_tags = RowTags::Flat; + dst.hash_salt = src.hash_salt; + } + dst.hash_table3.fill(0); + + dst.window = src.window; + dst.next_to_update = src.next_to_update; + dst.loaded_dict_end = src.loaded_dict_end; + + self.dict_id = cdict.dict_id; + self.dict_content_size = cdict.dict_content.len(); + block_state_mut(&mut self.prev_c_block).clone_from(&cdict.c_block_state); + Ok(sizes) + } + + /// `ZSTD_compressBegin_internal`: `dict` is content to start the window with, when there is no `input.cdict`. + pub(crate) fn begin( + &mut self, + input: &Input<'_>, + dict: Option, + params: &CCtxParams, + pledged_src_size: u64, + buffered: bool, + ) -> Result { + if let Some(cdict) = input.cdict + && !cdict.dict_content.is_empty() + { + // `ZSTD_resetCCtx_usingCDict` + return if cdict.should_attach(params, pledged_src_size) { + self.reset_by_attaching_cdict(cdict, *params, pledged_src_size, buffered) + } else { + self.reset_by_copying_cdict(cdict, *params, pledged_src_size, buffered) + }; + } + + let dict_content_size = dict.map_or(0, |dict| dict.len()); + let sizes = self.reset_internal( + params, + pledged_src_size, + dict_content_size, + CompResetPolicy::MakeClean, + buffered, + )?; + self.dict_id = match dict { + Some(_) => compress_insert_dictionary( + block_state_mut(&mut self.prev_c_block), + &mut self.match_state, + &self.applied_params, + &input.bufs, + dict, + TableFillPurpose::ForCCtx, + )?, + None => 0, + }; + self.dict_content_size = dict_content_size; + Ok(sizes) + } + + /// `ZSTD_referenceExternalSequences`: long distance matches for the input to come. + pub(crate) fn reference_external_sequences(&mut self, seq: Vec) { + self.extern_seq_store = RawSeqStore { + pos: 0, + pos_in_sequence: 0, + size: seq.len(), + capacity: seq.len(), + seq, + }; + } + + /// Takes back what [`Self::reference_external_sequences`] was given. + pub(crate) fn take_external_sequences(&mut self) -> Vec { + core::mem::replace(&mut self.extern_seq_store, empty_raw_seq_store()).seq + } + + /// `ZSTD_blockState_confirmRepcodesAndEntropyTables` + fn confirm_repcodes_and_entropy_tables(&mut self) { + core::mem::swap(&mut self.prev_c_block, &mut self.next_c_block); + } + + /// A table for offsets that comes from a dictionary is only known to be valid for the first block. + fn downgrade_offcode_repeat_mode(&mut self) { + if let Some(prev_c_block) = &mut self.prev_c_block { + let mode = &mut prev_c_block.entropy.fse.offcode_repeat_mode; + if *mode == FseRepeat::Valid { + *mode = FseRepeat::Check; + } + } + } + + /// `ZSTD_buildSeqStore` + fn build_seq_store(&mut self, input: &Input<'_>, src: Span) -> Result { + Ok(match self.find_sequences(input, src)? { + Some((rep, last_ll_size)) => { + (self.seq_store).store_last_literals(src.bytes(&input.bufs)?, last_ll_size)?; + block_state_mut(&mut self.next_c_block).rep = rep; + BuildSeqStore::Compress + } + None => BuildSeqStore::NoCompress, + }) + } + + /// `ZSTD_buildSeqStore`, but for the repeat offsets after the block and the literals after the last sequence, which + /// are not stored: it returns the former and the number of the latter. `None` is `ZSTDbss_noCompress`. + fn find_sequences( + &mut self, + input: &Input<'_>, + src: Span, + ) -> Result> { + let params = &self.applied_params; + let ms = &mut self.match_state; + let src_size = src.len(); + if src_size < MIN_CBLOCK_SIZE + ZSTD_BLOCKHEADERSIZE + 1 + 1 { + if params.c_params.strat() >= Strategy::BtOpt { + ldm::skip_raw_seq_store_bytes(&mut self.extern_seq_store, src_size)?; + } else { + ldm::skip_sequences( + &mut self.extern_seq_store, + src_size, + params.c_params.min_match, + )?; + } + return Ok(None); + } + self.seq_store.start_block(src_size)?; + ms.opt.literal_compression_mode = ParamSwitch::Auto; + + { + // Not to spend too long catching up after incompressible data. + let curr = ms.window.index_of(src.start_addr()); + if curr > ms.next_to_update.wrapping_add(384) { + let behind = curr.wrapping_sub(ms.next_to_update).wrapping_sub(384); + ms.next_to_update = curr - behind.min(192); + } + } + + let window = ms.window; + let block = BlockSrc { + buf: input.bufs.get(window.base_buf), + base: window.base.wrapping_sub(window.base_buf.start()), + dict_buf: input.bufs.get(window.dict_buf), + dict_base: window.dict_base.wrapping_sub(window.dict_buf.start()), + start: src.start, + end: src.end, + dms: match input.cdict { + Some(cdict) if ms.has_dict_match_state => Some(cdict.dict_match_state()), + _ => None, + }, + symbol_costs: &block_state(&self.prev_c_block).entropy, + ldm_seq_store: None, + }; + let dict_mode = ms.dict_mode(block.dms.as_ref()); + let use_row_match_finder = params.use_row_match_finder == ParamSwitch::Enable; + let mut rep_after = block_state(&self.prev_c_block).rep; + let rep = &mut rep_after; + + let last_ll_size = if self.extern_seq_store.pos < self.extern_seq_store.size { + ldm::block_compress( + &mut self.extern_seq_store, + ms, + &mut self.seq_store, + rep, + use_row_match_finder, + &block, + )? + } else if params.ldm_params.enable_ldm == ParamSwitch::Enable { + let ldm_seq_store = &mut self.ldm_sequences; + ldm_seq_store.pos = 0; + ldm_seq_store.pos_in_sequence = 0; + ldm_seq_store.size = 0; + ldm::generate_sequences( + &mut self.ldm_state, + ldm_seq_store, + ¶ms.ldm_params, + &input.bufs, + src.start, + src.end, + )?; + ldm::block_compress( + ldm_seq_store, + ms, + &mut self.seq_store, + rep, + use_row_match_finder, + &block, + )? + } else { + compress_block( + ms, + &mut self.seq_store, + rep, + &block, + params.c_params.strat(), + use_row_match_finder, + dict_mode, + )? + }; + Ok(Some((rep_after, last_ll_size))) + } + + /// Whether `ZSTD_compressLiterals` would leave `literals` as they are without a look at all of them. A block that is + /// nothing else then comes out larger than it went in: to find that out, the literals need not be copied and the + /// tables for the next block need not be made. + fn literals_stay_as_they_are(&self, literals: &[u8]) -> Result { + let prev_huf = &block_state(&self.prev_c_block).entropy.huf; + let strategy = self.applied_params.c_params.strat(); + Ok( + literals.len() < min_literals_to_compress(strategy, prev_huf.repeat_mode) + || literals_compression_is_disabled(&self.applied_params) + || huf::samples_are_incompressible(literals)?, + ) + } + + /// `ZSTD_compressSeqStore_singleBlock` + fn compress_seq_store_single_block( + &mut self, + chunk: &SeqChunk, + d_rep: &mut [u32; ZSTD_REP_NUM], + c_rep: &mut [u32; ZSTD_REP_NUM], + dst: &mut [u8], + src: &[u8], + last_block: bool, + is_partition: bool, + ) -> Result { + const RLE_MAX_LENGTH: usize = 25; + let d_rep_original = *d_rep; + if is_partition { + seq_store_resolve_off_codes(d_rep, c_rep, &mut self.seq_store, chunk)?; + } + + if dst.len() < ZSTD_BLOCKHEADERSIZE { + return Err(Error::DstSizeTooSmall); + } + let mut c_seqs_size = entropy_compress_seq_store( + &mut self.seq_store, + chunk, + &block_state(&self.prev_c_block).entropy, + &mut block_state_mut(&mut self.next_c_block).entropy, + &self.applied_params, + dst.sub_mut(ZSTD_BLOCKHEADERSIZE..)?, + src.len(), + None, + )?; + + if !self.is_first_block && c_seqs_size < RLE_MAX_LENGTH && is_rle(src) { + // Decoders up to 1.4.3 do not take a first block that is a run. + c_seqs_size = 1; + } + + let c_size = match c_seqs_size { + 0 => { + *d_rep = d_rep_original; + no_compress_block(dst, src, last_block)? + } + 1 => { + *d_rep = d_rep_original; + rle_compress_block(dst, src.at(0)?, src.len(), last_block)? + } + _ => { + self.confirm_repcodes_and_entropy_tables(); + write_block_header(dst, c_seqs_size, src.len(), last_block)?; + ZSTD_BLOCKHEADERSIZE + c_seqs_size + } + }; + + self.downgrade_offcode_repeat_mode(); + Ok(c_size) + } + + /// `ZSTD_deriveBlockSplitsHelper`: splits the sequences from `start_idx` to `end_idx` in two if that looks + /// smaller, then each half likewise. + fn derive_block_splits_helper( + &mut self, + splits: &mut SeqStoreSplits, + start_idx: usize, + end_idx: usize, + entropy_metadata: &mut EntropyCTablesMetadata, + ) -> Result<()> { + const MIN_SEQUENCES_BLOCK_SPLITTING: usize = 300; + let mid_idx = (start_idx + end_idx) / 2; + + if end_idx - start_idx < MIN_SEQUENCES_BLOCK_SPLITTING + || splits.idx >= ZSTD_MAX_NB_BLOCK_SPLITS + { + return Ok(()); + } + let mut estimate = |from: usize, to: usize| -> Result> { + let chunk = derive_seq_store_chunk(&self.seq_store, from, to)?; + let size = build_entropy_statistics_and_estimate_sub_block_size( + &mut self.seq_store, + &chunk, + &block_state(&self.prev_c_block).entropy, + &mut block_state_mut(&mut self.next_c_block).entropy, + &self.applied_params, + entropy_metadata, + ); + match size { + // Not among what makes for no split: the C code allocates nothing here. + Err(Error::MemoryAllocation) => Err(Error::MemoryAllocation), + size => Ok(size), + } + }; + let estimated_original_size = estimate(start_idx, end_idx)?; + let estimated_first_half_size = estimate(start_idx, mid_idx)?; + let estimated_second_half_size = estimate(mid_idx, end_idx)?; + let (Ok(original), Ok(first_half), Ok(second_half)) = ( + estimated_original_size, + estimated_first_half_size, + estimated_second_half_size, + ) else { + return Ok(()); + }; + if first_half + second_half < original { + self.derive_block_splits_helper(splits, start_idx, mid_idx, entropy_metadata)?; + splits.split_locations.put(splits.idx, mid_idx as u32)?; + splits.idx += 1; + self.derive_block_splits_helper(splits, mid_idx, end_idx, entropy_metadata)?; + } + Ok(()) + } + + /// `ZSTD_compressBlock_splitBlock_internal` + fn compress_block_split_block_internal( + &mut self, + dst: &mut [u8], + src: &[u8], + last_block: bool, + ) -> Result { + let nb_seq = self.seq_store.sequences.len(); + let block_size = src.len(); + let mut c_size = 0usize; + let mut ip = 0usize; + let mut src_bytes_total = 0usize; + + // `ZSTD_deriveBlockSplits` + let mut splits = SeqStoreSplits { + split_locations: [0; PARTITIONS_LEN], + idx: 0, + }; + if nb_seq > 4 { + let mut entropy_metadata = EntropyCTablesMetadata::new(); + self.derive_block_splits_helper(&mut splits, 0, nb_seq, &mut entropy_metadata)?; + splits.split_locations.put(splits.idx, nb_seq as u32)?; + } + let num_splits = splits.idx; + let partitions = &splits.split_locations; + + let mut d_rep = block_state(&self.prev_c_block).rep; + let mut c_rep = d_rep; + + if num_splits == 0 { + let whole = self.seq_store.whole(); + return self.compress_seq_store_single_block( + &whole, &mut d_rep, &mut c_rep, dst, src, last_block, false, + ); + } + + let mut curr_seq_store = + derive_seq_store_chunk(&self.seq_store, 0, partitions.at(0)? as usize)?; + for i in 0..=num_splits { + let last_partition = i == num_splits; + let mut last_block_entire_src = false; + + let mut src_bytes = count_seq_store_literals_bytes(&self.seq_store, &curr_seq_store)? + + count_seq_store_match_bytes(&self.seq_store, &curr_seq_store)?; + src_bytes_total += src_bytes; + let mut next_seq_store = curr_seq_store; + if last_partition { + // The last literals are in no sequence. + src_bytes += block_size - src_bytes_total; + last_block_entire_src = last_block; + } else { + next_seq_store = derive_seq_store_chunk( + &self.seq_store, + partitions.at(i)? as usize, + partitions.at(i + 1)? as usize, + )?; + } + + let c_size_chunk = self.compress_seq_store_single_block( + &curr_seq_store, + &mut d_rep, + &mut c_rep, + dst.sub_mut(c_size..)?, + src.span(ip, src_bytes)?, + last_block_entire_src, + true, + )?; + + ip += src_bytes; + c_size += c_size_chunk; + curr_seq_store = next_seq_store; + } + // What the decoder ends up with, which is not what the sequences were found with if a part is uncompressed. + block_state_mut(&mut self.prev_c_block).rep = d_rep; + Ok(c_size) + } + + /// `ZSTD_compressBlock_splitBlock` + fn compress_block_split_block( + &mut self, + dst: &mut [u8], + input: &Input<'_>, + src: Span, + last_block: bool, + ) -> Result { + let bytes = src.bytes(&input.bufs)?; + if self.build_seq_store(input, src)? == BuildSeqStore::NoCompress { + self.downgrade_offcode_repeat_mode(); + return no_compress_block(dst, bytes, last_block); + } + self.compress_block_split_block_internal(dst, bytes, last_block) + } + + /// `ZSTD_compressBlock_internal`: the content of a block, which is 1 byte for a run. Returns 0 if the block is + /// better left uncompressed. + fn compress_block_internal( + &mut self, + dst: &mut Output<'_>, + op: usize, + input: &Input<'_>, + src: Span, + ) -> Result { + const RLE_MAX_LENGTH: usize = 25; + let bytes = src.bytes(&input.bufs)?; + + let c_size = match self.find_sequences(input, src)? { + None => Ok(0), + Some((rep, last_ll_size)) => { + let room = dst.room_from(op).saturating_sub(ZSTD_BLOCKHEADERSIZE); + // If they fit as they are, which spares thinking about what fails first. They stay out of the store. + let only_literals = + self.seq_store.sequences.is_empty() && bytes.len() + 3 + 4 <= room; + let mut c_size = if only_literals && self.literals_stay_as_they_are(bytes)? { + Ok(0) + } else { + let store = &mut self.seq_store; + let most = if only_literals { + huf::room_for_quick_loops(bytes.len()) + } else { + store.store_last_literals(bytes, last_ll_size)?; + // A block that comes out no smaller than it went in is stored as it is, so it makes no + // difference when an encoder runs out of that much room. + (huf::room_for_quick_loops(store.lit_len) + + sequences::max_size(store.sequences.len())) + .min(bytes.len()) + }; + let next = block_state_mut(&mut self.next_c_block); + next.rep = rep; + let whole = store.whole(); + entropy_compress_seq_store( + store, + &whole, + &block_state(&self.prev_c_block).entropy, + &mut next.entropy, + &self.applied_params, + (dst.from(op, most + ROOM_TO_SPARE)?).sub_mut(ZSTD_BLOCKHEADERSIZE..)?, + bytes.len(), + only_literals.then_some(bytes), + ) + }; + if !self.is_first_block + && c_size.is_ok_and(|c_size| c_size < RLE_MAX_LENGTH) + && is_rle(bytes) + { + dst.from(op, ZSTD_BLOCKHEADERSIZE + 1)? + .put(ZSTD_BLOCKHEADERSIZE, bytes.at(0)?)?; + c_size = Ok(1); + } + c_size + } + }; + + if c_size.is_ok_and(|c_size| c_size > 1) { + self.confirm_repcodes_and_entropy_tables(); + } + self.downgrade_offcode_repeat_mode(); + c_size + } + + /// `ZSTD_compressBlock_targetCBlockSize` + fn compress_block_target_c_block_size( + &mut self, + dst: &mut [u8], + input: &Input<'_>, + src: Span, + last_block: bool, + ) -> Result { + let bytes = src.bytes(&input.bufs)?; + let bss = self.build_seq_store(input, src)?; + let c_size = self.compress_block_target_c_block_size_body(dst, bytes, bss, last_block)?; + self.downgrade_offcode_repeat_mode(); + Ok(c_size) + } + + /// `ZSTD_compressBlock_targetCBlockSize_body` + fn compress_block_target_c_block_size_body( + &mut self, + dst: &mut [u8], + src: &[u8], + bss: BuildSeqStore, + last_block: bool, + ) -> Result { + if bss == BuildSeqStore::Compress { + if !self.is_first_block && maybe_rle(&self.seq_store) && is_rle(src) { + return rle_compress_block(dst, src.at(0)?, src.len(), last_block); + } + match compress_super_block( + &mut self.seq_store, + block_state(&self.prev_c_block), + block_state_mut(&mut self.next_c_block), + &self.applied_params, + dst, + src, + last_block, + ) { + Err(Error::DstSizeTooSmall) => {} + c_size => { + let c_size = c_size?; + let max_c_size = src + .len() + .wrapping_sub(min_gain(src.len(), self.applied_params.c_params.strat())); + if c_size != 0 && c_size < max_c_size.wrapping_add(ZSTD_BLOCKHEADERSIZE) { + self.confirm_repcodes_and_entropy_tables(); + return Ok(c_size); + } + } + } + } + no_compress_block(dst, src, last_block) + } + + /// `ZSTD_optimalBlockSize`: `src` is what remains of the input. + fn optimal_block_size(&self, src: &[u8], savings: i64) -> Result { + const SPLIT_LEVELS: [i32; 10] = [0, 0, 1, 2, 2, 3, 3, 4, 4, 4]; + let block_size_max = self.block_size_max; + // Only full blocks are worth looking into. + if src.len() < 128 << 10 || block_size_max < 128 << 10 { + return Ok(src.len().min(block_size_max)); + } + // Data that does not compress has to be in full blocks, for the output to stay within `compress_bound`. + if savings < 3 { + return Ok(128 << 10); + } + let split_level = SPLIT_LEVELS.at(self.applied_params.c_params.strat() as usize)?; + presplit::split_block(src.sub(..block_size_max)?, split_level) + } + + /// `ZSTD_compress_frameChunk` + fn compress_frame_chunk( + &mut self, + dst: &mut Output<'_>, + start: usize, + input: &Input<'_>, + src: Span, + last_frame_chunk: bool, + ) -> Result { + let bytes = src.bytes(&input.bufs)?; + let mut ip = src.start; + let mut op = start; + let max_dist = 1u32 << self.applied_params.c_params.window_log; + let mut savings = self.consumed_src_size as i64 - self.produced_c_size as i64; + + if self.applied_params.f_params.checksum_flag && !bytes.is_empty() { + self.xxh_state.update(bytes); + } + + while ip < src.end { + self.match_state.check_cancelled()?; + let remaining = src.end - ip; + let block_size = self.optimal_block_size(bytes.sub(ip - src.start..)?, savings)?; + let last_block = last_frame_chunk && block_size == remaining; + let block = Span { + buf: src.buf, + start: ip, + end: ip + block_size, + }; + if dst.room_from(op) < ZSTD_BLOCKHEADERSIZE + MIN_CBLOCK_SIZE + 1 { + return Err(Error::DstSizeTooSmall); + } + + let ms = &mut self.match_state; + overflow_correct_if_needed( + ms, + &self.applied_params, + block.start_addr(), + block.end_addr(), + ); + if (ms.window).dict_is_invalid(block.end_addr(), max_dist, ms.loaded_dict_end) { + ms.loaded_dict_end = 0; + ms.has_dict_match_state = false; + } + if (ms.window).enforce_max_dist(block.start_addr(), max_dist, ms.loaded_dict_end) { + ms.loaded_dict_end = 0; + ms.has_dict_match_state = false; + } + ms.next_to_update = ms.next_to_update.max(ms.window.low_limit); + + // Parts, which take long to make anyway, get all the room there is. + let c_size = if self.applied_params.target_c_block_size != 0 { + let out = dst.from(op, usize::MAX)?; + self.compress_block_target_c_block_size(out, input, block, last_block)? + } else if self.applied_params.post_block_splitter == ParamSwitch::Enable { + let out = dst.from(op, usize::MAX)?; + self.compress_block_split_block(out, input, block, last_block)? + } else { + let c_size = self.compress_block_internal(dst, op, input, block)?; + if c_size == 0 { + dst.store_block(op, block.bytes(&input.bufs)?, last_block)? + } else { + let out = dst.from(op, ZSTD_BLOCKHEADERSIZE)?; + write_block_header(out, c_size, block_size, last_block)?; + c_size + ZSTD_BLOCKHEADERSIZE + } + }; + + savings += block_size as i64 - c_size as i64; + ip += block_size; + op += c_size; + self.is_first_block = false; + } + + if last_frame_chunk && op > start { + self.stage = CompressionStage::Ending; + } + Ok(op - start) + } + + /// `ZSTD_compressContinue_internal` + fn compress_continue_internal( + &mut self, + dst: &mut Output<'_>, + input: &Input<'_>, + src: Span, + last_frame_chunk: bool, + ) -> Result { + let mut fh_size = 0usize; + if self.stage == CompressionStage::Created { + return Err(Error::StageWrong); + } + + if self.stage == CompressionStage::Init { + fh_size = write_frame_header( + dst.from(0, ZSTD_FRAMEHEADERSIZE_MAX)?, + &self.applied_params, + self.pledged_src_size_plus_one.wrapping_sub(1), + self.dict_id, + )?; + self.stage = CompressionStage::Ongoing; + } + + if src.len() == 0 { + return Ok(fh_size); + } + + let ms = &mut self.match_state; + if !(ms.window).update( + src.start_addr(), + src.buf, + src.len(), + ms.force_non_contiguous, + ) { + ms.force_non_contiguous = false; + ms.next_to_update = ms.window.dict_limit; + } + if self.applied_params.ldm_params.enable_ldm == ParamSwitch::Enable { + (self.ldm_state.window).update(src.start_addr(), src.buf, src.len(), false); + } + + let c_size = self.compress_frame_chunk(dst, fh_size, input, src, last_frame_chunk)?; + self.consumed_src_size += src.len() as u64; + self.produced_c_size += (c_size + fh_size) as u64; + if self.pledged_src_size_plus_one != 0 + && self.consumed_src_size + 1 > self.pledged_src_size_plus_one + { + return Err(Error::SrcSizeWrong); + } + Ok(c_size + fh_size) + } + + /// `ZSTD_compressContinue` + pub(crate) fn compress_continue( + &mut self, + dst: &mut [u8], + input: &Input<'_>, + src: Span, + ) -> Result { + self.compress_continue_internal(&mut Output::Slice(dst), input, src, false) + } + + /// `ZSTD_writeEpilogue` + fn write_epilogue(&mut self, dst: &mut [u8]) -> Result { + let mut op = 0usize; + if self.stage == CompressionStage::Created { + return Err(Error::StageWrong); + } + + // An empty frame. + if self.stage == CompressionStage::Init { + op += write_frame_header(dst, &self.applied_params, 0, 0)?; + self.stage = CompressionStage::Ongoing; + } + + if self.stage != CompressionStage::Ending { + // One last empty block, to mark the end of the frame. + if dst.len() - op < 3 { + return Err(Error::DstSizeTooSmall); + } + write_le24(dst, op, block_header(BlockType::Raw, 0, true))?; + op += ZSTD_BLOCKHEADERSIZE; + } + + if self.applied_params.f_params.checksum_flag { + let checksum = self.xxh_state.digest() as u32; + if dst.len() - op < 4 { + return Err(Error::DstSizeTooSmall); + } + write_le32(dst, op, checksum)?; + op += 4; + } + + self.stage = CompressionStage::Created; + Ok(op) + } + + /// `ZSTD_compressEnd` + pub(crate) fn compress_end( + &mut self, + dst: &mut [u8], + input: &Input<'_>, + src: Span, + ) -> Result { + self.compress_end_to(&mut Output::Slice(dst), input, src) + } + + /// [`Self::compress_end`] to either kind of output. + pub(crate) fn compress_end_to( + &mut self, + dst: &mut Output<'_>, + input: &Input<'_>, + src: Span, + ) -> Result { + const EPILOGUE_SIZE_MAX: usize = ZSTD_BLOCKHEADERSIZE + 4; + let c_size = self.compress_continue_internal(dst, input, src, true)?; + let end_result = self.write_epilogue(dst.from(c_size, EPILOGUE_SIZE_MAX)?)?; + if self.pledged_src_size_plus_one != 0 + && self.pledged_src_size_plus_one != self.consumed_src_size + 1 + { + return Err(Error::SrcSizeWrong); + } + Ok(c_size + end_result) + } +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use crate::common::ZSTD_WORKSPACETOOLARGE_MAXDURATION; + use crate::compress::params::{FrameParameters, Parameters}; + + /// With tables that are as small as can be, but for the hash table. + pub(crate) fn small_params(strategy: Strategy, hash_log: u32) -> CCtxParams { + let params = Parameters { + c_params: CompressionParameters { + window_log: 10, + chain_log: 6, + hash_log, + search_log: 1, + min_match: 4, + target_length: 0, + strategy: Some(strategy), + }, + f_params: FrameParameters { + content_size_flag: false, + checksum_flag: false, + no_dict_id_flag: false, + }, + }; + CCtxParams::from_params(¶ms, 0) + } + + /// It takes 3.5 GB through a context for where its indices stand to show in a frame. + #[test] + fn indices_start_again_where_the_c_code_makes_a_new_workspace() -> Result<()> { + const JOB_SIZE: u32 = 64; + let data = [0u8; JOB_SIZE as usize]; + let input = Input { + bufs: Buffers { + in_buff: &[], + user: &data, + dict: &[], + }, + cdict: None, + }; + let src = Span { + buf: BufId::User, + start: 0, + end: data.len(), + }; + let begin = |cctx: &mut FrameCompressor, hash_log| { + let params = small_params(Strategy::Fast, hash_log); + cctx.begin(&input, None, ¶ms, u64::from(JOB_SIZE), false) + }; + // Returns the index that the next byte would get. + let compress = |cctx: &mut FrameCompressor, hash_log| -> Result { + begin(cctx, hash_log)?; + cctx.compress_continue(&mut [0u8; 128], &input, src)?; + let window = &cctx.match_state.window; + Ok(window.index_of(window.next_src)) + }; + let after_jobs = |jobs: u32| ZSTD_WINDOW_START_INDEX + jobs * JOB_SIZE; + + let worker = FrameCompressor::for_worker(Arc::new(AtomicBool::new(false))); + for mut cctx in [FrameCompressor::new(), worker] { + assert_eq!(compress(&mut cctx, 6)?, after_jobs(1)); + assert_eq!(compress(&mut cctx, 6)?, after_jobs(2)); + // Too small. + assert_eq!(compress(&mut cctx, 15)?, after_jobs(1)); + // Much too large, for too long. + for _ in 1..ZSTD_WORKSPACETOOLARGE_MAXDURATION { + begin(&mut cctx, 6)?; + } + assert_eq!(compress(&mut cctx, 6)?, after_jobs(2)); + assert_eq!(compress(&mut cctx, 6)?, after_jobs(1)); + // As large as it takes, for however long. + for _ in 0..=ZSTD_WORKSPACETOOLARGE_MAXDURATION { + begin(&mut cctx, 6)?; + } + assert_eq!(compress(&mut cctx, 6)?, after_jobs(2)); + } + Ok(()) + } + + #[test] + #[cfg_attr(miri, ignore = "takes minutes")] + fn second_tags_are_given_up_within_a_block() -> Result<()> { + const SIZE: usize = 1 << 17; + // Many positions start like another one and go on otherwise. + let mut state = 1u32; + let few_symbols = Vec::from_iter((0..SIZE).map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + b'A' + (state >> 30) as u8 + })); + // What starts like another position goes on like it. + let records = Vec::from_iter((0..SIZE).map(|n| match n % 100 { + 0 => (n / 100) as u8, + 1 => (n / 25_600) as u8, + n => n as u8, + })); + let tags_after_a_block_of = |data: &[u8]| -> Result { + let input = Input { + bufs: Buffers { + in_buff: &[], + user: data, + dict: &[], + }, + cdict: None, + }; + let src = Span { + buf: BufId::User, + start: 0, + end: data.len(), + }; + let mut params = small_params(Strategy::Lazy2, 17); + params.c_params.window_log = 17; + params.c_params.search_log = 6; + params.use_row_match_finder = ParamSwitch::Enable; + let mut cctx = FrameCompressor::new(); + cctx.begin(&input, None, ¶ms, data.len() as u64, false)?; + assert!(cctx.match_state.row_tags != RowTags::Flat); + cctx.compress_continue(&mut vec![0; 2 * SIZE], &input, src)?; + Ok(cctx.match_state.row_tags) + }; + + assert!(tags_after_a_block_of(&few_symbols)? != RowTags::Flat); + assert!(tags_after_a_block_of(&records)? == RowTags::Flat); + // Over the whole block they rule out a lot, either way. There is no way back. + let (records, _) = records.split_at(16 << 10); + let (few_symbols, _) = few_symbols.split_at(112 << 10); + assert!(tags_after_a_block_of(&[few_symbols, records].concat())? != RowTags::Flat); + assert!(tags_after_a_block_of(&[records, few_symbols].concat())? == RowTags::Flat); + Ok(()) + } +} diff --git a/src/zstd/compress/double_fast.rs b/src/zstd/compress/double_fast.rs new file mode 100644 index 000000000000..bdfdd0658257 --- /dev/null +++ b/src/zstd/compress/double_fast.rs @@ -0,0 +1,624 @@ +//! `zstd_double_fast.c`: the `dfast` strategy, which looks up a table of long hashes and a table of short ones. + +use super::fast::{ + DictBlock, ExtDictBlock, Found, MatchSink, NoDictBlock, Segments, TaggedTable, ThirdPositions, + after, put_tagged, put_tagged_in_between, restore_rep, +}; +use super::internal::{ + BlockSrc, HASH_READ_SIZE, Hasher, K_SEARCH_STRENGTH, MatchState, Near, Prefix, + REPCODE1_TO_OFFBASE, SeqStore, TableFillPurpose, hashed_get, hashed_put, hashed_swap, le64_at, + offset_to_offbase, table_of_size, table_prefetch, table_put, +}; +use crate::common::ZSTD_REP_NUM; +use crate::error::Result; +use crate::util::{BUG, SliceExt}; + +/// `ZSTD_fillDoubleHashTableForCDict`: the tables are empty. +fn fill_double_hash_table_for_cdict(ms: &mut MatchState, prefix: Prefix<'_>, end: usize) { + let long = Hasher::new(ms.tagged_hash_log(), 8); + let small = Hasher::new(ms.tagged_chain_log(), ms.c_params.min_match); + let positions = ThirdPositions::new(prefix, ms.next_to_update, end); + let hash_large = ms.hash_table.as_mut_slice(); + let hash_small = ms.chain_table.as_mut_slice(); + put_tagged_in_between(hash_large, long, positions.clone()); + for at in positions { + let bytes = le64_at::<0, _>(at.bytes); + put_tagged(hash_small, small.hash(bytes), at.index); + put_tagged(hash_large, long.hash(bytes), at.index); + } +} + +/// `ZSTD_fillDoubleHashTableForCCtx` +fn fill_double_hash_table_for_cctx(ms: &mut MatchState, prefix: Prefix<'_>, end: usize) { + let long = Hasher::new(ms.c_params.hash_log, 8); + let small = Hasher::new(ms.c_params.chain_log, ms.c_params.min_match); + let positions = ThirdPositions::new(prefix, ms.next_to_update, end); + let hash_large = ms.hash_table.as_mut_slice(); + let hash_small = ms.chain_table.as_mut_slice(); + for at in positions { + let bytes = le64_at::<0, _>(at.bytes); + table_put(hash_small, small.hash(bytes), at.index); + table_put(hash_large, long.hash(bytes), at.index); + } +} + +/// `ZSTD_fillDoubleHashTable`: indexes the segment from `ms.next_to_update` up to the position `end`. +pub(crate) fn fill_double_hash_table( + ms: &mut MatchState, + prefix: Prefix<'_>, + end: usize, + tfp: TableFillPurpose, +) { + match tfp { + TableFillPurpose::ForCDict => fill_double_hash_table_for_cdict(ms, prefix, end), + TableFillPurpose::ForCCtx => fill_double_hash_table_for_cctx(ms, prefix, end), + } +} + +/// Has what a search from `pos` on looks at first in the tables on its way. `pos` is the end of a match that was just +/// found: there is enough to do about that one in the meantime. +#[inline(always)] +fn look_ahead( + near: Near<'_>, + pos: u32, + hash_long: &[u32], + long: Hasher, + hash_small: &[u32], + small: Hasher, +) { + if let Some(bytes) = near.read64(pos) { + table_prefetch(hash_long, long.hash(bytes)); + table_prefetch(hash_small, small.hash(bytes)); + } +} + +/// Its loops have no register to spare for a buffer that moves. +type Sink<'a, 'b> = MatchSink<'a, 'b, 16, false>; + +/// What `dfast` looks for matches with, when there is no dictionary and the window is one segment. +struct Finder { + /// The index of the first position. + base_index: u32, + long: Hasher, + small: Hasher, + /// Where to start. + first: u32, + offset_1: u32, + offset_2: u32, +} + +impl Finder { + /// `ZSTD_compressBlock_doubleFast_noDict_generic` + /// + /// The C code computes the long hash and reads its table one position ahead. Here that is left to the processor: a + /// turn of the inner loop only takes the position and the 8 bytes there, `bytes0`, from the one before. + /// + /// `LONG` and `SMALL` are how many bits `long` and `small` give, or both 0 for any: see [`table_of_size`]. + #[inline(never)] + fn find_matches( + &mut self, + hash_long: &mut [u32], + hash_small: &mut [u32], + sink: &mut Sink<'_, '_>, + ) { + const K_STEP_INCR: u32 = 1 << K_SEARCH_STRENGTH; + let Self { + base_index, + long, + small, + first: mut ip, + mut offset_1, + mut offset_2, + } = *self; + let near = sink.near; + let long = if LONG > 0 { Hasher::new(LONG, 8) } else { long }; + let small = small.of_size::(); + let Some(hash_long) = table_of_size::(hash_long) else { + return; + }; + let Some(hash_small) = table_of_size::(hash_small) else { + return; + }; + let Some(mut bytes0) = near.read64(ip) else { + return; + }; + + // One turn per match. + 'block: loop { + let mut step = 1; + let mut next_step = ip + K_STEP_INCR; + let mut ip1 = ip + step; + let mut curr; + + // One turn per position. + let end = loop { + let Some(bytes1) = near.read64(ip1) else { + break 'block; + }; + curr = base_index.wrapping_add(ip); + let idxl0 = hashed_swap(hash_long, long.hash(bytes0), curr); + let idxs0 = hashed_swap(hash_small, small.hash(bytes0), curr); + + if sink.read32_before(ip + 1, offset_1) == Some((bytes0 >> 8) as u32) + && after(offset_1 > 0) + { + ip += 1; + let end = ip + 4 + sink.count_before(ip + 4, offset_1); + // Small tables are close by as it is. + if LONG == 0 || LONG >= 17 { + look_ahead(near, end, hash_long, long, hash_small, small); + } + sink.store(ip, end, REPCODE1_TO_OFFBASE); + break end; + } + + // An index that is too low leads past the end. + let matchl0 = idxl0.wrapping_sub(base_index); + let matchs0 = idxs0.wrapping_sub(base_index); + let (match0, end) = if near.has::<{ u64::MAX }>(matchl0, bytes0) { + (matchl0, ip + 8 + near.count(ip + 8, matchl0 + 8)) + } else if near.has::<0xFFFF_FFFF>(matchs0, bytes0) { + // A short match: a longer one at the next position is preferred. + let end = ip + 4 + near.count(ip + 4, matchs0 + 4); + let idxl1 = hashed_get(hash_long, long.hash(bytes1)); + let matchl1 = idxl1.wrapping_sub(base_index); + if near.has::<{ u64::MAX }>(matchl1, bytes1) && matchl1 > 0 { + let end1 = ip1 + 8 + near.count(ip1 + 8, matchl1 + 8); + if end1 - ip1 > end - ip { + ip = ip1; + (matchl1, end1) + } else { + (matchs0, end) + } + } else { + (matchs0, end) + } + } else { + if ip1 >= next_step { + core::hint::cold_path(); + step += 1; + next_step += K_STEP_INCR; + } + ip = ip1; + ip1 += step; + bytes0 = bytes1; + continue; + }; + + if LONG == 0 || LONG >= 17 { + look_ahead(near, end, hash_long, long, hash_small, small); + } + offset_2 = offset_1; + offset_1 = ip - match0; + // With a larger step, ip1 may be past the end of the match. + if step < 4 { + hashed_put(hash_long, long.hash(bytes1), base_index.wrapping_add(ip1)); + } + let back = near.count_backwards(ip, sink.anchor, match0, 0); + sink.store(ip - back, end, offset_to_offbase(offset_1)); + break end; + }; + ip = end; + + // `ip` is not past the last position that can be looked at. + let Some(around) = near.window::<{ 2 + HASH_READ_SIZE }>(ip - 2) else { + break 'block; + }; + bytes0 = le64_at::<2, _>(around); + let index_to_insert = curr + 2; + if let Some(to_insert) = near.read64(index_to_insert.wrapping_sub(base_index)) { + hashed_put(hash_long, long.hash(to_insert), index_to_insert); + hashed_put(hash_small, small.hash(to_insert), index_to_insert); + } + hashed_put( + hash_long, + long.hash(le64_at::<0, _>(around)), + base_index.wrapping_add(ip - 2), + ); + hashed_put( + hash_small, + small.hash(le64_at::<1, _>(around)), + base_index.wrapping_add(ip - 1), + ); + + while offset_2 > 0 && sink.read32_before(ip, offset_2) == Some(bytes0 as u32) { + let end = ip + 4 + sink.count_before(ip + 4, offset_2); + core::mem::swap(&mut offset_1, &mut offset_2); + let index = base_index.wrapping_add(ip); + hashed_put(hash_small, small.hash(bytes0), index); + hashed_put(hash_long, long.hash(bytes0), index); + sink.store_repeat(end); + ip = end; + let Some(bytes) = near.read64(ip) else { + break 'block; + }; + bytes0 = bytes; + } + } + + self.offset_1 = offset_1; + self.offset_2 = offset_2; + } +} + +/// `ZSTD_compressBlock_doubleFast` +pub(crate) fn compress_block_double_fast( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + let block = NoDictBlock::new(ms, rep, src)?; + let h_bits_l = ms.c_params.hash_log; + let h_bits_s = ms.c_params.chain_log; + let mut finder = Finder { + base_index: block.prefix_lowest_index, + long: Hasher::new(h_bits_l, 8), + small: Hasher::new(h_bits_s, ms.c_params.min_match), + first: block.first, + offset_1: block.rep.offset_1, + offset_2: block.rep.offset_2, + }; + let mut sink = Sink::new(seq_store, &block); + let hash_long = ms.hash_table.sub_mut(..1 << h_bits_l)?; + let hash_small = ms.chain_table.sub_mut(..1 << h_bits_s)?; + // What the levels come with, for an input of any size. + match (h_bits_l, h_bits_s) { + (18, 18) => finder.find_matches::<18, 18>(hash_long, hash_small, &mut sink), + (17, 17) => finder.find_matches::<17, 17>(hash_long, hash_small, &mut sink), + (17, 16) => finder.find_matches::<17, 16>(hash_long, hash_small, &mut sink), + (16, 16) => finder.find_matches::<16, 16>(hash_long, hash_small, &mut sink), + (16, 15) => finder.find_matches::<16, 15>(hash_long, hash_small, &mut sink), + (15, 14) => finder.find_matches::<15, 14>(hash_long, hash_small, &mut sink), + (14, 14) => finder.find_matches::<14, 14>(hash_long, hash_small, &mut sink), + (14, 13) => finder.find_matches::<14, 13>(hash_long, hash_small, &mut sink), + (13, 12) => finder.find_matches::<13, 12>(hash_long, hash_small, &mut sink), + (12, 11) => finder.find_matches::<12, 11>(hash_long, hash_small, &mut sink), + (11, 10) => finder.find_matches::<11, 10>(hash_long, hash_small, &mut sink), + _ => finder.find_matches::<0, 0>(hash_long, hash_small, &mut sink), + } + let last_literals = sink.last_literals()?; + restore_rep( + rep, + finder.offset_1, + finder.offset_2, + block.rep.offset_saved1, + block.rep.offset_saved2, + ); + Ok(last_literals) +} + +impl Finder { + /// `ZSTD_compressBlock_doubleFast_dictMatchState_generic` + #[inline(never)] + fn find_matches_dict( + &mut self, + hash_long: &mut [u32], + hash_small: &mut [u32], + sink: &mut Sink<'_, '_>, + window: Segments<'_>, + dict_tables: &DictTables<'_>, + ) { + let Self { + base_index, + long, + small, + first: mut ip, + mut offset_1, + mut offset_2, + } = *self; + let near = window.near; + // As it is made, which the compiler does not see. + let long = long.with_mls(8); + if hash_long.is_empty() || hash_small.is_empty() { + return; + } + + // `ip` is before the last position that can be looked at. + 'search: while let Some(here) = near.window::<{ HASH_READ_SIZE + 1 }>(ip) { + let bytes0 = le64_at::<0, _>(here); + let curr = base_index.wrapping_add(ip); + let match_long = + hashed_swap(hash_long, long.hash(bytes0), curr).wrapping_sub(base_index); + let match_index_s = hashed_swap(hash_small, small.hash(bytes0), curr); + + if let Some(end) = window.repeat_end(ip + 1, offset_1, (bytes0 >> 8) as u32) { + sink.store(ip + 1, end, REPCODE1_TO_OFFBASE); + ip = end; + } else { + // `ip` is moved to where the match is for. + let found = 'found: { + if near.has::<{ u64::MAX }>(match_long, bytes0) { + break 'found window.found_near(ip, sink.anchor, match_long, 8); + } + if let Some(place) = dict_tables.long.get(long, bytes0) + && window.older.read64(place) == Some(bytes0) + { + break 'found window.found_older(ip, sink.anchor, place, 8); + } + + // The dictionary is only looked up if there is no candidate in the prefix. + let short = if match_index_s > base_index { + let pos = match_index_s - base_index; + (near.read32(pos) == Some(bytes0 as u32)).then(|| window.place_of(pos)) + } else { + (dict_tables.short.get(small, bytes0)) + .filter(|&place| window.older.read32(place) == Some(bytes0 as u32)) + }; + let Some(place_short) = short else { + ip += ((ip - sink.anchor) >> K_SEARCH_STRENGTH) + 1; + continue 'search; + }; + + // A short match: a long one at the next position is preferred. + let bytes1 = le64_at::<1, _>(here); + let match_long1 = hashed_swap(hash_long, long.hash(bytes1), curr + 1) + .wrapping_sub(base_index); + if near.has::<{ u64::MAX }>(match_long1, bytes1) { + ip += 1; + break 'found window.found_near(ip, sink.anchor, match_long1, 8); + } + if let Some(place) = dict_tables.long.get(long, bytes1) + && window.older.read64(place) == Some(bytes1) + { + ip += 1; + break 'found window.found_older(ip, sink.anchor, place, 8); + } + Found { + end: ip + 4 + window.count(ip + 4, place_short + 4), + back: window.count_backwards(ip, sink.anchor, place_short), + offset: window.place_of(ip) - place_short, + } + }; + offset_2 = offset_1; + offset_1 = found.offset; + sink.store(ip - found.back, found.end, offset_to_offbase(offset_1)); + ip = found.end; + } + + // `ip` is not past the last position that can be looked at. + let Some(around) = near.window::<{ 2 + HASH_READ_SIZE }>(ip - 2) else { + break; + }; + let index_to_insert = curr + 2; + if let Some(to_insert) = near.read64(index_to_insert.wrapping_sub(base_index)) { + hashed_put(hash_long, long.hash(to_insert), index_to_insert); + hashed_put(hash_small, small.hash(to_insert), index_to_insert); + } + hashed_put( + hash_long, + long.hash(le64_at::<0, _>(around)), + base_index.wrapping_add(ip - 2), + ); + hashed_put( + hash_small, + small.hash(le64_at::<1, _>(around)), + base_index.wrapping_add(ip - 1), + ); + + while let Some(bytes) = near.read64(ip) + && let Some(end) = window.repeat_end(ip, offset_2, bytes as u32) + { + core::mem::swap(&mut offset_1, &mut offset_2); + let index = base_index.wrapping_add(ip); + hashed_put(hash_small, small.hash(bytes), index); + hashed_put(hash_long, long.hash(bytes), index); + sink.store(ip, end, REPCODE1_TO_OFFBASE); + ip = end; + } + } + + self.offset_1 = offset_1; + self.offset_2 = offset_2; + } +} + +/// The tables of a dictionary. +struct DictTables<'a> { + long: TaggedTable<'a>, + short: TaggedTable<'a>, +} + +/// `ZSTD_compressBlock_doubleFast_dictMatchState` +pub(crate) fn compress_block_double_fast_dict_match_state( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + let prefix_lowest_index = + ms.get_lowest_prefix_index(src.index_of(src.end), ms.c_params.window_log); + let block = DictBlock::new(src, prefix_lowest_index)?; + let dms = src.dms.as_ref().ok_or(BUG)?; + let tables = &DictTables { + long: TaggedTable::new( + &dms.ms.hash_table, + dms.ms.tagged_hash_log(), + block.dict_start_index, + )?, + short: TaggedTable::new( + &dms.ms.chain_table, + dms.ms.tagged_chain_log(), + block.dict_start_index, + )?, + }; + let h_bits_l = ms.c_params.hash_log; + let h_bits_s = ms.c_params.chain_log; + let [offset_1, offset_2, _] = *rep; + let mut finder = Finder { + base_index: block.prefix_start_index, + long: Hasher::new(h_bits_l, 8), + small: Hasher::new(h_bits_s, ms.c_params.min_match), + first: block.first(), + offset_1, + offset_2, + }; + let sink = &mut Sink::with_near(seq_store, block.window.near, block.istart); + let long = ms.hash_table.sub_mut(..1 << h_bits_l)?; + let small = ms.chain_table.sub_mut(..1 << h_bits_s)?; + finder.find_matches_dict(long, small, sink, block.window, tables); + let last_literals = sink.last_literals()?; + let [rep0, rep1, _] = rep; + *rep0 = finder.offset_1; + *rep1 = finder.offset_2; + Ok(last_literals) +} + +impl Finder { + /// `ZSTD_compressBlock_doubleFast_extDict_generic`. `LONG` and `SMALL` are as for [`Self::find_matches`]. + #[inline(never)] + fn find_matches_ext_dict( + &mut self, + hash_long: &mut [u32], + hash_small: &mut [u32], + sink: &mut Sink<'_, '_>, + window: Segments<'_>, + ) { + let Self { + base_index, + long, + small, + first: mut ip, + mut offset_1, + mut offset_2, + } = *self; + let near = window.near; + let older_len = window.older.len(); + // The index of the first place. + let origin_index = base_index.wrapping_sub(older_len); + let long = if LONG > 0 { Hasher::new(LONG, 8) } else { long }; + let small = small.of_size::(); + let Some(hash_long) = table_of_size::(hash_long) else { + return; + }; + let Some(hash_small) = table_of_size::(hash_small) else { + return; + }; + + // `ip` is before the last position that can be looked at. + while let Some(here) = near.window::<{ HASH_READ_SIZE + 1 }>(ip) { + let bytes0 = le64_at::<0, _>(here); + let curr = base_index.wrapping_add(ip); + let place_long = + hashed_swap(hash_long, long.hash(bytes0), curr).wrapping_sub(origin_index); + let place_short = + hashed_swap(hash_small, small.hash(bytes0), curr).wrapping_sub(origin_index); + + if window.read32_before(ip + 1, offset_1) == Some((bytes0 >> 8) as u32) + && offset_1 <= window.place_of(ip + 1) + { + ip += 1; + let rep_place = window.place_of(ip) - offset_1; + let end = ip + 4 + window.count(ip + 4, rep_place + 4); + look_ahead(near, end, hash_long, long, hash_small, small); + sink.store(ip, end, REPCODE1_TO_OFFBASE); + ip = end; + } else { + // The match, and how much of it is known. `ip` is moved to where it is for. + let (place, known) = + if window.has::<{ u64::MAX }>(place_long, bytes0) && place_long > 0 { + (place_long, 8) + } else if window.has::<0xFFFF_FFFF>(place_short, bytes0) && place_short > 0 { + // A short match: a long one at the next position is preferred. + let bytes1 = le64_at::<1, _>(here); + let place3 = hashed_swap(hash_long, long.hash(bytes1), curr + 1) + .wrapping_sub(origin_index); + if window.has::<{ u64::MAX }>(place3, bytes1) && place3 > 0 { + ip += 1; + (place3, 8) + } else { + (place_short, 4) + } + } else { + ip += ((ip - sink.anchor) >> K_SEARCH_STRENGTH) + 1; + continue; + }; + let end = ip + known + window.count(ip + known, place + known); + look_ahead(near, end, hash_long, long, hash_small, small); + let back = window.count_backwards_if_any(ip, sink.anchor, place); + offset_2 = offset_1; + offset_1 = window.place_of(ip) - place; + sink.store(ip - back, end, offset_to_offbase(offset_1)); + ip = end; + } + + // `ip` is not past the last position that can be looked at. + let Some(around) = near.window::<{ 2 + HASH_READ_SIZE }>(ip - 2) else { + break; + }; + let index_to_insert = curr + 2; + if let Some(to_insert) = near.read64(index_to_insert.wrapping_sub(base_index)) { + hashed_put(hash_long, long.hash(to_insert), index_to_insert); + hashed_put(hash_small, small.hash(to_insert), index_to_insert); + } + hashed_put( + hash_long, + long.hash(le64_at::<0, _>(around)), + base_index.wrapping_add(ip - 2), + ); + hashed_put( + hash_small, + small.hash(le64_at::<1, _>(around)), + base_index.wrapping_add(ip - 1), + ); + + while let Some(bytes) = near.read64(ip) { + if !(window.read32_before(ip, offset_2) == Some(bytes as u32) + && offset_2 <= window.place_of(ip)) + { + break; + } + let rep_place = window.place_of(ip) - offset_2; + let end = ip + 4 + window.count(ip + 4, rep_place + 4); + core::mem::swap(&mut offset_1, &mut offset_2); + let index = base_index.wrapping_add(ip); + hashed_put(hash_small, small.hash(bytes), index); + hashed_put(hash_long, long.hash(bytes), index); + sink.store_repeat(end); + ip = end; + } + } + + self.offset_1 = offset_1; + self.offset_2 = offset_2; + } +} + +/// `ZSTD_compressBlock_doubleFast_extDict` +pub(crate) fn compress_block_double_fast_ext_dict( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + let Some(block) = ExtDictBlock::new(ms, rep, src)? else { + // Nothing of the older segment is within reach. + return compress_block_double_fast(ms, seq_store, rep, src); + }; + let h_bits_l = ms.c_params.hash_log; + let h_bits_s = ms.c_params.chain_log; + let [offset_1, offset_2, _] = *rep; + let mut finder = Finder { + base_index: block.prefix_start_index, + long: Hasher::new(h_bits_l, 8), + small: Hasher::new(h_bits_s, ms.c_params.min_match), + first: block.istart, + offset_1, + offset_2, + }; + let mut sink = Sink::with_near(seq_store, block.window.near, block.istart); + let hash_long = ms.hash_table.sub_mut(..1 << h_bits_l)?; + let hash_small = ms.chain_table.sub_mut(..1 << h_bits_s)?; + let (long, small, sink) = (hash_long, hash_small, &mut sink); + // What the levels come with, for an input that is large enough to get here. + match (h_bits_l, h_bits_s) { + (18, 18) => finder.find_matches_ext_dict::<18, 18>(long, small, sink, block.window), + (17, 16) => finder.find_matches_ext_dict::<17, 16>(long, small, sink, block.window), + _ => finder.find_matches_ext_dict::<0, 0>(long, small, sink, block.window), + } + let last_literals = sink.last_literals()?; + let [rep0, rep1, _] = rep; + *rep0 = finder.offset_1; + *rep1 = finder.offset_2; + Ok(last_literals) +} diff --git a/src/zstd/compress/fast.rs b/src/zstd/compress/fast.rs new file mode 100644 index 000000000000..3cde100c5c53 --- /dev/null +++ b/src/zstd/compress/fast.rs @@ -0,0 +1,1133 @@ +//! `zstd_fast.c`: the `fast` strategy, which looks up one hash table. + +use core::sync::atomic::{Ordering, compiler_fence}; + +use super::internal::{ + BlockSrc, CompressionParameters, HASH_READ_SIZE, Hasher, K_SEARCH_STRENGTH, MatchState, Near, + Prefix, REPCODE1_TO_OFFBASE, SeqSink, SeqStore, TableFillPurpose, ZSTD_SHORT_CACHE_TAG_BITS, + compare_packed_tags, hashed_get, hashed_put, hashed_swap, le32_at, le64_at, offset_to_offbase, + table_of_size, table_put, +}; +use crate::common::ZSTD_REP_NUM; +use crate::error::Result; +use crate::util::{BUG, SliceExt}; + +const FAST_HASH_FILL_STEP: u32 = 3; + +/// The repeat offsets at the start of a block. Those that reach further than `max_rep` are 0, and saved. +pub(super) struct SavedRep { + pub(super) offset_1: u32, + pub(super) offset_2: u32, + pub(super) offset_saved1: u32, + pub(super) offset_saved2: u32, +} + +impl SavedRep { + pub(super) const fn new(rep: &[u32; ZSTD_REP_NUM], max_rep: u32) -> Self { + let [mut offset_1, mut offset_2, _] = *rep; + let (mut offset_saved1, mut offset_saved2) = (0, 0); + if offset_2 > max_rep { + offset_saved2 = offset_2; + offset_2 = 0; + } + if offset_1 > max_rep { + offset_saved1 = offset_1; + offset_1 = 0; + } + Self { + offset_1, + offset_2, + offset_saved1, + offset_saved2, + } + } +} + +/// The repeat offsets to leave at the end of a block, at the start of which those that reached too far were saved and +/// set to 0. +#[inline(always)] +pub(super) fn restore_rep( + rep: &mut [u32; ZSTD_REP_NUM], + offset_1: u32, + offset_2: u32, + offset_saved1: u32, + offset_saved2: u32, +) { + // If offset_1 started invalid and became valid, offset_saved1 is what rotates into the second place. + let offset_saved2 = if offset_saved1 != 0 && offset_1 != 0 { + offset_saved1 + } else { + offset_saved2 + }; + let [rep0, rep1, _] = rep; + *rep0 = if offset_1 != 0 { + offset_1 + } else { + offset_saved1 + }; + *rep1 = if offset_2 != 0 { + offset_2 + } else { + offset_saved2 + }; +} + +/// What `fast` and `dfast` index a dictionary by: every third position, with the 10 bytes there, which the 8 bytes at +/// it and at the two positions in between are among. +/// `array_windows().step_by(3)` gives the same, and indexing a dictionary then takes 1.4 to 1.6 times as long. +#[derive(Clone)] +pub(super) struct ThirdPositions<'a> { + /// From the first position that is left up to the end of the 10 bytes at the last one. + bytes: &'a [u8], + /// The index of the first position that is left. + index: u32, +} + +/// One of [`ThirdPositions`]. +pub(super) struct ThirdPosition<'a> { + pub(super) index: u32, + pub(super) bytes: &'a [u8; THIRD_POSITION_SIZE], +} + +const THIRD_POSITION_SIZE: usize = FAST_HASH_FILL_STEP as usize - 1 + HASH_READ_SIZE; + +impl<'a> ThirdPositions<'a> { + /// From `first_index` up to the position `end` of the segment: none if that is before. + pub(super) fn new(prefix: Prefix<'a>, first_index: u32, end: usize) -> Self { + let bytes = prefix.buf.get(prefix.pos_of(first_index)..end); + let bytes = bytes.unwrap_or_default(); + let len = match bytes.len().checked_sub(THIRD_POSITION_SIZE) { + Some(more) => bytes.len() - more % FAST_HASH_FILL_STEP as usize, + None => 0, + }; + Self { + bytes: bytes.get(..len).unwrap_or_default(), + index: first_index, + } + } +} + +impl<'a> Iterator for ThirdPositions<'a> { + type Item = ThirdPosition<'a>; + + #[inline(always)] + fn next(&mut self) -> Option { + let item = ThirdPosition { + index: self.index, + bytes: self.bytes.first_chunk()?, + }; + self.bytes = self.bytes.get(FAST_HASH_FILL_STEP as usize..)?; + self.index += FAST_HASH_FILL_STEP; + Some(item) + } +} + +impl DoubleEndedIterator for ThirdPositions<'_> { + #[inline(always)] + fn next_back(&mut self) -> Option { + let (before, bytes) = self.bytes.split_last_chunk()?; + let item = ThirdPosition { + index: self.index + before.len() as u32, + bytes, + }; + let len = self.bytes.len() - FAST_HASH_FILL_STEP as usize; + self.bytes = self.bytes.get(..len)?; + Some(item) + } +} + +/// `ZSTD_writeTaggedIndex`: the tables of a dictionary keep 8 more bits of the hash beside each index. The size of +/// `table` is a power of 2 that the hash is less than. +#[inline(always)] +pub(super) fn put_tagged(table: &mut [u32], hash_and_tag: usize, index: u32) { + let hash = hash_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS; + if let Some(cell) = table.get_mut(hash & table.len().wrapping_sub(1)) { + *cell = (index << ZSTD_SHORT_CACHE_TAG_BITS) | u32::from(hash_and_tag as u8); + } +} + +/// Indexes the positions between `positions`, for which the C code only has the cells that are empty when it gets to +/// them, on its way from the first position to the last. As `table` is empty, it comes to the same to go from the last +/// to the first, and to index `positions` afterwards. Whether a cell is empty, which is as good as random, is then of +/// no concern. +#[inline(always)] +pub(super) fn put_tagged_in_between( + table: &mut [u32], + hasher: Hasher, + positions: ThirdPositions<'_>, +) { + debug_assert!(table.iter().all(|&cell| cell == 0)); + for at in positions.rev() { + put_tagged(table, hasher.hash(le64_at::<2, _>(at.bytes)), at.index + 2); + put_tagged(table, hasher.hash(le64_at::<1, _>(at.bytes)), at.index + 1); + } +} + +/// `ZSTD_fillHashTableForCDict`: the table is empty. +fn fill_hash_table_for_cdict(ms: &mut MatchState, prefix: Prefix<'_>, end: usize) { + let hasher = Hasher::new(ms.tagged_hash_log(), ms.c_params.min_match); + let positions = ThirdPositions::new(prefix, ms.next_to_update, end); + let hash_table = ms.hash_table.as_mut_slice(); + put_tagged_in_between(hash_table, hasher, positions.clone()); + for at in positions { + put_tagged(hash_table, hasher.hash(le64_at::<0, _>(at.bytes)), at.index); + } +} + +/// `ZSTD_fillHashTableForCCtx` +fn fill_hash_table_for_cctx(ms: &mut MatchState, prefix: Prefix<'_>, end: usize) { + let hasher = Hasher::new(ms.c_params.hash_log, ms.c_params.min_match); + let positions = ThirdPositions::new(prefix, ms.next_to_update, end); + let hash_table = ms.hash_table.as_mut_slice(); + for at in positions { + table_put(hash_table, hasher.hash(le64_at::<0, _>(at.bytes)), at.index); + } +} + +/// `ZSTD_fillHashTable`: indexes the segment from `ms.next_to_update` up to the position `end`. +pub(crate) fn fill_hash_table( + ms: &mut MatchState, + prefix: Prefix<'_>, + end: usize, + tfp: TableFillPurpose, +) { + match tfp { + TableFillPurpose::ForCDict => fill_hash_table_for_cdict(ms, prefix, end), + TableFillPurpose::ForCCtx => fill_hash_table_for_cctx(ms, prefix, end), + } +} + +/// `targetLength + !targetLength`: how far `fast` goes from one position to the next, before it speeds up. +const fn least_step(c_params: &CompressionParameters) -> u32 { + c_params.target_length + (c_params.target_length == 0) as u32 +} + +/// What `fast` and `dfast` work out before they look for matches in a block, when there is no dictionary and the window +/// is one segment. +pub(super) struct NoDictBlock<'a> { + /// The input from `prefix_lowest_index`, the lowest index a match can be at, up to the end of the block. Positions + /// are in this: a lower index, less that one, wraps around to a position past the end. + near: Near<'a>, + pub(super) prefix_lowest_index: u32, + /// The same with what is before it, as far as a repeat offset reaches: where the window started at the start of the + /// block. + older: Near<'a>, + /// How much that is. + older_len: u32, + istart: u32, + /// The first position to look at. + pub(super) first: u32, + pub(super) rep: SavedRep, +} + +impl<'a> NoDictBlock<'a> { + pub(super) fn new( + ms: &MatchState, + rep: &[u32; ZSTD_REP_NUM], + src: &BlockSrc<'a>, + ) -> Result { + let window_log = ms.c_params.window_log; + let start_index = src.index_of(src.start); + let prefix_lowest_index = ms.get_lowest_prefix_index(src.index_of(src.end), window_log); + let first_index = start_index + u32::from(start_index == prefix_lowest_index); + let window_low = ms.get_lowest_prefix_index(first_index, window_log); + + let older_index = window_low.min(prefix_lowest_index); + let older = src.buf.sub(src.pos_of(older_index)..src.end)?; + let older_len = prefix_lowest_index - older_index; + Ok(Self { + near: Near::new(older.sub(older_len as usize..)?)?, + prefix_lowest_index, + older: Near::new(older)?, + older_len, + istart: start_index - prefix_lowest_index, + first: first_index - prefix_lowest_index, + rep: SavedRep::new(rep, first_index.wrapping_sub(window_low)), + }) + } +} + +/// What `fast` and `dfast` work out before they look for matches in a block, when the window is two segments. +pub(super) struct ExtDictBlock<'a> { + pub(super) window: Segments<'a>, + /// The index of the first position of the current segment. + pub(super) prefix_start_index: u32, + /// Where the block starts in it. + pub(super) istart: u32, + pub(super) rep: SavedRep, +} + +impl<'a> ExtDictBlock<'a> { + /// `None` if nothing of the older segment is within reach. + pub(super) fn new( + ms: &MatchState, + rep: &[u32; ZSTD_REP_NUM], + src: &BlockSrc<'a>, + ) -> Result> { + let dict_start_index = + ms.get_lowest_match_index(src.index_of(src.end), ms.c_params.window_log); + let prefix_start_index = ms.window.dict_limit.max(dict_start_index); + if prefix_start_index == dict_start_index { + return Ok(None); + } + let start_index = src.index_of(src.start); + let older = (src.dict_buf) + .sub(src.dict_pos_of(dict_start_index)..src.dict_pos_of(prefix_start_index))?; + // As far as the start of the older segment is too far too. + let max_rep = start_index.wrapping_sub(dict_start_index); + Ok(Some(Self { + window: Segments { + near: Near::new(src.buf.sub(src.pos_of(prefix_start_index)..src.end)?)?, + older: Near::new(older)?, + }, + prefix_start_index, + istart: start_index - prefix_start_index, + rep: SavedRep::new(rep, max_rep.saturating_sub(1)), + })) + } +} + +/// Where `fast` and `dfast` put the matches they find in a block, when there is no dictionary and the window is one +/// segment. All of it is read and written once per match, so that the search for the next one has the registers. +/// +/// `SHORT` and `GROWS` are those of [`SeqSink`]. +pub(super) struct MatchSink<'a, 'b, const SHORT: usize, const GROWS: bool> { + seq_store: SeqSink<'b, SHORT, GROWS>, + /// The input from the lowest index a match can be at. Positions are in this. + pub(super) near: Near<'a>, + /// The same with what is before it, as far as a repeat offset can reach. + older: Near<'a>, + /// How much that is. + older_len: u32, + /// The end of the last match. + pub(super) anchor: u32, + failed: bool, +} + +impl<'a, 'b, const SHORT: usize, const GROWS: bool> MatchSink<'a, 'b, SHORT, GROWS> { + pub(super) fn new(seq_store: &'b mut SeqStore, block: &NoDictBlock<'a>) -> Self { + Self { + seq_store: SeqSink::new(seq_store), + near: block.near, + older: block.older, + older_len: block.older_len, + anchor: block.istart, + failed: false, + } + } + + /// For a block that starts at the position `anchor` of `near`, which nothing is read before. + pub(super) fn with_near(seq_store: &'b mut SeqStore, near: Near<'a>, anchor: u32) -> Self { + Self { + seq_store: SeqSink::new(seq_store), + near, + older: near, + older_len: 0, + anchor, + failed: false, + } + } + + /// How many bytes there are after the last match. + pub(super) fn last_literals(&self) -> Result { + if self.failed { + return Err(BUG); + } + Ok((self.near.len() - self.anchor) as usize) + } + + /// The 4 bytes that are `offset` before `pos`. + #[inline(always)] + pub(super) fn read32_before(&self, pos: u32, offset: u32) -> Option { + match self.near.read32(pos.wrapping_sub(offset)) { + Some(bytes) => Some(bytes), + None => { + core::hint::cold_path(); + self.older + .read32((pos + self.older_len).wrapping_sub(offset)) + } + } + } + + /// How long what is at `pos` is the same as what is `offset` before. + #[inline(always)] + pub(super) fn count_before(&self, pos: u32, offset: u32) -> u32 { + if offset <= pos { + self.near.count(pos, pos - offset) + } else { + core::hint::cold_path(); + let pos = pos + self.older_len; + self.older.count(pos, pos.wrapping_sub(offset)) + } + } + + /// Stores the match from `ip` to `end`, with the literals before it. + #[inline(always)] + pub(super) fn store(&mut self, ip: u32, end: u32, off_base: u32) { + let anchor = self.anchor; + self.anchor = end; + if (self.seq_store) + .store_seq(ip - anchor, self.near, anchor, off_base, end - ip) + .is_err() + { + core::hint::cold_path(); + self.failed = true; + } + } + + /// Stores the match with the last offset that goes on from the one before up to `end`. + #[inline(always)] + pub(super) fn store_repeat(&mut self, end: u32) { + let ip = self.anchor; + self.anchor = end; + if (self.seq_store) + .store_seq_only(0, REPCODE1_TO_OFFBASE, end - ip) + .is_err() + { + core::hint::cold_path(); + self.failed = true; + } + } + + /// Whether the bytes before `ip` and before what is `offset` before it are the same. + #[inline(always)] + fn same_byte_before(&self, ip: u32, offset: u32) -> bool { + let ip = ip + self.older_len; + self.older.read8(ip - 1) == self.older.read8((ip - 1).wrapping_sub(offset)) + } +} + +/// What can be matched when that is two segments: the current one, and before it either the older one of the window or +/// the content of a dictionary. A place is a position in the two put end to end. +#[derive(Clone, Copy)] +pub(super) struct Segments<'a> { + /// The current segment, up to the end of the block. + pub(super) near: Near<'a>, + /// The other one, as far as it is within reach. + pub(super) older: Near<'a>, +} + +/// Where a place is. +#[derive(Clone, Copy)] +struct Spot<'a> { + segment: Near<'a>, + pos: u32, +} + +impl<'a> Segments<'a> { + #[inline(always)] + fn spot(self, place: u32) -> Spot<'a> { + let older_len = self.older.len(); + if place < older_len { + Spot { + segment: self.older, + pos: place, + } + } else { + Spot { + segment: self.near, + pos: place - older_len, + } + } + } + + /// The place of a position of the current segment. + #[inline(always)] + pub(super) const fn place_of(self, pos: u32) -> u32 { + pos.wrapping_add(self.older.len()) + } + + #[inline(always)] + pub(super) fn read8(self, place: u32) -> Option { + let spot = self.spot(place); + spot.segment.read8(spot.pos) + } + + /// Whether the bits of `MASK` are the same in `bytes` and in the 8 bytes at `place`, which are in one segment, as they + /// are wherever a table leads to. + #[inline(always)] + pub(super) fn has(self, place: u32, bytes: u64) -> bool { + let spot = self.spot(place); + spot.segment.has::(spot.pos, bytes) + } + + /// The 4 bytes that are `offset` before the position `pos`, if they are in one segment. Which one that is does not + /// change much from one time to the next. + #[inline(always)] + pub(super) fn read32_before(self, pos: u32, offset: u32) -> Option { + let at = pos.wrapping_sub(offset); + match self.near.read32(at) { + Some(bytes) => Some(bytes), + None => self.older.read32(self.place_of(at)), + } + } + + /// `ZSTD_count_2segments`: how long what is at the position `ip` is the same as what is at `place`. + #[inline(always)] + pub(super) fn count(self, ip: u32, place: u32) -> u32 { + let spot = self.spot(place); + let len = self.near.count_in(ip, spot.segment, spot.pos); + if place.wrapping_add(len) != self.older.len() { + return len; + } + len + self.near.count(ip + len, 0) + } + + /// [`Self::count`] for a place of the older segment. + #[inline(always)] + fn count_in_older(self, ip: u32, place: u32) -> u32 { + let len = self.near.count_in(ip, self.older, place); + if place + len != self.older.len() { + return len; + } + len + self.near.count(ip + len, 0) + } + + /// Where the match of the position `ip` with what is `offset` before it ends, if there is one: `bytes` are the 4 + /// bytes at `ip`, and those before have to be in one segment. + #[inline(always)] + pub(super) fn repeat_end(self, ip: u32, offset: u32, bytes: u32) -> Option { + let at = ip.wrapping_sub(offset); + let len = match self.near.read32(at) { + Some(there) if there == bytes => self.near.count(ip + 4, at + 4), + Some(_) => return None, + None => { + let place = self.place_of(at); + if self.older.read32(place) != Some(bytes) { + return None; + } + self.count_in_older(ip + 4, place + 4) + } + }; + Some(ip + 4 + len) + } + + /// The match at the position `ip` with what is at `pos` of the current segment, of which `known` bytes are known. + #[inline(always)] + pub(super) fn found_near(self, ip: u32, anchor: u32, pos: u32, known: u32) -> Found { + Found { + end: ip + known + self.near.count(ip + known, pos + known), + back: self.near.count_backwards(ip, anchor, pos, 0), + offset: ip - pos, + } + } + + /// The same with what is at `place`, which is in the older segment. + #[inline(always)] + pub(super) fn found_older(self, ip: u32, anchor: u32, place: u32, known: u32) -> Found { + let len = known + self.count_in_older(ip + known, place + known); + let back = if ip == anchor { + 0 + } else { + (self.near).count_backwards_in(ip, self.older, place, (ip - anchor).min(place)) + }; + Found { + end: ip + len, + back, + offset: self.place_of(ip) - place, + } + } + + /// [`Self::count_backwards`], with a branch on whether `ip` is `anchor`: whoever got there without a step is likely + /// to do so again. + #[inline(always)] + pub(super) fn count_backwards_if_any(self, ip: u32, anchor: u32, place: u32) -> u32 { + if ip == anchor { + return 0; + } + self.count_backwards(ip, anchor, place) + } + + /// How many bytes before the position `ip`, down to `anchor`, are the same as those before `place` in its segment. + #[inline(always)] + pub(super) fn count_backwards(self, ip: u32, anchor: u32, place: u32) -> u32 { + let spot = self.spot(place); + (self.near).count_backwards_in(ip, spot.segment, spot.pos, (ip - anchor).min(spot.pos)) + } +} + +/// A match at a position. +pub(super) struct Found { + pub(super) end: u32, + /// How many bytes before the position belong to it as well. + pub(super) back: u32, + pub(super) offset: u32, +} + +/// `condition`, which the compiler then does not look at before what is on its left. +#[inline(always)] +pub(super) fn after(condition: bool) -> bool { + compiler_fence(Ordering::SeqCst); + condition +} + +type Sink<'a, 'b> = MatchSink<'a, 'b, 32, true>; + +/// What `fast` looks for matches with, when there is no dictionary and the window is one segment. +struct Finder { + /// The index of the first position. + base_index: u32, + step_size: u32, + hasher: Hasher, + /// Where to start. + first: u32, + rep_offset1: u32, + rep_offset2: u32, +} + +impl Finder { + /// `ZSTD_compressBlock_fast_noDict_generic` + /// + /// Each turn of the inner loop looks at two positions, that of index `current0` and the next one, for a match, and + /// at `ip2` for a repeat. The C code computes hashes and reads the table one position ahead. Here that is left to + /// the processor: a turn only takes the position and the 8 bytes there, `bytes0`, from the one before. + /// + /// `HLOG` is how many bits `hasher` gives, or 0 for any: see [`table_of_size`]. + #[inline(never)] + fn find_matches(&mut self, hash_table: &mut [u32], sink: &mut Sink<'_, '_>) { + const K_STEP_INCR: u32 = 1 << (K_SEARCH_STRENGTH - 1); + let Self { + base_index, + step_size, + hasher, + first: mut ip0, + mut rep_offset1, + mut rep_offset2, + } = *self; + let near = sink.near; + let hasher = hasher.of_size::(); + let Some(hash_table) = table_of_size::(hash_table) else { + return; + }; + let Some(mut bytes0) = near.read64(ip0) else { + return; + }; + + 'block: loop { + let mut step = step_size; + let mut next_step = ip0 + K_STEP_INCR; + let mut ip2 = ip0 + step; + let mut current0 = base_index.wrapping_add(ip0); + + // Ends with where a match starts and ends. + let (start, end, off_base) = loop { + // There is a position after `ip2` to look at, and that one is before the last that can be. + let Some(ahead) = near.window::<{ HASH_READ_SIZE + 2 }>(ip2) else { + break 'block; + }; + let idx0 = hashed_swap(hash_table, hasher.hash(bytes0), current0); + + if sink.read32_before(ip2, rep_offset1) == Some(le32_at::<0, _>(ahead)) + && after(rep_offset1 > 0) + { + hashed_put(hash_table, hasher.hash(bytes0 >> 8), current0 + 1); + let back = u32::from(sink.same_byte_before(ip2, rep_offset1)); + let end = ip2 + 4 + sink.count_before(ip2 + 4, rep_offset1); + break (ip2 - back, end, REPCODE1_TO_OFFBASE); + } + let mut match0 = idx0.wrapping_sub(base_index); + if near.has::<0xFFFF_FFFF>(match0, bytes0) { + hashed_put(hash_table, hasher.hash(bytes0 >> 8), current0 + 1); + } else { + // As far as they are hashed or compared. + let bytes1 = bytes0 >> 8; + current0 += 1; + let idx1 = hashed_swap(hash_table, hasher.hash(bytes1), current0); + match0 = idx1.wrapping_sub(base_index); + if !near.has::<0xFFFF_FFFF>(match0, bytes1) { + current0 = base_index.wrapping_add(ip2); + bytes0 = le64_at::<0, _>(ahead); + ip2 += step; + if ip2 >= next_step { + core::hint::cold_path(); + step += 1; + next_step += K_STEP_INCR; + } + continue; + } + // `ip2` is certain to be before the end of the match only if the step is small. + if step <= 4 { + hashed_put( + hash_table, + hasher.hash(le64_at::<0, _>(ahead)), + base_index.wrapping_add(ip2), + ); + } + } + let ip = current0.wrapping_sub(base_index); + rep_offset2 = rep_offset1; + rep_offset1 = ip - match0; + let end = ip + 4 + near.count(ip + 4, match0 + 4); + let back = near.count_backwards(ip, sink.anchor, match0, 0); + break (ip - back, end, offset_to_offbase(rep_offset1)); + }; + + ip0 = end; + sink.store(start, end, off_base); + + // `ip0` is not past the last position that can be looked at. + let Some(around) = near.window::<{ 2 + HASH_READ_SIZE }>(ip0 - 2) else { + break 'block; + }; + bytes0 = le64_at::<2, _>(around); + if let Some(to_insert) = near.read64((current0 + 2).wrapping_sub(base_index)) { + hashed_put(hash_table, hasher.hash(to_insert), current0 + 2); + } + hashed_put( + hash_table, + hasher.hash(le64_at::<0, _>(around)), + base_index.wrapping_add(ip0 - 2), + ); + + while rep_offset2 > 0 && sink.read32_before(ip0, rep_offset2) == Some(bytes0 as u32) { + let end = ip0 + 4 + sink.count_before(ip0 + 4, rep_offset2); + core::mem::swap(&mut rep_offset1, &mut rep_offset2); + hashed_put( + hash_table, + hasher.hash(bytes0), + base_index.wrapping_add(ip0), + ); + sink.store_repeat(end); + ip0 = end; + let Some(bytes) = near.read64(ip0) else { + break 'block; + }; + bytes0 = bytes; + } + } + + self.rep_offset1 = rep_offset1; + self.rep_offset2 = rep_offset2; + } +} + +impl Finder { + /// `ZSTD_compressBlock_fast_extDict_generic`, which goes about it as [`Self::find_matches`] does. + #[inline(never)] + fn find_matches_ext_dict( + &mut self, + hash_table: &mut [u32], + sink: &mut Sink<'_, '_>, + window: Segments<'_>, + ) { + const K_STEP_INCR: u32 = 1 << (K_SEARCH_STRENGTH - 1); + let Self { + base_index, + step_size, + hasher, + first: mut ip0, + mut rep_offset1, + mut rep_offset2, + } = *self; + let near = window.near; + let older_len = window.older.len(); + // The index of the first place. + let origin_index = base_index.wrapping_sub(older_len); + let hasher = hasher.of_size::(); + let Some(hash_table) = table_of_size::(hash_table) else { + return; + }; + let Some(mut bytes0) = near.read64(ip0) else { + return; + }; + + 'block: loop { + let mut step = step_size; + let mut next_step = ip0 + K_STEP_INCR; + let mut ip2 = ip0 + step; + let mut current0 = base_index.wrapping_add(ip0); + + // Ends with where 4 bytes are the same as earlier, and with the next position if it is still to be indexed. + let (ip, place, back, off_base, next) = loop { + let Some(ahead) = near.window::<{ HASH_READ_SIZE + 2 }>(ip2) else { + break 'block; + }; + let idx0 = hashed_swap(hash_table, hasher.hash(bytes0), current0); + + // The 4 bytes have to be in one segment, with a byte before them. + let rep_place = window.place_of(ip2).wrapping_sub(rep_offset1); + if window.read32_before(ip2, rep_offset1) == Some(le32_at::<0, _>(ahead)) + && rep_place != older_len + && rep_offset1 > 0 + { + hashed_put(hash_table, hasher.hash(bytes0 >> 8), current0 + 1); + let back = + u32::from(near.read8(ip2 - 1) == window.read8(rep_place.wrapping_sub(1))); + break (ip2, rep_place, back, REPCODE1_TO_OFFBASE, u32::MAX); + } + let mut place = idx0.wrapping_sub(origin_index); + let mut next = u32::MAX; + if window.has::<0xFFFF_FFFF>(place, bytes0) { + hashed_put(hash_table, hasher.hash(bytes0 >> 8), current0 + 1); + } else { + let bytes1 = bytes0 >> 8; + current0 += 1; + let idx1 = hashed_swap(hash_table, hasher.hash(bytes1), current0); + place = idx1.wrapping_sub(origin_index); + if !window.has::<0xFFFF_FFFF>(place, bytes1) { + current0 = base_index.wrapping_add(ip2); + bytes0 = le64_at::<0, _>(ahead); + ip2 += step; + if ip2 >= next_step { + core::hint::cold_path(); + step += 1; + next_step += K_STEP_INCR; + } + continue; + } + next = ip2; + } + let ip = current0.wrapping_sub(base_index); + rep_offset2 = rep_offset1; + rep_offset1 = window.place_of(ip) - place; + let back = window.count_backwards_if_any(ip, sink.anchor, place); + break (ip, place, back, offset_to_offbase(rep_offset1), next); + }; + + ip0 = ip + 4 + window.count(ip + 4, place + 4); + sink.store(ip - back, ip0, off_base); + + if next < ip0 + && let Some(bytes) = near.read64(next) + { + hashed_put( + hash_table, + hasher.hash(bytes), + base_index.wrapping_add(next), + ); + } + + // `ip0` is not past the last position that can be looked at. + let Some(around) = near.window::<{ 2 + HASH_READ_SIZE }>(ip0 - 2) else { + break 'block; + }; + bytes0 = le64_at::<2, _>(around); + if let Some(to_insert) = near.read64((current0 + 2).wrapping_sub(base_index)) { + hashed_put(hash_table, hasher.hash(to_insert), current0 + 2); + } + hashed_put( + hash_table, + hasher.hash(le64_at::<0, _>(around)), + base_index.wrapping_add(ip0 - 2), + ); + + while window.read32_before(ip0, rep_offset2) == Some(bytes0 as u32) && rep_offset2 > 0 { + let rep_place = window.place_of(ip0).wrapping_sub(rep_offset2); + let end = ip0 + 4 + window.count(ip0 + 4, rep_place + 4); + core::mem::swap(&mut rep_offset1, &mut rep_offset2); + hashed_put( + hash_table, + hasher.hash(bytes0), + base_index.wrapping_add(ip0), + ); + sink.store_repeat(end); + ip0 = end; + let Some(bytes) = near.read64(ip0) else { + break 'block; + }; + bytes0 = bytes; + } + } + + self.rep_offset1 = rep_offset1; + self.rep_offset2 = rep_offset2; + } +} + +/// `ZSTD_compressBlock_fast` +pub(crate) fn compress_block_fast( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + let block = NoDictBlock::new(ms, rep, src)?; + let hlog = ms.c_params.hash_log; + let mut finder = Finder { + base_index: block.prefix_lowest_index, + // The least step is 2, as two positions are looked at per turn. + step_size: least_step(&ms.c_params) + 1, + hasher: Hasher::new(hlog, ms.c_params.min_match), + first: block.first, + rep_offset1: block.rep.offset_1, + rep_offset2: block.rep.offset_2, + }; + let mut sink = Sink::new(seq_store, &block); + let hash_table = ms.hash_table.sub_mut(..1 << hlog)?; + // What the levels come with, for an input of any size. + match hlog { + 16 => finder.find_matches::<16>(hash_table, &mut sink), + 15 => finder.find_matches::<15>(hash_table, &mut sink), + 14 => finder.find_matches::<14>(hash_table, &mut sink), + 13 => finder.find_matches::<13>(hash_table, &mut sink), + 12 => finder.find_matches::<12>(hash_table, &mut sink), + 11 => finder.find_matches::<11>(hash_table, &mut sink), + _ => finder.find_matches::<0>(hash_table, &mut sink), + } + let last_literals = sink.last_literals()?; + restore_rep( + rep, + finder.rep_offset1, + finder.rep_offset2, + block.rep.offset_saved1, + block.rep.offset_saved2, + ); + Ok(last_literals) +} + +/// What `fast` and `dfast` work out before they look for matches in a block, when a dictionary has its own match state. +pub(super) struct DictBlock<'a> { + /// The current segment, and the content of the dictionary. + pub(super) window: Segments<'a>, + /// The index of the first position of the current segment. + pub(super) prefix_start_index: u32, + /// The index that the start of the content has in the dictionary's tables. + pub(super) dict_start_index: u32, + /// Where the block starts in the current segment. + pub(super) istart: u32, +} + +impl<'a> DictBlock<'a> { + pub(super) fn new(src: &BlockSrc<'a>, prefix_start_index: u32) -> Result { + let dms = src.dms.as_ref().ok_or(BUG)?; + let dict_start_index = dms.ms.window.dict_limit; + Ok(Self { + window: Segments { + near: Near::new(src.buf.sub(src.pos_of(prefix_start_index)..src.end)?)?, + older: Near::new(dms.buf.sub(dms.pos_of(dict_start_index)..dms.end)?)?, + }, + prefix_start_index, + dict_start_index, + istart: src.index_of(src.start).wrapping_sub(prefix_start_index), + }) + } + + /// The first position to look at: not `istart` if there is nothing before it to match. + pub(super) const fn first(&self) -> u32 { + self.istart + (self.window.place_of(self.istart) == 0) as u32 + } +} + +/// A table of a dictionary, where entries have tags. +#[derive(Clone, Copy)] +pub(super) struct TaggedTable<'a> { + table: &'a [u32], + /// How many bits a hash and a tag have. + h_bits: u32, + /// The index that the first place has in the table. + start_index: u32, +} + +impl<'a> TaggedTable<'a> { + pub(super) fn new(table: &'a [u32], h_bits: u32, start_index: u32) -> Result { + Ok(Self { + table: table.sub(..1 << (h_bits - ZSTD_SHORT_CACHE_TAG_BITS))?, + h_bits, + start_index, + }) + } + + /// The place that the table has for `bytes`. `None` if the tag tells that it is for something else, or if it is the + /// first place, which stands for none. + /// + /// `hasher` takes as many bytes as the table was made for. It is that of the table of the input, so that one + /// multiplication does for both. + #[inline(always)] + pub(super) fn get(self, hasher: Hasher, bytes: u64) -> Option { + let hash_and_tag = hasher.with_log(self.h_bits).hash(bytes); + let index_and_tag = hashed_get(self.table, hash_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS); + if !compare_packed_tags(index_and_tag as usize, hash_and_tag) { + return None; + } + // Costs nothing, and keeps the compiler from asking both questions before it goes on, as it mostly does here. + compiler_fence(Ordering::SeqCst); + let index = index_and_tag >> ZSTD_SHORT_CACHE_TAG_BITS; + (index > self.start_index).then(|| index - self.start_index) + } +} + +impl Finder { + /// `ZSTD_compressBlock_fast_dictMatchState_generic` + #[inline(never)] + fn find_matches_dict( + &mut self, + hash_table: &mut [u32], + sink: &mut Sink<'_, '_>, + window: Segments<'_>, + dict_table: TaggedTable<'_>, + ) { + const K_STEP_INCR: u32 = 1 << K_SEARCH_STRENGTH; + let Self { + base_index, + step_size, + hasher, + first: mut ip0, + mut rep_offset1, + mut rep_offset2, + } = *self; + let near = window.near; + if hash_table.is_empty() || dict_table.table.is_empty() { + return; + } + let Some(mut bytes0) = near.read64(ip0) else { + return; + }; + let mut ip1 = sink.anchor + step_size; + + 'block: loop { + let mut step = step_size; + let mut next_step = ip0 + K_STEP_INCR; + let mut curr; + + // One turn per position. Ends with a match, or with `None` once a repeat is stored. + let found = loop { + let Some(bytes1) = near.read64(ip1) else { + break 'block; + }; + curr = base_index.wrapping_add(ip0); + let match_index = hashed_swap(hash_table, hasher.hash(bytes0), curr); + + if let Some(end) = window.repeat_end(ip0 + 1, rep_offset1, (bytes0 >> 8) as u32) { + sink.store(ip0 + 1, end, REPCODE1_TO_OFFBASE); + ip0 = end; + break None; + } + + // A match in the dictionary is only taken if there is no candidate in the prefix. + if let Some(place) = dict_table.get(hasher, bytes0) + && window.older.read32(place) == Some(bytes0 as u32) + && match_index <= base_index + { + break Some(window.found_older(ip0, sink.anchor, place, 4)); + } + let match_pos = match_index.wrapping_sub(base_index); + if near.has::<0xFFFF_FFFF>(match_pos, bytes0) { + break Some(window.found_near(ip0, sink.anchor, match_pos, 4)); + } + + if ip1 >= next_step { + core::hint::cold_path(); + step += 1; + next_step += K_STEP_INCR; + } + ip0 = ip1; + ip1 += step; + bytes0 = bytes1; + }; + + if let Some(found) = found { + rep_offset2 = rep_offset1; + rep_offset1 = found.offset; + sink.store(ip0 - found.back, found.end, offset_to_offbase(rep_offset1)); + ip0 = found.end; + } + + // `ip0` is not past the last position that can be looked at. + let Some(around) = near.window::<{ 2 + HASH_READ_SIZE }>(ip0 - 2) else { + break 'block; + }; + bytes0 = le64_at::<2, _>(around); + if let Some(to_insert) = near.read64((curr + 2).wrapping_sub(base_index)) { + hashed_put(hash_table, hasher.hash(to_insert), curr + 2); + } + hashed_put( + hash_table, + hasher.hash(le64_at::<0, _>(around)), + base_index.wrapping_add(ip0 - 2), + ); + + while let Some(end) = window.repeat_end(ip0, rep_offset2, bytes0 as u32) { + core::mem::swap(&mut rep_offset1, &mut rep_offset2); + hashed_put( + hash_table, + hasher.hash(bytes0), + base_index.wrapping_add(ip0), + ); + sink.store_repeat(end); + ip0 = end; + let Some(bytes) = near.read64(ip0) else { + break 'block; + }; + bytes0 = bytes; + } + ip1 = ip0 + step_size; + } + + self.rep_offset1 = rep_offset1; + self.rep_offset2 = rep_offset2; + } +} + +/// `ZSTD_compressBlock_fast_dictMatchState` +pub(crate) fn compress_block_fast_dict_match_state( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + let block = DictBlock::new(src, ms.window.dict_limit)?; + let dms = src.dms.as_ref().ok_or(BUG)?; + let dict_table = TaggedTable::new( + &dms.ms.hash_table, + dms.ms.tagged_hash_log(), + block.dict_start_index, + )?; + let hlog = ms.c_params.hash_log; + let [rep_offset1, rep_offset2, _] = *rep; + let mut finder = Finder { + base_index: block.prefix_start_index, + step_size: least_step(&ms.c_params), + hasher: Hasher::new(hlog, ms.c_params.min_match), + first: block.first(), + rep_offset1, + rep_offset2, + }; + let mut sink = Sink::with_near(seq_store, block.window.near, block.istart); + let hash_table = ms.hash_table.sub_mut(..1 << hlog)?; + finder.find_matches_dict(hash_table, &mut sink, block.window, dict_table); + let last_literals = sink.last_literals()?; + let [rep0, rep1, _] = rep; + *rep0 = finder.rep_offset1; + *rep1 = finder.rep_offset2; + Ok(last_literals) +} + +/// `ZSTD_compressBlock_fast_extDict` +pub(crate) fn compress_block_fast_ext_dict( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + let Some(block) = ExtDictBlock::new(ms, rep, src)? else { + // Nothing of the older segment is within reach. + return compress_block_fast(ms, seq_store, rep, src); + }; + let hlog = ms.c_params.hash_log; + let mut finder = Finder { + base_index: block.prefix_start_index, + step_size: least_step(&ms.c_params) + 1, + hasher: Hasher::new(hlog, ms.c_params.min_match), + first: block.istart, + rep_offset1: block.rep.offset_1, + rep_offset2: block.rep.offset_2, + }; + let mut sink = Sink::with_near(seq_store, block.window.near, block.istart); + let hash_table = ms.hash_table.sub_mut(..1 << hlog)?; + // What the levels come with, for an input that is large enough to get here. + match hlog { + 16 => finder.find_matches_ext_dict::<16>(hash_table, &mut sink, block.window), + 14 => finder.find_matches_ext_dict::<14>(hash_table, &mut sink, block.window), + 13 => finder.find_matches_ext_dict::<13>(hash_table, &mut sink, block.window), + _ => finder.find_matches_ext_dict::<0>(hash_table, &mut sink, block.window), + } + let last_literals = sink.last_literals()?; + restore_rep( + rep, + finder.rep_offset1, + finder.rep_offset2, + block.rep.offset_saved1, + block.rep.offset_saved2, + ); + Ok(last_literals) +} diff --git a/src/zstd/compress/fse.rs b/src/zstd/compress/fse.rs new file mode 100644 index 000000000000..6d536fdf0c6a --- /dev/null +++ b/src/zstd/compress/fse.rs @@ -0,0 +1,734 @@ +//! `fse_compress.c` and the encoding half of `fse.h`: Finite State Entropy coding. + +use crate::common::bitstream::BitCStream; +use crate::common::{FSE_MAX_TABLELOG, FSE_MIN_TABLELOG, MAX_SEQ, ML_FSE_LOG, fse_table_step}; +use crate::error::{Error, Result}; +use crate::util::{BUG, SliceExt, highbit32, write_le16}; + +pub(crate) const FSE_NCOUNTBOUND: usize = 512; +const FSE_DEFAULT_TABLELOG: u32 = 11; + +/// `FSE_BLOCKBOUND` +const fn block_bound(size: usize) -> usize { + size + (size >> 7) + 4 + size_of::() +} + +/// `FSE_repeat` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum FseRepeat { + /// The table cannot be used again. + None, + /// It can, if it has a code for every symbol to encode. + Check, + /// It can. + Valid, +} + +/// `FSE_symbolCompressionTransform` +#[derive(Clone, Copy, Default)] +pub(crate) struct FseSymbolTransform { + pub(crate) delta_find_state: i32, + pub(crate) delta_nb_bits: u32, +} + +/// `FSE_CTable`, with room for the largest table zstd makes, which is one for match lengths. +#[derive(Clone)] +pub(crate) struct FseCTable { + pub(crate) table_log: u32, + pub(crate) max_symbol_value: u32, + pub(crate) state_table: [u16; 1 << ML_FSE_LOG], + /// More than `MAX_SEQ + 1`, so that a masked symbol is known to be in bounds. + pub(crate) symbol_tt: [FseSymbolTransform; SYMBOL_TT_SIZE], +} + +const SYMBOL_TT_SIZE: usize = (MAX_SEQ as usize + 1).next_power_of_two(); + +/// A table for a distribution that is known in advance, of a table log of at most 6. +pub(crate) struct PredefinedCTable { + table_log: u32, + max_symbol_value: u32, + state_table: [u16; 64], + symbol_tt: [FseSymbolTransform; SYMBOL_TT_SIZE], +} + +impl PredefinedCTable { + pub(crate) const fn new( + table_log: u32, + states: [u16; STATES], + transforms: [FseSymbolTransform; SYMBOLS], + ) -> Self { + let mut table = Self { + table_log, + max_symbol_value: SYMBOLS as u32 - 1, + state_table: [0; 64], + symbol_tt: FseCTable::new().symbol_tt, + }; + table + .state_table + .split_at_mut(STATES) + .0 + .copy_from_slice(&states); + table + .symbol_tt + .split_at_mut(SYMBOLS) + .0 + .copy_from_slice(&transforms); + table + } +} + +impl FseCTable { + pub(crate) const fn new() -> Self { + Self { + table_log: 0, + max_symbol_value: 0, + state_table: [0; 1 << ML_FSE_LOG], + symbol_tt: [FseSymbolTransform { + delta_find_state: 0, + delta_nb_bits: 0, + }; SYMBOL_TT_SIZE], + } + } + + /// `FSE_buildCTable_wksp`: `normalized_counter` adds up to `1 << table_log`, a count of -1 standing for 1. + /// [`Error::TableLogTooLarge`] if the table has no room for it. + pub(crate) fn build( + &mut self, + normalized_counter: &[i16], + max_symbol_value: u32, + table_log: u32, + ) -> Result<()> { + if table_log > ML_FSE_LOG || max_symbol_value > MAX_SEQ { + return Err(Error::TableLogTooLarge); + } + let table_size = 1usize << table_log; + let table_mask = table_size - 1; + let step = fse_table_step(table_size); + let counts = normalized_counter.sub(..=max_symbol_value as usize)?; + + const STATE_TABLE_SIZE: usize = 1 << ML_FSE_LOG; + const CUMUL_SIZE: usize = (MAX_SEQ as usize + 1).next_power_of_two(); + let mut cumul = [0u16; CUMUL_SIZE]; + let mut table_symbol = [0u8; STATE_TABLE_SIZE]; + let mut high_threshold = table_size - 1; + + self.table_log = table_log; + self.max_symbol_value = max_symbol_value; + + // Where the states of each symbol start. The symbols of low probability get the highest positions. + let mut total = 0u16; + for (symbol, (&count, start)) in counts.iter().zip(&mut cumul).enumerate() { + *start = total; + if count == -1 { + total = total.wrapping_add(1); + table_symbol.put(high_threshold, symbol as u8)?; + high_threshold = high_threshold.wrapping_sub(1); + } else { + total = total.wrapping_add(count as u16); + } + } + + // Spread the symbols. + if high_threshold == table_size - 1 { + // No symbol of low probability: lay the symbols down in order, and then every position is used. + let mut spread = [0u8; (1 << ML_FSE_LOG) + 8]; + let mut pos = 0usize; + let mut sv = 0u64; + for &count in counts { + let n = usize::try_from(count).map_err(|_| BUG)?; + *spread.array_at_mut(pos)? = sv.to_le_bytes(); + for i in (8..n).step_by(8) { + *spread.array_at_mut(pos + i)? = sv.to_le_bytes(); + } + pos += n; + sv = sv.wrapping_add(0x0101_0101_0101_0101); + } + + let mut position = 0usize; + let (pairs, _) = spread.sub(..table_size)?.as_chunks::<2>(); + for &[first, second] in pairs { + table_symbol.put(position, first)?; + table_symbol.put((position + step) & table_mask, second)?; + position = (position + 2 * step) & table_mask; + } + } else { + let mut position = 0usize; + for (symbol, &freq) in counts.iter().enumerate() { + for _ in 0..freq { + table_symbol.put(position, symbol as u8)?; + position = (position + step) & table_mask; + while position > high_threshold { + position = (position + step) & table_mask; + } + } + } + } + + // The states, sorted by symbol. + for (u, &symbol) in table_symbol.sub(..table_size)?.iter().enumerate() { + let Some(next) = cumul.get_mut(usize::from(symbol) & (CUMUL_SIZE - 1)) else { + continue; + }; + let state = usize::from(*next) & (STATE_TABLE_SIZE - 1); + *next = next.wrapping_add(1); + if let Some(state) = self.state_table.get_mut(state) { + *state = (table_size + u) as u16; + } + } + + let mut total = 0u32; + for (&count, transform) in counts.iter().zip(&mut self.symbol_tt) { + match count { + // For `get_max_nb_bits` and `bit_cost`. + 0 => transform.delta_nb_bits = ((table_log + 1) << 16) - (1 << table_log), + -1 | 1 => { + transform.delta_nb_bits = (table_log << 16).wrapping_sub(1 << table_log); + transform.delta_find_state = total.wrapping_sub(1) as i32; + total += 1; + } + _ => { + let count = count as u32; + let max_bits_out = table_log.wrapping_sub(highbit32(count.wrapping_sub(1))); + let min_state_plus = count.wrapping_shl(max_bits_out); + transform.delta_nb_bits = (max_bits_out << 16).wrapping_sub(min_state_plus); + transform.delta_find_state = total.wrapping_sub(count) as i32; + total = total.wrapping_add(count); + } + } + } + Ok(()) + } + + /// What [`Self::build`] does for the distribution that `table` was made for. + pub(crate) fn set_predefined(&mut self, table: &PredefinedCTable) { + self.table_log = table.table_log; + self.max_symbol_value = table.max_symbol_value; + if let Some(states) = self.state_table.first_chunk_mut() { + *states = table.state_table; + } + self.symbol_tt = table.symbol_tt; + } + + /// `FSE_buildCTable_rle`: a table that encodes `symbol_value`, and nothing else, in 0 bits. + pub(crate) fn build_rle(&mut self, symbol_value: u8) -> Result<()> { + self.table_log = 0; + self.max_symbol_value = u32::from(symbol_value); + self.state_table.sub_mut(..2)?.fill(0); + self.symbol_tt + .put(usize::from(symbol_value), FseSymbolTransform::default()) + } + + /// `FSE_initCState2`: the state that it costs the fewest bits to start from, given that the first symbol is `symbol`, + /// which the table has a code for. + #[inline] + pub(crate) fn first_state(&self, symbol: u8) -> u32 { + FseCState::new2(self, u32::from(symbol)).value as u32 + } + + /// `FSE_getMaxNbBits`: the most bits it takes to encode `symbol_value`, rounded up. One more than the table log + /// if the table has no code for it. + #[inline(always)] + pub(crate) fn get_max_nb_bits(&self, symbol_value: u32) -> u32 { + let transform = self.symbol_tt.at(symbol_value as usize).unwrap_or_default(); + transform.delta_nb_bits.wrapping_add((1 << 16) - 1) >> 16 + } + + /// `FSE_bitCost`: about how many bits it takes to encode `symbol_value`, in units of `1 / (1 << accuracy_log)`. + /// The table has a code for it. + #[inline(always)] + pub(crate) fn bit_cost(&self, symbol_value: u32, accuracy_log: u32) -> u32 { + let transform = self.symbol_tt.at(symbol_value as usize).unwrap_or_default(); + let min_nb_bits = transform.delta_nb_bits >> 16; + let threshold = (min_nb_bits + 1) << 16; + let table_size = 1u32 << self.table_log; + let delta_from_threshold = + threshold.wrapping_sub(transform.delta_nb_bits.wrapping_add(table_size)); + // A linear interpolation, which is very approximate. + let normalized_delta_from_threshold = + (delta_from_threshold << accuracy_log) >> self.table_log; + let bit_multiplier = 1u32 << accuracy_log; + ((min_nb_bits + 1) * bit_multiplier).wrapping_sub(normalized_delta_from_threshold) + } +} + +/// `FSE_CState_t` +pub(crate) struct FseCState<'a> { + value: usize, + table: &'a FseCTable, +} + +impl<'a> FseCState<'a> { + /// `FSE_initCState` + #[inline] + pub(crate) fn new(table: &'a FseCTable) -> Self { + Self { + value: 1 << (table.table_log & 15), + table, + } + } + + /// `FSE_initCState2`: the first symbol costs the fewest bits this way. The table has a code for `symbol`. + #[inline] + pub(crate) fn new2(table: &'a FseCTable, symbol: u32) -> Self { + let mut state = Self::new(table); + let transform = table.symbol_tt.at(symbol as usize).unwrap_or_default(); + let nb_bits_out = transform.delta_nb_bits.wrapping_add(1 << 15) >> 16; + state.value = (nb_bits_out << 16).wrapping_sub(transform.delta_nb_bits) as usize; + state.step(nb_bits_out, transform.delta_find_state); + state + } + + #[inline(always)] + fn step(&mut self, nb_bits_out: u32, delta_find_state: i32) { + let state_table = &self.table.state_table; + let index = (self.value >> (nb_bits_out & (usize::BITS - 1))) + .wrapping_add_signed(delta_find_state as isize); + let next = state_table.get(index & (state_table.len() - 1)); + self.value = usize::from(next.copied().unwrap_or(0)); + } + + /// `FSE_encodeSymbol`: the table has a code for `symbol`. + #[inline(always)] + pub(crate) fn encode_symbol(&mut self, bit_c: &mut BitCStream<'_>, symbol: u32) { + let symbol_tt = &self.table.symbol_tt; + let transform = symbol_tt + .at(symbol as usize & (SYMBOL_TT_SIZE - 1)) + .unwrap_or_default(); + let nb_bits_out = ((self.value + transform.delta_nb_bits as usize) >> 16) as u32; + bit_c.add_bits(self.value as u64, nb_bits_out); + self.step(nb_bits_out, transform.delta_find_state); + } + + /// `FSE_flushCState` + #[inline(always)] + pub(crate) fn flush(&self, bit_c: &mut BitCStream<'_>) { + bit_c.add_bits(self.value as u64, self.table.table_log); + bit_c.flush_bits(); + } +} + +/// `FSE_NCountWriteBound` +pub(crate) fn ncount_write_bound(max_symbol_value: u32, table_log: u32) -> usize { + let max_header_size = + ((max_symbol_value as usize + 1) * table_log as usize + 4 + 2) / 8 + 1 + 2; + if max_symbol_value != 0 { + max_header_size + } else { + FSE_NCOUNTBOUND + } +} + +/// `FSE_writeNCount_generic`: `table_log` is from 5 to 12. +fn write_ncount_generic( + header: &mut [u8], + normalized_counter: &[i16], + max_symbol_value: u32, + table_log: u32, + write_is_safe: bool, +) -> Result { + let counts = normalized_counter.sub(..=max_symbol_value as usize)?; + let alphabet_size = counts.len(); + let mut out = 0usize; + let table_size = 1i32 << table_log; + let mut bit_stream = table_log - FSE_MIN_TABLELOG; + let mut bit_count = 4i32; + let mut symbol = 0usize; + let mut previous_is_0 = false; + + // The lowest 16 bits of the stream leave it. + let mut write16 = |bit_stream: &mut u32, out: &mut usize| -> Result<()> { + if !write_is_safe && *out + 2 > header.len() { + return Err(Error::DstSizeTooSmall); + } + write_le16(header, *out, *bit_stream as u16)?; + *out += 2; + *bit_stream >>= 16; + Ok(()) + }; + let shifted = |value: u32, bit_count: i32| value.wrapping_shl(bit_count as u32); + + let mut remaining = table_size + 1; + let mut threshold = table_size; + let mut nb_bits = table_log as i32 + 1; + + while symbol < alphabet_size && remaining > 1 { + if previous_is_0 { + // After a count of 0 comes how many more follow it. + let mut start = symbol; + while symbol < alphabet_size && counts.at(symbol)? == 0 { + symbol += 1; + } + if symbol == alphabet_size { + // The counts do not add up. + break; + } + while symbol >= start + 24 { + start += 24; + bit_stream = bit_stream.wrapping_add(shifted(0xFFFF, bit_count)); + write16(&mut bit_stream, &mut out)?; + } + while symbol >= start + 3 { + start += 3; + bit_stream = bit_stream.wrapping_add(shifted(3, bit_count)); + bit_count += 2; + } + bit_stream = bit_stream.wrapping_add(shifted((symbol - start) as u32, bit_count)); + bit_count += 2; + if bit_count > 16 { + write16(&mut bit_stream, &mut out)?; + bit_count -= 16; + } + } + { + let mut count = i32::from(counts.at(symbol)?); + symbol += 1; + let max = (2 * threshold - 1) - remaining; + remaining -= count.abs(); + // The stored value is the count plus one, since a count can be -1. + count += 1; + if count >= threshold { + count += max; + } + bit_stream = bit_stream.wrapping_add(shifted(count as u32, bit_count)); + bit_count += nb_bits; + bit_count -= i32::from(count < max); + previous_is_0 = count == 1; + if remaining < 1 { + return Err(Error::Generic); + } + while remaining < threshold { + nb_bits -= 1; + threshold >>= 1; + } + } + if bit_count > 16 { + write16(&mut bit_stream, &mut out)?; + bit_count -= 16; + } + } + + if remaining != 1 { + return Err(Error::Generic); + } + + let last = out; + write16(&mut bit_stream, &mut out)?; + Ok(last + ((bit_count + 7) / 8) as usize) +} + +/// `FSE_writeNCount`: writes the header that [`crate::common::entropy::read_ncount`] reads, and returns its size. +pub(crate) fn write_ncount( + buffer: &mut [u8], + normalized_counter: &[i16], + max_symbol_value: u32, + table_log: u32, +) -> Result { + if table_log > FSE_MAX_TABLELOG { + return Err(Error::TableLogTooLarge); + } + if table_log < FSE_MIN_TABLELOG { + return Err(Error::Generic); + } + let write_is_safe = buffer.len() >= ncount_write_bound(max_symbol_value, table_log); + write_ncount_generic( + buffer, + normalized_counter, + max_symbol_value, + table_log, + write_is_safe, + ) +} + +/// `FSE_minTableLog`: the smallest table log that can represent the distribution. `src_size` is more than 1. +fn min_table_log(src_size: usize, max_symbol_value: u32) -> u32 { + let min_bits_src = highbit32(src_size as u32) + 1; + let min_bits_symbols = highbit32(max_symbol_value) + 2; + min_bits_src.min(min_bits_symbols) +} + +/// `FSE_optimalTableLog_internal`: `src_size` is more than 1. +pub(crate) fn optimal_table_log_internal( + max_table_log: u32, + src_size: usize, + max_symbol_value: u32, + minus: u32, +) -> u32 { + let max_bits_src = highbit32(src_size.wrapping_sub(1) as u32).wrapping_sub(minus); + let mut table_log = max_table_log; + let min_bits = min_table_log(src_size, max_symbol_value); + if table_log == 0 { + table_log = FSE_DEFAULT_TABLELOG; + } + // More accuracy than the input has symbols for is of no use. + if max_bits_src < table_log { + table_log = max_bits_src; + } + if min_bits > table_log { + table_log = min_bits; + } + table_log.clamp(FSE_MIN_TABLELOG, FSE_MAX_TABLELOG) +} + +/// `FSE_optimalTableLog` +pub(crate) fn optimal_table_log(max_table_log: u32, src_size: usize, max_symbol_value: u32) -> u32 { + optimal_table_log_internal(max_table_log, src_size, max_symbol_value, 2) +} + +/// `FSE_normalizeM2`: the second normalization method, for when the first one would take too much from the most +/// frequent symbol. `norm` and `count` are of the same length. +fn normalize_m2( + norm: &mut [i16], + table_log: u32, + count: &[u32], + mut total: usize, + low_prob_count: i16, +) -> Result<()> { + const NOT_YET_ASSIGNED: i16 = -2; + let table_size = 1u32 << table_log; + let mut distributed = 0u32; + + // The symbols of low probability first. + let low_threshold = (total >> table_log) as u32; + let mut low_one = (total.wrapping_mul(3) >> (table_log + 1)) as u32; + + for (norm, &count) in norm.iter_mut().zip(count) { + if count == 0 { + *norm = 0; + } else if count <= low_threshold { + *norm = low_prob_count; + distributed += 1; + total = total.wrapping_sub(count as usize); + } else if count <= low_one { + *norm = 1; + distributed += 1; + total = total.wrapping_sub(count as usize); + } else { + *norm = NOT_YET_ASSIGNED; + } + } + let mut to_distribute = table_size.wrapping_sub(distributed); + + if to_distribute == 0 { + return Ok(()); + } + + if total / to_distribute as usize > low_one as usize { + // The rest would get too much each: more symbols get 1. + low_one = total + .wrapping_mul(3) + .checked_div(to_distribute.wrapping_mul(2) as usize) + .ok_or(BUG)? as u32; + for (norm, &count) in norm.iter_mut().zip(count) { + if *norm == NOT_YET_ASSIGNED && count <= low_one { + *norm = 1; + distributed += 1; + total = total.wrapping_sub(count as usize); + } + } + to_distribute = table_size.wrapping_sub(distributed); + } + + if distributed as usize == count.len() { + // All the values are small. The first symbol of the largest count gets what is left. + let mut max_v = 0usize; + let mut max_c = 0u32; + for (s, &count) in count.iter().enumerate() { + if count > max_c { + max_v = s; + max_c = count; + } + } + let norm = norm.at_mut(max_v)?; + *norm = norm.wrapping_add(to_distribute as i16); + return Ok(()); + } + + if total == 0 { + // All the symbols were of low enough probability to get 1 or less: what is left goes round. + if !norm.iter().any(|&norm| norm > 0) { + return Err(BUG); + } + while to_distribute > 0 { + for norm in norm.iter_mut().filter(|norm| **norm > 0) { + if to_distribute == 0 { + break; + } + to_distribute -= 1; + *norm = norm.wrapping_add(1); + } + } + return Ok(()); + } + + let v_step_log = 62 - table_log; + let mid = (1u64 << (v_step_log - 1)) - 1; + // A scale on what remains. + let r_step = ((1u64 << v_step_log) * u64::from(to_distribute) + mid) + .checked_div(u64::from(total as u32)) + .ok_or(BUG)?; + let mut tmp_total = mid; + for (norm, &count) in norm.iter_mut().zip(count) { + if *norm == NOT_YET_ASSIGNED { + let end = tmp_total.wrapping_add(u64::from(count).wrapping_mul(r_step)); + let s_start = (tmp_total >> v_step_log) as u32; + let s_end = (end >> v_step_log) as u32; + let weight = s_end.wrapping_sub(s_start); + if weight < 1 { + return Err(Error::Generic); + } + *norm = weight as i16; + tmp_total = end; + } + } + Ok(()) +} + +/// `FSE_normalizeCount`: scales `count`, which adds up to `total`, to add up to `1 << table_log`. Returns the table +/// log, or 0 if there is only one symbol, which is a case for run-length encoding. +pub(crate) fn normalize_count( + normalized_counter: &mut [i16], + mut table_log: u32, + count: &[u32], + total: usize, + max_symbol_value: u32, + use_low_prob_count: bool, +) -> Result { + if table_log == 0 { + table_log = FSE_DEFAULT_TABLELOG; + } + if table_log < FSE_MIN_TABLELOG { + return Err(Error::Generic); + } + if table_log > FSE_MAX_TABLELOG { + return Err(Error::TableLogTooLarge); + } + if table_log < min_table_log(total, max_symbol_value) { + return Err(Error::Generic); + } + + const RTB_TABLE: [u32; 8] = [0, 473195, 504333, 520860, 550000, 700000, 750000, 830000]; + let low_prob_count: i16 = if use_low_prob_count { -1 } else { 1 }; + let scale = 62 - table_log; + let step = (1u64 << 62) + .checked_div(u64::from(total as u32)) + .ok_or(BUG)?; + let v_step = 1u64 << (scale - 20); + let mut still_to_distribute = 1i32 << table_log; + let mut largest = 0usize; + let mut largest_p = 0i16; + let low_threshold = (total >> table_log) as u32; + + let count = count.sub(..=max_symbol_value as usize)?; + let norm = normalized_counter.sub_mut(..=max_symbol_value as usize)?; + + for (s, (norm, &count)) in norm.iter_mut().zip(count).enumerate() { + if count as usize == total { + return Ok(0); + } + if count == 0 { + *norm = 0; + } else if count <= low_threshold { + *norm = low_prob_count; + still_to_distribute -= 1; + } else { + let scaled = u64::from(count).wrapping_mul(step); + let mut proba = (scaled >> scale) as i16; + if proba < 8 { + let rest_to_beat = v_step * u64::from(RTB_TABLE.at(proba as usize)?); + proba += i16::from(scaled.wrapping_sub((proba as u64) << scale) > rest_to_beat); + } + if proba > largest_p { + largest_p = proba; + largest = s; + } + *norm = proba; + still_to_distribute -= i32::from(proba); + } + } + let largest = norm.at_mut(largest)?; + if -still_to_distribute >= i32::from(*largest >> 1) { + // The correction is too large for one symbol to take. + normalize_m2(norm, table_log, count, total, low_prob_count)?; + } else { + *largest = largest.wrapping_add(still_to_distribute as i16); + } + Ok(table_log as usize) +} + +/// `FSE_compress_usingCTable_generic` +#[inline(always)] +fn compress_using_ctable_generic( + dst: &mut [u8], + src: &[u8], + ct: &FseCTable, +) -> usize { + let flush = |bit_c: &mut BitCStream<'_>| { + if FAST { + bit_c.flush_bits_fast(); + } else { + bit_c.flush_bits(); + } + }; + + if src.len() <= 2 { + return 0; + } + let Ok(mut bit_c) = BitCStream::new(dst) else { + return 0; + }; + let encode = |state: &mut FseCState<'_>, bit_c: &mut BitCStream<'_>, symbol: u8| { + state.encode_symbol(bit_c, u32::from(symbol)); + }; + + // From the last symbol to the first, two states taking turns. + let mut state1; + let mut state2; + let mut rest; + if src.len() & 1 != 0 { + let [head @ .., c, b, a] = src else { + return 0; + }; + state1 = FseCState::new2(ct, u32::from(*a)); + state2 = FseCState::new2(ct, u32::from(*b)); + encode(&mut state1, &mut bit_c, *c); + flush(&mut bit_c); + rest = head; + } else { + let [head @ .., b, a] = src else { + return 0; + }; + state2 = FseCState::new2(ct, u32::from(*a)); + state1 = FseCState::new2(ct, u32::from(*b)); + rest = head; + } + + if let ([head @ .., b, a], true) = (rest, rest.len() & 2 != 0) { + encode(&mut state2, &mut bit_c, *a); + encode(&mut state1, &mut bit_c, *b); + flush(&mut bit_c); + rest = head; + } + + let (_, quads) = rest.as_rchunks::<4>(); + for &[d, c, b, a] in quads.iter().rev() { + encode(&mut state2, &mut bit_c, a); + encode(&mut state1, &mut bit_c, b); + encode(&mut state2, &mut bit_c, c); + encode(&mut state1, &mut bit_c, d); + flush(&mut bit_c); + } + + state2.flush(&mut bit_c); + state1.flush(&mut bit_c); + bit_c.close() +} + +/// `FSE_compress_usingCTable`: the table has a code for every symbol in `src`. Returns the size of the output, or 0 +/// if `src` has less than 3 symbols or `dst` is too small. +pub(crate) fn compress_using_ctable(dst: &mut [u8], src: &[u8], ct: &FseCTable) -> usize { + if dst.len() >= block_bound(src.len()) { + compress_using_ctable_generic::(dst, src, ct) + } else { + compress_using_ctable_generic::(dst, src, ct) + } +} diff --git a/src/zstd/compress/hist.rs b/src/zstd/compress/hist.rs new file mode 100644 index 000000000000..9ea10424765c --- /dev/null +++ b/src/zstd/compress/hist.rs @@ -0,0 +1,132 @@ +//! `hist.c`: how many times each byte value occurs. + +use crate::error::{Error, Result}; +use crate::util::{BUG, SliceExt}; + +/// `HIST_add`: `count` has room for every value in `src`. +pub(crate) fn add(count: &mut [u32], src: &[u8]) -> Result<()> { + for &byte in src { + *count.at_mut(usize::from(byte))? += 1; + } + Ok(()) +} + +/// `HIST_count_simple`: `count` has room for the values up to `max_symbol_value`, and `src` has none above it. +/// Lowers `max_symbol_value` to the largest value present, and returns the largest count. +pub(crate) fn count_simple( + count: &mut [u32], + max_symbol_value: &mut u32, + src: &[u8], +) -> Result { + let mut max_symbol = *max_symbol_value as usize; + count.sub_mut(..=max_symbol)?.fill(0); + if src.is_empty() { + *max_symbol_value = 0; + return Ok(0); + } + add(count, src)?; + while count.at(max_symbol)? == 0 { + max_symbol = max_symbol.checked_sub(1).ok_or(BUG)?; + } + *max_symbol_value = max_symbol as u32; + Ok(count.sub(..=max_symbol)?.iter().copied().max().unwrap_or(0)) +} + +#[inline(always)] +fn increment(counting: &mut [u32; 256], byte: u8) { + if let Some(count) = counting.get_mut(usize::from(byte)) { + *count = count.wrapping_add(1); + } +} + +/// `HIST_count_parallel_wksp`: four tables, so that a run of one value does not wait for its own counter. +/// `max_symbol_value` is at most 255. With `check`, a value above it is an error rather than trusted not to occur. +fn count_parallel( + count: &mut [u32], + max_symbol_value: &mut u32, + src: &[u8], + check: bool, +) -> Result { + let count_size = *max_symbol_value as usize + 1; + if src.is_empty() { + count.sub_mut(..count_size)?.fill(0); + *max_symbol_value = 0; + return Ok(0); + } + + let mut counting1 = [0u32; 256]; + let mut counting2 = [0u32; 256]; + let mut counting3 = [0u32; 256]; + let mut counting4 = [0u32; 256]; + + // Each word is read before the bytes of the one before it are counted. + let (words, rest) = src.as_chunks::<8>(); + if let Some((&first, words)) = words.split_first() { + let mut cached = u64::from_le_bytes(first); + let mut count_word = |next: [u8; 8]| { + let c = cached; + cached = u64::from_le_bytes(next); + increment(&mut counting1, c as u8); + increment(&mut counting2, (c >> 8) as u8); + increment(&mut counting3, (c >> 16) as u8); + increment(&mut counting4, (c >> 24) as u8); + increment(&mut counting1, (c >> 32) as u8); + increment(&mut counting2, (c >> 40) as u8); + increment(&mut counting3, (c >> 48) as u8); + increment(&mut counting4, (c >> 56) as u8); + }; + let (stripes, words) = words.as_chunks::<2>(); + for &[a, b] in stripes { + count_word(a); + count_word(b); + } + for &word in words { + count_word(word); + } + count_word([0; 8]); + } + for &byte in rest { + increment(&mut counting1, byte); + } + + let mut max = 0u32; + let others = counting2.iter().zip(&counting3).zip(&counting4); + for (sum, ((&c2, &c3), &c4)) in counting1.iter_mut().zip(others) { + *sum = sum.wrapping_add(c2).wrapping_add(c3).wrapping_add(c4); + max = max.max(*sum); + } + + let max_symbol = counting1.iter().rposition(|&c| c != 0).ok_or(BUG)? as u32; + if check && max_symbol > *max_symbol_value { + return Err(Error::MaxSymbolValueTooSmall); + } + *max_symbol_value = max_symbol; + count.write_at(0, counting1.sub(..count_size)?)?; + Ok(max as usize) +} + +/// `HIST_countFast_wksp`: as [`count_simple`], faster on large inputs. +pub(crate) fn count_fast_wksp( + count: &mut [u32], + max_symbol_value: &mut u32, + src: &[u8], +) -> Result { + if src.len() < 1500 { + return Ok(count_simple(count, max_symbol_value, src)? as usize); + } + count_parallel(count, max_symbol_value, src, false) +} + +/// `HIST_count_wksp`: as [`count_fast_wksp`], but a value above `max_symbol_value` in `src` is +/// [`Error::MaxSymbolValueTooSmall`]. +pub(crate) fn count_wksp( + count: &mut [u32], + max_symbol_value: &mut u32, + src: &[u8], +) -> Result { + if *max_symbol_value < 255 { + return count_parallel(count, max_symbol_value, src, true); + } + *max_symbol_value = 255; + count_fast_wksp(count, max_symbol_value, src) +} diff --git a/src/zstd/compress/huf.rs b/src/zstd/compress/huf.rs new file mode 100644 index 000000000000..5f77bbe8c076 --- /dev/null +++ b/src/zstd/compress/huf.rs @@ -0,0 +1,1183 @@ +//! `huf_compress.c`: Huffman coding of literals. + +use super::fse::{self, FseCTable}; +use super::hist; +use super::internal::Strategy; +use crate::common::entropy::huf_read_stats; +use crate::common::{HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_MAX, LIT_HUF_LOG}; +use crate::error::{Error, Result}; +use crate::util::{BUG, SliceExt, highbit32, write_le16}; + +pub(crate) const HUF_BLOCKSIZE_MAX: usize = 128 * 1024; +pub(crate) const HUF_TABLELOG_DEFAULT: u32 = 11; +/// The strategies from this one on look for the best depth of the tree. +pub(crate) const HUF_OPTIMAL_DEPTH_THRESHOLD: Strategy = Strategy::BtUltra; + +/// `HUF_flags_e` +#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] +pub(crate) struct HufFlags { + /// Try every depth of the tree and keep the one that gives the smallest output, instead of a guess. + pub(crate) optimal_depth: bool, + /// Use the previous table again whenever it is possible, without looking for a better one. + pub(crate) prefer_repeat: bool, + /// Sample the input first, and give up at once if it looks like it cannot be compressed. + pub(crate) suspect_uncompressible: bool, +} + +/// `HUF_repeat` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum HufRepeat { + /// The table cannot be used again. + None, + /// It can, if it has a code for every symbol to encode. + Check, + /// It can. + Valid, +} + +/// `HUF_CElt[HUF_CTABLE_SIZE_ST(255)]`. The first element is the header: the table log in its lowest byte, and the +/// largest symbol value in the next. Element `symbol + 1` has the number of bits in its lowest byte, and the code in +/// its highest bits. +#[derive(Clone)] +pub(crate) struct HufCTable(pub(crate) [u64; HUF_SYMBOLVALUE_MAX as usize + 2]); + +impl HufCTable { + pub(crate) const fn new() -> Self { + Self([0; HUF_SYMBOLVALUE_MAX as usize + 2]) + } + + /// The elements, by symbol. + #[inline(always)] + fn elements(&self) -> &[u64; HUF_SYMBOLVALUE_MAX as usize + 1] { + let [_, elements @ ..] = &self.0; + elements + } + + #[inline(always)] + fn elements_mut(&mut self) -> &mut [u64; HUF_SYMBOLVALUE_MAX as usize + 1] { + let [_, elements @ ..] = &mut self.0; + elements + } + + /// `HUF_writeCTableHeader` + fn write_header(&mut self, table_log: u32, max_symbol_value: u32) { + let [header, ..] = &mut self.0; + *header = u64::from(table_log as u8) | (u64::from(max_symbol_value as u8) << 8); + } + + /// `HUF_getNbBitsFromCTable`: 0 if the table has no code for `symbol_value`. + #[inline] + pub(crate) fn get_nb_bits(&self, symbol_value: u32) -> u32 { + let [header, elements @ ..] = &self.0; + let max_symbol_value = (header >> 8) as u8; + if symbol_value > u32::from(max_symbol_value) { + return 0; + } + (elements.at(symbol_value as usize).unwrap_or(0) & 0xFF) as u32 + } +} + +/// `HUF_CTableHeader` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) struct HufCTableHeader { + pub(crate) table_log: u8, + pub(crate) max_symbol_value: u8, +} + +/// `HUF_readCTableHeader` +#[inline] +pub(crate) fn read_ctable_header(ctable: &HufCTable) -> HufCTableHeader { + let [header, ..] = ctable.0; + HufCTableHeader { + table_log: header as u8, + max_symbol_value: (header >> 8) as u8, + } +} + +/// `HUF_getNbBits` +#[inline(always)] +const fn get_nb_bits(elt: u64) -> u64 { + elt & 0xFF +} + +/// `HUF_setValue`: `elt` has its number of bits, and nothing else. +#[inline(always)] +fn set_value(elt: &mut u64, value: u64) { + let nb_bits = get_nb_bits(*elt) as u32; + if nb_bits > 0 { + *elt |= value.wrapping_shl(u64::BITS.wrapping_sub(nb_bits)); + } +} + +/// The first code of each number of bits up to `table_log`, given how many symbols there are of each: the longest codes +/// have the lowest values. +fn first_values( + nb_per_rank: &[u16; RANKS], + table_log: u32, +) -> Result<[u16; RANKS]> { + let mut val_per_rank = [0u16; RANKS]; + let mut min = 0u16; + for n in (1..=table_log as usize).rev() { + val_per_rank.put(n, min)?; + min = min.wrapping_add(nb_per_rank.at(n)?) >> 1; + } + Ok(val_per_rank) +} + +/// Gives `elt`, which has its number of bits, the next code of that many bits. +#[inline(always)] +fn set_next_value(elt: &mut u64, val_per_rank: &mut [u16]) -> Result<()> { + let value = val_per_rank.at_mut(get_nb_bits(*elt) as usize)?; + set_value(elt, u64::from(*value)); + *value = value.wrapping_add(1); + Ok(()) +} + +const MAX_FSE_TABLELOG_FOR_HUFF_HEADER: u32 = 6; + +/// `HUF_compressWeights`: returns 0 if the weights cannot be compressed, and 1 if they are all the same. +fn compress_weights(dst: &mut [u8], weight_table: &[u8]) -> Result { + let mut max_symbol_value = HUF_TABLELOG_MAX; + let mut count = [0u32; HUF_TABLELOG_MAX as usize + 1]; + let mut norm = [0i16; HUF_TABLELOG_MAX as usize + 1]; + let wt_size = weight_table.len(); + + if wt_size <= 1 { + return Ok(0); + } + + let max_count = hist::count_simple(&mut count, &mut max_symbol_value, weight_table)?; + if max_count as usize == wt_size { + return Ok(1); + } + if max_count == 1 { + return Ok(0); + } + + let table_log = + fse::optimal_table_log(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, wt_size, max_symbol_value); + fse::normalize_count( + &mut norm, + table_log, + &count, + wt_size, + max_symbol_value, + false, + )?; + + let h_size = fse::write_ncount(dst, &norm, max_symbol_value, table_log)?; + + let mut ctable = FseCTable::new(); + ctable.build(&norm, max_symbol_value, table_log)?; + let c_size = fse::compress_using_ctable(dst.sub_mut(h_size..)?, weight_table, &ctable); + if c_size == 0 { + return Ok(0); + } + Ok(h_size + c_size) +} + +/// `HUF_writeCTable_wksp`: writes the header that [`huf_read_stats`] reads, and returns its size. `max_symbol_value` +/// and `huff_log` are those of `ctable`. +pub(crate) fn write_ctable( + dst: &mut [u8], + ctable: &HufCTable, + max_symbol_value: u32, + huff_log: u32, +) -> Result { + let mut bits_to_weight = [0u8; HUF_TABLELOG_MAX as usize + 1]; + let mut huff_weight = [0u8; HUF_SYMBOLVALUE_MAX as usize]; + + if max_symbol_value > HUF_SYMBOLVALUE_MAX { + return Err(Error::MaxSymbolValueTooLarge); + } + // The weight of the last symbol is not stored. + let nb_weights = max_symbol_value as usize; + + for n in 1..=huff_log { + bits_to_weight.put(n as usize, (huff_log + 1 - n) as u8)?; + } + let weights = huff_weight.sub_mut(..nb_weights)?; + for (weight, &elt) in weights.iter_mut().zip(ctable.elements()) { + *weight = bits_to_weight.at(get_nb_bits(elt) as usize)?; + } + + let max_dst_size = dst.len(); + let Some((first, rest)) = dst.split_first_mut() else { + return Err(Error::DstSizeTooSmall); + }; + let h_size = compress_weights(rest, weights)?; + if h_size > 1 && h_size < nb_weights / 2 { + *first = h_size as u8; + return Ok(h_size + 1); + } + + // Not compressed: 4 bits to a weight. + if max_symbol_value > 256 - 128 { + return Err(Error::Generic); + } + let raw_size = nb_weights.div_ceil(2); + if raw_size + 1 > max_dst_size { + return Err(Error::DstSizeTooSmall); + } + *first = 128u32.wrapping_add(max_symbol_value.wrapping_sub(1)) as u8; + // An odd number of weights is completed with a 0, which `huff_weight` still has there. + let (pairs, _) = huff_weight.sub(..raw_size * 2)?.as_chunks::<2>(); + for (out, &[high, low]) in rest.iter_mut().zip(pairs) { + *out = (high << 4).wrapping_add(low); + } + Ok(raw_size + 1) +} + +/// What [`read_ctable`] found besides the table. +pub(crate) struct HufCTableStats { + /// How many bytes of the input the header takes. + pub(crate) size: usize, + pub(crate) max_symbol_value: u32, + /// Some symbol up to `max_symbol_value` has no code. + pub(crate) has_zero_weights: bool, +} + +/// `HUF_readCTable`: [`Error::MaxSymbolValueTooSmall`] if the table has symbols above `max_symbol_value`. +pub(crate) fn read_ctable( + ctable: &mut HufCTable, + max_symbol_value: u32, + src: &[u8], +) -> Result { + let mut huff_weight = [0u8; HUF_SYMBOLVALUE_MAX as usize + 1]; + let mut rank_val = [0u32; HUF_TABLELOG_MAX as usize + 1]; + + let stats = huf_read_stats(&mut huff_weight, &mut rank_val, src)?; + let table_log = stats.table_log; + let [zero_weights, ..] = rank_val; + + if table_log > HUF_TABLELOG_MAX { + return Err(Error::TableLogTooLarge); + } + if stats.nb_symbols > max_symbol_value.wrapping_add(1) { + return Err(Error::MaxSymbolValueTooSmall); + } + let max_symbol_value = stats.nb_symbols - 1; + ctable.write_header(table_log, max_symbol_value); + + let mut nb_per_rank = [0u16; HUF_TABLELOG_MAX as usize + 2]; + let ct = ctable.elements_mut().sub_mut(..stats.nb_symbols as usize)?; + for (elt, &weight) in ct.iter_mut().zip(&huff_weight) { + let nb_bits = if weight != 0 { + (table_log + 1).wrapping_sub(u32::from(weight)) as u8 + } else { + 0 + }; + *elt = u64::from(nb_bits); + *nb_per_rank.at_mut(usize::from(nb_bits))? += 1; + } + let mut val_per_rank = first_values(&nb_per_rank, table_log)?; + for elt in ct { + set_next_value(elt, &mut val_per_rank)?; + } + + Ok(HufCTableStats { + size: stats.size, + max_symbol_value, + has_zero_weights: zero_weights > 0, + }) +} + +/// `nodeElt` +#[derive(Clone, Copy, Default)] +struct NodeElt { + count: u32, + parent: u16, + byte: u8, + nb_bits: u8, +} + +/// The leaves of `huffNodeTable`, sorted from the most frequent symbol to the rarest. What there is at -1, or anywhere +/// else out of the table, is all zeros. +struct HuffNodes([NodeElt; HUF_SYMBOLVALUE_MAX as usize + 1]); + +impl HuffNodes { + #[inline(always)] + fn get(&self, n: i32) -> NodeElt { + self.0.get(n as usize).copied().unwrap_or_default() + } + + #[inline(always)] + fn set(&mut self, n: i32, node: NodeElt) { + if let Some(slot) = self.0.get_mut(n as usize) { + *slot = node; + } + } + + #[inline(always)] + fn count(&self, n: i32) -> u32 { + self.get(n).count + } + + #[inline(always)] + fn nb_bits(&self, n: i32) -> u32 { + u32::from(self.get(n).nb_bits) + } + + #[inline(always)] + fn set_nb_bits(&mut self, n: i32, nb_bits: u32) { + if let Some(node) = self.0.get_mut(n as usize) { + node.nb_bits = nb_bits as u8; + } + } +} + +/// `HUF_setMaxHeight`: makes no code longer than `target_nb_bits`, which is not 0 and is enough for a code for every +/// symbol. `last_non_null` is the rarest symbol, which has the longest code. Returns the length of the longest code. +fn set_max_height( + huff_node: &mut HuffNodes, + last_non_null: u32, + target_nb_bits: u32, +) -> Result { + let largest_bits = huff_node.nb_bits(last_non_null as i32); + if largest_bits <= target_nb_bits { + return Ok(largest_bits); + } + + // The unit of cost is the share of the code space that a code of `largest_bits` takes. + let cost_of = |nb_bits: u32| 1u32.wrapping_shl(largest_bits.wrapping_sub(nb_bits)); + let mut total_cost = 0i32; + let base_cost = cost_of(target_nb_bits); + let mut n = last_non_null as i32; + + // Shorten the codes that are too long. `total_cost` is how much more than the whole code space is taken then. + while huff_node.nb_bits(n) > target_nb_bits { + let cost = base_cost.wrapping_sub(cost_of(huff_node.nb_bits(n))); + total_cost = total_cost.wrapping_add(cost as i32); + huff_node.set_nb_bits(n, target_nb_bits); + n -= 1; + } + // `n` ends at the rarest symbol with a code shorter than `target_nb_bits`, or at the sentinel. + while huff_node.nb_bits(n) == target_nb_bits { + n -= 1; + } + + // From here on the unit is the share of a code of `target_nb_bits`. + total_cost = total_cost.wrapping_shr(largest_bits - target_nb_bits); + + // Repay the cost by making other codes longer. + const NO_SYMBOL: u32 = 0xF0F0_F0F0; + // The rarest symbol whose code is that many bits shorter than `target_nb_bits`. + let mut rank_last = [NO_SYMBOL; HUF_TABLELOG_MAX as usize + 2]; + + let mut current_nb_bits = target_nb_bits; + for pos in (0..=n).rev() { + let nb_bits = huff_node.nb_bits(pos); + if nb_bits >= current_nb_bits { + continue; + } + current_nb_bits = nb_bits; + rank_last.put((target_nb_bits - current_nb_bits) as usize, pos as u32)?; + } + + while total_cost > 0 { + // One more bit on a code that is `n_bits_to_decrease` shorter repays `1 << (n_bits_to_decrease - 1)`. + let mut n_bits_to_decrease = highbit32(total_cost as u32) as usize + 1; + while n_bits_to_decrease > 1 { + let high_pos = rank_last.at(n_bits_to_decrease)?; + let low_pos = rank_last.at(n_bits_to_decrease - 1)?; + if high_pos != NO_SYMBOL { + if low_pos == NO_SYMBOL { + break; + } + // It is cheaper to lengthen this one than two of the rank below. + let high_total = huff_node.count(high_pos as i32); + let low_total = huff_node.count(low_pos as i32).wrapping_mul(2); + if high_total <= low_total { + break; + } + } + n_bits_to_decrease -= 1; + } + while n_bits_to_decrease <= HUF_TABLELOG_MAX as usize + && rank_last.at(n_bits_to_decrease)? == NO_SYMBOL + { + n_bits_to_decrease += 1; + } + let pos = rank_last.at(n_bits_to_decrease)?; + if pos == NO_SYMBOL { + return Err(BUG); + } + total_cost -= 1 << (n_bits_to_decrease - 1); + huff_node.set_nb_bits(pos as i32, huff_node.nb_bits(pos as i32) + 1); + + // The symbol is now of the rank below, where it is the rarest only if there was none. + let below = rank_last.at_mut(n_bits_to_decrease - 1)?; + if *below == NO_SYMBOL { + *below = pos; + } + // The next symbol takes its place, if it is of the same rank. + let same_rank = target_nb_bits.wrapping_sub(n_bits_to_decrease as u32); + let next = if pos != 0 && huff_node.nb_bits(pos as i32 - 1) == same_rank { + pos - 1 + } else { + NO_SYMBOL + }; + rank_last.put(n_bits_to_decrease, next)?; + } + + // Too much may have been repaid: shorten codes of `target_nb_bits`, the most frequent symbols first. + let [_, rank_last_1, ..] = &mut rank_last; + while total_cost < 0 { + if *rank_last_1 == NO_SYMBOL { + while huff_node.nb_bits(n) == target_nb_bits { + n -= 1; + } + *rank_last_1 = n as u32; + } + *rank_last_1 = rank_last_1.wrapping_add(1); + let pos = *rank_last_1 as i32; + huff_node.set_nb_bits(pos, huff_node.nb_bits(pos).wrapping_sub(1)); + total_cost += 1; + } + + Ok(target_nb_bits) +} + +/// `rankPos` +#[derive(Clone, Copy, Default)] +struct RankPos { + base: u16, + curr: u16, +} + +/// How many buckets [`sort`] has. +const RANK_POSITION_TABLE_SIZE: usize = 192; +const RANK_POSITION_MAX_COUNT_LOG: u32 = 32; +const RANK_POSITION_LOG_BUCKETS_BEGIN: u32 = + (RANK_POSITION_TABLE_SIZE as u32 - 1) - RANK_POSITION_MAX_COUNT_LOG - 1; +/// The counts below this one have a bucket each. The others share one with the counts of the same binary logarithm. +const RANK_POSITION_DISTINCT_COUNT_CUTOFF: u32 = + RANK_POSITION_LOG_BUCKETS_BEGIN + highbit32(RANK_POSITION_LOG_BUCKETS_BEGIN); + +/// `HUF_getIndex`: the bucket of a count. +#[inline(always)] +fn get_index(count: u32) -> usize { + if count < RANK_POSITION_DISTINCT_COUNT_CUTOFF { + count as usize + } else { + (highbit32(count) + RANK_POSITION_LOG_BUCKETS_BEGIN) as usize + } +} + +/// `HUF_swapNodes` +#[inline(always)] +fn swap_nodes(arr: &mut HuffNodes, a: i32, b: i32) { + let tmp = arr.get(a); + arr.set(a, arr.get(b)); + arr.set(b, tmp); +} + +/// `HUF_insertionSort`: by decreasing count, from `low` to `high` included. +#[inline] +fn insertion_sort(huff_node: &mut HuffNodes, low: i32, high: i32) { + for i in low + 1..=high { + let key = huff_node.get(i); + let mut j = i - 1; + while j >= low && huff_node.count(j) < key.count { + huff_node.set(j + 1, huff_node.get(j)); + j -= 1; + } + huff_node.set(j + 1, key); + } +} + +/// `HUF_quickSortPartition`: the pivot is the last element. Returns where it ends up. +fn quick_sort_partition(arr: &mut HuffNodes, low: i32, high: i32) -> i32 { + let pivot = arr.count(high); + let mut i = low - 1; + for j in low..high { + if arr.count(j) > pivot { + i += 1; + swap_nodes(arr, i, j); + } + } + swap_nodes(arr, i + 1, high); + i + 1 +} + +/// `HUF_simpleQuickSort`: by decreasing count, from `low` to `high` included. It recurses into the smaller part only, +/// which bounds the depth. The order it leaves symbols of the same count in decides their codes. +fn simple_quick_sort(arr: &mut HuffNodes, mut low: i32, mut high: i32) { + const INSERTION_SORT_THRESHOLD: i32 = 8; + if high - low < INSERTION_SORT_THRESHOLD { + insertion_sort(arr, low, high); + return; + } + while low < high { + let idx = quick_sort_partition(arr, low, high); + if idx - low < high - idx { + simple_quick_sort(arr, low, idx - 1); + low = idx + 1; + } else { + simple_quick_sort(arr, idx + 1, high); + high = idx - 1; + } + } +} + +/// The symbols whose count is not 0, from the lowest to the highest. +struct PresentSymbols { + symbols: [u8; HUF_SYMBOLVALUE_MAX as usize + 1], + len: usize, +} + +impl PresentSymbols { + fn get(&self) -> &[u8] { + self.symbols.get(..self.len).unwrap_or_default() + } +} + +/// `HUF_sort`: makes the first nodes, which are all zeros, the symbols by decreasing count, but for those whose count +/// is 0, which would come last. At most 256 counts. +fn sort(huff_node: &mut HuffNodes, count: &[u32]) -> Result { + let mut rank_position = [RankPos::default(); RANK_POSITION_TABLE_SIZE]; + let mut present = PresentSymbols { + symbols: [0; _], + len: 0, + }; + + // A bucket starts after the symbols of all the buckets above it. + let mut highest = 0; + for (symbol, &c) in count.iter().enumerate() { + present.symbols.put(present.len, symbol as u8)?; + present.len += usize::from(c != 0); + let index = get_index(c); + rank_position.at_mut(index)?.base += 1; + highest = highest.max(index); + } + let mut above = 0u16; + for rank in rank_position.sub_mut(..=highest)?.iter_mut().rev() { + rank.base += above; + rank.curr = rank.base; + above = rank.base; + } + + for &symbol in present.get() { + let c = count.at(usize::from(symbol))?; + let rank = rank_position.at_mut(get_index(c) + 1)?; + let node = NodeElt { + count: c, + byte: symbol, + ..NodeElt::default() + }; + huff_node.set(i32::from(rank.curr), node); + rank.curr += 1; + } + + let shared = RANK_POSITION_DISTINCT_COUNT_CUTOFF as usize..RANK_POSITION_TABLE_SIZE - 1; + for rank in rank_position + .sub(shared)? + .iter() + .take_while(|rank| rank.curr != 0) + { + if rank.curr > rank.base + 1 { + simple_quick_sort(huff_node, i32::from(rank.base), i32::from(rank.curr) - 1); + } + } + Ok(present) +} + +/// `HUF_buildTree`: sets the depth of `leaves`, which are sorted by decreasing count. There are at most 256. +fn build_tree(leaves: &mut [NodeElt]) { + /// The count of a parent that is not made yet, which is more than any count, + const NO_PARENT: u32 = 1 << 30; + /// and what there is past the last leaf, which is more than that. + const NO_LEAF: u32 = 1 << 31; + type PerParent = [T; HUF_SYMBOLVALUE_MAX as usize + 1]; + #[inline(always)] + fn at(table: &PerParent, n: u8) -> T { + table.get(usize::from(n)).copied().unwrap_or_default() + } + #[inline(always)] + fn put(table: &mut PerParent, n: u8, value: T) { + if let Some(slot) = table.get_mut(usize::from(n)) { + *slot = value; + } + } + + // The parents, in the order they are made, which is by increasing count. + let nb_parents = leaves.len().saturating_sub(1) as u8; + let mut counts: PerParent = [NO_PARENT; _]; + let mut parents: PerParent = [0; _]; + + // Join the two nodes of the lowest counts, which are among the next two leaves and the next two parents. With the + // same count, a parent goes first. + let mut low_s = leaves.len().wrapping_sub(1); + let mut low_n = 0u8; + for parent in 0..nb_parents { + let leaf = |n: usize| leaves.get(n).map_or(NO_LEAF, |leaf| leaf.count); + let (s0, s1) = (leaf(low_s), leaf(low_s.wrapping_sub(1))); + let (n0, n1) = (at(&counts, low_n), at(&counts, low_n.wrapping_add(1))); + let first_is_leaf = s0 < n0; + let second_is_leaf = if first_is_leaf { s1 < n0 } else { s0 < n1 }; + let first = if first_is_leaf { s0 } else { n0 }; + let second = match (first_is_leaf, second_is_leaf) { + (true, true) => s1, + (true, false) => n0, + (false, true) => s0, + (false, false) => n1, + }; + put(&mut counts, parent, first.wrapping_add(second)); + // The nodes that are not taken this time get their parent when they are. + for n in [low_s, low_s.wrapping_sub(1)] { + if let Some(leaf) = leaves.get_mut(n) { + leaf.parent = u16::from(parent); + } + } + put(&mut parents, low_n, parent); + put(&mut parents, low_n.wrapping_add(1), parent); + let nb_leaves = u8::from(first_is_leaf) + u8::from(second_is_leaf); + low_s = low_s.wrapping_sub(usize::from(nb_leaves)); + low_n = low_n.wrapping_add(2 - nb_leaves); + } + + // The parents come after their children, so that this goes from the root down. + let mut depths: PerParent = [0; _]; + for n in (0..nb_parents.saturating_sub(1)).rev() { + let depth = at(&depths, at(&parents, n)).wrapping_add(1); + put(&mut depths, n, depth); + } + for leaf in leaves { + leaf.nb_bits = at(&depths, leaf.parent as u8).wrapping_add(1); + } +} + +/// `HUF_buildCTableFromTree`: `leaves` are the symbols `present`, none deeper than `max_nb_bits`, which is at most 12. +fn build_ctable_from_tree( + ctable: &mut HufCTable, + leaves: &[NodeElt], + present: &[u8], + max_symbol_value: u32, + max_nb_bits: u32, +) -> Result<()> { + let ct = ctable.elements_mut(); + ct.sub_mut(..=max_symbol_value as usize)?.fill(0); + let mut nb_per_rank = [0u16; HUF_TABLELOG_MAX as usize + 1]; + for node in leaves { + *nb_per_rank.at_mut(usize::from(node.nb_bits))? += 1; + ct.put(usize::from(node.byte), u64::from(node.nb_bits))?; + } + let mut val_per_rank = first_values(&nb_per_rank, max_nb_bits)?; + for &symbol in present { + set_next_value(ct.at_mut(usize::from(symbol))?, &mut val_per_rank)?; + } + ctable.write_header(max_nb_bits, max_symbol_value); + Ok(()) +} + +/// `HUF_buildCTable_wksp`: some count up to `max_symbol_value` is not 0, and `max_nb_bits` is enough for a code for +/// each that is not, or 0 for the default. Returns the length of the longest code. +pub(crate) fn build_ctable( + ctable: &mut HufCTable, + count: &[u32], + max_symbol_value: u32, + mut max_nb_bits: u32, +) -> Result { + if max_nb_bits == 0 { + max_nb_bits = HUF_TABLELOG_DEFAULT; + } + if max_symbol_value > HUF_SYMBOLVALUE_MAX { + return Err(Error::MaxSymbolValueTooLarge); + } + let mut huff_node = HuffNodes([NodeElt::default(); _]); + + let present = sort(&mut huff_node, count.sub(..=max_symbol_value as usize)?)?; + let non_null_rank = present.len.checked_sub(1).ok_or(BUG)?; + build_tree(huff_node.0.sub_mut(..present.len)?); + max_nb_bits = set_max_height(&mut huff_node, non_null_rank as u32, max_nb_bits)?; + if max_nb_bits > HUF_TABLELOG_MAX { + return Err(Error::Generic); + } + build_ctable_from_tree( + ctable, + huff_node.0.sub(..present.len)?, + present.get(), + max_symbol_value, + max_nb_bits, + )?; + Ok(max_nb_bits as usize) +} + +/// `HUF_estimateCompressedSize`: how many bytes the symbols counted in `count` take, without the last partial one. +pub(crate) fn estimate_compressed_size( + ctable: &HufCTable, + count: &[u32], + max_symbol_value: u32, +) -> usize { + let nb_bits: usize = (ctable.elements().iter().zip(count)) + .take(max_symbol_value as usize + 1) + .map(|(&elt, &count)| get_nb_bits(elt) as usize * count as usize) + .sum(); + nb_bits >> 3 +} + +/// `HUF_validateCTable`: whether the table has a code for every symbol whose count is not 0. +pub(crate) fn validate_ctable(ctable: &HufCTable, count: &[u32], max_symbol_value: u32) -> bool { + if u32::from(read_ctable_header(ctable).max_symbol_value) < max_symbol_value { + return false; + } + (ctable.elements().iter().zip(count)) + .take(max_symbol_value as usize + 1) + .all(|(&elt, &count)| count == 0 || get_nb_bits(elt) != 0) +} + +/// One `bitContainer` of `HUF_CStream_t` and its `bitPos`. +#[derive(Clone, Copy, Default)] +struct BitContainer { + /// The bits are at the top. The lowest bits can be dirty. + bits: u64, + /// The number of bits is in the lowest byte. The rest is dirty. + pos: u64, +} + +impl BitContainer { + /// `HUF_addBits`: there is room for the code of `elt`. With `FAST`, the lowest byte of `elt` is not cleared + /// first, which takes that there is room for 4 bits more. + #[inline(always)] + fn add_bits(&mut self, elt: u64) { + self.bits >>= get_nb_bits(elt) & 63; + self.bits |= if FAST { elt } else { elt & !0xFF }; + self.pos = self.pos.wrapping_add(elt); + } + + #[inline(always)] + const fn nb_bits(self) -> u64 { + self.pos & 0xFF + } +} + +/// `HUF_CStream_t`. Its second container is a local variable of the loop that uses it. +struct HufCStream<'a> { + container: BitContainer, + dst: &'a mut [u8], + pos: usize, + /// The last position a whole container can be written to. + end: usize, +} + +impl<'a> HufCStream<'a> { + /// `HUF_initCStream` + #[inline] + fn new(dst: &'a mut [u8]) -> Result { + if dst.len() <= size_of::() { + return Err(Error::DstSizeTooSmall); + } + let end = dst.len() - size_of::(); + Ok(Self { + container: BitContainer::default(), + dst, + pos: 0, + end, + }) + } + + /// `HUF_mergeIndex1`: there is room for the bits of `second`, which come after those the stream has. + #[inline(always)] + fn merge(&mut self, second: BitContainer) { + self.container.bits >>= second.nb_bits() & 63; + self.container.bits |= second.bits; + self.container.pos = self.container.pos.wrapping_add(second.pos); + } + + /// `HUF_flushBits`: writes the whole bytes. At the end of the output the stream stops advancing, which + /// [`Self::close`] reports. The C code has a variant without that limit for when the output is known to be large + /// enough, but here the limit is what spares a check on the write. + #[inline(always)] + fn flush_bits(&mut self) { + let nb_bits = self.container.nb_bits(); + let nb_bytes = (nb_bits >> 3) as usize; + let bits = self.container.bits >> (u64::from(u64::BITS).wrapping_sub(nb_bits) & 63); + self.container.pos &= 7; + let pos = self.pos.min(self.end); + if let Ok(out) = self.dst.array_at_mut(pos) { + *out = bits.to_le_bytes(); + } + self.pos = pos + nb_bytes; + } + + /// `HUF_closeCStream`: the size of the stream, or 0 if it did not fit. + #[inline] + fn close(mut self) -> usize { + /// `HUF_endMark`: one bit, set. + const END_MARK: u64 = 1 | (1 << 63); + self.container.add_bits::(END_MARK); + self.flush_bits(); + if self.pos >= self.end { + return 0; + } + self.pos + usize::from(self.container.nb_bits() > 0) + } +} + +type Elements = [u64; HUF_SYMBOLVALUE_MAX as usize + 1]; + +/// `HUF_encodeSymbol` +#[inline(always)] +fn encode_symbol(container: &mut BitContainer, symbol: u8, ct: &Elements) { + container.add_bits::(ct.get(usize::from(symbol)).copied().unwrap_or(0)); +} + +/// Encodes `symbols` from the last to the first. +#[inline(always)] +fn encode_symbols( + container: &mut BitContainer, + symbols: &[u8; UNROLL], + ct: &Elements, +) { + if let Some((&last, symbols)) = symbols.split_first() { + for &symbol in symbols.iter().rev() { + encode_symbol::(container, symbol, ct); + } + encode_symbol::(container, last, ct); + } +} + +/// `HUF_compress1X_usingCTable_internal_body_loop`: a container has room for `UNROLL` symbols on top of the 7 bits +/// that a flush can leave, and, with `LAST_FAST`, for 4 bits more. +#[inline(always)] +fn compress_1x_body_loop( + bit_c: &mut HufCStream<'_>, + src: &[u8], + ct: &Elements, +) { + let (groups, rem) = src.as_chunks::(); + if !rem.is_empty() { + for &symbol in rem.iter().rev() { + encode_symbol::(&mut bit_c.container, symbol, ct); + } + bit_c.flush_bits(); + } + let (pairs, odd) = groups.as_chunks::<2>(); + if let [group] = odd { + encode_symbols::(&mut bit_c.container, group, ct); + bit_c.flush_bits(); + } + for [second, first] in pairs.iter().rev() { + encode_symbols::(&mut bit_c.container, first, ct); + bit_c.flush_bits(); + // The second group does not depend on the flush, since it has a container of its own. + let mut container = BitContainer::default(); + encode_symbols::(&mut container, second, ct); + bit_c.merge(container); + bit_c.flush_bits(); + } +} + +/// `HUF_tightCompressBound`: no output is larger, given the longest code. +const fn tight_compress_bound(src_size: usize, table_log: usize) -> usize { + ((src_size * table_log) >> 3) + 8 +} + +/// With that much room, the streams of `nb_literals` are written the quick way, whatever their code and be it one stream +/// or four. As they are, the literals take less. +pub(crate) const fn room_for_quick_loops(nb_literals: usize) -> usize { + 4 * tight_compress_bound(nb_literals.div_ceil(4), LIT_HUF_LOG as usize) +} + +/// `HUF_compress1X_usingCTable`: the table has a code for every symbol in `src`. Returns the size of the output, or 0 +/// if `dst` is too small. +pub(crate) fn compress_1x_using_ctable( + dst: &mut [u8], + src: &[u8], + ctable: &HufCTable, +) -> Result { + let table_log = read_ctable_header(ctable).table_log; + let ct = ctable.elements(); + let dst_size = dst.len(); + + let Ok(mut bit_c) = HufCStream::new(dst) else { + return Ok(0); + }; + let bit_c_ref = &mut bit_c; + + if dst_size < tight_compress_bound(src.len(), usize::from(table_log)) || table_log > 11 { + compress_1x_body_loop::<4, false>(bit_c_ref, src, ct); + } else { + match table_log { + 11 => compress_1x_body_loop::<5, false>(bit_c_ref, src, ct), + 10 => compress_1x_body_loop::<5, true>(bit_c_ref, src, ct), + 9 => compress_1x_body_loop::<6, false>(bit_c_ref, src, ct), + 8 => compress_1x_body_loop::<7, false>(bit_c_ref, src, ct), + 7 => compress_1x_body_loop::<8, false>(bit_c_ref, src, ct), + _ => compress_1x_body_loop::<9, true>(bit_c_ref, src, ct), + } + } + Ok(bit_c.close()) +} + +/// `HUF_compress4X_usingCTable`: four streams, after the sizes of the first three. The table has a code for every +/// symbol in `src`. Returns the size of the output, or 0 if `src` is too small for it, `dst` is, or a stream is too +/// large. +pub(crate) fn compress_4x_using_ctable( + dst: &mut [u8], + src: &[u8], + ctable: &HufCTable, +) -> Result { + let segment_size = src.len().div_ceil(4); + if dst.len() < 6 + 1 + 1 + 1 + 8 { + return Ok(0); + } + if src.len() < 12 { + return Ok(0); + } + let mut op = 6usize; + for (stream, segment) in src.chunks(segment_size).enumerate() { + let c_size = compress_1x_using_ctable(dst.sub_mut(op..)?, segment, ctable)?; + if c_size == 0 || c_size > 65535 { + return Ok(0); + } + if stream < 3 { + write_le16(dst, stream * 2, c_size as u16)?; + } + op += c_size; + } + Ok(op) +} + +/// `HUF_nbStreams_e` +#[derive(Clone, Copy, PartialEq, Eq)] +enum NbStreams { + Single, + Four, +} + +/// `HUF_compressCTable_internal`: writes from `op` on. Returns how much of `dst` is used, or 0 if that saves nothing. +fn compress_ctable_internal( + dst: &mut [u8], + op: usize, + src: &[u8], + nb_streams: NbStreams, + ctable: &HufCTable, +) -> Result { + let out = dst.sub_mut(op..)?; + let c_size = match nb_streams { + NbStreams::Single => compress_1x_using_ctable(out, src, ctable)?, + NbStreams::Four => compress_4x_using_ctable(out, src, ctable)?, + }; + if c_size == 0 { + return Ok(0); + } + if op + c_size >= src.len().wrapping_sub(1) { + return Ok(0); + } + Ok(op + c_size) +} + +const SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE: usize = 4096; +const SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO: usize = 10; + +/// `HUF_cardinality`: how many counts up to `max_symbol_value` are not 0. +pub(crate) fn cardinality(count: &[u32], max_symbol_value: u32) -> u32 { + let counts = count.iter().take(max_symbol_value as usize + 1); + counts.filter(|&&count| count != 0).count() as u32 +} + +/// `HUF_minTableLog`: the shortest that the longest code can be. `symbol_cardinality` is not 0. +pub(crate) fn min_table_log(symbol_cardinality: u32) -> u32 { + highbit32(symbol_cardinality) + 1 +} + +/// `HUF_optimalTableLog`: the limit to put on the length of the codes. `src_size` is more than 1. With +/// `flags.optimal_depth`, `table` is overwritten. +pub(crate) fn optimal_table_log( + max_table_log: u32, + src_size: usize, + max_symbol_value: u32, + table: &mut HufCTable, + count: &[u32], + flags: HufFlags, +) -> u32 { + if !flags.optimal_depth { + return fse::optimal_table_log_internal(max_table_log, src_size, max_symbol_value, 1); + } + + // Large enough that no step of `write_ctable` is short of room. + let mut dst = [0u8; 512]; + let symbol_cardinality = cardinality(count, max_symbol_value); + let min_table_log = min_table_log(symbol_cardinality); + let mut opt_size = usize::MAX - 1; + let mut opt_log = max_table_log; + + // Search until the size increases. + for opt_log_guess in min_table_log..=max_table_log { + let Ok(max_bits) = build_ctable(table, count, max_symbol_value, opt_log_guess) else { + continue; + }; + if max_bits < opt_log_guess as usize && opt_log_guess > min_table_log { + break; + } + let Ok(h_size) = write_ctable(&mut dst, table, max_symbol_value, max_bits as u32) else { + continue; + }; + let new_size = estimate_compressed_size(table, count, max_symbol_value) + h_size; + if new_size > opt_size + 1 { + break; + } + if new_size < opt_size { + opt_size = new_size; + opt_log = opt_log_guess; + } + } + opt_log +} + +/// Whether `src` is long enough to tell by its start and its end, and they say that it does not compress. +pub(crate) fn samples_are_incompressible(src: &[u8]) -> Result { + let src_size = src.len(); + if src_size < SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE * SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO { + return Ok(false); + } + let mut count = [0u32; HUF_SYMBOLVALUE_MAX as usize + 1]; + let mut largest_total = 0usize; + for sample in [ + src.sub(..SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE)?, + src.sub(src_size - SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE..)?, + ] { + let mut max_symbol_value = HUF_SYMBOLVALUE_MAX; + largest_total += hist::count_simple(&mut count, &mut max_symbol_value, sample)? as usize; + } + Ok(largest_total <= ((2 * SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE) >> 7) + 4) +} + +/// `HUF_compress_internal` +fn compress_internal( + dst: &mut [u8], + src: &[u8], + mut max_symbol_value: u32, + mut huff_log: u32, + nb_streams: NbStreams, + old_huf_table: &mut HufCTable, + repeat: &mut HufRepeat, + flags: HufFlags, +) -> Result { + let src_size = src.len(); + let Some(&first_byte) = src.first() else { + return Ok(0); + }; + if dst.is_empty() { + return Ok(0); + } + if src_size > HUF_BLOCKSIZE_MAX { + return Err(Error::SrcSizeWrong); + } + if huff_log > HUF_TABLELOG_MAX { + return Err(Error::TableLogTooLarge); + } + if max_symbol_value > HUF_SYMBOLVALUE_MAX { + return Err(Error::MaxSymbolValueTooLarge); + } + if max_symbol_value == 0 { + max_symbol_value = HUF_SYMBOLVALUE_MAX; + } + if huff_log == 0 { + huff_log = HUF_TABLELOG_DEFAULT; + } + + if flags.prefer_repeat && *repeat == HufRepeat::Valid { + return compress_ctable_internal(dst, 0, src, nb_streams, old_huf_table); + } + + // If the input may well be incompressible, look at its start and its end before all of it. + if flags.suspect_uncompressible && samples_are_incompressible(src)? { + return Ok(0); + } + + let mut count = [0u32; HUF_SYMBOLVALUE_MAX as usize + 1]; + let largest = hist::count_wksp(&mut count, &mut max_symbol_value, src)?; + if largest == src_size { + // A single symbol: run-length encoding. + dst.put(0, first_byte)?; + return Ok(1); + } + if largest <= (src_size >> 7) + 4 { + // Probably not compressible enough. + return Ok(0); + } + + if *repeat == HufRepeat::Check && !validate_ctable(old_huf_table, &count, max_symbol_value) { + *repeat = HufRepeat::None; + } + if flags.prefer_repeat && *repeat != HufRepeat::None { + return compress_ctable_internal(dst, 0, src, nb_streams, old_huf_table); + } + + let mut ctable = HufCTable::new(); + huff_log = optimal_table_log( + huff_log, + src_size, + max_symbol_value, + &mut ctable, + &count, + flags, + ); + huff_log = build_ctable(&mut ctable, &count, max_symbol_value, huff_log)? as u32; + + let h_size = write_ctable(dst, &ctable, max_symbol_value, huff_log)?; + // Is the previous table the better choice, since it takes no header? + if *repeat != HufRepeat::None { + let old_size = estimate_compressed_size(old_huf_table, &count, max_symbol_value); + let new_size = estimate_compressed_size(&ctable, &count, max_symbol_value); + if old_size <= h_size + new_size || h_size + 12 >= src_size { + return compress_ctable_internal(dst, 0, src, nb_streams, old_huf_table); + } + } + + if h_size + 12 >= src_size { + return Ok(0); + } + *repeat = HufRepeat::None; + old_huf_table.clone_from(&ctable); + compress_ctable_internal(dst, h_size, src, nb_streams, &ctable) +} + +/// `HUF_compress1X_repeat`: one stream, after the header of its table, or without one if `huf_table` is used again, +/// which `repeat` says is allowed. If it is not used again, `repeat` becomes [`HufRepeat::None`] and `huf_table` the +/// new table. Returns the size of the output, 0 if `src` is not compressible, or 1 if it is one byte repeated, which +/// is then the output. +pub(crate) fn compress_1x_repeat( + dst: &mut [u8], + src: &[u8], + max_symbol_value: u32, + huff_log: u32, + huf_table: &mut HufCTable, + repeat: &mut HufRepeat, + flags: HufFlags, +) -> Result { + compress_internal( + dst, + src, + max_symbol_value, + huff_log, + NbStreams::Single, + huf_table, + repeat, + flags, + ) +} + +/// `HUF_compress4X_repeat`: as [`compress_1x_repeat`], with four streams. +pub(crate) fn compress_4x_repeat( + dst: &mut [u8], + src: &[u8], + max_symbol_value: u32, + huff_log: u32, + huf_table: &mut HufCTable, + repeat: &mut HufRepeat, + flags: HufFlags, +) -> Result { + compress_internal( + dst, + src, + max_symbol_value, + huff_log, + NbStreams::Four, + huf_table, + repeat, + flags, + ) +} diff --git a/src/zstd/compress/internal.rs b/src/zstd/compress/internal.rs new file mode 100644 index 000000000000..2fd7276be78f --- /dev/null +++ b/src/zstd/compress/internal.rs @@ -0,0 +1,3035 @@ +//! `zstd_compress_internal.h`: what the parts of the compressor share. + +use core::hint::{Locality, prefetch_read}; +use core::marker::ConstParamTy; +use core::sync::atomic::{AtomicBool, Ordering, compiler_fence}; +use std::sync::Arc; + +use super::fse::{FseCTable, FseRepeat}; +use super::huf::{HufCTable, HufRepeat}; +use super::sequences::SeqCodes; +use crate::common::{ + BlockType, MINMATCH, REP_START_VALUE, SymbolEncodingType, WILDCOPY_OVERLENGTH, + ZSTD_BLOCKHEADERSIZE, ZSTD_REP_NUM, +}; +use crate::error::{Error, Result}; +use crate::util::{ + BUG, SliceExt, append_at, copy, highbit32, make_room, reset_table, try_zeroed, write_le24, +}; + +pub(crate) const K_SEARCH_STRENGTH: u32 = 8; +pub(crate) const HASH_READ_SIZE: usize = 8; +/// Marks an unsorted position of a binary tree. Less than any index, since those start at [`ZSTD_WINDOW_START_INDEX`]. +pub(crate) const ZSTD_DUBT_UNSORTED_MARK: u32 = 1; + +pub(crate) const ZSTD_OPT_NUM: usize = 1 << 12; +pub(crate) const ZSTD_OPT_SIZE: usize = ZSTD_OPT_NUM + 3; + +pub(crate) const ZSTD_MAX_HUF_HEADER_SIZE: usize = 128; +pub(crate) const ZSTD_MAX_FSE_HEADERS_SIZE: usize = (53usize * 9 + 36 * 9 + 32 * 8).div_ceil(8); + +/// `ZSTD_strategy` +#[repr(u8)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)] +pub enum Strategy { + Fast = 1, + DFast = 2, + Greedy = 3, + Lazy = 4, + Lazy2 = 5, + BtLazy2 = 6, + BtOpt = 7, + BtUltra = 8, + BtUltra2 = 9, +} + +impl Strategy { + pub(crate) const fn from_int(value: i32) -> Option { + Some(match value { + 1 => Self::Fast, + 2 => Self::DFast, + 3 => Self::Greedy, + 4 => Self::Lazy, + 5 => Self::Lazy2, + 6 => Self::BtLazy2, + 7 => Self::BtOpt, + 8 => Self::BtUltra, + 9 => Self::BtUltra2, + _ => return None, + }) + } +} + +/// `ZSTD_ParamSwitch_e` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum ParamSwitch { + Auto = 0, + Enable = 1, + Disable = 2, +} + +/// `ZSTD_compressionParameters` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) struct CompressionParameters { + pub(crate) window_log: u32, + pub(crate) chain_log: u32, + pub(crate) hash_log: u32, + pub(crate) search_log: u32, + pub(crate) min_match: u32, + pub(crate) target_length: u32, + /// `None` stands for 0, which is "not set" among requested parameters. Applied parameters have one. + pub(crate) strategy: Option, +} + +impl CompressionParameters { + /// The strategy of parameters that are ready to be applied. + #[inline(always)] + pub(crate) fn strat(&self) -> Strategy { + self.strategy.unwrap_or(Strategy::Fast) + } +} + +/// `ZSTD_dictMode_e`. Nothing can ask for `ZSTD_dedicatedDictSearch`. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum DictMode { + NoDict, + ExtDict, + DictMatchState, +} + +/// `ZSTD_tableFillPurpose_e`. A `CDict` is loaded with `ZSTD_dtlm_full` and `ZSTD_dct_auto`, and nothing else is. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum TableFillPurpose { + ForCCtx, + ForCDict, +} + +// ── entropy tables ──────────────────────────────────────────────────────── + +/// `ZSTD_hufCTables_t` +#[derive(Clone)] +pub(crate) struct HufCTables { + pub(crate) c_table: HufCTable, + pub(crate) repeat_mode: HufRepeat, +} + +impl HufCTables { + /// `*self = from.clone()`, but for a table that cannot be used again, which nothing reads. + pub(crate) fn copy_from(&mut self, from: &Self) { + self.repeat_mode = from.repeat_mode; + if from.repeat_mode != HufRepeat::None { + self.c_table.clone_from(&from.c_table); + } + } +} + +/// `ZSTD_fseCTables_t` +#[derive(Clone)] +pub(crate) struct FseCTables { + pub(crate) offcode_c_table: FseCTable, + pub(crate) matchlength_c_table: FseCTable, + pub(crate) litlength_c_table: FseCTable, + pub(crate) offcode_repeat_mode: FseRepeat, + pub(crate) matchlength_repeat_mode: FseRepeat, + pub(crate) litlength_repeat_mode: FseRepeat, +} + +impl FseCTables { + /// `*self = from.clone()`, but for the tables that cannot be used again, which nothing reads. + pub(crate) fn copy_from(&mut self, from: &Self) { + self.offcode_repeat_mode = from.offcode_repeat_mode; + if from.offcode_repeat_mode != FseRepeat::None { + self.offcode_c_table.clone_from(&from.offcode_c_table); + } + self.matchlength_repeat_mode = from.matchlength_repeat_mode; + if from.matchlength_repeat_mode != FseRepeat::None { + (self.matchlength_c_table).clone_from(&from.matchlength_c_table); + } + self.litlength_repeat_mode = from.litlength_repeat_mode; + if from.litlength_repeat_mode != FseRepeat::None { + self.litlength_c_table.clone_from(&from.litlength_c_table); + } + } +} + +/// `ZSTD_entropyCTables_t` +#[derive(Clone)] +pub(crate) struct EntropyCTables { + pub(crate) huf: HufCTables, + pub(crate) fse: FseCTables, +} + +/// `ZSTD_compressedBlockState_t` +#[derive(Clone)] +pub(crate) struct CompressedBlockState { + pub(crate) entropy: EntropyCTables, + pub(crate) rep: [u32; ZSTD_REP_NUM], +} + +impl CompressedBlockState { + /// `ZSTD_reset_compressedBlockState`, on tables that are all zeros. + pub(crate) const fn new() -> Self { + Self { + entropy: EntropyCTables { + huf: HufCTables { + c_table: HufCTable::new(), + repeat_mode: HufRepeat::None, + }, + fse: FseCTables { + offcode_c_table: FseCTable::new(), + matchlength_c_table: FseCTable::new(), + litlength_c_table: FseCTable::new(), + offcode_repeat_mode: FseRepeat::None, + matchlength_repeat_mode: FseRepeat::None, + litlength_repeat_mode: FseRepeat::None, + }, + }, + rep: REP_START_VALUE, + } + } + + /// `ZSTD_reset_compressedBlockState` + pub(crate) fn reset(&mut self) { + self.rep = REP_START_VALUE; + self.entropy.huf.repeat_mode = HufRepeat::None; + self.entropy.fse.offcode_repeat_mode = FseRepeat::None; + self.entropy.fse.matchlength_repeat_mode = FseRepeat::None; + self.entropy.fse.litlength_repeat_mode = FseRepeat::None; + } +} + +/// `ZSTD_hufCTablesMetadata_t` +pub(crate) struct HufCTablesMetadata { + pub(crate) h_type: SymbolEncodingType, + pub(crate) huf_des_buffer: [u8; ZSTD_MAX_HUF_HEADER_SIZE], + pub(crate) huf_des_size: usize, +} + +/// `ZSTD_fseCTablesMetadata_t` +pub(crate) struct FseCTablesMetadata { + pub(crate) ll_type: SymbolEncodingType, + pub(crate) of_type: SymbolEncodingType, + pub(crate) ml_type: SymbolEncodingType, + pub(crate) fse_tables_buffer: [u8; ZSTD_MAX_FSE_HEADERS_SIZE], + pub(crate) fse_tables_size: usize, + /// Accounts for a bug of the decoders up to 1.3.4: see `ZSTD_entropyCompressSeqStore_internal`. + pub(crate) last_count_size: usize, +} + +/// `ZSTD_entropyCTablesMetadata_t` +pub(crate) struct EntropyCTablesMetadata { + pub(crate) huf_metadata: HufCTablesMetadata, + pub(crate) fse_metadata: FseCTablesMetadata, +} + +impl EntropyCTablesMetadata { + pub(crate) const fn new() -> Self { + Self { + huf_metadata: HufCTablesMetadata { + h_type: SymbolEncodingType::Basic, + huf_des_buffer: [0; ZSTD_MAX_HUF_HEADER_SIZE], + huf_des_size: 0, + }, + fse_metadata: FseCTablesMetadata { + ll_type: SymbolEncodingType::Basic, + of_type: SymbolEncodingType::Basic, + ml_type: SymbolEncodingType::Basic, + fse_tables_buffer: [0; ZSTD_MAX_FSE_HEADERS_SIZE], + fse_tables_size: 0, + last_count_size: 0, + }, + } + } +} + +// ── sequences ───────────────────────────────────────────────────────────── + +/// `SeqDef` +#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] +pub(crate) struct SeqDef { + /// A repeat code (1 to 3), or an offset plus 3. + pub(crate) off_base: u32, + pub(crate) lit_length: u16, + /// The match length minus [`MINMATCH`]. + pub(crate) ml_base: u16, +} + +/// `ZSTD_longLengthType_e`: at most one length of a block does not fit 16 bits. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum LongLengthType { + None, + LiteralLength, + MatchLength, +} + +/// `SeqStore_t`: the sequences and the literals of a block. +pub(crate) struct SeqStore { + /// Never grows beyond the capacity it is given by [`Self::start_block`]. + pub(crate) sequences: Vec, + /// It has room for as many literals as a block has bytes, and [`WILDCOPY_OVERLENGTH`] to copy in chunks. Its length + /// follows the literals: to clear all of it for a few of them takes longer than many a block. + pub(crate) lit_buffer: Vec, + /// How much of `lit_buffer` is literals. + pub(crate) lit_len: usize, + /// As many as the most sequences that a block had so far. + pub(crate) codes: Vec, + pub(crate) max_nb_seq: usize, + pub(crate) max_nb_lit: usize, + pub(crate) long_length_type: LongLengthType, + /// Which sequence has the long length. + pub(crate) long_length_pos: u32, +} + +/// `ZSTD_SequenceLength` +#[derive(Clone, Copy)] +pub(crate) struct SequenceLength { + pub(crate) lit_length: u32, + pub(crate) match_length: u32, +} + +/// Some consecutive sequences of a [`SeqStore`] and their literals, which the C code has a `SeqStore_t` that points +/// into the other one for. +#[derive(Clone, Copy)] +pub(crate) struct SeqChunk { + pub(crate) seq_start: usize, + pub(crate) seq_end: usize, + pub(crate) lit_start: usize, + pub(crate) lit_end: usize, + pub(crate) long_length_type: LongLengthType, + /// Counted from `seq_start`. + pub(crate) long_length_pos: u32, +} + +impl SeqChunk { + #[inline] + pub(crate) const fn nb_seq(&self) -> usize { + self.seq_end - self.seq_start + } + + #[inline] + pub(crate) const fn lit_size(&self) -> usize { + self.lit_end - self.lit_start + } +} + +impl SeqStore { + /// All of it. + pub(crate) const fn whole(&self) -> SeqChunk { + SeqChunk { + seq_start: 0, + seq_end: self.sequences.len(), + lit_start: 0, + lit_end: self.lit_len, + long_length_type: self.long_length_type, + long_length_pos: self.long_length_pos, + } + } + + /// `ZSTD_getSequenceLength` + #[inline] + pub(crate) fn get_sequence_length(&self, index: usize) -> Result { + let seq = self.sequences.at(index)?; + let mut seq_len = SequenceLength { + lit_length: u32::from(seq.lit_length), + match_length: u32::from(seq.ml_base) + MINMATCH, + }; + if self.long_length_pos as usize == index { + match self.long_length_type { + LongLengthType::None => {} + LongLengthType::LiteralLength => seq_len.lit_length += 0x10000, + LongLengthType::MatchLength => seq_len.match_length += 0x10000, + } + } + Ok(seq_len) + } + + pub(crate) const fn new() -> Self { + Self { + sequences: Vec::new(), + lit_buffer: Vec::new(), + lit_len: 0, + codes: Vec::new(), + max_nb_seq: 0, + max_nb_lit: 0, + long_length_type: LongLengthType::None, + long_length_pos: 0, + } + } + + /// A block has at most `max_nb_seq` sequences and `max_nb_lit` bytes. + pub(crate) fn set_limits(&mut self, max_nb_seq: usize, max_nb_lit: usize) { + self.max_nb_seq = max_nb_seq; + self.max_nb_lit = max_nb_lit; + } + + /// `ZSTD_resetSeqStore`, with room for the sequences and the literals of a block of `block_size` bytes. + pub(crate) fn start_block(&mut self, block_size: usize) -> Result<()> { + self.reset(); + let block_size = block_size.min(self.max_nb_lit); + let nb_seq = (block_size / MINMATCH as usize).min(self.max_nb_seq); + make_room(&mut self.sequences, nb_seq, self.max_nb_seq)?; + make_room( + &mut self.lit_buffer, + block_size + WILDCOPY_OVERLENGTH, + self.max_nb_lit + WILDCOPY_OVERLENGTH, + ) + } + + /// Makes `codes` at least as long as `sequences`. + pub(crate) fn make_room_for_codes(&mut self) -> Result<()> { + let len = self.sequences.len(); + make_room(&mut self.codes, len, self.max_nb_seq)?; + if self.codes.len() < len { + self.codes.resize(len, SeqCodes::default()); + } + Ok(()) + } + + /// `ZSTD_resetSeqStore` + #[inline] + pub(crate) fn reset(&mut self) { + self.lit_len = 0; + self.sequences.clear(); + self.long_length_type = LongLengthType::None; + } + + #[inline] + pub(crate) fn literals(&self) -> Result<&[u8]> { + self.lit_buffer.sub(..self.lit_len) + } + + /// `ZSTD_storeSeqOnly`: `match_length` is at least [`MINMATCH`]. + #[inline(always)] + pub(crate) fn store_seq_only(&mut self, lit_length: usize, off_base: u32, match_length: usize) { + debug_assert!(self.sequences.len() < self.max_nb_seq); + if lit_length > 0xFFFF { + self.long_length_type = LongLengthType::LiteralLength; + self.long_length_pos = self.sequences.len() as u32; + } + let ml_base = match_length.wrapping_sub(MINMATCH as usize); + if ml_base > 0xFFFF { + self.long_length_type = LongLengthType::MatchLength; + self.long_length_pos = self.sequences.len() as u32; + } + self.sequences.push(SeqDef { + off_base, + lit_length: lit_length as u16, + ml_base: ml_base as u16, + }); + } + + /// `ZSTD_storeSeq`: the literals are the `lit_length` bytes of `buf` from `literals` on. + #[inline(always)] + pub(crate) fn store_seq( + &mut self, + lit_length: usize, + buf: &[u8], + literals: usize, + off_base: u32, + match_length: usize, + ) -> Result<()> { + match ( + buf.array_at::<16>(literals), + self.lit_buffer.array_at_mut::<16>(self.lit_len), + ) { + (Ok(from), Ok(to)) if lit_length <= 16 => *to = *from, + _ => write_literals( + &mut self.lit_buffer, + self.lit_len, + buf.span(literals, lit_length)?, + )?, + } + self.lit_len += lit_length; + self.store_seq_only(lit_length, off_base, match_length); + Ok(()) + } + + /// Appends the literals that follow the last sequence of `block`, `count` of them. + #[inline] + pub(crate) fn store_last_literals(&mut self, block: &[u8], count: usize) -> Result<()> { + let literals = block.sub(block.len().checked_sub(count).ok_or(BUG)?..)?; + write_literals(&mut self.lit_buffer, self.lit_len, literals)?; + self.lit_len += literals.len(); + Ok(()) + } +} + +/// Writes `literals` at `lit_len` in `lit_buffer`. +#[inline(always)] +fn write_literals(lit_buffer: &mut Vec, lit_len: usize, literals: &[u8]) -> Result<()> { + if let Some(room) = lit_buffer.get_mut(lit_len..lit_len + literals.len()) { + return copy(room, literals); + } + // Moved there and back: a buffer that is lent could not stay in the registers of a hot loop. + let result; + (*lit_buffer, result) = append_literals(core::mem::take(lit_buffer), lit_len, literals); + result +} + +/// How much of the buffer of the literals is cleared ahead of them. +const LITERALS_AHEAD: usize = 1 << 12; + +/// [`write_literals`] beyond the length of `lit_buffer`, which gets longer than that takes. +#[cold] +#[inline(never)] +fn append_literals( + mut lit_buffer: Vec, + lit_len: usize, + literals: &[u8], +) -> (Vec, Result<()>) { + let result = append_at(&mut lit_buffer, lit_len, literals); + if result.is_ok() { + let len = (lit_buffer.len() + LITERALS_AHEAD).min(lit_buffer.capacity()); + lit_buffer.resize(len, 0); + } + (lit_buffer, result) +} + +/// A [`SeqStore`] while a hot loop adds to it: what changes with every sequence is here, in a local variable, which can +/// stay in registers. It goes back to the store on drop. +/// +/// Up to `SHORT` literals, which is at most [`WILDCOPY_OVERLENGTH`], that many bytes are copied whatever their number. +/// `GROWS` is whether `lit_buffer` gets longer as literals are added. Else all of it is cleared first, for a loop that has +/// no register to spare for a buffer that moves. +pub(crate) struct SeqSink<'a, const SHORT: usize, const GROWS: bool> { + store: &'a mut SeqStore, + sequences: Vec, + lit_buffer: Vec, + lit_len: u32, +} + +impl<'a, const SHORT: usize, const GROWS: bool> SeqSink<'a, SHORT, GROWS> { + #[inline(always)] + pub(crate) fn new(store: &'a mut SeqStore) -> Self { + let mut lit_buffer = core::mem::take(&mut store.lit_buffer); + if !GROWS { + lit_buffer.resize(lit_buffer.capacity(), 0); + } + Self { + sequences: core::mem::take(&mut store.sequences), + lit_buffer, + lit_len: store.lit_len as u32, + store, + } + } + + /// `ZSTD_storeSeq`: the literals are the `lit_length` bytes of `src` from `literals` on. + #[inline(always)] + pub(crate) fn store_seq( + &mut self, + lit_length: u32, + src: Near<'_>, + literals: u32, + off_base: u32, + match_length: u32, + ) -> Result<()> { + let lit_len = self.lit_len as usize; + let room = self.lit_buffer.get_mut(lit_len..lit_len + SHORT); + match ( + src.window::(literals), + room.and_then(<[u8]>::first_chunk_mut::), + ) { + (Some(from), Some(to)) if lit_length as usize <= SHORT => *to = *from, + _ => { + let literals = src.bytes().span(literals as usize, lit_length as usize)?; + if GROWS { + write_literals(&mut self.lit_buffer, lit_len, literals)?; + } else { + self.lit_buffer.write_at(lit_len, literals)?; + } + } + } + self.lit_len += lit_length; + self.store_seq_only(lit_length, off_base, match_length) + } + + /// `ZSTD_storeSeqOnly` + #[inline(always)] + pub(crate) fn store_seq_only( + &mut self, + lit_length: u32, + off_base: u32, + match_length: u32, + ) -> Result<()> { + let ml_base = match_length.wrapping_sub(MINMATCH); + if lit_length > 0xFFFF || ml_base > 0xFFFF { + self.store.long_length_type = if ml_base > 0xFFFF { + LongLengthType::MatchLength + } else { + LongLengthType::LiteralLength + }; + self.store.long_length_pos = self.sequences.len() as u32; + } + self.sequences + .push_within_capacity(SeqDef { + off_base, + lit_length: lit_length as u16, + ml_base: ml_base as u16, + }) + .map(drop) + .map_err(|_| BUG) + } +} + +impl Drop for SeqSink<'_, SHORT, GROWS> { + #[inline(always)] + fn drop(&mut self) { + self.store.sequences = core::mem::take(&mut self.sequences); + self.store.lit_buffer = core::mem::take(&mut self.lit_buffer); + self.store.lit_len = self.lit_len as usize; + } +} + +pub(crate) const REPCODE1_TO_OFFBASE: u32 = 1; + +/// `OFFSET_TO_OFFBASE`: `offset` is not 0. +#[inline(always)] +pub(crate) const fn offset_to_offbase(offset: u32) -> u32 { + offset.wrapping_add(ZSTD_REP_NUM as u32) +} + +/// `OFFBASE_IS_OFFSET` +#[inline(always)] +pub(crate) const fn offbase_is_offset(off_base: u32) -> bool { + off_base > ZSTD_REP_NUM as u32 +} + +/// `OFFBASE_IS_REPCODE` +#[inline(always)] +pub(crate) const fn offbase_is_repcode(off_base: u32) -> bool { + 1 <= off_base && off_base <= ZSTD_REP_NUM as u32 +} + +/// `OFFBASE_TO_OFFSET` +#[inline(always)] +pub(crate) const fn offbase_to_offset(off_base: u32) -> u32 { + off_base.wrapping_sub(ZSTD_REP_NUM as u32) +} + +/// `ZSTD_updateRep`: `ll0` is whether the sequence has no literals. +#[inline(always)] +pub(crate) fn update_rep(rep: &mut [u32; ZSTD_REP_NUM], off_base: u32, ll0: bool) { + let [rep0, rep1, rep2] = *rep; + if offbase_is_offset(off_base) { + *rep = [offbase_to_offset(off_base), rep0, rep1]; + } else { + match off_base - 1 + u32::from(ll0) { + 0 => {} + 1 => *rep = [rep1, rep0, rep2], + 2 => *rep = [rep2, rep0, rep1], + _ => *rep = [rep0.wrapping_sub(1), rep0, rep1], + } + } +} + +/// `ZSTD_newRep` +#[inline(always)] +pub(crate) fn new_rep(rep: &[u32; ZSTD_REP_NUM], off_base: u32, ll0: bool) -> [u32; ZSTD_REP_NUM] { + let mut new_reps = *rep; + update_rep(&mut new_reps, off_base, ll0); + new_reps +} + +/// `ZSTD_LLcode` +#[inline(always)] +pub(crate) fn ll_code(lit_length: u32) -> u32 { + #[rustfmt::skip] + const LL_CODE: [u8; 64] = [ + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 16, 17, 17, 18, 18, 19, 19, + 20, 20, 20, 20, 21, 21, 21, 21, + 22, 22, 22, 22, 22, 22, 22, 22, + 23, 23, 23, 23, 23, 23, 23, 23, + 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, + ]; + const LL_DELTA_CODE: u32 = 19; + match LL_CODE.get(lit_length as usize) { + Some(&code) => u32::from(code), + None => highbit32(lit_length) + LL_DELTA_CODE, + } +} + +/// `ZSTD_MLcode`: `ml_base` is the match length minus [`MINMATCH`]. +#[inline(always)] +pub(crate) fn ml_code(ml_base: u32) -> u32 { + #[rustfmt::skip] + const ML_CODE: [u8; 128] = [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, + 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, + 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, + 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + ]; + const ML_DELTA_CODE: u32 = 36; + match ML_CODE.get(ml_base as usize) { + Some(&code) => u32::from(code), + None => highbit32(ml_base) + ML_DELTA_CODE, + } +} + +/// `ZSTD_noCompressBlock` +pub(crate) fn no_compress_block(dst: &mut [u8], src: &[u8], last_block: bool) -> Result { + if src.len() + ZSTD_BLOCKHEADERSIZE > dst.len() { + return Err(Error::DstSizeTooSmall); + } + write_le24(dst, 0, block_header(BlockType::Raw, src.len(), last_block))?; + dst.write_at(ZSTD_BLOCKHEADERSIZE, src)?; + Ok(ZSTD_BLOCKHEADERSIZE + src.len()) +} + +/// The 3 bytes before a block. `size` is that of its content, or for a run what it stands for. +pub(crate) const fn block_header(block_type: BlockType, size: usize, last_block: bool) -> u32 { + last_block as u32 + ((block_type as u32) << 1) + ((size as u32) << 3) +} + +/// `ZSTD_rleCompressBlock` +pub(crate) fn rle_compress_block( + dst: &mut [u8], + byte: u8, + src_size: usize, + last_block: bool, +) -> Result { + if dst.len() < 4 { + return Err(Error::DstSizeTooSmall); + } + write_le24(dst, 0, block_header(BlockType::Rle, src_size, last_block))?; + dst.put(3, byte)?; + Ok(4) +} + +/// `ZSTD_minGain`: how much smaller than `src_size` a compressed block has to be, to be worth it. +#[inline] +pub(crate) fn min_gain(src_size: usize, strat: Strategy) -> usize { + let minlog = if strat >= Strategy::BtUltra { + strat as u32 - 1 + } else { + 6 + }; + (src_size >> minlog) + 2 +} + +// ── matching ────────────────────────────────────────────────────────────── + +/// How many bytes `a` and `b` have in common at their start. +#[inline(always)] +pub(crate) fn common_prefix(a: &[u8], b: &[u8]) -> usize { + let (a_words, _) = a.as_chunks::<8>(); + let (b_words, _) = b.as_chunks::<8>(); + let mut len = 0usize; + for (a_word, b_word) in a_words.iter().zip(b_words) { + let diff = u64::from_le_bytes(*a_word) ^ u64::from_le_bytes(*b_word); + if diff != 0 { + return len + (diff.trailing_zeros() >> 3) as usize; + } + len += 8; + } + let a_tail = a.get(len..).unwrap_or_default(); + let b_tail = b.get(len..).unwrap_or_default(); + len + a_tail + .iter() + .zip(b_tail) + .take_while(|(a_byte, b_byte)| a_byte == b_byte) + .count() +} + +/// Input whose positions fit in 32 bits, which makes a bounds check a single comparison. The hot loops of the match +/// finders read through this. +#[derive(Clone, Copy)] +pub(crate) struct Near<'a>(&'a [u8]); + +/// What [`Near::compare_chunks`] does with the first 16 bytes, where most positions differ. +#[derive(Clone, Copy, PartialEq, Eq, ConstParamTy)] +pub(crate) enum First { + /// The same as with the others. + Alike, + /// They have code of their own. + Apart, + /// That code takes them 8 at a time. On x86-64, HTML and records take 5 to 13% less time where positions are only + /// added to the trees. Where matches are looked for, text takes 1% longer. + Words, +} + +/// A bit for each of the 16 bytes that is not the same in both. A comparison of vectors. +#[inline(always)] +fn differences(bytes: &[u8; 16], match_bytes: &[u8; 16]) -> u32 { + let mut differ = 0u32; + for (n, (byte, match_byte)) in (0..16).zip(bytes.iter().zip(match_bytes)) { + differ |= u32::from(byte != match_byte).wrapping_shl(n); + } + differ +} + +impl<'a> Near<'a> { + #[inline] + pub(crate) fn new(bytes: &'a [u8]) -> Result { + if u32::try_from(bytes.len()).is_err() { + return Err(BUG); + } + Ok(Self(bytes)) + } + + #[inline(always)] + pub(crate) const fn bytes(self) -> &'a [u8] { + self.0 + } + + #[inline(always)] + pub(crate) const fn len(self) -> u32 { + self.0.len() as u32 + } + + #[inline(always)] + pub(crate) fn read32(self, pos: u32) -> Option { + let pos = pos as usize; + let bytes = self.0.get(pos..pos + 4)?.first_chunk()?; + Some(u32::from_le_bytes(*bytes)) + } + + /// [`Self::read32`] at a position that may not fit in 32 bits, but is far from the most a `usize` can be. + #[inline(always)] + pub(crate) fn read32_at(self, pos: usize) -> Option { + let bytes = self.0.get(pos..pos + 4)?.first_chunk()?; + Some(u32::from_le_bytes(*bytes)) + } + + /// What there is from `start` on. + #[inline(always)] + pub(crate) fn tail(self, start: u32) -> Option { + self.0.get(start as usize..).map(Self) + } + + #[inline(always)] + pub(crate) fn read64(self, pos: u32) -> Option { + let pos = pos as usize; + let bytes = self.0.get(pos..pos + 8)?.first_chunk()?; + Some(u64::from_le_bytes(*bytes)) + } + + /// Whether the bits of `MASK` are the same in `bytes` and in the 8 bytes at `pos`. If `pos` is too far to read them, + /// that is only found out once other bytes turned out to be the same. + #[inline(always)] + pub(crate) fn has(self, pos: u32, bytes: u64) -> bool { + let Some(last) = self.0.len().checked_sub(8) else { + return false; + }; + let pos = pos as usize; + let at = pos.min(last); + let Some(there) = self.0.get(at..at + 8).and_then(<[u8]>::first_chunk) else { + return false; + }; + if (u64::from_le_bytes(*there) ^ bytes) & MASK != 0 { + return false; + } + // Costs nothing, and keeps the compiler from asking the second question first. + compiler_fence(Ordering::SeqCst); + pos <= last + } + + #[inline(always)] + fn chunk16(self, pos: u32) -> Option<&'a [u8; 16]> { + self.window(pos) + } + + /// The `N` bytes at `pos`, which can then be read with no further check. + #[inline(always)] + pub(crate) fn window(self, pos: u32) -> Option<&'a [u8; N]> { + let pos = pos as usize; + self.0.get(pos..pos + N)?.first_chunk() + } + + #[inline(always)] + pub(crate) fn read8(self, pos: u32) -> Option { + self.0.get(pos as usize).copied() + } + + /// `ZSTD_count`, up to the end. `match_pos` is less than `pos`. + #[inline(always)] + pub(crate) fn count(self, pos: u32, match_pos: u32) -> u32 { + self.count_in(pos, self, match_pos) + } + + /// How long what is at `pos` is the same as what is at `other_pos` in `other`, up to the end of either. + #[inline(always)] + pub(crate) fn count_in(self, pos: u32, other: Near<'_>, other_pos: u32) -> u32 { + let mut len = 0; + // Where 16 bytes differ takes no branch, and few matches are longer: what a loop has to prepare is left for those. + if let (Some(bytes), Some(other_bytes)) = (self.chunk16(pos), other.chunk16(other_pos)) { + let differ = differences(bytes, other_bytes); + if differ != 0 { + return differ.trailing_zeros(); + } + len = 16; + } + while let (Some(bytes), Some(other_bytes)) = + (self.chunk16(pos + len), other.chunk16(other_pos + len)) + { + let differ = differences(bytes, other_bytes); + if differ != 0 { + return len + differ.trailing_zeros(); + } + len += 16; + if len == 64 { + len = self.same_blocks(pos, other, other_pos, len); + } + } + while let Some(byte) = self.read8(pos + len) + && other.read8(other_pos + len) == Some(byte) + { + len += 1; + } + len + } + + /// `len`, which is how many bytes are known to be the same at `pos` and at `other_pos` in `other`, plus the blocks of + /// 64 bytes after them that are the same too. For the matches that are long. + #[inline(never)] + fn same_blocks(self, pos: u32, other: Near<'_>, other_pos: u32, mut len: u32) -> u32 { + while let Some(block) = self.window::<64>(pos + len) + && let Some(other_block) = other.window::<64>(other_pos + len) + { + // A vector. + let mut differ = [0u8; 16]; + let (chunks, other_chunks) = + (block.as_chunks::<16>().0, other_block.as_chunks::<16>().0); + for (bytes, other_bytes) in chunks.iter().zip(other_chunks) { + for ((differ, byte), other_byte) in differ.iter_mut().zip(bytes).zip(other_bytes) { + *differ |= byte ^ other_byte; + } + } + if differ != [0; 16] { + break; + } + len += 64; + } + len + } + + /// [`Self::count`] of positions that are known to have `from` bytes in common, and which one is before the other in + /// the order of what follows them. Only the sum of `match_pos` and `from` has to be a position. + #[inline(always)] + pub(crate) fn compare(self, pos: u32, match_pos: u32, from: u32) -> Common { + let mut len = match self.compare_chunks::<{ First::Alike }>(pos, match_pos, from) { + Ok(common) => return common, + Err(len) => len, + }; + while let Some(byte) = self.read8(pos.wrapping_add(len)) + && let Some(match_byte) = self.read8(match_pos.wrapping_add(len)) + { + if byte != match_byte { + return Common::followed_by(len, match_byte, byte); + } + len += 1; + } + Common::up_to_the_end(len) + } + + /// [`Self::compare`], 16 bytes at a time as long as there are as many. Fails with how many bytes are known to be in + /// common when there are not. + #[inline(always)] + pub(crate) fn compare_chunks( + self, + pos: u32, + match_pos: u32, + from: u32, + ) -> core::result::Result { + self.compare_chunks_with::(pos, self, match_pos, from) + } + + /// [`Self::compare_chunks`], for a match that is at `match_pos` in `other`. + #[inline(always)] + pub(crate) fn compare_chunks_with( + self, + pos: u32, + other: Self, + match_pos: u32, + from: u32, + ) -> core::result::Result { + let mut len = from; + let is_apart = match FIRST { + First::Alike => false, + First::Apart | First::Words => true, + }; + if is_apart + && let (Some(bytes), Some(match_bytes)) = ( + self.chunk16(pos.wrapping_add(len)), + other.chunk16(match_pos.wrapping_add(len)), + ) + { + let common = match FIRST { + First::Words => Common::within_words(len, match_bytes, bytes), + First::Alike | First::Apart => Common::within_chunk(len, match_bytes, bytes), + }; + if let Some(common) = common { + return Ok(common); + } + len += 16; + } + while let (Some(bytes), Some(match_bytes)) = ( + self.chunk16(pos.wrapping_add(len)), + other.chunk16(match_pos.wrapping_add(len)), + ) { + let differ = differences(bytes, match_bytes); + if differ != 0 { + let first = differ.trailing_zeros(); + if !is_apart { + return Ok(Common::within(len + first, match_bytes, bytes)); + } + // After 16 bytes that are the same, that takes longer than it saves. + let byte = bytes.get(first as usize & 15).copied().unwrap_or(0); + let match_byte = match_bytes.get(first as usize & 15).copied().unwrap_or(0); + return Ok(Common::followed_by(len + first, match_byte, byte)); + } + len += 16; + } + Err(len) + } + + /// [`Self::compare`] of a position that starts with `head`, if these bytes are enough to tell. What it reads does not + /// depend on `from`, so it does not wait for the comparison that `from` comes from. + #[inline(always)] + pub(crate) fn compare_head(self, head: &[u8; 16], match_pos: u32, from: u32) -> Option { + let match_bytes = self.chunk16(match_pos)?; + let first = differences(head, match_bytes).trailing_zeros(); + if first >= 16 || first < from { + return None; + } + Some(Common::within(first, match_bytes, head)) + } + + /// A hint that `pos`, which may be anything, is about to be read. + #[inline(always)] + pub(crate) fn prefetch(self, pos: u32) { + prefetch_read(self.0.as_ptr().wrapping_add(pos as usize), Locality::L1); + } + + /// How many of the 16 bytes at `match_pos` are the same as `bytes`, from the first on. It takes no branch. + #[inline(always)] + fn count16(self, bytes: &[u8; 16], match_pos: u32) -> Option { + let differ = differences(bytes, self.chunk16(match_pos)?); + Some((differ | 1 << 16).trailing_zeros()) + } + + /// [`Self::count`], of which only the comparison of the first 16 bytes is inlined. It takes few registers of a loop + /// that has other uses for them. + #[inline(always)] + pub(crate) fn count_short(self, pos: u32, match_pos: u32) -> u32 { + match self + .chunk16(pos) + .and_then(|bytes| self.count16(bytes, match_pos)) + { + None => self.count_outlined(pos, match_pos), + Some(16) => 16 + self.count_outlined(pos + 16, match_pos + 16), + Some(len) => len, + } + } + + #[inline(never)] + fn count_outlined(self, pos: u32, match_pos: u32) -> u32 { + self.count(pos, match_pos) + } + + /// How many bytes before `pos`, down to `anchor`, are the same as those before `match_pos`, down to `match_low`. + #[inline(always)] + pub(crate) fn count_backwards( + self, + pos: u32, + anchor: u32, + match_pos: u32, + match_low: u32, + ) -> u32 { + // Whoever got here without a step is likely to do so again. + if pos == anchor { + return 0; + } + let most = (pos - anchor).min(match_pos - match_low); + self.count_backwards_in(pos, self, match_pos, most) + } + + /// How many bytes before `pos` are the same as those before `other_pos` in `other`, up to `most`. + #[inline(always)] + pub(crate) fn count_backwards_in( + self, + pos: u32, + other: Near<'_>, + other_pos: u32, + most: u32, + ) -> u32 { + // Whether there is anything to compare is as good as random: it takes no branch to find out. + if let (Some(bytes), Some(other_bytes)) = ( + self.read64(pos.wrapping_sub(8)), + other.read64(other_pos.wrapping_sub(8)), + ) { + let same = (bytes ^ other_bytes).leading_zeros() >> 3; + if same < 8 || most <= 8 { + return same.min(most); + } + } + self.count_backwards_bytewise(pos, other, other_pos, most) + } + + #[cold] + fn count_backwards_bytewise(self, pos: u32, other: Near<'_>, other_pos: u32, most: u32) -> u32 { + let mut len = 0; + while len < most + && let Some(byte) = self.read8(pos - len - 1) + && other.read8(other_pos - len - 1) == Some(byte) + { + len += 1; + } + len + } +} + +/// `table`, of `1 << BITS` cells, with that known to the compiler, which then neither has to keep the size around nor +/// to check a hash of `BITS` bits against it. With 0, `table` as it is. +#[inline(always)] +pub(crate) fn table_of_size(table: &mut [u32]) -> Option<&mut [u32]> { + if BITS > 0 { + table.get_mut(..1 << BITS) + } else { + Some(table) + } +} + +/// Has `table[hash]` on its way. +#[inline(always)] +pub(crate) fn table_prefetch(table: &[u32], hash: usize) { + prefetch_read(table.as_ptr().wrapping_add(hash), Locality::L1); +} + +/// What a position and a match for it have in common. +#[derive(Clone, Copy)] +pub(crate) struct Common { + pub(crate) len: u32, + /// Whether the match is before the position in the order of what follows them. `None` if the input ends before + /// they differ. + pub(crate) match_is_smaller: Option, +} + +impl Common { + #[inline(always)] + pub(crate) const fn followed_by(len: u32, match_byte: u8, byte: u8) -> Self { + Self { + len, + match_is_smaller: Some(match_byte < byte), + } + } + + /// [`Self::followed_by`] for bytes that differ somewhere. Which ones are smaller does not wait for where they differ. + #[inline(always)] + pub(crate) fn within(len: u32, match_bytes: &[u8; 16], bytes: &[u8; 16]) -> Self { + debug_assert!(match_bytes != bytes); + Self { + len, + match_is_smaller: Some(u128::from_be_bytes(*match_bytes) < u128::from_be_bytes(*bytes)), + } + } + + /// [`Self::within`] after `from` bytes in common, unless the bytes are the same. + #[inline(always)] + fn within_chunk(from: u32, match_bytes: &[u8; 16], bytes: &[u8; 16]) -> Option { + let differ = differences(bytes, match_bytes); + (differ != 0).then(|| Self::within(from + differ.trailing_zeros(), match_bytes, bytes)) + } + + /// [`Self::within_chunk`], 8 bytes at a time. + #[inline(always)] + fn within_words(from: u32, match_bytes: &[u8; 16], bytes: &[u8; 16]) -> Option { + let (bytes, match_bytes) = ( + u128::from_le_bytes(*bytes), + u128::from_le_bytes(*match_bytes), + ); + for (half, from) in [(0, from), (64, from + 8)] { + let (word, match_word) = ((bytes >> half) as u64, (match_bytes >> half) as u64); + let differ = word ^ match_word; + if differ != 0 { + return Some(Self { + len: from + (differ.trailing_zeros() >> 3), + match_is_smaller: Some(match_word.swap_bytes() < word.swap_bytes()), + }); + } + } + None + } + + #[inline(always)] + pub(crate) const fn up_to_the_end(len: u32) -> Self { + Self { + len, + match_is_smaller: None, + } + } +} + +/// A table of binary trees of positions, in the order of what follows them: for each position, the roots of the smaller +/// ones and of the larger ones below it. Its size is a power of 2. +pub(crate) type Nodes = [[u32; 2]]; + +/// `chainTable`, as the strategies that use binary trees see it. +#[inline(always)] +pub(crate) fn bt_nodes(chain_table: &mut [u32]) -> &mut Nodes { + let (nodes, _) = chain_table.as_chunks_mut(); + nodes +} + +/// What is below `index`, the smaller ones first. +#[inline(always)] +pub(crate) fn bt_children(bt: &Nodes, index: u32) -> [u32; 2] { + let node = bt.get(index as usize & bt.len().wrapping_sub(1)); + node.copied().unwrap_or_default() +} + +/// Sets what [`bt_children`] returns. +#[inline(always)] +pub(crate) fn bt_set_children(bt: &mut Nodes, index: u32, children: [u32; 2]) { + let mask = bt.len().wrapping_sub(1); + if let Some(node) = bt.get_mut(index as usize & mask) { + *node = children; + } +} + +/// A hint that [`bt_children`] of `index` is about to be read. +#[inline(always)] +pub(crate) fn bt_prefetch(bt: &Nodes, index: u32) { + let node = index as usize & bt.len().wrapping_sub(1); + prefetch_read(bt.as_ptr().wrapping_add(node), Locality::L1); +} + +/// The way down a tree for a position that becomes its root: what is met on the way goes to its left or to its right. +pub(crate) struct Descent { + /// The positions from there down have been overwritten in the table. + bt_low: u32, + /// Where the next smaller position goes, as twice the position that it goes below, plus 1 for its larger side. And + /// the next larger one. + smaller_slot: usize, + larger_slot: usize, + /// What all the smaller positions from here on are sure to have in common with the new root, and the larger ones. + common_smaller: u32, + common_larger: u32, +} + +impl Descent { + #[inline(always)] + pub(crate) const fn new(bt: &Nodes, curr: u32) -> Self { + Self::down_to(curr, bt_low(bt, curr)) + } + + /// [`Self::new`], which does not go below the positions from `bt_low` down. + #[inline(always)] + pub(crate) const fn down_to(curr: u32, bt_low: u32) -> Self { + let slot = (curr as usize) << 1; + Self { + bt_low, + smaller_slot: slot, + larger_slot: slot + 1, + common_smaller: 0, + common_larger: 0, + } + } + + /// What the next position is sure to have in common with the new root. + #[inline(always)] + pub(crate) fn common(&self) -> u32 { + self.common_smaller.min(self.common_larger) + } + + /// Goes past `match_index`, which has `len` bytes in common with the new root and `children` below it. Returns the + /// next position, if the table still has it. + #[inline(always)] + pub(crate) fn step( + &mut self, + bt: &mut Nodes, + match_index: u32, + children: [u32; 2], + len: u32, + match_is_smaller: bool, + ) -> Option { + let slots = bt.as_flattened_mut(); + let slot = (match_index as usize) << 1; + let [smaller_child, larger_child] = children; + if match_is_smaller { + table_put(slots, self.smaller_slot, match_index); + self.common_smaller = len; + if match_index <= self.bt_low { + // This slot keeps its value: only the other one is cleared. + self.smaller_slot = self.larger_slot; + return None; + } + // What is larger than it may be smaller than the new root. + self.smaller_slot = slot + 1; + Some(larger_child) + } else { + table_put(slots, self.larger_slot, match_index); + self.common_larger = len; + if match_index <= self.bt_low { + self.larger_slot = self.smaller_slot; + return None; + } + self.larger_slot = slot; + Some(smaller_child) + } + } + + /// Nothing more goes to the left or to the right of the new root. + #[inline(always)] + pub(crate) fn end(&self, bt: &mut Nodes) { + let slots = bt.as_flattened_mut(); + table_put(slots, self.smaller_slot, 0); + table_put(slots, self.larger_slot, 0); + } +} + +/// The positions from there down have been overwritten in the table when `curr` is added. +#[inline(always)] +pub(crate) const fn bt_low(bt: &Nodes, curr: u32) -> u32 { + curr.saturating_sub(bt.len().wrapping_sub(1) as u32) +} + +/// What the match finders that use binary trees read. +#[derive(Clone, Copy)] +pub(crate) struct TreeInput<'a> { + /// The current segment, up to where matches end. A position in it is an index minus `dict_limit`. + pub(crate) near: Near<'a>, + pub(crate) dict_limit: u32, + /// The older segment, with which its buffer ends. + dict: Prefix<'a>, + /// Whether there is enough of it for hints at what is about to be read to be worth giving. + pub(crate) far: bool, +} + +impl<'a> TreeInput<'a> { + /// `prefix` is the current segment, which starts at the index `dict_limit`, and `dict` the older one. + pub(crate) fn new( + prefix: Prefix<'a>, + dict: Prefix<'a>, + dict_limit: u32, + iend: usize, + ) -> Result { + let near = Near::new(prefix.buf.sub(prefix.pos_of(dict_limit)..iend)?)?; + let older = dict.buf.get(..dict.pos_of(dict_limit)); + let dict = Prefix { + buf: older.unwrap_or_default(), + base: dict.base, + }; + Ok(Self { + near, + dict_limit, + dict, + far: Self::is_far(near), + }) + } + + /// The older segment, with which its buffer ends. + #[inline(always)] + pub(crate) const fn dict(&self) -> Prefix<'a> { + self.dict + } + + /// What `far` is for a current segment. + #[inline(always)] + pub(crate) const fn is_far(near: Near<'_>) -> bool { + near.len() > 1 << 16 + } + + /// `ZSTD_count_2segments`, for the position `ip` of `near` and a match that starts in another segment. + #[inline(always)] + pub(crate) fn count_2segments( + &self, + ip: u32, + match_buf: &[u8], + match_pos: usize, + match_end: usize, + ) -> Result { + let buf = self.near.bytes(); + let len = count_2segments( + buf, + ip as usize, + match_buf, + match_pos, + buf.len(), + match_end, + 0, + )?; + Ok(len as u32) + } + + /// [`Self::count_2segments`] for a match at `match_index`, in the older segment. + #[inline(always)] + pub(crate) fn count_ext_dict(&self, ip: u32, match_index: u32) -> Result { + self.count_2segments( + ip, + self.dict.buf, + self.dict.pos_of(match_index), + self.dict.pos_of(self.dict_limit), + ) + } + + /// Whether a match at `match_index` is in the current segment once it is `len` long. + #[inline(always)] + fn is_current(&self, match_index: u32, len: u32) -> bool { + u64::from(match_index) + u64::from(len) >= u64::from(self.dict_limit) + } + + /// [`Near::compare_chunks`] for the next 16 bytes alone, if the match has them in the older segment. In words, + /// whatever the walk does in the current segment. + #[inline(always)] + fn compare_older( + &self, + ip: u32, + match_index: u32, + from: u32, + ) -> core::result::Result { + let pos = self.dict.pos_of(match_index).wrapping_add(from as usize); + let (Some(bytes), Some(match_bytes)) = ( + self.near.chunk16(ip.wrapping_add(from)), + (self.dict.buf.get(pos..pos.wrapping_add(16))).and_then(<[u8]>::first_chunk), + ) else { + return Err(from); + }; + Common::within_words(from, match_bytes, bytes).ok_or(from + 16) + } + + /// [`Near::compare`] wherever the match is, for what [`Near::compare_chunks`] leaves. Small, for a cheap call. + #[cold] + #[inline(never)] + fn compare_rest(self, ip: u32, match_index: u32, from: u32) -> Result { + match self.compare_older(ip, match_index, from) { + Ok(common) => Ok(common), + Err(from) => self.compare_to_the_end(ip, match_index, from), + } + } + + /// [`Self::compare_rest`], however long the match is. + #[inline(never)] + fn compare_to_the_end(&self, ip: u32, match_index: u32, from: u32) -> Result { + // A position of the older segment, as one of the current segment, only makes sense with a length that brings + // it there. + if self.is_current(match_index, from) { + let match_pos = match_index.wrapping_sub(self.dict_limit); + return Ok(self.near.compare(ip, match_pos, from)); + } + + let len = from + self.count_ext_dict(ip + from, match_index.wrapping_add(from))?; + let Some(byte) = self.near.read8(ip.wrapping_add(len)) else { + return Ok(Common::up_to_the_end(len)); + }; + let differs = match_index.wrapping_add(len); + let match_byte = if differs < self.dict_limit { + self.dict.buf.at(self.dict.pos_of(differs))? + } else { + let pos = differs.wrapping_sub(self.dict_limit); + self.near.read8(pos).ok_or(BUG)? + }; + Ok(Common::followed_by(len, match_byte, byte)) + } +} + +/// Hints at what the ways down the trees start with for the positions after `ip`, which are likely to be the next: the +/// cell of the hash table for the one 8 bytes ahead, the root that it has for the one `ROOT` bytes ahead, and what is +/// below the root for the one `BELOW` bytes ahead. +#[inline(always)] +pub(crate) fn bt_look_ahead( + hash_table: &[u32], + bt: &Nodes, + input: &TreeInput<'_>, + hasher: Hasher, + ip: u32, +) { + let near = input.near; + if !input.far { + return; + } + let Some(bytes) = near.chunk16(ip) else { + return; + }; + let bytes = u128::from_le_bytes(*bytes); + let hash_of = |ahead: u32| hasher.hash((bytes >> (8 * ahead)) as u64); + prefetch_read(hash_table.as_ptr().wrapping_add(hash_of(8)), Locality::L1); + let root = table_get(hash_table, hash_of(ROOT)); + bt_prefetch(bt, root); + near.prefetch(root.wrapping_sub(input.dict_limit)); + for child in bt_children(bt, table_get(hash_table, hash_of(BELOW))) { + bt_prefetch(bt, child); + near.prefetch(child.wrapping_sub(input.dict_limit)); + } +} + +/// What each step of [`bt_walk`] tells, where `input.far`, of what the next one may read. Not [`bt_look_ahead`]. +#[derive(Clone, Copy, PartialEq, Eq, ConstParamTy)] +pub(crate) enum Hints { + /// The nodes and the bytes. + All, + Bytes, + None, +} + +/// What [`bt_walk`] does with a position of the older segment. With a dictionary of 64 KB there, records take 12 to 19% +/// less time without the call, HTML and JSON 6%. +#[derive(Clone, Copy, PartialEq, Eq, ConstParamTy)] +pub(crate) enum Older { + /// A call, once the position is found not to be in the current segment. + Called, + /// The same without a call. + TriedLast, + /// Before the current segment is tried, which takes a cycle for a position that is there. + TriedFirst, +} + +/// Goes down the tree whose root is `root` for the position `ip` of `input.near`, which `descent` is for and which +/// becomes the root. `visit` is called with each position that is met on the way, which are all at least `low` and at +/// most `nb_compares`, and what it has in common with `ip`, and returns whether to go on. Returns how many more positions +/// could have been compared. With [`First::Alike`], [`Near::compare_head`] is tried before [`Near::compare_chunks`], which +/// is for ways down that are short. `older` is a constant. +#[inline(always)] +pub(crate) fn bt_walk( + bt: &mut Nodes, + input: &TreeInput<'_>, + ip: u32, + root: u32, + low: u32, + mut nb_compares: u32, + mut descent: Descent, + older: Older, + mut visit: impl FnMut(u32, Common) -> Result, +) -> Result { + if bt.is_empty() { + return Err(BUG); + } + let near = input.near; + let dict_limit = input.dict_limit; + let by_head = match FIRST { + First::Alike => true, + First::Apart | First::Words => false, + }; + let head = near.chunk16(ip).filter(|_| by_head); + let mut match_index = root; + while nb_compares != 0 && match_index >= low { + let children = bt_children(bt, match_index); + let from = descent.common(); + // Whichever way it goes, the next step does not wait as long for memory. + if input.far { + for child in children { + if matches!(HINTS, Hints::All) { + bt_prefetch(bt, child); + } + if matches!(HINTS, Hints::All | Hints::Bytes) { + near.prefetch(child.wrapping_sub(dict_limit).wrapping_add(from)); + } + } + } + let match_pos = match_index.wrapping_sub(dict_limit); + let common = match head.and_then(|head| near.compare_head(head, match_pos, from)) { + Some(common) => common, + None => { + let at_once = matches!(older, Older::TriedFirst) && match_index < dict_limit; + let by_chunks = if at_once { + Err(from) + } else { + near.compare_chunks::(ip, match_pos, from) + }; + match by_chunks { + Ok(common) => common, + Err(from) if at_once || matches!(older, Older::TriedLast) => { + // As a function that `compare_rest` calls too, it takes 1 to 2% more instructions. + match input.compare_older(ip, match_index, from) { + Ok(common) => common, + Err(from) => input.compare_to_the_end(ip, match_index, from)?, + } + } + Err(from) => input.compare_rest(ip, match_index, from)?, + } + } + }; + if !visit(match_index, common)? { + break; + } + let match_is_smaller = common.match_is_smaller.ok_or(BUG)?; + let Some(next) = descent.step(bt, match_index, children, common.len, match_is_smaller) + else { + break; + }; + match_index = next; + nb_compares -= 1; + } + descent.end(bt); + Ok(nb_compares) +} + +/// The tree of a dictionary that a search goes on in. It is only read. +#[derive(Clone, Copy)] +pub(crate) struct DictTree<'a> { + dms: DictMatchState<'a>, + bt: &'a Nodes, + /// The content of the dictionary. A position in it is an index minus `low_limit`. + content: Near<'a>, + low_limit: u32, + high_limit: u32, + bt_low: u32, + /// What to add to an index of the dictionary for an index of the window. + pub(crate) index_delta: u32, +} + +impl<'a> DictTree<'a> { + /// `window_low` is the index of the window that the end of the dictionary is at. + pub(crate) fn new(dms: DictMatchState<'a>, window_low: u32) -> Result { + let (bt, _) = dms.ms.chain_table.as_chunks(); + if bt.len() != 1usize.wrapping_shl(dms.ms.c_params.chain_log.wrapping_sub(1)) { + return Err(BUG); + } + let bt_mask = bt.len().wrapping_sub(1) as u32; + let low_limit = dms.ms.window.low_limit; + let high_limit = dms.end.wrapping_sub(dms.base) as u32; + Ok(Self { + dms, + bt, + content: Near::new(dms.buf.sub(dms.pos_of(low_limit)..dms.end)?)?, + low_limit, + high_limit, + bt_low: if bt_mask < high_limit.wrapping_sub(low_limit) { + high_limit.wrapping_sub(bt_mask) + } else { + low_limit + }, + index_delta: window_low.wrapping_sub(high_limit), + }) + } + + /// The root of the tree of `dms` for `hash`, or 0 if there is none, and a hint that the way down is about to start. + #[inline(always)] + pub(crate) fn root(dms: DictMatchState<'_>, hash: usize) -> Result { + let root = dms.ms.hash_table.at(hash)?; + if root <= dms.ms.window.low_limit { + return Ok(0); + } + let (bt, _) = dms.ms.chain_table.as_chunks(); + bt_prefetch(bt, root); + prefetch_read( + dms.buf.as_ptr().wrapping_add(dms.pos_of(root)), + Locality::L1, + ); + Ok(root) + } + + /// [`Self::root`] with the hint before it is known whether there is a root, which is as good as random where there + /// are few matches. + #[inline(always)] + pub(crate) fn root_at_once(dms: DictMatchState<'_>, hash: usize) -> Result { + let root = dms.ms.hash_table.at(hash)?; + let (bt, _) = dms.ms.chain_table.as_chunks(); + bt_prefetch(bt, root); + prefetch_read( + dms.buf.as_ptr().wrapping_add(dms.pos_of(root)), + Locality::L1, + ); + let is_root = root > dms.ms.window.low_limit; + Ok(if is_root { root } else { 0 }) + } + + /// [`Near::compare`] for a match in the dictionary, for what [`Near::compare_chunks_with`] leaves. A match that + /// reaches the end of the dictionary goes on at the start of the current segment. + #[cold] + #[inline(never)] + fn compare_rest( + self, + input: &TreeInput<'_>, + ip: u32, + dict_match_index: u32, + from: u32, + ) -> Result { + let dms = &self.dms; + let match_pos = dms.pos_of(dict_match_index.wrapping_add(from)); + let len = from + input.count_2segments(ip + from, dms.buf, match_pos, dms.end)?; + let Some(byte) = input.near.read8(ip.wrapping_add(len)) else { + return Ok(Common::up_to_the_end(len)); + }; + let differs = dict_match_index.wrapping_add(len); + let match_byte = if differs >= self.high_limit { + let index = differs.wrapping_add(self.index_delta); + let pos = index.wrapping_sub(input.dict_limit); + input.near.read8(pos).ok_or(BUG)? + } else { + dms.buf.at(dms.pos_of(differs))? + }; + Ok(Common::followed_by(len, match_byte, byte)) + } + + /// Goes down the tree for the position `ip` of `input.near`, from what [`Self::root`] returns for it. `visit` is called with each position that is met on the way, at most `nb_compares` of them, and what it has in + /// common with `ip`, and returns whether to go on. + #[inline(always)] + pub(crate) fn walk( + &self, + input: &TreeInput<'_>, + ip: u32, + root: u32, + mut nb_compares: u32, + mut visit: impl FnMut(u32, Common) -> Result, + ) -> Result<()> { + let mut dict_match_index = root; + let mut common_smaller = 0u32; + let mut common_larger = 0u32; + while nb_compares != 0 && dict_match_index > self.low_limit { + let [smaller_child, larger_child] = bt_children(self.bt, dict_match_index); + let from = common_smaller.min(common_larger); + // Whichever way it goes, the next step does not wait as long for memory. + for child in [smaller_child, larger_child] { + bt_prefetch(self.bt, child); + let pos = child.wrapping_sub(self.low_limit); + self.content.prefetch(pos.wrapping_add(from)); + } + let match_pos = dict_match_index.wrapping_sub(self.low_limit); + let chunks = input.near.compare_chunks_with::<{ First::Alike }>( + ip, + self.content, + match_pos, + from, + ); + let common = match chunks { + Ok(common) => common, + Err(from) => self.compare_rest(input, ip, dict_match_index, from)?, + }; + if !visit(dict_match_index, common)? || dict_match_index <= self.bt_low { + break; + } + if common.match_is_smaller.ok_or(BUG)? { + common_smaller = common.len; + dict_match_index = larger_child; + } else { + common_larger = common.len; + dict_match_index = smaller_child; + } + nb_compares -= 1; + } + Ok(()) + } +} + +/// `table[hash]`. The size of `table` is a power of 2 that `hash` is less than, which the mask tells the compiler. +#[inline(always)] +pub(crate) fn table_get(table: &[u32], hash: usize) -> u32 { + let cell = table.get(hash & table.len().wrapping_sub(1)); + cell.copied().unwrap_or(0) +} + +/// `table[hash] = index`, likewise. +#[inline(always)] +pub(crate) fn table_put(table: &mut [u32], hash: usize, index: u32) { + let mask = table.len().wrapping_sub(1); + if let Some(cell) = table.get_mut(hash & mask) { + *cell = index; + } +} + +/// `table[hash]`. `hash` is less than the size of `table`: to check that delays nothing, which a mask would. +#[inline(always)] +pub(crate) fn hashed_get(table: &[u32], hash: usize) -> u32 { + table.get(hash).copied().unwrap_or(0) +} + +/// `table[hash] = index`, likewise. +#[inline(always)] +pub(crate) fn hashed_put(table: &mut [u32], hash: usize, index: u32) { + if let Some(cell) = table.get_mut(hash) { + *cell = index; + } +} + +/// `table[hash] = index`, likewise. Returns what was there. +#[inline(always)] +pub(crate) fn hashed_swap(table: &mut [u32], hash: usize, index: u32) -> u32 { + match table.get_mut(hash) { + Some(cell) => core::mem::replace(cell, index), + None => 0, + } +} + +/// The 4 bytes of `window` from `AT` on. +#[inline(always)] +pub(crate) fn le32_at(window: &[u8; N]) -> u32 { + let bytes = window.get(AT..).and_then(<[u8]>::first_chunk); + bytes.map_or(0, |bytes| u32::from_le_bytes(*bytes)) +} + +/// The 8 bytes of `window` from `AT` on. +#[inline(always)] +pub(crate) fn le64_at(window: &[u8; N]) -> u64 { + let bytes = window.get(AT..).and_then(<[u8]>::first_chunk); + bytes.map_or(0, |bytes| u64::from_le_bytes(*bytes)) +} + +/// `ZSTD_count`: how long `buf` from `p_in` on, up to `p_in_limit`, is the same as `match_buf` from `p_match` on. +#[inline(always)] +pub(crate) fn count( + buf: &[u8], + p_in: usize, + match_buf: &[u8], + p_match: usize, + p_in_limit: usize, +) -> Result { + Ok(common_prefix( + buf.sub(p_in..p_in_limit)?, + match_buf.sub(p_match..)?, + )) +} + +/// `ZSTD_count_2segments`: the match starts in `match_buf`, which it may reach the end `m_end` of. It then goes on at +/// `i_start` in `buf`. +#[inline(always)] +pub(crate) fn count_2segments( + buf: &[u8], + ip: usize, + match_buf: &[u8], + match_pos: usize, + i_end: usize, + m_end: usize, + i_start: usize, +) -> Result { + let match_length = common_prefix(buf.sub(ip..i_end)?, match_buf.sub(match_pos..m_end)?); + if match_pos + match_length != m_end { + return Ok(match_length); + } + Ok(match_length + count(buf, ip + match_length, buf, i_start, i_end)?) +} + +const PRIME_3_BYTES: u32 = 506_832_829; +const PRIME_4_BYTES: u32 = 2_654_435_761; +const PRIME_5_BYTES: u64 = 889_523_592_379; +const PRIME_6_BYTES: u64 = 227_718_039_650_203; +const PRIME_7_BYTES: u64 = 58_295_818_150_454_627; +const PRIME_8_BYTES: u64 = 0xCF1B_BCDC_B7A5_6463; + +/// `ZSTD_hash3Ptr` of the position that the 4 bytes `bytes` were read at. `h` is from 1 to 32, as for all these. +#[inline(always)] +pub(crate) const fn hash3_of(bytes: u32, h: u32) -> usize { + ((bytes << (32 - 24)).wrapping_mul(PRIME_3_BYTES) >> ((32 - h) & 31)) as usize +} + +/// `ZSTD_hashPtr`, out of the 8 bytes at the position: a hash of `h_bits` bits of the first `mls` of them, or 4 if that is +/// less. +#[derive(Clone, Copy)] +pub(crate) struct Hasher { + /// Takes the bytes that do not count out, on the left. + factor: u64, + shift: u32, +} + +impl Hasher { + pub(crate) const fn new(h_bits: u32, mls: u32) -> Self { + let factor = match mls { + 5 => PRIME_5_BYTES << (64 - 40), + 6 => PRIME_6_BYTES << (64 - 48), + 7 => PRIME_7_BYTES << (64 - 56), + 8 => PRIME_8_BYTES, + _ => (PRIME_4_BYTES as u64) << 32, + }; + Self { factor, shift: 0 }.with_log(h_bits) + } + + /// The same for a table of another size. + pub(crate) const fn with_log(self, h_bits: u32) -> Self { + Self { + shift: 64u32.wrapping_sub(h_bits) & 63, + ..self + } + } + + /// The same for what [`table_of_size`] gives. + #[inline(always)] + pub(crate) const fn of_size(self) -> Self { + if BITS > 0 { self.with_log(BITS) } else { self } + } + + /// The same of `mls` bytes. + pub(crate) const fn with_mls(self, mls: u32) -> Self { + Self { + shift: self.shift, + ..Self::new(0, mls) + } + } + + /// The same from the first 4 of the 8 bytes, if no more than these count. + pub(crate) const fn of_4_bytes(self) -> Option { + let h_bits = 64 - self.shift; + if self.factor == (PRIME_4_BYTES as u64) << 32 && h_bits <= 32 { + Some(Hash4 { h_bits }) + } else { + None + } + } + + /// The hash of the position that the 8 bytes `bytes` were read at. + #[inline(always)] + pub(crate) const fn hash(self, bytes: u64) -> usize { + (bytes.wrapping_mul(self.factor) >> self.shift) as usize + } +} + +/// What a [`Hasher`] gives for 4 bytes. To read no more than these crosses fewer lines of the cache than a [`Hasher`] does. +#[derive(Clone, Copy)] +pub(crate) struct Hash4 { + h_bits: u32, +} + +impl Hash4 { + #[inline(always)] + pub(crate) const fn hash(self, bytes: u32) -> usize { + (bytes.wrapping_mul(PRIME_4_BYTES) >> ((32 - self.h_bits) & 31)) as usize + } +} + +/// `ZSTD_hashPtrSalted` of 4 to 6 bytes, out of the 8 bytes at the position: the multiplication also does the shift that +/// leaves the other bytes out. +#[derive(Clone, Copy)] +pub(crate) struct Hash8 { + mul: u64, + salt: u64, + shift: u32, +} + +impl Hash8 { + /// `mls` is 4 if it is not 5 or 6, and `h_bits` is from 1 to 32. + pub(crate) const fn new(mls: u32, h_bits: u32, hash_salt: u64) -> Self { + let shift = 64 - h_bits; + match mls { + 5 => Self { + mul: PRIME_5_BYTES << (64 - 40), + salt: hash_salt, + shift, + }, + 6 => Self { + mul: PRIME_6_BYTES << (64 - 48), + salt: hash_salt, + shift, + }, + _ => Self { + mul: (PRIME_4_BYTES as u64) << 32, + salt: hash_salt << 32, + shift, + }, + } + } + + #[inline(always)] + pub(crate) const fn of(self, bytes: u64) -> u32 { + (bytes.wrapping_mul(self.mul) ^ self.salt).wrapping_shr(self.shift) as u32 + } +} + +// ── the window ──────────────────────────────────────────────────────────── + +/// The C code tells where a segment of the window lies by its address, and does arithmetic on addresses that are +/// outside of any buffer. Here each buffer the compressor reads from has a range of made-up addresses, far from the +/// others, and an address is turned into a position in the buffer only to read. +pub(crate) type Addr = usize; + +/// The buffers that a window can have a segment in. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum BufId { + /// Where an empty window points. + None = 0, + /// The context's own buffer of input. + InBuff = 1, + /// What the caller gave to the call in progress. + User = 2, + /// The content of the dictionary, or the prefix. + Dict = 3, +} + +impl BufId { + /// The address of the first byte of the buffer. + #[inline(always)] + pub(crate) const fn start(self) -> Addr { + (self as usize + 1) << 44 + } +} + +/// The buffers themselves, for as long as a call lasts. +#[derive(Clone, Copy)] +pub(crate) struct Buffers<'a> { + pub(crate) in_buff: &'a [u8], + pub(crate) user: &'a [u8], + pub(crate) dict: &'a [u8], +} + +impl<'a> Buffers<'a> { + #[inline] + pub(crate) fn get(&self, id: BufId) -> &'a [u8] { + match id { + BufId::None => &[], + BufId::InBuff => self.in_buff, + BufId::User => self.user, + BufId::Dict => self.dict, + } + } +} + +/// Indices start here, so that 0 and [`ZSTD_DUBT_UNSORTED_MARK`] are never the index of a position. +pub(crate) const ZSTD_WINDOW_START_INDEX: u32 = 2; + +/// The index past which indices are brought down. +pub(crate) const ZSTD_CURRENT_MAX: u32 = 3500 << 20; +/// The most input that can be indexed at once. +pub(crate) const ZSTD_CHUNKSIZE_MAX: u32 = u32::MAX - ZSTD_CURRENT_MAX; + +/// `ZSTD_window_t`. A position is known by its index, which is its address minus `base`, or minus `dict_base` if the +/// index is less than `dict_limit`. +#[derive(Clone, Copy, Debug)] +pub(crate) struct Window { + /// Where input has to continue to be contiguous. + pub(crate) next_src: Addr, + pub(crate) base: Addr, + pub(crate) dict_base: Addr, + /// The buffer of the segment that starts at `dict_limit`. + pub(crate) base_buf: BufId, + /// The buffer of the segment from `low_limit` to `dict_limit`. + pub(crate) dict_buf: BufId, + pub(crate) dict_limit: u32, + /// Below this, there is nothing valid. + pub(crate) low_limit: u32, + pub(crate) nb_overflow_corrections: u32, +} + +impl Window { + /// `ZSTD_window_init` + pub(crate) const fn new() -> Self { + let base = BufId::None.start(); + Self { + next_src: base + ZSTD_WINDOW_START_INDEX as usize, + base, + dict_base: base, + base_buf: BufId::None, + dict_buf: BufId::None, + dict_limit: ZSTD_WINDOW_START_INDEX, + low_limit: ZSTD_WINDOW_START_INDEX, + nb_overflow_corrections: 0, + } + } + + /// The index of `addr`, which is in the current segment or at its end. + #[inline(always)] + pub(crate) const fn index_of(&self, addr: Addr) -> u32 { + addr.wrapping_sub(self.base) as u32 + } + + /// `ZSTD_window_clear`: forgets all there is, but goes on counting from where it is. + pub(crate) fn clear(&mut self) { + let end = self.index_of(self.next_src); + self.low_limit = end; + self.dict_limit = end; + } + + /// `ZSTD_window_hasExtDict` + #[inline] + pub(crate) fn has_ext_dict(&self) -> bool { + self.low_limit < self.dict_limit + } + + /// `ZSTD_window_needOverflowCorrection` + #[inline] + pub(crate) fn need_overflow_correction(&self, src_end: Addr) -> bool { + self.index_of(src_end) > ZSTD_CURRENT_MAX + } + + /// `ZSTD_window_correctOverflow`: brings the indices down, keeping them the same modulo `1 << cycle_log`. Returns + /// what to subtract from every index that is stored. + pub(crate) fn correct_overflow(&mut self, cycle_log: u32, max_dist: u32, src: Addr) -> u32 { + let cycle_size = 1u32 << cycle_log; + let cycle_mask = cycle_size - 1; + let curr = self.index_of(src); + let current_cycle = curr & cycle_mask; + // Indices must not fall below the start index. + let current_cycle_correction = if current_cycle < ZSTD_WINDOW_START_INDEX { + cycle_size.max(ZSTD_WINDOW_START_INDEX) + } else { + 0 + }; + let new_current = current_cycle + current_cycle_correction + max_dist.max(cycle_size); + let correction = curr.wrapping_sub(new_current); + + self.base = self.base.wrapping_add(correction as usize); + self.dict_base = self.dict_base.wrapping_add(correction as usize); + self.low_limit = if self.low_limit < correction.wrapping_add(ZSTD_WINDOW_START_INDEX) { + ZSTD_WINDOW_START_INDEX + } else { + self.low_limit - correction + }; + self.dict_limit = if self.dict_limit < correction.wrapping_add(ZSTD_WINDOW_START_INDEX) { + ZSTD_WINDOW_START_INDEX + } else { + self.dict_limit - correction + }; + self.nb_overflow_corrections += 1; + correction + } + + /// `ZSTD_window_enforceMaxDist`: nothing farther than `max_dist` from `block_end` stays valid, once the + /// dictionary is that far. Returns whether that is so, and the dictionary is to be forgotten. + pub(crate) fn enforce_max_dist( + &mut self, + block_end: Addr, + max_dist: u32, + loaded_dict_end: u32, + ) -> bool { + let block_end_idx = self.index_of(block_end); + if block_end_idx > max_dist.wrapping_add(loaded_dict_end) { + let new_low_limit = block_end_idx - max_dist; + if self.low_limit < new_low_limit { + self.low_limit = new_low_limit; + } + if self.dict_limit < self.low_limit { + self.dict_limit = self.low_limit; + } + return true; + } + false + } + + /// `ZSTD_checkDictValidity`: whether the dictionary is too far from `block_end` to be referred to, or is not + /// right before the current segment, and is to be forgotten. + pub(crate) fn dict_is_invalid( + &self, + block_end: Addr, + max_dist: u32, + loaded_dict_end: u32, + ) -> bool { + let block_end_idx = self.index_of(block_end); + block_end_idx > loaded_dict_end.wrapping_add(max_dist) || loaded_dict_end != self.dict_limit + } + + /// `ZSTD_window_update`: the input goes on with `src_size` bytes at `src`, which is in `src_buf`. If that is not + /// where it left off, what there was becomes the older segment. Returns whether the input is contiguous. + pub(crate) fn update( + &mut self, + src: Addr, + src_buf: BufId, + src_size: usize, + force_non_contiguous: bool, + ) -> bool { + let mut contiguous = true; + if src_size == 0 { + return contiguous; + } + if src != self.next_src || force_non_contiguous { + let distance_from_base = self.next_src.wrapping_sub(self.base); + self.low_limit = self.dict_limit; + self.dict_limit = distance_from_base as u32; + self.dict_base = self.base; + self.dict_buf = self.base_buf; + self.base = src.wrapping_sub(distance_from_base); + self.base_buf = src_buf; + if self.dict_limit.wrapping_sub(self.low_limit) < HASH_READ_SIZE as u32 { + // Too small to be of use. + self.low_limit = self.dict_limit; + } + contiguous = false; + } + self.next_src = src + src_size; + // Where the input overlaps the older segment, it has overwritten it. + if (src + src_size > self.dict_base.wrapping_add(self.low_limit as usize)) + & (src < self.dict_base.wrapping_add(self.dict_limit as usize)) + { + let high_input_idx = (src + src_size).wrapping_sub(self.dict_base); + self.low_limit = if high_input_idx > self.dict_limit as usize { + self.dict_limit + } else { + high_input_idx as u32 + }; + } + contiguous + } +} + +/// `ZSTD_index_overlap_check`: whether a repeat offset that leads to `rep_index` stays clear of the last 3 bytes +/// before `prefix_lowest_index`, from which 4 bytes cannot be read in one segment. +#[inline(always)] +pub(crate) const fn index_overlap_check(prefix_lowest_index: u32, rep_index: u32) -> bool { + prefix_lowest_index.wrapping_sub(1).wrapping_sub(rep_index) >= 3 +} + +pub(crate) const ZSTD_SHORT_CACHE_TAG_BITS: u32 = 8; +pub(crate) const ZSTD_SHORT_CACHE_TAG_MASK: u32 = (1 << ZSTD_SHORT_CACHE_TAG_BITS) - 1; + +/// `ZSTD_comparePackedTags` +#[inline(always)] +pub(crate) const fn compare_packed_tags(packed_tag1: usize, packed_tag2: usize) -> bool { + (packed_tag1 as u32 & ZSTD_SHORT_CACHE_TAG_MASK) + == (packed_tag2 as u32 & ZSTD_SHORT_CACHE_TAG_MASK) +} + +// ── the state of the match finders ──────────────────────────────────────── + +/// `ZSTD_match_t` +#[derive(Clone, Copy, Default)] +#[repr(C)] +pub(crate) struct Match { + /// As [`SeqDef::off_base`]. + pub(crate) off: u32, + pub(crate) len: u32, +} + +/// `rawSeq`: a sequence found by long distance matching. +#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] +#[repr(C)] +pub(crate) struct RawSeq { + pub(crate) offset: u32, + pub(crate) lit_length: u32, + pub(crate) match_length: u32, +} + +/// `RawSeqStore_t` +#[derive(Clone, Default)] +pub(crate) struct RawSeqStore { + /// `capacity` of them, of which `size` are in use. + pub(crate) seq: Vec, + /// The sequence that is being read. + pub(crate) pos: usize, + /// How many of its bytes, literals first, are read. + pub(crate) pos_in_sequence: usize, + pub(crate) size: usize, + pub(crate) capacity: usize, +} + +/// `ZSTD_optimal_t`, one array per member: how to get to each of the positions that the optimal parser looks at, by a +/// match followed by literals. +#[derive(Default)] +pub(crate) struct PriceTable { + /// The cost of getting to this position. + pub(crate) price: Vec, + pub(crate) off: Vec, + /// `mlen` and `litlen`. + pub(crate) lens: Vec, + pub(crate) rep: Vec<[u32; ZSTD_REP_NUM]>, + /// The price of each length of match. + pub(crate) ml_price: Vec, +} + +impl PriceTable { + /// Makes room for `len` positions. + pub(crate) fn resize(&mut self, len: usize) -> Result<()> { + reset_table(&mut self.price, len, false)?; + reset_table(&mut self.off, len, false)?; + reset_table(&mut self.lens, len, false)?; + reset_table(&mut self.rep, len, false)?; + reset_table(&mut self.ml_price, len, false) + } +} + +/// `optState_t`. `symbolCosts` is in [`BlockSrc`]. +pub(crate) struct OptState { + pub(crate) lit_freq: Vec, + pub(crate) lit_length_freq: Vec, + pub(crate) match_length_freq: Vec, + pub(crate) off_code_freq: Vec, + pub(crate) match_table: Vec, + pub(crate) price_table: PriceTable, + pub(crate) lit_sum: u32, + pub(crate) lit_length_sum: u32, + pub(crate) match_length_sum: u32, + pub(crate) off_code_sum: u32, + pub(crate) literal_compression_mode: ParamSwitch, +} + +impl OptState { + pub(crate) const fn new() -> Self { + Self { + lit_freq: Vec::new(), + lit_length_freq: Vec::new(), + match_length_freq: Vec::new(), + off_code_freq: Vec::new(), + match_table: Vec::new(), + price_table: PriceTable { + price: Vec::new(), + off: Vec::new(), + lens: Vec::new(), + rep: Vec::new(), + ml_price: Vec::new(), + }, + lit_sum: 0, + lit_length_sum: 0, + match_length_sum: 0, + off_code_sum: 0, + literal_compression_mode: ParamSwitch::Auto, + } + } +} + +pub(crate) const ZSTD_ROW_HASH_CACHE_SIZE: usize = 8; + +/// A table of a match finder. +/// +/// It starts where a cache line does, as those of the C code do: a row of the row-based match finder is then in as few +/// cache lines as can be. +/// +/// A frame that needs a shorter one than a frame before it had uses the start of that one. The rest stays as it is, for +/// a frame that needs more again, so that nothing is cleared as long as indices go on. +pub(crate) struct Table { + /// Starts before the table, by what it takes to have such a start, and ends where the longest one so far did. + storage: Vec, + start: usize, + len: usize, +} + +impl Table { + const CACHE_LINE: usize = 64; + + pub(crate) const fn new() -> Self { + Self { + storage: Vec::new(), + start: 0, + len: 0, + } + } + + /// For when indices get lower: the ones behind the part in use would seem valid later. + pub(crate) fn forget_rest(&mut self) { + self.storage.truncate(self.start + self.len); + } + + #[inline] + pub(crate) fn as_slice(&self) -> &[T] { + let table = self.storage.get(self.start..self.start + self.len); + table.unwrap_or_default() + } + + #[inline] + pub(crate) fn as_mut_slice(&mut self) -> &mut [T] { + let table = self.storage.get_mut(self.start..self.start + self.len); + table.unwrap_or_default() + } +} + +impl Table { + /// Gives it the length `len`. What it holds is 0, or if `clean` is false possibly what it held at some time. Returns + /// whether it is new memory. + pub(crate) fn reset(&mut self, len: usize, clean: bool) -> Result { + let end = self.start + len; + if end > self.storage.capacity() { + // First, so that both are not allocated at once. + *self = Self::new(); + let slack = Self::CACHE_LINE / size_of::(); + let storage = try_zeroed::(len + slack)?; + let start = storage.as_ptr().align_offset(Self::CACHE_LINE); + *self = Self { + storage, + start: if start <= slack { start } else { 0 }, + len, + }; + return Ok(true); + } + if clean { + self.storage.truncate(self.start); + } + if self.storage.len() < end { + self.storage.resize(end, T::default()); + } + self.len = len; + Ok(false) + } +} + +impl core::ops::Deref for Table { + type Target = [T]; + + #[inline] + fn deref(&self) -> &[T] { + self.as_slice() + } +} + +impl core::ops::DerefMut for Table { + #[inline] + fn deref_mut(&mut self) -> &mut [T] { + self.as_mut_slice() + } +} + +/// What the second tags of the row-based match finder have been good for, in a frame. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) struct SecondTagStats { + /// How many positions were inserted since it was last asked whether to keep them. + pub(crate) nb_new_positions: u32, + /// How many positions were inserted. + pub(crate) nb_positions: u64, + /// How many candidates they ruled out. + pub(crate) nb_ruled_out: u64, +} + +/// How `MatchState::tag_table` is laid out. See `lazy.rs`. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum RowTags { + /// As in the C code. + Flat, + /// The tags of each row are followed by its second tags. + Second(SecondTagStats), +} + +/// `ZSTD_MatchState_t`. `dictMatchState`, `ldmSeqStore` and `opt.symbolCosts`, which point to what other structures +/// own, are in [`BlockSrc`]. +pub(crate) struct MatchState { + pub(crate) window: Window, + /// The index of the end of the dictionary, as long as it is right before the input and within reach. Else 0. + pub(crate) loaded_dict_end: u32, + /// The index from which the tables are still to be filled. + pub(crate) next_to_update: u32, + /// 0 if there is no table of 3-byte hashes. + pub(crate) hash_log3: u32, + pub(crate) row_hash_log: u32, + /// For the row-based match finder: a byte of hash for each entry of `hash_table`. + pub(crate) tag_table: Table, + pub(crate) row_tags: RowTags, + /// Makes what a previous frame left in `tag_table` meaningless. + pub(crate) hash_salt: u64, + pub(crate) hash_salt_entropy: u32, + pub(crate) hash_table: Table, + pub(crate) hash_table3: Table, + pub(crate) chain_table: Table, + pub(crate) force_non_contiguous: bool, + pub(crate) opt: OptState, + /// `dictMatchState != NULL`: the match state of the dictionary is searched beside this one. + pub(crate) has_dict_match_state: bool, + pub(crate) c_params: CompressionParameters, + /// For a worker: whether its frame is given up. + pub(crate) cancelled: Option>, +} + +impl MatchState { + pub(crate) const fn new() -> Self { + Self { + window: Window::new(), + loaded_dict_end: 0, + next_to_update: 0, + hash_log3: 0, + row_hash_log: 0, + tag_table: Table::new(), + row_tags: RowTags::Flat, + hash_salt: 0, + hash_salt_entropy: 0, + hash_table: Table::new(), + hash_table3: Table::new(), + chain_table: Table::new(), + force_non_contiguous: false, + opt: OptState::new(), + has_dict_match_state: false, + c_params: CompressionParameters { + window_log: 0, + chain_log: 0, + hash_log: 0, + search_log: 0, + min_match: 0, + target_length: 0, + strategy: None, + }, + cancelled: None, + } + } + + /// Fails if there is no use in going on. + pub(crate) fn check_cancelled(&self) -> Result<()> { + match &self.cancelled { + Some(cancelled) if cancelled.load(Ordering::SeqCst) => Err(Error::Generic), + _ => Ok(()), + } + } + + /// How many bits to hash to for `hash_table` when it is a dictionary's, where entries have tags. + pub(crate) const fn tagged_hash_log(&self) -> u32 { + self.c_params.hash_log + ZSTD_SHORT_CACHE_TAG_BITS + } + + /// The same for `chain_table`, as `dfast` uses it. + pub(crate) const fn tagged_chain_log(&self) -> u32 { + self.c_params.chain_log + ZSTD_SHORT_CACHE_TAG_BITS + } + + /// `ZSTD_matchState_dictMode`: `dms` is the match state of the dictionary, if there is one to search. + pub(crate) fn dict_mode(&self, dms: Option<&DictMatchState<'_>>) -> DictMode { + if self.window.has_ext_dict() { + return DictMode::ExtDict; + } + if dms.is_some() { + DictMode::DictMatchState + } else { + DictMode::NoDict + } + } + + /// `ZSTD_getLowestMatchIndex`: the lowest index a match for the position `curr` can be at. + #[inline(always)] + pub(crate) fn get_lowest_match_index(&self, curr: u32, window_log: u32) -> u32 { + let max_distance = 1u32 << window_log; + let lowest_valid = self.window.low_limit; + let within_window = if curr.wrapping_sub(lowest_valid) > max_distance { + curr - max_distance + } else { + lowest_valid + }; + // A dictionary is valid as a whole until it is invalidated as a whole. + if self.loaded_dict_end != 0 { + lowest_valid + } else { + within_window + } + } + + /// `ZSTD_getLowestPrefixIndex`: the same, in the current segment. + #[inline(always)] + pub(crate) fn get_lowest_prefix_index(&self, curr: u32, window_log: u32) -> u32 { + let max_distance = 1u32 << window_log; + let lowest_valid = self.window.dict_limit; + let within_window = if curr.wrapping_sub(lowest_valid) > max_distance { + curr - max_distance + } else { + lowest_valid + }; + if self.loaded_dict_end != 0 { + lowest_valid + } else { + within_window + } + } +} + +/// The match state of a dictionary, and where its indices lead. +#[derive(Clone, Copy)] +pub(crate) struct DictMatchState<'a> { + pub(crate) ms: &'a MatchState, + /// The buffer its window is in, which is one segment. + pub(crate) buf: &'a [u8], + /// `window.base`, as a position in `buf`. It wraps around: only its sum with an index is a position. + pub(crate) base: usize, + /// `window.nextSrc`, as a position in `buf`. + pub(crate) end: usize, +} + +/// The current segment of a window, for the functions that index a dictionary. +#[derive(Clone, Copy)] +pub(crate) struct Prefix<'a> { + /// The buffer that the segment is in. + pub(crate) buf: &'a [u8], + /// `window.base`, as a position in `buf`. It wraps around: only its sum with an index is a position. + pub(crate) base: usize, +} + +impl Prefix<'_> { + /// The index of the position `pos` of `buf`. + #[inline(always)] + pub(crate) const fn index_of(&self, pos: usize) -> u32 { + pos.wrapping_sub(self.base) as u32 + } + + /// The position in `buf` of `index`. + #[inline(always)] + pub(crate) const fn pos_of(&self, index: u32) -> usize { + self.base.wrapping_add(index as usize) + } +} + +/// What a block compressor reads: `src` and `srcSize`, and what the pointers of `ZSTD_MatchState_t` lead to. +#[derive(Clone, Copy)] +pub(crate) struct BlockSrc<'a> { + /// The buffer that the current segment of the window is in, and so the block. + pub(crate) buf: &'a [u8], + /// `window.base`, as a position in `buf`. It wraps around: only its sum with an index is a position. + pub(crate) base: usize, + /// The buffer that the older segment of the window is in. + pub(crate) dict_buf: &'a [u8], + /// `window.dictBase`, as a position in `dict_buf`. It wraps around too. + pub(crate) dict_base: usize, + /// Where the block starts in `buf`. + pub(crate) start: usize, + /// Where it ends. + pub(crate) end: usize, + pub(crate) dms: Option>, + /// `opt.symbolCosts`: the tables of the previous block. + pub(crate) symbol_costs: &'a EntropyCTables, + /// `ldmSeqStore`: matches that the optimal parser is to consider. + pub(crate) ldm_seq_store: Option<&'a RawSeqStore>, +} + +impl<'a> BlockSrc<'a> { + #[inline(always)] + pub(crate) const fn prefix(&self) -> Prefix<'a> { + Prefix { + buf: self.buf, + base: self.base, + } + } + + /// The index of the position `pos` of `buf`. + #[inline(always)] + pub(crate) const fn index_of(&self, pos: usize) -> u32 { + pos.wrapping_sub(self.base) as u32 + } + + /// The position in `buf` of `index`, which is in the current segment. + #[inline(always)] + pub(crate) const fn pos_of(&self, index: u32) -> usize { + self.base.wrapping_add(index as usize) + } + + /// The position in `dict_buf` of `index`, which is in the older segment. + #[inline(always)] + pub(crate) const fn dict_pos_of(&self, index: u32) -> usize { + self.dict_base.wrapping_add(index as usize) + } +} + +impl DictMatchState<'_> { + /// The position in `buf` of `index`. + #[inline(always)] + pub(crate) const fn pos_of(&self, index: u32) -> usize { + self.base.wrapping_add(index as usize) + } +} + +// ── long distance matching ──────────────────────────────────────────────── + +/// `ldmEntry_t` +#[derive(Clone, Copy, Default)] +#[repr(C)] +pub(crate) struct LdmEntry { + pub(crate) offset: u32, + pub(crate) checksum: u32, +} + +/// `ldmParams_t` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) struct LdmParams { + pub(crate) enable_ldm: ParamSwitch, + pub(crate) hash_log: u32, + pub(crate) bucket_size_log: u32, + pub(crate) min_match_length: u32, + pub(crate) hash_rate_log: u32, + pub(crate) window_log: u32, +} + +/// `ldmState_t`. Its scratch arrays are locals of the functions that use them. +pub(crate) struct LdmState { + pub(crate) window: Window, + pub(crate) hash_table: Vec, + pub(crate) loaded_dict_end: u32, + /// The next entry to replace in each bucket. + pub(crate) bucket_offsets: Vec, +} + +impl LdmState { + pub(crate) const fn new() -> Self { + Self { + window: Window::new(), + hash_table: Vec::new(), + loaded_dict_end: 0, + bucket_offsets: Vec::new(), + } + } +} + +bun_pod::impl_zeroable!(Match, RawSeq, LdmEntry); + +#[cfg(test)] +mod tests { + use super::*; + + const BLOCK_SIZE: usize = 16_000; + const FULL_LIT_BUFFER: usize = BLOCK_SIZE + WILDCOPY_OVERLENGTH; + + fn block() -> Vec { + let mut block = Vec::from_iter(0..251).repeat(BLOCK_SIZE.div_ceil(251)); + block.truncate(BLOCK_SIZE); + block + } + + fn store_at_the_start_of_a_block() -> Result { + let mut store = SeqStore::new(); + store.set_limits(BLOCK_SIZE / 3, BLOCK_SIZE); + store.start_block(BLOCK_SIZE)?; + Ok(store) + } + + fn the_buffer_of_the_literals_follows_them() -> Result<()> { + const LAST: usize = 4000; + let block = block(); + let mut store = store_at_the_start_of_a_block()?; + // Of the second block, which finds the buffer as the first one left it, more is literals. + for lit_lengths in [ + [0, 3, 16, 33, 4200, 1, 0, 100], + [1, 9000, 32, 0, 2, 2, 2, 2], + ] { + store.start_block(BLOCK_SIZE)?; + let len_before = store.lit_buffer.len(); + let mut expected = Vec::new(); + let mut sink = SeqSink::::new(&mut store); + let mut pos = 0; + for lit_length in lit_lengths { + sink.store_seq(lit_length, Near::new(&block)?, pos, 1, MINMATCH)?; + expected.extend_from_slice(block.span(pos as usize, lit_length as usize)?); + pos += lit_length + MINMATCH; + } + drop(sink); + assert_eq!(store.literals()?, expected); + let len = store.lit_buffer.len(); + assert!(len >= expected.len() + SHORT); + assert!(len <= len_before.max(expected.len() + LITERALS_AHEAD)); + + store.store_last_literals(&block, LAST)?; + expected.extend_from_slice(block.sub(BLOCK_SIZE - LAST..)?); + assert_eq!(store.literals()?, expected); + } + Ok(()) + } + + #[test] + fn the_buffer_of_the_literals_follows_them_in_a_sink() -> Result<()> { + the_buffer_of_the_literals_follows_them::<16>()?; + the_buffer_of_the_literals_follows_them::<32>() + } + + #[test] + fn the_buffer_of_the_literals_follows_them_up_to_a_whole_block() -> Result<()> { + let block = block(); + let mut store = store_at_the_start_of_a_block()?; + let mut expected = Vec::new(); + let mut pos = 0; + for lit_length in [5, 17, 6000] { + store.store_seq(lit_length, &block, pos, 1, MINMATCH as usize)?; + expected.extend_from_slice(block.span(pos, lit_length)?); + pos += lit_length + MINMATCH as usize; + } + assert_eq!(store.literals()?, expected); + assert!(store.lit_buffer.len() < FULL_LIT_BUFFER); + store.store_last_literals(&block, BLOCK_SIZE - expected.len())?; + expected.extend_from_slice(block.sub(expected.len()..)?); + assert_eq!(store.literals()?, expected); + + // What does not fit leaves the buffer where it is. + let too_many = WILDCOPY_OVERLENGTH + 1; + assert!(store.store_last_literals(&block, too_many).is_err()); + assert_eq!(store.lit_buffer.capacity(), FULL_LIT_BUFFER); + + let mut store = store_at_the_start_of_a_block()?; + store.lit_len = 1; + assert!(store.store_last_literals(&block, 1).is_err()); + assert!(store.literals().is_err()); + Ok(()) + } + + #[test] + fn a_buffer_of_literals_that_does_not_grow_is_as_long_as_it_gets() -> Result<()> { + let block = block(); + // Whatever it was left like. + for left in [0, 100] { + let mut store = store_at_the_start_of_a_block()?; + store.store_last_literals(&block, left)?; + let mut sink = SeqSink::<16, false>::new(&mut store); + sink.store_seq(9000, Near::new(&block)?, 0, 1, MINMATCH)?; + drop(sink); + assert_eq!(store.lit_buffer.len(), FULL_LIT_BUFFER); + let (before, literals) = store.literals()?.split_at_checked(left).ok_or(BUG)?; + assert_eq!(before, block.sub(BLOCK_SIZE - left..)?); + assert_eq!(literals, block.sub(..9000)?); + } + Ok(()) + } + + #[test] + fn the_hash_of_4_bytes_is_the_same_from_4_bytes() -> Result<()> { + for h_bits in [6, 17, 32] { + let hasher = Hasher::new(h_bits, 4); + let of_4_bytes = hasher.of_4_bytes().ok_or(BUG)?; + for bytes in [0, 1, 0x0123_4567_89AB_CDEF, u64::MAX] { + assert_eq!(of_4_bytes.hash(bytes as u32), hasher.hash(bytes)); + } + } + assert!(Hasher::new(33, 4).of_4_bytes().is_none()); + assert!(Hasher::new(17, 5).of_4_bytes().is_none()); + Ok(()) + } + + #[test] + fn the_head_of_a_position_is_compared_as_all_of_it() -> Result<()> { + const POS: u32 = 40; + for len in 0..=17 { + for (byte, match_byte) in [(1, 2), (2, 1), (0x7F, 0x80), (0x80, 0x7F)] { + // What follows the difference is in the other order. + let mut bytes = [vec![byte; POS as usize], vec![match_byte; POS as usize]].concat(); + bytes.sub_mut(..len as usize)?.fill(9); + bytes.sub_mut(POS as usize..(POS + len) as usize)?.fill(9); + *bytes.at_mut(len as usize)? = match_byte; + *bytes.at_mut((POS + len) as usize)? = byte; + let near = Near::new(&bytes)?; + let head = near.chunk16(POS).ok_or(BUG)?; + for from in [0, len / 2, len] { + for common in [ + near.compare_chunks::<{ First::Alike }>(POS, 0, from), + near.compare_chunks::<{ First::Apart }>(POS, 0, from), + near.compare_chunks::<{ First::Words }>(POS, 0, from), + ] { + assert_eq!( + common.map(|common| (common.len, common.match_is_smaller)), + Ok((len, Some(match_byte < byte))), + "{len} {from}" + ); + } + } + for from in [0, len, len + 1] { + let common = near.compare_head(head, 0, from); + let common = common.map(|common| (common.len, common.match_is_smaller)); + let told = len < 16 && from <= len; + assert_eq!( + common, + told.then_some((len, Some(match_byte < byte))), + "{len} {from}" + ); + } + } + } + Ok(()) + } + + /// `older`, whose first `dict_limit - low` bytes are the older segment, and then the first `end` bytes of `current`. + fn two_segments<'a>( + older: &'a [u8], + low: u32, + current: &'a [u8], + dict_limit: u32, + end: u32, + ) -> Result> { + let base_of = |index: u32| 0usize.wrapping_sub(index as usize); + let (buf, base) = (current, base_of(dict_limit)); + let prefix = Prefix { buf, base }; + let (buf, base) = (older, base_of(low)); + TreeInput::new(prefix, Prefix { buf, base }, dict_limit, end as usize) + } + + #[test] + fn the_next_16_bytes_of_a_match_in_the_older_segment_are_compared_at_once() -> Result<()> { + const LOW: u32 = 100; + const OLDER: u32 = 56; + const DICT_LIMIT: u32 = LOW + OLDER; + const IP: u32 = 3; + let input = two_segments(&[0; OLDER as usize], LOW, &[0; 64], DICT_LIMIT, 64)?; + // Not in the older segment, or only up to `from`, also where the sums wrap around. + for (index, from) in [ + (DICT_LIMIT, 0), + (DICT_LIMIT - 2, 2), + (LOW + 5, u32::MAX - 1), + ] { + let common = input.compare_older(IP, index, from); + assert!(common.is_err_and(|known| known == from)); + } + // The first position of the current segment is not looked for in an older one, which there may not be. + let alone = two_segments(&[], 0, &[0; 64], DICT_LIMIT, 64)?; + let common = alone.compare_rest(IP, DICT_LIMIT, 0)?; + assert_eq!((common.len, common.match_is_smaller), (64 - IP, None)); + for len in [0, 1, 8, 15, 16, 17] { + for (byte, match_byte) in [(1, 2), (0x80, 0x7F)] { + for start in [0, OLDER - len - 16, OLDER - len - 15, OLDER - len - 1] { + // What follows the difference is in the other order, and the buffer goes on after the segment. + let mut older = vec![byte; OLDER as usize + 32]; + older + .sub_mut(start as usize..(start + len) as usize)? + .fill(9); + *older.at_mut((start + len) as usize)? = match_byte; + let mut current = vec![match_byte; 64]; + current.sub_mut(IP as usize..(IP + len) as usize)?.fill(9); + *current.at_mut((IP + len) as usize)? = byte; + let whole = (len, Some(match_byte < byte)); + // The input ends after the difference, or later. + for end in [IP + len + 1, 64] { + let input = two_segments(&older, LOW, ¤t, DICT_LIMIT, end)?; + for from in [0, len / 2, len] { + let expected = if start + from + 16 > OLDER || IP + from + 16 > end { + Err(from) + } else if len < from + 16 { + Ok(whole) + } else { + Err(from + 16) + }; + let common = input.compare_older(IP, LOW + start, from); + assert_eq!( + common.map(|common| (common.len, common.match_is_smaller)), + expected, + "{len} {start} {end} {from}" + ); + for common in [ + input.compare_rest(IP, LOW + start, from)?, + input.compare_to_the_end(IP, LOW + start, from)?, + ] { + assert_eq!((common.len, common.match_is_smaller), whole); + } + } + } + } + } + } + Ok(()) + } + + #[test] + fn a_match_that_leaves_the_older_segment_goes_on_in_the_current_one() -> Result<()> { + const LOW: u32 = 100; + const OLDER: u32 = 40; + const DICT_LIMIT: u32 = LOW + OLDER; + const IP: u32 = 60; + const END: u32 = IP + 56; + for in_older in [1, 16, 17, 33] { + let lens = [0, 1, 15, 16, 17].map(|more| in_older + more); + for len in [&[0, in_older - 1, END - IP - 1, END - IP], &lens[..]].concat() { + for match_is_smaller in [false, true] { + // The buffer goes on after the segment, with what no match goes on with. + let mut older: Vec = (1..=OLDER as u8).collect(); + older.extend_from_slice(&[0xEE; 32]); + let mut current: Vec = (0..END as u8).map(|pos| 100 + pos).collect(); + let window = [older.sub(..OLDER as usize)?, ¤t].concat(); + let matched = window.sub((OLDER - in_older) as usize..)?; + current + .sub_mut(IP as usize..(IP + len) as usize)? + .copy_from_slice(matched.sub(..len as usize)?); + if let Some(byte) = current.get_mut((IP + len) as usize) { + let match_byte = matched.at(len as usize)?; + *byte = if match_is_smaller { + match_byte + 1 + } else { + match_byte - 1 + }; + } + let input = two_segments(&older, LOW, ¤t, DICT_LIMIT, END)?; + let match_index = DICT_LIMIT - in_older; + for from in [0, len / 2, len] { + let match_pos = match_index.wrapping_sub(DICT_LIMIT); + let near = input.near; + let called = [ + near.compare_chunks::<{ First::Alike }>(IP, match_pos, from), + near.compare_chunks::<{ First::Apart }>(IP, match_pos, from), + near.compare_chunks::<{ First::Words }>(IP, match_pos, from), + ] + .map(|by_chunks| match by_chunks { + Ok(common) => Ok(common), + Err(from) => input.compare_rest(IP, match_index, from), + }); + let inline = match input.compare_older(IP, match_index, from) { + Ok(common) => Ok(common), + Err(from) => input.compare_to_the_end(IP, match_index, from), + }; + for common in called.into_iter().chain([inline]) { + let common = common?; + assert_eq!( + (common.len, common.match_is_smaller), + (len, (IP + len < END).then_some(match_is_smaller)), + "{in_older} {len} {from}" + ); + } + } + } + } + } + Ok(()) + } + + /// Adds the positions from `from` up to `to` to one tree, and compares what each step finds with `window`, which + /// starts at the index `low`. Returns how many matches start in the older segment and end in the current one. + fn fill_a_tree( + bt: &mut Nodes, + input: &TreeInput<'_>, + window: &[u8], + low: u32, + (from, to): (u32, u32), + older: Older, + ) -> Result { + let mut leave_the_older_segment = 0; + for curr in from..to { + let visit = |index: u32, common: Common| { + let here = window.sub((curr - low) as usize..)?; + let there = window.sub((index - low) as usize..)?; + let len = here.iter().zip(there).take_while(|(a, b)| a == b).count(); + let is_smaller = here.get(len).map(|byte| there.get(len) < Some(byte)); + assert_eq!( + (common.len as usize, common.match_is_smaller), + (len, is_smaller), + "{curr} {index}" + ); + let limit = input.dict_limit; + leave_the_older_segment += u32::from(index < limit && index + common.len > limit); + Ok(is_smaller.is_some()) + }; + let (ip, descent) = (curr - input.dict_limit, Descent::new(bt, curr)); + let root = curr - 1; + bt_walk::(bt, input, ip, root, low, 64, descent, older, visit)?; + } + Ok(leave_the_older_segment) + } + + #[test] + #[cfg_attr(miri, ignore = "takes minutes")] + fn a_way_down_a_tree_is_the_same_whatever_is_done_with_the_older_segment() -> Result<()> { + const LOW: u32 = 100; + const OLDER: u32 = 300; + const DICT_LIMIT: u32 = LOW + OLDER; + const END: u32 = OLDER; + // A period with a byte altered now and then: what two positions have in common is often more than 16 bytes. + let mut seed = 5u32; + let window = Vec::from_iter((0..OLDER + END).map(|at| { + seed = seed.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + (at % 37) as u8 ^ u8::from(seed >> 27 == 0) + })); + let (older, current) = window.split_at(OLDER as usize); + // The older segment gets into the tree while it is the current one. + let mut at_first = vec![[0; 2]; 1024]; + let input = two_segments(&[], LOW, older, LOW, OLDER)?; + let positions = (LOW + 1, DICT_LIMIT); + fill_a_tree::<{ First::Words }>( + &mut at_first, + &input, + older, + LOW, + positions, + Older::Called, + )?; + + let input = two_segments(older, LOW, current, DICT_LIMIT, END)?; + let positions = (DICT_LIMIT, DICT_LIMIT + END); + let mut trees = Vec::new(); + for how in [Older::Called, Older::TriedLast, Older::TriedFirst] { + let mut by_first = [(); 3].map(|()| at_first.clone()); + let [alike, apart, words] = &mut by_first; + let counts = [ + fill_a_tree::<{ First::Alike }>(alike, &input, &window, LOW, positions, how)?, + fill_a_tree::<{ First::Apart }>(apart, &input, &window, LOW, positions, how)?, + fill_a_tree::<{ First::Words }>(words, &input, &window, LOW, positions, how)?, + ]; + assert!(counts.iter().all(|&count| count > 30), "{counts:?}"); + trees.extend(by_first); + } + assert!(trees.iter().all(|tree| Some(tree) == trees.first())); + Ok(()) + } + + #[test] + fn matches_are_counted_up_to_where_they_end() -> Result<()> { + const DISTANCE: u32 = 7; + let lengths: &[u32] = if cfg!(miri) { + &[15, 16, 63, 64, 65, 80, 127, 128, 129, 192, 193] + } else { + &Vec::from_iter(0..=400) + }; + for &len in lengths { + for room in [0, 1, 16, 64, 200] { + let mut bytes: Vec = (0..DISTANCE + len + room) + .map(|n| (n % DISTANCE) as u8) + .collect(); + if room != 0 { + *bytes.at_mut((DISTANCE + len) as usize)? ^= 1; + } + assert_eq!( + Near::new(&bytes)?.count(DISTANCE, 0), + len, + "{room} more bytes" + ); + } + } + Ok(()) + } +} diff --git a/src/zstd/compress/lazy.rs b/src/zstd/compress/lazy.rs new file mode 100644 index 000000000000..fa0f42397c4c --- /dev/null +++ b/src/zstd/compress/lazy.rs @@ -0,0 +1,2718 @@ +//! `zstd_lazy.c`: the `greedy`, `lazy`, `lazy2` and `btlazy2` strategies. + +use core::hint::{Locality, cold_path, prefetch_read}; +use core::num::NonZeroU32; +use core::ops::Range; + +use super::internal::{ + BlockSrc, CompressionParameters, Descent, DictMatchState, DictMode, DictTree, First, + HASH_READ_SIZE, Hash8, Hasher, Hints, K_SEARCH_STRENGTH, MatchState, Near, Nodes, Older, + Prefix, REPCODE1_TO_OFFBASE, RowTags, SecondTagStats, SeqSink, SeqStore, Strategy, TreeInput, + ZSTD_DUBT_UNSORTED_MARK, ZSTD_ROW_HASH_CACHE_SIZE, bt_children, bt_look_ahead, bt_low, + bt_nodes, bt_set_children, bt_walk, count_2segments, hashed_swap, index_overlap_check, + offbase_is_offset, offbase_to_offset, offset_to_offbase, table_get, table_put, +}; +use crate::common::{ZSTD_BLOCKSIZE_MAX, ZSTD_REP_NUM}; +use crate::error::Result; +use crate::util::{BUG, SliceExt, highbit32, read_le32}; + +const K_LAZY_SKIPPING_STEP: u32 = 8; + +// `ZSTD_dictMode_e`, as an argument of a const generic. +const NO_DICT: u8 = DictMode::NoDict as u8; +const EXT_DICT: u8 = DictMode::ExtDict as u8; +const DICT_MATCH_STATE: u8 = DictMode::DictMatchState as u8; + +/// `PREFETCH_L1`: a hint that `slice[index]` is about to be read. `index` may be anything. +#[inline(always)] +fn prefetch_l1(slice: &[T], index: usize) { + prefetch_read(slice.as_ptr().wrapping_add(index), Locality::L1); +} + +/// `dms->window.nextSrc - dms->window.base`: the index of the end of the dictionary, among its own indices. +#[inline(always)] +const fn dms_end_index(dms: &DictMatchState<'_>) -> u32 { + dms.end.wrapping_sub(dms.base) as u32 +} + +// ── binary tree search ──────────────────────────────────────────────────── + +/// What the searches in the binary trees work on for the time of a block: the tables of a [`MatchState`], and what does +/// not change of it. +struct Dubt<'a> { + hash_table: &'a mut [u32], + bt: &'a mut Nodes, + input: TreeInput<'a>, + /// The dictionary, if searches go on in it. + dms: Option<&'a DictMatchState<'a>>, + hasher: Hasher, + /// `window.lowLimit` + low_limit: u32, + max_distance: u32, + /// How far a match can be, if that is not down to `low_limit`. + reach: u32, + nb_compares: u32, + next_to_update: &'a mut u32, +} + +impl<'a> Dubt<'a> { + #[inline(always)] + fn new(ms: &'a mut MatchState, src: &'a BlockSrc<'a>, search_dms: bool) -> Result { + let MatchState { + hash_table, + chain_table, + next_to_update, + window, + c_params, + loaded_dict_end, + .. + } = ms; + if hash_table.len() != 1usize.wrapping_shl(c_params.hash_log) + || chain_table.len() != 1usize.wrapping_shl(c_params.chain_log) + { + return Err(BUG); + } + let dict = Prefix { + buf: src.dict_buf, + base: src.dict_base, + }; + let max_distance = 1u32.wrapping_shl(c_params.window_log); + Ok(Self { + hash_table: hash_table.as_mut_slice(), + bt: bt_nodes(chain_table), + input: TreeInput::new(src.prefix(), dict, window.dict_limit, src.end)?, + dms: src.dms.as_ref().filter(|_| search_dms), + hasher: Hasher::new(c_params.hash_log, c_params.min_match.clamp(4, 6)), + low_limit: window.low_limit, + max_distance, + // A dictionary is valid as a whole until it is invalidated as a whole. + reach: if *loaded_dict_end != 0 { + u32::MAX + } else { + max_distance + }, + nb_compares: 1u32.wrapping_shl(c_params.search_log), + next_to_update, + }) + } + + /// `ZSTD_updateDUBT`: adds the positions from `next_to_update` up to `curr` to the trees, as unsorted chains. + #[inline(always)] + fn update(&mut self, curr: u32) -> Result<()> { + let near = self.input.near; + let next_to_update = *self.next_to_update; + let long_update = if self.input.far { + LONG_UPDATE_FAR + } else { + LONG_UPDATE + }; + if curr.saturating_sub(next_to_update) >= long_update { + let first = next_to_update..curr; + let dict_limit = self.input.dict_limit; + return update_many( + self.hash_table, + self.bt, + near, + self.hasher, + first, + dict_limit, + ); + } + for idx in next_to_update..curr { + let bytes = near.read64(idx.wrapping_sub(self.input.dict_limit)); + let h = self.hasher.hash(bytes.ok_or(BUG)?); + let match_index = table_get(self.hash_table, h); + table_put(self.hash_table, h, idx); + bt_set_children(self.bt, idx, [match_index, ZSTD_DUBT_UNSORTED_MARK]); + let ahead = idx.wrapping_sub(self.input.dict_limit).wrapping_add(8); + if self.input.far + && let Some(bytes) = near.read64(ahead) + { + prefetch_l1(self.hash_table, self.hasher.hash(bytes)); + } + } + Ok(()) + } + + /// `ZSTD_insertDUBT1`: sorts the position `curr`, which is in a tree but unsorted. `curr` is at least `bt_low`. + #[inline(always)] + fn insert_dubt1(&mut self, curr: u32, nb_compares: u32, bt_low: u32) -> Result<()> { + let window_low = if curr.wrapping_sub(self.low_limit) > self.max_distance { + curr.wrapping_sub(self.max_distance) + } else { + self.low_limit + }; + if curr < self.input.dict_limit { + return sort_dubt1_in_dict( + self.bt, + self.input, + self.low_limit, + curr, + nb_compares, + bt_low, + window_low, + ); + } + sort_dubt1(self.bt, &self.input, curr, nb_compares, bt_low, window_low) + } + + /// `ZSTD_DUBT_findBestMatch` for the position `ip` of `input.near`, but for the dictionary. Returns the length of the + /// best match, or 0, and how many more positions could have been compared if the dictionary is to be searched. + #[inline(always)] + fn find_best_match(&mut self, ip: u32, off_base_ptr: &mut u32) -> Result { + let curr = ip.wrapping_add(self.input.dict_limit); + // `ZSTD_getLowestMatchIndex` + let window_low = if curr.wrapping_sub(self.low_limit) > self.reach { + curr.wrapping_sub(self.reach) + } else { + self.low_limit + }; + let h = self.hasher.hash(self.input.near.read64(ip).ok_or(BUG)?); + let mut match_index = table_get(self.hash_table, h); + bt_look_ahead::<2, 1>(self.hash_table, self.bt, &self.input, self.hasher, ip); + + let low = window_low.saturating_add(1); + if match_index < low { + // There is no candidate: the position is the whole of its tree. + table_put(self.hash_table, h, curr); + bt_set_children(self.bt, curr, [0, 0]); + *self.next_to_update = curr.wrapping_add(1); + return Ok(DubtFound { + best_length: 0, + nb_compares: self.nb_compares, + }); + } + + let unsort_limit = bt_low(self.bt, curr).max(window_low); + let mut nb_candidates = self.nb_compares; + let mut previous_candidate = 0u32; + + // Reach the end of the list of unsorted candidates. The marks become a chain back to where it starts. + let mut slots = bt_children(self.bt, match_index); + while let [next_candidate, ZSTD_DUBT_UNSORTED_MARK] = slots + && match_index > unsort_limit + { + if nb_candidates <= 1 { + // The last candidate is still unsorted: cut the list there. + bt_set_children(self.bt, match_index, [0, 0]); + break; + } + bt_set_children(self.bt, match_index, [next_candidate, previous_candidate]); + previous_candidate = match_index; + match_index = next_candidate; + slots = bt_children(self.bt, match_index); + nb_candidates -= 1; + } + + // Sort the candidates, the oldest first. + match_index = previous_candidate; + while match_index != 0 { + let [_, next_candidate_idx] = bt_children(self.bt, match_index); + self.insert_dubt1(match_index, nb_candidates, unsort_limit)?; + match_index = next_candidate_idx; + nb_candidates += 1; + } + + // Find the longest match. + let mut match_end_idx = curr.wrapping_add(8 + 1); + let mut best_length = 0u32; + let mut search_dict = true; + match_index = table_get(self.hash_table, h); + table_put(self.hash_table, h, curr); + let nb_compares = bt_walk::<{ First::Alike }, { Hints::All }>( + self.bt, + &self.input, + ip, + match_index, + low, + self.nb_compares, + Descent::new(self.bt, curr), + Older::Called, + |match_index, common| { + if common.len > best_length { + if common.len > match_end_idx.wrapping_sub(match_index) { + match_end_idx = match_index.wrapping_add(common.len); + } + if 4 * (common.len - best_length) as i32 + > highbit32(curr.wrapping_sub(match_index).wrapping_add(1)) as i32 + - highbit32(*off_base_ptr) as i32 + { + best_length = common.len; + *off_base_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); + } + if common.match_is_smaller.is_none() { + // Equal: no way to know which is smaller. Nothing in the dictionary can be longer. + search_dict = false; + return Ok(false); + } + } + Ok(true) + }, + )?; + + // Skip repetitive patterns. + *self.next_to_update = match_end_idx.wrapping_sub(8); + Ok(DubtFound { + best_length, + nb_compares: if search_dict { nb_compares } else { 0 }, + }) + } +} + +impl Dubt<'_> { + /// `ZSTD_BtFindBestMatch`: updates the trees up to the position `ip` of `input.near` and returns the best match there. + /// Its length is 0 if there is none. + #[inline(always)] + fn search(&mut self, ip: u32) -> Result { + let mut off_base = 999_999_999u32; + let curr = ip.wrapping_add(self.input.dict_limit); + let len = if curr < *self.next_to_update { + // In an area that is skipped. + 0 + } else { + let dict_root = match self.dms { + Some(dms) => self.dict_root(dms, ip)?, + None => 0, + }; + self.update(curr)?; + let found = self.find_best_match(ip, &mut off_base)?; + match self.dms { + Some(dms) if dict_root != 0 && found.nb_compares != 0 => find_better_dict_match( + dms, + self.input, + self.low_limit, + ip, + dict_root, + found, + &mut off_base, + )?, + _ => found.best_length, + } + }; + Ok(Found { + len, + off_base: NonZeroU32::new(off_base).ok_or(BUG)?, + }) + } + + /// [`Self::search`], where the time goes to waiting for memory. + #[inline(never)] + fn search_far(&mut self, ip: u32) -> Result { + self.search(ip) + } + + /// [`DictTree::root`] for the position `ip` of `input.near`. + #[inline(always)] + fn dict_root(&self, dms: &DictMatchState<'_>, ip: u32) -> Result { + let bytes = self.input.near.read64(ip).ok_or(BUG)?; + let hasher = self.hasher.with_log(dms.ms.c_params.hash_log); + DictTree::root(*dms, hasher.hash(bytes)) + } +} + +/// From that many positions on, [`Dubt::update`] leaves them to [`update_many`], whose loop is tighter. +const LONG_UPDATE: u32 = 8; +/// The same where `update` gives hints at what is about to be read, which `update_many` does not. +const LONG_UPDATE_FAR: u32 = 32; + +/// `ZSTD_updateDUBT` of the indices `indices`, which are in `near`, whose first index is `dict_limit`. The sizes of the +/// tables are powers of 2, and that of `hash_table` is what `hasher` is for. +#[cold] +#[inline(never)] +fn update_many( + hash_table: &mut [u32], + bt: &mut Nodes, + near: Near<'_>, + hasher: Hasher, + indices: Range, + dict_limit: u32, +) -> Result<()> { + let bytes = near + .tail(indices.start.wrapping_sub(dict_limit)) + .ok_or(BUG)?; + match hasher.of_4_bytes() { + Some(hasher) => update_many_with(hash_table, bt, bytes, indices, |bytes| { + hasher.hash(u32::from_le_bytes(bytes)) + }), + None => update_many_with(hash_table, bt, bytes, indices, |bytes| { + hasher.hash(u64::from_le_bytes(bytes)) + }), + } +} + +/// [`update_many`] of `indices`, the first of which is at the start of `bytes`. `hash` gives the hash of a position from +/// its first `N` bytes. +#[inline(always)] +fn update_many_with( + hash_table: &mut [u32], + bt: &mut Nodes, + bytes: Near<'_>, + mut indices: Range, + hash: impl Fn([u8; N]) -> usize, +) -> Result<()> { + if bt.is_empty() { + return Err(BUG); + } + let mut bytes = bytes.bytes().sub(..indices.len() + 7)?; + while !indices.is_empty() { + // The nodes of indices that follow each other do so too, up to the end of the table. + let nodes = bt.sub_mut(indices.start as usize & (bt.len() - 1)..)?; + let nb_nodes = nodes.len().min(indices.len()); + for ((idx, node), bytes) in indices.clone().zip(nodes).zip(bytes.array_windows()) { + let match_index = hashed_swap(hash_table, hash(*bytes), idx); + *node = [match_index, ZSTD_DUBT_UNSORTED_MARK]; + } + indices.start += nb_nodes as u32; + bytes = bytes.sub(nb_nodes..)?; + } + Ok(()) +} + +/// `ZSTD_DUBT_findBetterDictMatch`: goes on, in the tree of the dictionary whose root is `root`, with a search for the +/// position `ip` of `input.near` that found `found`. `low_limit` is `window.lowLimit`. Returns the length of the best +/// match. +#[inline(never)] +fn find_better_dict_match( + dms: &DictMatchState<'_>, + input: TreeInput<'_>, + low_limit: u32, + ip: u32, + root: u32, + found: DubtFound, + offset_ptr: &mut u32, +) -> Result { + let tree = DictTree::new(*dms, low_limit)?; + let curr = ip.wrapping_add(input.dict_limit); + let mut best_length = found.best_length; + tree.walk( + &input, + ip, + root, + found.nb_compares, + |dict_match_index, common| { + if common.len > best_length { + let match_index = dict_match_index.wrapping_add(tree.index_delta); + if 4 * (common.len - best_length) as i32 + > highbit32(curr.wrapping_sub(match_index).wrapping_add(1)) as i32 + - highbit32(offset_ptr.wrapping_add(1)) as i32 + { + best_length = common.len; + *offset_ptr = offset_to_offbase(curr.wrapping_sub(match_index)); + } + // At the end, there is no way to know which is smaller. + return Ok(common.match_is_smaller.is_some()); + } + Ok(true) + }, + )?; + Ok(best_length) +} + +/// [`sort_dubt1`] of a position of the older segment of `input`, which starts at the index `low_limit`. All that is +/// before the position is in that segment too. +#[cold] +#[inline(never)] +fn sort_dubt1_in_dict( + bt: &mut Nodes, + input: TreeInput<'_>, + low_limit: u32, + curr: u32, + nb_compares: u32, + bt_low: u32, + window_low: u32, +) -> Result<()> { + let older = input.dict(); + let nothing = Prefix { buf: &[], base: 0 }; + let older = TreeInput::new(older, nothing, low_limit, older.pos_of(input.dict_limit))?; + sort_dubt1(bt, &older, curr, nb_compares, bt_low, window_low) +} + +/// The body of `ZSTD_insertDUBT1`, for a position of `input.near`. +#[inline(always)] +fn sort_dubt1( + bt: &mut Nodes, + input: &TreeInput<'_>, + curr: u32, + nb_compares: u32, + bt_low: u32, + window_low: u32, +) -> Result<()> { + // The next sorted candidate is in the first slot. The other one has the previous unsorted candidate. + let [next_sorted, _] = bt_children(bt, curr); + bt_walk::<{ First::Alike }, { Hints::All }>( + bt, + input, + curr.wrapping_sub(input.dict_limit), + next_sorted, + window_low.saturating_add(1), + nb_compares, + Descent::down_to(curr, bt_low), + Older::Called, + // If they are equal, there is no way to know which is smaller. Dropped, to keep the tree consistent. + |_, common| Ok(common.match_is_smaller.is_some()), + )?; + Ok(()) +} + +/// What [`Dubt::find_best_match`] returns. +#[derive(Clone, Copy)] +struct DubtFound { + best_length: u32, + nb_compares: u32, +} + +// ── what the match finders share ────────────────────────────────────────── + +/// What an index below the current segment leads to: the older segment of the window, or the dictionary whose match +/// state is searched. +#[derive(Clone, Copy)] +struct DictSegment<'a> { + /// All of it, and nothing else. + bytes: &'a [u8], + /// The index of its first byte. + start_index: u32, +} + +/// `ZSTD_getLowestMatchIndex`, with what it reads from the match state. +#[derive(Clone, Copy)] +struct Reach { + lowest_valid: u32, + /// More than any distance if there is a dictionary, which is valid as a whole until it is invalidated as a whole. + max_distance: u32, +} + +impl Reach { + const fn new(ms: &MatchState) -> Self { + Self { + lowest_valid: ms.window.low_limit, + max_distance: if ms.loaded_dict_end != 0 { + u32::MAX + } else { + 1 << ms.c_params.window_log + }, + } + } + + /// The lowest index a match for the position `curr` can be at. + #[inline(always)] + const fn low_limit(self, curr: u32) -> u32 { + if curr.wrapping_sub(self.lowest_valid) > self.max_distance { + curr - self.max_distance + } else { + self.lowest_valid + } + } +} + +/// The block that a parser works on. Positions are counted from the start of the current segment of the window. +#[derive(Clone, Copy)] +struct LazyBlock<'a> { + /// The current segment, up to the end of the block. + near: Near<'a>, + /// The index of the position 0. + base_index: u32, + /// Where the block starts. + istart: u32, + dict: DictSegment<'a>, + reach: Reach, +} + +impl<'a> LazyBlock<'a> { + fn new(ms: &MatchState, src: &BlockSrc<'a>, dict_mode: DictMode) -> Result { + let base_index = ms.window.dict_limit; + let prefix_start = src.pos_of(base_index); + let dict = match dict_mode { + DictMode::NoDict => DictSegment { + bytes: &[], + start_index: base_index, + }, + DictMode::ExtDict => { + let start_index = ms.window.low_limit; + let segment = src.dict_pos_of(start_index)..src.dict_pos_of(base_index); + DictSegment { + bytes: src.dict_buf.sub(segment)?, + start_index, + } + } + DictMode::DictMatchState => { + let dms = src.dms.ok_or(BUG)?; + let bytes = dms.buf.sub(dms.pos_of(dms.ms.window.dict_limit)..dms.end)?; + DictSegment { + bytes, + start_index: base_index.wrapping_sub(bytes.len() as u32), + } + } + }; + Ok(Self { + near: Near::new(src.buf.sub(prefix_start..src.end)?)?, + base_index, + istart: (src.start - prefix_start) as u32, + dict, + reach: Reach::new(ms), + }) + } + + /// `ZSTD_count_2segments`: how long a match is that starts at `match_pos` in the dictionary. + #[inline(never)] + fn count_from_dict(&self, ip: u32, match_pos: usize) -> Result { + let bytes = self.near.bytes(); + let dict = self.dict.bytes; + let len = count_2segments( + bytes, + ip as usize, + dict, + match_pos, + bytes.len(), + dict.len(), + 0, + )?; + Ok(len as u32) + } +} + +/// A match. +#[derive(Clone, Copy)] +struct Found { + len: u32, + off_base: NonZeroU32, +} + +impl Found { + /// With the first repeat offset. + #[inline(always)] + const fn repeat(len: u32) -> Self { + Self { + len, + off_base: NonZeroU32::MIN, + } + } + + /// What the offset costs, in what the parsers compare matches by. + #[inline(always)] + const fn cost(self) -> i32 { + highbit32(self.off_base.get()) as i32 + } + + /// What the parsers compare matches by. + #[inline(always)] + const fn gain(self) -> i32 { + (self.len * 4) as i32 - self.cost() + } +} + +/// What a match has to be like to be better than what the parser has. +#[derive(Clone, Copy)] +struct Wanted { + /// It is longer than this, which is at least 3. + longer_than: u32, + /// Its [`Found::gain`] is more than this. + gain: i32, +} + +impl Wanted { + /// Less than the gain of any match. + const ANY_GAIN: i32 = i32::MIN / 2; + + /// What a match has to be longer than, given that no candidate is at a higher index than `closest`. `BY_GAIN` is + /// whether `gain` may be something else than [`Self::ANY_GAIN`]. + #[inline(always)] + fn floor(self, curr: u32, closest: u32) -> u32 { + if !BY_GAIN { + return self.longer_than; + } + let cost = highbit32(offset_to_offbase(curr.wrapping_sub(closest))) as i32; + ((self.gain + cost) >> 2).max(self.longer_than as i32) as u32 + } +} + +/// What became of a candidate. +#[derive(PartialEq, Eq)] +enum Offered { + /// It is not in the segment. + Outside, + Considered, + /// It goes up to the end of the block: there is nothing longer. + Longest, +} + +/// What a match has to be longer than, which is at least 3: the best one so far, if there is one. +#[derive(Clone, Copy)] +struct Best { + len: u32, + /// 0 if there is no match. + off_base: u32, +} + +impl Best { + #[inline(always)] + fn found(self) -> Option { + Some(Found { + len: self.len, + off_base: NonZeroU32::new(self.off_base)?, + }) + } +} + +/// What [`Near::count_short`] compares at first. From there on, it took more than a glance to find out that a candidate +/// is not long enough. +const LONG_WAY: u32 = 16; + +/// The longest match so far for a position, while candidates are compared with it. +struct Longest<'a> { + /// The current segment, from the lowest index that a match can be at or from its start. + segment: Near<'a>, + /// The index of its first byte. + start_index: u32, + /// The position that matches are for, in `segment`. + ip: u32, + /// A match has to be more than 3 bytes longer than this, unless it is the first and [`LONG_WAY`] bytes. + tail_start: u32, + /// `segment` from `tail_start` on. + tails: Near<'a>, + /// What such a match has in common with `ip`: the 4 bytes from `tail_start` on. + tail: u32, + /// 0 as long as there is no match. + off_base: u32, +} + +impl<'a> Longest<'a> { + /// `None` if there is nothing after `best.len` bytes, up to the end of the block. + #[inline(always)] + fn new(segment: Near<'a>, start_index: u32, curr: u32, best: Best) -> Option { + let ip = curr.wrapping_sub(start_index); + let tail_start = best.len - 3; + Some(Self { + segment, + start_index, + ip, + tail_start, + tails: segment.tail(tail_start)?, + tail: segment.read32(ip + tail_start)?, + off_base: best.off_base, + }) + } + + /// There is a match of `len` bytes. + #[inline(always)] + fn offer_match(&mut self, len: u32, off_base: u32) -> Offered { + if len > self.tail_start + 3 { + return self.take(len, off_base); + } + // Records that differ in one field are the same again after it. If that is where the tail is, it tells none of + // them from a longer match. Where this one ends does, as in the C code: it is taken, short as it is. + if len >= LONG_WAY && self.off_base == 0 { + cold_path(); + return self.take(len, off_base); + } + Offered::Considered + } + + /// From now on, a match has to be longer than `len`. + #[inline(always)] + fn take(&mut self, len: u32, off_base: u32) -> Offered { + self.tail_start = len - 3; + self.off_base = off_base; + let (Some(tails), Some(tail)) = ( + self.segment.tail(self.tail_start), + self.segment.read32(self.ip + self.tail_start), + ) else { + return Offered::Longest; + }; + self.tails = tails; + self.tail = tail; + Offered::Considered + } + + /// There may be a match for the position `ip` of `block` at `match_pos` in its dictionary. + #[inline(always)] + fn offer_in_dict( + &mut self, + block: &LazyBlock<'_>, + ip: u32, + match_pos: usize, + ) -> Result { + // Where the match would go on in the current segment, it takes more to tell. + let tail_pos = match_pos + self.tail_start as usize; + let tail = block.dict.bytes.get(tail_pos..tail_pos + 4); + if let Some(tail) = tail.and_then(<[u8]>::first_chunk) + && u32::from_le_bytes(*tail) != self.tail + { + return Ok(Offered::Considered); + } + let len = block.count_from_dict(ip, match_pos)?; + let dict_len = block.base_index.wrapping_sub(block.dict.start_index); + let offset = (ip + dict_len).wrapping_sub(match_pos as u32); + Ok(self.offer_match(len, offset_to_offbase(offset))) + } + + /// There may be a match at `match_index`. `IN_TAILS` is which of two ways its tail is read: the one that takes fewer + /// instructions, or the one that has less to wait for after a match. + #[inline(always)] + fn offer(&mut self, match_index: u32) -> Offered { + let match_pos = match_index.wrapping_sub(self.start_index); + let tail = if IN_TAILS { + self.tails.read32(match_pos) + } else { + self.segment + .read32_at(match_pos as usize + self.tail_start as usize) + }; + let Some(tail) = tail else { + return Offered::Outside; + }; + if tail != self.tail { + return Offered::Considered; + } + let len = self.segment.count_short(self.ip, match_pos); + self.offer_match(len, offset_to_offbase(self.ip.wrapping_sub(match_pos))) + } + + /// It may be shorter than what [`Self::new`] was given. + #[inline(always)] + const fn best(&self) -> Best { + Best { + len: self.tail_start + 3, + off_base: self.off_base, + } + } +} + +/// Where the parser is, when it has no match. +#[derive(Clone, Copy)] +struct LiteralRun { + ip: u32, + /// Where the literals start. + anchor: u32, + /// Positions are searched up to this one, which is not. + ilimit: u32, + offset_1: u32, +} + +/// Where the parser goes on after [`Finder::skip_literals`]. +struct Skipped { + ip: u32, + /// The repeat offset is known to give no match at the next position. + no_repeat: bool, +} + +/// `ZSTD_searchMax`, with the state of the search in local variables for as long as a block lasts. +trait Finder<'a>: Sized { + /// How far from the end of the block the last search is. + const MARGIN: u32; + /// After how many bytes the parser stops, for another finder to go on if that is better. + const TRIAL: Option = None; + + /// `block` is at least `MARGIN` bytes. + fn new(ms: &'a mut MatchState, src: &'a BlockSrc<'a>, block: LazyBlock<'a>) -> Result; + + /// The longest match for `ip`, the most recent one of that length, if it is as `wanted`. If it is not, `None` or any + /// match of at least 4 bytes that is not either. Positions are searched in increasing order. While `skipping`, those + /// in between are not indexed. + /// `BY_GAIN` is whether `wanted.gain` may be something else than [`Wanted::ANY_GAIN`]. + fn find( + &mut self, + ip: u32, + wanted: Wanted, + skipping: bool, + ) -> Result>; + + /// Goes past positions where the parser would find nothing, neither with [`Self::find`] nor with the repeat offset + /// at the next position: goes on from `run.ip`, or from a later position that the parser would come to. + #[inline(always)] + fn skip_literals(&mut self, run: LiteralRun) -> Result { + Ok(Skipped { + ip: run.ip, + no_repeat: false, + }) + } + + /// Every position is indexed again from now on. + fn stop_skipping(&mut self) -> Result<()>; + + /// The end of the block, or of the trial. + fn finish(self) -> Result<()>; +} + +// ── binary trees, for the parser ────────────────────────────────────────── + +/// `FAR` is [`TreeInput::far`]. +struct TreeFinder<'a, const DICT_MODE: u8, const FAR: bool>(Dubt<'a>); + +impl<'a, const DICT_MODE: u8, const FAR: bool> Finder<'a> for TreeFinder<'a, DICT_MODE, FAR> { + const MARGIN: u32 = 8; + + #[inline(always)] + fn new(ms: &'a mut MatchState, src: &'a BlockSrc<'a>, _block: LazyBlock<'a>) -> Result { + let mut dubt = Dubt::new(ms, src, DICT_MODE == DICT_MATCH_STATE)?; + dubt.input.far = FAR; + Ok(Self(dubt)) + } + + #[inline(always)] + fn find( + &mut self, + ip: u32, + wanted: Wanted, + _skipping: bool, + ) -> Result> { + let found = if FAR { + self.0.search_far(ip)? + } else { + self.0.search(ip)? + }; + Ok(Some(found).filter(|found| found.len > wanted.longer_than)) + } + + #[inline(always)] + fn stop_skipping(&mut self) -> Result<()> { + Ok(()) + } + + #[inline(always)] + fn finish(self) -> Result<()> { + Ok(()) + } +} + +// ── hash chains ─────────────────────────────────────────────────────────── + +/// `ZSTD_insertAndFindFirstIndex`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the hash +/// chains. `ip` is not before that, and there are 7 bytes from it on. For a job that is 16 KB at most, as rows take over +/// from a window of 32 KB on. +pub(crate) fn insert_and_find_first_index( + ms: &mut MatchState, + prefix: Prefix<'_>, + ip: usize, +) -> Result<()> { + if ms.hash_table.is_empty() || ms.chain_table.is_empty() { + return Err(BUG); + } + let first = prefix.pos_of(ms.next_to_update); + let positions = prefix.buf.sub(first..ip + HASH_READ_SIZE - 1)?; + let chains = Chains { + hash_table: &mut ms.hash_table, + chain_table: &mut ms.chain_table, + }; + let hasher = Hasher::new(ms.c_params.hash_log, ms.c_params.min_match); + chains.insert_all(hasher, positions, ms.next_to_update); + ms.next_to_update = prefix.index_of(ip); + Ok(()) +} + +/// The hash chains of a dictionary. +#[derive(Clone, Copy)] +struct ChainDict<'a> { + hash_table: &'a [u32], + chain_table: &'a [u32], + hasher: Hasher, + /// The index, among its own, of its first byte. + lowest_index: u32, + /// Below this, an index is no longer in the chain table. + min_chain: u32, +} + +impl<'a> ChainDict<'a> { + fn new(dms: &DictMatchState<'a>, mls: u32) -> Self { + Self { + hash_table: &dms.ms.hash_table, + chain_table: &dms.ms.chain_table, + hasher: Hasher::new(dms.ms.c_params.hash_log, mls), + lowest_index: dms.ms.window.dict_limit, + min_chain: dms_end_index(dms).saturating_sub(1 << dms.ms.c_params.chain_log), + } + } +} + +/// The hash table and the chain table. +struct Chains<'a> { + hash_table: &'a mut [u32], + chain_table: &'a mut [u32], +} + +impl Chains<'_> { + /// Adds the position that `bytes` is the 8 bytes of. + #[inline(always)] + fn insert(&mut self, hasher: Hasher, bytes: u64, index: u32) { + let hash = hasher.hash(bytes) & self.hash_table.len().wrapping_sub(1); + let in_chain = index as usize & self.chain_table.len().wrapping_sub(1); + if let Some(first) = self.hash_table.get_mut(hash) + && let Some(next) = self.chain_table.get_mut(in_chain) + { + *next = core::mem::replace(first, index); + } + } + + /// Adds every position that has 8 bytes in `positions`. The first one has the index `index`. + #[inline(never)] + fn insert_all(self, hasher: Hasher, positions: &[u8], index: u32) { + let (Some(hash_mask), Some(chain_mask)) = ( + self.hash_table.len().checked_sub(1), + self.chain_table.len().checked_sub(1), + ) else { + return; + }; + for (n, bytes) in positions.array_windows().enumerate() { + let index = index as usize + n; + let hash = hasher.hash(u64::from_le_bytes(*bytes)); + if let Some(head) = self.hash_table.get_mut(hash & hash_mask) + && let Some(next) = self.chain_table.get_mut(index & chain_mask) + { + *next = core::mem::replace(head, index as u32); + } + } + } +} + +/// Where the search of a position is at. +#[derive(Clone, Copy)] +struct Candidate { + /// The first candidate that is a match. + match_index: u32, + /// How many there may be, with that one. + nb_attempts: u32, +} + +/// `ZSTD_HcFindBestMatch`. `SIMPLE` tells that all along the block, every index from that of the position 0 on is within +/// reach and in the chain table, and none below it is valid. +struct ChainFinder<'a, const DICT_MODE: u8, const SIMPLE: bool> { + block: LazyBlock<'a>, + chains: Chains<'a>, + hasher: Hasher, + mls: u32, + nb_attempts: u32, + /// `ms.next_to_update`, as a position. + next_to_update: u32, + /// The hash of `next_to_update`, if it was searched. Else it is out of the hash table. + next_hash: usize, + /// What the hash table had for it, and still has. + next_head: u32, + /// The index from which every position is in the chains, up to `next_to_update`. + dense_from: u32, + /// Where the search of `next_to_update` is at, if [`Finder::skip_literals`] started it. + started: Option, + ms_next_to_update: &'a mut u32, + dms: Option>, +} + +/// Whether [`ChainFinder`] can be `SIMPLE`. +fn chains_are_simple(ms: &MatchState, block: &LazyBlock<'_>) -> bool { + let reach = block.reach.max_distance.min(ms.chain_table.len() as u32); + block.reach.lowest_valid == block.base_index && block.near.len() <= reach +} + +impl ChainFinder<'_, DICT_MODE, SIMPLE> { + /// `ZSTD_insertAndFindFirstIndex_internal`: returns the index of the first candidate for `ip`. + #[inline(always)] + fn insert_and_find_first_index(&mut self, ip: u32, skipping: bool) -> Result { + let near = self.block.near; + let first = self.next_to_update; + if first < ip { + let index = self.block.base_index.wrapping_add(first); + if let Some(head) = self.chains.hash_table.get_mut(self.next_hash) { + *head = index; + table_put(self.chains.chain_table, index as usize, self.next_head); + } else { + let bytes = near.read64(first).ok_or(BUG)?; + self.chains.insert(self.hasher, bytes, index); + } + if skipping { + // Not every position is inserted. + self.dense_from = self.block.base_index.wrapping_add(ip); + } else if first + 1 < ip { + let positions = near.bytes().sub(first as usize + 1..ip as usize + 7)?; + let chains = Chains { + hash_table: self.chains.hash_table, + chain_table: self.chains.chain_table, + }; + chains.insert_all(self.hasher, positions, index.wrapping_add(1)); + } + } + self.next_to_update = ip; + let hash = self.hasher.hash(near.read64(ip).ok_or(BUG)?); + let hash = hash & self.chains.hash_table.len().wrapping_sub(1); + let head = self.chains.hash_table.get(hash).copied().unwrap_or(0); + self.next_hash = hash; + self.next_head = head; + Ok(head) + } +} + +impl<'a, const DICT_MODE: u8, const SIMPLE: bool> Finder<'a> + for ChainFinder<'a, DICT_MODE, SIMPLE> +{ + const MARGIN: u32 = 8; + + #[inline(always)] + fn new(ms: &'a mut MatchState, src: &'a BlockSrc<'a>, block: LazyBlock<'a>) -> Result { + let mls = ms.c_params.min_match.clamp(4, 6); + Ok(Self { + started: None, + block, + chains: Chains { + hash_table: &mut ms.hash_table, + chain_table: &mut ms.chain_table, + }, + hasher: Hasher::new(ms.c_params.hash_log, mls), + mls, + nb_attempts: 1 << ms.c_params.search_log, + next_to_update: ms.next_to_update.wrapping_sub(block.base_index), + next_hash: usize::MAX, + next_head: 0, + dense_from: ms.next_to_update, + ms_next_to_update: &mut ms.next_to_update, + dms: src.dms.as_ref().map(|dms| ChainDict::new(dms, mls)), + }) + } + + #[inline(always)] + fn find( + &mut self, + ip: u32, + wanted: Wanted, + skipping: bool, + ) -> Result> { + let block = self.block; + let curr = block.base_index.wrapping_add(ip); + let low_limit = if SIMPLE { + block.base_index + } else { + block.reach.low_limit(curr) + }; + let min_chain = curr.saturating_sub(self.chains.chain_table.len() as u32); + let mut nb_attempts = self.nb_attempts; + let mut match_index; + if let Some(started) = self.started.take() { + match_index = started.match_index; + nb_attempts = started.nb_attempts; + } else { + match_index = self.insert_and_find_first_index(ip, skipping)?; + if DICT_MODE == NO_DICT && match_index < low_limit { + return Ok(None); + } + } + + let closest = if DICT_MODE == DICT_MATCH_STATE { + match_index.max(low_limit) + } else { + match_index + }; + let start_index = if DICT_MODE == EXT_DICT { + low_limit.max(block.base_index) + } else { + low_limit + }; + let segment = block + .near + .tail(start_index.wrapping_sub(block.base_index)) + .ok_or(BUG)?; + let best = Best { + len: wanted.floor::(curr, closest), + off_base: 0, + }; + // A match that long has a position in the chains for each of its bytes, but the last ones. + if BY_GAIN + && DICT_MODE == NO_DICT + && best.len > 3 + && low_limit >= self.dense_from + && let Some(end) = (best.len + 1).checked_sub(self.mls) + && let Some(bytes) = block.near.read64(ip + end) + && table_get(self.chains.hash_table, self.hasher.hash(bytes)) < low_limit + { + return Ok(None); + } + let Some(mut longest) = Longest::new(segment, start_index, curr, best) else { + return Ok(None); + }; + while nb_attempts > 0 { + let mut offered = longest.offer::(match_index); + if offered == Offered::Outside { + if DICT_MODE != EXT_DICT || match_index < low_limit { + break; + } + let match_pos = (match_index - block.dict.start_index) as usize; + offered = longest.offer_in_dict(&block, ip, match_pos)?; + } + if offered == Offered::Longest || (!SIMPLE && match_index <= min_chain) { + break; + } + match_index = table_get(self.chains.chain_table, match_index as usize); + nb_attempts -= 1; + } + + if DICT_MODE == DICT_MATCH_STATE { + let dms = self.dms.ok_or(BUG)?; + let hash = dms.hasher.hash(block.near.read64(ip).ok_or(BUG)?); + match_index = dms.hash_table.at(hash)?; + + while match_index >= dms.lowest_index && nb_attempts > 0 { + let match_pos = (match_index - dms.lowest_index) as usize; + if longest.offer_in_dict(&block, ip, match_pos)? == Offered::Longest { + break; + } + if match_index <= dms.min_chain { + break; + } + match_index = table_get(dms.chain_table, match_index as usize); + nb_attempts -= 1; + } + } + + Ok(longest.best().found()) + } + + #[inline(always)] + fn skip_literals(&mut self, run: LiteralRun) -> Result { + let near = self.block.near; + let mut ip = run.ip; + let no_repeat = false; + if !SIMPLE { + return Ok(Skipped { ip, no_repeat }); + } + while ip < run.ilimit { + let step = (ip - run.anchor) >> K_SEARCH_STRENGTH; + let bytes = near.read64(ip).ok_or(BUG)?; + if step + 1 > K_LAZY_SKIPPING_STEP + || (run.offset_1 > 0 + && near.read32((ip + 1).wrapping_sub(run.offset_1)) + == Some((bytes >> 8) as u32)) + { + break; + } + + let mut match_index = self.insert_and_find_first_index(ip, false)?; + let mut nb_attempts = self.nb_attempts; + while let Some(match_bytes) = + near.read32(match_index.wrapping_sub(self.block.base_index)) + { + if match_bytes == bytes as u32 { + self.started = Some(Candidate { + match_index, + nb_attempts, + }); + return Ok(Skipped { + ip, + no_repeat: true, + }); + } + nb_attempts -= 1; + if nb_attempts == 0 { + break; + } + match_index = table_get(self.chains.chain_table, match_index as usize); + } + ip += step + 1; + } + Ok(Skipped { ip, no_repeat }) + } + + #[inline(always)] + fn stop_skipping(&mut self) -> Result<()> { + Ok(()) + } + + #[inline(always)] + fn finish(self) -> Result<()> { + *self.ms_next_to_update = self.block.base_index.wrapping_add(self.next_to_update); + Ok(()) + } +} + +// ── row-based match finder ──────────────────────────────────────────────── +// +// The hash table is made of rows of 16, 32 or 64 entries, and the tag table has one byte for each entry: 8 more bits +// of the hash of the position. A row is a circular buffer. Its entry 0 is not used, and the byte for it in the tag +// table is the "head": which entry was written last. +// +// Unlike that of the C code, the tag table may have a second byte for each entry, after the tags of the row: a hash of +// the bytes 4 to 7 of the position. An entry that has another one than the position that is searched for is a match of +// less than 8 bytes, which is often enough to know without reading what it leads to. + +pub(crate) const ZSTD_ROW_HASH_TAG_BITS: u32 = 8; +const ZSTD_ROW_HASH_CACHE_MASK: u32 = ZSTD_ROW_HASH_CACHE_SIZE as u32 - 1; + +/// How long a match is at least, if it has the same second tag for a reason. +const SECOND_TAG_LENGTH: u32 = 8; +/// After how many bytes the parser stops for [`SecondTagStats::record_trial`]. +const SECOND_TAG_TRIAL: u32 = 1 << 13; + +/// How the tag table is laid out at the start of a frame of `src_size` bytes, if that is known. +pub(crate) const fn row_tags_at_start( + c_params: &CompressionParameters, + src_size: Option, +) -> RowTags { + // In shorter rows the candidates are few, and the second tags cost more than they save. As does the memory for + // them, for a short input. + match src_size { + Some(src_size) if c_params.search_log >= 6 && src_size >> c_params.hash_log != 0 => { + RowTags::Second(SecondTagStats::AT_START) + } + _ => RowTags::Flat, + } +} + +impl SecondTagStats { + const AT_START: Self = Self { + nb_new_positions: 0, + nb_positions: 0, + nb_ruled_out: 0, + }; + + /// The second tags have ruled out `nb_ruled_out` more candidates while `nb_positions` more were inserted. Returns + /// whether to keep them. + fn record_trial(&mut self, nb_positions: u32, nb_ruled_out: u32) -> bool { + self.nb_new_positions += nb_positions; + self.nb_positions += u64::from(nb_positions); + self.nb_ruled_out += u64::from(nb_ruled_out); + // Where few positions are inserted, they cost little, and it takes some to tell. + if self.nb_new_positions < SECOND_TAG_TRIAL / 2 { + return true; + } + self.nb_new_positions = 0; + // Below that, they cost more than they save. There is no way back, and less to rule out while the rows fill up: + // at first they are only given up where there is next to nothing. + if self.nb_positions < ZSTD_BLOCKSIZE_MAX as u64 { + 2 * self.nb_ruled_out >= self.nb_positions + } else { + 2 * self.nb_ruled_out >= 5 * self.nb_positions + } + } +} + +/// The size of the tag table that goes with a hash table of `h_size` entries. +pub(crate) const fn row_tag_table_size(tags: RowTags, h_size: usize) -> usize { + match tags { + RowTags::Flat => h_size, + RowTags::Second(_) => 2 * h_size, + } +} + +/// Gives up the second tags, if there are: the tags of each row move to where they are when there are none. +fn row_flatten_tags(layout: &mut RowTags, tag_table: &mut [u8]) -> Result<()> { + const ROW_ENTRIES: usize = 64; + if core::mem::replace(layout, RowTags::Flat) == RowTags::Flat { + return Ok(()); + } + for row in 1..tag_table.len() / (2 * ROW_ENTRIES) { + tag_table.move_within(2 * ROW_ENTRIES * row, ROW_ENTRIES * row, ROW_ENTRIES)?; + } + Ok(()) +} + +/// `ZSTD_row_nextIndex`, for every byte that a head can be. +const fn row_next_heads(row_entries: usize) -> [u8; 256] { + let row_mask = row_entries - 1; + let mut next_heads = [0; 256]; + let mut rest: &mut [u8] = &mut next_heads; + let mut head = 0usize; + while let Some((next_head, others)) = rest.split_first_mut() { + let next = head.wrapping_sub(1) & row_mask; + *next_head = if next == 0 { row_mask } else { next } as u8; + rest = others; + head += 1; + } + next_heads +} + +/// What tells where a position goes in the tables. +#[derive(Clone, Copy)] +struct RowHash { + /// The number of the row, and below it the tag. + hash: u32, + /// 0 if there are none. + second_tag: u8, +} + +/// What it takes to hash a position. +#[derive(Clone, Copy)] +struct RowHasher(Hash8); + +impl RowHasher { + /// The hash of the position that `bytes` is the 8 bytes of. + #[inline(always)] + const fn of(self, bytes: u64) -> RowHash { + RowHash { + hash: self.0.of(bytes), + // The tag depends on the first bytes already. + second_tag: if SECOND_TAGS { + (((bytes >> 32) as u32).wrapping_mul(2_654_435_761) >> 24) as u8 + } else { + 0 + }, + } + } +} + +/// A row of the tables. +struct Row<'a, const ROW_ENTRIES: usize> { + indices: &'a mut [u32; ROW_ENTRIES], + tags: &'a mut [u8; ROW_ENTRIES], + /// If they are in use. + second_tags: Option<&'a mut [u8; ROW_ENTRIES]>, +} + +impl Row<'_, ROW_ENTRIES> { + const NEXT_HEADS: [u8; 256] = row_next_heads(ROW_ENTRIES); + + /// Moves the head, and writes the entry that it is at. + #[inline(always)] + fn insert(&mut self, hash: RowHash, index: u32) { + let Some(head) = self.tags.first_mut() else { + return; + }; + let next = Self::NEXT_HEADS.get(usize::from(*head)).copied(); + let next = next.unwrap_or(1); + *head = next; + let next = usize::from(next) & (ROW_ENTRIES - 1); + if let Some(tag) = self.tags.get_mut(next) { + *tag = hash.hash as u8; + } + if let Some(second_tags) = &mut self.second_tags + && let Some(second_tag) = second_tags.get_mut(next) + { + *second_tag = hash.second_tag; + } + if let Some(entry) = self.indices.get_mut(next) { + *entry = index; + } + } +} + +/// Where the row for `hash` starts in the hash table. +#[inline(always)] +const fn rel_row(hash: u32) -> usize { + ((hash >> ZSTD_ROW_HASH_TAG_BITS) as usize) << ROW_ENTRIES.trailing_zeros() +} + +/// Where it starts in the tag table. +#[inline(always)] +const fn rel_tag_row(hash: u32) -> usize { + if SECOND_TAGS { + 2 * rel_row::(hash) + } else { + rel_row::(hash) + } +} + +/// The hash table and the tag table. +struct Rows<'a> { + indices: &'a mut [u32], + tags: &'a mut [u8], +} + +impl Rows<'_> { + /// The row for `hash`. + #[inline(always)] + fn row( + &mut self, + hash: u32, + ) -> Option> { + let start = rel_row::(hash); + let indices = self.indices.get_mut(start..start + ROW_ENTRIES)?; + let start = rel_tag_row::(hash); + let (tags, second_tags) = if SECOND_TAGS { + let tags = self.tags.get_mut(start..start + 2 * ROW_ENTRIES)?; + let (tags, second_tags) = tags.split_at_mut_checked(ROW_ENTRIES)?; + (tags, Some(second_tags.first_chunk_mut()?)) + } else { + (self.tags.get_mut(start..start + ROW_ENTRIES)?, None) + }; + Some(Row { + indices: indices.first_chunk_mut()?, + tags: tags.first_chunk_mut()?, + second_tags, + }) + } + + /// `ZSTD_row_prefetch`: of the row for `hash`. + #[inline(always)] + fn prefetch(&self, hash: u32) { + row_prefetch::(self.indices, self.tags, hash); + } +} + +/// `ZSTD_row_prefetch`: of the row for `hash`. +#[inline(always)] +fn row_prefetch( + indices: &[u32], + tags: &[u8], + hash: u32, +) { + if ROW_ENTRIES == 16 { + // Somewhere in the row, which is no more than a cache line. + let entry = (hash >> (ZSTD_ROW_HASH_TAG_BITS - 4)) as usize; + prefetch_l1(indices, entry); + prefetch_l1(tags, entry); + return; + } + let start = rel_row::(hash); + prefetch_l1(indices, start); + prefetch_l1(indices, start + 16); + if ROW_ENTRIES == 64 { + // The C code leaves this half out. + prefetch_l1(indices, start + 32); + prefetch_l1(indices, start + 48); + } + let start = rel_tag_row::(hash); + prefetch_l1(tags, start); + if SECOND_TAGS { + prefetch_l1(tags, start + ROW_ENTRIES); + } +} + +/// `ZSTD_row_update`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the rows of the +/// row-based match finder. There are 7 bytes from `ip` on. +#[inline(never)] +pub(crate) fn row_update(ms: &mut MatchState, prefix: Prefix<'_>, ip: usize) -> Result<()> { + #[inline(always)] + fn update( + mut rows: Rows<'_>, + hasher: RowHasher, + prefix: Prefix<'_>, + indices: Range, + ) -> Result<()> { + let first = prefix.pos_of(indices.start); + let positions = (prefix.buf).span(first, indices.len() + HASH_READ_SIZE - 1)?; + for (idx, bytes) in indices.zip(positions.array_windows()) { + let hash = hasher.of::(u64::from_le_bytes(*bytes)); + rows.row::(hash.hash) + .ok_or(BUG)? + .insert(hash, idx); + } + Ok(()) + } + + /// The prefix of a job can take 0.1 s, so a worker looks every so many positions whether there is any use in going on. + const PART: u32 = 1 << 16; + + row_flatten_tags(&mut ms.row_tags, &mut ms.tag_table)?; + let mls = ms.c_params.min_match.min(6); + let hash_bits = ms.row_hash_log + ZSTD_ROW_HASH_TAG_BITS; + let hasher = RowHasher(Hash8::new(mls, hash_bits, ms.hash_salt)); + let target = prefix.index_of(ip); + while ms.next_to_update < target { + ms.check_cancelled()?; + let indices = ms.next_to_update..ms.next_to_update.saturating_add(PART).min(target); + ms.next_to_update = indices.end; + let rows = Rows { + indices: &mut ms.hash_table, + tags: &mut ms.tag_table, + }; + match ms.c_params.search_log.clamp(4, 6) { + 4 => update::<16>(rows, hasher, prefix, indices)?, + 5 => update::<32>(rows, hasher, prefix, indices)?, + _ => update::<64>(rows, hasher, prefix, indices)?, + } + } + ms.next_to_update = target; + Ok(()) +} + +/// Bit `n` is set if the entry `n` has the tag `tag`. This becomes a comparison of vectors. +#[inline(always)] +fn tags_match_mask(tags: &[u8; 16], tag: u8) -> u16 { + let mut matches = 0; + for (n, &entry) in tags.iter().enumerate() { + matches |= u16::from(entry == tag) << n; + } + matches +} + +/// Bit `n` is set if the entry `n` has the tag `tag`. As `a | b << 16 | ..`, LTO makes 10 times the instructions of it. +#[inline(always)] +fn row_tags_match_mask(tags: &[u8; ROW_ENTRIES], tag: u8) -> u64 { + const { assert!(matches!(ROW_ENTRIES, 16 | 32 | 64)) } + let (chunks, _) = tags.as_chunks::<16>(); + chunks.iter().rev().fold(0, |matches, tags| { + matches << 16 | u64::from(tags_match_mask(tags, tag)) + }) +} + +/// `ZSTD_row_getMatchMask`: bit `n` is set if the entry `(head + n) % ROW_ENTRIES` has the tag `tag`. +#[inline(always)] +fn row_get_match_mask( + tags: &[u8; ROW_ENTRIES], + tag: u8, + head: u32, +) -> u64 { + let matches = row_tags_match_mask(tags, tag); + match ROW_ENTRIES { + 16 => u64::from((matches as u16).rotate_right(head)), + 32 => u64::from((matches as u32).rotate_right(head)), + _ => matches.rotate_right(head), + } +} + +/// [`row_get_match_mask`]. Inlined, levels 11 and 12 take a tenth longer for text. +#[inline(never)] +fn row_get_second_match_mask( + tags: &[u8; ROW_ENTRIES], + tag: u8, + head: u32, +) -> u64 { + row_get_match_mask(tags, tag, head) +} + +/// The bits of [`row_get_match_mask`], but for the one of the entry 0. +#[inline(always)] +const fn row_entries_mask(head: u32) -> u64 { + !(1 << (head.wrapping_neg() & (ROW_ENTRIES as u32 - 1))) +} + +/// The rows of a dictionary. +#[derive(Clone, Copy)] +struct RowDict<'a> { + indices: &'a [u32], + tags: &'a [u8], + hasher: Hash8, + /// The index, among its own, of its first byte. + lowest_index: u32, +} + +impl<'a> RowDict<'a> { + fn new(dms: &DictMatchState<'a>, mls: u32) -> Self { + Self { + indices: &dms.ms.hash_table, + tags: &dms.ms.tag_table, + hasher: Hash8::new(mls, dms.ms.row_hash_log + ZSTD_ROW_HASH_TAG_BITS, 0), + lowest_index: dms.ms.window.dict_limit, + } + } + + /// The entries of the row for `hash` that have its tag. + #[inline(always)] + fn candidates( + &self, + hash: u32, + ) -> Option> { + let start = rel_row::(hash); + let tags = self.tags.get(start..start + ROW_ENTRIES)?; + let tags = tags.first_chunk::()?; + let indices = self.indices.get(start..start + ROW_ENTRIES)?; + RowCandidates::new(indices.first_chunk()?, tags, hash as u8) + } +} + +/// The tables of the row-based match finder, and what it takes to add to them. +struct RowIndex<'a> { + rows: Rows<'a>, + /// The hashes of the 8 positions from `next_to_update` on, each at its index modulo 8. + hash_cache: [u32; ZSTD_ROW_HASH_CACHE_SIZE], + second_tag_cache: [u8; ZSTD_ROW_HASH_CACHE_SIZE], + hasher: RowHasher, + near: Near<'a>, + base_index: u32, + /// `ms.next_to_update`, as a position. + next_to_update: u32, + /// How many positions were inserted, if there are second tags. + nb_inserted: u32, +} + +/// The hash of the position at `index`, from the caches of [`RowIndex`], where `new_hash` takes its place: the hash of the +/// position 8 bytes further. +#[inline(always)] +fn replace_cached_hash( + hash_cache: &mut [u32; ZSTD_ROW_HASH_CACHE_SIZE], + second_tag_cache: &mut [u8; ZSTD_ROW_HASH_CACHE_SIZE], + index: u32, + new_hash: RowHash, +) -> RowHash { + let cached = (index & ZSTD_ROW_HASH_CACHE_MASK) as usize; + let (Some(hash), Some(second_tag)) = + (hash_cache.get_mut(cached), second_tag_cache.get_mut(cached)) + else { + return new_hash; + }; + RowHash { + hash: core::mem::replace(hash, new_hash.hash), + second_tag: if SECOND_TAGS { + core::mem::replace(second_tag, new_hash.second_tag) + } else { + 0 + }, + } +} + +impl RowIndex<'_> { + /// `ZSTD_row_nextCachedHash`: [`replace_cached_hash`] for `pos`, and a prefetch of the row for `new_hash`. + #[inline(always)] + fn next_cached_hash( + &mut self, + pos: u32, + new_hash: RowHash, + ) -> RowHash { + self.rows + .prefetch::(new_hash.hash); + replace_cached_hash::( + &mut self.hash_cache, + &mut self.second_tag_cache, + self.base_index.wrapping_add(pos), + new_hash, + ) + } + + /// `ZSTD_row_fillHashCache`: the hashes of the 8 positions from `next_to_update` on, or fewer so as not to go past + /// `limit`. + #[inline(never)] + fn fill_hash_cache( + &mut self, + limit: u32, + ) -> Option<()> { + let from = self.next_to_update; + let max_elems = (limit + 1).saturating_sub(from); + for pos in from..from + max_elems.min(ZSTD_ROW_HASH_CACHE_SIZE as u32) { + let hash = self.hasher.of::(self.near.read64(pos)?); + self.next_cached_hash::(pos, hash); + } + Some(()) + } + + /// `ZSTD_row_update_internalImpl`: inserts the positions from `next_to_update` up to `end`. `COPIES` is for the + /// largest functions, where the loop would read the fields again after every write to the tables. In the others it + /// does not, and with copies, 4 MB of base64 take 3% longer at level 11. + #[inline(always)] + fn insert_up_to( + &mut self, + end: u32, + ) -> Option<()> { + let bytes = self.near.bytes(); + if end as usize + 2 * ZSTD_ROW_HASH_CACHE_SIZE > bytes.len() { + return None; + } + if COPIES { + let (hasher, base_index) = (self.hasher, self.base_index); + let mut rows = Rows { + indices: &mut *self.rows.indices, + tags: &mut *self.rows.tags, + }; + for pos in self.next_to_update..end { + let ahead = pos as usize + ZSTD_ROW_HASH_CACHE_SIZE; + let ahead = bytes.get(ahead..ahead + 8)?.first_chunk()?; + let new_hash = hasher.of::(u64::from_le_bytes(*ahead)); + rows.prefetch::(new_hash.hash); + let index = base_index.wrapping_add(pos); + let hash = replace_cached_hash::( + &mut self.hash_cache, + &mut self.second_tag_cache, + index, + new_hash, + ); + rows.row::(hash.hash)? + .insert(hash, index); + } + } else { + for pos in self.next_to_update..end { + let ahead = pos as usize + ZSTD_ROW_HASH_CACHE_SIZE; + let ahead = bytes.get(ahead..ahead + 8)?.first_chunk()?; + let new_hash = self.hasher.of::(u64::from_le_bytes(*ahead)); + let hash = self.next_cached_hash::(pos, new_hash); + let index = self.base_index.wrapping_add(pos); + self.rows + .row::(hash.hash)? + .insert(hash, index); + } + } + if SECOND_TAGS { + self.nb_inserted += end.saturating_sub(self.next_to_update); + } + self.next_to_update = end; + Some(()) + } + + /// `ZSTD_row_update_internal`: inserts the positions from `next_to_update` up to `ip`, but not all of them if they + /// are many. See [`Self::insert_up_to`] for `COPIES`. + #[inline(always)] + fn update( + &mut self, + ip: u32, + ) -> Option<()> { + const K_SKIP_THRESHOLD: u32 = 384; + + if ip.wrapping_sub(self.next_to_update) > K_SKIP_THRESHOLD { + self.skip::(ip)?; + } + self.insert_up_to::(ip) + } + + /// The part of `ZSTD_row_update_internal` for a long way up to `ip`. + #[cold] + #[inline(never)] + fn skip(&mut self, ip: u32) -> Option<()> { + const K_MAX_MATCH_START_POSITIONS_TO_UPDATE: u32 = 96; + const K_MAX_MATCH_END_POSITIONS_TO_UPDATE: u32 = 32; + + let end = self.next_to_update + K_MAX_MATCH_START_POSITIONS_TO_UPDATE; + self.insert_up_to::(end)?; + self.next_to_update = ip - K_MAX_MATCH_END_POSITIONS_TO_UPDATE; + self.fill_hash_cache::(ip + 1) + } +} + +/// The entries of a row that may be matches for a position. +#[derive(Clone, Copy)] +struct RowCandidates<'a, const ROW_ENTRIES: usize> { + indices: &'a [u32; ROW_ENTRIES], + head: u32, + /// See [`row_get_match_mask`]. + matches: u64, +} + +impl<'a, const ROW_ENTRIES: usize> RowCandidates<'a, ROW_ENTRIES> { + /// The entries of a row that have the tag `tag`. + #[inline(always)] + fn new(indices: &'a [u32; ROW_ENTRIES], tags: &[u8; ROW_ENTRIES], tag: u8) -> Option { + let head = u32::from(*tags.first()?) & (ROW_ENTRIES as u32 - 1); + let matches = row_get_match_mask(tags, tag, head); + Some(Self { + indices, + head, + matches: matches & row_entries_mask::(head), + }) + } + + /// The most recent one. There is one. + #[inline(always)] + fn first(&self) -> u32 { + let entry = (self.head + self.matches.trailing_zeros()) as usize & (ROW_ENTRIES - 1); + self.indices.get(entry).copied().unwrap_or(0) + } + + #[inline(always)] + const fn remove_first(&mut self) { + self.matches &= self.matches.wrapping_sub(1); + } +} + +/// What the row-based match finder looks for matches in. +struct RowSearch<'a> { + block: LazyBlock<'a>, + nb_attempts: u32, + dms: Option>, + /// How many candidates the second tags ruled out. + nb_ruled_out: u32, +} + +impl RowSearch<'_> { + /// The part of `ZSTD_RowFindBestMatch` that follows the update of the tables. `candidates` is the entries of the + /// row that have the tag of `ip`, and `long_matches` those that have its second tag as well, if there are second + /// tags. `dict_hash` is the hash of `ip` for the rows of the dictionary, if there are. See [`Finder::find`]. + #[inline(always)] + fn longest_match( + &mut self, + mut candidates: RowCandidates<'_, ROW_ENTRIES>, + long_matches: Option, + dict_hash: u32, + ip: u32, + wanted: Wanted, + ) -> Option { + let block = &self.block; + let matches = candidates.matches; + let curr = block.base_index.wrapping_add(ip); + let low_limit = block.reach.low_limit(curr); + let closest = if DICT_MODE == DICT_MATCH_STATE { + candidates.first().max(low_limit) + } else { + candidates.first() + }; + let start_index = if DICT_MODE == EXT_DICT { + low_limit.max(block.base_index) + } else { + low_limit + }; + let segment = block + .near + .tail(start_index.wrapping_sub(block.base_index))?; + let floor = Best { + len: wanted.floor::(curr, closest), + off_base: 0, + }; + + // If there is a match that long, the others are of no interest. + if let Some(long_matches) = long_matches + && (long_matches != 0 || floor.len + 1 >= SECOND_TAG_LENGTH) + { + candidates.matches = long_matches; + self.nb_ruled_out += (matches ^ long_matches).count_ones(); + } + + loop { + let mut longest = Longest::new(segment, start_index, curr, floor)?; + let mut nb_attempts = self.nb_attempts; + let all = candidates.matches == matches; + + while candidates.matches != 0 && nb_attempts > 0 { + let match_index = candidates.first(); + candidates.remove_first(); + let mut offered = longest.offer::(match_index); + if offered == Offered::Outside { + if DICT_MODE != EXT_DICT || match_index < low_limit { + break; + } + let match_pos = (match_index - block.dict.start_index) as usize; + offered = longest.offer_in_dict(block, ip, match_pos).ok()?; + } + if offered == Offered::Longest { + return longest.best().found(); + } + nb_attempts -= 1; + } + + if DICT_MODE == DICT_MATCH_STATE { + let in_dict = self.dms?.candidates::(dict_hash)?; + self.longest_dict_match(in_dict, ip, nb_attempts, &mut longest)?; + } + let best = longest.best(); + if all || best.len >= SECOND_TAG_LENGTH || floor.len + 1 >= SECOND_TAG_LENGTH { + return best.found(); + } + // The second tags were the same for no reason. + candidates.matches = matches; + } + } + + /// [`Self::longest_match`] where the row has no candidate, and that of the dictionary has `in_dict`. + #[inline(always)] + fn longest_match_in_dict_alone( + &self, + in_dict: RowCandidates<'_, ROW_ENTRIES>, + ip: u32, + wanted: Wanted, + ) -> Option { + let block = &self.block; + let curr = block.base_index.wrapping_add(ip); + let low_limit = block.reach.low_limit(curr); + let segment = block.near.tail(low_limit.wrapping_sub(block.base_index))?; + let floor = Best { + len: wanted.floor::(curr, low_limit), + off_base: 0, + }; + let mut longest = Longest::new(segment, low_limit, curr, floor)?; + self.longest_dict_match(in_dict, ip, self.nb_attempts, &mut longest)?; + longest.best().found() + } + + /// The part of `ZSTD_RowFindBestMatch` that searches the dictionary. + #[inline(always)] + fn longest_dict_match( + &self, + mut candidates: RowCandidates<'_, ROW_ENTRIES>, + ip: u32, + mut nb_attempts: u32, + longest: &mut Longest<'_>, + ) -> Option<()> { + let block = &self.block; + let dms = self.dms?; + while candidates.matches != 0 && nb_attempts > 0 { + let match_index = candidates.first(); + candidates.remove_first(); + if match_index < dms.lowest_index { + break; + } + nb_attempts -= 1; + let match_pos = (match_index - dms.lowest_index) as usize; + if longest.offer_in_dict(block, ip, match_pos).ok()? == Offered::Longest { + break; + } + } + Some(()) + } +} + +/// What the row-based match finder leaves in the match state at the end of a block. +struct RowsSaved<'a> { + next_to_update: &'a mut u32, + hash_salt_entropy: &'a mut u32, + tags: &'a mut RowTags, +} + +/// The state of [`RowFinder`], which does not depend on the size of the rows. +struct RowState<'a> { + index: RowIndex<'a>, + search: RowSearch<'a>, + hash_salt_entropy: u32, + saved: RowsSaved<'a>, +} + +impl<'a> RowState<'a> { + fn new(ms: &'a mut MatchState, src: &BlockSrc<'a>, block: LazyBlock<'a>) -> Self { + let mls = ms.c_params.min_match.clamp(4, 6); + let hash_bits = ms.row_hash_log + ZSTD_ROW_HASH_TAG_BITS; + Self { + index: RowIndex { + rows: Rows { + indices: &mut ms.hash_table, + tags: &mut ms.tag_table, + }, + hash_cache: [0; ZSTD_ROW_HASH_CACHE_SIZE], + second_tag_cache: [0; ZSTD_ROW_HASH_CACHE_SIZE], + hasher: RowHasher(Hash8::new(mls, hash_bits, ms.hash_salt)), + near: block.near, + base_index: block.base_index, + next_to_update: ms.next_to_update.wrapping_sub(block.base_index), + nb_inserted: 0, + }, + search: RowSearch { + block, + // There are no more searches than entries in a row. + nb_attempts: 1 << ms.c_params.search_log.min(6), + dms: src.dms.as_ref().map(|dms| RowDict::new(dms, mls)), + nb_ruled_out: 0, + }, + hash_salt_entropy: ms.hash_salt_entropy, + saved: RowsSaved { + next_to_update: &mut ms.next_to_update, + hash_salt_entropy: &mut ms.hash_salt_entropy, + tags: &mut ms.row_tags, + }, + } + } + + /// The end of a block, or of a trial of the second tags. + fn finish(self) -> Result<()> { + let Self { + index, + search, + hash_salt_entropy, + saved, + } = self; + *saved.next_to_update = index.base_index.wrapping_add(index.next_to_update); + *saved.hash_salt_entropy = hash_salt_entropy; + if let RowTags::Second(stats) = saved.tags + && !stats.record_trial(index.nb_inserted, search.nb_ruled_out) + { + row_flatten_tags(saved.tags, index.rows.tags)?; + } + Ok(()) + } +} + +/// `ZSTD_RowFindBestMatch` +struct RowFinder<'a, const ROW_ENTRIES: usize, const SECOND_TAGS: bool, const DICT_MODE: u8>( + RowState<'a>, +); + +impl<'a, const ROW_ENTRIES: usize, const SECOND_TAGS: bool, const DICT_MODE: u8> Finder<'a> + for RowFinder<'a, ROW_ENTRIES, SECOND_TAGS, DICT_MODE> +{ + const MARGIN: u32 = 8 + ZSTD_ROW_HASH_CACHE_SIZE as u32; + const TRIAL: Option = if SECOND_TAGS { + Some(SECOND_TAG_TRIAL) + } else { + None + }; + + #[inline(always)] + fn new(ms: &'a mut MatchState, src: &'a BlockSrc<'a>, block: LazyBlock<'a>) -> Result { + let mut finder = Self(RowState::new(ms, src, block)); + finder.stop_skipping()?; + Ok(finder) + } + + #[inline(always)] + fn find( + &mut self, + ip: u32, + wanted: Wanted, + skipping: bool, + ) -> Result> { + let RowState { + index, + search, + hash_salt_entropy, + .. + } = &mut self.0; + + let mut dict_hash = 0; + if DICT_MODE == DICT_MATCH_STATE { + let dms = search.dms.ok_or(BUG)?; + dict_hash = dms.hasher.of(index.near.read64(ip).ok_or(BUG)?); + row_prefetch::(dms.indices, dms.tags, dict_hash); + } + + // Update the tables up to, and not including, `ip`. + let hash = if skipping { + let bytes = index.near.read64(ip).ok_or(BUG)?; + index.hasher.of::(bytes) + } else { + if index.next_to_update != ip { + if DICT_MODE == DICT_MATCH_STATE { + index + .update::(ip) + .ok_or(BUG)?; + } else { + index + .update::(ip) + .ok_or(BUG)?; + } + } + let ahead = index.near.read64(ip + ZSTD_ROW_HASH_CACHE_SIZE as u32); + let new_hash = index.hasher.of::(ahead.ok_or(BUG)?); + index.next_cached_hash::(ip, new_hash) + }; + *hash_salt_entropy = hash_salt_entropy.wrapping_add(hash.hash); + index.next_to_update = ip + 1; + if SECOND_TAGS { + index.nb_inserted += 1; + } + + let mut row = index + .rows + .row::(hash.hash) + .ok_or(BUG)?; + let candidates = RowCandidates::new(row.indices, row.tags, hash.hash as u8).ok_or(BUG)?; + let RowCandidates { head, matches, .. } = candidates; + let curr = index.base_index.wrapping_add(ip); + if matches == 0 { + if DICT_MODE == DICT_MATCH_STATE { + let dms = search.dms.ok_or(BUG)?; + let in_dict = dms.candidates::(dict_hash).ok_or(BUG)?; + if in_dict.matches != 0 { + let found = search + .longest_match_in_dict_alone::(in_dict, ip, wanted); + row.insert(hash, curr); + return Ok(found); + } + } + row.insert(hash, curr); + return Ok(None); + } + let second_tags = row.second_tags.as_ref(); + let long_matches = second_tags + .map(|tags| matches & row_get_second_match_mask(tags, hash.second_tag, head)); + let found = search.longest_match::( + candidates, + long_matches, + dict_hash, + ip, + wanted, + ); + row.insert(hash, curr); + Ok(found) + } + + #[inline(always)] + fn stop_skipping(&mut self) -> Result<()> { + let ilimit = self.0.index.near.len() - Self::MARGIN; + self.0 + .index + .fill_hash_cache::(ilimit) + .ok_or(BUG) + } + + #[inline(always)] + fn finish(self) -> Result<()> { + self.0.finish() + } +} + +// ── the parsers ─────────────────────────────────────────────────────────── + +/// How long a match the repeat offset `offset` gives at `ip`: at least 4, or 0 if it gives none. +#[inline(always)] +fn rep_length(block: &LazyBlock<'_>, ip: u32, offset: u32) -> Result { + let near = block.near; + let ip_bytes = near.read32(ip).ok_or(BUG)?; + if DICT_MODE == NO_DICT { + let rep_match = ip.wrapping_sub(offset); + if offset > 0 && near.read32(rep_match) == Some(ip_bytes) { + return Ok(near.count(ip + 4, rep_match + 4) + 4); + } + return Ok(0); + } + + let curr = block.base_index.wrapping_add(ip); + let rep_index = curr.wrapping_sub(offset); + if !index_overlap_check(block.base_index, rep_index) { + return Ok(0); + } + // The offset may lead to what is no longer in the window. + if DICT_MODE == EXT_DICT && offset > curr.wrapping_sub(block.reach.low_limit(curr)) { + return Ok(0); + } + if rep_index < block.base_index { + let rep_match = rep_index.wrapping_sub(block.dict.start_index) as usize; + if read_le32(block.dict.bytes, rep_match)? != ip_bytes { + return Ok(0); + } + return Ok(block.count_from_dict(ip + 4, rep_match + 4)? + 4); + } + let rep_match = rep_index - block.base_index; + if near.read32(rep_match) != Some(ip_bytes) { + return Ok(0); + } + Ok(near.count(ip + 4, rep_match + 4) + 4) +} + +/// The repeat offsets at the start of a block. +#[derive(Clone, Copy)] +struct Offsets { + offset_1: u32, + offset_2: u32, + /// What they were, for those that are too far to be of use and are 0. + saved_1: u32, + saved_2: u32, +} + +/// Where the parser starts in a block, or goes on, and with what. +#[derive(Clone, Copy)] +struct Start { + ip: u32, + anchor: u32, + skipping: bool, + offsets: Offsets, +} + +/// How far the parser got. +enum Parsed { + /// To the end of the block, the last literals of which are this many. + Block(usize), + /// To the end of [`Finder::TRIAL`]. + Trial(Start), +} + +impl Start { + fn new( + ms: &MatchState, + block: &LazyBlock<'_>, + rep: &[u32; ZSTD_REP_NUM], + dict_mode: DictMode, + ) -> Self { + let [offset_1, offset_2, _] = *rep; + let mut offsets = Offsets { + offset_1, + offset_2, + saved_1: 0, + saved_2: 0, + }; + // As in the C code, what counts as nothing to refer to is not the same in all modes. + let nothing_before = + block.istart == 0 && (dict_mode == DictMode::ExtDict || block.dict.bytes.is_empty()); + let ip = block.istart + u32::from(nothing_before); + if dict_mode == DictMode::NoDict { + let curr = block.base_index.wrapping_add(ip); + let window_low = ms.get_lowest_prefix_index(curr, ms.c_params.window_log); + let max_rep = curr - window_low; + if offset_2 > max_rep { + offsets.saved_2 = offset_2; + offsets.offset_2 = 0; + } + if offset_1 > max_rep { + offsets.saved_1 = offset_1; + offsets.offset_1 = 0; + } + } + Self { + ip, + anchor: block.istart, + skipping: false, + offsets, + } + } +} + +/// `ZSTD_compressBlock_lazy_generic` and `ZSTD_compressBlock_lazy_extDict_generic`. `depth` is how many positions +/// further a better match is looked for, once there is one, and `LAZY` whether that is any. From `args.start` up to the end +/// of the block, or of [`Finder::TRIAL`]. +#[inline(never)] +fn compress_block_lazy_part<'a, F: Finder<'a>, const LAZY: bool, const DICT_MODE: u8>( + args: LazyArgs<'a>, + depth: u32, +) -> Result { + let LazyArgs { + ms, + seq_store, + rep, + src, + block, + start, + } = args; + let block = █ + let near = block.near; + let Some(ilimit) = near.len().checked_sub(F::MARGIN) else { + // There is no position to search, nor to hash. + return Ok(Parsed::Block((near.len() - start.anchor) as usize)); + }; + let mut finder = F::new(ms, src, *block)?; + let mut seq_store = SeqSink::<16, true>::new(seq_store); + let mut ip = start.ip; + let mut anchor = start.anchor; + let stop = match F::TRIAL { + Some(trial) => ilimit.min(ip + trial), + None => ilimit, + }; + let Offsets { + mut offset_1, + mut offset_2, + saved_1, + mut saved_2, + } = start.offsets; + let mut skipping = start.skipping; + + 'parse: while ip < stop { + let mut no_repeat = false; + if !skipping { + let skipped = finder.skip_literals(LiteralRun { + ip, + anchor, + ilimit, + offset_1, + })?; + ip = skipped.ip; + no_repeat = skipped.no_repeat; + if ip >= ilimit { + break; + } + } + + // Check the repeat offset. + let mut best = Found::repeat(if !LAZY && no_repeat { + 0 + } else { + rep_length::(block, ip + 1, offset_1)? + }); + let mut start = ip + 1; + + if LAZY || best.len == 0 { + // How many positions were searched since the last match that was taken, that one included. + let mut since_best = 0; + let mut wanted = Wanted { + longer_than: best.len.max(3), + gain: Wanted::ANY_GAIN, + }; + + loop { + since_best += 1; + let found = if LAZY { + finder.find::(ip, wanted, skipping)? + } else { + finder.find::(ip, wanted, skipping)? + }; + // At first any gain will do, that of a match that is not wanted too: one that is no longer than the repeat + // at the next position. It loses to that repeat at the next step. + if let Some(found) = found + && (!LAZY || found.gain() > wanted.gain) + { + debug_assert!( + found.len > wanted.longer_than || (LAZY && depth > 0 && ip < ilimit) + ); + best = found; + start = ip; + since_best = 1; + } + + if best.len < 4 { + // Jump faster over incompressible sections. + let step = (ip - anchor) >> K_SEARCH_STRENGTH; + ip += step + 1; + // Far into one, positions are no longer all inserted: it is faster, and keeps positions that are + // of no use out of the tables. As in the C code, it does not start at the same step in all modes. + skipping = if DICT_MODE == EXT_DICT { + step > K_LAZY_SKIPPING_STEP + } else { + step + 1 > K_LAZY_SKIPPING_STEP + }; + continue 'parse; + } + + // Look for something better at the next positions. + if !LAZY || since_best > depth || ip >= ilimit { + break; + } + ip += 1; + let rep_len = rep_length::(block, ip, offset_1)?; + if rep_len >= 4 { + let weight = 2 + since_best; + let gain2 = (rep_len * weight) as i32; + let gain1 = (best.len * weight) as i32 - best.cost() + 1; + if gain2 > gain1 { + best = Found::repeat(rep_len); + start = ip; + } + } + wanted = Wanted { + longer_than: 3, + gain: best.gain() + 1 + 3 * since_best as i32, + }; + } + + // Catch up: the match may start earlier. + if offbase_is_offset(best.off_base.get()) { + let offset = offbase_to_offset(best.off_base.get()); + let match_index = block.base_index.wrapping_add(start).wrapping_sub(offset); + let (segment, mut match_pos) = + if DICT_MODE != NO_DICT && match_index < block.base_index { + (block.dict.bytes, match_index - block.dict.start_index) + } else { + (near.bytes(), start - offset) + }; + while start > anchor + && match_pos > 0 + && near.read8(start - 1) == segment.get(match_pos as usize - 1).copied() + { + start -= 1; + match_pos -= 1; + best.len += 1; + } + offset_2 = offset_1; + offset_1 = offset; + } + } + + // Store the sequence. + seq_store.store_seq(start - anchor, near, anchor, best.off_base.get(), best.len)?; + ip = start + best.len; + anchor = ip; + + if skipping { + // There was a match: every position is inserted again. + finder.stop_skipping()?; + skipping = false; + } + + // Check the second repeat offset right away. + while ip <= ilimit { + let rep_len = rep_length::(block, ip, offset_2)?; + if rep_len == 0 { + break; + } + (offset_1, offset_2) = (offset_2, offset_1); + seq_store.store_seq(0, near, anchor, REPCODE1_TO_OFFBASE, rep_len)?; + ip += rep_len; + anchor = ip; + } + } + finder.finish()?; + if F::TRIAL.is_some() && ip < ilimit { + return Ok(Parsed::Trial(Start { + ip, + anchor, + skipping, + offsets: Offsets { + offset_1, + offset_2, + saved_1, + saved_2, + }, + })); + } + + // If `offset_1` was not valid at first and now is, what it was becomes the second repeat offset. + if saved_1 != 0 && offset_1 != 0 { + saved_2 = saved_1; + } + + // Save the repeat offsets for the next block. + let [rep_0, rep_1, _] = rep; + *rep_0 = if offset_1 != 0 { offset_1 } else { saved_1 }; + *rep_1 = if offset_2 != 0 { offset_2 } else { saved_2 }; + + Ok(Parsed::Block((near.len() - anchor) as usize)) +} + +/// [`compress_block_lazy_part`] with a finder that is not on trial. Returns the size of the last literals. +#[inline(always)] +fn compress_block_lazy_generic<'a, F: Finder<'a>, const LAZY: bool, const DICT_MODE: u8>( + args: LazyArgs<'a>, + depth: u32, +) -> Result { + const { assert!(F::TRIAL.is_none()) }; + match compress_block_lazy_part::(args, depth)? { + Parsed::Block(last_literals) => Ok(last_literals), + Parsed::Trial(_) => Err(BUG), + } +} + +/// What [`compress_block_lazy_part`] works on. +struct LazyArgs<'a> { + ms: &'a mut MatchState, + seq_store: &'a mut SeqStore, + rep: &'a mut [u32; ZSTD_REP_NUM], + src: &'a BlockSrc<'a>, + block: LazyBlock<'a>, + start: Start, +} + +impl LazyArgs<'_> { + fn reborrow(&mut self) -> LazyArgs<'_> { + LazyArgs { + ms: self.ms, + seq_store: self.seq_store, + rep: self.rep, + src: self.src, + block: self.block, + start: self.start, + } + } +} + +/// [`compress_block_lazy_generic`], with the hash chains or the rows. +#[inline(always)] +fn compress_block_lazy_hashed( + mut args: LazyArgs<'_>, + depth: u32, + use_row: bool, +) -> Result { + if !use_row { + if DICT_MODE == NO_DICT && chains_are_simple(args.ms, &args.block) { + return compress_block_lazy_generic::, LAZY, NO_DICT>( + args, depth, + ); + } + return compress_block_lazy_generic::, LAZY, DICT_MODE>( + args, depth, + ); + } + // With a dictionary, the second tags are given up: how many candidates there are matters if it is searched next. + if DICT_MODE != NO_DICT { + row_flatten_tags(&mut args.ms.row_tags, &mut args.ms.tag_table)?; + } + match args.ms.c_params.search_log.clamp(4, 6) { + 4 => compress_block_lazy_generic::, LAZY, DICT_MODE>( + args, depth, + ), + 5 => compress_block_lazy_generic::, LAZY, DICT_MODE>( + args, depth, + ), + _ => { + while DICT_MODE == NO_DICT && args.ms.row_tags != RowTags::Flat { + let part = args.reborrow(); + match compress_block_lazy_part::, LAZY, NO_DICT>( + part, depth, + )? { + Parsed::Block(last_literals) => return Ok(last_literals), + Parsed::Trial(start) => args.start = start, + } + } + compress_block_lazy_generic::, LAZY, DICT_MODE>( + args, depth, + ) + } + } +} + +/// [`compress_block_lazy`], for a mode that is a constant. +#[inline(always)] +fn compress_block_lazy_any_strategy( + args: LazyArgs<'_>, + strategy: Strategy, + use_row: bool, +) -> Result { + match strategy { + Strategy::Greedy => compress_block_lazy_hashed::(args, 0, use_row), + Strategy::Lazy => compress_block_lazy_hashed::(args, 1, use_row), + Strategy::Lazy2 => compress_block_lazy_hashed::(args, 2, use_row), + // In the first block, the hints take more time than they save. + Strategy::BtLazy2 if !use_row && args.block.near.len() > ZSTD_BLOCKSIZE_MAX as u32 => { + compress_block_lazy_generic::, true, DICT_MODE>(args, 2) + } + Strategy::BtLazy2 if !use_row => { + compress_block_lazy_generic::, true, DICT_MODE>(args, 2) + } + _ => Err(BUG), + } +} + +/// `ZSTD_compressBlock_greedy`, `ZSTD_compressBlock_lazy2_dictMatchState_row` and all the others of the file: +/// `strategy` is one of the four, and `use_row` is false for `btlazy2`. +pub(crate) fn compress_block_lazy( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + strategy: Strategy, + use_row: bool, + dict_mode: DictMode, +) -> Result { + let block = LazyBlock::new(ms, src, dict_mode)?; + let args = LazyArgs { + start: Start::new(ms, &block, rep, dict_mode), + ms, + seq_store, + rep, + src, + block, + }; + match dict_mode { + DictMode::NoDict => compress_block_lazy_any_strategy::(args, strategy, use_row), + DictMode::ExtDict => compress_block_lazy_any_strategy::(args, strategy, use_row), + DictMode::DictMatchState => { + compress_block_lazy_any_strategy::(args, strategy, use_row) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn second_tags_are_kept_for_as_long_as_they_rule_out_enough() { + const TRIAL: u32 = SECOND_TAG_TRIAL; + const TRIALS_IN_A_BLOCK: usize = ZSTD_BLOCKSIZE_MAX / TRIAL as usize; + // How many of `trials`, of so many positions and candidates that were ruled out, they get through. + let nb_kept = |trials: &[(u32, u32)]| { + let mut stats = SecondTagStats::AT_START; + trials + .iter() + .take_while(|&&(nb_positions, nb_ruled_out)| { + stats.record_trial(nb_positions, nb_ruled_out) + }) + .count() + }; + let (none, little, some, a_lot) = ( + (TRIAL, 0), + (TRIAL, TRIAL / 2), + (TRIAL, TRIAL), + (TRIAL, 4 * TRIAL), + ); + + assert_eq!(nb_kept(&[none]), 0); + assert_eq!(nb_kept(&[(TRIAL, TRIAL / 2 - 1)]), 0); + assert_eq!(nb_kept(&[little, none]), 1); + // What came before makes up for a stretch with little. + assert_eq!(nb_kept(&[some, none, a_lot, none, none, none, none]), 7); + assert_eq!(nb_kept(&[some, none, none]), 2); + // More is asked for once the rows had the time to fill up. + assert_eq!( + nb_kept(&[some; 2 * TRIALS_IN_A_BLOCK]), + TRIALS_IN_A_BLOCK - 1 + ); + assert_eq!( + nb_kept(&[a_lot; 2 * TRIALS_IN_A_BLOCK]), + 2 * TRIALS_IN_A_BLOCK + ); + // It takes some positions to tell. + let few = (TRIAL / 4, 0); + assert_eq!(nb_kept(&[few, few]), 1); + assert_eq!(nb_kept(&[few, (TRIAL / 4, TRIAL), few]), 3); + } + + #[test] + fn positions_are_inserted_in_the_rows_the_same_with_copies_of_the_fields() -> Result<()> { + type Inserted = (Vec, Vec, [u32; 8], [u8; 8], u32, u32); + fn inserted( + bytes: &[u8], + ) -> Result { + const ROW_HASH_LOG: u32 = 4; + let mut indices = vec![0; ROW_ENTRIES << ROW_HASH_LOG]; + let mut tags = vec![0; 2 * indices.len()]; + let hash_bits = ROW_HASH_LOG + ZSTD_ROW_HASH_TAG_BITS; + let mut index = RowIndex { + rows: Rows { + indices: &mut indices, + tags: &mut tags, + }, + hash_cache: [0; ZSTD_ROW_HASH_CACHE_SIZE], + second_tag_cache: [0; ZSTD_ROW_HASH_CACHE_SIZE], + hasher: RowHasher(Hash8::new(4, hash_bits, 7)), + near: Near::new(bytes)?, + base_index: 1_003, + next_to_update: 2, + nb_inserted: 0, + }; + index + .fill_hash_cache::(400) + .ok_or(BUG)?; + for end in [3, 60, 60, 400] { + index + .insert_up_to::(end) + .ok_or(BUG)?; + } + let (hashes, second_tags) = (index.hash_cache, index.second_tag_cache); + let (next, nb_inserted) = (index.next_to_update, index.nb_inserted); + Ok((indices, tags, hashes, second_tags, next, nb_inserted)) + } + + let mut x = 9_u32; + let bytes: Vec = core::iter::repeat_with(|| { + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + (x >> 30) as u8 + }) + .take(500) + .collect(); + let with_copies = inserted::<16, false, true>(&bytes)?; + assert_eq!(with_copies.4, 400); + assert!(with_copies.0.contains(&(1_003 + 399))); + assert_eq!(with_copies, inserted::<16, false, false>(&bytes)?); + assert_eq!( + inserted::<32, false, true>(&bytes)?, + inserted::<32, false, false>(&bytes)? + ); + assert_eq!( + inserted::<64, false, true>(&bytes)?, + inserted::<64, false, false>(&bytes)? + ); + assert_eq!( + inserted::<64, true, true>(&bytes)?, + inserted::<64, true, false>(&bytes)? + ); + Ok(()) + } + + #[test] + fn many_positions_are_added_to_the_trees_as_one_by_one() -> Result<()> { + const DICT_LIMIT: u32 = 2; + const HASH_LOG: u32 = 6; + let bytes: Vec = (0..300u32).map(|n| (n * n % 7) as u8).collect(); + let near = Near::new(&bytes)?; + for min_match in [4, 5] { + let hasher = Hasher::new(HASH_LOG, min_match); + // They go around the table more than once, or stop at its end, or short of it. + for indices in [5..280, 40..64, 70..90] { + let (mut hash_table, mut chain_table) = ([1; 1 << HASH_LOG], [9; 64]); + let (mut expected_hash_table, mut expected_chain_table) = (hash_table, chain_table); + for idx in indices.clone() { + let h = hasher.hash(near.read64(idx - DICT_LIMIT).ok_or(BUG)?); + let children = [table_get(&expected_hash_table, h), ZSTD_DUBT_UNSORTED_MARK]; + bt_set_children(bt_nodes(&mut expected_chain_table), idx, children); + table_put(&mut expected_hash_table, h, idx); + } + + let bt = bt_nodes(&mut chain_table); + update_many(&mut hash_table, bt, near, hasher, indices, DICT_LIMIT)?; + assert_eq!(hash_table, expected_hash_table); + assert_eq!(chain_table, expected_chain_table); + } + } + Ok(()) + } + + #[test] + fn a_long_candidate_that_falls_short_is_what_the_next_ones_have_to_beat() -> Result<()> { + const RECORD: u32 = 200; + let best = |longest: &Longest<'_>| (longest.best().len, longest.best().off_base); + + for field in [LONG_WAY - 1, LONG_WAY, 100] { + let mut bytes: Vec = (0..5 * RECORD).map(|n| (n % RECORD) as u8).collect(); + for (record, value) in (0..).zip([250, 251, 252, 250, 253]) { + *bytes.at_mut((record * RECORD + field) as usize)? = value; + } + // The tail is after the field, where all records are the same. + let floor = Best { + len: field + 50, + off_base: 0, + }; + let mut longest = Longest::new(Near::new(&bytes)?, 0, 3 * RECORD, floor).ok_or(BUG)?; + let up_to_the_field = if field >= LONG_WAY { + (field, offset_to_offbase(RECORD)) + } else { + (floor.len, 0) + }; + for record in [2, 1] { + assert!(longest.offer::(record * RECORD) == Offered::Considered); + assert_eq!(best(&longest), up_to_the_field); + } + if field > LONG_WAY { + assert!(longest.offer_match(field - 1, 1) == Offered::Considered); + assert_eq!(best(&longest), up_to_the_field); + } + + let past_the_field = (RECORD + field, offset_to_offbase(3 * RECORD)); + assert!(longest.offer::(0) == Offered::Considered); + assert_eq!(best(&longest), past_the_field); + assert!(longest.offer_match(RECORD, 1) == Offered::Considered); + assert_eq!(best(&longest), past_the_field); + } + Ok(()) + } +} diff --git a/src/zstd/compress/ldm.rs b/src/zstd/compress/ldm.rs new file mode 100644 index 000000000000..70b058b95571 --- /dev/null +++ b/src/zstd/compress/ldm.rs @@ -0,0 +1,754 @@ +//! `zstd_ldm.c`: long distance matching. + +use super::internal::{ + BlockSrc, Buffers, CompressionParameters, HASH_READ_SIZE, LdmEntry, LdmParams, LdmState, + MatchState, ParamSwitch, Prefix, RawSeq, RawSeqStore, SeqStore, Strategy, TableFillPurpose, + count, count_2segments, offset_to_offbase, +}; +use super::params::{ZSTD_HASHLOG_MAX, ZSTD_HASHLOG_MIN, ZSTD_LDM_BUCKETSIZELOG_MAX}; +use super::{compress_block, double_fast, fast}; +use crate::common::ZSTD_REP_NUM; +use crate::common::xxhash::Xxh64; +use crate::error::{Error, Result}; +use crate::util::{BUG, SliceExt}; + +const LDM_BUCKET_SIZE_LOG: u32 = 4; +const LDM_MIN_MATCH_LENGTH: u32 = 64; +const LDM_BATCH_SIZE: usize = 64; + +/// `ZSTD_ldm_gearTab` +#[rustfmt::skip] +static GEAR_TAB: [u64; 256] = [ + 0xf5b8_f72c_5f77_775c, 0x8493_5f26_6b7a_c412, 0xb647_ada9_ca73_0ccc, 0xb065_bb4b_114f_b1de, + 0x3458_4e7e_8c3a_9fd0, 0x4e97_e17c_6ae2_6b05, 0x3a03_d743_bc99_a604, 0xcecd_0424_22c4_044f, + 0x76de_76c5_8524_259e, 0x9c85_28f6_5bad_eaca, 0x8656_3706_e209_7529, 0x2902_475f_a375_d889, + 0xafb3_2a97_39a5_ebe6, 0xce27_14da_3883_e639, 0x021e_af82_1722_e69e, 0x0003_7b62_8620_b628, + 0x049a_8d45_5d88_caf5, 0x8556_d711_e695_8140, 0x04f7_ae74_fc60_5c1f, 0x829f_0c34_68bd_3a20, + 0x4ffd_c885_c625_179e, 0x8473_de04_8a3d_af1b, 0x5100_8822_b056_46b2, 0x69d7_5d12_b2d1_cc5f, + 0x8c9d_4a19_1591_54bc, 0xc3cc_10f4_abbd_4003, 0xd06d_dc1c_ecb9_7391, 0xbe48_e6e7_ed80_302e, + 0x3481_db31_cee0_3547, 0xacc3_f67c_daa1_d210, 0x65cb_771d_8c7f_96cc, 0x8eb2_7177_0557_23dd, + 0xc789_950d_44cd_94be, 0x934f_eadc_3700_b12b, 0x5e48_5f11_edbd_f182, 0x1e2e_2a46_fd64_767a, + 0x2969_ca71_d82e_fa7c, 0x9d46_e993_5ebb_ba2e, 0xe056_b67e_05e6_822b, 0x94d7_3f55_739d_03a0, + 0xcd70_10bd_b69b_5a03, 0x455e_f9fc_d79b_82f4, 0x869c_b54a_8749_c161, 0x38d1_a4fa_6185_d225, + 0xb475_166f_94bb_e9bb, 0xa414_3548_7209_59f1, 0x7aed_4780_ba6b_26ba, 0xd0ce_2644_39e0_2312, + 0x8436_6d74_6078_d508, 0xa8ce_973c_72ed_17be, 0x21c3_23a2_9a43_0b01, 0x9962_d617_e3af_80ee, + 0xab0c_e91d_9c8c_f75b, 0x530e_8ee6_d19a_4dbc, 0x2ef6_8c0c_f53f_5d72, 0xc03a_6816_40a8_5506, + 0x496e_4e9f_9c31_0967, 0x7858_0472_b59b_14a0, 0x2738_24c2_3b38_8577, 0x66bf_923a_d45c_b553, + 0x47ae_1a5a_2492_ba86, 0x35e3_0456_9e22_9659, 0x4765_182a_4687_0b6f, 0x6cba_b625_e909_9412, + 0xddac_9a2e_5985_22c1, 0x7172_086e_6666_24f2, 0xdf50_03ca_503b_7837, 0x88c0_c1db_7856_3d09, + 0x58d5_1865_acfc_289d, 0x1776_71ae_c652_24f1, 0xfb79_d8a2_41e9_67d7, 0x2be1_e101_cad9_a49a, + 0x6625_682f_6e29_186b, 0x3995_5345_7ac0_6e50, 0x035d_ffb4_c23a_bb74, 0x429d_b259_1f54_aade, + 0xc528_02a8_037d_1009, 0x6acb_2738_1f0b_25f3, 0xf45e_2551_ee4f_823b, 0x8b0e_a2d9_9580_c2f7, + 0x3bed_519c_bcb4_e1e1, 0x0ff4_5282_3dbb_010a, 0x9d42_ed61_4f3d_d267, 0x5b93_13c0_6257_c57b, + 0xa114_b800_8b5e_1442, 0xc1fe_311c_11c1_3d4b, 0x66e8_763e_a34c_5568, 0x8b98_2af1_c262_f05d, + 0xee88_76fa_aa75_fbb7, 0x8a62_a4d0_d172_bb2a, 0xc13d_94a3_b744_9a97, 0x6dbb_ba9d_c15d_037c, + 0xc786_101f_1d92_e0f1, 0xd786_81a9_07a0_b79b, 0xf61a_af29_62c9_abb9, 0x2cfd_16fc_d3cb_7ad9, + 0x868c_5b67_4462_4d21, 0x25e6_5089_9c74_ddd7, 0xba04_2af4_a7c3_7463, 0x4eb1_a539_465a_3eca, + 0xbe09_dbf0_3b05_d5ca, 0x774e_5a36_2b54_72ba, 0x47a1_2212_29d1_83cd, 0x504b_0ca1_8ef5_a2df, + 0xdffb_dfbd_e245_6eb9, 0x46cd_2b2f_bee3_4634, 0xf2ae_f8fe_819d_98c3, 0x357f_5276_d459_9d61, + 0x24a5_4838_79c4_53e3, 0x0880_2688_9192_b4b9, 0x28da_9667_1782_dbec, 0x4ef3_7c40_588e_9aaa, + 0x8837_b906_51bc_9fb3, 0xc164_f741_d3f0_e5d6, 0xbc13_5a0a_704b_70ba, 0x069c_d868_f762_2ada, + 0xbc37_ba89_e0b9_c0ab, 0x47c1_4a01_3235_52f6, 0x4f00_794b_acee_98bb, 0x7107_de7d_637a_69d5, + 0x88af_793b_b6f2_255e, 0xf3c6_466b_8799_b598, 0xc288_c616_aa7f_3b59, 0x81ca_63cf_42fc_a3fd, + 0x88d8_5ace_36a2_674b, 0x0d05_6bd3_7923_89e7, 0xe55c_396c_4e9d_d32d, 0xbefb_5045_71e6_c0a6, + 0x96ab_3211_5e91_e8cc, 0xbf8a_cb18_de8f_38d1, 0x66da_e588_0167_2606, 0x833b_6017_8723_17fb, + 0xb87c_16f2_d1c9_2864, 0xdb76_6a74_e58b_669c, 0x8965_9f85_c614_17be, 0xc8da_ad85_6011_ea0c, + 0x76a4_b565_b6fe_7eae, 0xa469_d085_f623_7312, 0xaaf0_3656_83a3_e96c, 0x4dbb_746f_8424_f7b8, + 0x0638_755a_f4e4_acc1, 0x3d78_07f5_bde6_4486, 0x17be_6d8f_5bbb_7639, 0x0903_f0cd_44dc_35dc, + 0x67b6_72ea_fdf1_196c, 0xa676_ff93_ed4c_82f1, 0x521d_1004_c505_3d9d, 0x37ba_9ad0_9ccc_9202, + 0x84e5_4d29_7aac_fb51, 0x0a0b_4b77_6a14_3445, 0x0820_d471_e20b_348e, 0x1874_383c_b83d_46dc, + 0x97ed_eec7_a1ef_e11c, 0xb330_e50b_1bdc_42aa, 0x1dd9_1955_ce70_e032, 0xa514_cdb8_8f29_39d5, + 0x2791_233f_d90d_b9d3, 0x7b67_0a4c_c50f_7a9b, 0x77c0_7d2a_05c6_dfa5, 0xe377_8b66_46d0_a6fa, + 0xb39c_8eda_47b5_6749, 0x933e_d448_addb_ef28, 0xaf84_6af6_ab7d_0bf4, 0x0e5a_f208_eb66_6e49, + 0x5e66_22f7_3534_cd6a, 0x297d_aeca_42ef_5b6e, 0x862d_aef3_d355_39a6, 0xe687_2249_8f8e_1ea9, + 0x981c_5309_3dc0_d572, 0xfa09_b0bf_bf86_fbf5, 0x30b1_e961_6621_9f15, 0x70e7_d466_bdc4_fb83, + 0x5a66_736e_35f2_a8e9, 0xcddb_59d2_b7c1_baef, 0xd6c7_d247_d26d_8996, 0xea4e_39ea_c8de_1ba3, + 0x539c_8bb1_9fa3_aff2, 0x09f9_0e4c_5fd5_08d8, 0xa34e_5956_fbaf_3385, 0x2e2f_8e15_1d3e_f375, + 0x1736_91e9_b83f_aec1, 0xb85a_8d56_bf01_6379, 0x8382_3812_6740_8ae3, 0xb90f_901b_bdc0_096d, + 0x7c6a_d329_33bc_ec65, 0x76bb_5e2f_2c8a_d595, 0x390f_851a_6cf4_6d28, 0xc3e6_064d_a1c2_da72, + 0xc52a_0c10_1cfa_5389, 0xd78e_af84_a3fb_c530, 0x3781_b9e2_288b_997e, 0x73c2_f6de_a83d_05c4, + 0x0422_8e36_4c5b_5ed7, 0x9d7a_3edf_0da4_3911, 0x8edc_feda_2468_6756, 0x5e76_67a7_b7a9_b3a1, + 0x4c4f_389f_a143_791d, 0xb08b_c102_3da7_cddc, 0x7ab4_be3a_e529_b1cc, 0x754e_6132_dbe7_4ff9, + 0x7163_5442_a839_df45, 0x2f6f_b164_3fbe_52de, 0x961e_0a42_cf7a_8177, 0xf3b4_5d83_d89e_f2ea, + 0xee3d_e4cf_4a6e_3e9b, 0xcd68_4854_2c32_95e7, 0xe4ce_e166_4c78_662f, 0x9947_548b_474c_68c4, + 0x25d7_3777_a5ed_8b0b, 0x00c9_15b1_d636_b7fc, 0x21c2_ba75_d9b0_d2da, 0x5f6b_5dcf_608a_64a1, + 0xdcf3_3325_5ff9_570c, 0x633b_9224_18ce_d4ee, 0xc136_dde0_b004_b34a, 0x58cc_83b0_5d4b_2f5a, + 0x5eb4_24dd_a28e_42d2, 0x62df_4736_9739_cd98, 0xb4e0_b424_85e4_ce17, 0x16e1_f0c1_f9a8_d1e7, + 0x8ec3_9167_0756_0ebf, 0x62ba_6e2d_f2cc_9db3, 0xcbf9_f4ff_77d8_3a16, 0x78d9_d7d0_7d2b_bcc4, + 0xef55_4ce1_e02c_41f4, 0x8d75_8112_7ecc_f94d, 0xa9b5_3336_cb3c_8a05, 0x38c4_2c0b_f45c_4f91, + 0x6408_93cd_f448_8863, 0x80ec_34bc_575e_a568, 0x39f3_24f5_b48e_aa40, 0xe9d9_ed1f_8eff_527f, + 0x9224_fc05_8cc5_a214, 0xbaba_00b0_4cfe_7741, 0x309a_9f12_0fcf_52af, 0xa558_f3ec_6562_6212, + 0x424b_ec8b_7ada_be2f, 0x4162_2513_a6ae_a433, 0xb88d_a2d5_324c_a798, 0xd287_733b_2455_28a4, + 0x9a44_697e_6d68_aec3, 0x7b10_93be_2f49_bb28, 0x50bb_ec63_2e3d_8aad, 0x6cd9_0723_e1ea_8283, + 0x897b_9e74_31b0_2bf3, 0x219e_fdcb_338a_7047, 0x3b03_11f0_a27c_0656, 0xdb17_bf91_c0db_96e7, + 0x8cd4_fd6b_4e85_a5b2, 0xfab0_7105_4ba6_409d, 0x40d6_fe83_1fa9_dfd9, 0xaf35_8deb_ad7d_791e, + 0xeb8d_0e25_a65e_3e58, 0xbbcb_d3df_14e0_8580, 0x0cf7_51f2_7ecd_ab2b, 0x2b4d_a14f_2613_d8f4, +]; + +/// `ldmRollingHashState_t` +struct RollingHashState { + rolling: u64, + stop_mask: u64, +} + +impl RollingHashState { + /// `ZSTD_ldm_gear_init` + fn new(params: &LdmParams) -> Self { + let max_bits_in_mask = params.min_match_length.min(64); + let hash_rate_log = params.hash_rate_log; + let rate_mask = (1u64 << (hash_rate_log & 63)) - 1; + // A split is to happen every `1 << hash_rate_log` bytes on average, and bit n of the hash depends on the last + // n bytes: the mask has the highest bits that depend on no more than `min_match_length` bytes. + let stop_mask = if hash_rate_log > 0 && hash_rate_log <= max_bits_in_mask { + rate_mask << ((max_bits_in_mask - hash_rate_log) & 63) + } else { + rate_mask + }; + Self { + rolling: u64::from(u32::MAX), + stop_mask, + } + } + + /// `ZSTD_ldm_gear_feed`: writes to `splits` the split points of `data`, as positions in it. Stops at the end of + /// `data` or once `splits` is full. + #[inline] + fn feed(&mut self, data: &[u8], splits: &mut [usize; LDM_BATCH_SIZE]) -> Fed { + let mut hash = self.rolling; + let mask = self.stop_mask; + let mut n = 0usize; + let mut num_splits = 0usize; + // Returns whether `splits` is full. + let mut iter_once = |byte: u8| { + let gear = GEAR_TAB.get(usize::from(byte)).copied().unwrap_or(0); + hash = (hash << 1).wrapping_add(gear); + n += 1; + if hash & mask == 0 { + if let Some(split) = splits.get_mut(num_splits) { + *split = n; + } + num_splits += 1; + return num_splits == LDM_BATCH_SIZE; + } + false + }; + let (quads, rest) = data.as_chunks::<4>(); + let full = quads + .iter() + .any(|&[a, b, c, d]| iter_once(a) || iter_once(b) || iter_once(c) || iter_once(d)); + if !full { + for &byte in rest { + if iter_once(byte) { + break; + } + } + } + self.rolling = hash; + Fed { + hashed: n, + num_splits, + } + } +} + +/// What [`RollingHashState::feed`] did. +struct Fed { + /// How many bytes it went through. + hashed: usize, + /// How many splits it found. + num_splits: usize, +} + +/// `XXH64`, with a seed of 0. +#[inline] +fn xxh64(data: &[u8]) -> u64 { + let mut state = Xxh64::new(); + state.update(data); + state.digest() +} + +/// `ZSTD_ldm_adjustParameters`: gives a value to the parameters that are 0. `c_params` are those that are applied. +pub(crate) fn adjust_parameters(params: &mut LdmParams, c_params: &CompressionParameters) { + let strategy = c_params.strat(); + params.window_log = c_params.window_log; + if params.hash_rate_log == 0 { + if params.hash_log > 0 { + if params.window_log > params.hash_log { + params.hash_rate_log = params.window_log - params.hash_log; + } + } else { + // From 7 for `fast` to 4 for `btultra2`. + params.hash_rate_log = 7 - (strategy as u32 / 3); + } + } + if params.hash_log == 0 { + // Not as in 1.5.7, where the difference wraps around and the table is then of the largest size there is. + params.hash_log = params + .window_log + .saturating_sub(params.hash_rate_log) + .clamp(ZSTD_HASHLOG_MIN as u32, ZSTD_HASHLOG_MAX as u32); + } + if params.min_match_length == 0 { + params.min_match_length = LDM_MIN_MATCH_LENGTH; + if strategy >= Strategy::BtUltra { + params.min_match_length /= 2; + } + } + if params.bucket_size_log == 0 { + params.bucket_size_log = + (strategy as u32).clamp(LDM_BUCKET_SIZE_LOG, ZSTD_LDM_BUCKETSIZELOG_MAX as u32); + } + params.bucket_size_log = params.bucket_size_log.min(params.hash_log); +} + +/// `ZSTD_ldm_getMaxNbSeq`: the most sequences that `max_chunk_size` bytes can give. +pub(crate) fn get_max_nb_seq(params: LdmParams, max_chunk_size: usize) -> usize { + if params.enable_ldm == ParamSwitch::Enable { + max_chunk_size + .checked_div(params.min_match_length as usize) + .unwrap_or(0) + } else { + 0 + } +} + +/// The mask that keeps, of a hash, the number of a bucket. +#[inline] +const fn bucket_mask(params: &LdmParams) -> u32 { + (1u32 << (params.hash_log.wrapping_sub(params.bucket_size_log) & 31)) - 1 +} + +/// `ZSTD_ldm_getBucket`: where the bucket `hash` starts in the hash table. +#[inline(always)] +const fn get_bucket(hash: usize, bucket_size_log: u32) -> usize { + hash << (bucket_size_log & 31) +} + +/// `ZSTD_ldm_insertEntry`: in place of the oldest entry of the bucket `hash`. +#[inline] +fn insert_entry( + ldm_state: &mut LdmState, + hash: usize, + entry: LdmEntry, + bucket_size_log: u32, +) -> Result<()> { + let p_offset = ldm_state.bucket_offsets.at_mut(hash)?; + let offset = u32::from(*p_offset); + *p_offset = ((offset + 1) & ((1u32 << (bucket_size_log & 31)) - 1)) as u8; + ldm_state + .hash_table + .put(get_bucket(hash, bucket_size_log) + offset as usize, entry) +} + +/// `ZSTD_ldm_countBackwardsMatch`: how many bytes before `p_in` in `buf`, down to `p_anchor`, are the same as those +/// before `p_match` in `match_buf`, down to `p_match_base`. +#[inline] +fn count_backwards_match( + buf: &[u8], + p_in: usize, + p_anchor: usize, + match_buf: &[u8], + p_match: usize, + p_match_base: usize, +) -> Result { + Ok(buf + .sub(p_anchor..p_in)? + .iter() + .rev() + .zip(match_buf.sub(p_match_base..p_match)?.iter().rev()) + .take_while(|(in_byte, match_byte)| in_byte == match_byte) + .count()) +} + +/// `ZSTD_ldm_countBackwardsMatch_2segments`, for a match in the current segment, which starts at `p_match_base`: from +/// there, the match goes on at the end of the older segment. A match in the older segment has nowhere to go on. +#[inline] +fn count_backwards_match_2segments( + buf: &[u8], + p_in: usize, + p_anchor: usize, + p_match: usize, + p_match_base: usize, + dict_buf: &[u8], + p_ext_dict_start: usize, + p_ext_dict_end: usize, +) -> Result { + let match_length = count_backwards_match(buf, p_in, p_anchor, buf, p_match, p_match_base)?; + if p_match - match_length != p_match_base { + return Ok(match_length); + } + Ok(match_length + + count_backwards_match( + buf, + p_in - match_length, + p_anchor, + dict_buf, + p_ext_dict_end, + p_ext_dict_start, + )?) +} + +/// `ZSTD_ldm_fillFastTables`: the strategies that index only what they search have to be told about what they skip. +fn fill_fast_tables(ms: &mut MatchState, prefix: Prefix<'_>, end: usize) { + match ms.c_params.strat() { + Strategy::Fast => fast::fill_hash_table(ms, prefix, end, TableFillPurpose::ForCCtx), + Strategy::DFast => { + double_fast::fill_double_hash_table(ms, prefix, end, TableFillPurpose::ForCCtx) + } + Strategy::Greedy + | Strategy::Lazy + | Strategy::Lazy2 + | Strategy::BtLazy2 + | Strategy::BtOpt + | Strategy::BtUltra + | Strategy::BtUltra2 => {} + } +} + +/// `ZSTD_ldm_limitTableUpdate`: if the tables of `ms` are far behind `anchor`, skips most of the way. +fn limit_table_update(ms: &mut MatchState, src: &BlockSrc<'_>, anchor: usize) { + let curr = src.index_of(anchor); + if curr > ms.next_to_update.wrapping_add(1024) { + let behind = curr.wrapping_sub(ms.next_to_update).wrapping_sub(1024); + ms.next_to_update = curr.wrapping_sub(behind.min(512)); + } +} + +/// `ldmMatchCandidate_t` +#[derive(Clone, Copy, Default)] +struct MatchCandidate { + split: usize, + hash: u32, + checksum: u32, + /// Where the bucket starts in the hash table. + bucket: usize, +} + +/// `ZSTD_ldm_generateSequences_internal`: the input is from `istart` to `iend` in the buffer of the current segment. +/// Returns how many bytes at its end are in no sequence. +fn generate_sequences_internal( + ldm_state: &mut LdmState, + raw_seq_store: &mut RawSeqStore, + params: &LdmParams, + bufs: &Buffers<'_>, + istart: usize, + iend: usize, +) -> Result { + let window = ldm_state.window; + let ext_dict = window.has_ext_dict(); + let min_match_length = params.min_match_length as usize; + let ents_per_bucket = 1usize << (params.bucket_size_log & 31); + let bucket_mask = bucket_mask(params); + let dict_limit = window.dict_limit; + let lowest_index = if ext_dict { + window.low_limit + } else { + dict_limit + }; + let prefix = Prefix { + buf: bufs.get(window.base_buf), + base: window.base.wrapping_sub(window.base_buf.start()), + }; + let buf = prefix.buf; + let dict_buf = bufs.get(window.dict_buf); + let dict_base = window.dict_base.wrapping_sub(window.dict_buf.start()); + let dict_start = dict_base.wrapping_add(lowest_index as usize); + let dict_end = dict_base.wrapping_add(dict_limit as usize); + let low_prefix = prefix.pos_of(dict_limit); + let ilimit = iend.saturating_sub(HASH_READ_SIZE); + let mut anchor = istart; + let mut ip = istart; + let mut splits = [0usize; LDM_BATCH_SIZE]; + let mut candidates = [MatchCandidate::default(); LDM_BATCH_SIZE]; + + if iend.checked_sub(istart).ok_or(BUG)? < min_match_length { + return Ok(iend - anchor); + } + + // Here and where it skips over a match, the C code calls `ZSTD_ldm_gear_reset`, which computes the hash of the + // `min_match_length` bytes before `ip` and drops it: the rolling hash goes on from what it was. + let mut hash_state = RollingHashState::new(params); + ip += min_match_length; + + while ip < ilimit { + let Fed { hashed, num_splits } = hash_state.feed(buf.sub(ip..ilimit)?, &mut splits); + + for (candidate, &split) in candidates.iter_mut().zip(&splits).take(num_splits) { + let split = ip + split - min_match_length; + let xxhash = xxh64(buf.span(split, min_match_length)?); + let hash = xxhash as u32 & bucket_mask; + *candidate = MatchCandidate { + split, + hash, + checksum: (xxhash >> 32) as u32, + bucket: get_bucket(hash as usize, params.bucket_size_log), + }; + } + + for candidate in candidates.iter().take(num_splits) { + let MatchCandidate { + split, + hash, + checksum, + bucket, + } = *candidate; + let hash = hash as usize; + let mut forward_match_length = 0usize; + let mut backward_match_length = 0usize; + let mut best_match_length = 0usize; + let mut best_entry = None; + let new_entry = LdmEntry { + offset: prefix.index_of(split), + checksum, + }; + + // A split inside the previous match is indexed, but not searched. + if split < anchor { + insert_entry(ldm_state, hash, new_entry, params.bucket_size_log)?; + continue; + } + + for cur in ldm_state.hash_table.span(bucket, ents_per_bucket)? { + if cur.checksum != checksum || cur.offset <= lowest_index { + continue; + } + let (cur_forward_match_length, cur_backward_match_length) = if ext_dict + && cur.offset < dict_limit + { + let p_match = dict_base.wrapping_add(cur.offset as usize); + let forward = + count_2segments(buf, split, dict_buf, p_match, iend, dict_end, low_prefix)?; + if forward < min_match_length { + continue; + } + let backward = + count_backwards_match(buf, split, anchor, dict_buf, p_match, dict_start)?; + (forward, backward) + } else { + let p_match = prefix.pos_of(cur.offset); + let forward = count(buf, split, buf, p_match, iend)?; + if forward < min_match_length { + continue; + } + let backward = if ext_dict { + count_backwards_match_2segments( + buf, split, anchor, p_match, low_prefix, dict_buf, dict_start, dict_end, + )? + } else { + count_backwards_match(buf, split, anchor, buf, p_match, low_prefix)? + }; + (forward, backward) + }; + let cur_total_match_length = cur_forward_match_length + cur_backward_match_length; + + if cur_total_match_length > best_match_length { + best_match_length = cur_total_match_length; + forward_match_length = cur_forward_match_length; + backward_match_length = cur_backward_match_length; + best_entry = Some(*cur); + } + } + + let Some(best_entry) = best_entry else { + insert_entry(ldm_state, hash, new_entry, params.bucket_size_log)?; + continue; + }; + + let offset = new_entry.offset.wrapping_sub(best_entry.offset); + let m_length = forward_match_length + backward_match_length; + if raw_seq_store.size == raw_seq_store.capacity { + return Err(Error::DstSizeTooSmall); + } + raw_seq_store.seq.put( + raw_seq_store.size, + RawSeq { + offset, + lit_length: (split - backward_match_length - anchor) as u32, + match_length: m_length as u32, + }, + )?; + raw_seq_store.size += 1; + + // Only now, so that the position does not match itself. + insert_entry(ldm_state, hash, new_entry, params.bucket_size_log)?; + + anchor = split + forward_match_length; + + // The match goes beyond what was hashed: what is left of it is skipped, and the other candidates dropped. + if anchor > ip + hashed { + ip = anchor - hashed; + break; + } + } + + ip += hashed; + } + + Ok(iend - anchor) +} + +/// `ZSTD_ldm_reduceTable` +fn reduce_table(table: &mut [LdmEntry], reducer_value: u32) { + for entry in table { + entry.offset = entry.offset.saturating_sub(reducer_value); + } +} + +/// `ZSTD_ldm_generateSequences`: adds to `sequences` the long distance matches of the input, which is the bytes from +/// `start` to `end` of the buffer of the current segment of the window, and which the window was updated with. +/// Stops early, without an error, at a chunk boundary where `sequences` is full. +pub(crate) fn generate_sequences( + ldm_state: &mut LdmState, + sequences: &mut RawSeqStore, + params: &LdmParams, + bufs: &Buffers<'_>, + start: usize, + end: usize, +) -> Result<()> { + const K_MAX_CHUNK_SIZE: usize = 1 << 20; + let max_dist = 1u32 << (params.window_log & 31); + let buf_start = ldm_state.window.base_buf.start(); + let mut leftover_size = 0usize; + let mut chunk_start = start; + + while chunk_start < end && sequences.size < sequences.capacity { + let chunk_end = end.min(chunk_start + K_MAX_CHUNK_SIZE); + let chunk_size = chunk_end - chunk_start; + let prev_size = sequences.size; + + if ldm_state + .window + .need_overflow_correction(buf_start + chunk_end) + { + let ldm_h_size = 1usize << (params.hash_log & 31); + let correction = + ldm_state + .window + .correct_overflow(0, max_dist, buf_start + chunk_start); + reduce_table(ldm_state.hash_table.sub_mut(..ldm_h_size)?, correction); + // An overflow correction invalidates the dictionary. + ldm_state.loaded_dict_end = 0; + } + // The whole chunk is searched with the window that its end has, so that no match is too far for any position + // of it, at the cost of some matches at its start. + if ldm_state.window.enforce_max_dist( + buf_start + chunk_end, + max_dist, + ldm_state.loaded_dict_end, + ) { + ldm_state.loaded_dict_end = 0; + } + let new_leftover_size = generate_sequences_internal( + ldm_state, + sequences, + params, + bufs, + chunk_start, + chunk_end, + )?; + // What the previous chunks left goes to the literals of the first sequence of this one. + if prev_size < sequences.size { + let first = sequences.seq.at_mut(prev_size)?; + first.lit_length = first.lit_length.wrapping_add(leftover_size as u32); + leftover_size = new_leftover_size; + } else { + leftover_size += chunk_size; + } + chunk_start = chunk_end; + } + Ok(()) +} + +/// `ZSTD_ldm_skipSequences`: moves `src_size` bytes forward in the sequences. A match of which less than `min_match` +/// is left becomes literals of the next sequence. +pub(crate) fn skip_sequences( + raw_seq_store: &mut RawSeqStore, + mut src_size: usize, + min_match: u32, +) -> Result<()> { + while src_size > 0 && raw_seq_store.pos < raw_seq_store.size { + let seq = raw_seq_store.seq.at_mut(raw_seq_store.pos)?; + if src_size <= seq.lit_length as usize { + seq.lit_length -= src_size as u32; + return Ok(()); + } + src_size -= seq.lit_length as usize; + seq.lit_length = 0; + if src_size < seq.match_length as usize { + seq.match_length -= src_size as u32; + if seq.match_length < min_match { + let match_length = seq.match_length; + if raw_seq_store.pos + 1 < raw_seq_store.size { + let next = raw_seq_store.seq.at_mut(raw_seq_store.pos + 1)?; + next.lit_length = next.lit_length.wrapping_add(match_length); + } + raw_seq_store.pos += 1; + } + return Ok(()); + } + src_size -= seq.match_length as usize; + seq.match_length = 0; + raw_seq_store.pos += 1; + } + Ok(()) +} + +/// `maybeSplitSequence`: the next sequence, cut to the `remaining` bytes of the block. What is cut off stays in the +/// store. If there is no match of `min_match` left in the block, the offset is 0. +fn maybe_split_sequence( + raw_seq_store: &mut RawSeqStore, + remaining: u32, + min_match: u32, +) -> Result { + let mut sequence = raw_seq_store.seq.at(raw_seq_store.pos)?; + let length = sequence.lit_length.wrapping_add(sequence.match_length); + if remaining >= length { + raw_seq_store.pos += 1; + return Ok(sequence); + } + if remaining <= sequence.lit_length { + sequence.offset = 0; + } else { + sequence.match_length = remaining - sequence.lit_length; + if sequence.match_length < min_match { + sequence.offset = 0; + } + } + skip_sequences(raw_seq_store, remaining as usize, min_match)?; + Ok(sequence) +} + +/// `ZSTD_ldm_skipRawSeqStoreBytes`: moves `nb_bytes` forward in the sequences, leaving them as they are. +pub(crate) fn skip_raw_seq_store_bytes( + raw_seq_store: &mut RawSeqStore, + nb_bytes: usize, +) -> Result<()> { + let mut curr_pos = raw_seq_store.pos_in_sequence.wrapping_add(nb_bytes) as u32; + while curr_pos != 0 && raw_seq_store.pos < raw_seq_store.size { + let curr_seq = raw_seq_store.seq.at(raw_seq_store.pos)?; + let length = curr_seq.lit_length.wrapping_add(curr_seq.match_length); + if curr_pos >= length { + curr_pos -= length; + raw_seq_store.pos += 1; + } else { + raw_seq_store.pos_in_sequence = curr_pos as usize; + break; + } + } + if curr_pos == 0 || raw_seq_store.pos == raw_seq_store.size { + raw_seq_store.pos_in_sequence = 0; + } + Ok(()) +} + +/// `ZSTD_ldm_blockCompress`: finds the sequences of the block `src`, given the long distance matches, which it moves +/// forward in by the size of the block. Returns how many bytes at the end of the block are in no sequence. +pub(crate) fn block_compress( + raw_seq_store: &mut RawSeqStore, + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + use_row_match_finder: bool, + src: &BlockSrc<'_>, +) -> Result { + let strategy = ms.c_params.strat(); + let min_match = ms.c_params.min_match; + let dict_mode = ms.dict_mode(src.dms.as_ref()); + let iend = src.end; + let mut ip = src.start; + + // The optimal parser weighs the long distance matches against its own. + if strategy >= Strategy::BtOpt { + let block = BlockSrc { + ldm_seq_store: Some(&*raw_seq_store), + ..*src + }; + let last_ll_size = compress_block( + ms, + seq_store, + rep, + &block, + strategy, + use_row_match_finder, + dict_mode, + )?; + skip_raw_seq_store_bytes(raw_seq_store, iend.wrapping_sub(ip))?; + return Ok(last_ll_size); + } + + while raw_seq_store.pos < raw_seq_store.size && ip < iend { + let sequence = maybe_split_sequence(raw_seq_store, (iend - ip) as u32, min_match)?; + if sequence.offset == 0 { + break; + } + + limit_table_update(ms, src, ip); + fill_fast_tables(ms, src.prefix(), ip); + // The block compressor takes care of the literals, and what it leaves goes with the long distance match. + let literals = BlockSrc { + start: ip, + end: ip + sequence.lit_length as usize, + ..*src + }; + let new_lit_length = compress_block( + ms, + seq_store, + rep, + &literals, + strategy, + use_row_match_finder, + dict_mode, + )?; + ip += sequence.lit_length as usize; + let [rep0, rep1, _] = *rep; + *rep = [sequence.offset, rep0, rep1]; + seq_store.store_seq( + new_lit_length, + src.buf, + ip.checked_sub(new_lit_length).ok_or(BUG)?, + offset_to_offbase(sequence.offset), + sequence.match_length as usize, + )?; + ip += sequence.match_length as usize; + } + limit_table_update(ms, src, ip); + fill_fast_tables(ms, src.prefix(), ip); + let last = BlockSrc { start: ip, ..*src }; + compress_block( + ms, + seq_store, + rep, + &last, + strategy, + use_row_match_finder, + dict_mode, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_hash_rate_above_the_window_makes_for_the_smallest_table() { + let c_params = CompressionParameters { + window_log: 10, + chain_log: 10, + hash_log: 10, + search_log: 1, + min_match: 4, + target_length: 0, + strategy: Some(Strategy::Fast), + }; + let mut params = LdmParams { + enable_ldm: ParamSwitch::Enable, + hash_log: 0, + bucket_size_log: 0, + min_match_length: 0, + hash_rate_log: 11, + window_log: 0, + }; + adjust_parameters(&mut params, &c_params); + assert_eq!(params.hash_log, ZSTD_HASHLOG_MIN as u32); + } +} diff --git a/src/zstd/compress/literals.rs b/src/zstd/compress/literals.rs new file mode 100644 index 000000000000..14450d85c865 --- /dev/null +++ b/src/zstd/compress/literals.rs @@ -0,0 +1,147 @@ +//! `zstd_compress_literals.c`: the literals section of a block. + +use super::huf::{self, HUF_OPTIMAL_DEPTH_THRESHOLD, HufFlags, HufRepeat}; +use super::internal::{HufCTables, Strategy, min_gain}; +use crate::common::{HUF_SYMBOLVALUE_MAX, LIT_HUF_LOG, SymbolEncodingType}; +use crate::error::{Error, Result}; +use crate::util::{SliceExt, write_le16, write_le24, write_le32}; + +/// The header of a section of raw or run-length encoded literals. Returns its size. +fn write_short_header(dst: &mut [u8], ty: SymbolEncodingType, src_size: usize) -> Result { + let fl_size = 1 + usize::from(src_size > 31) + usize::from(src_size > 4095); + let ty = ty as u32; + let src_size = src_size as u32; + match fl_size { + 1 => dst.put(0, (ty + (src_size << 3)) as u8)?, + 2 => write_le16(dst, 0, (ty + (1 << 2) + (src_size << 4)) as u16)?, + _ => write_le24(dst, 0, ty + (3 << 2) + (src_size << 4))?, + } + Ok(fl_size) +} + +/// `ZSTD_noCompressLiterals` +pub(crate) fn no_compress_literals(dst: &mut [u8], src: &[u8]) -> Result { + let fl_size = 1 + usize::from(src.len() > 31) + usize::from(src.len() > 4095); + if src.len() + fl_size > dst.len() { + return Err(Error::DstSizeTooSmall); + } + write_short_header(dst, SymbolEncodingType::Basic, src.len())?; + dst.write_at(fl_size, src)?; + Ok(src.len() + fl_size) +} + +/// `ZSTD_compressRleLiteralsBlock`: `src` is one byte repeated, and `dst` has room for 4 bytes. +pub(crate) fn compress_rle_literals_block(dst: &mut [u8], src: &[u8]) -> Result { + let fl_size = write_short_header(dst, SymbolEncodingType::Rle, src.len())?; + dst.put(fl_size, src.at(0)?)?; + Ok(fl_size + 1) +} + +/// `ZSTD_minLiteralsToCompress`: fewer literals than this are not worth a try. +pub(crate) fn min_literals_to_compress(strategy: Strategy, huf_repeat: HufRepeat) -> usize { + let shift = (9 - strategy as i32).min(3); + if huf_repeat == HufRepeat::Valid { + 6 + } else { + 8 << shift + } +} + +/// `ZSTD_compressLiterals` +pub(crate) fn compress_literals( + dst: &mut [u8], + src: &[u8], + prev_huf: &HufCTables, + next_huf: &mut HufCTables, + strategy: Strategy, + disable_literal_compression: bool, + suspect_uncompressible: bool, +) -> Result { + let src_size = src.len(); + let lh_size = 3 + usize::from(src_size >= 1 << 10) + usize::from(src_size >= 16 << 10); + let mut single_stream = src_size < 256; + let mut h_type = SymbolEncodingType::Compressed; + + next_huf.copy_from(prev_huf); + + if disable_literal_compression { + return no_compress_literals(dst, src); + } + if src_size < min_literals_to_compress(strategy, prev_huf.repeat_mode) { + return no_compress_literals(dst, src); + } + + if dst.len() < lh_size + 1 { + return Err(Error::DstSizeTooSmall); + } + let c_lit_size = { + let mut repeat = prev_huf.repeat_mode; + let flags = HufFlags { + prefer_repeat: strategy < Strategy::Lazy && src_size <= 1024, + optimal_depth: strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD, + suspect_uncompressible, + }; + if repeat == HufRepeat::Valid && lh_size == 3 { + single_stream = true; + } + let huf_compress = if single_stream { + huf::compress_1x_repeat + } else { + huf::compress_4x_repeat + }; + let c_lit_size = huf_compress( + dst.sub_mut(lh_size..)?, + src, + HUF_SYMBOLVALUE_MAX, + LIT_HUF_LOG, + &mut next_huf.c_table, + &mut repeat, + flags, + ); + if repeat != HufRepeat::None { + h_type = SymbolEncodingType::Repeat; + } + c_lit_size + }; + + let min_gain = min_gain(src_size, strategy); + let c_lit_size = match c_lit_size { + Ok(size) if size != 0 && size < src_size.wrapping_sub(min_gain) => size, + _ => { + next_huf.copy_from(prev_huf); + return no_compress_literals(dst, src); + } + }; + if c_lit_size == 1 { + // With few literals, a size of 1 can also be that of a Huffman stream. + let first = src.at(0)?; + if src_size >= 8 || src.iter().all(|&b| b == first) { + next_huf.copy_from(prev_huf); + return compress_rle_literals_block(dst, src); + } + } + + if h_type == SymbolEncodingType::Compressed { + next_huf.repeat_mode = HufRepeat::Check; + } + + let h_type = h_type as u32; + let src_size = src_size as u32; + let c_lit = c_lit_size as u32; + match lh_size { + 3 => { + let lhc = h_type + (u32::from(!single_stream) << 2) + (src_size << 4) + (c_lit << 14); + write_le24(dst, 0, lhc)?; + } + 4 => { + let lhc = h_type + (2 << 2) + (src_size << 4) + (c_lit << 18); + write_le32(dst, 0, lhc)?; + } + _ => { + let lhc = h_type + (3 << 2) + (src_size << 4) + (c_lit << 22); + write_le32(dst, 0, lhc)?; + dst.put(4, (c_lit_size >> 10) as u8)?; + } + } + Ok(lh_size + c_lit_size) +} diff --git a/src/zstd/compress/mod.rs b/src/zstd/compress/mod.rs new file mode 100644 index 000000000000..fd2986adf53a --- /dev/null +++ b/src/zstd/compress/mod.rs @@ -0,0 +1,76 @@ +//! `lib/compress` + +pub(crate) mod block_entropy; +pub(crate) mod cctx; +pub(crate) mod double_fast; +pub(crate) mod fast; +pub(crate) mod fse; +pub(crate) mod hist; +pub(crate) mod huf; +pub(crate) mod internal; +pub(crate) mod lazy; +pub(crate) mod ldm; +pub(crate) mod literals; +pub(crate) mod mt; +pub(crate) mod opt; +pub(crate) mod params; +pub(crate) mod presplit; +pub(crate) mod sequences; +pub(crate) mod stream; +pub(crate) mod superblock; + +use internal::{BlockSrc, DictMode, MatchState, SeqStore, Strategy}; + +use crate::common::ZSTD_REP_NUM; +use crate::error::Result; + +/// `ZSTD_rowMatchFinderSupported` +pub(crate) fn row_match_finder_supported(strategy: Strategy) -> bool { + strategy >= Strategy::Greedy && strategy <= Strategy::Lazy2 +} + +/// What `ZSTD_selectBlockCompressor` selects, called: finds the sequences of the block `src` and adds them to +/// `seq_store`. `use_row_match_finder` is whether the row-based match finder is enabled, which only the strategies +/// that support it look at. Returns how many bytes at the end of the block are in no sequence. +pub(crate) fn compress_block( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + strategy: Strategy, + use_row_match_finder: bool, + dict_mode: DictMode, +) -> Result { + match strategy { + Strategy::Fast => match dict_mode { + DictMode::NoDict => fast::compress_block_fast(ms, seq_store, rep, src), + DictMode::ExtDict => fast::compress_block_fast_ext_dict(ms, seq_store, rep, src), + DictMode::DictMatchState => { + fast::compress_block_fast_dict_match_state(ms, seq_store, rep, src) + } + }, + Strategy::DFast => match dict_mode { + DictMode::NoDict => double_fast::compress_block_double_fast(ms, seq_store, rep, src), + DictMode::ExtDict => { + double_fast::compress_block_double_fast_ext_dict(ms, seq_store, rep, src) + } + DictMode::DictMatchState => { + double_fast::compress_block_double_fast_dict_match_state(ms, seq_store, rep, src) + } + }, + Strategy::Greedy | Strategy::Lazy | Strategy::Lazy2 | Strategy::BtLazy2 => { + lazy::compress_block_lazy( + ms, + seq_store, + rep, + src, + strategy, + use_row_match_finder && row_match_finder_supported(strategy), + dict_mode, + ) + } + Strategy::BtOpt | Strategy::BtUltra | Strategy::BtUltra2 => { + opt::compress_block_opt(ms, seq_store, rep, src, strategy, dict_mode) + } + } +} diff --git a/src/zstd/compress/mt.rs b/src/zstd/compress/mt.rs new file mode 100644 index 000000000000..6c6548e9fa53 --- /dev/null +++ b/src/zstd/compress/mt.rs @@ -0,0 +1,957 @@ +//! `zstdmt_compress.c`: a frame made of sections that are compressed in parallel. +//! +//! The input is cut in the same sections as the C code cuts it in, each of which starts from the end of the previous +//! one as a prefix, so the frames are the same. What differs is how memory is shared. The C code has one round buffer +//! for the input, which workers read while it is being filled, one output buffer for each job, which is flushed while +//! it is being written, and a state that jobs lock in turn. Here a job owns its input, sends its output in pieces, and +//! hands that state over to the next job. + +use std::collections::VecDeque; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpmc::{Receiver, Sender, TryRecvError, channel}; + +use super::cctx::{CDict, FrameCompressor, Input, Span, write_last_empty_block}; +use super::internal::{ + BufId, Buffers, LdmParams, LdmState, ParamSwitch, RawSeq, RawSeqStore, Strategy, Window, +}; +use super::ldm; +use super::params::{ + CCtxParams, NO_LDM_PARAMS, ZSTDMT_JOBLOG_MAX, ZSTDMT_JOBSIZE_MAX, ZSTDMT_JOBSIZE_MIN, + ZSTDMT_NBWORKERS_MAX, compress_bound, cycle_log, +}; +use super::stream::EndDirective; +use crate::common::ZSTD_BLOCKSIZE_MAX; +use crate::common::xxhash::Xxh64; +use crate::error::{Error, Result}; +use crate::util::{BUG, SliceExt, append, make_room, reset_table, try_copy}; +use crate::{InBuffer, OutBuffer}; + +/// `SerialState`: what has to see the input in order. +struct SerialState { + ldm_params: LdmParams, + checksum_flag: bool, + ldm_state: LdmState, + xxh_state: Xxh64, + /// The input, where the C code has it in its round buffer: long distance matches reach further back than the + /// prefix of a job. It is as long as it has been written to. + round_buff: Vec, +} + +impl SerialState { + fn new() -> Self { + Self { + ldm_params: NO_LDM_PARAMS, + checksum_flag: false, + ldm_state: LdmState::new(), + xxh_state: Xxh64::new(), + round_buff: Vec::new(), + } + } + + /// `ZSTDMT_serialState_reset` + fn reset(&mut self, params: &CCtxParams, round_buff_capacity: usize) -> Result<()> { + let mut ldm_params = params.ldm_params; + if ldm_params.enable_ldm == ParamSwitch::Enable { + ldm::adjust_parameters(&mut ldm_params, ¶ms.c_params); + } else { + ldm_params = NO_LDM_PARAMS; + } + self.checksum_flag = params.f_params.checksum_flag; + self.xxh_state = Xxh64::new(); + if ldm_params.enable_ldm == ParamSwitch::Enable { + let bucket_log = ldm_params.hash_log - ldm_params.bucket_size_log; + self.ldm_state.window = Window::new(); + reset_table( + &mut self.ldm_state.hash_table, + 1 << ldm_params.hash_log, + true, + )?; + reset_table(&mut self.ldm_state.bucket_offsets, 1 << bucket_log, true)?; + self.ldm_state.loaded_dict_end = 0; + self.round_buff.clear(); + make_room( + &mut self.round_buff, + round_buff_capacity, + round_buff_capacity, + )?; + } else { + self.ldm_state = LdmState::new(); + self.round_buff = Vec::new(); + } + self.ldm_params = ldm_params; + Ok(()) + } + + /// `ZSTDMT_serialState_genSequences`: `src` is the input of the next job, and `round_pos` where it goes in the + /// round buffer. + fn gen_sequences( + &mut self, + seq_store: &mut RawSeqStore, + src: &[u8], + round_pos: usize, + ) -> Result<()> { + if self.ldm_params.enable_ldm == ParamSwitch::Enable { + let written = self.round_buff.get_mut(round_pos..).ok_or(BUG)?; + let (over, beyond) = src.split_at(written.len().min(src.len())); + written.write_at(0, over)?; + (self.round_buff.try_reserve(beyond.len())).map_err(|_| Error::MemoryAllocation)?; + append(&mut self.round_buff, beyond)?; + (self.ldm_state.window).update( + BufId::InBuff.start() + round_pos, + BufId::InBuff, + src.len(), + false, + ); + ldm::generate_sequences( + &mut self.ldm_state, + seq_store, + &self.ldm_params, + &Buffers { + in_buff: &self.round_buff, + user: &[], + dict: &[], + }, + round_pos, + round_pos + src.len(), + )?; + } + if self.checksum_flag && !src.is_empty() { + self.xxh_state.update(src); + } + Ok(()) + } +} + +/// `ZSTDMT_CCtxPool`. A job takes the context that was given back last, so jobs that come one after the other all have +/// the same one, and what it has been through shows in their output, see [`Workspace`](super::params::Workspace). +#[derive(Clone)] +struct CCtxPool { + /// `totalCCtx` + total: usize, + /// Holds the contexts that are free, but for while someone looks at them. + free: (Sender>, Receiver>), +} + +impl CCtxPool { + /// `ZSTDMT_createCCtxPool` + fn new(total: usize) -> Result { + let mut contexts = Vec::new(); + (contexts.try_reserve_exact(total)).map_err(|_| Error::MemoryAllocation)?; + let free = channel(); + let _ = free.0.send(contexts); + Ok(Self { total, free }) + } + + fn with_free(&self, look: impl FnOnce(&mut Vec) -> R) -> R { + let mut contexts = self.free.1.recv().unwrap_or_default(); + let result = look(&mut contexts); + let _ = self.free.0.send(contexts); + result + } + + /// `ZSTDMT_getCCtx` + fn take(&self, cancelled: &Arc) -> FrameCompressor { + self.with_free(Vec::pop) + .unwrap_or_else(|| FrameCompressor::for_worker(Arc::clone(cancelled))) + } + + /// `ZSTDMT_releaseCCtx` + fn give_back(&self, cctx: FrameCompressor) { + self.with_free(|contexts| { + if contexts.len() < self.total { + contexts.push(cctx); + } + }); + } +} + +/// What a job tells about itself. +enum Progress { + /// Some more output, and how much of the input is consumed by now. + Output { data: Vec, consumed: usize }, + /// The job stops here. + Failed(Error), +} + +/// `ZSTDMT_jobDescription`, as far as the worker is concerned. +struct Job { + /// The prefix, then the input. + data: Vec, + prefix_size: usize, + round_pos: usize, + first_job: bool, + last_job: bool, + params: CCtxParams, + cdict: Option>, + full_frame_size: u64, + /// How many long distance matches the C code makes room for, which is as many as a whole section can have. + max_nb_seq: usize, + /// How many the input of this job can have. + nb_seq_bound: usize, + /// From the previous job. + serial_in: Receiver>, + /// To the next one. + serial_out: Sender>, + progress: Sender, +} + +/// What a worker keeps from one job to the next. +struct WorkerState { + seq: Vec, + dst: Vec, +} + +const CHUNK_SIZE: usize = 4 * ZSTD_BLOCKSIZE_MAX; + +/// `ZSTDMT_compressionJob`, up to `_endJob`. Returns the end of the output. +fn compression_job( + state: &mut WorkerState, + cctx: &mut FrameCompressor, + job: &Job, + cancelled: &AtomicBool, +) -> Result> { + let mut job_params = job.params; + let src_size = job.data.len() - job.prefix_size; + let src = job.data.sub(job.prefix_size..)?; + + // Whatever happens, the next job waits for this. + let mut raw_seq_store = RawSeqStore { + seq: core::mem::take(&mut state.seq), + pos: 0, + pos_in_sequence: 0, + size: 0, + capacity: job.max_nb_seq, + }; + let mut serial = job.serial_in.recv().map_err(|_| Error::Generic)?; + let generated = if cancelled.load(Ordering::SeqCst) { + Err(Error::Generic) + } else { + reset_table(&mut raw_seq_store.seq, job.nb_seq_bound, true) + .and_then(|()| serial.gen_sequences(&mut raw_seq_store, src, job.round_pos)) + }; + // The receiver is gone if the frame is given up. + let _ = job.serial_out.send(serial); + generated?; + + reset_table(&mut state.dst, compress_bound(CHUNK_SIZE) + 1024, false)?; + + // Only the first job writes the frame header, and the checksum is not that of one job. + if !job.first_job { + job_params.f_params.checksum_flag = false; + } + job_params.ldm_params.enable_ldm = ParamSwitch::Disable; + job_params.nb_workers = 0; + + let cdict = job.cdict.as_deref(); + let input = Input { + bufs: Buffers { + in_buff: &[], + user: &job.data, + dict: cdict.map_or(&[], CDict::dict_content), + }, + cdict, + }; + if cdict.is_some() { + cctx.begin(&input, None, &job_params, job.full_frame_size, false)?; + } else { + let pledged_src_size = if job.first_job { + job.full_frame_size + } else { + src_size as u64 + }; + job_params.force_window = !job.first_job; + cctx.begin( + &input, + Some(Span { + buf: BufId::User, + start: 0, + end: job.prefix_size, + }), + &job_params, + pledged_src_size, + false, + )?; + } + + raw_seq_store.seq.truncate(raw_seq_store.size); + cctx.reference_external_sequences(raw_seq_store.seq); + + if !job.first_job { + // This frame header is not for the output. + let nothing = Span { + buf: BufId::User, + start: job.prefix_size, + end: job.prefix_size, + }; + cctx.compress_continue(&mut state.dst, &input, nothing)?; + cctx.invalidate_rep_codes(); + } + + let nb_chunks = src_size.div_ceil(CHUNK_SIZE); + let mut ip = job.prefix_size; + let send = |dst: &[u8], consumed: usize| -> Result<()> { + let data = try_copy(dst)?; + let _ = job.progress.send(Progress::Output { data, consumed }); + Ok(()) + }; + for chunk_nb in 1..nb_chunks { + let chunk = Span { + buf: BufId::User, + start: ip, + end: ip + CHUNK_SIZE, + }; + let c_size = cctx.compress_continue(&mut state.dst, &input, chunk)?; + ip += CHUNK_SIZE; + send(state.dst.sub(..c_size)?, CHUNK_SIZE * chunk_nb)?; + } + let mut last_c_block_size = 0; + if nb_chunks > 0 || job.last_job { + let chunk = Span { + buf: BufId::User, + start: ip, + end: job.data.len(), + }; + last_c_block_size = if job.last_job { + cctx.compress_end(&mut state.dst, &input, chunk)? + } else { + cctx.compress_continue(&mut state.dst, &input, chunk)? + }; + } + try_copy(state.dst.sub(..last_c_block_size)?) +} + +fn worker_main(jobs: &Receiver, cctx_pool: &CCtxPool, cancelled: &Arc) { + let mut state = WorkerState { + seq: Vec::new(), + dst: Vec::new(), + }; + while let Ok(job) = jobs.recv() { + let mut cctx = cctx_pool.take(cancelled); + let end = compression_job(&mut state, &mut cctx, &job, cancelled); + state.seq = cctx.take_external_sequences(); + // First, so that a job that is created once this one is known to be over finds its context. + cctx_pool.give_back(cctx); + let _ = job.progress.send(match end { + Ok(data) => Progress::Output { + data, + consumed: job.data.len() - job.prefix_size, + }, + Err(code) => Progress::Failed(code), + }); + } +} + +/// `ZSTDMT_jobDescription`, as far as flushing is concerned. +struct PendingJob { + /// `None` for a job that needs no worker. + progress: Option>, + /// Output that is received and not flushed. The first piece is flushed up to `flushed`. + output: VecDeque>, + flushed: usize, + /// How many bytes there are in `output` beyond `flushed`. + to_flush: usize, + consumed: usize, + src_size: usize, + frame_checksum_needed: bool, + error: Option, +} + +impl PendingJob { + fn take(&mut self, progress: Progress) { + match progress { + Progress::Output { data, consumed } => { + self.to_flush += data.len(); + self.consumed = consumed; + if !data.is_empty() { + self.output.push_back(data); + } + } + Progress::Failed(code) => { + self.error = Some(code); + self.consumed = self.src_size; + self.progress = None; + } + } + } + + /// `consumed == src.size` is how the C code tells, which does not do for a job that has no input. + fn is_complete(&self) -> bool { + self.progress.is_none() + } + + /// Takes what the worker has sent. If `wait` is set and there is nothing to flush, waits for some. + fn poll(&mut self, wait: bool) { + while let Some(receiver) = &self.progress { + let received = if wait && self.to_flush == 0 { + receiver.recv().ok() + } else { + match receiver.try_recv() { + Ok(progress) => Some(progress), + Err(TryRecvError::Empty) => return, + Err(TryRecvError::Disconnected) => None, + } + }; + match received { + Some(progress) => { + let last = matches!(&progress, Progress::Output { consumed, .. } if *consumed == self.src_size); + self.take(progress); + if last { + self.progress = None; + } + } + // The worker is gone. + None => self.take(Progress::Failed(Error::Generic)), + } + } + } +} + +/// `ZSTDMT_CCtx` +pub(crate) struct MtCtx { + /// The workers have it too. Before `queue`, so that it is the last of them to leave that frees the contexts. + cctx_pool: CCtxPool, + /// They are on their own, and leave once nothing can be sent to their queue. + nb_workers: usize, + /// Whichever worker is free takes the next job, so that a job does not wait for one thread to be scheduled. + queue: Sender, + cancelled: Arc, + /// The jobs from `doneJobID` to `nextJobID`. + jobs: VecDeque, + params: CCtxParams, + target_section_size: usize, + target_prefix_size: usize, + /// `inBuff`: the prefix of the next job, then as much of its input as there is. + in_buff: Vec, + in_buff_prefix_size: usize, + /// `inBuff.buffer.start != NULL`: there is room for the input of the next job. + in_buff_ready: bool, + round_buff_capacity: usize, + round_buff_pos: usize, + /// Where the last job that was created leaves the state. + serial: Option>>, + next_job_id: u32, + frame_ended: bool, + all_jobs_completed: bool, + frame_content_size: u64, + cdict: Option>, +} + +/// `ZSTDMT_computeTargetJobLog` +fn compute_target_job_log(params: &CCtxParams) -> u32 { + let job_log = if params.ldm_params.enable_ldm == ParamSwitch::Enable { + // Long distance matching makes up for the matches that cutting loses, so sections can be smaller. + 21.max(cycle_log(params.c_params.chain_log, params.c_params.strat()) + 3) + } else { + 20.max(params.c_params.window_log + 2) + }; + job_log.min(ZSTDMT_JOBLOG_MAX) +} + +/// `ZSTDMT_overlapLog` +fn overlap_log(ovlog: i32, strat: Strategy) -> i32 { + if ovlog != 0 { + return ovlog; + } + match strat { + Strategy::BtUltra2 => 9, + Strategy::BtUltra | Strategy::BtOpt => 8, + Strategy::BtLazy2 | Strategy::Lazy2 => 7, + Strategy::Lazy | Strategy::Greedy | Strategy::DFast | Strategy::Fast => 6, + } +} + +/// `ZSTDMT_computeOverlapSize` +fn compute_overlap_size(params: &CCtxParams) -> usize { + let overlap_r_log = 9 - overlap_log(params.overlap_log, params.c_params.strat()); + let mut ov_log = if overlap_r_log >= 8 { + 0 + } else { + params.c_params.window_log as i32 - overlap_r_log + }; + if params.ldm_params.enable_ldm == ParamSwitch::Enable { + ov_log = (params.c_params.window_log).min(compute_target_job_log(params) - 2) as i32 + - overlap_r_log; + } + if ov_log == 0 { 0 } else { 1 << ov_log } +} + +impl MtCtx { + /// `ZSTDMT_createCCtx_advanced` + pub(crate) fn new(nb_workers: u32) -> Result { + let mut mtctx = Self { + cctx_pool: CCtxPool::new(0)?, + nb_workers: 0, + queue: channel().0, + cancelled: Arc::new(AtomicBool::new(false)), + jobs: VecDeque::new(), + params: CCtxParams::new(0), + target_section_size: 0, + target_prefix_size: 0, + in_buff: Vec::new(), + in_buff_prefix_size: 0, + in_buff_ready: false, + round_buff_capacity: 0, + round_buff_pos: 0, + serial: None, + next_job_id: 0, + frame_ended: false, + all_jobs_completed: true, + frame_content_size: 0, + cdict: None, + }; + mtctx.resize(nb_workers.clamp(1, ZSTDMT_NBWORKERS_MAX as u32) as usize)?; + Ok(mtctx) + } + + /// `ZSTDMT_resize`: no job is under way. + fn resize(&mut self, nb_workers: usize) -> Result<()> { + // `ZSTDMT_expandCCtxPool` + if nb_workers > self.cctx_pool.total { + // Now, and not when the last of the workers has left: the new ones are to have the memory. + self.cctx_pool.with_free(Vec::clear); + self.cctx_pool = CCtxPool::new(nb_workers)?; + } + // The workers there are leave, as nothing can be sent to the queue they have. + let queue_end; + (self.queue, queue_end) = channel(); + self.nb_workers = 0; + while self.nb_workers < nb_workers { + let queue_end = queue_end.clone(); + let cctx_pool = self.cctx_pool.clone(); + let cancelled = Arc::clone(&self.cancelled); + std::thread::Builder::new() + .name("zstd".into()) + .stack_size(2 << 20) + .spawn(move || worker_main(&queue_end, &cctx_pool, &cancelled)) + .map_err(|_| Error::MemoryAllocation)?; + self.nb_workers += 1; + } + Ok(()) + } + + /// `ZSTDMT_waitForAllJobsCompleted` and `ZSTDMT_releaseAllJobResources`, for a frame that is given up. + fn abandon_jobs(&mut self) { + self.cancelled.store(true, Ordering::SeqCst); + for job in &mut self.jobs { + while !job.is_complete() { + job.to_flush = 0; + job.output.clear(); + job.poll(true); + } + } + self.cancelled.store(false, Ordering::SeqCst); + self.jobs.clear(); + self.in_buff = Vec::new(); + self.in_buff_prefix_size = 0; + self.in_buff_ready = false; + self.all_jobs_completed = true; + } + + /// The state, once all the jobs that were created have had it. + fn take_serial(&mut self) -> Box { + (self.serial.take()) + .and_then(|serial| serial.recv().ok()) + .unwrap_or_else(|| Box::new(SerialState::new())) + } + + /// `ZSTDMT_initCStream_internal` + pub(crate) fn init_c_stream_internal( + &mut self, + cdict: Option>, + mut params: CCtxParams, + pledged_src_size: u64, + ) -> Result<()> { + if !self.all_jobs_completed { + self.abandon_jobs(); + } + if params.nb_workers as usize != self.nb_workers { + self.resize(params.nb_workers as usize)?; + } + + if params.job_size != 0 && params.job_size < ZSTDMT_JOBSIZE_MIN as usize { + params.job_size = ZSTDMT_JOBSIZE_MIN as usize; + } + params.job_size = params.job_size.min(ZSTDMT_JOBSIZE_MAX as usize); + + self.params = params; + self.frame_content_size = pledged_src_size; + self.target_prefix_size = compute_overlap_size(¶ms); + self.target_section_size = params.job_size; + if self.target_section_size == 0 { + self.target_section_size = 1 << compute_target_job_log(¶ms); + } + self.target_section_size = self.target_section_size.max(self.target_prefix_size); + { + let window_size = if params.ldm_params.enable_ldm == ParamSwitch::Enable { + 1usize << params.c_params.window_log + } else { + 0 + }; + let nb_slack_buffers = 2 + usize::from(self.target_prefix_size > 0); + let slack_size = self.target_section_size * nb_slack_buffers; + let nb_workers = params.nb_workers.max(1) as usize; + let sections_size = self.target_section_size * nb_workers; + let capacity = window_size.max(sections_size) + slack_size; + self.round_buff_capacity = self.round_buff_capacity.max(capacity); + } + self.round_buff_pos = 0; + self.in_buff = Vec::new(); + self.in_buff_prefix_size = 0; + self.in_buff_ready = false; + self.next_job_id = 0; + self.frame_ended = false; + self.all_jobs_completed = false; + self.cdict = cdict; + + let mut serial = self.take_serial(); + serial.reset(¶ms, self.round_buff_capacity)?; + let (sender, receiver) = channel(); + let _ = sender.send(serial); + self.serial = Some(receiver); + Ok(()) + } + + /// The most jobs at a time, which bounds memory as the size of the round buffer does in the C code. + fn max_jobs(&self) -> usize { + self.nb_workers + 2 + } + + /// `ZSTDMT_createCompressionJob`: of the input there is in `in_buff`. + fn create_compression_job(&mut self, end_op: EndDirective) -> Result<()> { + let end_frame = end_op == EndDirective::End; + if self.jobs.len() >= self.max_jobs() { + // Later. + return Ok(()); + } + + let data = core::mem::take(&mut self.in_buff); + let prefix_size = self.in_buff_prefix_size; + let src_size = data.len() - prefix_size; + let round_pos = self.round_buff_pos; + let first_job = self.next_job_id == 0; + let params = self.params; + let frame_checksum_needed = params.f_params.checksum_flag && end_frame && !first_job; + + self.round_buff_pos += src_size; + self.in_buff_ready = false; + self.in_buff_prefix_size = 0; + if !end_frame { + let new_prefix_size = src_size.min(self.target_prefix_size); + self.in_buff = try_copy(data.sub(data.len() - new_prefix_size..)?)?; + self.in_buff_prefix_size = new_prefix_size; + } else { + self.frame_ended = true; + if first_job { + // The only job writes the checksum itself. + self.params.f_params.checksum_flag = false; + } + } + + let mut pending = PendingJob { + progress: None, + output: VecDeque::new(), + flushed: 0, + to_flush: 0, + consumed: 0, + src_size, + frame_checksum_needed, + error: None, + }; + + if src_size == 0 && !first_job { + // `ZSTDMT_writeLastEmptyBlock`: all there is left to do is to end the frame. + let mut block = [0u8; 3]; + let size = write_last_empty_block(&mut block)?; + pending.take(Progress::Output { + data: block.sub(..size)?.to_vec(), + consumed: 0, + }); + self.jobs.push_back(pending); + self.next_job_id += 1; + return Ok(()); + } + + let (progress, progress_receiver) = channel(); + let (serial_out, serial_next) = channel(); + let serial_in = self.serial.replace(serial_next).ok_or(Error::Generic)?; + pending.progress = Some(progress_receiver); + let mut ldm_params = params.ldm_params; + if ldm_params.enable_ldm == ParamSwitch::Enable { + ldm::adjust_parameters(&mut ldm_params, ¶ms.c_params); + } + let job = Job { + data, + prefix_size, + round_pos, + first_job, + last_job: end_frame, + params, + cdict: if first_job { self.cdict.clone() } else { None }, + full_frame_size: self.frame_content_size, + max_nb_seq: ldm::get_max_nb_seq(ldm_params, self.target_section_size), + nb_seq_bound: ldm::get_max_nb_seq(ldm_params, src_size), + serial_in, + serial_out, + progress, + }; + self.queue.send(job).map_err(|_| Error::Generic)?; + self.jobs.push_back(pending); + self.next_job_id += 1; + Ok(()) + } + + /// `ZSTDMT_flushProduced`: returns the least number of bytes that are left to flush. + fn flush_produced( + &mut self, + output: &mut OutBuffer<'_>, + block_to_flush: bool, + end: EndDirective, + ) -> Result { + if let Some(job) = self.jobs.front_mut() { + job.poll(block_to_flush); + if let Some(code) = job.error { + self.abandon_jobs(); + return Err(code); + } + } + if self + .jobs + .front() + .is_some_and(|job| job.is_complete() && job.frame_checksum_needed) + { + let serial = self.take_serial(); + let checksum = serial.xxh_state.digest() as u32; + let (sender, receiver) = channel(); + let _ = sender.send(serial); + self.serial = Some(receiver); + if let Some(job) = self.jobs.front_mut() { + job.frame_checksum_needed = false; + let consumed = job.consumed; + job.take(Progress::Output { + data: checksum.to_le_bytes().to_vec(), + consumed, + }); + } + } + if let Some(job) = self.jobs.front_mut() { + while let Some(piece) = job.output.front() { + let room = output.dst.sub_mut(output.pos..)?; + let rest = piece.sub(job.flushed..)?; + let n = rest.len().min(room.len()); + room.write_at(0, rest.sub(..n)?)?; + output.pos += n; + job.to_flush -= n; + if n < rest.len() { + job.flushed += n; + break; + } + job.flushed = 0; + job.output.pop_front(); + } + let (to_flush, complete) = (job.to_flush, job.is_complete()); + if complete && to_flush == 0 { + self.jobs.pop_front(); + } + if to_flush > 0 { + return Ok(to_flush); + } + if !complete { + return Ok(1); + } + } + if !self.jobs.is_empty() { + return Ok(1); + } + if self.in_buff.len() > self.in_buff_prefix_size { + return Ok(1); + } + self.all_jobs_completed = self.frame_ended; + if end == EndDirective::End { + return Ok(usize::from(!self.frame_ended)); + } + Ok(0) + } + + /// `ZSTDMT_tryGetInputRange`: whether the next job can start taking input. + fn try_get_input_range(&mut self) -> Result { + if self.jobs.len() >= self.max_jobs() { + return Ok(false); + } + let space_left = self.round_buff_capacity - self.round_buff_pos; + if space_left < self.target_section_size { + // The C code moves the prefix to the start of the round buffer. + self.round_buff_pos = self.in_buff_prefix_size; + } + self.in_buff_ready = true; + Ok(true) + } + + /// Room for `more` input. A section that is flushed early is not to cost as much as a whole one. + fn reserve_input(&mut self, more: usize) -> Result<()> { + let needed = self.in_buff.len() + more; + if needed <= self.in_buff.capacity() { + return Ok(()); + } + let capacity = needed + .max(self.in_buff.capacity().saturating_mul(2)) + .min(self.in_buff_prefix_size + self.target_section_size); + self.in_buff + .try_reserve_exact(capacity - self.in_buff.len()) + .map_err(|_| Error::MemoryAllocation) + } + + /// `ZSTDMT_compressStream_generic` + pub(crate) fn compress_stream_generic( + &mut self, + output: &mut OutBuffer<'_>, + input: &mut InBuffer<'_>, + mut end_op: EndDirective, + ) -> Result { + let mut forward_input_progress = false; + + if self.frame_ended && end_op == EndDirective::Continue { + // The frame is complete and still to be flushed. + return Err(Error::StageWrong); + } + + if input.src.len() > input.pos && (self.in_buff_ready || self.try_get_input_range()?) { + let filled = self.in_buff.len() - self.in_buff_prefix_size; + let to_load = (input.src.len() - input.pos).min(self.target_section_size - filled); + self.reserve_input(to_load)?; + append(&mut self.in_buff, input.src.span(input.pos, to_load)?)?; + input.pos += to_load; + forward_input_progress = to_load > 0; + } + + if input.pos < input.src.len() && end_op == EndDirective::End { + // Not before all the input is consumed. + end_op = EndDirective::Flush; + } + + let filled = self.in_buff.len() - self.in_buff_prefix_size; + if filled >= self.target_section_size + || (end_op != EndDirective::Continue && filled > 0) + || (end_op == EndDirective::End && !self.frame_ended) + { + self.create_compression_job(end_op)?; + } + + // Waits for a job to output something if this call did nothing else. + let remaining_to_flush = self.flush_produced(output, !forward_input_progress, end_op)?; + if input.pos < input.src.len() { + return Ok(remaining_to_flush.max(1)); + } + Ok(remaining_to_flush) + } +} + +impl Drop for MtCtx { + fn drop(&mut self) { + // Nobody waits for them, which could take as long as a job does. + self.cancelled.store(true, Ordering::SeqCst); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::common::ZSTD_CONTENTSIZE_UNKNOWN; + use crate::compress::cctx::tests::small_params; + + #[test] + fn the_context_that_was_given_back_last_is_taken_first() -> Result<()> { + let cancelled = Arc::new(AtomicBool::new(false)); + let pool = CCtxPool::new(2)?; + let mut contexts = [(); 3].map(|()| pool.take(&cancelled)); + for (dict_id, cctx) in (1..).zip(&mut contexts) { + cctx.dict_id = dict_id; + } + // One more than there are workers, which does not happen. + contexts.into_iter().for_each(|cctx| pool.give_back(cctx)); + assert_eq!([(); 3].map(|()| pool.take(&cancelled).dict_id), [2, 1, 0]); + Ok(()) + } + + #[test] + fn jobs_that_come_one_after_the_other_have_the_same_context() -> Result<()> { + let mut mtctx = MtCtx::new(1)?; + // A frame in three jobs, each of which is over before the next one is created. + let compress = |mtctx: &mut MtCtx, nb_workers| -> Result<()> { + let mut params = small_params(Strategy::Fast, 6); + params.nb_workers = nb_workers; + mtctx.init_c_stream_internal(None, params, ZSTD_CONTENTSIZE_UNKNOWN)?; + for end_op in [EndDirective::Flush, EndDirective::Flush, EndDirective::End] { + let (mut dst, src) = ([0u8; 64], [0u8; 16]); + let mut output = OutBuffer { + dst: &mut dst, + pos: 0, + }; + let mut input = InBuffer { src: &src, pos: 0 }; + while mtctx.compress_stream_generic(&mut output, &mut input, end_op)? != 0 {} + } + Ok(()) + }; + let nb_contexts = |mtctx: &MtCtx| mtctx.cctx_pool.with_free(|contexts| contexts.len()); + + compress(&mut mtctx, 4)?; + compress(&mut mtctx, 4)?; + assert_eq!(nb_contexts(&mtctx), 1); + // With fewer workers the C code keeps its contexts, with more than ever it starts with new ones. + compress(&mut mtctx, 2)?; + assert_eq!((nb_contexts(&mtctx), mtctx.cctx_pool.total), (1, 4)); + let mut params = small_params(Strategy::Fast, 6); + params.nb_workers = 5; + mtctx.init_c_stream_internal(None, params, 0)?; + assert_eq!((nb_contexts(&mtctx), mtctx.cctx_pool.total), (0, 5)); + Ok(()) + } + + #[test] + fn a_worker_stops_at_a_prefix_or_a_block_of_a_frame_that_is_given_up() { + a_worker_stops(&small_params(Strategy::BtOpt, 6)); + let mut with_rows = small_params(Strategy::Lazy, 6); + with_rows.use_row_match_finder = ParamSwitch::Enable; + a_worker_stops(&with_rows); + } + + fn a_worker_stops(params: &CCtxParams) { + let data = [0u8; 64]; + let input = Input { + bufs: Buffers { + in_buff: &[], + user: &data, + dict: &[], + }, + cdict: None, + }; + let span = |start, end| Span { + buf: BufId::User, + start, + end, + }; + let cancelled = Arc::new(AtomicBool::new(false)); + let mut cctx = FrameCompressor::for_worker(Arc::clone(&cancelled)); + let begin = |cctx: &mut FrameCompressor| { + cctx.begin(&input, Some(span(0, 32)), params, 32, false) + .map(|_| ()) + }; + let mut dst = [0u8; 128]; + + cancelled.store(true, Ordering::SeqCst); + assert_eq!(begin(&mut cctx), Err(Error::Generic)); + + cancelled.store(false, Ordering::SeqCst); + assert_eq!(begin(&mut cctx), Ok(())); + cancelled.store(true, Ordering::SeqCst); + let given_up = cctx.compress_continue(&mut dst, &input, span(32, 64)); + assert_eq!(given_up, Err(Error::Generic)); + + cancelled.store(false, Ordering::SeqCst); + assert_eq!(begin(&mut cctx), Ok(())); + assert!( + cctx.compress_continue(&mut dst, &input, span(32, 64)) + .is_ok() + ); + } +} diff --git a/src/zstd/compress/opt.rs b/src/zstd/compress/opt.rs new file mode 100644 index 000000000000..1c2ba4957957 --- /dev/null +++ b/src/zstd/compress/opt.rs @@ -0,0 +1,2154 @@ +//! `zstd_opt.c`: the `btopt`, `btultra` and `btultra2` strategies, which look for the cheapest parse. + +use core::hint::{Locality, black_box, cold_path, prefetch_read}; +use core::mem; + +use super::hist; +use super::huf::HufRepeat; +use super::internal::{ + BlockSrc, Common, Descent, DictMatchState, DictMode, DictTree, EntropyCTables, First, Hasher, + Hints, Match, MatchState, Near, Nodes, Older, OptState, ParamSwitch, Prefix, PriceTable, + RawSeq, RawSeqStore, SeqStore, Strategy, TreeInput, ZSTD_OPT_NUM, bt_look_ahead, bt_nodes, + bt_set_children, bt_walk, hash3_of, index_overlap_check, ll_code, ml_code, new_rep, + offset_to_offbase, table_get, table_put, +}; +use crate::common::{ + LL_BITS, MAX_LL, MAX_ML, MAX_OFF, MINMATCH, ML_BITS, ZSTD_BLOCKSIZE_MAX, ZSTD_REP_NUM, +}; +use crate::error::Result; +use crate::util::{BUG, SliceExt, highbit32, read_le32}; + +/// How much a literal adds to its frequency, so that frequencies adapt fast to new statistics. +const ZSTD_LITFREQ_ADD: u32 = 2; +/// From that many positions on, [`Trees::update_tree`] gives fewer hints on the way down. +const LONG_RUN: u32 = 32; +/// And none if the way down is at most that many positions: further down, it gets to bytes that are not at hand. +const SHORT_WAY_DOWN: u32 = 32; +const ZSTD_MAX_PRICE: i32 = 1 << 30; +/// A first block of at most this size is priced by fixed rules, not by statistics. +const ZSTD_PREDEF_THRESHOLD: usize = 8; + +const MAX_LIT: u32 = 255; + +// ── prices ──────────────────────────────────────────────────────────────── + +/// Prices are in units of `1 / BITCOST_MULTIPLIER` of a bit. +const BITCOST_ACCURACY: u32 = 8; +const BITCOST_MULTIPLIER: u32 = 1 << BITCOST_ACCURACY; + +/// `WEIGHT`: approximate for `btopt`, accurate for `btultra`. +#[inline(always)] +const fn weight(stat: u32, opt_level: i32) -> u32 { + if opt_level != 0 { + frac_weight(stat) + } else { + bit_weight(stat) + } +} + +/// `ZSTD_bitWeight`: the cost of a statistic, in whole bits. +#[inline(always)] +const fn bit_weight(stat: u32) -> u32 { + highbit32(stat.wrapping_add(1)) * BITCOST_MULTIPLIER +} + +/// `ZSTD_fracWeight`: the cost of a statistic, with a fraction of a bit that is interpolated linearly. +#[inline(always)] +const fn frac_weight(raw_stat: u32) -> u32 { + let stat = raw_stat.wrapping_add(1); + let hb = highbit32(stat); + let b_weight = hb * BITCOST_MULTIPLIER; + // From 1 to 2, in fixed point. + let f_weight = (stat << BITCOST_ACCURACY) >> hb; + b_weight.wrapping_add(f_weight) +} + +/// `ZSTD_compressedLiterals` +#[inline(always)] +fn compressed_literals(opt_ptr: &OptState) -> bool { + opt_ptr.literal_compression_mode != ParamSwitch::Disable +} + +/// `sum_u32` +fn sum_u32(table: &[u32]) -> u32 { + table.iter().fold(0, |total, &elt| total.wrapping_add(elt)) +} + +/// `base_directive_e`: the least a scaled down statistic can be. +#[derive(Clone, Copy, PartialEq, Eq)] +enum BaseDirective { + /// 0, if it is 0 already. + Base0Possible, + Base1Guaranteed, +} + +/// `ZSTD_downscaleStats`: returns the new sum of the elements. +fn downscale_stats( + table: &mut [u32], + last_elt_index: u32, + shift: u32, + base1: BaseDirective, +) -> Result { + let mut sum = 0u32; + for elt in table.sub_mut(..=last_elt_index as usize)? { + let base = u32::from(base1 == BaseDirective::Base1Guaranteed || *elt > 0); + let new_stat = base.wrapping_add(*elt >> (shift & 31)); + sum = sum.wrapping_add(new_stat); + *elt = new_stat; + } + Ok(sum) +} + +/// `ZSTD_scaleStats`: scales all the elements down if their sum is too large. Returns the sum that results. +fn scale_stats(table: &mut [u32], last_elt_index: u32, log_target: u32) -> Result { + let prevsum = sum_u32(table.sub(..=last_elt_index as usize)?); + let factor = prevsum >> log_target; + if factor <= 1 { + return Ok(prevsum); + } + downscale_stats( + table, + last_elt_index, + highbit32(factor), + BaseDirective::Base1Guaranteed, + ) +} + +/// Frequencies, out of about `1 << scale_log`, that make each symbol cost the bits `bit_cost` tells. Returns their sum. +fn freqs_from_bit_costs( + freqs: &mut [u32], + last_elt_index: u32, + scale_log: u32, + bit_cost: impl Fn(u32) -> u32, +) -> Result { + let mut sum = 0u32; + for (symbol, freq) in (0u32..).zip(freqs.sub_mut(..=last_elt_index as usize)?) { + let bit_cost = bit_cost(symbol); + // At least 1, to have a cost. A code of a dictionary can have one bit more than `scale_log`: the C code then + // shifts by -1, which x86-64 and aarch64 take for 31. + *freq = if bit_cost != 0 { + 1 << (scale_log.wrapping_sub(bit_cost) & 31) + } else { + 1 + }; + sum = sum.wrapping_add(*freq); + } + Ok(sum) +} + +/// `ZSTD_rescaleFreqs`: for the first block, which no statistics tell, starts them from the tables of the dictionary +/// if there is one, else from the literals of `src` and from fixed values for the sequences. For the next ones, scales +/// down what the blocks before have accumulated. +fn rescale_freqs(opt_ptr: &mut OptState, symbol_costs: &EntropyCTables, src: &[u8]) -> Result<()> { + #[rustfmt::skip] + const BASE_LL_FREQS: [u32; MAX_LL as usize + 1] = [ + 4, 2, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, + ]; + #[rustfmt::skip] + const BASE_OFC_FREQS: [u32; MAX_OFF as usize + 1] = [ + 6, 2, 1, 1, 2, 3, 4, 4, + 4, 3, 2, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + ]; + + let compressed_literals = compressed_literals(opt_ptr); + + // `zop_predef`, the fixed prices for a first block of at most `ZSTD_PREDEF_THRESHOLD` bytes, are of no use: the last + // 8 bytes of a block are not searched for matches. + if opt_ptr.lit_length_sum == 0 { + if symbol_costs.huf.repeat_mode == HufRepeat::Valid { + // A Huffman table that has all the values is presumed to come from a dictionary. + if compressed_literals { + let huf = &symbol_costs.huf.c_table; + opt_ptr.lit_sum = + freqs_from_bit_costs(&mut opt_ptr.lit_freq, MAX_LIT, 11, |lit| { + huf.get_nb_bits(lit) + })?; + } + let fse = &symbol_costs.fse; + opt_ptr.lit_length_sum = + freqs_from_bit_costs(&mut opt_ptr.lit_length_freq, MAX_LL, 10, |ll| { + fse.litlength_c_table.get_max_nb_bits(ll) + })?; + opt_ptr.match_length_sum = + freqs_from_bit_costs(&mut opt_ptr.match_length_freq, MAX_ML, 10, |ml| { + fse.matchlength_c_table.get_max_nb_bits(ml) + })?; + opt_ptr.off_code_sum = + freqs_from_bit_costs(&mut opt_ptr.off_code_freq, MAX_OFF, 10, |of| { + fse.offcode_c_table.get_max_nb_bits(of) + })?; + } else { + if compressed_literals { + let mut lit = MAX_LIT; + hist::count_simple(&mut opt_ptr.lit_freq, &mut lit, src)?; + opt_ptr.lit_sum = downscale_stats( + &mut opt_ptr.lit_freq, + MAX_LIT, + 8, + BaseDirective::Base0Possible, + )?; + } + + opt_ptr.lit_length_freq.write_at(0, &BASE_LL_FREQS)?; + opt_ptr.lit_length_sum = sum_u32(&BASE_LL_FREQS); + + opt_ptr + .match_length_freq + .sub_mut(..=MAX_ML as usize)? + .fill(1); + opt_ptr.match_length_sum = MAX_ML + 1; + + opt_ptr.off_code_freq.write_at(0, &BASE_OFC_FREQS)?; + opt_ptr.off_code_sum = sum_u32(&BASE_OFC_FREQS); + } + } else { + if compressed_literals { + opt_ptr.lit_sum = scale_stats(&mut opt_ptr.lit_freq, MAX_LIT, 12)?; + } + opt_ptr.lit_length_sum = scale_stats(&mut opt_ptr.lit_length_freq, MAX_LL, 11)?; + opt_ptr.match_length_sum = scale_stats(&mut opt_ptr.match_length_freq, MAX_ML, 11)?; + opt_ptr.off_code_sum = scale_stats(&mut opt_ptr.off_code_freq, MAX_OFF, 11)?; + } + + Ok(()) +} + +/// The least length, minus [`MINMATCH`], that each code of match length stands for, and what follows the last one. +#[rustfmt::skip] +const ML_CODE_START: [u32; MAX_ML as usize + 2] = [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 34, 36, 38, 40, 44, 48, 56, 64, 80, 96, + 1 << 7, 1 << 8, 1 << 9, 1 << 10, 1 << 11, 1 << 12, 1 << 13, 1 << 14, 1 << 15, 1 << 16, 1 << 17, +]; + +/// The same for the codes of literal length. +#[rustfmt::skip] +const LL_CODE_START: [u32; MAX_LL as usize + 2] = [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 18, 20, 22, 24, 28, 32, 40, 48, + 1 << 6, 1 << 7, 1 << 8, 1 << 9, 1 << 10, 1 << 11, 1 << 12, 1 << 13, 1 << 14, 1 << 15, 1 << 16, 1 << 17, +]; + +/// The lengths of literals below this have their `LL_INCPRICE` in a table. +const LL_INC_LENGTHS: usize = 64; + +/// The statistics, and the prices that they give to the symbols. A price is only worked out when the frequency of its +/// symbol changes. What depends on the sum of the frequencies, which changes more often, is kept apart. +struct Prices<'a> { + opt_level: i32, + compressed_literals: bool, + lit_freq: &'a mut [u32; MAX_LIT as usize + 1], + lit_length_freq: &'a mut [u32; MAX_LL as usize + 1], + match_length_freq: &'a mut [u32; MAX_ML as usize + 1], + off_code_freq: &'a mut [u32; MAX_OFF as usize + 1], + sums: Sums, + /// `litSumBasePrice` + lit_base: u32, + /// `litLengthSumBasePrice` + lit_length_base: u32, + /// `matchLengthSumBasePrice` plus `offCodeSumBasePrice` + match_base: u32, + /// `WEIGHT(litFreq[literal])` + lit_weight: [u32; MAX_LIT as usize + 1], + /// By code. + lit_length: [u32; MAX_LL as usize + 1], + /// `LL_INCPRICE` of the short lengths. + lit_length_inc: [u32; LL_INC_LENGTHS], + /// By code. + offset: [u32; MAX_OFF as usize + 1], + /// By length, up to the longest that is priced. + ml: &'a mut [u32], +} + +/// `litSum`, `litLengthSum`, `matchLengthSum` and `offCodeSum` +#[derive(Clone, Copy)] +struct Sums { + lit: u32, + lit_length: u32, + match_length: u32, + off_code: u32, +} + +impl<'a> Prices<'a> { + fn new(stats: &'a mut OptState, ml: &'a mut [u32], opt_level: i32) -> Result { + let mut prices = Self { + opt_level, + compressed_literals: compressed_literals(stats), + sums: Sums { + lit: stats.lit_sum, + lit_length: stats.lit_length_sum, + match_length: stats.match_length_sum, + off_code: stats.off_code_sum, + }, + lit_freq: stats.lit_freq.first_chunk_mut().ok_or(BUG)?, + lit_length_freq: stats.lit_length_freq.first_chunk_mut().ok_or(BUG)?, + match_length_freq: stats.match_length_freq.first_chunk_mut().ok_or(BUG)?, + off_code_freq: stats.off_code_freq.first_chunk_mut().ok_or(BUG)?, + // 8 bits per literal, if they are not compressed. + lit_base: 8 * BITCOST_MULTIPLIER, + lit_length_base: 0, + match_base: 0, + lit_weight: [0; MAX_LIT as usize + 1], + lit_length: [0; MAX_LL as usize + 1], + lit_length_inc: [0; LL_INC_LENGTHS], + offset: [0; MAX_OFF as usize + 1], + ml, + }; + if prices.compressed_literals { + prices.weigh_literals(); + } + for code in 0..=MAX_LL { + prices.price_lit_length(code)?; + } + for code in 0..=MAX_OFF { + prices.price_offset(code)?; + } + for code in 0..=MAX_ML { + prices.price_match_length(code)?; + } + prices.set_base_prices(); + Ok(prices) + } + + #[inline(always)] + fn price_lit_length(&mut self, code: u32) -> Result<()> { + let code = code as usize; + let bits = u32::from(LL_BITS.at(code)?) * BITCOST_MULTIPLIER; + let price = bits.wrapping_sub(weight(self.lit_length_freq.at(code)?, self.opt_level)); + self.lit_length.put(code, price)?; + // The price changes where the lengths of this code start, and where those of the next one do. + if let Some(inc) = self + .lit_length_inc + .get_mut(LL_CODE_START.at(code)? as usize) + && let Some(&before) = self.lit_length.get(code.wrapping_sub(1)) + { + *inc = price.wrapping_sub(before); + } + if let Some(inc) = self + .lit_length_inc + .get_mut(LL_CODE_START.at(code + 1)? as usize) + && let Some(&after) = self.lit_length.get(code + 1) + { + *inc = after.wrapping_sub(price); + } + Ok(()) + } + + /// Below level 2, small offsets are favoured, which are faster to decompress. + #[inline(always)] + fn price_offset(&mut self, code: u32) -> Result<()> { + let freq = self.off_code_freq.at(code as usize)?; + let mut price = (code * BITCOST_MULTIPLIER).wrapping_sub(weight(freq, self.opt_level)); + if self.opt_level < 2 && code >= 20 { + price = price.wrapping_add((code - 19) * 2 * BITCOST_MULTIPLIER); + } + // Matches cost a bit more, to have fewer sequences, which is faster to decompress. + self.offset + .put(code as usize, price.wrapping_add(BITCOST_MULTIPLIER / 5)) + } + + #[inline(always)] + fn price_match_length(&mut self, code: u32) -> Result<()> { + let code = code as usize; + let bits = u32::from(ML_BITS.at(code)?) * BITCOST_MULTIPLIER; + let freq = self.match_length_freq.at(code)?; + let price = bits.wrapping_sub(weight(freq, self.opt_level)); + let most = self.ml.len(); + let start = (ML_CODE_START.at(code)? + MINMATCH) as usize; + let end = (ML_CODE_START.at(code + 1)? + MINMATCH) as usize; + self.ml.sub_mut(start.min(most)..end.min(most))?.fill(price); + Ok(()) + } + + /// `ZSTD_setBasePrices` + #[inline(always)] + fn set_base_prices(&mut self) { + if self.compressed_literals { + self.lit_base = weight(self.sums.lit, self.opt_level); + } + self.lit_length_base = weight(self.sums.lit_length, self.opt_level); + self.match_base = weight(self.sums.match_length, self.opt_level) + .wrapping_add(weight(self.sums.off_code, self.opt_level)); + } + + /// `LIT_PRICE`, which is `ZSTD_rawLiteralsCost` of one literal. + #[inline(always)] + fn literal(&self, literal: u8) -> i32 { + let lit_price_max = self.lit_base.wrapping_sub(BITCOST_MULTIPLIER); + let lit_price = self.lit_weight.get(usize::from(literal)).copied(); + self.lit_base + .wrapping_sub(lit_price.unwrap_or(0).min(lit_price_max)) as i32 + } + + /// `ZSTD_litLengthPrice`, but for what does not depend on the length. + #[inline(always)] + fn lit_length_part(&self, lit_length: u32) -> u32 { + // The format has no code for the length of a whole block of the largest size, which is then all literals. It + // is made to cost 1 bit more than the length before it. + let (lit_length, extra) = if lit_length == ZSTD_BLOCKSIZE_MAX as u32 { + (ZSTD_BLOCKSIZE_MAX as u32 - 1, BITCOST_MULTIPLIER) + } else { + (lit_length, 0) + }; + let price = self.lit_length.get(ll_code(lit_length) as usize).copied(); + extra.wrapping_add(price.unwrap_or(0)) + } + + /// `LL_PRICE` + #[inline(always)] + fn lit_length(&self, lit_length: u32) -> i32 { + self.lit_length_base + .wrapping_add(self.lit_length_part(lit_length)) as i32 + } + + /// `LL_INCPRICE`: what one more literal, which makes `lit_length` of them, adds to the price of their length. + #[inline(always)] + fn lit_length_inc(&self, lit_length: u32) -> i32 { + if let Some(&inc) = self.lit_length_inc.get(lit_length as usize) { + return inc as i32; + } + cold_path(); + self.lit_length_part(lit_length) + .wrapping_sub(self.lit_length_part(lit_length.wrapping_sub(1))) as i32 + } + + /// `ZSTD_getMatchPrice`, the price of the offset and of the match length of a sequence, but for what depends on the + /// length, which is in `ml`. `off_base` is as in [`SeqStore::store_seq`]. + #[inline(always)] + fn offset(&self, off_base: u32) -> i32 { + let price = self.offset.get(highbit32(off_base) as usize).copied(); + self.match_base.wrapping_add(price.unwrap_or(0)) as i32 + } + + /// Sets `lit_weight` for all of `lit_freq`. + #[inline(never)] + fn weigh_literals(&mut self) { + for (weight_of, &freq) in self.lit_weight.iter_mut().zip(&*self.lit_freq) { + *weight_of = weight(freq, self.opt_level); + } + } + + /// `ZSTD_updateStats` + #[inline(always)] + fn update_stats(&mut self, literals: &[u8], off_base: u32, match_length: u32) -> Result<()> { + #[inline(always)] + fn increment(freqs: &mut [u32], code: u32, sum: &mut u32) -> Result<()> { + let freq = freqs.at_mut(code as usize)?; + *freq = freq.wrapping_add(1); + *sum = sum.wrapping_add(1); + Ok(()) + } + + let lit_length = literals.len() as u32; + if self.compressed_literals { + if literals.len() > self.lit_weight.len() { + // There are fewer weights to compute than literals. + for &literal in literals { + let freq = self.lit_freq.at_mut(usize::from(literal))?; + *freq = freq.wrapping_add(ZSTD_LITFREQ_ADD); + } + self.weigh_literals(); + } else { + for &literal in literals { + let literal = usize::from(literal); + let freq = self.lit_freq.at(literal)?.wrapping_add(ZSTD_LITFREQ_ADD); + self.lit_freq.put(literal, freq)?; + self.lit_weight.put(literal, weight(freq, self.opt_level))?; + } + } + self.sums.lit = self + .sums + .lit + .wrapping_add(lit_length.wrapping_mul(ZSTD_LITFREQ_ADD)); + } + + let ll_code = ll_code(lit_length); + increment(self.lit_length_freq, ll_code, &mut self.sums.lit_length)?; + let off_code = highbit32(off_base); + increment(self.off_code_freq, off_code, &mut self.sums.off_code)?; + let ml_code = ml_code(match_length.wrapping_sub(MINMATCH)); + increment(self.match_length_freq, ml_code, &mut self.sums.match_length)?; + self.price_lit_length(ll_code)?; + self.price_offset(off_code)?; + self.price_match_length(ml_code) + } +} + +// ── binary tree search ──────────────────────────────────────────────────── + +/// What a parser has to search before the current segment. +#[derive(Clone, Copy)] +enum Before { + Nothing, + /// Of the window. + OlderSegment, + /// One that is searched in place. + Dictionary, +} + +/// Where the repeat offsets lead, for [`Trees::skip_matchless_generic`]. +#[derive(Clone, Copy)] +enum Reps { + /// Into the current segment, all three. + Near, + /// Before it, at least one of them. + Far, +} + +/// What the match finders work on for the time of a block: the tables of a [`MatchState`], and what does not change +/// of it. +struct Trees<'a> { + hash_table: &'a mut [u32], + hash_table3: &'a mut [u32], + bt: &'a mut Nodes, + input: TreeInput<'a>, + dms: Option>, + low_limit: u32, + /// How far a match can be, if that is not down to `low_limit`. + max_distance: u32, + hasher: Hasher, + hash_log3: u32, + /// 3 or 4. + min_match: u32, + nb_compares: u32, + sufficient_len: u32, + next_to_update: u32, + next_to_update3: u32, +} + +impl<'a> Trees<'a> { + fn new( + ms: &'a mut MatchState, + prefix: Prefix<'a>, + dict: Prefix<'a>, + dms: Option>, + iend: usize, + mls: u32, + ) -> Result { + let c_params = ms.c_params; + let dict_limit = ms.window.dict_limit; + if ms.hash_table.len() != 1usize.wrapping_shl(c_params.hash_log) + || ms.chain_table.len() != 1usize.wrapping_shl(c_params.chain_log) + { + return Err(BUG); + } + Ok(Self { + hash_table: ms.hash_table.as_mut_slice(), + hash_table3: ms.hash_table3.as_mut_slice(), + bt: bt_nodes(&mut ms.chain_table), + input: TreeInput::new(prefix, dict, dict_limit, iend)?, + dms, + low_limit: ms.window.low_limit, + // A dictionary is valid as a whole until it is invalidated as a whole. + max_distance: if ms.loaded_dict_end != 0 { + u32::MAX + } else { + 1u32.wrapping_shl(c_params.window_log) + }, + hasher: Hasher::new(c_params.hash_log, mls), + hash_log3: ms.hash_log3, + min_match: if c_params.min_match == 3 { 3 } else { 4 }, + nb_compares: 1u32.wrapping_shl(c_params.search_log), + sufficient_len: c_params.target_length.min(ZSTD_OPT_NUM as u32 - 1), + next_to_update: ms.next_to_update, + next_to_update3: ms.next_to_update, + }) + } + + /// `ZSTD_getLowestMatchIndex` + #[inline(always)] + const fn lowest_match_index(&self, curr: u32) -> u32 { + if curr.wrapping_sub(self.low_limit) > self.max_distance { + curr.wrapping_sub(self.max_distance) + } else { + self.low_limit + } + } + + /// `ZSTD_insertAndFindFirstIndexHash3`: indexes the 3-byte hashes up to `curr`, which is not included and starts + /// with `bytes`, and looks it up. All of it is in the current segment. + #[inline(always)] + fn insert_and_find_first_index_hash3(&mut self, curr: u32, bytes: u32) -> Result { + let input = self.input; + for idx in self.next_to_update3..curr { + let bytes = input.near.read32(idx.wrapping_sub(input.dict_limit)); + let hash3 = hash3_of(bytes.ok_or(BUG)?, self.hash_log3); + table_put(self.hash_table3, hash3, idx); + } + self.next_to_update3 = curr; + Ok(table_get(self.hash_table3, hash3_of(bytes, self.hash_log3))) + } + + /// Adds the position `ip` of `near`, which starts with `bytes`, to its tree. `visit` is called with each position + /// that is met on the way, which are all at least `low`, and what it has in common with `ip`, and returns whether + /// to go on. Returns how many more positions could have been compared. The rest is for [`bt_walk`]. + #[inline(always)] + fn walk( + &mut self, + ip: u32, + bytes: u64, + low: u32, + older: Older, + visit: impl FnMut(u32, Common) -> Result, + ) -> Result { + let curr = ip.wrapping_add(self.input.dict_limit); + if self.hash_table.is_empty() { + return Err(BUG); + } + + let h = self.hasher.hash(bytes); + let match_index = table_get(self.hash_table, h); + table_put(self.hash_table, h, curr); + bt_look_ahead::<4, 2>(self.hash_table, self.bt, &self.input, self.hasher, ip); + if match_index < low { + bt_set_children(self.bt, curr, [0, 0]); + return Ok(self.nb_compares); + } + + let descent = Descent::new(self.bt, curr); + bt_walk::( + self.bt, + &self.input, + ip, + match_index, + low, + self.nb_compares, + descent, + older, + visit, + ) + } + + /// `ZSTD_insertBt1`: adds `curr`, which is at least 8 bytes before the end, to its tree. `target` is up to where + /// the caller is filling them. Returns how many positions to move forward. + #[inline(always)] + fn insert_bt1( + &mut self, + curr: u32, + target: u32, + ) -> Result { + let ip = curr.wrapping_sub(self.input.dict_limit); + let bytes = self.input.near.read64(ip).ok_or(BUG)?; + // Only the positions that are still in the window once the trees are filled are of use. + let window_low = self.lowest_match_index(target); + let mut match_end_idx = curr.wrapping_add(8 + 1); + let mut best_length = 8u32; + + let visit = |match_index: u32, common: Common| -> Result { + if common.len > best_length { + cold_path(); + best_length = common.len; + if common.len > match_end_idx.wrapping_sub(match_index) { + match_end_idx = match_index.wrapping_add(common.len); + } + } + // If they are equal, there is no telling which is smaller. Dropping it keeps the tree consistent. + Ok(common.match_is_smaller.is_some()) + }; + self.walk::<{ First::Words }, HINTS>(ip, bytes, window_low, OLDER, visit)?; + + let positions = if best_length > 384 { + 192.min(best_length - 384) + } else { + 0 + }; + Ok(positions.max(match_end_idx.wrapping_sub(curr.wrapping_add(8)))) + } + + /// `ZSTD_updateTree_internal`: adds the positions from `next_to_update` up to `target` to the trees. `OLDER` is not + /// for [`Hints::Bytes`], where HTML takes 2.5% longer without the call, though records of 1 KB 15% less. + #[inline(always)] + fn update_tree(&mut self, target: u32) -> Result<()> { + if target.saturating_sub(self.next_to_update) < LONG_RUN || !self.input.far { + self.update_tree_in_part::<{ Hints::All }, OLDER>(target, target)?; + // A match skipped them: they go down past its positions, which follow each other too and so are mostly at hand. + } else if self.nb_compares > SHORT_WAY_DOWN { + self.update_tree_in_part::<{ Hints::Bytes }, { Older::Called }>(target, target)?; + } else { + self.update_tree_in_part::<{ Hints::None }, OLDER>(target, target)?; + } + self.next_to_update = target; + Ok(()) + } + + /// The part of [`Self::update_tree`] that gets to `up_to`, or somewhat beyond. + #[inline(never)] + fn update_tree_in_part( + &mut self, + up_to: u32, + target: u32, + ) -> Result<()> { + let mut idx = self.next_to_update; + while idx < up_to { + let forward = self.insert_bt1::(idx, target)?; + // A branch, so that the next position does not wait for what this one met: on x86-64, records take up to + // 20% less time. Without `black_box` it is folded into one addition. + idx = if forward == 1 { + idx.wrapping_add(1) + } else { + cold_path(); + idx.wrapping_add(black_box(forward)) + }; + } + self.next_to_update = idx; + Ok(()) + } + + /// The length of the match for `ip` at the repeat offset `rep_offset`, which is at least `min_match` if it leads to + /// the current segment. Else it is 0 if it is less. + #[inline(never)] + fn rep_len(&self, ip: u32, rep_offset: u32, window_low: u32) -> Result { + let input = self.input; + let min_match = self.min_match; + if rep_offset.wrapping_sub(1) < ip { + let more = input + .near + .count(ip + min_match, ip - rep_offset + min_match); + return Ok(more + min_match); + } + let Some((buf, pos, end)) = self.far_rep(self.dms, ip, rep_offset, window_low)? else { + return Ok(0); + }; + let after = pos + min_match as usize; + Ok(input.count_2segments(ip + min_match, buf, after, end)? + min_match) + } + + /// Where the repeat offset `rep_offset` of `ip` leads, which is before the current segment: a buffer, the position + /// in it, and that of the end of the segment. `None` unless what is there starts like `ip`. `dms` is `self.dms`, + /// of which a loop has a copy at hand, or `None` where the caller knows that there is none. + #[inline(always)] + fn far_rep( + &self, + dms: Option>, + ip: u32, + rep_offset: u32, + window_low: u32, + ) -> Result> { + let input = self.input; + let curr = ip.wrapping_add(input.dict_limit); + let rep_index = curr.wrapping_sub(rep_offset); + if !index_overlap_check(input.dict_limit, rep_index) { + return Ok(None); + } + let (buf, pos, end) = if let Some(dms) = dms { + let dms_high_limit = dms.end.wrapping_sub(dms.base) as u32; + let dms_index_delta = window_low.wrapping_sub(dms_high_limit); + let dms_low = dms.ms.window.low_limit.wrapping_add(dms_index_delta); + // `curr > rep_index >= dms_low` + if rep_offset.wrapping_sub(1) >= curr.wrapping_sub(dms_low) { + return Ok(None); + } + let pos = dms.pos_of(rep_index.wrapping_sub(dms_index_delta)); + (dms.buf, pos, dms.end) + } else { + // `curr > rep_index >= window_low` + if self.low_limit >= input.dict_limit + || rep_offset.wrapping_sub(1) >= curr.wrapping_sub(window_low) + { + return Ok(None); + } + let dict = input.dict(); + ( + dict.buf, + dict.pos_of(rep_index), + dict.pos_of(input.dict_limit), + ) + }; + let here = input.near.read32(ip).ok_or(BUG)?; + let is_alike = minmatch_differs(read_le32(buf, pos)?, here, self.min_match) == 0; + Ok(is_alike.then_some((buf, pos, end))) + } + + /// Whether what is at `index` of the older segment may start with the first 3 bytes of `bytes`. + #[inline(always)] + fn older_starts_alike(&self, index: u32, bytes: u32) -> Result { + let input = self.input; + // Not met: the last 8 positions of a segment do not get into the table. + if input.dict_limit.wrapping_sub(index) < 4 { + return Ok(true); + } + let there = read_le32(input.dict().buf, input.dict().pos_of(index))?; + Ok(minmatch_differs(there, bytes, 3) == 0) + } + + /// `ZSTD_btGetAllMatches_internal`: adds the positions up to `ip`, a position of `near`, to the trees, and stores in + /// `matches` the matches for it that are found on the way, each longer than the one before and all at least + /// `min_match` long. `ll0` is whether no literals come before `ip`. Returns how many there are. + #[inline(never)] + fn get_all_matches( + &mut self, + matches: &mut [Match], + ip: u32, + rep: &[u32; ZSTD_REP_NUM], + ll0: bool, + ) -> Result { + self.get_all_matches_generic(matches, ip, rep, ll0, Before::Nothing) + } + + /// [`Self::get_all_matches`] where a dictionary is searched in place. + #[inline(never)] + fn get_all_matches_in_place( + &mut self, + matches: &mut [Match], + ip: u32, + rep: &[u32; ZSTD_REP_NUM], + ll0: bool, + ) -> Result { + self.get_all_matches_generic(matches, ip, rep, ll0, Before::Dictionary) + } + + /// [`Self::get_all_matches`] where the window has an older segment. + #[inline(never)] + fn get_all_matches_older( + &mut self, + matches: &mut [Match], + ip: u32, + rep: &[u32; ZSTD_REP_NUM], + ll0: bool, + ) -> Result { + self.get_all_matches_generic(matches, ip, rep, ll0, Before::OlderSegment) + } + + /// The one of these three for what there is before the current segment. + #[inline(always)] + fn get_all_matches_after( + &mut self, + before: Before, + matches: &mut [Match], + ip: u32, + rep: &[u32; ZSTD_REP_NUM], + ll0: bool, + ) -> Result { + match before { + Before::Nothing => self.get_all_matches(matches, ip, rep, ll0), + Before::OlderSegment => self.get_all_matches_older(matches, ip, rep, ll0), + Before::Dictionary => self.get_all_matches_in_place(matches, ip, rep, ll0), + } + } + + /// Whether `ip`, which starts with `bytes`, starts like what is at `rep_offset`, before the current segment too. + /// `None` is a bug. + #[inline(always)] + fn starts_like_rep( + &self, + before: Before, + ip: u32, + bytes: u32, + rep_offset: u32, + most_in_prefix: u32, + window_low: u32, + ) -> Option { + if rep_offset.wrapping_sub(1) < most_in_prefix { + let rep_bytes = self.input.near.read32(ip.wrapping_sub(rep_offset))?; + Some(minmatch_differs(rep_bytes, bytes, self.min_match) == 0) + } else if rep_offset.wrapping_sub(1) >= ip { + let far = match before { + Before::Dictionary => self.far_rep(self.dms, ip, rep_offset, window_low), + Before::Nothing | Before::OlderSegment => { + debug_assert!(self.dms.is_none()); + self.far_rep(None, ip, rep_offset, window_low) + } + }; + Some(far.ok()?.is_some()) + } else { + Some(false) + } + } + + /// [`Self::get_all_matches`], of which there is a copy for each `before`. + #[inline(always)] + fn get_all_matches_generic( + &mut self, + matches: &mut [Match], + ip: u32, + rep: &[u32; ZSTD_REP_NUM], + ll0: bool, + before: Before, + ) -> Result { + let near = self.input.near; + let dict_limit = self.input.dict_limit; + let curr = ip.wrapping_add(dict_limit); + if curr < self.next_to_update { + // An area that is skipped. + return Ok(0); + } + if self.next_to_update < curr { + match before { + // Periodic data takes 1 to 2% longer with `TriedFirst`. + Before::OlderSegment => self.update_tree::<{ Older::TriedLast }>(curr)?, + Before::Nothing | Before::Dictionary => { + self.update_tree::<{ Older::Called }>(curr)?; + } + } + } + + let min_match = self.min_match; + let bytes = near.read64(ip).ok_or(BUG)?; + let dict_root = match self.dms { + Some(dms) => { + let hasher = self.hasher.with_log(dms.ms.c_params.hash_log); + DictTree::root(dms, hasher.hash(bytes))? + } + None => 0, + }; + let window_low = self.lowest_match_index(curr); + let mut mnum = 0u32; + let mut best_length = min_match - 1; + + // The repeat offsets. + let [rep0, rep1, rep2] = *rep; + let rep_offsets = if ll0 { + [rep1, rep2, rep0.wrapping_sub(1)] + } else { + [rep0, rep1, rep2] + }; + // With a dictionary, the range of valid offsets shrinks when the dictionary gets out of reach. + let most_in_prefix = ip.min(curr.wrapping_sub(window_low)); + let [a, b, c] = rep_offsets; + let candidates = match before { + Before::OlderSegment | Before::Dictionary => [ + self.starts_like_rep(before, ip, bytes as u32, a, most_in_prefix, window_low), + self.starts_like_rep(before, ip, bytes as u32, b, most_in_prefix, window_low), + self.starts_like_rep(before, ip, bytes as u32, c, most_in_prefix, window_low), + ], + // With the other case in this closure, it is not inlined: all data takes 10 to 18% longer. + Before::Nothing => { + rep_offsets.map(|rep_offset| { + // The subtraction wraps around to discard 0 and -1: `curr > rep_index >= dict_limit`. + if rep_offset.wrapping_sub(1) < most_in_prefix { + let rep_bytes = near.read32(ip.wrapping_sub(rep_offset))?; + Some(minmatch_differs(rep_bytes, bytes as u32, min_match) == 0) + } else { + Some(rep_offset.wrapping_sub(1) >= ip) + } + }) + } + }; + if candidates != [Some(false); 3] { + for ((off, rep_offset), is_candidate) in (1..).zip(rep_offsets).zip(candidates) { + if !is_candidate.ok_or(BUG)? { + continue; + } + let rep_len = self.rep_len(ip, rep_offset, window_low)?; + if rep_len > best_length { + best_length = rep_len; + matches.put(mnum as usize, Match { off, len: rep_len })?; + mnum += 1; + if rep_len > self.sufficient_len || rep_len == near.len().wrapping_sub(ip) { + // The best there can be. + return Ok(mnum); + } + } + } + } + + // The table of 3-byte hashes. That of a dictionary is not filled. + let match_low = window_low.max(1); + if best_length < 3 { + let match_index3 = self.insert_and_find_first_index_hash3(curr, bytes as u32)?; + // Farther than that is likely to cost too much. + if match_index3 >= match_low && curr.wrapping_sub(match_index3) < (1 << 18) { + let mlen = if match_index3 >= dict_limit { + let match_pos = match_index3.wrapping_sub(dict_limit); + let match_bytes = near.read32(match_pos).ok_or(BUG)?; + if minmatch_differs(match_bytes, bytes as u32, 3) != 0 { + 0 + } else { + count(near, ip, match_pos) + } + } else { + self.input.count_ext_dict(ip, match_index3)? + }; + + if mlen >= 3 { + best_length = mlen; + matches.put( + 0, + Match { + off: offset_to_offbase(curr.wrapping_sub(match_index3)), + len: mlen, + }, + )?; + mnum = 1; + if mlen > self.sufficient_len || mlen == near.len().wrapping_sub(ip) { + // The best there can be: `ip` is not inserted. + self.next_to_update = curr.wrapping_add(1); + return Ok(1); + } + } + } + } + + let mut found = Found::new(mnum, best_length, curr); + let older = match before { + Before::OlderSegment => Older::TriedFirst, + Before::Nothing | Before::Dictionary => Older::Called, + }; + let nb_compares = self.tree_matches(matches, ip, bytes, match_low, &mut found, older)?; + if dict_root != 0 && nb_compares != 0 { + // A copy, for the one that the loop above works on to need no place in memory. + let mut with_dict = found; + self.search_dms( + matches, + ip, + dict_root, + window_low, + nb_compares, + &mut with_dict, + )?; + found = with_dict; + } + + // Repetitive patterns are skipped. + self.next_to_update = found.match_end_idx.wrapping_sub(8); + Ok(found.mnum) + } + + /// The part of `ZSTD_insertBtAndGetAllMatches` that adds the position `ip` of `near`, which starts with `bytes`, to + /// its tree, and stores the matches that are met on the way. Returns how many more positions can be compared in the + /// tree of a dictionary. + #[inline(always)] + fn tree_matches( + &mut self, + matches: &mut [Match], + ip: u32, + bytes: u64, + match_low: u32, + found: &mut Found, + older: Older, + ) -> Result { + let curr = ip.wrapping_add(self.input.dict_limit); + let mut is_over = false; + let visit = |match_index: u32, common: Common| -> Result { + if common.len > found.best_length { + if common.len > found.match_end_idx.wrapping_sub(match_index) { + found.match_end_idx = match_index.wrapping_add(common.len); + } + found.best_length = common.len; + matches.put( + found.mnum as usize, + Match { + off: offset_to_offbase(curr.wrapping_sub(match_index)), + len: common.len, + }, + )?; + found.mnum += 1; + if common.len > ZSTD_OPT_NUM as u32 || common.match_is_smaller.is_none() { + // If they are equal, there is no telling which is smaller. Dropping it keeps the tree consistent. + is_over = true; + return Ok(false); + } + } + Ok(true) + }; + let left = + self.walk::<{ First::Apart }, { Hints::All }>(ip, bytes, match_low, older, visit)?; + Ok(if is_over { 0 } else { left }) + } + + /// [`Self::get_all_matches`] for `ip`, which comes after literals, and then for the positions after it, before + /// `limit`, for as long as there is no match. It stops before a position that starts like what is at a repeat + /// offset, or where the table of 3-byte hashes leads: that one is left to [`Self::get_all_matches`]. Returns where + /// it stops, and how many matches it has found there. + #[inline(never)] + fn skip_matchless( + &mut self, + matches: &mut [Match], + ip: u32, + limit: u32, + rep: &[u32; ZSTD_REP_NUM], + ) -> Result<(u32, u32)> { + self.skip_matchless_generic(matches, ip, limit, rep, Before::Nothing, Reps::Near) + } + + /// [`Self::skip_matchless`] where the window has an older segment. + #[inline(never)] + fn skip_matchless_older( + &mut self, + matches: &mut [Match], + ip: u32, + limit: u32, + rep: &[u32; ZSTD_REP_NUM], + ) -> Result<(u32, u32)> { + self.skip_matchless_far_then_near(matches, ip, limit, rep, Before::OlderSegment) + } + + /// [`Self::skip_matchless`] where a dictionary is searched in place. + #[inline(never)] + fn skip_matchless_in_place( + &mut self, + matches: &mut [Match], + ip: u32, + limit: u32, + rep: &[u32; ZSTD_REP_NUM], + ) -> Result<(u32, u32)> { + self.skip_matchless_far_then_near(matches, ip, limit, rep, Before::Dictionary) + } + + /// [`Self::skip_matchless`] with repeat offsets that lead before the current segment, which after a match there is + /// most of the time: one loop up to where none does any more, then the other. + #[inline(always)] + fn skip_matchless_far_then_near( + &mut self, + matches: &mut [Match], + mut ip: u32, + limit: u32, + rep: &[u32; ZSTD_REP_NUM], + before: Before, + ) -> Result<(u32, u32)> { + let in_place = matches!(before, Before::Dictionary); + let [rep0, rep1, rep2] = *rep; + let farthest = rep0.max(rep1).max(rep2); + if ip < farthest { + // What was at the last offset goes on after a few bytes that differ: nothing to pass. + let window_low = self.lowest_match_index(ip.wrapping_add(self.input.dict_limit)); + let dms = if in_place { self.dms } else { None }; + if ip < rep0 && self.far_rep(dms, ip, rep0, window_low)?.is_some() { + return Ok((ip, 0)); + } + let found; + (ip, found) = if in_place { + self.skip_matchless_far_reps_in_place(matches, ip, limit.min(farthest), rep)? + } else { + self.skip_matchless_far_reps_older(matches, ip, limit.min(farthest), rep)? + }; + if ip < farthest { + return Ok((ip, found)); + } + } + if in_place { + self.skip_matchless_generic(matches, ip, limit, rep, before, Reps::Near) + } else if self.min_match == 3 { + self.skip_matchless_near_reps_older(matches, ip, limit, rep) + } else { + // The same loop but for the table of 3 bytes. A copy comes out 3 to 8% slower on noise. + self.skip_matchless(matches, ip, limit, rep) + } + } + + /// The first part of [`Self::skip_matchless_older`]. + #[inline(never)] + fn skip_matchless_far_reps_older( + &mut self, + matches: &mut [Match], + ip: u32, + limit: u32, + rep: &[u32; ZSTD_REP_NUM], + ) -> Result<(u32, u32)> { + self.skip_matchless_generic(matches, ip, limit, rep, Before::OlderSegment, Reps::Far) + } + + /// The second part of [`Self::skip_matchless_older`], where a match can be 3 bytes. + #[inline(never)] + fn skip_matchless_near_reps_older( + &mut self, + matches: &mut [Match], + ip: u32, + limit: u32, + rep: &[u32; ZSTD_REP_NUM], + ) -> Result<(u32, u32)> { + self.skip_matchless_generic(matches, ip, limit, rep, Before::OlderSegment, Reps::Near) + } + + /// The first part of [`Self::skip_matchless_in_place`]. + #[inline(never)] + fn skip_matchless_far_reps_in_place( + &mut self, + matches: &mut [Match], + ip: u32, + limit: u32, + rep: &[u32; ZSTD_REP_NUM], + ) -> Result<(u32, u32)> { + self.skip_matchless_generic(matches, ip, limit, rep, Before::Dictionary, Reps::Far) + } + + /// `before` and `reps` are not const parameters: an instance of a generic function is compiled apart from the rest + /// of this file, after which [`Self::search_dms`] takes 2% longer on short records. + #[inline(always)] + fn skip_matchless_generic( + &mut self, + matches: &mut [Match], + mut ip: u32, + limit: u32, + rep: &[u32; ZSTD_REP_NUM], + before: Before, + reps: Reps, + ) -> Result<(u32, u32)> { + let in_place = matches!(before, Before::Dictionary); + let far_reps = matches!(reps, Reps::Far); + let near = self.input.near; + let dict_limit = self.input.dict_limit; + let min_match = self.min_match; + // Unless `far_reps`, no repeat offset leads before the current segment, from `ip` on. + if ip.wrapping_add(dict_limit) != self.next_to_update + || (!in_place && self.dms.is_some()) + || (!far_reps && rep.iter().any(|rep_offset| *rep_offset > ip)) + { + return Ok((ip, 0)); + } + let dms = if in_place { self.dms } else { None }; + let dict = dms.map(|dms| (dms, self.hasher.with_log(dms.ms.c_params.hash_log))); + + while ip < limit + && let Some(bytes) = near.read64(ip) + { + let curr = ip.wrapping_add(dict_limit); + let mut dict_root = 0; + if let Some((dms, hasher)) = dict { + if let Some(next) = near.read64(ip + 1) { + let slot = dms.ms.hash_table.as_ptr().wrapping_add(hasher.hash(next)); + prefetch_read(slot, Locality::L1); + } + dict_root = DictTree::root_at_once(dms, hasher.hash(bytes))?; + } + let window_low = self.lowest_match_index(curr); + let match_low = window_low.max(1); + let starts_alike = |pos: u32, min_match: u32| { + let there = near.read32(pos); + there.is_none_or(|there| minmatch_differs(there, bytes as u32, min_match) == 0) + }; + if far_reps { + let mut is_repeat = false; + for &rep_offset in rep { + is_repeat |= if rep_offset <= ip { + starts_alike(ip - rep_offset, min_match) + } else { + self.far_rep(dms, ip, rep_offset, window_low)?.is_some() + }; + } + if is_repeat { + break; + } + } else if (rep.iter()).any(|rep_offset| starts_alike(ip - rep_offset, min_match)) { + break; + } + if min_match == 3 { + let match_index3 = self.insert_and_find_first_index_hash3(curr, bytes as u32)?; + if match_index3 >= match_low + && curr.wrapping_sub(match_index3) < (1 << 18) + && if match_index3 >= dict_limit { + starts_alike(match_index3 - dict_limit, 3) + } else if matches!(before, Before::OlderSegment) { + self.older_starts_alike(match_index3, bytes as u32)? + } else { + true + } + { + break; + } + } + + let mut found = Found::new(0, min_match - 1, curr); + // Noise takes 2 to 4% longer with anything but a call. + let older = Older::Called; + let nb_compares = + self.tree_matches(matches, ip, bytes, match_low, &mut found, older)?; + if dict_root != 0 && nb_compares != 0 { + // A copy, so that `found` need not be in memory: 1 to 2% where there are no matches. + let mut with_dict = found; + self.search_dms( + matches, + ip, + dict_root, + window_low, + nb_compares, + &mut with_dict, + )?; + found = with_dict; + } + if found.mnum != 0 { + self.next_to_update = found.match_end_idx.wrapping_sub(8); + return Ok((ip, found.mnum)); + } + ip += 1; + } + self.next_to_update = ip.wrapping_add(dict_limit); + Ok((ip, 0)) + } + + /// The end of `ZSTD_insertBtAndGetAllMatches`: goes on in the tree of the dictionary whose root is `root`, for + /// `nb_compares` positions at most. + #[inline(never)] + fn search_dms( + &self, + matches: &mut [Match], + ip: u32, + root: u32, + window_low: u32, + nb_compares: u32, + found: &mut Found, + ) -> Result<()> { + let dms = self.dms.ok_or(BUG)?; + let input = self.input; + let curr = ip.wrapping_add(input.dict_limit); + let tree = DictTree::new(dms, window_low)?; + tree.walk(&input, ip, root, nb_compares, |dict_match_index, common| { + if common.len > found.best_length { + let match_index = dict_match_index.wrapping_add(tree.index_delta); + if common.len > found.match_end_idx.wrapping_sub(match_index) { + found.match_end_idx = match_index.wrapping_add(common.len); + } + found.best_length = common.len; + matches.put( + found.mnum as usize, + Match { + off: offset_to_offbase(curr.wrapping_sub(match_index)), + len: common.len, + }, + )?; + found.mnum += 1; + if common.len > ZSTD_OPT_NUM as u32 || common.match_is_smaller.is_none() { + return Ok(false); + } + } + Ok(true) + }) + } +} + +/// [`Near::count`], where few candidates get that far. +#[inline(never)] +fn count(near: Near<'_>, pos: u32, match_pos: u32) -> u32 { + near.count(pos, match_pos) +} + +/// What a search has found so far. +#[derive(Clone, Copy)] +struct Found { + mnum: u32, + best_length: u32, + /// The farthest position any match refers to, which detects repetitive patterns. + match_end_idx: u32, +} + +impl Found { + /// What the search for `curr` has before it gets to the trees. + #[inline(always)] + const fn new(mnum: u32, best_length: u32, curr: u32) -> Self { + Self { + mnum, + best_length, + match_end_idx: curr.wrapping_add(8 + 1), + } + } +} + +/// `ZSTD_readMINMATCH`: 0 if the first `length` bytes, 3 or 4, of the 4 in `a` and in `b` are the same. +#[inline(always)] +const fn minmatch_differs(a: u32, b: u32, length: u32) -> u32 { + (a ^ b) << ((4 - length) * 8) +} + +/// `ZSTD_updateTree`: indexes the segment from `ms.next_to_update` up to the position `ip`, in the binary trees. +/// Matches are not compared past the position `iend`. +pub(crate) fn update_tree( + ms: &mut MatchState, + prefix: Prefix<'_>, + ip: usize, + iend: usize, +) -> Result<()> { + let no_dict = Prefix { buf: &[], base: 0 }; + let mls = ms.c_params.min_match; + let target = prefix.index_of(ip); + // The prefix of a job can take seconds. + while ms.next_to_update < target { + ms.check_cancelled()?; + let up_to = ms.next_to_update.saturating_add(1 << 12).min(target); + let mut trees = Trees::new(ms, prefix, no_dict, None, iend, mls)?; + trees.update_tree_in_part::<{ Hints::All }, { Older::Called }>(up_to, target)?; + ms.next_to_update = trees.next_to_update; + } + ms.next_to_update = target; + Ok(()) +} + +// ── candidates from long distance matching ──────────────────────────────── + +/// `ZSTD_optLdm_t`: what it takes to decide whether to use a long distance match. +struct OptLdm<'a> { + /// `seqStore.seq`: the `seqStore.size` candidates for this block and the next ones. + seq: &'a [RawSeq], + /// `seqStore.pos` + pos: usize, + /// `seqStore.posInSequence` + pos_in_sequence: usize, + /// Where the current candidate starts. + start_pos_in_block: u32, + /// Where it ends. + end_pos_in_block: u32, + offset: u32, +} + +impl<'a> OptLdm<'a> { + fn new(seq_store: Option<&'a RawSeqStore>) -> Self { + Self { + seq: seq_store + .and_then(|store| store.seq.get(..store.size)) + .unwrap_or_default(), + pos: seq_store.map_or(0, |store| store.pos), + pos_in_sequence: seq_store.map_or(0, |store| store.pos_in_sequence), + start_pos_in_block: 0, + end_pos_in_block: 0, + offset: 0, + } + } + + /// `ZSTD_optLdm_skipRawSeqStoreBytes`: moves forward by `nb_bytes`. + fn skip_raw_seq_store_bytes(&mut self, nb_bytes: usize) { + let mut curr_pos = self.pos_in_sequence.wrapping_add(nb_bytes) as u32; + while curr_pos != 0 { + let Some(curr_seq) = self.seq.get(self.pos) else { + break; + }; + let seq_length = curr_seq.lit_length.wrapping_add(curr_seq.match_length); + if curr_pos >= seq_length { + curr_pos -= seq_length; + self.pos += 1; + } else { + self.pos_in_sequence = curr_pos as usize; + break; + } + } + if curr_pos == 0 || self.pos == self.seq.len() { + self.pos_in_sequence = 0; + } + } + + /// `ZSTD_opt_getNextMatchAndUpdateSeqStore`: finds where the next candidate starts and ends in the block, and + /// moves past it. + fn get_next_match_and_update_seq_store( + &mut self, + curr_pos_in_block: u32, + block_bytes_remaining: u32, + ) { + let Some(&curr_seq) = self.seq.get(self.pos) else { + // No position gets there, so that there is no candidate in this block. + self.start_pos_in_block = u32::MAX; + self.end_pos_in_block = u32::MAX; + return; + }; + // What is left of the literals and of the match. + let pos_in_sequence = self.pos_in_sequence as u32; + let curr_block_end_pos = curr_pos_in_block.wrapping_add(block_bytes_remaining); + let literals_bytes_remaining = curr_seq.lit_length.saturating_sub(pos_in_sequence); + let match_bytes_remaining = if literals_bytes_remaining == 0 { + curr_seq + .match_length + .wrapping_sub(pos_in_sequence.wrapping_sub(curr_seq.lit_length)) + } else { + curr_seq.match_length + }; + + if literals_bytes_remaining >= block_bytes_remaining { + // The match starts after the block. + self.start_pos_in_block = u32::MAX; + self.end_pos_in_block = u32::MAX; + self.skip_raw_seq_store_bytes(block_bytes_remaining as usize); + return; + } + + // What is left of a match can be too short to use, which is checked later. + self.start_pos_in_block = curr_pos_in_block.wrapping_add(literals_bytes_remaining); + self.end_pos_in_block = self.start_pos_in_block.wrapping_add(match_bytes_remaining); + self.offset = curr_seq.offset; + + if self.end_pos_in_block > curr_block_end_pos { + // The match ends after the block, which can only use its start. + self.end_pos_in_block = curr_block_end_pos; + self.skip_raw_seq_store_bytes( + curr_block_end_pos.wrapping_sub(curr_pos_in_block) as usize + ); + } else { + self.skip_raw_seq_store_bytes( + literals_bytes_remaining.wrapping_add(match_bytes_remaining) as usize, + ); + } + } + + /// `ZSTD_optLdm_maybeAddMatch`: adds what is left of the candidate at `curr_pos_in_block` to `matches`, if it is + /// long enough and longer than all of them, since they are in order of length. + fn maybe_add_match( + &self, + matches: &mut [Match], + nb_matches: &mut u32, + curr_pos_in_block: u32, + min_match: u32, + ) -> Result<()> { + let pos_diff = curr_pos_in_block.wrapping_sub(self.start_pos_in_block); + let candidate_match_length = self + .end_pos_in_block + .wrapping_sub(self.start_pos_in_block) + .wrapping_sub(pos_diff); + + if curr_pos_in_block < self.start_pos_in_block + || curr_pos_in_block >= self.end_pos_in_block + || candidate_match_length < min_match + { + return Ok(()); + } + + if *nb_matches == 0 + || (candidate_match_length > matches.at(*nb_matches as usize - 1)?.len + && (*nb_matches as usize) < ZSTD_OPT_NUM) + { + matches.put( + *nb_matches as usize, + Match { + off: offset_to_offbase(self.offset), + len: candidate_match_length, + }, + )?; + *nb_matches += 1; + } + Ok(()) + } + + /// Whether there is no candidate left. + #[inline(always)] + const fn is_over(&self) -> bool { + self.pos >= self.seq.len() + } + + /// `ZSTD_optLdm_processMatchCandidate` + #[inline(always)] + fn process_match_candidate( + &mut self, + matches: &mut [Match], + nb_matches: &mut u32, + curr_pos_in_block: u32, + remaining_bytes: u32, + min_match: u32, + ) -> Result<()> { + if self.is_over() { + return Ok(()); + } + + if curr_pos_in_block >= self.end_pos_in_block { + if curr_pos_in_block > self.end_pos_in_block { + // The parser does not stop at the end of each candidate: it is often some bytes past it. + let pos_overshoot = curr_pos_in_block - self.end_pos_in_block; + self.skip_raw_seq_store_bytes(pos_overshoot as usize); + } + self.get_next_match_and_update_seq_store(curr_pos_in_block, remaining_bytes); + } + self.maybe_add_match(matches, nb_matches, curr_pos_in_block, min_match) + } +} + +// ── the optimal parser ──────────────────────────────────────────────────── + +/// `ZSTD_optimal_t`, as [`PriceTable`] has it: a match followed by literals. +#[derive(Clone, Copy, Default)] +struct Stretch { + off: u32, + mlen: u32, + litlen: u32, + rep: [u32; ZSTD_REP_NUM], +} + +/// [`PriceTable::lens`] has the length of the match in its low bits, which are enough since the table has +/// [`ZSTD_OPT_NUM`] positions, and the number of literals above. +const LITLEN_SHIFT: u32 = 12; +const ONE_LITERAL: u32 = 1 << LITLEN_SHIFT; + +#[inline(always)] +const fn litlen_of(lens: u32) -> u32 { + lens >> LITLEN_SHIFT +} + +#[inline(always)] +const fn mlen_of(lens: u32) -> u32 { + lens & (ONE_LITERAL - 1) +} + +/// With fewer lengths than this, [`try_lengths`] is not worth calling. +const FEW_LENGTHS: u32 = 4; + +/// The positions that the lengths from `first_len` on of a match at the offset `off` lead to take it if that is cheaper +/// than what leads to them so far. `ml_prices` has the price of each of these lengths, and `base_price` is the rest. +#[inline(never)] +fn try_lengths( + prices: &mut [i32], + offs: &mut [u32], + lens: &mut [u32], + ml_prices: &[u32], + first_len: u32, + base_price: i32, + off: u32, +) { + let reached = prices.iter_mut().zip(offs).zip(lens); + for (mlen, (((price, best_off), best_lens), &ml_price)) in + (first_len..).zip(reached.zip(ml_prices)) + { + // Written so that it is done on vectors. + let new_price = base_price.wrapping_add(ml_price as i32); + let cheaper = u32::from(new_price < *price).wrapping_neg(); + *price = new_price.min(*price); + *best_off = (*best_off & !cheaper) | (off & cheaper); + *best_lens = (*best_lens & !cheaper) | (mlen & cheaper); + } +} + +/// `ZSTD_compressBlock_opt_generic`. `before` is a constant: what [`compress_block_opt`] finds before the input. +#[inline(always)] +fn compress_block_opt_generic( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + opt_level: i32, + dict_mode: DictMode, + before: Before, +) -> Result { + // The match finders borrow the rest of `ms`. + let mut opt_state = mem::replace(&mut ms.opt, OptState::new()); + let mut match_table = mem::take(&mut opt_state.match_table); + let mut price_table = mem::take(&mut opt_state.price_table); + let result = parse_block( + ms, + &mut opt_state, + &mut match_table, + &mut price_table, + seq_store, + rep, + src, + opt_level, + dict_mode, + before, + ); + opt_state.match_table = match_table; + opt_state.price_table = price_table; + ms.opt = opt_state; + result +} + +/// The body of `ZSTD_compressBlock_opt_generic`. `opt_state` is `ms.opt`, without `matches` and `price_table`. +#[inline(always)] +fn parse_block( + ms: &mut MatchState, + opt_state: &mut OptState, + matches: &mut [Match], + price_table: &mut PriceTable, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + opt_level: i32, + dict_mode: DictMode, + before: Before, +) -> Result { + let buf = src.buf; + let istart = src.start; + let mut ip = istart; + let mut anchor = istart; + let iend = src.end; + let ilimit = iend.saturating_sub(8); + let prefix_start = src.pos_of(ms.window.dict_limit); + + let sufficient_len = ms.c_params.target_length.min(ZSTD_OPT_NUM as u32 - 1); + let min_match = if ms.c_params.min_match == 3 { 3 } else { 4 }; + let dict = Prefix { + buf: src.dict_buf, + base: src.dict_base, + }; + let dms = if dict_mode == DictMode::DictMatchState { + Some(src.dms.ok_or(BUG)?) + } else { + None + }; + let mls = ms.c_params.min_match.clamp(3, 6); + let mut trees = Trees::new(ms, src.prefix(), dict, dms, iend, mls)?; + + let mut last_stretch = Stretch::default(); + let mut opt_ldm = OptLdm::new(src.ldm_seq_store); + opt_ldm.get_next_match_and_update_seq_store(0, (iend - ip) as u32); + + rescale_freqs(opt_state, src.symbol_costs, buf.sub(istart..iend)?)?; + // Of the same size, so that one check is enough for all. + let nb_positions = price_table.price.len(); + let opt_price = price_table.price.sub_mut(..nb_positions)?; + let opt_off = price_table.off.sub_mut(..nb_positions)?; + let opt_lens = price_table.lens.sub_mut(..nb_positions)?; + let opt_rep = price_table.rep.sub_mut(..nb_positions)?; + // No match is longer than a block. + let ml_prices = + (price_table.ml_price).sub_mut(..(sufficient_len as usize + 1).min(nb_positions))?; + let mut prices = Prices::new(opt_state, ml_prices, opt_level)?; + ip += usize::from(ip == prefix_start); + + 'series: while ip < ilimit { + let mut last_pos: u32; + let ll0_price = prices.lit_length(0); + let ll1_inc_price = prices.lit_length_inc(1); + + // The forward pass. The table holds stretches, each a match followed by literals, unlike a sequence, which is + // literals followed by a match: that way each position of a run of literals can have another match before it. + // It ends with the last stretch in `last_stretch`, which starts at `cur` and ends at `last_pos`. + let mut cur = 'forward: { + // Find a first match. + { + let mut nb_matches = 0; + if ip != anchor && opt_ldm.is_over() { + let first = (ip - prefix_start) as u32; + let limit = (ilimit - prefix_start) as u32; + let (stop, found) = match before { + Before::Nothing => trees.skip_matchless(matches, first, limit, rep)?, + Before::OlderSegment => { + trees.skip_matchless_older(matches, first, limit, rep)? + } + Before::Dictionary => { + trees.skip_matchless_in_place(matches, first, limit, rep)? + } + }; + ip += (stop - first) as usize; + if ip >= ilimit { + break 'series; + } + nb_matches = found; + } + let litlen = (ip - anchor) as u32; + if nb_matches == 0 { + let pos = (ip - prefix_start) as u32; + nb_matches = + trees.get_all_matches_after(before, matches, pos, rep, litlen == 0)?; + } + opt_ldm.process_match_candidate( + matches, + &mut nb_matches, + (ip - istart) as u32, + (iend - ip) as u32, + min_match, + )?; + if nb_matches == 0 { + ip += 1; + continue 'series; + } + let matches = matches.sub(..nb_matches as usize)?; + + // The price of the literals before the first match is the same whatever comes next, so it is left + // out. But how the price of their length changes depends on their length. + let first_price = prices.lit_length(litlen); + opt_price.put(0, first_price)?; + opt_lens.put(0, litlen << LITLEN_SHIFT)?; + opt_rep.put(0, *rep)?; + + // A large match is encoded at once. + let longest = matches.at(matches.len() - 1)?; + if longest.len > sufficient_len { + last_stretch.litlen = 0; + last_stretch.mlen = longest.len; + last_stretch.off = longest.off; + last_pos = longest.len; + break 'forward 0; + } + + // Set the prices of the first matches. + let mut pos = 1u32; + while pos < min_match { + opt_price.put(pos as usize, ZSTD_MAX_PRICE)?; + opt_lens.put(pos as usize, litlen.wrapping_add(pos) << LITLEN_SHIFT)?; + pos += 1; + } + for candidate in matches { + let base_price = first_price + .wrapping_add(ll0_price) + .wrapping_add(prices.offset(candidate.off)); + while pos <= candidate.len { + let price = base_price.wrapping_add(prices.ml.at(pos as usize)? as i32); + opt_price.put(pos as usize, price)?; + opt_off.put(pos as usize, candidate.off)?; + opt_lens.put(pos as usize, pos)?; + pos += 1; + } + } + last_pos = pos - 1; + opt_price.put(pos as usize, ZSTD_MAX_PRICE)?; + } + + // Check the positions that follow. + let mut cur = 0u32; + while cur < last_pos { + cur += 1; + let at = cur as usize; + let inr = ip + at; + if at + 1 >= nb_positions { + return Err(BUG); + } + + // Get to this position by a literal, if it is cheaper. + { + let lens = opt_lens.at(at - 1)?.wrapping_add(ONE_LITERAL); + let litlen = litlen_of(lens); + let price = opt_price + .at(at - 1)? + .wrapping_add(prices.literal(buf.at(inr - 1)?)) + .wrapping_add(prices.lit_length_inc(litlen)); + let match_price = opt_price.at(at)?; + if price <= match_price { + let match_lens = opt_lens.at(at)?; + let match_off = opt_off.at(at)?; + opt_price.put(at, price)?; + opt_lens.put(at, lens)?; + let off = opt_off.at(at - 1)?; + opt_off.put(at, off)?; + let rep = opt_rep.at(at - 1)?; + opt_rep.put(at, rep)?; + // If this replaces a match, and 1 literal is cheaper than none, the match followed by a + // literal may be a cheaper way to the next position. + if opt_level >= 1 + && litlen_of(match_lens) == 0 + && ll1_inc_price < 0 + && inr < iend + { + let next_lit_price = prices.literal(buf.at(inr)?); + let with_1_literal = match_price + .wrapping_add(next_lit_price) + .wrapping_add(ll1_inc_price); + let with_more_literals = price + .wrapping_add(next_lit_price) + .wrapping_add(prices.lit_length_inc(litlen.wrapping_add(1))); + if with_1_literal < with_more_literals + && with_1_literal < opt_price.at(at + 1)? + { + let prev = cur.wrapping_sub(match_lens) as usize; + let ll0 = litlen_of(opt_lens.at(prev)?) == 0; + opt_price.put(at + 1, with_1_literal)?; + opt_lens.put(at + 1, match_lens + ONE_LITERAL)?; + opt_off.put(at + 1, match_off)?; + let rep = new_rep(&opt_rep.at(prev)?, match_off, ll0); + opt_rep.put(at + 1, rep)?; + last_pos = last_pos.max(cur + 1); + } + } + } + } + + // The repeat offsets are not updated while matches are compared, but here, now that one is selected. + let lens = opt_lens.at(at)?; + let ll0 = litlen_of(lens) == 0; + if ll0 { + let prev = cur.wrapping_sub(lens) as usize; + let prev_ll0 = litlen_of(opt_lens.at(prev)?) == 0; + let rep = new_rep(&opt_rep.at(prev)?, opt_off.at(at)?, prev_ll0); + opt_rep.put(at, rep)?; + } + + // The last match starts at least 8 bytes before the end. + if inr > ilimit { + continue; + } + + if cur == last_pos { + break; + } + + let price = opt_price.at(at)?; + if opt_level == 0 + && opt_price.at(at + 1)? <= price.wrapping_add(BITCOST_MULTIPLIER as i32 / 2) + { + // The next position is cheaper to get to: this one is not promising. + continue; + } + + { + let (pos, rep) = ((inr - prefix_start) as u32, opt_rep.get(at).ok_or(BUG)?); + let mut nb_matches = + trees.get_all_matches_after(before, matches, pos, rep, ll0)?; + opt_ldm.process_match_candidate( + matches, + &mut nb_matches, + (inr - istart) as u32, + (iend - inr) as u32, + min_match, + )?; + if nb_matches == 0 { + continue; + } + let matches = matches.sub(..nb_matches as usize)?; + + let longest = matches.at(matches.len() - 1)?; + if longest.len > sufficient_len + || cur.wrapping_add(longest.len) as usize >= ZSTD_OPT_NUM + || inr + longest.len as usize >= iend + { + last_stretch.mlen = longest.len; + last_stretch.off = longest.off; + last_stretch.litlen = 0; + last_pos = cur.wrapping_add(longest.len); + break 'forward cur; + } + + // The positions that nothing leads to yet. + let furthest = cur + longest.len; + if furthest > last_pos { + let new = last_pos as usize + 1..=furthest as usize; + opt_price.sub_mut(new.clone())?.fill(ZSTD_MAX_PRICE); + // Not the end of a match. + opt_lens.sub_mut(new)?.fill(ONE_LITERAL); + last_pos = furthest; + } + + // Set the prices that the matches found here give. + let base_price = price.wrapping_add(ll0_price); + let mut start_ml = min_match; + for candidate in matches { + if candidate.len < start_ml { + start_ml = candidate.len + 1; + continue; + } + let base_price = base_price.wrapping_add(prices.offset(candidate.off)); + if opt_level == 0 || candidate.len - start_ml < FEW_LENGTHS { + // From the longest length to the shortest. + for mlen in (start_ml..=candidate.len).rev() { + let ml_price = prices.ml.at(mlen as usize)?; + let price = base_price.wrapping_add(ml_price as i32); + let pos = (cur + mlen) as usize; + if price < opt_price.at(pos)? { + opt_price.put(pos, price)?; + opt_off.put(pos, candidate.off)?; + opt_lens.put(pos, mlen)?; + } else if opt_level == 0 { + // Shorter lengths are not tried. + break; + } + } + } else { + let reached = + (cur + start_ml) as usize..=(cur + candidate.len) as usize; + try_lengths( + opt_price.sub_mut(reached.clone())?, + opt_off.sub_mut(reached.clone())?, + opt_lens.sub_mut(reached)?, + prices.ml.sub(start_ml as usize..=candidate.len as usize)?, + start_ml, + base_price, + candidate.off, + ); + } + start_ml = candidate.len + 1; + } + } + opt_price.put(last_pos as usize + 1, ZSTD_MAX_PRICE)?; + } + + let lens = opt_lens.at(last_pos as usize)?; + last_stretch = Stretch { + off: opt_off.at(last_pos as usize)?, + mlen: mlen_of(lens), + litlen: litlen_of(lens), + rep: opt_rep.at(last_pos as usize)?, + }; + last_pos.wrapping_sub(last_stretch.mlen) + }; + + if last_stretch.mlen == 0 { + // All the matches have been turned into literals. + ip += last_pos as usize; + continue; + } + + if last_stretch.litlen == 0 { + // It ends with a match. + let ll0 = litlen_of(opt_lens.at(cur as usize)?) == 0; + *rep = new_rep(&opt_rep.at(cur as usize)?, last_stretch.off, ll0); + } else { + *rep = last_stretch.rep; + cur = cur.wrapping_sub(last_stretch.litlen); + } + + // Follow the cheapest path backwards, and turn its stretches into sequences, which go where the repeat offsets + // were, from the end. The literals that end the last stretch are in no sequence: they are parsed again. + let mut first_seq = opt_rep.len(); + let mut stretch_pos = cur; + let mut mlen = last_stretch.mlen; + let mut off = last_stretch.off; + loop { + let lens = opt_lens.at(stretch_pos as usize)?; + first_seq = first_seq.wrapping_sub(1); + opt_rep.put(first_seq, [litlen_of(lens), mlen, off])?; + mlen = mlen_of(lens); + if mlen == 0 { + // The start of the series. + break; + } + off = opt_off.at(stretch_pos as usize)?; + stretch_pos = stretch_pos.wrapping_sub(litlen_of(lens).wrapping_add(mlen)); + } + + for &[llen, mlen, off] in opt_rep.sub(first_seq..)? { + let llen = llen as usize; + prices.update_stats(buf.span(anchor, llen)?, off, mlen)?; + seq_store.store_seq(llen, buf, anchor, off, mlen as usize)?; + anchor += llen + mlen as usize; + ip = anchor; + } + + prices.set_base_prices(); + } + + let sums = prices.sums; + opt_state.lit_sum = sums.lit; + opt_state.lit_length_sum = sums.lit_length; + opt_state.match_length_sum = sums.match_length; + opt_state.off_code_sum = sums.off_code; + ms.next_to_update = trees.next_to_update; + Ok(iend - anchor) +} + +/// `ZSTD_compressBlock_opt0` +#[inline(never)] +fn compress_block_opt0( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + dict_mode: DictMode, +) -> Result { + compress_block_opt_generic(ms, seq_store, rep, src, 0, dict_mode, Before::Nothing) +} + +/// [`compress_block_opt0`] with what [`Trees`] has for a dictionary in place. A copy: one parser that chooses takes 0.4 +/// to 1.4% longer on most data without a dictionary. +#[inline(never)] +fn compress_block_opt0_in_place( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + dict_mode: DictMode, +) -> Result { + compress_block_opt_generic(ms, seq_store, rep, src, 0, dict_mode, Before::Dictionary) +} + +/// [`compress_block_opt0`] with what [`Trees`] has for an older segment, likewise. +#[inline(never)] +fn compress_block_opt0_older( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + dict_mode: DictMode, +) -> Result { + compress_block_opt_generic(ms, seq_store, rep, src, 0, dict_mode, Before::OlderSegment) +} + +/// `ZSTD_compressBlock_opt2` +fn compress_block_opt2( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + dict_mode: DictMode, +) -> Result { + compress_block_opt_generic(ms, seq_store, rep, src, 2, dict_mode, Before::Nothing) +} + +/// [`compress_block_opt2`], likewise. +#[inline(never)] +fn compress_block_opt2_in_place( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + dict_mode: DictMode, +) -> Result { + compress_block_opt_generic(ms, seq_store, rep, src, 2, dict_mode, Before::Dictionary) +} + +/// [`compress_block_opt2`], likewise. +#[inline(never)] +fn compress_block_opt2_older( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + dict_mode: DictMode, +) -> Result { + compress_block_opt_generic(ms, seq_store, rep, src, 2, dict_mode, Before::OlderSegment) +} + +/// `ZSTD_initStats_ultra`: a first pass over the block, only to have better statistics to start from. It has to be the +/// first block, with no dictionary and no long distance matching. Returns the block as the window then sees it, since +/// its indices change. +fn init_stats_ultra<'a>( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &[u32; ZSTD_REP_NUM], + src: &BlockSrc<'a>, +) -> Result> { + // The repeat offsets it leaves are of no use. + let mut tmp_rep = *rep; + compress_block_opt2(ms, seq_store, &mut tmp_rep, src, DictMode::NoDict)?; + + // Of the first pass, only the statistics are kept: what the tables have is made out of reach. + let src_size = src.end - src.start; + seq_store.reset(); + ms.window.base = ms.window.base.wrapping_sub(src_size); + ms.window.dict_limit = ms.window.dict_limit.wrapping_add(src_size as u32); + ms.window.low_limit = ms.window.dict_limit; + ms.next_to_update = ms.window.dict_limit; + Ok(BlockSrc { + base: src.base.wrapping_sub(src_size), + ..*src + }) +} + +/// `ZSTD_compressBlock_btultra2` +fn compress_block_btultra2( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, +) -> Result { + let curr = src.index_of(src.start); + + // The first block is parsed twice, the first time to collect statistics for the second. Since the first pass is + // then forgotten, nothing must be in the tables before it: no dictionary, no prefix, no long distance matching. + if ms.opt.lit_length_sum == 0 + && seq_store.sequences.is_empty() + && ms.window.dict_limit == ms.window.low_limit + && curr == ms.window.dict_limit + && src.end - src.start > ZSTD_PREDEF_THRESHOLD + { + let src = init_stats_ultra(ms, seq_store, rep, src)?; + return compress_block_opt2(ms, seq_store, rep, &src, DictMode::NoDict); + } + + compress_block_opt2(ms, seq_store, rep, src, DictMode::NoDict) +} + +/// `ZSTD_compressBlock_btopt`, `ZSTD_compressBlock_btultra`, `ZSTD_compressBlock_btultra2` and their `_dictMatchState` +/// and `_extDict` variants. With a dictionary of either kind, `btultra2` is `btultra`. +/// +/// `btultra2` can move `ms.window.base`, which `src.base` comes from. +pub(crate) fn compress_block_opt( + ms: &mut MatchState, + seq_store: &mut SeqStore, + rep: &mut [u32; ZSTD_REP_NUM], + src: &BlockSrc<'_>, + strategy: Strategy, + dict_mode: DictMode, +) -> Result { + let in_place = src.dms.is_some(); + match (strategy, dict_mode) { + (Strategy::BtUltra2, DictMode::NoDict) => compress_block_btultra2(ms, seq_store, rep, src), + (Strategy::BtUltra | Strategy::BtUltra2, _) if in_place => { + compress_block_opt2_in_place(ms, seq_store, rep, src, dict_mode) + } + (Strategy::BtUltra | Strategy::BtUltra2, DictMode::ExtDict) => { + compress_block_opt2_older(ms, seq_store, rep, src, dict_mode) + } + (Strategy::BtUltra | Strategy::BtUltra2, _) => { + compress_block_opt2(ms, seq_store, rep, src, dict_mode) + } + _ if in_place => compress_block_opt0_in_place(ms, seq_store, rep, src, dict_mode), + (_, DictMode::ExtDict) => compress_block_opt0_older(ms, seq_store, rep, src, dict_mode), + _ => compress_block_opt0(ms, seq_store, rep, src, dict_mode), + } +} diff --git a/src/zstd/compress/params.rs b/src/zstd/compress/params.rs new file mode 100644 index 000000000000..7c2c81e2c5e8 --- /dev/null +++ b/src/zstd/compress/params.rs @@ -0,0 +1,901 @@ +//! The parameters of `zstd_compress.c`: what can be asked for, and how it turns into what is applied. + +use core::ops::RangeInclusive; + +use super::internal::{ + CompressionParameters, LdmParams, ParamSwitch, Strategy, ZSTD_OPT_SIZE, + ZSTD_SHORT_CACHE_TAG_BITS, +}; +use super::lazy::ZSTD_ROW_HASH_TAG_BITS; +use super::{ldm, row_match_finder_supported}; +use crate::common::{ + LITBITS, MAX_LL, MAX_ML, MAX_OFF, WILDCOPY_OVERLENGTH, ZSTD_BLOCKSIZE_MAX, + ZSTD_CONTENTSIZE_UNKNOWN, ZSTD_WINDOWLOG_ABSOLUTEMIN, ZSTD_WINDOWLOG_LIMIT_DEFAULT, + ZSTD_WINDOWLOG_MAX, ZSTD_WORKSPACETOOLARGE_FACTOR, ZSTD_WORKSPACETOOLARGE_MAXDURATION, +}; +use crate::error::{Error, Result}; +use crate::util::highbit32; + +pub const ZSTD_CLEVEL_DEFAULT: i32 = 3; +const ZSTD_MAX_CLEVEL: i32 = 22; +const ZSTD_MIN_CLEVEL: i32 = -(ZSTD_TARGETLENGTH_MAX); + +const ZSTD_WINDOWLOG_MIN: i32 = 10; +pub(crate) const ZSTD_HASHLOG_MAX: i32 = 30; +pub(crate) const ZSTD_HASHLOG_MIN: i32 = 6; +const ZSTD_CHAINLOG_MAX: i32 = 30; +const ZSTD_CHAINLOG_MIN: i32 = ZSTD_HASHLOG_MIN; +const ZSTD_SEARCHLOG_MAX: i32 = ZSTD_WINDOWLOG_MAX as i32 - 1; +const ZSTD_SEARCHLOG_MIN: i32 = 1; +const ZSTD_MINMATCH_MAX: i32 = 7; +const ZSTD_MINMATCH_MIN: i32 = 3; +const ZSTD_TARGETLENGTH_MAX: i32 = ZSTD_BLOCKSIZE_MAX as i32; +const ZSTD_TARGETLENGTH_MIN: i32 = 0; +const ZSTD_OVERLAPLOG_MIN: i32 = 0; +const ZSTD_OVERLAPLOG_MAX: i32 = 9; +const ZSTD_LDM_HASHLOG_MIN: i32 = ZSTD_HASHLOG_MIN; +const ZSTD_LDM_HASHLOG_MAX: i32 = ZSTD_HASHLOG_MAX; +const ZSTD_LDM_MINMATCH_MIN: i32 = 4; +const ZSTD_LDM_MINMATCH_MAX: i32 = 4096; +const ZSTD_LDM_BUCKETSIZELOG_MIN: i32 = 1; +pub(crate) const ZSTD_LDM_BUCKETSIZELOG_MAX: i32 = 8; +const ZSTD_LDM_HASHRATELOG_MIN: i32 = 0; +const ZSTD_LDM_HASHRATELOG_MAX: i32 = ZSTD_WINDOWLOG_MAX as i32 - ZSTD_HASHLOG_MIN; +const ZSTD_TARGETCBLOCKSIZE_MIN: i32 = 1340; +const ZSTD_TARGETCBLOCKSIZE_MAX: i32 = ZSTD_BLOCKSIZE_MAX as i32; +const ZSTD_LDM_DEFAULT_WINDOW_LOG: u32 = ZSTD_WINDOWLOG_LIMIT_DEFAULT; +pub(crate) const ZSTD_HASHLOG3_MAX: u32 = 17; + +pub(crate) const ZSTDMT_NBWORKERS_MAX: i32 = 256; +pub(crate) const ZSTDMT_JOBSIZE_MIN: i32 = 512 << 10; +pub(crate) const ZSTDMT_JOBLOG_MAX: u32 = 30; +pub(crate) const ZSTDMT_JOBSIZE_MAX: i32 = 1024 << 20; + +/// `ZSTD_cParameter`, as far as `node:zlib` lets a parameter through. +pub mod c_param { + pub(crate) const FORMAT: i32 = 10; + pub const COMPRESSION_LEVEL: i32 = 100; + pub const WINDOW_LOG: i32 = 101; + pub const HASH_LOG: i32 = 102; + pub const CHAIN_LOG: i32 = 103; + pub const SEARCH_LOG: i32 = 104; + pub const MIN_MATCH: i32 = 105; + pub const TARGET_LENGTH: i32 = 106; + pub const STRATEGY: i32 = 107; + pub(crate) const TARGET_C_BLOCK_SIZE: i32 = 130; + pub const ENABLE_LONG_DISTANCE_MATCHING: i32 = 160; + pub const LDM_HASH_LOG: i32 = 161; + pub const LDM_MIN_MATCH: i32 = 162; + pub const LDM_BUCKET_SIZE_LOG: i32 = 163; + pub const LDM_HASH_RATE_LOG: i32 = 164; + pub const CONTENT_SIZE_FLAG: i32 = 200; + pub const CHECKSUM_FLAG: i32 = 201; + pub const DICT_ID_FLAG: i32 = 202; + pub const NB_WORKERS: i32 = 400; + pub const JOB_SIZE: i32 = 401; + pub const OVERLAP_LOG: i32 = 402; +} + +/// `ZSTD_format_e` +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum Format { + Zstd1, + Zstd1Magicless, +} + +/// `ZSTD_frameParameters` +#[derive(Clone, Copy)] +pub(crate) struct FrameParameters { + pub(crate) content_size_flag: bool, + pub(crate) checksum_flag: bool, + pub(crate) no_dict_id_flag: bool, +} + +/// `ZSTD_parameters` +#[derive(Clone, Copy)] +pub(crate) struct Parameters { + pub(crate) c_params: CompressionParameters, + pub(crate) f_params: FrameParameters, +} + +/// `ZSTD_CParamMode_e`: what the parameters are selected for. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum CParamMode { + /// No dictionary, or one that is copied or loaded into the context's own tables. + NoAttachDict, + /// The dictionary keeps its own tables, so its size does not count. + AttachDict, + CreateCDict, +} + +/// `ZSTD_CCtx_params`, without the fields that nothing can change from their default. +#[derive(Clone, Copy)] +pub(crate) struct CCtxParams { + pub(crate) format: Format, + pub(crate) c_params: CompressionParameters, + pub(crate) f_params: FrameParameters, + pub(crate) compression_level: i32, + /// Only the jobs of a multithreaded compression set it. + pub(crate) force_window: bool, + pub(crate) target_c_block_size: usize, + pub(crate) nb_workers: i32, + pub(crate) job_size: usize, + pub(crate) overlap_log: i32, + pub(crate) ldm_params: LdmParams, + pub(crate) post_block_splitter: ParamSwitch, + pub(crate) use_row_match_finder: ParamSwitch, +} + +const NO_C_PARAMS: CompressionParameters = CompressionParameters { + window_log: 0, + chain_log: 0, + hash_log: 0, + search_log: 0, + min_match: 0, + target_length: 0, + strategy: None, +}; + +pub(crate) const NO_LDM_PARAMS: LdmParams = LdmParams { + enable_ldm: ParamSwitch::Auto, + hash_log: 0, + bucket_size_log: 0, + min_match_length: 0, + hash_rate_log: 0, + window_log: 0, +}; + +/// `ZSTD_cParam_getBounds` +fn bounds(param: i32) -> Result> { + Ok(match param { + c_param::COMPRESSION_LEVEL => ZSTD_MIN_CLEVEL..=ZSTD_MAX_CLEVEL, + c_param::WINDOW_LOG => ZSTD_WINDOWLOG_MIN..=ZSTD_WINDOWLOG_MAX as i32, + c_param::HASH_LOG => ZSTD_HASHLOG_MIN..=ZSTD_HASHLOG_MAX, + c_param::CHAIN_LOG => ZSTD_CHAINLOG_MIN..=ZSTD_CHAINLOG_MAX, + c_param::SEARCH_LOG => ZSTD_SEARCHLOG_MIN..=ZSTD_SEARCHLOG_MAX, + c_param::MIN_MATCH => ZSTD_MINMATCH_MIN..=ZSTD_MINMATCH_MAX, + c_param::TARGET_LENGTH => ZSTD_TARGETLENGTH_MIN..=ZSTD_TARGETLENGTH_MAX, + c_param::STRATEGY => Strategy::Fast as i32..=Strategy::BtUltra2 as i32, + c_param::CONTENT_SIZE_FLAG | c_param::CHECKSUM_FLAG | c_param::DICT_ID_FLAG => 0..=1, + c_param::NB_WORKERS => 0..=ZSTDMT_NBWORKERS_MAX, + c_param::JOB_SIZE => 0..=ZSTDMT_JOBSIZE_MAX, + c_param::OVERLAP_LOG => ZSTD_OVERLAPLOG_MIN..=ZSTD_OVERLAPLOG_MAX, + c_param::ENABLE_LONG_DISTANCE_MATCHING => { + ParamSwitch::Auto as i32..=ParamSwitch::Disable as i32 + } + c_param::LDM_HASH_LOG => ZSTD_LDM_HASHLOG_MIN..=ZSTD_LDM_HASHLOG_MAX, + c_param::LDM_MIN_MATCH => ZSTD_LDM_MINMATCH_MIN..=ZSTD_LDM_MINMATCH_MAX, + c_param::LDM_BUCKET_SIZE_LOG => ZSTD_LDM_BUCKETSIZELOG_MIN..=ZSTD_LDM_BUCKETSIZELOG_MAX, + c_param::LDM_HASH_RATE_LOG => ZSTD_LDM_HASHRATELOG_MIN..=ZSTD_LDM_HASHRATELOG_MAX, + c_param::FORMAT => 0..=1, + c_param::TARGET_C_BLOCK_SIZE => ZSTD_TARGETCBLOCKSIZE_MIN..=ZSTD_TARGETCBLOCKSIZE_MAX, + _ => return Err(Error::ParameterUnsupported), + }) +} + +/// `ZSTD_cParam_clampBounds` +fn clamp_bounds(param: i32, value: i32) -> Result { + let bounds = bounds(param)?; + Ok(value.clamp(*bounds.start(), *bounds.end())) +} + +/// `BOUNDCHECK` +fn bound_check(param: i32, value: i32) -> Result<()> { + if !bounds(param)?.contains(&value) { + return Err(Error::ParameterOutOfBound); + } + Ok(()) +} + +/// `BOUNDCHECK`, for the parameters of which 0 means "use the default". +fn bound_check_unless_0(param: i32, value: i32) -> Result { + if value != 0 { + bound_check(param, value)?; + } + Ok(value as u32) +} + +impl CCtxParams { + /// `ZSTD_CCtxParams_init` + pub(crate) const fn new(compression_level: i32) -> Self { + Self { + format: Format::Zstd1, + c_params: NO_C_PARAMS, + f_params: FrameParameters { + content_size_flag: true, + checksum_flag: false, + no_dict_id_flag: false, + }, + compression_level, + force_window: false, + target_c_block_size: 0, + nb_workers: 0, + job_size: 0, + overlap_log: 0, + ldm_params: NO_LDM_PARAMS, + post_block_splitter: ParamSwitch::Auto, + use_row_match_finder: ParamSwitch::Auto, + } + } + + /// `ZSTD_CCtxParams_init_internal` + pub(crate) fn from_params(params: &Parameters, compression_level: i32) -> Self { + let mut this = Self::new(compression_level); + this.c_params = params.c_params; + this.f_params = params.f_params; + this.use_row_match_finder = + resolve_row_match_finder_mode(this.use_row_match_finder, ¶ms.c_params); + this.post_block_splitter = + resolve_block_splitter_mode(this.post_block_splitter, ¶ms.c_params); + this.ldm_params.enable_ldm = + resolve_enable_ldm(this.ldm_params.enable_ldm, ¶ms.c_params); + this + } + + /// `ZSTD_CCtxParams_setParameter` + pub(crate) fn set_parameter(&mut self, param: i32, value: i32) -> Result { + match param { + c_param::FORMAT => { + bound_check(param, value)?; + self.format = if value == 0 { + Format::Zstd1 + } else { + Format::Zstd1Magicless + }; + Ok(value as usize) + } + c_param::COMPRESSION_LEVEL => { + let value = clamp_bounds(param, value)?; + self.compression_level = if value == 0 { + ZSTD_CLEVEL_DEFAULT + } else { + value + }; + Ok(self.compression_level.max(0) as usize) + } + c_param::WINDOW_LOG => { + self.c_params.window_log = bound_check_unless_0(param, value)?; + Ok(value as usize) + } + c_param::HASH_LOG => { + self.c_params.hash_log = bound_check_unless_0(param, value)?; + Ok(value as usize) + } + c_param::CHAIN_LOG => { + self.c_params.chain_log = bound_check_unless_0(param, value)?; + Ok(value as usize) + } + c_param::SEARCH_LOG => { + self.c_params.search_log = bound_check_unless_0(param, value)?; + Ok(value as usize) + } + c_param::MIN_MATCH => { + self.c_params.min_match = bound_check_unless_0(param, value)?; + Ok(value as usize) + } + c_param::TARGET_LENGTH => { + bound_check(param, value)?; + self.c_params.target_length = value as u32; + Ok(value as usize) + } + c_param::STRATEGY => { + bound_check_unless_0(param, value)?; + self.c_params.strategy = Strategy::from_int(value); + Ok(value as usize) + } + c_param::CONTENT_SIZE_FLAG => { + self.f_params.content_size_flag = value != 0; + Ok(usize::from(value != 0)) + } + c_param::CHECKSUM_FLAG => { + self.f_params.checksum_flag = value != 0; + Ok(usize::from(value != 0)) + } + c_param::DICT_ID_FLAG => { + self.f_params.no_dict_id_flag = value == 0; + Ok(usize::from(value != 0)) + } + c_param::NB_WORKERS => { + self.nb_workers = clamp_bounds(param, value)?; + Ok(self.nb_workers as usize) + } + c_param::JOB_SIZE => { + let value = if value != 0 && value < ZSTDMT_JOBSIZE_MIN { + ZSTDMT_JOBSIZE_MIN + } else { + value + }; + self.job_size = clamp_bounds(param, value)? as usize; + Ok(self.job_size) + } + c_param::OVERLAP_LOG => { + self.overlap_log = clamp_bounds(param, value)?; + Ok(self.overlap_log as usize) + } + c_param::ENABLE_LONG_DISTANCE_MATCHING => { + bound_check(param, value)?; + self.ldm_params.enable_ldm = match value { + 0 => ParamSwitch::Auto, + 1 => ParamSwitch::Enable, + _ => ParamSwitch::Disable, + }; + Ok(value as usize) + } + c_param::LDM_HASH_LOG => { + self.ldm_params.hash_log = bound_check_unless_0(param, value)?; + Ok(value as usize) + } + c_param::LDM_MIN_MATCH => { + self.ldm_params.min_match_length = bound_check_unless_0(param, value)?; + Ok(value as usize) + } + c_param::LDM_BUCKET_SIZE_LOG => { + self.ldm_params.bucket_size_log = bound_check_unless_0(param, value)?; + Ok(value as usize) + } + c_param::LDM_HASH_RATE_LOG => { + self.ldm_params.hash_rate_log = bound_check_unless_0(param, value)?; + Ok(value as usize) + } + c_param::TARGET_C_BLOCK_SIZE => { + let value = if value != 0 { + value.max(ZSTD_TARGETCBLOCKSIZE_MIN) + } else { + value + }; + self.target_c_block_size = bound_check_unless_0(param, value)? as usize; + Ok(self.target_c_block_size) + } + _ => Err(Error::ParameterUnsupported), + } + } +} + +/// `ZSTD_rowMatchFinderUsed` +pub(crate) fn row_match_finder_used(strategy: Strategy, mode: ParamSwitch) -> bool { + row_match_finder_supported(strategy) && mode == ParamSwitch::Enable +} + +/// `ZSTD_resolveRowMatchFinderMode` +pub(crate) fn resolve_row_match_finder_mode( + mode: ParamSwitch, + c_params: &CompressionParameters, +) -> ParamSwitch { + if mode != ParamSwitch::Auto { + return mode; + } + if row_match_finder_supported(c_params.strat()) && c_params.window_log > 14 { + ParamSwitch::Enable + } else { + ParamSwitch::Disable + } +} + +/// `ZSTD_resolveBlockSplitterMode` +pub(crate) fn resolve_block_splitter_mode( + mode: ParamSwitch, + c_params: &CompressionParameters, +) -> ParamSwitch { + if mode != ParamSwitch::Auto { + return mode; + } + if c_params.strat() >= Strategy::BtOpt && c_params.window_log >= 17 { + ParamSwitch::Enable + } else { + ParamSwitch::Disable + } +} + +/// `ZSTD_allocateChainTable` +pub(crate) fn allocate_chain_table(strategy: Strategy, use_row_match_finder: ParamSwitch) -> bool { + strategy != Strategy::Fast && !row_match_finder_used(strategy, use_row_match_finder) +} + +/// `ZSTD_resolveEnableLdm` +pub(crate) fn resolve_enable_ldm( + mode: ParamSwitch, + c_params: &CompressionParameters, +) -> ParamSwitch { + if mode != ParamSwitch::Auto { + return mode; + } + if c_params.strat() >= Strategy::BtOpt && c_params.window_log >= 27 { + ParamSwitch::Enable + } else { + ParamSwitch::Disable + } +} + +/// `ZSTD_CDictIndicesAreTagged` +pub(crate) fn cdict_indices_are_tagged(c_params: &CompressionParameters) -> bool { + matches!(c_params.strat(), Strategy::Fast | Strategy::DFast) +} + +/// `ZSTD_cycleLog` +pub(crate) fn cycle_log(hash_log: u32, strat: Strategy) -> u32 { + hash_log - u32::from(strat >= Strategy::BtLazy2) +} + +/// `ZSTD_dictAndWindowLog`: the log of a window that also covers the dictionary. +fn dict_and_window_log(window_log: u32, src_size: u64, dict_size: u64) -> u32 { + let max_window_size = 1u64 << ZSTD_WINDOWLOG_MAX; + if dict_size == 0 { + return window_log; + } + let window_size = 1u64 << window_log; + let dict_and_window_size = dict_size + window_size; + if window_size >= dict_size.wrapping_add(src_size) { + window_log + } else if dict_and_window_size >= max_window_size { + ZSTD_WINDOWLOG_MAX + } else { + highbit32(dict_and_window_size as u32 - 1) + 1 + } +} + +/// `ZSTD_adjustCParams_internal` +pub(crate) fn adjust_c_params_internal( + mut c_par: CompressionParameters, + mut src_size: u64, + mut dict_size: usize, + mode: CParamMode, + mut use_row_match_finder: ParamSwitch, +) -> CompressionParameters { + const MIN_SRC_SIZE: u64 = 513; + const MAX_WINDOW_RESIZE: u64 = 1 << (ZSTD_WINDOWLOG_MAX - 1); + + match mode { + CParamMode::NoAttachDict => {} + CParamMode::CreateCDict => { + if dict_size != 0 && src_size == ZSTD_CONTENTSIZE_UNKNOWN { + src_size = MIN_SRC_SIZE; + } + } + CParamMode::AttachDict => dict_size = 0, + } + + if src_size <= MAX_WINDOW_RESIZE && dict_size as u64 <= MAX_WINDOW_RESIZE { + let t_size = (src_size + dict_size as u64) as u32; + const HASH_SIZE_MIN: u32 = 1 << ZSTD_HASHLOG_MIN; + let src_log = if t_size < HASH_SIZE_MIN { + ZSTD_HASHLOG_MIN as u32 + } else { + highbit32(t_size - 1) + 1 + }; + c_par.window_log = c_par.window_log.min(src_log); + } + if src_size != ZSTD_CONTENTSIZE_UNKNOWN { + let dict_and_window_log = dict_and_window_log(c_par.window_log, src_size, dict_size as u64); + let cycle_log = cycle_log(c_par.chain_log, c_par.strat()); + c_par.hash_log = c_par.hash_log.min(dict_and_window_log + 1); + if cycle_log > dict_and_window_log { + c_par.chain_log -= cycle_log - dict_and_window_log; + } + } + + c_par.window_log = c_par.window_log.max(ZSTD_WINDOWLOG_ABSOLUTEMIN); + + if mode == CParamMode::CreateCDict && cdict_indices_are_tagged(&c_par) { + let max_short_cache_hash_log = 32 - ZSTD_SHORT_CACHE_TAG_BITS; + c_par.hash_log = c_par.hash_log.min(max_short_cache_hash_log); + c_par.chain_log = c_par.chain_log.min(max_short_cache_hash_log); + } + + if use_row_match_finder == ParamSwitch::Auto { + use_row_match_finder = ParamSwitch::Enable; + } + if row_match_finder_used(c_par.strat(), use_row_match_finder) { + // The rows' hashes are 32 bits, some of which are the tag. + let row_log = c_par.search_log.clamp(4, 6); + let max_row_hash_log = 32 - ZSTD_ROW_HASH_TAG_BITS; + c_par.hash_log = c_par.hash_log.min(max_row_hash_log + row_log); + } + + c_par +} + +/// `ZSTD_overrideCParams` +fn override_c_params(c_params: &mut CompressionParameters, overrides: &CompressionParameters) { + if overrides.window_log != 0 { + c_params.window_log = overrides.window_log; + } + if overrides.hash_log != 0 { + c_params.hash_log = overrides.hash_log; + } + if overrides.chain_log != 0 { + c_params.chain_log = overrides.chain_log; + } + if overrides.search_log != 0 { + c_params.search_log = overrides.search_log; + } + if overrides.min_match != 0 { + c_params.min_match = overrides.min_match; + } + if overrides.target_length != 0 { + c_params.target_length = overrides.target_length; + } + if overrides.strategy.is_some() { + c_params.strategy = overrides.strategy; + } +} + +/// `ZSTD_getCParamsFromCCtxParams` +pub(crate) fn get_c_params_from_cctx_params( + cctx_params: &CCtxParams, + src_size_hint: u64, + dict_size: usize, + mode: CParamMode, +) -> CompressionParameters { + let mut c_params = get_c_params_internal( + cctx_params.compression_level, + src_size_hint, + dict_size, + mode, + ); + if cctx_params.ldm_params.enable_ldm == ParamSwitch::Enable { + c_params.window_log = ZSTD_LDM_DEFAULT_WINDOW_LOG; + } + override_c_params(&mut c_params, &cctx_params.c_params); + adjust_c_params_internal( + c_params, + src_size_hint, + dict_size, + mode, + cctx_params.use_row_match_finder, + ) +} + +/// `ZSTD_getCParamRowSize`: the size that selects a table of [`DEFAULT_C_PARAMETERS`]. +fn get_c_param_row_size(src_size_hint: u64, mut dict_size: usize, mode: CParamMode) -> u64 { + if mode == CParamMode::AttachDict { + dict_size = 0; + } + let unknown = src_size_hint == ZSTD_CONTENTSIZE_UNKNOWN; + let added_size: u64 = if unknown && dict_size > 0 { 500 } else { 0 }; + if unknown && dict_size == 0 { + ZSTD_CONTENTSIZE_UNKNOWN + } else { + src_size_hint + .wrapping_add(dict_size as u64) + .wrapping_add(added_size) + } +} + +/// `ZSTD_getCParams_internal` +pub(crate) fn get_c_params_internal( + compression_level: i32, + src_size_hint: u64, + dict_size: usize, + mode: CParamMode, +) -> CompressionParameters { + let r_size = get_c_param_row_size(src_size_hint, dict_size, mode); + let table_id = usize::from(r_size <= 256 << 10) + + usize::from(r_size <= 128 << 10) + + usize::from(r_size <= 16 << 10); + let row = match compression_level { + 0 => ZSTD_CLEVEL_DEFAULT, + ..0 => 0, + _ => compression_level.min(ZSTD_MAX_CLEVEL), + }; + let mut cp = DEFAULT_C_PARAMETERS + .get(table_id) + .and_then(|table| table.get(row as usize)) + .copied() + .unwrap_or(NO_C_PARAMS); + if compression_level < 0 { + // Negative levels go faster by skipping more. + cp.target_length = compression_level.max(ZSTD_MIN_CLEVEL).unsigned_abs(); + } + adjust_c_params_internal(cp, src_size_hint, dict_size, mode, ParamSwitch::Auto) +} + +/// `ZSTD_getParams_internal` +pub(crate) fn get_params_internal( + compression_level: i32, + src_size_hint: u64, + dict_size: usize, + mode: CParamMode, +) -> Parameters { + Parameters { + c_params: get_c_params_internal(compression_level, src_size_hint, dict_size, mode), + f_params: FrameParameters { + content_size_flag: true, + checksum_flag: false, + no_dict_id_flag: false, + }, + } +} + +/// `ZSTD_compressBound`: the most that `src_size` bytes can take once compressed. No slice is of a size that is too +/// large for it, which gives more than can be allocated, as the error code of the C function is. +pub const fn compress_bound(src_size: usize) -> usize { + const ZSTD_MAX_INPUT_SIZE: usize = 0xFF00_FF00_FF00_FF00; + if src_size >= ZSTD_MAX_INPUT_SIZE { + return usize::MAX; + } + // A margin for small inputs, so that the bound of two inputs is at most that of both together. + let margin = if src_size < (128 << 10) { + ((128 << 10) - src_size) >> 11 + } else { + 0 + }; + src_size + (src_size >> 8) + margin +} + +// ── the size of the workspace ──────────────────────────────────────────────── +// +// The C code carves all the tables and buffers of a context out of one allocation. A context only goes on counting +// indexes from where the previous frame left them as long as that allocation is kept, so this keeps its books. + +const SIZEOF_COMPRESSED_BLOCK_STATE: usize = 5632; +const TMP_WORKSPACE_SIZE: usize = 8920; +const SIZEOF_SEQ_DEF: usize = 8; +const SIZEOF_MATCH: usize = 8; +const SIZEOF_OPTIMAL: usize = 28; +const SIZEOF_RAW_SEQ: usize = 12; +const SIZEOF_LDM_ENTRY: usize = 8; +const ZSTD_CWKSP_ALIGNMENT_BYTES: usize = 64; + +/// `ZSTD_cwksp_aligned64_alloc_size` +const fn aligned64(size: usize) -> usize { + size.next_multiple_of(ZSTD_CWKSP_ALIGNMENT_BYTES) +} + +/// `ZSTD_maxNbSeq` +pub(crate) const fn max_nb_seq(block_size: usize, min_match: u32) -> usize { + block_size / if min_match == 3 { 3 } else { 4 } +} + +/// The sizes of the tables of a context's match state. +pub(crate) struct MatchStateSizes { + pub(crate) chain_size: usize, + pub(crate) h_size: usize, + pub(crate) hash_log3: u32, + pub(crate) h3_size: usize, +} + +impl MatchStateSizes { + pub(crate) fn new( + c_params: &CompressionParameters, + use_row_match_finder: ParamSwitch, + for_cctx: bool, + ) -> Self { + let hash_log3 = if for_cctx && c_params.min_match == 3 { + ZSTD_HASHLOG3_MAX.min(c_params.window_log) + } else { + 0 + }; + Self { + chain_size: if allocate_chain_table(c_params.strat(), use_row_match_finder) { + 1 << c_params.chain_log + } else { + 0 + }, + h_size: 1 << c_params.hash_log, + hash_log3, + h3_size: if hash_log3 != 0 { 1 << hash_log3 } else { 0 }, + } + } +} + +/// The books of a `ZSTD_cwksp`, as offsets from its start, which is taken to be aligned. +pub(crate) struct Workspace { + size: usize, + object_end: usize, + table_end: usize, + alloc_start: usize, + oversized_duration: usize, +} + +/// What a context needs room for. +pub(crate) struct WorkspaceNeeds<'a> { + pub(crate) c_params: &'a CompressionParameters, + pub(crate) ldm_params: &'a LdmParams, + pub(crate) use_row_match_finder: ParamSwitch, + pub(crate) buff_in_size: usize, + pub(crate) buff_out_size: usize, + pub(crate) block_size: usize, +} + +impl WorkspaceNeeds<'_> { + /// The tables, which the C code allocates from the bottom. + fn tables(&self) -> usize { + let sizes = MatchStateSizes::new(self.c_params, self.use_row_match_finder, true); + (sizes.chain_size + sizes.h_size + sizes.h3_size) * size_of::() + } + + /// Everything else, which the C code allocates from the top. + fn allocs(&self) -> usize { + let max_nb_seq = max_nb_seq(self.block_size, self.c_params.min_match); + let mut total = 0; + if row_match_finder_used(self.c_params.strat(), self.use_row_match_finder) { + total += aligned64(1 << self.c_params.hash_log); + } + if self.c_params.strat() >= Strategy::BtOpt { + total += aligned64((MAX_ML as usize + 1) * size_of::()) + + aligned64((MAX_LL as usize + 1) * size_of::()) + + aligned64((MAX_OFF as usize + 1) * size_of::()) + + aligned64((1 << LITBITS) * size_of::()) + + aligned64(ZSTD_OPT_SIZE * SIZEOF_MATCH) + + aligned64(ZSTD_OPT_SIZE * SIZEOF_OPTIMAL); + } + total += aligned64(max_nb_seq * SIZEOF_SEQ_DEF); + if self.ldm_params.enable_ldm == ParamSwitch::Enable { + total += aligned64((1usize << self.ldm_params.hash_log) * SIZEOF_LDM_ENTRY) + + aligned64( + ldm::get_max_nb_seq(*self.ldm_params, self.block_size) * SIZEOF_RAW_SEQ, + ) + + (1usize << (self.ldm_params.hash_log - self.ldm_params.bucket_size_log)); + } + total + + (self.block_size + WILDCOPY_OVERLENGTH) + + self.buff_in_size + + self.buff_out_size + + 3 * max_nb_seq + } + + /// `ZSTD_estimateCCtxSize_usingCCtxParams_internal` + fn needed_space(&self) -> usize { + let objects = TMP_WORKSPACE_SIZE + 2 * SIZEOF_COMPRESSED_BLOCK_STATE; + let slack_space = ZSTD_CWKSP_ALIGNMENT_BYTES * 2; + objects + self.tables() + self.allocs() + slack_space + } +} + +impl Workspace { + pub(crate) const fn new() -> Self { + Self { + size: 0, + object_end: 0, + table_end: 0, + alloc_start: 0, + oversized_duration: 0, + } + } + + /// What `ZSTD_resetCCtx_internal` does to the workspace. Returns whether it made a new one. + pub(crate) fn reset(&mut self, needs: &WorkspaceNeeds<'_>) -> bool { + let needed_space = needs.needed_space(); + self.oversized_duration += 1; + let too_small = self.size < needed_space; + let wasteful = self.alloc_start - self.table_end + >= needed_space.saturating_mul(ZSTD_WORKSPACETOOLARGE_FACTOR) + && self.oversized_duration > ZSTD_WORKSPACETOOLARGE_MAXDURATION; + let resize = too_small || wasteful; + if resize { + self.size = needed_space; + self.object_end = aligned64(TMP_WORKSPACE_SIZE + 2 * SIZEOF_COMPRESSED_BLOCK_STATE); + self.oversized_duration = 0; + } + self.table_end = self.object_end + needs.tables(); + self.alloc_start = + (self.size - self.size % ZSTD_CWKSP_ALIGNMENT_BYTES).saturating_sub(needs.allocs()); + self.alloc_start = self.alloc_start.max(self.table_end); + resize + } +} + +const fn cp( + window_log: u32, + chain_log: u32, + hash_log: u32, + search_log: u32, + min_match: u32, + target_length: u32, + strategy: Strategy, +) -> CompressionParameters { + CompressionParameters { + window_log, + chain_log, + hash_log, + search_log, + min_match, + target_length, + strategy: Some(strategy), + } +} + +/// `ZSTD_defaultCParameters`: for any size of input, then for at most 256 KB, 128 KB and 16 KB. +#[rustfmt::skip] +static DEFAULT_C_PARAMETERS: [[CompressionParameters; ZSTD_MAX_CLEVEL as usize + 1]; 4] = [ + [ + cp(19, 12, 13, 1, 6, 1, Strategy::Fast), + cp(19, 13, 14, 1, 7, 0, Strategy::Fast), + cp(20, 15, 16, 1, 6, 0, Strategy::Fast), + cp(21, 16, 17, 1, 5, 0, Strategy::DFast), + cp(21, 18, 18, 1, 5, 0, Strategy::DFast), + cp(21, 18, 19, 3, 5, 2, Strategy::Greedy), + cp(21, 18, 19, 3, 5, 4, Strategy::Lazy), + cp(21, 19, 20, 4, 5, 8, Strategy::Lazy), + cp(21, 19, 20, 4, 5, 16, Strategy::Lazy2), + cp(22, 20, 21, 4, 5, 16, Strategy::Lazy2), + cp(22, 21, 22, 5, 5, 16, Strategy::Lazy2), + cp(22, 21, 22, 6, 5, 16, Strategy::Lazy2), + cp(22, 22, 23, 6, 5, 32, Strategy::Lazy2), + cp(22, 22, 22, 4, 5, 32, Strategy::BtLazy2), + cp(22, 22, 23, 5, 5, 32, Strategy::BtLazy2), + cp(22, 23, 23, 6, 5, 32, Strategy::BtLazy2), + cp(22, 22, 22, 5, 5, 48, Strategy::BtOpt), + cp(23, 23, 22, 5, 4, 64, Strategy::BtOpt), + cp(23, 23, 22, 6, 3, 64, Strategy::BtUltra), + cp(23, 24, 22, 7, 3, 256, Strategy::BtUltra2), + cp(25, 25, 23, 7, 3, 256, Strategy::BtUltra2), + cp(26, 26, 24, 7, 3, 512, Strategy::BtUltra2), + cp(27, 27, 25, 9, 3, 999, Strategy::BtUltra2), + ], + [ + cp(18, 12, 13, 1, 5, 1, Strategy::Fast), + cp(18, 13, 14, 1, 6, 0, Strategy::Fast), + cp(18, 14, 14, 1, 5, 0, Strategy::DFast), + cp(18, 16, 16, 1, 4, 0, Strategy::DFast), + cp(18, 16, 17, 3, 5, 2, Strategy::Greedy), + cp(18, 17, 18, 5, 5, 2, Strategy::Greedy), + cp(18, 18, 19, 3, 5, 4, Strategy::Lazy), + cp(18, 18, 19, 4, 4, 4, Strategy::Lazy), + cp(18, 18, 19, 4, 4, 8, Strategy::Lazy2), + cp(18, 18, 19, 5, 4, 8, Strategy::Lazy2), + cp(18, 18, 19, 6, 4, 8, Strategy::Lazy2), + cp(18, 18, 19, 5, 4, 12, Strategy::BtLazy2), + cp(18, 19, 19, 7, 4, 12, Strategy::BtLazy2), + cp(18, 18, 19, 4, 4, 16, Strategy::BtOpt), + cp(18, 18, 19, 4, 3, 32, Strategy::BtOpt), + cp(18, 18, 19, 6, 3, 128, Strategy::BtOpt), + cp(18, 19, 19, 6, 3, 128, Strategy::BtUltra), + cp(18, 19, 19, 8, 3, 256, Strategy::BtUltra), + cp(18, 19, 19, 6, 3, 128, Strategy::BtUltra2), + cp(18, 19, 19, 8, 3, 256, Strategy::BtUltra2), + cp(18, 19, 19, 10, 3, 512, Strategy::BtUltra2), + cp(18, 19, 19, 12, 3, 512, Strategy::BtUltra2), + cp(18, 19, 19, 13, 3, 999, Strategy::BtUltra2), + ], + [ + cp(17, 12, 12, 1, 5, 1, Strategy::Fast), + cp(17, 12, 13, 1, 6, 0, Strategy::Fast), + cp(17, 13, 15, 1, 5, 0, Strategy::Fast), + cp(17, 15, 16, 2, 5, 0, Strategy::DFast), + cp(17, 17, 17, 2, 4, 0, Strategy::DFast), + cp(17, 16, 17, 3, 4, 2, Strategy::Greedy), + cp(17, 16, 17, 3, 4, 4, Strategy::Lazy), + cp(17, 16, 17, 3, 4, 8, Strategy::Lazy2), + cp(17, 16, 17, 4, 4, 8, Strategy::Lazy2), + cp(17, 16, 17, 5, 4, 8, Strategy::Lazy2), + cp(17, 16, 17, 6, 4, 8, Strategy::Lazy2), + cp(17, 17, 17, 5, 4, 8, Strategy::BtLazy2), + cp(17, 18, 17, 7, 4, 12, Strategy::BtLazy2), + cp(17, 18, 17, 3, 4, 12, Strategy::BtOpt), + cp(17, 18, 17, 4, 3, 32, Strategy::BtOpt), + cp(17, 18, 17, 6, 3, 256, Strategy::BtOpt), + cp(17, 18, 17, 6, 3, 128, Strategy::BtUltra), + cp(17, 18, 17, 8, 3, 256, Strategy::BtUltra), + cp(17, 18, 17, 10, 3, 512, Strategy::BtUltra), + cp(17, 18, 17, 5, 3, 256, Strategy::BtUltra2), + cp(17, 18, 17, 7, 3, 512, Strategy::BtUltra2), + cp(17, 18, 17, 9, 3, 512, Strategy::BtUltra2), + cp(17, 18, 17, 11, 3, 999, Strategy::BtUltra2), + ], + [ + cp(14, 12, 13, 1, 5, 1, Strategy::Fast), + cp(14, 14, 15, 1, 5, 0, Strategy::Fast), + cp(14, 14, 15, 1, 4, 0, Strategy::Fast), + cp(14, 14, 15, 2, 4, 0, Strategy::DFast), + cp(14, 14, 14, 4, 4, 2, Strategy::Greedy), + cp(14, 14, 14, 3, 4, 4, Strategy::Lazy), + cp(14, 14, 14, 4, 4, 8, Strategy::Lazy2), + cp(14, 14, 14, 6, 4, 8, Strategy::Lazy2), + cp(14, 14, 14, 8, 4, 8, Strategy::Lazy2), + cp(14, 15, 14, 5, 4, 8, Strategy::BtLazy2), + cp(14, 15, 14, 9, 4, 8, Strategy::BtLazy2), + cp(14, 15, 14, 3, 4, 12, Strategy::BtOpt), + cp(14, 15, 14, 4, 3, 24, Strategy::BtOpt), + cp(14, 15, 14, 5, 3, 32, Strategy::BtUltra), + cp(14, 15, 15, 6, 3, 64, Strategy::BtUltra), + cp(14, 15, 15, 7, 3, 256, Strategy::BtUltra), + cp(14, 15, 15, 5, 3, 48, Strategy::BtUltra2), + cp(14, 15, 15, 6, 3, 128, Strategy::BtUltra2), + cp(14, 15, 15, 7, 3, 256, Strategy::BtUltra2), + cp(14, 15, 15, 8, 3, 256, Strategy::BtUltra2), + cp(14, 15, 15, 8, 3, 512, Strategy::BtUltra2), + cp(14, 15, 15, 9, 3, 512, Strategy::BtUltra2), + cp(14, 15, 15, 10, 3, 999, Strategy::BtUltra2), + ], +]; diff --git a/src/zstd/compress/presplit.rs b/src/zstd/compress/presplit.rs new file mode 100644 index 000000000000..00ab8c5a217f --- /dev/null +++ b/src/zstd/compress/presplit.rs @@ -0,0 +1,181 @@ +//! `zstd_preSplit.c`: where to cut a block in two, before it is compressed. + +use super::hist; +use crate::error::Result; +use crate::util::SliceExt; + +const THRESHOLD_PENALTY_RATE: u64 = 16; +const THRESHOLD_BASE: i32 = THRESHOLD_PENALTY_RATE as i32 - 2; +const THRESHOLD_PENALTY: i32 = 3; + +const HASHLENGTH: usize = 2; +const KNUTH: u32 = 0x9e37_79b9; + +const CHUNKSIZE: usize = 8 << 10; +const SEGMENT_SIZE: usize = 512; + +/// `Fingerprint`, with a table of `SIZE` events, which is `1 << hashLog`: the C code leaves the rest of its table at 0. +struct Fingerprint { + events: [u32; SIZE], + nb_events: usize, +} + +impl Fingerprint { + const fn new() -> Self { + Self { + events: [0; SIZE], + nb_events: 0, + } + } + + /// `hash2`: a table of 256 takes the first byte as it is, a larger one a hash of the 2 bytes. + #[inline(always)] + const fn hash2(pair: [u8; HASHLENGTH]) -> usize { + if SIZE == 256 { + let [first, _] = pair; + return first as usize; + } + ((u16::from_le_bytes(pair) as u32).wrapping_mul(KNUTH) >> (32 - SIZE.trailing_zeros())) + as usize + } + + /// `addEvents_generic`: `src` has at least 2 bytes. + #[inline(always)] + fn add_events(&mut self, src: &[u8]) { + let limit = src.len().saturating_sub(HASHLENGTH - 1); + let mut n = 0; + while n < limit { + if let Some(pair) = src.get(n..).and_then(<[u8]>::first_chunk) + && let Some(event) = self.events.get_mut(Self::hash2(*pair) & (SIZE - 1)) + { + *event += 1; + } + n += SAMPLING_RATE; + } + self.nb_events += limit / SAMPLING_RATE; + } + + /// `recordFingerprint_generic` + #[inline(always)] + fn record(&mut self, src: &[u8]) { + self.events.fill(0); + self.nb_events = 0; + self.add_events::(src); + } + + /// `fpDistance` + fn distance(&self, other: &Self) -> u64 { + let nb_events1 = self.nb_events as i64; + let nb_events2 = other.nb_events as i64; + self.events + .iter() + .zip(&other.events) + .map(|(&event1, &event2)| { + (i64::from(event1) * nb_events2 - i64::from(event2) * nb_events1).unsigned_abs() + }) + .sum() + } + + /// `compareFingerprints`: whether `newfp` is too different from this one. + fn differs_from(&self, newfp: &Self, penalty: i32) -> bool { + let p50 = self.nb_events as u64 * newfp.nb_events as u64; + let deviation = self.distance(newfp); + let threshold = p50 * (THRESHOLD_BASE + penalty) as u64 / THRESHOLD_PENALTY_RATE; + deviation >= threshold + } + + /// `mergeEvents` + fn merge(&mut self, newfp: &Self) { + for (acc, &event) in self.events.iter_mut().zip(&newfp.events) { + *acc += event; + } + self.nb_events += newfp.nb_events; + } +} + +/// `ZSTD_splitBlock_byChunks`: cuts before the first chunk that is too different from all there is before it. +#[inline(always)] +fn split_block_by_chunks(block: &[u8]) -> usize { + let mut past_events = Fingerprint::::new(); + let mut new_events = Fingerprint::::new(); + let mut penalty = THRESHOLD_PENALTY; + let (chunks, _) = block.as_chunks::(); + let Some((first, chunks)) = chunks.split_first() else { + return block.len(); + }; + + past_events.record::(first); + let mut pos = CHUNKSIZE; + for chunk in chunks { + new_events.record::(chunk); + if past_events.differs_from(&new_events, penalty) { + return pos; + } + past_events.merge(&new_events); + if penalty > 0 { + penalty -= 1; + } + pos += CHUNKSIZE; + } + block.len() +} + +fn split_block_by_chunks_43(block: &[u8]) -> usize { + split_block_by_chunks::<43, { 1 << 8 }>(block) +} + +fn split_block_by_chunks_11(block: &[u8]) -> usize { + split_block_by_chunks::<11, { 1 << 9 }>(block) +} + +fn split_block_by_chunks_5(block: &[u8]) -> usize { + split_block_by_chunks::<5, { 1 << 10 }>(block) +} + +fn split_block_by_chunks_1(block: &[u8]) -> usize { + split_block_by_chunks::<1, { 1 << 10 }>(block) +} + +/// `ZSTD_splitBlock_fromBorders`: cuts if the start and the end of the block are too different. In the middle, or at a +/// quarter from the border that the middle is clearly the most different from. +fn split_block_from_borders(block: &[u8]) -> Result { + fn segment(block: &[u8], start: usize) -> Result> { + let mut fingerprint = Fingerprint::new(); + hist::add(&mut fingerprint.events, block.span(start, SEGMENT_SIZE)?)?; + fingerprint.nb_events = SEGMENT_SIZE; + Ok(fingerprint) + } + + let block_size = block.len(); + let past_events = segment(block, 0)?; + let new_events = segment(block, block_size.wrapping_sub(SEGMENT_SIZE))?; + if !past_events.differs_from(&new_events, 0) { + return Ok(block_size); + } + + let middle_events = segment(block, (block_size / 2).wrapping_sub(SEGMENT_SIZE / 2))?; + let dist_from_begin = past_events.distance(&middle_events); + let dist_from_end = new_events.distance(&middle_events); + let min_distance = (SEGMENT_SIZE * SEGMENT_SIZE / 3) as u64; + if dist_from_begin.abs_diff(dist_from_end) < min_distance { + return Ok(64 << 10); + } + Ok(if dist_from_begin > dist_from_end { + 32 << 10 + } else { + 96 << 10 + }) +} + +/// `ZSTD_splitBlock`: `block` is 128 KB, and `level` is from 0 to 4: the higher, the more work is put in. Returns the +/// size of the first part, which is the whole if the block is better left in one piece. +pub(crate) fn split_block(block: &[u8], level: i32) -> Result { + debug_assert!(block.len() == 128 << 10); + Ok(match level { + 0 => split_block_from_borders(block)?, + 1 => split_block_by_chunks_43(block), + 2 => split_block_by_chunks_11(block), + 3 => split_block_by_chunks_5(block), + _ => split_block_by_chunks_1(block), + }) +} diff --git a/src/zstd/compress/sequences.rs b/src/zstd/compress/sequences.rs new file mode 100644 index 000000000000..4b1f9131194c --- /dev/null +++ b/src/zstd/compress/sequences.rs @@ -0,0 +1,826 @@ +//! `zstd_compress_sequences.c`: how the sequences of a block are encoded. + +use core::num::NonZero; + +use super::fse::{ + self, FSE_NCOUNTBOUND, FseCTable, FseRepeat, FseSymbolTransform, PredefinedCTable, +}; +use super::internal::{FseCTables, LongLengthType, SeqDef, Strategy}; +use crate::common::{ + DEFAULT_MAX_OFF, LL_BITS, LL_DEFAULT_NORM, LL_DEFAULT_NORM_LOG, LL_FSE_LOG, MAX_LL, MAX_ML, + MAX_SEQ, ML_BITS, ML_DEFAULT_NORM, ML_DEFAULT_NORM_LOG, ML_FSE_LOG, OF_DEFAULT_NORM, + OF_DEFAULT_NORM_LOG, OFF_FSE_LOG, SymbolEncodingType, +}; +use crate::error::{Error, Result}; +use crate::util::{SliceExt, highbit32}; + +/// `-log2(x / 256)` in units of 1/256 of a bit, for `x` from 1 to 255. The entry for 0 is not used. +#[rustfmt::skip] +static K_INVERSE_PROBABILITY_LOG256: [u32; 256] = [ + 0, 2048, 1792, 1642, 1536, 1453, 1386, 1329, 1280, 1236, 1197, 1162, 1130, 1100, 1073, 1047, + 1024, 1001, 980, 960, 941, 923, 906, 889, 874, 859, 844, 830, 817, 804, 791, 779, + 768, 756, 745, 734, 724, 714, 704, 694, 685, 676, 667, 658, 650, 642, 633, 626, + 618, 610, 603, 595, 588, 581, 574, 567, 561, 554, 548, 542, 535, 529, 523, 517, + 512, 506, 500, 495, 489, 484, 478, 473, 468, 463, 458, 453, 448, 443, 438, 434, + 429, 424, 420, 415, 411, 407, 402, 398, 394, 390, 386, 382, 377, 373, 370, 366, + 362, 358, 354, 350, 347, 343, 339, 336, 332, 329, 325, 322, 318, 315, 311, 308, + 305, 302, 298, 295, 292, 289, 286, 282, 279, 276, 273, 270, 267, 264, 261, 258, + 256, 253, 250, 247, 244, 241, 239, 236, 233, 230, 228, 225, 222, 220, 217, 215, + 212, 209, 207, 204, 202, 199, 197, 194, 192, 190, 187, 185, 182, 180, 178, 175, + 173, 171, 168, 166, 164, 162, 159, 157, 155, 153, 151, 149, 146, 144, 142, 140, + 138, 136, 134, 132, 130, 128, 126, 123, 121, 119, 117, 115, 114, 112, 110, 108, + 106, 104, 102, 100, 98, 96, 94, 93, 91, 89, 87, 85, 83, 82, 80, 78, + 76, 74, 73, 71, 69, 67, 66, 64, 62, 61, 59, 57, 55, 54, 52, 50, + 49, 47, 46, 44, 42, 41, 39, 37, 36, 34, 33, 31, 30, 28, 26, 25, + 23, 22, 20, 19, 17, 16, 14, 13, 11, 10, 8, 7, 5, 4, 2, 1, +]; + +/// What the C code has for a cost that could not be computed: an error code, which is more than any cost. +fn cost_or_error(cost: Result) -> usize { + cost.unwrap_or_else(|code| 0usize.wrapping_sub(code as usize)) +} + +/// `ZSTD_useLowProbCount`: with few sequences, counts of -1 hurt more than they help. +const fn use_low_prob_count(nb_seq: usize) -> bool { + nb_seq >= 2048 +} + +/// `ZSTD_NCountCost`: how many bytes the header of a table for `count` takes. +fn ncount_cost(count: &[u32], max: u32, nb_seq: usize, fse_log: u32) -> Result { + let mut wksp = [0u8; FSE_NCOUNTBOUND]; + let mut norm = [0i16; MAX_SEQ as usize + 1]; + let table_log = fse::optimal_table_log(fse_log, nb_seq, max); + fse::normalize_count( + &mut norm, + table_log, + count, + nb_seq, + max, + use_low_prob_count(nb_seq), + )?; + fse::write_ncount(&mut wksp, &norm, max, table_log) +} + +/// `ZSTD_entropyCost`: how many bits it takes to encode the distribution `count`, which adds up to `total`, with a +/// table made for it. +fn entropy_cost(count: &[u32], max: u32, total: usize) -> Result { + let mut cost: u32 = 0; + for &c in count.sub(..=max as usize)? { + let mut norm = 256u32.wrapping_mul(c) as usize / total.max(1); + if c != 0 && norm == 0 { + norm = 1; + } + cost = cost.wrapping_add(c.wrapping_mul(K_INVERSE_PROBABILITY_LOG256.at(norm)?)); + } + Ok((cost >> 8) as usize) +} + +/// `ZSTD_fseBitCost`: how many bits it takes to encode the distribution `count` with `ctable`, which is an error if +/// the table lacks one of the symbols. +pub(crate) fn fse_bit_cost(ctable: &FseCTable, count: &[u32], max: u32) -> Result { + const K_ACCURACY_LOG: u32 = 8; + let mut cost: usize = 0; + if ctable.max_symbol_value < max { + return Err(Error::Generic); + } + let bad_cost = (ctable.table_log + 1) << K_ACCURACY_LOG; + for (s, &c) in count.sub(..=max as usize)?.iter().enumerate() { + if c == 0 { + continue; + } + let bit_cost = ctable.bit_cost(s as u32, K_ACCURACY_LOG); + if bit_cost >= bad_cost { + return Err(Error::Generic); + } + cost += c as usize * bit_cost as usize; + } + Ok(cost >> K_ACCURACY_LOG) +} + +/// `ZSTD_crossEntropyCost`: how many bits it takes to encode the distribution `count` with a table made from `norm`. +pub(crate) fn cross_entropy_cost( + norm: &[i16], + accuracy_log: u32, + count: &[u32], + max: u32, +) -> Result { + let shift = 8 - accuracy_log; + let mut cost: usize = 0; + let len = max as usize + 1; + for (&n, &c) in norm.sub(..len)?.iter().zip(count.sub(..len)?) { + let norm_acc = if n != -1 { n as u32 } else { 1 }; + let norm256 = norm_acc << shift; + cost += c.wrapping_mul(K_INVERSE_PROBABILITY_LOG256.at(norm256 as usize)?) as usize; + } + Ok(cost >> 8) +} + +/// `ZSTD_selectEncodingType` +pub(crate) fn select_encoding_type( + repeat_mode: &mut FseRepeat, + count: &[u32], + max: u32, + most_frequent: usize, + nb_seq: usize, + fse_log: u32, + prev_ctable: &FseCTable, + default_norm: &[i16], + default_norm_log: u32, + is_default_allowed: bool, + strategy: Strategy, +) -> Result { + if most_frequent == nb_seq { + *repeat_mode = FseRepeat::None; + if is_default_allowed && nb_seq <= 2 { + // A run of 1 or 2 costs more than the predefined table does. + return Ok(SymbolEncodingType::Basic); + } + return Ok(SymbolEncodingType::Rle); + } + if strategy < Strategy::Lazy { + if is_default_allowed { + const STATIC_FSE_NB_SEQ_MAX: usize = 1000; + let mult = 10 - strategy as usize; + const BASE_LOG: usize = 3; + let dynamic_fse_nb_seq_min = ((1usize << default_norm_log) * mult) >> BASE_LOG; + if *repeat_mode == FseRepeat::Valid && nb_seq < STATIC_FSE_NB_SEQ_MAX { + return Ok(SymbolEncodingType::Repeat); + } + if nb_seq < dynamic_fse_nb_seq_min || most_frequent < (nb_seq >> (default_norm_log - 1)) + { + *repeat_mode = FseRepeat::None; + return Ok(SymbolEncodingType::Basic); + } + } + } else { + let basic_cost = cost_or_error(if is_default_allowed { + cross_entropy_cost(default_norm, default_norm_log, count, max) + } else { + Err(Error::Generic) + }); + let repeat_cost = cost_or_error(if *repeat_mode != FseRepeat::None { + fse_bit_cost(prev_ctable, count, max) + } else { + Err(Error::Generic) + }); + let ncount_cost = cost_or_error(ncount_cost(count, max, nb_seq, fse_log)); + let compressed_cost = (ncount_cost << 3).wrapping_add(entropy_cost(count, max, nb_seq)?); + + if basic_cost <= repeat_cost && basic_cost <= compressed_cost { + *repeat_mode = FseRepeat::None; + return Ok(SymbolEncodingType::Basic); + } + if repeat_cost <= compressed_cost { + return Ok(SymbolEncodingType::Repeat); + } + } + *repeat_mode = FseRepeat::Check; + Ok(SymbolEncodingType::Compressed) +} + +const fn t(delta_find_state: i32, delta_nb_bits: u32) -> FseSymbolTransform { + FseSymbolTransform { + delta_find_state, + delta_nb_bits, + } +} + +/// What [`FseCTable::build`] makes of [`LL_DEFAULT_NORM`]. +#[rustfmt::skip] +static LL_DEFAULT_CTABLE: PredefinedCTable = PredefinedCTable::new( + LL_DEFAULT_NORM_LOG, + [ + 64, 65, 86, 107, 66, 87, 108, 88, 109, 67, 110, 68, 89, 90, 111, 69, + 112, 70, 91, 92, 113, 71, 114, 72, 93, 94, 115, 73, 116, 95, 74, 117, + 75, 96, 97, 118, 76, 119, 77, 98, 99, 120, 78, 121, 79, 100, 101, 122, + 80, 123, 81, 102, 103, 82, 104, 83, 105, 84, 106, 85, 127, 126, 125, 124, + ], + [ + t(-4, 327552), t( 1, 327584), t( 5, 393088), t( 7, 393088), t( 9, 393088), t(11, 393088), + t(13, 393088), t(15, 393088), t(17, 393088), t(19, 393088), t(21, 393088), t(23, 393088), + t(25, 393088), t(28, 393152), t(29, 393152), t(30, 393152), t(30, 393088), t(32, 393088), + t(34, 393088), t(36, 393088), t(38, 393088), t(40, 393088), t(42, 393088), t(44, 393088), + t(46, 393088), t(47, 327584), t(51, 393088), t(54, 393152), t(55, 393152), t(56, 393152), + t(57, 393152), t(58, 393152), t(59, 393152), t(60, 393152), t(61, 393152), t(62, 393152), + ], +); + +/// What [`FseCTable::build`] makes of [`OF_DEFAULT_NORM`]. +#[rustfmt::skip] +static OF_DEFAULT_CTABLE: PredefinedCTable = PredefinedCTable::new( + OF_DEFAULT_NORM_LOG, + [ + 32, 55, 46, 37, 51, 42, 33, 56, 38, 47, 43, 52, 34, 57, 48, 39, + 53, 44, 35, 58, 49, 40, 54, 45, 36, 50, 41, 63, 62, 61, 60, 59, + ], + [ + t(-1, 327648), t( 0, 327648), t( 1, 327648), t( 2, 327648), t( 3, 327648), t( 4, 327648), + t( 4, 327616), t( 6, 327616), t( 8, 327616), t(11, 327648), t(12, 327648), t(13, 327648), + t(14, 327648), t(15, 327648), t(16, 327648), t(17, 327648), t(18, 327648), t(19, 327648), + t(20, 327648), t(21, 327648), t(22, 327648), t(23, 327648), t(24, 327648), t(25, 327648), + t(26, 327648), t(27, 327648), t(28, 327648), t(29, 327648), t(30, 327648), + ], +); + +/// What [`FseCTable::build`] makes of [`ML_DEFAULT_NORM`]. +#[rustfmt::skip] +static ML_DEFAULT_CTABLE: PredefinedCTable = PredefinedCTable::new( + ML_DEFAULT_NORM_LOG, + [ + 64, 65, 86, 107, 108, 66, 87, 109, 67, 88, 89, 110, 68, 111, 69, 90, + 91, 112, 70, 113, 92, 71, 114, 93, 72, 115, 94, 73, 116, 95, 74, 117, + 96, 75, 118, 97, 76, 119, 98, 77, 120, 99, 78, 100, 79, 101, 80, 102, + 81, 103, 82, 104, 83, 105, 84, 106, 85, 127, 126, 125, 124, 123, 122, 121, + ], + [ + t(-1, 393152), t(-3, 327552), t( 2, 327584), t( 6, 393088), t( 8, 393088), t(10, 393088), + t(12, 393088), t(14, 393088), t(16, 393088), t(19, 393152), t(20, 393152), t(21, 393152), + t(22, 393152), t(23, 393152), t(24, 393152), t(25, 393152), t(26, 393152), t(27, 393152), + t(28, 393152), t(29, 393152), t(30, 393152), t(31, 393152), t(32, 393152), t(33, 393152), + t(34, 393152), t(35, 393152), t(36, 393152), t(37, 393152), t(38, 393152), t(39, 393152), + t(40, 393152), t(41, 393152), t(42, 393152), t(43, 393152), t(44, 393152), t(45, 393152), + t(46, 393152), t(47, 393152), t(48, 393152), t(49, 393152), t(50, 393152), t(51, 393152), + t(52, 393152), t(53, 393152), t(54, 393152), t(55, 393152), t(56, 393152), t(57, 393152), + t(58, 393152), t(59, 393152), t(60, 393152), t(61, 393152), t(62, 393152), + ], +); + +/// What is particular to literal lengths, to offsets or to match lengths in how a table is made. +pub(crate) struct CodeKind { + pub(crate) fse_log: u32, + pub(crate) default_norm: &'static [i16], + pub(crate) default_norm_log: u32, + /// The largest code that the predefined table has. + pub(crate) default_max: u32, + default_ctable: &'static PredefinedCTable, +} + +pub(crate) static LL_KIND: CodeKind = CodeKind { + fse_log: LL_FSE_LOG, + default_norm: &LL_DEFAULT_NORM, + default_norm_log: LL_DEFAULT_NORM_LOG, + default_max: MAX_LL, + default_ctable: &LL_DEFAULT_CTABLE, +}; + +pub(crate) static OF_KIND: CodeKind = CodeKind { + fse_log: OFF_FSE_LOG, + default_norm: &OF_DEFAULT_NORM, + default_norm_log: OF_DEFAULT_NORM_LOG, + default_max: DEFAULT_MAX_OFF, + default_ctable: &OF_DEFAULT_CTABLE, +}; + +pub(crate) static ML_KIND: CodeKind = CodeKind { + fse_log: ML_FSE_LOG, + default_norm: &ML_DEFAULT_NORM, + default_norm_log: ML_DEFAULT_NORM_LOG, + default_max: MAX_ML, + default_ctable: &ML_DEFAULT_CTABLE, +}; + +/// `ZSTD_buildCTable`: makes the table for `nb_seq` codes, the last of which is `last_code` and of which `count` is the +/// distribution, and writes what the decoder needs to make the same one. Returns how many bytes that takes. +pub(crate) fn build_ctable( + dst: &mut [u8], + next_ctable: &mut FseCTable, + kind: &CodeKind, + ty: SymbolEncodingType, + count: &mut [u32], + max: u32, + nb_seq: usize, + last_code: u8, + prev_ctable: &FseCTable, +) -> Result { + match ty { + SymbolEncodingType::Rle => { + next_ctable.build_rle(max as u8)?; + let Some(first) = dst.first_mut() else { + return Err(Error::DstSizeTooSmall); + }; + *first = max as u8; + Ok(1) + } + SymbolEncodingType::Repeat => { + next_ctable.clone_from(prev_ctable); + Ok(0) + } + SymbolEncodingType::Basic => { + next_ctable.set_predefined(kind.default_ctable); + Ok(0) + } + SymbolEncodingType::Compressed => { + let mut norm = [0i16; MAX_SEQ as usize + 1]; + let table_log = fse::optimal_table_log(kind.fse_log, nb_seq, max); + // The last code is the first state, which takes no transition. + let last_count = count.at_mut(usize::from(last_code))?; + let without_last = usize::from(*last_count > 1); + *last_count -= without_last as u32; + let nb_seq_1 = nb_seq - without_last; + let normalized = fse::normalize_count( + &mut norm, + table_log, + count, + nb_seq_1, + max, + use_low_prob_count(nb_seq_1), + ); + *count.at_mut(usize::from(last_code))? += without_last as u32; + normalized?; + let ncount_size = fse::write_ncount(dst, &norm, max, table_log)?; + next_ctable.build(&norm, max, table_log)?; + Ok(ncount_size) + } + } +} + +/// The code of a length or of an offset. As the index into a table of 64, it takes no check. +#[rustfmt::skip] +#[derive(Clone, Copy, Default)] +#[repr(u8)] +pub(crate) enum Code { + #[default] + C0, C1, C2, C3, C4, C5, C6, C7, C8, C9, C10, C11, C12, C13, C14, C15, + C16, C17, C18, C19, C20, C21, C22, C23, C24, C25, C26, C27, C28, C29, C30, C31, + C32, C33, C34, C35, C36, C37, C38, C39, C40, C41, C42, C43, C44, C45, C46, C47, + C48, C49, C50, C51, C52, C53, C54, C55, C56, C57, C58, C59, C60, C61, C62, C63, +} +use Code::*; + +impl Code { + #[rustfmt::skip] + const ALL: [Self; 64] = [ + C0, C1, C2, C3, C4, C5, C6, C7, C8, C9, C10, C11, C12, C13, C14, C15, + C16, C17, C18, C19, C20, C21, C22, C23, C24, C25, C26, C27, C28, C29, C30, C31, + C32, C33, C34, C35, C36, C37, C38, C39, C40, C41, C42, C43, C44, C45, C46, C47, + C48, C49, C50, C51, C52, C53, C54, C55, C56, C57, C58, C59, C60, C61, C62, C63, + ]; + + /// `code` is less than 64. + #[inline(always)] + fn new(code: u32) -> Self { + (Self::ALL.get(code as usize & 63).copied()).unwrap_or_default() + } +} + +/// The code of a length, and the bits that follow the code in the stream. +#[derive(Clone, Copy)] +struct LengthCode { + code: Code, + nb_bits: u32, + bits: u32, +} + +/// [`LengthCode`] for the lengths below `N`. +struct LengthCodes { + code: [Code; N], + nb_bits: [u8; N], + bits: [u8; N], +} + +impl LengthCodes { + #[inline(always)] + fn get(&self, length: u16) -> Option { + let index = usize::from(length); + Some(LengthCode { + code: *self.code.get(index)?, + nb_bits: u32::from(*self.nb_bits.get(index)?), + bits: u32::from(*self.bits.get(index)?), + }) + } + + /// From `N` on, a code is for the lengths of one binary logarithm, to which `delta` is added. + fn get_any(&self, length: u16, delta: u32) -> LengthCode { + self.get(length).unwrap_or_else(|| { + let log = highbit32(u32::from(length)); + LengthCode { + code: Code::new(log + delta), + nb_bits: log, + bits: u32::from(length) & !(1 << log), + } + }) + } +} + +/// `ZSTD_LLcode` +#[rustfmt::skip] +static LL_CODES: LengthCodes<64> = LengthCodes { + code: [ + C0, C1, C2, C3, C4, C5, C6, C7, C8, C9, C10, C11, C12, C13, C14, C15, + C16, C16, C17, C17, C18, C18, C19, C19, C20, C20, C20, C20, C21, C21, C21, C21, + C22, C22, C22, C22, C22, C22, C22, C22, C23, C23, C23, C23, C23, C23, C23, C23, + C24, C24, C24, C24, C24, C24, C24, C24, C24, C24, C24, C24, C24, C24, C24, C24, + ], + nb_bits: [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + ], + bits: [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 0, 1, 2, 3, + 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + ], +}; + +/// `ZSTD_MLcode` +#[rustfmt::skip] +static ML_CODES: LengthCodes<128> = LengthCodes { + code: [ + C0, C1, C2, C3, C4, C5, C6, C7, C8, C9, C10, C11, C12, C13, C14, C15, + C16, C17, C18, C19, C20, C21, C22, C23, C24, C25, C26, C27, C28, C29, C30, C31, + C32, C32, C33, C33, C34, C34, C35, C35, C36, C36, C36, C36, C37, C37, C37, C37, + C38, C38, C38, C38, C38, C38, C38, C38, C39, C39, C39, C39, C39, C39, C39, C39, + C40, C40, C40, C40, C40, C40, C40, C40, C40, C40, C40, C40, C40, C40, C40, C40, + C41, C41, C41, C41, C41, C41, C41, C41, C41, C41, C41, C41, C41, C41, C41, C41, + C42, C42, C42, C42, C42, C42, C42, C42, C42, C42, C42, C42, C42, C42, C42, C42, + C42, C42, C42, C42, C42, C42, C42, C42, C42, C42, C42, C42, C42, C42, C42, C42, + ], + nb_bits: [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + ], + bits: [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 0, 1, 2, 3, + 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + ], +}; + +const LL_DELTA_CODE: u32 = 19; +const ML_DELTA_CODE: u32 = 36; + +/// The codes of a sequence, and the bits that follow them. +#[derive(Clone, Copy, Default)] +pub(crate) struct SeqCodes { + /// The bits for the literal length, above them those for the match length, above them those for the offset, and + /// above them a bit set. If there are too many for the sequence to be written at once, only the bit set. + extra_bits: u32, + pub(crate) ll: Code, + pub(crate) of: Code, + pub(crate) ml: Code, +} + +impl SeqCodes { + /// With more than this on top of the bits that leave the three states, a sequence may not fit the container. + const MAX_EXTRA_BITS: u32 = 64 - 7 - (LL_FSE_LOG + ML_FSE_LOG + OFF_FSE_LOG) - 1; + + /// `None` if there are too many extra bits. + #[inline(always)] + fn new(ll: LengthCode, ml: LengthCode, off_base: u32) -> Option { + let of = highbit32(off_base); + if ll.nb_bits + ml.nb_bits + of > Self::MAX_EXTRA_BITS { + return None; + } + // The highest bit of `off_base`, which the code stands for, is the bit set above the others. + let above_ll = (off_base << (ml.nb_bits & 31)) | ml.bits; + Some(Self { + extra_bits: (above_ll << (ll.nb_bits & 31)) | ll.bits, + ll: ll.code, + of: Code::new(of), + ml: ml.code, + }) + } + + /// `*self = codes`, without putting the fields together first. + #[inline(always)] + fn set(&mut self, codes: Self) { + self.extra_bits = codes.extra_bits; + self.ll = codes.ll; + self.of = codes.of; + self.ml = codes.ml; + } + + fn ll_bits(self) -> u32 { + u32::from(LL_BITS.get(self.ll as usize).copied().unwrap_or(0)) + } + + fn ml_bits(self) -> u32 { + u32::from(ML_BITS.get(self.ml as usize).copied().unwrap_or(0)) + } + + /// Whether `extra_bits` lacks the extra bits. + fn has_many_bits(self) -> bool { + self.extra_bits == 1 + && self.ll_bits() + self.ml_bits() + self.of as u32 > Self::MAX_EXTRA_BITS + } +} + +/// How many times each code occurs. +pub(crate) type CodeCounts = [u32; 64]; + +#[inline(always)] +fn count_code(counts: &mut CodeCounts, code: Code, delta: u32) { + if let Some(count) = counts.get_mut(code as usize) { + *count = count.wrapping_add(delta); + } +} + +/// What `HIST_count` tells about counts. +#[derive(Clone, Copy)] +pub(crate) struct CountsSummary { + pub(crate) max_symbol_value: u32, + pub(crate) largest: usize, +} + +pub(crate) fn summarize(counts: &CodeCounts) -> CountsSummary { + CountsSummary { + max_symbol_value: counts.iter().rposition(|&c| c != 0).unwrap_or(0) as u32, + largest: counts.iter().copied().max().unwrap_or(0) as usize, + } +} + +/// The distributions of the codes of some sequences. +pub(crate) struct SeqHistograms { + pub(crate) ll: CodeCounts, + pub(crate) of: CodeCounts, + pub(crate) ml: CodeCounts, + /// Whether some sequence has too many extra bits to be written at once. + pub(crate) many_bits: bool, +} + +impl SeqHistograms { + #[inline(always)] + fn add(&mut self, codes: SeqCodes) { + count_code(&mut self.ll, codes.ll, 1); + count_code(&mut self.of, codes.of, 1); + count_code(&mut self.ml, codes.ml, 1); + } + + /// The codes of any sequence, which are not counted. + fn codes_of(&mut self, ll: LengthCode, ml: LengthCode, off_base: u32) -> SeqCodes { + SeqCodes::new(ll, ml, off_base).unwrap_or_else(|| { + self.many_bits = true; + SeqCodes { + extra_bits: 1, + ll: ll.code, + of: Code::new(highbit32(off_base)), + ml: ml.code, + } + }) + } + + /// For a sequence that takes more than the tables of [`seq_to_codes`]. + #[cold] + #[inline(never)] + fn add_unusual(&mut self, seq: SeqDef) -> SeqCodes { + let codes = self.codes_of( + LL_CODES.get_any(seq.lit_length, LL_DELTA_CODE), + ML_CODES.get_any(seq.ml_base, ML_DELTA_CODE), + seq.off_base, + ); + self.add(codes); + codes + } +} + +/// `ZSTD_seqToCodes`, and the distributions of the codes. `codes` is as long as `sequences`. One length of the sequence +/// `long_length_pos`, if there is such a sequence, is 0x10000 more than it says, unless `long_length_type` is `None`. +pub(crate) fn seq_to_codes( + codes: &mut [SeqCodes], + sequences: &[SeqDef], + long_length_type: LongLengthType, + long_length_pos: usize, +) -> SeqHistograms { + let mut histograms = SeqHistograms { + ll: [0; 64], + of: [0; 64], + ml: [0; 64], + many_bits: false, + }; + for (codes, seq) in codes.iter_mut().zip(sequences) { + if let (Some(ll), Some(ml)) = (LL_CODES.get(seq.lit_length), ML_CODES.get(seq.ml_base)) + && let Some(usual) = SeqCodes::new(ll, ml, seq.off_base) + { + codes.set(usual); + histograms.add(usual); + } else { + *codes = histograms.add_unusual(*seq); + } + } + if long_length_type != LongLengthType::None + && let (Some(codes), Some(seq)) = ( + codes.get_mut(long_length_pos), + sequences.get(long_length_pos), + ) + { + let mut ll = LL_CODES.get_any(seq.lit_length, LL_DELTA_CODE); + let mut ml = ML_CODES.get_any(seq.ml_base, ML_DELTA_CODE); + let (length, counts, code, bits) = if long_length_type == LongLengthType::LiteralLength { + (&mut ll, &mut histograms.ll, MAX_LL, seq.lit_length) + } else { + (&mut ml, &mut histograms.ml, MAX_ML, seq.ml_base) + }; + count_code(counts, length.code, 1u32.wrapping_neg()); + *length = LengthCode { + code: Code::new(code), + nb_bits: 16, + bits: u32::from(bits), + }; + count_code(counts, length.code, 1); + *codes = histograms.codes_of(ll, ml, seq.off_base); + } + histograms +} + +/// Bits on their way to the stream, with a bit set above them: the first to be written are the lowest. +#[derive(Clone, Copy)] +struct SeqBits(u64); + +impl SeqBits { + /// The lowest `nb_bits` of `bits`, which is less than 32. + #[inline(always)] + fn new(bits: u32, nb_bits: u32) -> Self { + let above = 1 << (nb_bits & 31); + Self((u64::from(bits) & (above - 1)) | above) + } + + /// `FSE_encodeSymbol`: the bits that leave the state go below the others. + #[inline(always)] + fn encode_below(&mut self, table: &FseCTable, state: &mut u32, code: Code) { + let transform = (table.symbol_tt.get(code as usize).copied()).unwrap_or_default(); + let nb_bits = state.wrapping_add(transform.delta_nb_bits) >> 16; + let kept = state.wrapping_shr(nb_bits); + // Taking `kept` from what comes above the state cancels the bits that stay in it. + self.0 = (self.0.wrapping_sub(u64::from(kept))) + .wrapping_shl(nb_bits) + .wrapping_add(u64::from(*state)); + let next = kept.wrapping_add_signed(transform.delta_find_state) as usize; + let state_table = &table.state_table; + *state = u32::from((state_table.get(next & (state_table.len() - 1)).copied()).unwrap_or(0)); + } +} + +/// The states of the three encoders. +#[derive(Clone, Copy)] +struct SeqStates { + lit_length: u32, + match_length: u32, + offset_bits: u32, +} + +/// `BIT_CStream_t`, without a container: what a byte lacks is added to it in the output. +struct SeqStream<'a> { + /// In the byte that `bit_pos` is in, that bit is set, and those above it are not. + dst: &'a mut [u8], + bit_pos: usize, +} + +impl SeqStream<'_> { + /// `BIT_addBits` and `BIT_flushBits`: `seq` makes at most 56 bits. `None` if the output is too small, which is found + /// out first so that there is less to keep while `seq` is at work. + #[inline(always)] + fn write_with(&mut self, seq: impl FnOnce() -> SeqBits) -> Option<()> { + let pos = self.bit_pos >> 3; + if pos > self.dst.len().checked_sub(size_of::())? { + return None; + } + let out: &mut [u8; 8] = self.dst.get_mut(pos..)?.first_chunk_mut()?; + let seq = NonZero::new(seq().0)?; + let [partial, ..] = *out; + // The lowest of the new bits replaces the bit set. + let bits = (seq.get() - 1).wrapping_shl(self.bit_pos as u32 & 7); + *out = bits.wrapping_add(u64::from(partial)).to_le_bytes(); + self.bit_pos += seq.ilog2() as usize; + Some(()) + } + + #[inline(always)] + fn write(&mut self, bits: u32, nb_bits: u32) -> Option<()> { + self.write_with(|| SeqBits::new(bits, nb_bits)) + } +} + +/// Where [`encode_run`] leaves things. +struct Encoded { + states: SeqStates, + bit_pos: usize, +} + +/// From the last of `codes` to the first. `None` if the output is too small. +/// +/// On its own, so that nothing else competes for the registers. +#[inline(never)] +fn encode_run( + dst: &mut [u8], + bit_pos: usize, + tables: &FseCTables, + states: SeqStates, + codes: &[SeqCodes], +) -> Option { + let mut stream = SeqStream { dst, bit_pos }; + let mut states = SeqStates { + lit_length: states.lit_length, + match_length: states.match_length, + offset_bits: states.offset_bits, + }; + for seq_codes in codes.iter().rev() { + stream.write_with(|| { + let mut seq = SeqBits(u64::from(seq_codes.extra_bits)); + seq.encode_below( + &tables.litlength_c_table, + &mut states.lit_length, + seq_codes.ll, + ); + seq.encode_below( + &tables.matchlength_c_table, + &mut states.match_length, + seq_codes.ml, + ); + seq.encode_below( + &tables.offcode_c_table, + &mut states.offset_bits, + seq_codes.of, + ); + seq + })?; + } + Some(Encoded { + states, + bit_pos: stream.bit_pos, + }) +} + +/// Returns the size of the stream in bits, or `None` if the output is too small. +fn encode_sequences_into( + dst: &mut [u8], + tables: &FseCTables, + codes: &[SeqCodes], + sequences: &[SeqDef], + many_bits: bool, +) -> Option { + *dst.first_mut()? = 1; + let mut stream = SeqStream { dst, bit_pos: 0 }; + let last_codes = codes.last()?; + let mut states = SeqStates { + lit_length: (tables.litlength_c_table).first_state(last_codes.ll as u8), + match_length: (tables.matchlength_c_table).first_state(last_codes.ml as u8), + offset_bits: tables.offcode_c_table.first_state(last_codes.of as u8), + }; + // The sequence whose extra bits are to be written: the last one, which has nothing else to write, + let mut apart = codes.len() - 1; + loop { + let (seq_codes, seq) = (codes.get(apart)?, sequences.get(apart)?); + stream.write(u32::from(seq.lit_length), seq_codes.ll_bits())?; + stream.write(u32::from(seq.ml_base), seq_codes.ml_bits())?; + stream.write(seq.off_base, seq_codes.of as u32)?; + + // then each that has too many. + let left = codes.get(..apart)?; + let next = if many_bits { + left.iter().rposition(|codes| codes.has_many_bits()) + } else { + None + }; + let run = encode_run( + stream.dst, + stream.bit_pos, + tables, + states, + left.get(next.unwrap_or(0)..)?, + )?; + states = run.states; + stream.bit_pos = run.bit_pos; + match next { + Some(next) => apart = next, + None => break, + } + } + + // `FSE_flushCState` + stream.write(states.match_length, tables.matchlength_c_table.table_log)?; + stream.write(states.offset_bits, tables.offcode_c_table.table_log)?; + stream.write(states.lit_length, tables.litlength_c_table.table_log)?; + // `BIT_closeCStream`: a bit set, which is there. + Some(stream.bit_pos + 1) +} + +/// `nb_seq` sequences and the descriptions of their three tables take no more. A sequence has up to 16, 16 and 31 bits, +/// and 9, 9 and 8 for the states. +pub(crate) const fn max_size(nb_seq: usize) -> usize { + 3 * FSE_NCOUNTBOUND + nb_seq * 12 +} + +/// `ZSTD_encodeSequences`: `codes` are those of `sequences`, of which there is at least one. Without `many_bits`, none +/// has too many extra bits to be written at once. +pub(crate) fn encode_sequences( + dst: &mut [u8], + tables: &FseCTables, + codes: &[SeqCodes], + sequences: &[SeqDef], + many_bits: bool, +) -> Result { + let capacity = dst.len(); + encode_sequences_into(dst, tables, codes, sequences, many_bits) + // The stream has to end before the last position that a container can be written to. + .filter(|nb_bits| (nb_bits >> 3) + size_of::() < capacity) + .map(|nb_bits| nb_bits.div_ceil(8)) + .ok_or(Error::DstSizeTooSmall) +} diff --git a/src/zstd/compress/stream.rs b/src/zstd/compress/stream.rs new file mode 100644 index 000000000000..1f449b3de233 --- /dev/null +++ b/src/zstd/compress/stream.rs @@ -0,0 +1,545 @@ +//! The public side of `zstd_compress.c`: `ZSTD_CCtx`, its parameters, `ZSTD_compressStream2` and `ZSTD_compress`. + +use std::sync::Arc; + +use super::cctx::{CDict, FrameCompressor, Input, Output, Span}; +use super::internal::{BufId, Buffers, CompressedBlockState}; +use super::mt::MtCtx; +use super::params::{ + CCtxParams, CParamMode, ZSTD_CLEVEL_DEFAULT, ZSTDMT_JOBSIZE_MIN, compress_bound, + get_c_params_from_cctx_params, get_params_internal, resolve_block_splitter_mode, + resolve_enable_ldm, resolve_row_match_finder_mode, +}; +use crate::error::{Error, Result}; +use crate::util::{SliceExt, append, try_copy}; +use crate::{InBuffer, OutBuffer}; + +/// `ZSTD_EndDirective` +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum EndDirective { + /// Collect more input: the encoder decides when to output. + Continue = 0, + /// Output all that was consumed, so that a decoder can get it. + Flush = 1, + /// Flush, and close the frame. + End = 2, +} + +/// `ZSTD_cStreamStage` +#[derive(Clone, Copy, PartialEq, Eq)] +enum StreamStage { + Init, + Load, + Flush, + /// The frame is given up. The C library then starts another if it has workers and else goes on with this one, and + /// `zstd.h` calls a call after one that has failed undefined behavior. + Failed(Error), +} + +/// `ZSTD_CCtx` +pub struct CCtx { + frame: FrameCompressor, + requested_params: CCtxParams, + /// As long as it was ever filled, with room for `in_buff_size`. + in_buff: Vec, + in_buff_size: usize, + in_to_compress: usize, + in_buff_pos: usize, + in_buff_target: usize, + /// Empty until it is needed, with room for `out_buff_size`. + out_buff: Vec, + out_buff_size: usize, + out_buff_content_size: usize, + out_buff_flushed_size: usize, + stream_stage: StreamStage, + frame_ended: bool, + /// `localDict.dict` + local_dict: Option>>, + /// `localDict.cdict`, which is made when the first frame starts. + cdict: Option>, + mtctx: Option>, +} + +impl Default for CCtx { + fn default() -> Self { + Self::new() + } +} + +/// `ZSTD_limitCopy` +fn limit_copy(dst: &mut [u8], src: &[u8]) -> Result { + let length = dst.len().min(src.len()); + dst.write_at(0, src.sub(..length)?)?; + Ok(length) +} + +/// Gives `buffer` room for `new_size` bytes instead of `size`, none of which is touched. +fn resize_buffer(buffer: &mut Vec, size: &mut usize, new_size: usize) -> Result<()> { + if *size != new_size { + // First, so that both are not allocated at once. + *buffer = Vec::new(); + *size = 0; + buffer + .try_reserve_exact(new_size) + .map_err(|_| Error::MemoryAllocation)?; + *size = new_size; + } + Ok(()) +} + +/// Writes `src` at `pos`, which is at most the length of `buffer`, over what is there and beyond, for which it has the +/// capacity. +fn write_or_append(buffer: &mut Vec, pos: usize, src: &[u8]) -> Result<()> { + let over = buffer.sub_mut(pos..)?; + let (first, rest) = src.split_at_checked(over.len()).unwrap_or((src, &[])); + over.write_at(0, first)?; + append(buffer, rest) +} + +impl CCtx { + /// How much memory a new one takes. + pub const SIZE: usize = size_of::() + 2 * size_of::(); + + /// `ZSTD_createCCtx` + pub fn new() -> Self { + Self { + frame: FrameCompressor::new(), + requested_params: CCtxParams::new(ZSTD_CLEVEL_DEFAULT), + in_buff: Vec::new(), + in_buff_size: 0, + in_to_compress: 0, + in_buff_pos: 0, + in_buff_target: 0, + out_buff: Vec::new(), + out_buff_size: 0, + out_buff_content_size: 0, + out_buff_flushed_size: 0, + stream_stage: StreamStage::Init, + frame_ended: false, + local_dict: None, + cdict: None, + mtctx: None, + } + } + + /// `ZSTD_CCtx_setParameter`: `param` is one of [`super::params::c_param`]. + pub fn set_parameter(&mut self, param: i32, value: i32) -> Result { + if self.stream_stage != StreamStage::Init { + return Err(Error::StageWrong); + } + self.requested_params.set_parameter(param, value) + } + + /// `ZSTD_CCtx_setPledgedSrcSize` + pub fn set_pledged_src_size(&mut self, pledged_src_size: u64) -> Result<()> { + if self.stream_stage != StreamStage::Init { + return Err(Error::StageWrong); + } + self.frame.pledged_src_size_plus_one = pledged_src_size.wrapping_add(1); + Ok(()) + } + + /// `ZSTD_CCtx_loadDictionary`: every frame from now on starts with `dict`, of which a copy is kept. If it is empty, + /// with no dictionary. + pub fn load_dictionary(&mut self, dict: &[u8]) -> Result<()> { + if self.stream_stage != StreamStage::Init { + return Err(Error::StageWrong); + } + self.local_dict = None; + self.cdict = None; + if !dict.is_empty() { + self.local_dict = Some(Arc::new(try_copy(dict)?)); + } + Ok(()) + } + + /// `ZSTD_CCtx_reset` with `ZSTD_reset_session_only` + fn reset_session(&mut self) { + self.stream_stage = StreamStage::Init; + self.frame.pledged_src_size_plus_one = 0; + } + + /// `ZSTD_initLocalDict` + fn init_local_dict(&mut self) -> Result<()> { + let Some(dict) = &self.local_dict else { + return Ok(()); + }; + if self.cdict.is_some() { + return Ok(()); + } + // Whatever is wrong with the dictionary, the C code only knows that it got no `ZSTD_CDict`. + let cdict = CDict::new(Arc::clone(dict), &self.requested_params) + .map_err(|_| Error::MemoryAllocation)?; + self.cdict = Some(Arc::new(cdict)); + Ok(()) + } + + /// `ZSTD_CCtx_init_compressStream2` + fn init_compress_stream2(&mut self, end_op: EndDirective, in_size: usize) -> Result<()> { + let mut params = self.requested_params; + self.init_local_dict()?; + if end_op == EndDirective::End { + // All the input is there. + self.frame.pledged_src_size_plus_one = in_size as u64 + 1; + } + let pledged_src_size = self.frame.pledged_src_size_plus_one.wrapping_sub(1); + let cdict = self.cdict.as_deref(); + + { + let dict_size = cdict.map_or(0, |cdict| cdict.dict_content().len()); + // `ZSTD_getCParamMode` + let mode = match cdict { + Some(cdict) if cdict.should_attach(¶ms, pledged_src_size) => { + CParamMode::AttachDict + } + _ => CParamMode::NoAttachDict, + }; + params.c_params = + get_c_params_from_cctx_params(¶ms, pledged_src_size, dict_size, mode); + } + + params.post_block_splitter = + resolve_block_splitter_mode(params.post_block_splitter, ¶ms.c_params); + params.ldm_params.enable_ldm = + resolve_enable_ldm(params.ldm_params.enable_ldm, ¶ms.c_params); + params.use_row_match_finder = + resolve_row_match_finder_mode(params.use_row_match_finder, ¶ms.c_params); + + if pledged_src_size <= ZSTDMT_JOBSIZE_MIN as u64 { + // Not worth several threads. + params.nb_workers = 0; + } + if params.nb_workers > 0 { + if self.mtctx.is_none() { + self.mtctx = Some(Box::new(MtCtx::new(params.nb_workers as u32)?)); + } + let mtctx = self.mtctx.as_mut().ok_or(Error::MemoryAllocation)?; + mtctx.init_c_stream_internal(self.cdict.clone(), params, pledged_src_size)?; + self.frame.dict_id = cdict.map_or(0, CDict::dict_id); + self.frame.dict_content_size = cdict.map_or(0, |cdict| cdict.dict_content().len()); + self.frame.consumed_src_size = 0; + self.frame.produced_c_size = 0; + self.stream_stage = StreamStage::Load; + self.frame.applied_params = params; + } else { + let input = Input { + bufs: Buffers { + in_buff: &[], + user: &[], + dict: cdict.map_or(&[], CDict::dict_content), + }, + cdict, + }; + let sizes = self + .frame + .begin(&input, None, ¶ms, pledged_src_size, true)?; + // They are part of the workspace in the C code. + resize_buffer( + &mut self.in_buff, + &mut self.in_buff_size, + sizes.buff_in_size, + ) + .and_then(|()| { + resize_buffer( + &mut self.out_buff, + &mut self.out_buff_size, + sizes.buff_out_size, + ) + }) + .inspect_err(|_| self.frame.forget())?; + self.in_to_compress = 0; + self.in_buff_pos = 0; + // One more byte, to tell an input of exactly one block from a longer one. + self.in_buff_target = self.frame.block_size_max + + usize::from(self.frame.block_size_max as u64 == pledged_src_size); + self.out_buff_content_size = 0; + self.out_buff_flushed_size = 0; + self.stream_stage = StreamStage::Load; + self.frame_ended = false; + } + Ok(()) + } + + /// `ZSTD_nextInputSizeHint` + fn next_input_size_hint(&self) -> usize { + match self.in_buff_target - self.in_buff_pos { + 0 => self.frame.block_size_max, + hint_in_size => hint_in_size, + } + } + + /// `ZSTD_compressStream_generic` + fn compress_stream_generic( + &mut self, + output: &mut OutBuffer<'_>, + input: &mut InBuffer<'_>, + flush_mode: EndDirective, + ) -> Result { + let iend = input.src.len(); + let mut ip = input.pos; + let mut op = output.pos; + let cdict = self.cdict.clone(); + let cdict = cdict.as_deref(); + let dict = cdict.map_or(&[][..], CDict::dict_content); + + loop { + match self.stream_stage { + StreamStage::Init => return Err(Error::InitMissing), + StreamStage::Failed(error) => return Err(error), + + StreamStage::Load => { + if flush_mode == EndDirective::End + && output.dst.len() - op >= compress_bound(iend - ip) + && self.in_buff_pos == 0 + { + // There is room for the worst case: no need for the buffers. + let c_size = self.frame.compress_end( + output.dst.sub_mut(op..)?, + &Input { + bufs: Buffers { + in_buff: &self.in_buff, + user: input.src, + dict, + }, + cdict, + }, + Span { + buf: BufId::User, + start: ip, + end: iend, + }, + )?; + ip = iend; + op += c_size; + self.frame_ended = true; + self.reset_session(); + break; + } + { + let loaded = (self.in_buff_target - self.in_buff_pos).min(iend - ip); + write_or_append( + &mut self.in_buff, + self.in_buff_pos, + input.src.span(ip, loaded)?, + )?; + self.in_buff_pos += loaded; + ip += loaded; + if flush_mode == EndDirective::Continue + && self.in_buff_pos < self.in_buff_target + { + // Not enough for a block yet. + break; + } + if flush_mode == EndDirective::Flush + && self.in_buff_pos == self.in_to_compress + { + // Nothing to flush. + break; + } + } + let i_size = self.in_buff_pos - self.in_to_compress; + let direct = output.dst.len() - op >= compress_bound(i_size); + let c_dst = if direct { + output.dst.sub_mut(op..)? + } else { + self.out_buff.resize(self.out_buff_size, 0); + self.out_buff.as_mut_slice() + }; + let last_block = flush_mode == EndDirective::End && ip == iend; + let frame_input = Input { + bufs: Buffers { + in_buff: &self.in_buff, + user: &[], + dict, + }, + cdict, + }; + let src = Span { + buf: BufId::InBuff, + start: self.in_to_compress, + end: self.in_buff_pos, + }; + let c_size = if last_block { + self.frame.compress_end(c_dst, &frame_input, src)? + } else { + self.frame.compress_continue(c_dst, &frame_input, src)? + }; + self.frame_ended = last_block; + // The next block goes after this one, or at the start if there is no room left. + self.in_buff_target = self.in_buff_pos + self.frame.block_size_max; + if self.in_buff_target > self.in_buff_size { + self.in_buff_pos = 0; + self.in_buff_target = self.frame.block_size_max; + } + self.in_to_compress = self.in_buff_pos; + if direct { + op += c_size; + if self.frame_ended { + self.reset_session(); + break; + } + continue; + } + self.out_buff_content_size = c_size; + self.out_buff_flushed_size = 0; + self.stream_stage = StreamStage::Flush; + } + + StreamStage::Flush => { + let to_flush = self.out_buff_content_size - self.out_buff_flushed_size; + let flushed = limit_copy( + output.dst.sub_mut(op..)?, + self.out_buff.span(self.out_buff_flushed_size, to_flush)?, + )?; + op += flushed; + self.out_buff_flushed_size += flushed; + if to_flush != flushed { + // The output is full. + break; + } + self.out_buff_content_size = 0; + self.out_buff_flushed_size = 0; + if self.frame_ended { + self.reset_session(); + break; + } + self.stream_stage = StreamStage::Load; + } + } + } + + input.pos = ip; + output.pos = op; + if self.frame_ended { + return Ok(0); + } + Ok(self.next_input_size_hint()) + } + + /// `ZSTD_compressStream2`: returns the least number of bytes that are left to flush, which for + /// [`EndDirective::End`] is 0 once the frame is complete. Once a frame has made it fail, it fails in the same way + /// ever after. + pub fn compress_stream2( + &mut self, + output: &mut OutBuffer<'_>, + input: &mut InBuffer<'_>, + end_op: EndDirective, + ) -> Result { + if output.pos > output.dst.len() { + return Err(Error::DstSizeTooSmall); + } + if input.pos > input.src.len() { + return Err(Error::SrcSizeWrong); + } + self.compress_stream2_on(output, input, end_op) + .inspect_err(|&error| { + // A frame that could not start can be tried again, as with the C library. + if self.stream_stage != StreamStage::Init { + self.stream_stage = StreamStage::Failed(error); + self.mtctx = None; + } + }) + } + + /// [`Self::compress_stream2`], but for what becomes of the stream if it fails. + fn compress_stream2_on( + &mut self, + output: &mut OutBuffer<'_>, + input: &mut InBuffer<'_>, + end_op: EndDirective, + ) -> Result { + if let StreamStage::Failed(error) = self.stream_stage { + return Err(error); + } + if self.stream_stage == StreamStage::Init { + self.init_compress_stream2(end_op, input.src.len() - input.pos)?; + } + + if self.frame.applied_params.nb_workers > 0 { + let mtctx = self.mtctx.as_mut().ok_or(Error::Generic)?; + loop { + let ipos = input.pos; + let opos = output.pos; + let mut flush_min = mtctx.compress_stream_generic(output, input, end_op); + self.frame.consumed_src_size += (input.pos - ipos) as u64; + self.frame.produced_c_size += (output.pos - opos) as u64; + // Not in 1.5.7, where only a frame of one job is checked, and any other is one that cannot be decoded. + let pledged = self.frame.pledged_src_size_plus_one; + let consumed = self.frame.consumed_src_size + 1; + let is_complete = end_op == EndDirective::End && flush_min == Ok(0); + if pledged != 0 && (consumed > pledged || (is_complete && consumed != pledged)) { + flush_min = flush_min.and(Err(Error::SrcSizeWrong)); + } + let flush_min = flush_min?; + if end_op == EndDirective::End && flush_min == 0 { + // `reset_session`, but `mtctx` is borrowed. + self.stream_stage = StreamStage::Init; + self.frame.pledged_src_size_plus_one = 0; + } + + if end_op == EndDirective::Continue { + // Some progress is enough. Otherwise, wait for a job to make some. + if input.pos != ipos + || output.pos != opos + || input.pos == input.src.len() + || output.pos == output.dst.len() + { + return Ok(flush_min); + } + } else if flush_min == 0 || output.pos == output.dst.len() { + // Flushing goes on for as long as there is room. + return Ok(flush_min); + } + } + } + self.compress_stream_generic(output, input, end_op)?; + Ok(self.out_buff_content_size - self.out_buff_flushed_size) + } +} + +/// `ZSTD_compress`: returns the size of the frame. +pub fn compress(dst: &mut [u8], src: &[u8], compression_level: i32) -> Result { + let dst = &mut Output::Slice(dst); + compress_to(&mut FrameCompressor::new(), dst, src, compression_level) +} + +/// `ZSTD_compressCCtx`, to either kind of output. +pub(crate) fn compress_to( + frame: &mut FrameCompressor, + dst: &mut Output<'_>, + src: &[u8], + compression_level: i32, +) -> Result { + let params = get_params_internal( + compression_level, + src.len() as u64, + 0, + CParamMode::NoAttachDict, + ); + let cctx_params = CCtxParams::from_params( + ¶ms, + if compression_level == 0 { + ZSTD_CLEVEL_DEFAULT + } else { + compression_level + }, + ); + let input = Input { + bufs: Buffers { + in_buff: &[], + user: src, + dict: &[], + }, + cdict: None, + }; + frame.begin(&input, None, &cctx_params, src.len() as u64, false)?; + frame.compress_end_to( + dst, + &input, + Span { + buf: BufId::User, + start: 0, + end: src.len(), + }, + ) +} diff --git a/src/zstd/compress/superblock.rs b/src/zstd/compress/superblock.rs new file mode 100644 index 000000000000..b78065c5b7dc --- /dev/null +++ b/src/zstd/compress/superblock.rs @@ -0,0 +1,582 @@ +//! `zstd_compress_superblock.c`: a block written as several smaller ones, of about the size asked for. + +use super::block_entropy::{additional_bits_of, build_block_entropy_stats, write_nb_seq}; +use super::fse::FseCTable; +use super::internal::{ + CompressedBlockState, EntropyCTables, EntropyCTablesMetadata, FseCTables, FseCTablesMetadata, + HufCTables, HufCTablesMetadata, SeqDef, SeqStore, block_header, no_compress_block, update_rep, +}; +use super::literals::{compress_rle_literals_block, no_compress_literals}; +use super::params::CCtxParams; +use super::sequences::{ + CodeCounts, SeqCodes, SeqHistograms, cross_entropy_cost, encode_sequences, fse_bit_cost, + summarize, +}; +use super::{hist, huf}; +use crate::common::{ + BlockType, DEFAULT_MAX_OFF, HUF_SYMBOLVALUE_MAX, LL_BITS, LL_DEFAULT_NORM, LL_DEFAULT_NORM_LOG, + MAX_LL, MAX_ML, MINMATCH, ML_BITS, ML_DEFAULT_NORM, ML_DEFAULT_NORM_LOG, OF_DEFAULT_NORM, + OF_DEFAULT_NORM_LOG, SymbolEncodingType, ZSTD_BLOCKHEADERSIZE, +}; +use crate::error::{Error, Result}; +use crate::util::{SliceExt, write_le24, write_le32}; + +const ZSTD_TARGETCBLOCKSIZE_MIN: usize = 1340; + +/// `ZSTD_compressSubBlock_literal`: the literals section of a sub-block. Returns its size, which is 0 if it could not +/// be compressed, and whether it holds the description of the Huffman table. +fn compress_sub_block_literal( + huf_table: &huf::HufCTable, + huf_metadata: &HufCTablesMetadata, + literals: &[u8], + dst: &mut [u8], + write_entropy: bool, +) -> Result<(usize, bool)> { + let lit_size = literals.len(); + let header = if write_entropy { 200 } else { 0 }; + let lh_size = 3 + + usize::from(lit_size >= (1 << 10) - header) + + usize::from(lit_size >= (16 << 10) - header); + let mut op = lh_size; + let single_stream = lh_size == 3; + let h_type = if write_entropy { + huf_metadata.h_type + } else { + SymbolEncodingType::Repeat + }; + let mut c_lit_size = 0usize; + + if lit_size == 0 || huf_metadata.h_type == SymbolEncodingType::Basic { + return Ok((no_compress_literals(dst, literals)?, false)); + } else if huf_metadata.h_type == SymbolEncodingType::Rle { + return Ok((compress_rle_literals_block(dst, literals)?, false)); + } + + if write_entropy && huf_metadata.h_type == SymbolEncodingType::Compressed { + let description = huf_metadata + .huf_des_buffer + .sub(..huf_metadata.huf_des_size)?; + dst.write_at(op, description)?; + op += description.len(); + c_lit_size += description.len(); + } + + { + let out = dst.sub_mut(op..)?; + let c_size = if single_stream { + huf::compress_1x_using_ctable(out, literals, huf_table) + } else { + huf::compress_4x_using_ctable(out, literals, huf_table) + }; + let Ok(c_size @ 1..) = c_size else { + return Ok((0, false)); + }; + op += c_size; + c_lit_size += c_size; + // With the table already written, literals that expand can still be left as they are. + if !write_entropy && c_lit_size >= lit_size { + return Ok((no_compress_literals(dst, literals)?, false)); + } + // The size of the header was a guess. + if lh_size < 3 + usize::from(c_lit_size >= 1 << 10) + usize::from(c_lit_size >= 16 << 10) { + return Ok((no_compress_literals(dst, literals)?, false)); + } + } + + let h_type = h_type as u32; + let lit = lit_size as u32; + let c_lit = c_lit_size as u32; + match lh_size { + 3 => { + let lhc = h_type + (u32::from(!single_stream) << 2) + (lit << 4) + (c_lit << 14); + write_le24(dst, 0, lhc)?; + } + 4 => { + let lhc = h_type + (2 << 2) + (lit << 4) + (c_lit << 18); + write_le32(dst, 0, lhc)?; + } + _ => { + let lhc = h_type + (3 << 2) + (lit << 4) + (c_lit << 22); + write_le32(dst, 0, lhc)?; + dst.put(4, (c_lit_size >> 10) as u8)?; + } + } + Ok((op, true)) +} + +/// `ZSTD_seqDecompressedSize`: how many bytes the `nb_seqs` sequences from `first` on stand for, with `lit_size` bytes +/// of literals. +fn seq_decompressed_size( + seq_store: &SeqStore, + first: usize, + nb_seqs: usize, + lit_size: usize, +) -> Result { + let mut match_length_sum = 0usize; + for n in first..first + nb_seqs { + match_length_sum += seq_store.get_sequence_length(n)?.match_length as usize; + } + Ok(match_length_sum + lit_size) +} + +/// `ZSTD_compressSubBlock_sequences`: the sequences section of a sub-block. Returns its size, which is 0 if it could +/// not be compressed, and whether it holds the descriptions of the tables. +fn compress_sub_block_sequences( + fse_tables: &FseCTables, + fse_metadata: &FseCTablesMetadata, + sequences: &[SeqDef], + codes: &[SeqCodes], + dst: &mut [u8], + write_entropy: bool, +) -> Result<(usize, bool)> { + let nb_seq = sequences.len(); + if dst.len() < 3 + 1 { + return Err(Error::DstSizeTooSmall); + } + let mut op = write_nb_seq(dst, nb_seq)?; + if nb_seq == 0 { + return Ok((op, false)); + } + + let seq_head = op; + op += 1; + + if write_entropy { + dst.put( + seq_head, + (((fse_metadata.ll_type as u32) << 6) + + ((fse_metadata.of_type as u32) << 4) + + ((fse_metadata.ml_type as u32) << 2)) as u8, + )?; + let tables = fse_metadata + .fse_tables_buffer + .sub(..fse_metadata.fse_tables_size)?; + dst.write_at(op, tables)?; + op += tables.len(); + } else { + let repeat = SymbolEncodingType::Repeat as u32; + dst.put( + seq_head, + ((repeat << 6) + (repeat << 4) + (repeat << 2)) as u8, + )?; + } + + let bitstream_size = encode_sequences(dst.sub_mut(op..)?, fse_tables, codes, sequences, true)?; + op += bitstream_size; + // Decoders up to 1.3.4 fail on a table header with fewer than 4 bytes after it, + if write_entropy + && fse_metadata.last_count_size != 0 + && fse_metadata.last_count_size + bitstream_size < 4 + { + return Ok((0, false)); + } + // and those up to 1.4.0 on a sequences section of fewer than 4 bytes. + if op - seq_head < 4 { + return Ok((0, false)); + } + Ok((op, true)) +} + +/// What [`compress_sub_block`] wrote. +struct SubBlock { + /// 0 if it could not be compressed. + c_size: usize, + lit_entropy_written: bool, + seq_entropy_written: bool, +} + +/// `ZSTD_compressSubBlock` +fn compress_sub_block( + entropy: &EntropyCTables, + entropy_metadata: &EntropyCTablesMetadata, + sequences: &[SeqDef], + literals: &[u8], + codes: &[SeqCodes], + dst: &mut [u8], + write_lit_entropy: bool, + write_seq_entropy: bool, + last_block: bool, +) -> Result { + let mut result = SubBlock { + c_size: 0, + lit_entropy_written: false, + seq_entropy_written: false, + }; + let mut op = ZSTD_BLOCKHEADERSIZE; + { + let (c_lit_size, written) = compress_sub_block_literal( + &entropy.huf.c_table, + &entropy_metadata.huf_metadata, + literals, + dst.sub_mut(op..)?, + write_lit_entropy, + )?; + result.lit_entropy_written = written; + if c_lit_size == 0 { + return Ok(result); + } + op += c_lit_size; + } + { + let (c_seq_size, written) = compress_sub_block_sequences( + &entropy.fse, + &entropy_metadata.fse_metadata, + sequences, + codes, + dst.sub_mut(op..)?, + write_seq_entropy, + )?; + result.seq_entropy_written = written; + if c_seq_size == 0 { + return Ok(result); + } + op += c_seq_size; + } + let c_size = op - ZSTD_BLOCKHEADERSIZE; + write_le24( + dst, + 0, + block_header(BlockType::Compressed, c_size, last_block), + )?; + result.c_size = op; + Ok(result) +} + +/// `ZSTD_estimateSubBlockSize_literal` +fn estimate_sub_block_size_literal( + literals: &[u8], + huf: &HufCTables, + huf_metadata: &HufCTablesMetadata, + write_entropy: bool, +) -> usize { + let mut count = [0u32; HUF_SYMBOLVALUE_MAX as usize + 1]; + let mut max_symbol_value = HUF_SYMBOLVALUE_MAX; + const LITERAL_SECTION_HEADER_SIZE: usize = 3; + match huf_metadata.h_type { + SymbolEncodingType::Basic => literals.len(), + SymbolEncodingType::Rle => 1, + SymbolEncodingType::Compressed | SymbolEncodingType::Repeat => { + if hist::count_wksp(&mut count, &mut max_symbol_value, literals).is_err() { + return literals.len(); + } + let mut c_lit_size_estimate = + huf::estimate_compressed_size(&huf.c_table, &count, max_symbol_value); + if write_entropy { + c_lit_size_estimate += huf_metadata.huf_des_size; + } + c_lit_size_estimate + LITERAL_SECTION_HEADER_SIZE + } + } +} + +/// `ZSTD_estimateSubBlockSize_symbolType`: `count` is the distribution of `nb_seq` codes. +fn estimate_sub_block_size_symbol_type( + ty: SymbolEncodingType, + count: &CodeCounts, + nb_seq: usize, + fse_c_table: &FseCTable, + additional_bits: Option<&[u8]>, + default_norm: &[i16], + default_norm_log: u32, + default_max: u32, +) -> usize { + let max = summarize(count).max_symbol_value; + let estimate_in_bits = match ty { + SymbolEncodingType::Basic if max <= default_max => { + cross_entropy_cost(default_norm, default_norm_log, count, max) + } + SymbolEncodingType::Basic => Err(Error::Generic), + SymbolEncodingType::Rle => Ok(0), + SymbolEncodingType::Compressed | SymbolEncodingType::Repeat => { + fse_bit_cost(fse_c_table, count, max) + } + }; + let Ok(estimate_in_bits) = estimate_in_bits else { + return nb_seq * 10; + }; + (estimate_in_bits + additional_bits_of(count, additional_bits)) / 8 +} + +/// `ZSTD_estimateSubBlockSize_sequences` +fn estimate_sub_block_size_sequences( + histograms: &SeqHistograms, + nb_seq: usize, + fse_tables: &FseCTables, + fse_metadata: &FseCTablesMetadata, + write_entropy: bool, +) -> usize { + const SEQUENCES_SECTION_HEADER_SIZE: usize = 3; + if nb_seq == 0 { + return SEQUENCES_SECTION_HEADER_SIZE; + } + let mut c_seq_size_estimate = estimate_sub_block_size_symbol_type( + fse_metadata.of_type, + &histograms.of, + nb_seq, + &fse_tables.offcode_c_table, + None, + &OF_DEFAULT_NORM, + OF_DEFAULT_NORM_LOG, + DEFAULT_MAX_OFF, + ); + c_seq_size_estimate += estimate_sub_block_size_symbol_type( + fse_metadata.ll_type, + &histograms.ll, + nb_seq, + &fse_tables.litlength_c_table, + Some(&LL_BITS), + &LL_DEFAULT_NORM, + LL_DEFAULT_NORM_LOG, + MAX_LL, + ); + c_seq_size_estimate += estimate_sub_block_size_symbol_type( + fse_metadata.ml_type, + &histograms.ml, + nb_seq, + &fse_tables.matchlength_c_table, + Some(&ML_BITS), + &ML_DEFAULT_NORM, + ML_DEFAULT_NORM_LOG, + MAX_ML, + ); + if write_entropy { + c_seq_size_estimate += fse_metadata.fse_tables_size; + } + c_seq_size_estimate + SEQUENCES_SECTION_HEADER_SIZE +} + +/// `ZSTD_needSequenceEntropyTables` +fn need_sequence_entropy_tables(fse_metadata: &FseCTablesMetadata) -> bool { + [ + fse_metadata.ll_type, + fse_metadata.ml_type, + fse_metadata.of_type, + ] + .iter() + .any(|ty| matches!(ty, SymbolEncodingType::Compressed | SymbolEncodingType::Rle)) +} + +/// `countLiterals` +fn count_literals(seq_store: &SeqStore, first: usize, seq_count: usize) -> Result { + let mut total = 0usize; + for n in first..first + seq_count { + total += seq_store.get_sequence_length(n)?.lit_length as usize; + } + Ok(total) +} + +const BYTESCALE: usize = 256; + +/// `sizeBlockSequences`: how many of the sequences `sp`, of which there is at least one, fit the budget. +fn size_block_sequences( + sp: &[SeqDef], + target_budget: usize, + avg_lit_cost: usize, + avg_seq_cost: usize, + first_sub_block: bool, +) -> Result { + let (first, rest) = sp.split_first().ok_or(Error::Generic)?; + // The first sub-block also has the tables. + let header_size = usize::from(first_sub_block) * 120 * BYTESCALE; + let mut budget = header_size; + budget += usize::from(first.lit_length) * avg_lit_cost + avg_seq_cost; + if budget > target_budget { + return Ok(1); + } + let mut in_size = + usize::from(first.lit_length) + usize::from(first.ml_base) + MINMATCH as usize; + + let mut n = 1; + for seq in rest { + budget += usize::from(seq.lit_length) * avg_lit_cost + avg_seq_cost; + in_size += usize::from(seq.lit_length) + usize::from(seq.ml_base) + MINMATCH as usize; + // Only stop where the sub-block is expected to compress. + if budget > target_budget && budget < in_size * BYTESCALE { + break; + } + n += 1; + } + Ok(n) +} + +/// `ZSTD_compressSubBlock_multi` +fn compress_sub_block_multi( + seq_store: &SeqStore, + histograms: &SeqHistograms, + prev_c_block: &CompressedBlockState, + next_c_block: &mut CompressedBlockState, + entropy_metadata: &EntropyCTablesMetadata, + cctx_params: &CCtxParams, + dst: &mut [u8], + src: &[u8], + last_block: bool, +) -> Result { + let sequences = seq_store.sequences.as_slice(); + let nb_seqs = sequences.len(); + let mut sp = 0usize; + let literals = seq_store.literals()?; + let nb_literals = literals.len(); + let mut lp = 0usize; + let mut ip = 0usize; + let mut op = 0usize; + let target_c_block_size = ZSTD_TARGETCBLOCKSIZE_MIN.max(cctx_params.target_c_block_size); + let mut write_lit_entropy = + entropy_metadata.huf_metadata.h_type == SymbolEncodingType::Compressed; + let mut write_seq_entropy = true; + let codes = |from: usize, count: usize| seq_store.codes.span(from, count); + + if nb_seqs > 0 { + let est_lit_size = estimate_sub_block_size_literal( + literals, + &next_c_block.entropy.huf, + &entropy_metadata.huf_metadata, + write_lit_entropy, + ); + let est_block_size = estimate_sub_block_size_sequences( + histograms, + nb_seqs, + &next_c_block.entropy.fse, + &entropy_metadata.fse_metadata, + write_seq_entropy, + ) + est_lit_size + + ZSTD_BLOCKHEADERSIZE; + let avg_lit_cost = (est_lit_size * BYTESCALE) + .checked_div(nb_literals) + .unwrap_or(BYTESCALE); + let avg_seq_cost = ((est_block_size - est_lit_size) * BYTESCALE) / nb_seqs; + let nb_sub_blocks = + ((est_block_size + target_c_block_size / 2) / target_c_block_size).max(1); + let avg_block_budget = (est_block_size * BYTESCALE) / nb_sub_blocks; + if est_block_size > src.len() { + return Ok(0); + } + + for n in 0..nb_sub_blocks - 1 { + let seq_count = size_block_sequences( + sequences.sub(sp..)?, + avg_block_budget, + avg_lit_cost, + avg_seq_cost, + n == 0, + )?; + // The last sequences go with the last literals. + if sp + seq_count == nb_seqs { + break; + } + let lit_size = count_literals(seq_store, sp, seq_count)?; + let decompressed_size = seq_decompressed_size(seq_store, sp, seq_count, lit_size)?; + let sub_block = compress_sub_block( + &next_c_block.entropy, + entropy_metadata, + sequences.span(sp, seq_count)?, + literals.span(lp, lit_size)?, + codes(sp, seq_count)?, + dst.sub_mut(op..)?, + write_lit_entropy, + write_seq_entropy, + false, + )?; + if sub_block.c_size > 0 && sub_block.c_size < decompressed_size { + ip += decompressed_size; + lp += lit_size; + op += sub_block.c_size; + if sub_block.lit_entropy_written { + write_lit_entropy = false; + } + if sub_block.seq_entropy_written { + write_seq_entropy = false; + } + sp += seq_count; + } + } + } + + { + let lit_size = nb_literals - lp; + let seq_count = nb_seqs - sp; + let decompressed_size = seq_decompressed_size(seq_store, sp, seq_count, lit_size)?; + let sub_block = compress_sub_block( + &next_c_block.entropy, + entropy_metadata, + sequences.span(sp, seq_count)?, + literals.span(lp, lit_size)?, + codes(sp, seq_count)?, + dst.sub_mut(op..)?, + write_lit_entropy, + write_seq_entropy, + last_block, + )?; + if sub_block.c_size > 0 && sub_block.c_size < decompressed_size { + ip += decompressed_size; + op += sub_block.c_size; + if sub_block.lit_entropy_written { + write_lit_entropy = false; + } + if sub_block.seq_entropy_written { + write_seq_entropy = false; + } + sp += seq_count; + } + } + + if write_lit_entropy { + // The table was not written, so the next block cannot use it again. + next_c_block + .entropy + .huf + .copy_from(&prev_c_block.entropy.huf); + } + if write_seq_entropy && need_sequence_entropy_tables(&entropy_metadata.fse_metadata) { + // Neither were these, and there is no going back to the previous ones. + return Ok(0); + } + + if ip < src.len() { + // What did not compress. + op += no_compress_block(dst.sub_mut(op..)?, src.sub(ip..)?, last_block)?; + if sp < nb_seqs { + // The sequences that were dropped did not update the repeat offsets. + let mut rep = prev_c_block.rep; + for (n, seq) in sequences.sub(..sp)?.iter().enumerate() { + let ll0 = seq_store.get_sequence_length(n)?.lit_length == 0; + update_rep(&mut rep, seq.off_base, ll0); + } + next_c_block.rep = rep; + } + } + Ok(op) +} + +/// `ZSTD_compressSuperBlock`: writes the block `src`, of which `seq_store` has the sequences, header included. Returns +/// 0 if it is better left uncompressed. +pub(crate) fn compress_super_block( + seq_store: &mut SeqStore, + prev_c_block: &CompressedBlockState, + next_c_block: &mut CompressedBlockState, + cctx_params: &CCtxParams, + dst: &mut [u8], + src: &[u8], + last_block: bool, +) -> Result { + let mut entropy_metadata = EntropyCTablesMetadata::new(); + let whole = seq_store.whole(); + let histograms = build_block_entropy_stats( + seq_store, + &whole, + &prev_c_block.entropy, + &mut next_c_block.entropy, + cctx_params, + &mut entropy_metadata, + )?; + compress_sub_block_multi( + seq_store, + &histograms, + prev_c_block, + next_c_block, + &entropy_metadata, + cctx_params, + dst, + src, + last_block, + ) +} diff --git a/src/zstd/decompress/block.rs b/src/zstd/decompress/block.rs new file mode 100644 index 000000000000..419c9685e13d --- /dev/null +++ b/src/zstd/decompress/block.rs @@ -0,0 +1,2211 @@ +//! `zstd_decompress_block.c`: one compressed block, which is literals and the sequences that mix them with matches. + +use super::huf::{self, HufDTable}; +use super::seq_table::{ + Cell, Codes, LL_CODES, ML_CODES, OF_CODES, OffsetTable, SeqTable, SeqTableRefs, + SeqTableSources, SeqTables, TableSource, +}; +use core::cell; +use core::hint::{Locality, prefetch_read}; +use core::ops::Range; + +use crate::common::bitstream::BitDStream; +use crate::common::entropy::read_ncount; +use crate::common::fse_decompress::Spread; +use crate::common::{ + BlockType, LL_FSE_LOG, LONGNBSEQ, MAX_LL, MAX_ML, MAX_OFF, MIN_CBLOCK_SIZE, + MIN_LITERALS_FOR_4_STREAMS, MIN_SEQUENCES_SIZE, MINMATCH, ML_FSE_LOG, OFF_FSE_LOG, + REP_START_VALUE, SymbolEncodingType, WILDCOPY_OVERLENGTH, ZSTD_BLOCKHEADERSIZE, + ZSTD_BLOCKSIZE_MAX, ZSTD_REP_NUM, +}; +use crate::error::{Error, Result}; +use crate::util::{ + BUG, SliceExt, append, append_within, copy, make_room, read_le16, read_le24, read_le32, +}; + +/// `blockProperties_t` +pub(crate) struct BlockProperties { + pub(crate) block_type: BlockType, + pub(crate) last_block: bool, + pub(crate) orig_size: u32, +} + +/// `ZSTD_getcBlockSize`: reads a block header, and returns how many bytes of input the block takes after it. +pub(crate) fn get_c_block_size(src: &[u8]) -> Result<(usize, BlockProperties)> { + if src.len() < ZSTD_BLOCKHEADERSIZE { + return Err(Error::SrcSizeWrong); + } + let c_block_header = read_le24(src, 0)?; + let c_size = c_block_header >> 3; + let block_type = match (c_block_header >> 1) & 3 { + 0 => BlockType::Raw, + 1 => BlockType::Rle, + 2 => BlockType::Compressed, + _ => BlockType::Reserved, + }; + let properties = BlockProperties { + block_type, + last_block: c_block_header & 1 != 0, + orig_size: c_size, + }; + match block_type { + BlockType::Rle => Ok((1, properties)), + BlockType::Reserved => Err(Error::CorruptionDetected), + _ => Ok((c_size as usize, properties)), + } +} + +/// `ZSTD_entropyDTables_t` +#[derive(Clone)] +pub(crate) struct EntropyDTables { + pub(crate) seq: SeqTables, + pub(crate) huf_table: HufDTable, + pub(crate) workspace: Spread, +} + +impl EntropyDTables { + pub(crate) const fn new() -> Self { + Self { + seq: SeqTables::new(), + huf_table: HufDTable::new(), + workspace: Spread::new(), + } + } +} + +/// What precedes the contiguous output, for matches to reach into: `dictEnd` and `virtualStart`. +#[derive(Clone, Copy)] +pub(crate) enum ExtDict<'a> { + /// The content of a dictionary. Empty if there is none. + Dict(&'a [u8]), + /// The `len` bytes up to `end` of the buffer that the output goes to, which is the streaming decoder's and has + /// wrapped around. What is further back than `window` is being overwritten. + Within { + end: usize, + len: usize, + window: usize, + }, +} + +impl ExtDict<'_> { + #[inline(always)] + fn len(&self) -> usize { + match *self { + ExtDict::Dict(dict) => dict.len(), + ExtDict::Within { len, .. } => len, + } + } + + /// The largest offset there can be, `in_prefix` bytes after it. In 1.5.7 that is all there is in either case, and + /// a frame whose offsets go beyond its window gets whatever copies that go too far have left there. + #[inline(always)] + fn reach(&self, in_prefix: usize) -> usize { + match *self { + ExtDict::Dict(dict) => in_prefix + dict.len(), + ExtDict::Within { len, window, .. } => window.min(in_prefix + len), + } + } + + /// `memmove(&out[dst], dictEnd - back, len)` + #[inline] + fn copy_to( + &self, + out: &mut OutputBuffer<'_>, + dst: usize, + back: usize, + len: usize, + growth: Growth, + ) -> Result<()> { + match *self { + ExtDict::Dict(dict) => out.write(dst, dict.span(dict.len() - back, len)?, growth), + ExtDict::Within { end, .. } => { + let Growth::Zeros { limit } = growth else { + return Err(BUG); + }; + out.extend_to(dst + len, limit); + out.bytes().move_within(end - back, dst, len) + } + } + } +} + +/// A buffer that output goes to. +pub(crate) enum OutputBuffer<'a> { + Whole(&'a mut [u8]), + /// What has been written so far of a buffer, or somewhat more. It has the capacity for all that a block is let to + /// write. + Used(&'a mut Vec), +} + +impl OutputBuffer<'_> { + #[inline(always)] + pub(crate) fn bytes(&mut self) -> &mut [u8] { + match self { + Self::Whole(bytes) => bytes, + Self::Used(bytes) => bytes, + } + } + + #[inline(always)] + pub(crate) fn reborrow(&mut self) -> OutputBuffer<'_> { + match self { + Self::Whole(bytes) => OutputBuffer::Whole(bytes), + Self::Used(bytes) => OutputBuffer::Used(bytes), + } + } + + /// Makes `bytes` reach `end`, and somewhat further if `limit` allows. + fn extend_to(&mut self, end: usize, limit: usize) { + if let Self::Used(bytes) = self + && bytes.len() < end + { + let ahead = (bytes.len() / 2).clamp(1 << 12, 1 << 17) + 2 * WILDCOPY_OVERLENGTH; + bytes.resize((end + ahead).min(limit), 0); + } + } + + /// Where the `len` bytes from `start` on go. + fn room(&mut self, start: usize, len: usize, growth: Growth) -> Result> { + if let Growth::Zeros { limit } = growth { + self.extend_to(start + len, limit); + } + match self { + Self::Used(bytes) if start + len > bytes.len() => { + if start > bytes.len() || !matches!(growth, Growth::Append) { + return Err(BUG); + } + bytes.truncate(start); + Ok(Room::AtEnd(bytes)) + } + _ => Ok(Room::There(self.bytes().span_mut(start, len)?)), + } + } + + /// `memcpy(&self[start], from, from.len())` + pub(crate) fn write(&mut self, start: usize, from: &[u8], growth: Growth) -> Result<()> { + match self.room(start, from.len(), growth)? { + Room::There(room) => room.write_at(0, from), + Room::AtEnd(bytes) => append(bytes, from), + } + } + + /// `memset(&self[start], byte, len)` + pub(crate) fn fill( + &mut self, + start: usize, + byte: u8, + len: usize, + growth: Growth, + ) -> Result<()> { + match self.room(start, len, growth)? { + Room::There(room) => room.fill(byte), + Room::AtEnd(bytes) if len > bytes.capacity() - start => return Err(BUG), + Room::AtEnd(bytes) => bytes.resize(start + len, byte), + } + Ok(()) + } + + /// [`copy_match_exact`] + fn repeat(&mut self, src: usize, dst: usize, len: usize, growth: Growth) -> Result<()> { + if src >= dst { + return Err(BUG); + } + if let Room::AtEnd(bytes) = self.room(dst, len, growth)? { + return in_whole_periods(dst - src, len, |_, chunk| append_within(bytes, src, chunk)); + } + copy_match_exact(self.bytes(), src, dst, len) + } +} + +/// What to do about a buffer that is `Used` only up to before the end of what is to be written. +#[derive(Clone, Copy)] +pub(crate) enum Growth { + /// Zeros up to there, and somewhat further if `limit` allows. + Zeros { limit: usize }, + /// What is written becomes its end, without being zeroed first. Only where nothing that comes after what is being + /// written is of any use, as in output in one call. + Append, +} + +enum Room<'a> { + There(&'a mut [u8]), + AtEnd(&'a mut Vec), +} + +/// Where a block is written. +pub(crate) struct BlockOutput<'a> { + /// Holds the contiguous output so far, from `prefix_start` to `start`. + pub(crate) buf: OutputBuffer<'a>, + pub(crate) prefix_start: usize, + pub(crate) start: usize, + /// How many bytes from `start` on may be written. + pub(crate) capacity: usize, + pub(crate) ext_dict: ExtDict<'a>, +} + +/// `streaming_operation` +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum Streaming { + No, + Yes, +} + +/// `ZSTD_LITBUFFEREXTRASIZE` +const LIT_BUFFER_EXTRA_SIZE: usize = 1 << 16; + +/// `ZSTD_litLocation_e`. The C code saves memory by keeping the literals of a block in the part of the output that the +/// block has not reached yet. Here they never are, but where they would have been limits how far the block may write. +#[derive(Clone, Copy, PartialEq, Eq)] +enum LitLocation { + NotInDst, + /// Past the largest block, so that is as far as the block may write. + InDst, + /// All but the last `LIT_BUFFER_EXTRA_SIZE` at the end of the block, where the output must not catch up with them. + Split, +} + +/// `ZSTD_allocateLiteralsBuffer` +fn lit_location( + block_size_max: usize, + dst_capacity: usize, + lit_size: usize, + streaming: Streaming, +) -> LitLocation { + if streaming == Streaming::No + && dst_capacity > block_size_max + WILDCOPY_OVERLENGTH + lit_size + WILDCOPY_OVERLENGTH + { + LitLocation::InDst + } else if lit_size <= LIT_BUFFER_EXTRA_SIZE { + LitLocation::NotInDst + } else { + LitLocation::Split + } +} + +/// The literals of a block, once decoded. +struct Literals { + /// How many bytes of the block they take. + c_size: usize, + size: usize, + /// Where they start in the block, if they are stored there as they are with enough after them to copy in chunks. + /// Otherwise they are in the context's buffer. + in_src: Option, + location: LitLocation, +} + +/// The state of a `ZSTD_DCtx` that lasts from block to block. +pub(crate) struct BlockDecoder { + /// There once a block takes it. + scratch: Option>, + pub(crate) rep: [u32; ZSTD_REP_NUM], + pub(crate) seq_sources: SeqTableSources, + pub(crate) huf_source: TableSource, + /// There is a Huffman table to repeat. + pub(crate) lit_entropy: bool, + /// There are sequence tables to repeat. + pub(crate) fse_entropy: bool, + pub(crate) ddict_is_cold: bool, +} + +/// What it takes to decode blocks that are compressed. +struct Scratch { + entropy: EntropyDTables, + /// Grows to the literals of the largest block so far, and `WILDCOPY_OVERLENGTH`. + lit_buffer: Vec, + /// Sequences are decoded that many at a time, after 3 that only stand for the last offsets. + batch: [RawSequence; 3 + 128], + /// Where a block that is all of a small output is decoded: with room to copy in chunks up to its end. + small_output: [u8; Self::MAX_SMALL_OUTPUT + WILDCOPY_OVERLENGTH], + /// All of `small_output` from here on is zeros. + small_output_used: usize, +} + +thread_local! { + /// That of the last decoder that was dropped, without its literals and output, for the next one. Nothing depends on + /// what is in the tables, and it takes longer to clear new ones than to decode a small frame. + static SPARE_SCRATCH: cell::Cell>> = const { cell::Cell::new(None) }; +} + +impl Scratch { + const MAX_SMALL_OUTPUT: usize = 1 << 12; + /// A thread keeps no more than that for its next decoder. + const MAX_SPARE_LIT_BUFFER: usize = 1 << 14; + + fn reserve_literals(&mut self, lit_size: usize) -> Result<()> { + make_room( + &mut self.lit_buffer, + lit_size + WILDCOPY_OVERLENGTH, + ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH, + ) + } + + /// Makes `lit_buffer` long enough for `lit_size` literals. + fn extend_lit_buffer(&mut self, lit_size: usize) -> Result<()> { + let needed = lit_size + WILDCOPY_OVERLENGTH; + self.reserve_literals(lit_size)?; + if self.lit_buffer.len() < needed { + self.lit_buffer.resize(needed, 0); + } + // Literals are copied by more than there are: this much can be read along with the last ones. + self.lit_buffer.sub_mut(lit_size..needed)?.fill(0); + Ok(()) + } + + /// [`Self::extend_lit_buffer`], for literals that `write` writes from the start of the buffer on, all `lit_size` of + /// them: what the buffer gets longer by for them is then not cleared first. + fn write_literals( + &mut self, + lit_size: usize, + write: impl FnOnce(OutputBuffer<'_>, Growth) -> Result<()>, + ) -> Result<()> { + self.reserve_literals(lit_size)?; + write(OutputBuffer::Used(&mut self.lit_buffer), Growth::Append)?; + self.extend_lit_buffer(lit_size) + } +} + +impl Drop for BlockDecoder { + fn drop(&mut self) { + if let Some(mut scratch) = self.scratch.take() { + scratch.lit_buffer.clear(); + let used = core::mem::take(&mut scratch.small_output_used); + if used != 0 + && let Some(used) = scratch.small_output.get_mut(..used) + { + used.fill(0); + } + if scratch.lit_buffer.capacity() > Scratch::MAX_SPARE_LIT_BUFFER { + scratch.lit_buffer = Vec::new(); + } + // Not if the thread is at its end. + let _ = SPARE_SCRATCH.try_with(|spare| spare.set(Some(scratch))); + } + } +} + +impl BlockDecoder { + pub(crate) const SCRATCH_SIZE: usize = size_of::(); + + pub(crate) const fn new() -> Self { + Self { + scratch: None, + rep: REP_START_VALUE, + seq_sources: SeqTableSources::all(TableSource::Local), + huf_source: TableSource::Local, + lit_entropy: false, + fse_entropy: false, + ddict_is_cold: false, + } + } + + fn scratch(&mut self) -> &mut Scratch { + self.scratch.get_or_insert_with(|| { + let spare = SPARE_SCRATCH.try_with(cell::Cell::take).ok().flatten(); + spare.unwrap_or_else(|| { + Box::new(Scratch { + entropy: EntropyDTables::new(), + lit_buffer: Vec::new(), + batch: [RawSequence::EMPTY; _], + small_output: [0; _], + small_output_used: 0, + }) + }) + }) + } + + /// `ZSTD_decodeLiteralsBlock` + fn decode_literals_block( + &mut self, + src: &[u8], + dict_entropy: Option<&EntropyDTables>, + block_size_max: usize, + dst_capacity: usize, + streaming: Streaming, + ) -> Result { + if src.len() < MIN_CBLOCK_SIZE { + return Err(Error::CorruptionDetected); + } + let first = src.at(0)?; + let lit_enc_type = SymbolEncodingType::from_bits(first); + let lhl_code = (first >> 2) & 3; + let expected_write_size = block_size_max.min(dst_capacity); + + match lit_enc_type { + SymbolEncodingType::Repeat | SymbolEncodingType::Compressed => { + if lit_enc_type == SymbolEncodingType::Repeat && !self.lit_entropy { + return Err(Error::DictionaryCorrupted); + } + if src.len() < 5 { + return Err(Error::CorruptionDetected); + } + let lhc = read_le32(src, 0)? as usize; + let (single_stream, lh_size, lit_size, lit_c_size) = match lhl_code { + 2 => (false, 4, (lhc >> 4) & 0x3FFF, lhc >> 18), + 3 => ( + false, + 5, + (lhc >> 4) & 0x3FFFF, + (lhc >> 22) + (usize::from(src.at(4)?) << 10), + ), + _ => (lhl_code == 0, 3, (lhc >> 4) & 0x3FF, (lhc >> 14) & 0x3FF), + }; + if lit_size > block_size_max { + return Err(Error::CorruptionDetected); + } + if !single_stream && lit_size < MIN_LITERALS_FOR_4_STREAMS { + return Err(Error::LiteralsHeaderWrong); + } + if lit_c_size + lh_size > src.len() { + return Err(Error::CorruptionDetected); + } + if expected_write_size < lit_size { + return Err(Error::DstSizeTooSmall); + } + let location = lit_location(block_size_max, dst_capacity, lit_size, streaming); + let compressed = src.span(lh_size, lit_c_size)?; + + let huf_source = self.huf_source; + let scratch = self.scratch(); + scratch.extend_lit_buffer(lit_size)?; + let Scratch { + entropy, + lit_buffer, + .. + } = scratch; + let dst = lit_buffer.sub_mut(..lit_size)?; + let huf_result = if lit_enc_type == SymbolEncodingType::Repeat { + let table = match (huf_source, dict_entropy) { + (TableSource::Dict, Some(dict)) => &dict.huf_table, + _ => { + entropy.huf_table.prepare( + lit_size, + compressed.len(), + !single_stream, + )?; + &entropy.huf_table + } + }; + if single_stream { + huf::decompress_1x_using_dtable(dst, compressed, table) + } else { + huf::decompress_4x_using_dtable(dst, compressed, table) + } + } else if single_stream { + huf::decompress_1x1( + &mut entropy.huf_table, + dst, + compressed, + &mut entropy.workspace, + ) + } else { + huf::decompress_4x_huf_only( + &mut entropy.huf_table, + dst, + compressed, + &mut entropy.workspace, + ) + }; + if huf_result.is_err() { + return Err(Error::CorruptionDetected); + } + + self.lit_entropy = true; + if lit_enc_type == SymbolEncodingType::Compressed { + self.huf_source = TableSource::Local; + } + Ok(Literals { + c_size: lit_c_size + lh_size, + size: lit_size, + in_src: None, + location, + }) + } + + SymbolEncodingType::Basic => { + let (lh_size, lit_size) = match lhl_code { + 1 => (2, usize::from(read_le16(src, 0)? >> 4)), + 3 => { + if src.len() < 3 { + return Err(Error::CorruptionDetected); + } + (3, (read_le24(src, 0)? >> 4) as usize) + } + _ => (1, usize::from(first >> 3)), + }; + if lit_size > block_size_max { + return Err(Error::CorruptionDetected); + } + if expected_write_size < lit_size { + return Err(Error::DstSizeTooSmall); + } + if lh_size + lit_size + WILDCOPY_OVERLENGTH > src.len() { + if lit_size + lh_size > src.len() { + return Err(Error::CorruptionDetected); + } + let literals = src.span(lh_size, lit_size)?; + self.scratch() + .write_literals(lit_size, |mut to, growth| to.write(0, literals, growth))?; + return Ok(Literals { + c_size: lh_size + lit_size, + size: lit_size, + in_src: None, + location: lit_location(block_size_max, dst_capacity, lit_size, streaming), + }); + } + Ok(Literals { + c_size: lh_size + lit_size, + size: lit_size, + in_src: Some(lh_size), + location: LitLocation::NotInDst, + }) + } + + SymbolEncodingType::Rle => { + let (lh_size, lit_size) = match lhl_code { + 1 => { + if src.len() < 3 { + return Err(Error::CorruptionDetected); + } + (2, usize::from(read_le16(src, 0)? >> 4)) + } + 3 => { + if src.len() < 4 { + return Err(Error::CorruptionDetected); + } + (3, (read_le24(src, 0)? >> 4) as usize) + } + _ => (1, usize::from(first >> 3)), + }; + if lit_size > block_size_max { + return Err(Error::CorruptionDetected); + } + if expected_write_size < lit_size { + return Err(Error::DstSizeTooSmall); + } + let byte = src.at(lh_size)?; + self.scratch().write_literals(lit_size, |mut to, growth| { + to.fill(0, byte, lit_size, growth) + })?; + Ok(Literals { + c_size: lh_size + 1, + size: lit_size, + in_src: None, + location: lit_location(block_size_max, dst_capacity, lit_size, streaming), + }) + } + } + } + + /// `ZSTD_decodeSeqHeaders`: returns how many sequences there are, and the size of the header. + fn decode_seq_headers(&mut self, src: &[u8]) -> Result<(usize, usize)> { + if src.len() < MIN_SEQUENCES_SIZE { + return Err(Error::SrcSizeWrong); + } + let mut ip = 0usize; + let mut nb_seq = usize::from(src.at(ip)?); + ip += 1; + if nb_seq > 0x7F { + if nb_seq == 0xFF { + if ip + 2 > src.len() { + return Err(Error::SrcSizeWrong); + } + nb_seq = usize::from(read_le16(src, ip)?) + LONGNBSEQ; + ip += 2; + } else { + if ip >= src.len() { + return Err(Error::SrcSizeWrong); + } + nb_seq = ((nb_seq - 0x80) << 8) + usize::from(src.at(ip)?); + ip += 1; + } + } + + if nb_seq == 0 { + if ip != src.len() { + return Err(Error::CorruptionDetected); + } + return Ok((0, ip)); + } + + if ip + 1 > src.len() { + return Err(Error::SrcSizeWrong); + } + let modes = src.at(ip)?; + if modes & 3 != 0 { + return Err(Error::CorruptionDetected); + } + ip += 1; + + let fse_entropy = self.fse_entropy; + self.scratch(); + let Some(scratch) = &mut self.scratch else { + return Err(BUG); + }; + let EntropyDTables { seq, workspace, .. } = &mut scratch.entropy; + ip += build_seq_table( + &mut seq.ll, + &mut self.seq_sources.ll, + SymbolEncodingType::from_bits(modes >> 6), + MAX_LL, + LL_FSE_LOG, + src.sub(ip..)?, + &LL_CODES, + workspace, + fse_entropy, + ) + .map_err(|_| Error::CorruptionDetected)?; + ip += build_seq_table( + &mut seq.of, + &mut self.seq_sources.of, + SymbolEncodingType::from_bits(modes >> 4), + MAX_OFF, + OFF_FSE_LOG, + src.sub(ip..)?, + &OF_CODES, + workspace, + fse_entropy, + ) + .map_err(|_| Error::CorruptionDetected)?; + ip += build_seq_table( + &mut seq.ml, + &mut self.seq_sources.ml, + SymbolEncodingType::from_bits(modes >> 2), + MAX_ML, + ML_FSE_LOG, + src.sub(ip..)?, + &ML_CODES, + workspace, + fse_entropy, + ) + .map_err(|_| Error::CorruptionDetected)?; + + Ok((nb_seq, ip)) + } + + /// `ZSTD_decompressBlock_internal`: returns the size of the block once decompressed. + pub(crate) fn decompress_block( + &mut self, + out: BlockOutput<'_>, + src: &[u8], + dict_entropy: Option<&EntropyDTables>, + block_size_max: usize, + streaming: Streaming, + ) -> Result { + let BlockOutput { + buf, + prefix_start, + start, + capacity, + ext_dict, + } = out; + if src.len() > block_size_max { + return Err(Error::SrcSizeWrong); + } + + let literals = + self.decode_literals_block(src, dict_entropy, block_size_max, capacity, streaming)?; + let sequences = src.sub(literals.c_size..)?; + + let total_history_size = + start - prefix_start + ext_dict.len() + capacity.min(block_size_max); + let mut use_prefetch_decoder = self.ddict_is_cold; + let (nb_seq, seq_h_size) = self.decode_seq_headers(sequences)?; + let sequences = sequences.sub(seq_h_size..)?; + + if capacity == 0 && nb_seq > 0 { + return Err(Error::DstSizeTooSmall); + } + + self.scratch(); + let Self { + scratch: Some(scratch), + rep, + .. + } = self + else { + return Err(BUG); + }; + let Scratch { + entropy, + lit_buffer, + batch, + small_output, + small_output_used, + } = &mut **scratch; + let dict_tables = dict_entropy.map(|dict| &dict.seq); + let is_history_long = total_history_size > MIN_FAR_HISTORY; + let tables = if nb_seq <= MAX_ONE_BY_ONE && !use_prefetch_decoder && !is_history_long { + entropy.seq.pick(self.seq_sources, dict_tables) + } else { + entropy.seq.gather(&mut self.seq_sources, dict_tables) + }; + + // With matches far back in a long history, it pays to know the sequences ahead of time. Which of two problems in + // a damaged block is reported depends on that too. + if !use_prefetch_decoder && is_history_long && nb_seq > 8 { + const MIN_SHARE: u32 = 7; + use_prefetch_decoder = long_offset_share(tables.of) >= MIN_SHARE; + } + self.ddict_is_cold = false; + if nb_seq > 0 { + self.fse_entropy = true; + } + + let oend = if literals.location == LitLocation::InDst { + start + block_size_max + WILDCOPY_OVERLENGTH + } else { + start + capacity + }; + let split = literals.location == LitLocation::Split; + let lits = match literals.in_src { + Some(start) => src.sub(start..)?, + None => lit_buffer.as_slice(), + }; + // How many would be in the output buffer, if some. + let in_dst_size = if split { + literals.size - LIT_BUFFER_EXTRA_SIZE + } else { + literals.size + }; + // A block that has that room where it goes needs no other place. + let is_small = start == prefix_start + && capacity <= Scratch::MAX_SMALL_OUTPUT + && capacity < block_size_max + WILDCOPY_OVERLENGTH + && literals.location == LitLocation::NotInDst + && matches!(ext_dict, ExtDict::Dict(_)); + let (out, copy_to) = match buf { + OutputBuffer::Used(_) if streaming == Streaming::Yes => (buf, None), + buf if is_small => { + let used = capacity + WILDCOPY_OVERLENGTH; + *small_output_used = used.max(*small_output_used); + ( + OutputBuffer::Whole(small_output.sub_mut(..used)?), + Some((buf, start)), + ) + } + OutputBuffer::Whole(bytes) => (OutputBuffer::Whole(bytes.sub_mut(..oend)?), None), + // Sequences that are few can be long, and then what they make is best appended. A block whose literals are + // in `src` is let to write all there is room for. A frame before may have been let to write further. + OutputBuffer::Used(bytes) + if nb_seq > MAX_ONE_BY_ONE + && oend - start <= 2 * (block_size_max + WILDCOPY_OVERLENGTH) + || oend - start <= Scratch::MAX_SMALL_OUTPUT + || bytes.len() > oend => + { + if bytes.len() < oend { + bytes.resize(oend, 0); + } + (OutputBuffer::Whole(bytes.sub_mut(..oend)?), None) + } + OutputBuffer::Used(_) => (buf, None), + }; + let in_one_call = streaming == Streaming::No && matches!(out, OutputBuffer::Used(_)); + let (start, prefix_start, oend) = match copy_to { + Some(_) => (0, 0, capacity), + None => (start, prefix_start, oend), + }; + let mut exec = Executor { + out, + in_one_call, + oend, + op: start, + prefix_start, + ext_dict, + lits: lits.sub(..in_dst_size + WILDCOPY_OVERLENGTH)?, + split, + lits_extra: lits.sub(in_dst_size..literals.size + WILDCOPY_OVERLENGTH)?, + // `WILDCOPY_OVERLENGTH` short of how far the block is expected to write. + lit_end_in_dst: (start + capacity.min(block_size_max)) + .wrapping_sub(WILDCOPY_OVERLENGTH), + }; + + if nb_seq > 0 { + let seq_state = SeqState::new(sequences, tables, rep)?; + let prev_offset = if use_prefetch_decoder { + exec.run::(&seq_state, nb_seq, batch)? + } else { + exec.run::(&seq_state, nb_seq, batch)? + }; + *rep = prev_offset.map(|offset| offset as u32); + } + exec.last_literals()?; + let size = exec.op - start; + if let Some((mut to, start)) = copy_to { + to.write(start, exec.out.bytes().sub(..size)?, Growth::Append)?; + } + Ok(size) + } +} + +/// `ZSTD_buildSeqTable`: returns the size of the table's description in `src`. +fn build_seq_table( + table: &mut SeqTable, + source: &mut TableSource, + encoding_type: SymbolEncodingType, + max: u32, + max_log: u32, + src: &[u8], + codes: &Codes, + workspace: &mut Spread, + flag_repeat_table: bool, +) -> Result { + match encoding_type { + SymbolEncodingType::Rle => { + let Some(&symbol) = src.first() else { + return Err(Error::SrcSizeWrong); + }; + if u32::from(symbol) > max { + return Err(Error::CorruptionDetected); + } + table.build_rle(codes.at(usize::from(symbol))?); + *source = TableSource::Local; + Ok(1) + } + SymbolEncodingType::Basic => { + *source = TableSource::Predefined; + Ok(0) + } + SymbolEncodingType::Repeat => { + if !flag_repeat_table { + return Err(Error::CorruptionDetected); + } + Ok(0) + } + SymbolEncodingType::Compressed => { + let header = read_ncount(&mut workspace.counts, max, src) + .map_err(|_| Error::CorruptionDetected)?; + if header.table_log > max_log { + return Err(Error::CorruptionDetected); + } + table.build( + header.max_symbol_value as usize + 1, + codes, + header.table_log, + workspace, + )?; + *source = TableSource::Local; + Ok(header.size) + } + } +} + +/// `ZSTD_getOffsetInfo`: how many of 256 offset codes have more than 22 bits. +fn long_offset_share(of_table: &OffsetTable) -> u32 { + let count = of_table + .symbols() + .iter() + .filter(|symbol| symbol.nb_additional_bits > 22) + .count() as u32; + count << (OFF_FSE_LOG - of_table.table_log) +} + +/// `seq_t` +#[derive(Clone, Copy, Default)] +struct Sequence { + lit_length: u32, + match_length: u32, + offset: usize, +} + +/// A sequence whose offset is as it is coded: 1 to 3 for one of the last offsets, or 3 more than the offset. +#[derive(Clone, Copy)] +struct RawSequence { + offset_value: u32, + lit_length: u32, + /// Above `MINMATCH`. + match_length: u32, +} + +impl RawSequence { + const EMPTY: Self = Self { + offset_value: 0, + lit_length: 0, + match_length: 0, + }; +} + +/// Where matches are, for `ZSTD_prefetchMatch`. +struct History { + prefix_start: *const u8, + dict_end: *const u8, + /// How long the prefix is. + position: usize, +} + +impl History { + /// Of the match of the next sequence, if its offset is not one of the last. + #[inline(always)] + fn prefetch(&mut self, lit_length: u32, match_length: u32, offset_value: u32) { + self.position += lit_length as usize; + let offset = offset_value.saturating_sub(3) as usize; + let base = if offset > self.position { + self.dict_end + } else { + self.prefix_start + }; + let from = if offset == 0 { + self.dict_end + } else { + base.wrapping_add(self.position).wrapping_sub(offset) + }; + prefetch_read(from, Locality::L1); + prefetch_read(from.wrapping_add(64), Locality::L1); + self.position += (match_length + MINMATCH) as usize; + } + + /// The same, whatever its offset. + #[inline(always)] + fn prefetch_match(&mut self, sequence: Sequence) { + self.prefetch( + sequence.lit_length, + sequence.match_length - MINMATCH, + (sequence.offset as u32).saturating_add(3), + ); + } +} + +/// A copy of the start of a stream after zeroes, where it can be read as if its start were far. That it was not shows +/// afterwards. +struct PaddedStart([u8; Self::PADDING + Self::MAX_SIZE]); + +impl PaddedStart { + const PADDING: usize = 32; + const MAX_SIZE: usize = 224; + + const fn new() -> Self { + Self([0; _]) + } + + /// Whether little enough is left of `bit_d`. + fn fits(bit_d: &BitDStream<'_>) -> bool { + bit_d.bytes().len() <= Self::MAX_SIZE && bit_d.bits_consumed <= 64 + } + + /// `bit_d` on the copy, which is new. It fits. + fn set(&mut self, bit_d: &BitDStream<'_>) -> Option> { + let src = bit_d.bytes(); + self.0 + .get_mut(Self::PADDING..Self::PADDING + src.len())? + .copy_from_slice(src); + // A stream of less than 8 bytes counts what it lacks as consumed. + let lacking = 8usize.saturating_sub(src.len()) as u32 * 8; + let pos = Self::PADDING + src.len() - 8; + BitDStream::resume(&self.0, pos, bit_d.bits_consumed.checked_sub(lacking)?).ok() + } + + /// How many bits are left to read of the copy and of the zeroes. + fn bits_left(bit_d: &BitDStream<'_>) -> usize { + (bit_d.bytes().len() * 8).saturating_sub(bit_d.bits_consumed as usize) + } + + fn is_reached(bit_d: &BitDStream<'_>) -> bool { + Self::bits_left(bit_d) == Self::PADDING * 8 + } + + fn is_passed(bit_d: &BitDStream<'_>) -> bool { + Self::bits_left(bit_d) < Self::PADDING * 8 + } +} + +/// Up to how many sequences it is not worth it to decode them by the batch. +const MAX_ONE_BY_ONE: usize = 32; +/// How many sequences `ZSTD_decompressSequencesLong_body` decodes ahead of the one that it executes. +const ADVANCED_SEQS: usize = 8; +/// A history that is longer is not all in a cache. +const MIN_FAR_HISTORY: usize = 1 << 24; + +/// `seqState_t` +#[derive(Clone, Copy)] +struct SeqState<'a> { + bit_d: BitDStream<'a>, + state_ll: usize, + state_off: usize, + state_ml: usize, + tables: SeqTableRefs<'a>, + prev_offset: [usize; ZSTD_REP_NUM], +} + +impl<'a> SeqState<'a> { + fn new(src: &'a [u8], tables: SeqTableRefs<'a>, rep: &[u32; ZSTD_REP_NUM]) -> Result { + let mut bit_d = BitDStream::new(src).map_err(|_| Error::CorruptionDetected)?; + // `ZSTD_initFseState` + let mut init = |table_log: u32| { + let state = bit_d.read_bits(table_log) as usize; + bit_d.reload(); + state + }; + let state_ll = init(tables.ll.table_log); + let state_off = init(tables.of.table_log); + let state_ml = init(tables.ml.table_log); + Ok(Self { + bit_d, + state_ll, + state_off, + state_ml, + tables, + prev_offset: rep.map(|offset| offset as usize), + }) + } + + /// Decodes sequences for as long as the start of the stream is far. Returns how many that is. + /// With `prefetch`, `history` follows the output, and what matches refer to is on its way to the cache. + /// + /// `more_bits` is for when it cannot be predicted which lengths come with additional bits. + #[inline(never)] + fn decode_batch( + &mut self, + prefetch: bool, + more_bits: bool, + out: &mut [RawSequence], + history: &mut History, + ) -> Result { + match (prefetch, more_bits) { + (false, false) => self.decode_batch_as::(out, history), + (false, true) => self.decode_batch_as::(out, history), + (true, false) => self.decode_batch_as::(out, history), + (true, true) => self.decode_batch_as::(out, history), + } + } + + #[inline(always)] + fn decode_batch_as( + &mut self, + out: &mut [RawSequence], + history: &mut History, + ) -> Result { + /// The 64 bits that end at `position`, turned so that the next one to read is the highest. Those that are read + /// then go to the other end. + #[inline(always)] + fn load(containers: &[u8], position: u32) -> Option { + // Wraps around less than `FAR` bytes from the start. + let index = (position >> 3).wrapping_sub(FAR as u32) as usize; + let container = containers.get(index..index + 8)?; + let container = u64::from_le_bytes(container.try_into().ok()?); + Some(container.rotate_left(!position & 7)) + } + /// A sequence takes at most 12 bytes. + const FAR: usize = 24; + + let src = self.bit_d.bytes(); + // A block is not that large. + if src.len() < FAR || src.len() >= 1 << 28 || self.bit_d.bits_consumed > 64 { + return Ok(0); + } + // How many bits are left to read, and 7. + let mut position = src.len() as u32 * 8 + 7 - self.bit_d.bits_consumed; + let containers = src.get(FAR - 8..).unwrap_or_default(); + let Some(tables) = self.tables.side_by_side else { + return Ok(0); + }; + let mut state_ll = self.state_ll as u32 & 511; + let mut state_ml = self.state_ml as u32 & 511; + let mut state_off = self.state_off as u32 & 255; + + let room = out.len(); + let mut slots = out.iter_mut(); + let mut unused = 0; + for slot in slots.by_ref() { + let Some(mut bits) = load(containers, position) else { + unused = 1; + break; + }; + // Nothing waits for a check, as it would for a mask. + let (Some(ll_cell), Some(ml_cell), Some(of_cell)) = ( + tables.ll.table.get(state_ll as usize), + tables.ml.table.get(state_ml as usize), + tables.of.table.get(state_off as usize), + ) else { + unused = 1; + break; + }; + let of_bits = u32::from(of_cell.nb_additional_bits); + let ml_bits = u32::from(ml_cell.nb_additional_bits); + let ll_bits = u32::from(ll_cell.nb_additional_bits); + let ll_nb_bits = u32::from(ll_cell.nb_bits); + let ml_nb_bits = u32::from(ml_cell.nb_bits); + let of_nb_bits = u32::from(of_cell.nb_bits); + let nb_all_bits = + (ml_bits + ll_bits + ll_nb_bits + ml_nb_bits) + (of_bits + of_nb_bits); + + let mut match_length = u32::from(ml_cell.base_value); + let mut lit_length = u32::from(ll_cell.base_value); + let first_value = of_cell.first_value; + let mut loaded = bits; + bits = bits.rotate_left(of_bits); + let offset_value = first_value + (bits as u32 & (first_value - 1)); + if MORE_BITS || ml_bits + ll_bits != 0 { + bits = bits.rotate_left(ml_bits); + match_length = tables.length(ml_cell.base_value, ml_cell.nb_additional_bits, bits); + // All has to fit in 57 bits with the 26 of the next states. + if of_bits + ml_bits + ll_bits >= 31 { + let position = position - of_bits - ml_bits; + bits = load(src, position + (FAR as u32 - 8) * 8).unwrap_or(0); + loaded = bits.rotate_right(of_bits + ml_bits); + } + bits = bits.rotate_left(ll_bits); + lit_length = tables.length(ll_cell.base_value, ll_cell.nb_additional_bits, bits); + } + *slot = RawSequence { + offset_value, + lit_length, + match_length, + }; + if PREFETCH { + history.prefetch(lit_length, match_length, offset_value); + } + + position -= nb_all_bits; + bits = bits.rotate_left(ll_nb_bits); + state_ll = + u32::from(ll_cell.next_state) + (bits as u32 & u32::from(ll_cell.state_mask)); + bits = bits.rotate_left(ml_nb_bits); + state_ml = + u32::from(ml_cell.next_state) + (bits as u32 & u32::from(ml_cell.state_mask)); + // This is read last, and it is what the first bits of the next sequence depend on. So it does not wait + // for the rest to be read. + state_off = u32::from(of_cell.next_state) + + (loaded.rotate_left(nb_all_bits) as u32 & u32::from(of_cell.state_mask)); + } + + let count = room - slots.len() - unused; + if count > 0 { + self.bit_d = BitDStream::resume(src, (position >> 3) as usize - 8, !position & 7)?; + self.state_ll = state_ll as usize; + self.state_ml = state_ml as usize; + self.state_off = state_off as usize; + } + Ok(count) + } + + /// Decodes the last sequence, far from the start of the stream. + fn decode_last(&mut self) -> Option { + let ll_cell = self.tables.ll.table.get(self.state_ll)?; + let ml_cell = self.tables.ml.table.get(self.state_ml)?; + let of_cell = self.tables.of.table.get(self.state_off)?; + let of_bits = u32::from(of_cell.nb_additional_bits); + let ml_bits = u32::from(ml_cell.nb_additional_bits); + let ll_bits = u32::from(ll_cell.nb_additional_bits); + let offset_value = of_cell.first_value + self.bit_d.read_bits(of_bits) as u32; + let match_length = ml_cell.base_value() + self.bit_d.read_bits(ml_bits) as u32; + self.bit_d.reload(); + let lit_length = ll_cell.base_value() + self.bit_d.read_bits(ll_bits) as u32; + Some(RawSequence { + offset_value, + lit_length, + match_length, + }) + } + + /// `ZSTD_decodeSequence` + #[inline(always)] + fn decode_sequence(&mut self, is_last_seq: bool) -> Sequence { + let ll_d_info = self.tables.ll.get(self.state_ll); + let ml_d_info = self.tables.ml.get(self.state_ml); + let of_d_info = self.tables.of.get(self.state_off); + let mut match_length = ml_d_info.base_value + MINMATCH; + let mut lit_length = ll_d_info.base_value; + let of_base = of_d_info.base_value as usize; + let ll_bits = u32::from(ll_d_info.nb_additional_bits); + let ml_bits = u32::from(ml_d_info.nb_additional_bits); + let of_bits = u32::from(of_d_info.nb_additional_bits); + let total_bits = ll_bits + ml_bits + of_bits; + + let [prev0, prev1, prev2] = self.prev_offset; + let offset; + if of_bits > 1 { + offset = of_base + self.bit_d.read_bits_fast(of_bits) as usize; + self.prev_offset = [offset, prev0, prev1]; + } else { + let ll0 = ll_d_info.base_value == 0; + if of_bits == 0 { + if ll0 { + offset = prev1; + self.prev_offset = [prev1, prev0, prev2]; + } else { + offset = prev0; + } + } else { + let code = of_base + usize::from(ll0) + self.bit_d.read_bits_fast(1) as usize; + let temp = match code { + 1 => prev1, + 2 => prev2, + _ => prev0.wrapping_sub(1), + }; + // 0 is not valid: it becomes an offset that no history is long enough for. + offset = temp.wrapping_sub(usize::from(temp == 0)); + self.prev_offset = [offset, prev0, if code != 1 { prev1 } else { prev2 }]; + } + } + + if ml_bits > 0 { + match_length += self.bit_d.read_bits_fast(ml_bits) as u32; + } + const ACCUMULATOR_MIN: u32 = 57; + if total_bits >= ACCUMULATOR_MIN - (LL_FSE_LOG + ML_FSE_LOG + OFF_FSE_LOG) { + self.bit_d.reload(); + } + if ll_bits > 0 { + lit_length += self.bit_d.read_bits_fast(ll_bits) as u32; + } + + if !is_last_seq { + // `ZSTD_updateFseStateWithDInfo` + self.state_ll = usize::from(ll_d_info.next_state) + + self.bit_d.read_bits(u32::from(ll_d_info.nb_bits)) as usize; + self.state_ml = usize::from(ml_d_info.next_state) + + self.bit_d.read_bits(u32::from(ml_d_info.nb_bits)) as usize; + self.state_off = usize::from(of_d_info.next_state) + + self.bit_d.read_bits(u32::from(of_d_info.nb_bits)) as usize; + self.bit_d.reload(); + } + + Sequence { + lit_length, + match_length, + offset, + } + } +} + +const NO_EXT: u8 = 0; +const EXT_DICT: u8 = 1; +const EXT_WITHIN: u8 = 2; + +/// Copies `N` bytes within `buf`, between positions that do not overflow. Whether they are in `buf`. +#[inline(always)] +fn copy_chunk(buf: &mut [u8], src: usize, dst: usize) -> bool { + let Some(Ok(chunk)) = buf.get(src..src + N).map(<[u8; N]>::try_from) else { + return false; + }; + let Some(to) = buf.get_mut(dst..dst + N) else { + return false; + }; + to.copy_from_slice(&chunk); + true +} + +/// The least offset value of a match that can be copied 16 bytes at a time. +const MIN_FAR_OFFSET_VALUE: u32 = 16 + 3; +/// How many literals a sequence can have for [`Executor::exec_batch`]. +const MAX_BATCH_LITERALS: u32 = 32; +/// How long a match from closer can be for [`copy_short_close_match`]. +const MAX_CLOSE_MATCH: u32 = 32; + +/// Whether `sequences[index]` and the few after it look like they are for [`Executor::exec_batch`] with `CLOSE`. +#[inline(always)] +fn starts_close_run(sequences: &[RawSequence], index: usize) -> bool { + let Some(run) = sequences.get(index..index + 4) else { + return false; + }; + (run.first()).is_some_and(|first| first.offset_value > 3) + && run.iter().all(|next| { + next.offset_value < MIN_FAR_OFFSET_VALUE + && next.lit_length <= MAX_BATCH_LITERALS + && next.match_length + MINMATCH <= MAX_CLOSE_MATCH + }) +} + +/// [`Executor::copy_close_match`] for what is short: `offset` is below 16, and up to 7 bytes more than `len` are written. +/// Whether all that is in `buf`. +#[inline(always)] +fn copy_short_close_match(buf: &mut [u8], dst: usize, offset: usize, len: usize) -> bool { + /// For each offset below 8: what leaves as many bytes, what repeats them, and how many of the 8 are whole periods. + const REPEAT: [(u64, u64, usize); 8] = [ + (0, 0, 8), + (0xFF, 0x0101_0101_0101_0101, 8), + (0xFFFF, 0x0001_0001_0001_0001, 8), + (0xFF_FFFF, 0x0001_0000_0100_0001, 6), + (0xFFFF_FFFF, 0x0000_0001_0000_0001, 8), + (0xFF_FFFF_FFFF, 1 | 1 << 40, 5), + (0xFFFF_FFFF_FFFF, 1 | 1 << 48, 6), + (0xFF_FFFF_FFFF_FFFF, 1 | 1 << 56, 7), + ]; + let Some(src) = dst.checked_sub(offset).filter(|_| offset != 0) else { + return false; + }; + let mut done = 0; + if let Some(&(mask, repeat, step)) = REPEAT.get(offset) { + let Some(Ok(bytes)) = buf.get(src..src + 8).map(<[u8; 8]>::try_from) else { + return false; + }; + let pattern = (u64::from_le_bytes(bytes) & mask).wrapping_mul(repeat); + while done < len { + let Some(to) = buf.get_mut(dst + done..dst + done + 8) else { + return false; + }; + to.copy_from_slice(&pattern.to_le_bytes()); + done += step; + } + } else { + while done < len { + if !copy_chunk::<8>(buf, src + done, dst + done) { + return false; + } + done += 8; + } + } + true +} + +/// `buf[dst + i] = buf[src + i]`, one after the other, so that a match repeats itself where it overlaps its copy. +/// `src` is less than `dst`. +#[cold] +#[inline(never)] +fn copy_match_exact(buf: &mut [u8], src: usize, dst: usize, len: usize) -> Result<()> { + in_whole_periods(dst - src, len, |done, chunk| { + buf.move_within(src, dst + done, chunk) + }) +} + +/// For a match of `len` bytes `offset` bytes back: `copy` gets how much of it is copied and how much to copy next, from +/// where the match starts. What is copied so far is whole periods, so it can be copied again, twice as much each time. +#[inline(always)] +fn in_whole_periods( + offset: usize, + len: usize, + mut copy: impl FnMut(usize, usize) -> Result<()>, +) -> Result<()> { + let mut done = 0; + while done < len { + let chunk = (offset + done).min(len - done); + copy(done, chunk)?; + done += chunk; + } + Ok(()) +} + +/// Makes the offset value of `sequences[index]`, which refers to one of the last offsets, what it refers to, or 0 if that +/// is not valid, and returns it. The last offsets are those of the 3 sequences before, which are rewritten so that the +/// same holds for the next sequence. +#[inline(always)] +fn resolve_offset(sequences: &mut [RawSequence], index: usize) -> u32 { + let Some([last2, last1, last0, this]) = index + .checked_sub(3) + .and_then(|first| sequences.get_mut(first..)) + .and_then(<[RawSequence]>::first_chunk_mut) + else { + return 0; + }; + let rep0 = last0.offset_value; + let rep1 = last1.offset_value; + let rep2 = last2.offset_value; + let reps = match this.offset_value - 1 + u32::from(this.lit_length == 0) { + 0 => [rep0, rep1, rep2], + 1 => [rep1, rep0, rep2], + 2 => [rep2, rep0, rep1], + // An offset of 0 is not valid. + _ if rep0 == 1 + 3 => [0, rep0, rep1], + _ => [rep0 - 1, rep0, rep1], + }; + [this.offset_value, last0.offset_value, last1.offset_value] = reps; + this.offset_value +} + +/// What [`Executor::exec_batch`] works on. Positions in these fit in 32 bits, so that the compiler sees that nothing added +/// to them overflows. +struct BatchBuffers<'buf, 'a> { + /// Where in the output buffer `buf` starts. + start: usize, + buf: &'buf mut [u8], + lits: &'a [u8], + /// What comes right before `buf`, if anything of the common kind does: this or, of `buf`, `within`. + dict: &'a [u8], + within: Range, + /// [`ExtDict::Within::window`] + window: usize, + /// Where in `buf` the literals would be, which the output must not catch up with. What gets close to them is not of + /// the common kind. That is where `buf` ends, unless `within` is after it. + lit_ptr: usize, +} + +/// Turns sequences into output. +struct Executor<'buf, 'a> { + /// What is beyond `oend` is room for copies in chunks. + out: OutputBuffer<'buf>, + /// All the output is written in one call, so nothing that comes after `op` is of any use. + in_one_call: bool, + /// Where the output of the block has to end. + oend: usize, + op: usize, + prefix_start: usize, + ext_dict: ExtDict<'a>, + /// The literals that are left, up to `litBufferEnd`, and `WILDCOPY_OVERLENGTH` bytes after them. + lits: &'a [u8], + /// `litBufferLocation == ZSTD_split`: `lits` would be in the output buffer. + split: bool, + /// The literals that come after `lits` then, likewise. + lits_extra: &'a [u8], + /// Where in `buf` `lits` would end then. + lit_end_in_dst: usize, +} + +impl<'a> Executor<'_, 'a> { + /// How `out` is to grow for `size` bytes that are written with exact copies: if with zeros, up to `limit` at most. + #[inline(always)] + fn growth(&self, size: usize, limit: usize) -> Growth { + /// Less is likely followed by more of the kind, which is copied in chunks where there is room. + const MIN_SIZE_TO_APPEND: usize = 256; + if self.in_one_call && size >= MIN_SIZE_TO_APPEND { + Growth::Append + } else { + Growth::Zeros { limit } + } + } + + /// `litBufferEnd - litPtr` + #[inline(always)] + fn lits_left(&self) -> usize { + self.lits.len().saturating_sub(WILDCOPY_OVERLENGTH) + } + + /// `litPtr`, if `split`: where in the output buffer `lits` would start. + #[inline(always)] + fn lit_ptr(&self) -> usize { + self.lit_end_in_dst.wrapping_sub(self.lits_left()) + } + + /// The same, for a while. A loop works on a copy that is a local variable and whose address nothing takes, which + /// can stay in registers. What is behind a reference is written and read again for every sequence. + #[inline(always)] + fn fork(&mut self) -> Executor<'_, 'a> { + Executor { + out: self.out.reborrow(), + ..*self + } + } + + #[inline(always)] + fn join(&mut self, (op, lits, split): (usize, &'a [u8], bool)) { + (self.op, self.lits, self.split) = (op, lits, split); + } + + #[inline(always)] + fn exec_sequence_end(&mut self, sequence: Sequence) -> Result<()> { + let mut fork = self.fork(); + fork.exec_sequence_end_cold(sequence)?; + let state = (fork.op, fork.lits, fork.split); + self.join(state); + Ok(()) + } + + /// `ZSTD_execSequenceEnd` and `ZSTD_execSequenceEndSplitLitBuffer`: nothing is written past the sequence. Also what + /// `ZSTD_execSequence` does of a match that starts before the prefix. + #[cold] + #[inline(never)] + fn exec_sequence_end_cold(&mut self, sequence: Sequence) -> Result<()> { + let lit_length = sequence.lit_length as usize; + let mut match_length = sequence.match_length as usize; + let offset = sequence.offset; + let mut op = self.op; + + if lit_length + match_length > self.oend - op { + return Err(Error::DstSizeTooSmall); + } + if lit_length > self.lits_left() { + return Err(Error::CorruptionDetected); + } + if self.split { + let lit_ptr = self.lit_ptr(); + if op > lit_ptr && op < lit_ptr.wrapping_add(lit_length) { + return Err(Error::DstSizeTooSmall); + } + } + + let growth = self.growth(lit_length + match_length, self.oend); + self.out.write(op, self.lits.sub(..lit_length)?, growth)?; + self.lits = self.lits.sub(lit_length..)?; + op += lit_length; + self.op = op + match_length; + + let in_prefix = op - self.prefix_start; + let mut match_pos = op.wrapping_sub(offset); + if offset > in_prefix { + if offset > self.ext_dict.reach(in_prefix) { + return Err(Error::CorruptionDetected); + } + let back = offset - in_prefix; + if back >= match_length { + return self + .ext_dict + .copy_to(&mut self.out, op, back, match_length, growth); + } + // The match spans the two segments. + self.ext_dict + .copy_to(&mut self.out, op, back, back, growth)?; + op += back; + match_length -= back; + match_pos = self.prefix_start; + } + self.out.repeat(match_pos, op, match_length, growth) + } + + /// `ZSTD_execSequence` and `ZSTD_execSequenceSplitLitBuffer`, if there is room to copy in chunks and the match is + /// in the prefix. Otherwise `None`, and what is changed is only output that the sequence is to overwrite. + /// + /// The conditions are such that the compiler sees that most of the `?` cannot fail. + #[inline(always)] + fn exec_sequence_fast(&mut self, sequence: Sequence) -> Option<()> { + let lit_length = sequence.lit_length as usize; + let match_length = sequence.match_length as usize; + let offset = sequence.offset; + let op = self.op; + let lit_ptr = self.lit_ptr(); + + let room = self.out.bytes().get_mut(op..)?; + if lit_length + match_length + WILDCOPY_OVERLENGTH > room.len() + || lit_length + WILDCOPY_OVERLENGTH > self.lits.len() + { + return None; + } + if SPLIT { + // The output must not catch up with the literals. + let i_lit_end = lit_ptr.wrapping_add(lit_length); + if op + lit_length + match_length + WILDCOPY_OVERLENGTH > i_lit_end { + return None; + } + } + + *room.first_chunk_mut::<16>()? = *self.lits.first_chunk()?; + if lit_length > 16 { + copy( + room.get_mut(16..lit_length)?, + self.lits.get(16..lit_length)?, + ) + .ok()?; + } + + let o_lit_end = op + lit_length; + let (before, room) = self.out.bytes().split_at_mut_checked(o_lit_end)?; + let in_prefix = o_lit_end - self.prefix_start; + let from = if offset > in_prefix { + // All of it in a dictionary, and then some to copy in chunks. + let ExtDict::Dict(dict) = self.ext_dict else { + return None; + }; + let back = offset - in_prefix; + if back < match_length + 16 { + return None; + } + dict.get(dict.len().checked_sub(back)?..)? + } else { + before.get(before.len().checked_sub(offset)?..)? + }; + if offset >= 16 { + // `ZSTD_wildcopy(ZSTD_no_overlap)` + *room.first_chunk_mut::<16>()? = *from.first_chunk()?; + if match_length > 16 { + let rounded = (match_length + 15) & !15; + if offset >= rounded { + copy(room.get_mut(16..rounded)?, from.get(16..rounded)?).ok()?; + } else { + let buf = self.out.bytes(); + copy_match_exact(buf, o_lit_end - offset, o_lit_end, match_length).ok()?; + } + } + } else { + self.copy_close_match(o_lit_end, offset, match_length) + .ok()?; + } + + self.lits = self.lits.get(lit_length..)?; + self.op = o_lit_end + match_length; + Some(()) + } + + /// `ZSTD_overlapCopy8` and `ZSTD_wildcopy(ZSTD_overlap_src_before_dst)`: a match less than 16 bytes back, which is + /// not 0. Up to 16 bytes are written past its end. + #[inline(always)] + fn copy_close_match(&mut self, op: usize, offset: usize, match_length: usize) -> Result<()> { + /// The largest multiple of an offset that is at most 16. + const STEPS: [u8; 16] = [ + 16, 16, 16, 15, 16, 15, 12, 14, 16, 9, 10, 11, 12, 13, 14, 15, + ]; + let buf = self.out.bytes(); + // The `offset` bytes before the match, again and again. + let mut pattern = u128::from_le_bytes(*buf.array_at(op - offset)?); + let mut filled = offset as u32 * 8; + pattern &= !(u128::MAX << filled); + while filled < 128 { + pattern |= pattern << filled; + filled *= 2; + } + let pattern = pattern.to_le_bytes(); + let step = usize::from(STEPS.at(offset)?); + let room = buf.span_mut(op, match_length + 16)?; + let mut done = 0; + while let Some(to) = room.get_mut(done..done + 16) + && done < match_length + { + to.copy_from_slice(&pattern); + done += step; + } + Ok(()) + } + + #[inline(always)] + fn exec_sequence(&mut self, sequence: Sequence) -> Result<()> { + match self.exec_sequence_fast::(sequence) { + Some(()) => Ok(()), + None => self.exec_sequence_end(sequence), + } + } + + /// The first sequence whose literals do not end among those that would be in the output buffer. + #[inline(always)] + fn exec_sequence_leaving_split(&mut self, sequence: Sequence) -> Result<()> { + let mut fork = self.fork(); + fork.exec_sequence_leaving_split_cold(sequence)?; + let state = (fork.op, fork.lits, fork.split); + self.join(state); + Ok(()) + } + + #[cold] + #[inline(never)] + fn exec_sequence_leaving_split_cold(&mut self, mut sequence: Sequence) -> Result<()> { + let leftover_lit = self.lits_left(); + if leftover_lit > 0 { + if leftover_lit > self.oend - self.op { + return Err(Error::DstSizeTooSmall); + } + let growth = self.growth(leftover_lit, self.oend); + self.out + .write(self.op, self.lits.sub(..leftover_lit)?, growth)?; + sequence.lit_length -= leftover_lit as u32; + self.op += leftover_lit; + } + self.split = false; + self.lits = self.lits_extra; + self.exec_sequence::(sequence) + } + + #[inline(always)] + fn exec_sequence_maybe_split(&mut self, sequence: Sequence) -> Result<()> { + if !self.split { + self.exec_sequence::(sequence) + } else if sequence.lit_length as usize > self.lits_left() { + self.exec_sequence_leaving_split(sequence) + } else { + self.exec_sequence::(sequence) + } + } + + #[inline(always)] + fn batch_buffers(&mut self) -> BatchBuffers<'_, 'a> { + const MAX_BACK: usize = 3 << 30; + let start = self.prefix_start.max(self.op.saturating_sub(MAX_BACK)); + let is_whole = start == self.prefix_start; + let (dict, within, window) = match self.ext_dict { + ExtDict::Dict(dict) if is_whole => (dict, 0..0, 0), + ExtDict::Within { end, len, window } if is_whole && end - len >= start => { + (&[][..], end - len - start..end - start, window) + } + _ => (&[][..], 0..0, 0), + }; + let lits = self.lits.get(..u32::MAX as usize).unwrap_or(self.lits); + let lit_ptr = if self.split { + self.lit_ptr() + } else { + usize::MAX + }; + let end = if within.is_empty() { + lit_ptr + } else { + usize::MAX + }; + let buf = self.out.bytes(); + let end = buf.len().min(start + u32::MAX as usize).min(end); + let buf = buf.get_mut(start..end).unwrap_or_default(); + BatchBuffers { + start, + buf, + lits, + dict, + within, + window, + lit_ptr: lit_ptr.saturating_sub(start), + } + } + + /// Executes `sequences` from `index` on, for as long as nothing is close to its end and they are of the common kind. + /// Returns the index of the first that is not, whose offset value then is no reference to one of the last offsets: + /// it is 0 if that is not valid. + /// + /// The last offsets are those of the 3 sequences before, which are rewritten for that where it takes it. + /// + /// `EXT` tells which kind [`Self::ext_dict`] is of, if matches in it are of the common kind. So are, with `CLOSE`, short + /// matches from less than 16 bytes back. + #[inline(never)] + fn exec_batch( + &mut self, + sequences: &mut [RawSequence], + mut index: usize, + ) -> usize { + let op = self.op; + let BatchBuffers { + start, + buf, + lits, + dict, + within, + window, + lit_ptr, + } = self.batch_buffers(); + let mut op = op.wrapping_sub(start) as u32; + let mut lit_pos = 0u32; + + while let Some(raw) = sequences.get(index) { + let lit_length = raw.lit_length as usize; + let match_length = raw.match_length as usize + MINMATCH as usize; + let mut offset_value = raw.offset_value; + if offset_value < MIN_FAR_OFFSET_VALUE { + if offset_value <= 3 { + offset_value = resolve_offset(sequences, index); + } + if offset_value < MIN_FAR_OFFSET_VALUE + && !(CLOSE && offset_value > 3 && match_length <= MAX_CLOSE_MATCH as usize) + { + break; + } + } + + let o_lit_end = op as usize + lit_length; + let o_match_end = o_lit_end + match_length; + let lit_end = lit_pos as usize + lit_length; + if o_match_end + WILDCOPY_OVERLENGTH > buf.len() + || EXT == EXT_WITHIN && o_match_end + WILDCOPY_OVERLENGTH > lit_ptr + || lit_end + WILDCOPY_OVERLENGTH > lits.len() + || lit_length > MAX_BATCH_LITERALS as usize + || (EXT == NO_EXT || match_length > 64 || offset_value < MIN_FAR_OFFSET_VALUE) + && offset_value as usize > o_lit_end + 3 + { + break; + } + let (Some(to), Some(from)) = ( + buf.get_mut(op as usize..op as usize + 32), + lits.get(lit_pos as usize..lit_pos as usize + 32), + ) else { + break; + }; + if CLOSE && offset_value < MIN_FAR_OFFSET_VALUE { + to.copy_from_slice(from); + let offset = offset_value as usize - 3; + if !copy_short_close_match(buf, o_lit_end, offset, match_length) { + break; + } + op = o_match_end as u32; + lit_pos = lit_end as u32; + index += 1; + continue; + } + if EXT == EXT_DICT && offset_value as usize > o_lit_end + 3 { + let back = offset_value as usize - 3 - o_lit_end; + let Some(match_pos) = dict.len().checked_sub(back) else { + break; + }; + let Some(([first, second], _)) = dict + .get(match_pos..match_pos + 64) + .map(<[u8]>::as_chunks::<32>) + else { + break; + }; + to.copy_from_slice(from); + let Some(to) = buf.get_mut(o_lit_end..o_lit_end + 32) else { + break; + }; + to.copy_from_slice(first); + if match_length > 32 { + let Some(to) = buf.get_mut(o_lit_end + 32..o_lit_end + 64) else { + break; + }; + to.copy_from_slice(second); + } + op = o_match_end as u32; + lit_pos = lit_end as u32; + index += 1; + continue; + } + let mut match_end = o_lit_end + 3; + if EXT == EXT_WITHIN && offset_value as usize > o_lit_end + 3 { + let back = offset_value as usize - 3 - o_lit_end; + // All that gets copied is of the older part, which is more than the match. + if back > within.len() || back < 64 || offset_value as usize - 3 > window { + break; + } + match_end += within.end; + } + to.copy_from_slice(from); + let match_pos = match_end - offset_value as usize; + let is_copied = if match_length > 64 { + copy_match_exact(buf, match_pos, o_lit_end, match_length).is_ok() + } else { + copy_chunk::<16>(buf, match_pos, o_lit_end) + && copy_chunk::<16>(buf, match_pos + 16, o_lit_end + 16) + && (match_length <= 32 + || copy_chunk::<16>(buf, match_pos + 32, o_lit_end + 32) + && copy_chunk::<16>(buf, match_pos + 48, o_lit_end + 48)) + }; + if !is_copied { + break; + } + + op = o_match_end as u32; + lit_pos = lit_end as u32; + index += 1; + } + + self.op = op as usize + start; + self.lits = self.lits.get(lit_pos as usize..).unwrap_or_default(); + index + } + + /// Executes `sequences[index]` and those after it that are not for [`Self::exec_batch`]. Returns the index of the + /// first that is, or that [`starts_close_run`]. + #[inline(never)] + fn exec_others(&mut self, sequences: &mut [RawSequence], first: usize) -> Result { + let mut exec = self.fork(); + let mut index = first; + while let Some(&raw) = sequences.get(index) { + let mut offset_value = raw.offset_value; + if (1..=3).contains(&offset_value) { + offset_value = resolve_offset(sequences, index); + } + if index != first + && raw.lit_length <= MAX_BATCH_LITERALS + && (offset_value >= MIN_FAR_OFFSET_VALUE || starts_close_run(sequences, index)) + { + break; + } + exec.exec_sequence_maybe_split(Sequence { + lit_length: raw.lit_length, + match_length: raw.match_length + MINMATCH, + offset: (offset_value as usize).wrapping_sub(3), + })?; + index += 1; + } + let exec = (exec.op, exec.lits, exec.split); + self.join(exec); + Ok(index) + } + + /// `ZSTD_decompressSequencesLong_body` the way it is in C: a sequence is executed once [`ADVANCED_SEQS`] more are + /// decoded and their matches asked for. It is for long matches, which [`Self::run`] takes one at a time too, and knows + /// nothing of literals that would be in the output. Returns the last offsets. + #[inline(never)] + fn run_long_matches_ahead( + &mut self, + seq_state: &SeqState<'_>, + nb_seq: usize, + ) -> Result<[usize; ZSTD_REP_NUM]> { + let mut ahead = [Sequence::default(); ADVANCED_SEQS]; + let ahead = ahead.sub_mut(..nb_seq.min(ADVANCED_SEQS))?; + let mut history = self.history(); + let mut exec = self.fork(); + let mut seq_state: SeqState<'_> = *seq_state; + let mut remaining = nb_seq; + for sequence in &mut *ahead { + *sequence = seq_state.decode_sequence(remaining == 1); + remaining -= 1; + history.prefetch_match(*sequence); + } + let mut oldest = 0; + while remaining > 0 { + let next = seq_state.decode_sequence(remaining == 1); + remaining -= 1; + history.prefetch_match(next); + let slot = ahead.at_mut(oldest)?; + exec.exec_sequence::(*slot)?; + *slot = next; + oldest = (oldest + 1) % ADVANCED_SEQS; + } + if !seq_state.bit_d.is_at_end() { + return Err(Error::CorruptionDetected); + } + ahead.rotate_left(oldest); + for &sequence in &*ahead { + exec.exec_sequence::(sequence)?; + } + let exec = (exec.op, exec.lits, exec.split); + self.join(exec); + Ok(seq_state.prev_offset) + } + + /// `ZSTD_decompressSequences_body`, `ZSTD_decompressSequences_bodySplitLitBuffer` and, with `LONG`, + /// `ZSTD_decompressSequencesLong_body`. Returns the last offsets. + fn run( + &mut self, + seq_state: &SeqState<'_>, + nb_seq: usize, + batch: &mut [RawSequence], + ) -> Result<[usize; ZSTD_REP_NUM]> { + if LONG + && self.op - self.prefix_start + self.ext_dict.len() > MIN_FAR_HISTORY + && seq_state.tables.matches_are_long() + && !self.split + { + return self.run_long_matches_ahead(seq_state, nb_seq); + } + // With `LONG`, the C code decodes that many sequences ahead, so it sees whether the stream ends where it should + // before it executes the last ones. + let mut ahead = [Sequence::default(); ADVANCED_SEQS]; + let ahead = ahead.sub_mut(..if LONG { nb_seq.min(ADVANCED_SEQS) } else { 0 })?; + let mut start = None; + // There until decoding goes on from it. + let mut start = (!LONG).then_some(&mut start); + let mut seq_state: SeqState<'_> = *seq_state; + + let more_bits = seq_state.tables.lengths_often_have_more_bits(); + + let mut remaining = nb_seq; + // Long copies go on while the next sequences are decoded, if that is what comes next. And a batch of a few + // sequences is not worth it. This loop knows nothing of literals that would be in the output. + if (seq_state.tables.matches_are_long() || nb_seq <= MAX_ONE_BY_ONE && !LONG) && !self.split + { + let mut exec = self.fork(); + if LONG { + while remaining > ahead.len() { + let sequence = seq_state.decode_sequence(remaining == 1); + remaining -= 1; + exec.exec_sequence::(sequence)?; + } + } else { + // As for `fork`: `decode_batch` takes the address of `seq_state`. With `LONG`, HTML after a cold dictionary + // takes 7% longer with the copy. + let mut state = seq_state; + while remaining > ahead.len() { + let sequence = state.decode_sequence(remaining == 1); + remaining -= 1; + exec.exec_sequence::(sequence)?; + } + seq_state = state; + } + let exec = (exec.op, exec.lits, exec.split); + self.join(exec); + } + // Where decoding went on from a copy of the start of the stream, if it did. + let mut left_at = None; + while remaining > ahead.len() { + let mut count = 0; + if PaddedStart::fits(&seq_state.bit_d) + && let Some(start) = start.take() + && let Some(bit_d) = start.insert(PaddedStart::new()).set(&seq_state.bit_d) + { + left_at = Some((seq_state, remaining)); + seq_state.bit_d = bit_d; + } + // The last sequence is followed by no state. + let room = (batch.len() - 4).min(remaining - ahead.len().max(1)); + if room > 0 { + let history = &mut self.history(); + let room = batch.span_mut(3, room)?; + count = seq_state.decode_batch(LONG, more_bits, room, history)?; + } + if left_at.is_some() + && count + 1 == remaining + && let Some(last) = seq_state.decode_last() + { + batch.put(3 + count, last)?; + count += 1; + } + let mut sequence = None; + if count == 0 { + sequence = Some(seq_state.decode_sequence(remaining == 1)); + } + if let Some((state, at)) = left_at + && PaddedStart::is_passed(&seq_state.bit_d) + { + // What the C code makes of that depends on more than the bits. + seq_state = state; + for _ in remaining..at { + seq_state.decode_sequence(false); + } + left_at = None; + continue; + } + match sequence { + Some(sequence) => { + remaining -= 1; + self.exec_sequence_maybe_split(sequence)?; + } + None => { + remaining -= count; + self.exec_all(batch.sub_mut(..3 + count)?, &mut seq_state.prev_offset)?; + } + } + } + let history = &mut self.history(); + for sequence in &mut *ahead { + *sequence = seq_state.decode_sequence(remaining == 1); + remaining -= 1; + history.prefetch_match(*sequence); + } + let is_at_end = match left_at { + Some(_) => PaddedStart::is_reached(&seq_state.bit_d), + None => seq_state.bit_d.is_at_end(), + }; + if !is_at_end { + return Err(Error::CorruptionDetected); + } + for &sequence in &*ahead { + self.exec_sequence_maybe_split(sequence)?; + } + Ok(seq_state.prev_offset) + } + + fn history(&mut self) -> History { + let buf = self.out.bytes().as_ptr(); + History { + prefix_start: buf.wrapping_add(self.prefix_start), + dict_end: match self.ext_dict { + ExtDict::Dict(dict) => dict.as_ptr_range().end, + ExtDict::Within { end, .. } => buf.wrapping_add(end), + }, + position: self.op - self.prefix_start, + } + } + + /// Executes `sequences` but the first 3, which are there to stand for the last offsets. + fn exec_all( + &mut self, + sequences: &mut [RawSequence], + prev_offset: &mut [usize; ZSTD_REP_NUM], + ) -> Result<()> { + for (last, offset) in sequences.iter_mut().zip(prev_offset.iter().rev()) { + last.offset_value = *offset as u32 + 3; + } + match self.ext_dict { + ExtDict::Dict(dict) if !dict.is_empty() => self.exec_in_turns::(sequences), + ExtDict::Within { len, .. } if len != 0 => self.exec_in_turns::(sequences), + _ => self.exec_in_turns::(sequences), + }?; + for (last, offset) in sequences.iter().rev().zip(prev_offset) { + *offset = last.offset_value as usize - 3; + } + Ok(()) + } + + /// Executes `sequences` but the first 3. `EXT` is for [`Self::exec_batch`]. + #[inline(always)] + fn exec_in_turns(&mut self, sequences: &mut [RawSequence]) -> Result<()> { + let mut index = 3; + let mut close = false; + while index < sequences.len() { + index = if close { + self.exec_batch::(sequences, index) + } else { + self.exec_batch::(sequences, index) + }; + if index < sequences.len() { + index = self.exec_others(sequences, index)?; + close = (sequences.get(index)) + .is_some_and(|next| next.offset_value < MIN_FAR_OFFSET_VALUE); + } + } + Ok(()) + } + + /// The literals after the last sequence. + fn last_literals(&mut self) -> Result<()> { + let extra = if self.split { self.lits_extra } else { &[] }; + let parts = [self.lits, extra] + .map(|part| part.get(..part.len().saturating_sub(WILDCOPY_OVERLENGTH))); + let parts = parts.map(Option::unwrap_or_default); + let size = parts.iter().map(|part| part.len()).sum::(); + if size > self.oend - self.op { + return Err(Error::DstSizeTooSmall); + } + let growth = self.growth(size, self.op + size); + for part in parts { + if !part.is_empty() { + self.out.write(self.op, part, growth)?; + self.op += part.len(); + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn short_close_matches_repeat_what_is_before_them() -> Result<()> { + const DST: usize = 20; + let start: Vec = (0..DST as u8).collect(); + for offset in 1..16 { + for len in 1..=MAX_CLOSE_MATCH as usize { + let mut expected = start.clone(); + for i in 0..len { + expected.push(expected.at(DST - offset + i)?); + } + let mut buf = [start.as_slice(), &[0xEE; MAX_CLOSE_MATCH as usize + 8]].concat(); + assert!(copy_short_close_match(&mut buf, DST, offset, len)); + assert_eq!( + *buf.sub(..DST + len)?, + *expected, + "{len} bytes from {offset} back" + ); + assert!(buf.sub(DST + len + 7..)?.iter().all(|&byte| byte == 0xEE)); + let too_short = buf.sub_mut(..DST + len - 1)?; + assert!(!copy_short_close_match(too_short, DST, offset, len)); + } + } + assert!(!copy_short_close_match(&mut [0; 64], 3, 4, 5)); + assert!(!copy_short_close_match(&mut [0; 64], 3, 0, 5)); + Ok(()) + } + + #[test] + fn matches_in_a_dictionary_are_of_the_common_kind_wherever_the_output_starts() -> Result<()> { + let dict: Vec = (0..200).collect(); + let lits = [b'l'; 64]; + let sequence = |lit_length, match_length, offset: u32| RawSequence { + offset_value: offset + 3, + lit_length, + match_length: match_length - MINMATCH, + }; + for prefix_start in [0, 50] { + let mut sequences = [ + RawSequence::EMPTY, + RawSequence::EMPTY, + RawSequence::EMPTY, + sequence(0, 10, 150), + sequence(2, 40, 12 + 100), + sequence(1, 5, 4), + sequence(0, 8, 58 + 200), + ]; + let expected = [ + dict.sub(50..60)?, + b"ll", + dict.sub(100..140)?, + b"l", + &[137, 138, 139, b'l', 137], + dict.sub(..8)?, + ] + .concat(); + let mut buf = [0xEE; 250]; + let mut exec = Executor { + out: OutputBuffer::Whole(&mut buf), + in_one_call: true, + oend: 250, + op: prefix_start, + prefix_start, + ext_dict: ExtDict::Dict(&dict), + lits: &lits, + split: false, + lits_extra: &[], + lit_end_in_dst: 0, + }; + assert_eq!(exec.exec_batch::(&mut sequences, 3), 5); + assert_eq!(exec.exec_batch::(&mut sequences, 5), 7); + assert_eq!(exec.op, prefix_start + expected.len()); + assert_eq!( + *buf.sub(prefix_start..prefix_start + expected.len())?, + *expected + ); + assert!(buf.sub(..prefix_start)?.iter().all(|&byte| byte == 0xEE)); + } + Ok(()) + } +} diff --git a/src/zstd/decompress/huf.rs b/src/zstd/decompress/huf.rs new file mode 100644 index 000000000000..05e3970b21dc --- /dev/null +++ b/src/zstd/decompress/huf.rs @@ -0,0 +1,1346 @@ +//! Huffman decoding of literals. +//! +//! Streams that end exactly where they should decode to the same thing whichever way it is done, so that is done in +//! the way that is fastest here. The others are left to [`huf_exact`](super::huf_exact). + +use super::huf_exact::{self, TableType, select_decoder_x2}; +use crate::common::entropy::huf_read_stats_wksp; +use crate::common::fse_decompress::Spread; +use crate::common::{HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_MAX}; +use crate::error::{Error, Result}; +use crate::util::{BUG, SliceExt, read_le16}; + +const NB_SYMBOLS: usize = HUF_SYMBOLVALUE_MAX as usize + 1; +const NB_RANKS: usize = HUF_TABLELOG_MAX as usize + 1; +const TABLE_SIZE: usize = 1 << HUF_TABLELOG_MAX; + +/// The description of a Huffman code. +#[derive(Clone)] +pub(super) struct Code { + /// A symbol of weight `w` has a code of `table_log + 1 - w` bits, or none if that is 0. + pub(super) weights: [u8; NB_SYMBOLS], + /// How many symbols have each weight. + pub(super) rank_stats: [u32; NB_RANKS], + pub(super) nb_symbols: u32, + pub(super) table_log: u32, +} + +/// No fewer bits are looked up at once. +const MIN_LOG: u32 = 8; +/// How many are if there are enough literals to make up for the time that it takes to fill the table. +const FAST_LOG: u32 = 11; +/// How many literals are enough for that. +const FAST_MIN_LITERALS: usize = 2048; +/// From how many literals on the larger table is considered. +const LARGE_MIN_LITERALS: usize = 1 << 14; +/// The longest code of a table that yields one symbol to a lookup. +const SINGLE_MAX_LOG: u32 = 11; +/// Few literals are not worth a closer look at their code: from how many bits to a literal on few lookups of at most +/// `SINGLE_SMALL_LOG` bits would yield more than one. +const SINGLE_SMALL_MIN_BITS: usize = 5; +const SINGLE_SMALL_LOG: u32 = 9; + +impl Code { + /// How many symbols a lookup of `log` bits yields, in 65536ths, if symbols are as frequent as their codes are short. + /// Without `thirds` a third symbol does not count, which takes less time. + fn symbols_per_lookup(&self, log: u32, thirds: bool) -> u64 { + let table_log = self.table_log as usize; + // For each number of bits, out of `1 << table_log`, how many symbols have a code no longer than that. + let mut one = [0u64; NB_RANKS + 1]; + let mut share = [0u64; NB_RANKS + 1]; + let mut total = 0; + for nb_bits in 1..=log as usize { + let of_these = match (table_log + 1).saturating_sub(nb_bits) { + 0 => 0, + weight => { + u64::from(self.rank_stats.get(weight).copied().unwrap_or(0)) << (weight - 1) + } + }; + total += of_these; + if let (Some(share), Some(one)) = (share.get_mut(nb_bits), one.get_mut(nb_bits)) { + *share = of_these; + *one = total; + } + } + // The same for two symbols and for three, out of that squared and cubed. + let after = |fits: &[u64; NB_RANKS + 1], room: usize| -> u64 { + (1..room) + .map(|nb_bits| { + share.get(nb_bits).copied().unwrap_or(0) + * fits.get(room - nb_bits).copied().unwrap_or(0) + }) + .sum() + }; + if !thirds { + return (1 << 16) + ((after(&one, log as usize) << 16) >> (2 * table_log)); + } + let mut two = [0u64; NB_RANKS + 1]; + for (room, two) in two.iter_mut().enumerate() { + *two = after(&one, room); + } + let three = after(&two, log as usize); + let two = two.get(log as usize).copied().unwrap_or(0); + (1 << 16) + ((two << 16) >> (2 * table_log)) + ((three << 16) >> (3 * table_log)) + } + + /// How many bits to look up at once to decode `nb_literals`. + fn best_log(&self, nb_literals: usize) -> u32 { + if nb_literals < FAST_MIN_LITERALS { + return self.table_log.max(MIN_LOG); + } + // The larger table takes twice the time to fill, half of the cache, and a refill for 4 lookups and not 5. + if self.table_log > FAST_LOG + || nb_literals >= LARGE_MIN_LITERALS + && self.symbols_per_lookup(FAST_LOG + 1, true) * 4 + > self.symbols_per_lookup(FAST_LOG, true) * 5 + { + return FAST_LOG + 1; + } + FAST_LOG + } + + /// How many bits to look up at once to decode `nb_literals` from four streams of `nb_bytes` in all, and whether + /// that is sooner done one to a lookup. + fn best_form(&self, nb_literals: usize, nb_bytes: usize) -> (u32, bool) { + let log = self.best_log(nb_literals); + let single = if nb_literals < FAST_MIN_LITERALS { + log <= SINGLE_SMALL_LOG && 8 * nb_bytes >= SINGLE_SMALL_MIN_BITS * nb_literals + } else { + // A lookup that can yield several symbols takes a quarter more time. + self.table_log <= SINGLE_MAX_LOG && self.symbols_per_lookup(log, false) * 4 < 5 << 16 + }; + if single { + (self.table_log.max(MIN_LOG), true) + } else { + (log, false) + } + } +} + +/// What a lookup yields. From the lowest byte on: up to 3 symbols, then how many bits they take in 6 bits and how many +/// they are above. What is after the last symbol is not always 0: see [`single_entry`]. +type Entry = u32; + +const COUNT_SHIFT: u32 = 6; + +/// What a symbol adds to an entry that has `index` symbols. +const fn entry(symbol: u8, index: u32, nb_bits: u32) -> Entry { + ((symbol as u32) << (8 * index)) | ((nb_bits + (1 << COUNT_SHIFT)) << 24) +} + +/// An entry of one symbol that also has its highest byte and then the symbol in the middle, where there is no symbol: +/// one read yields both, the former where a shift takes its count from. `Streams::decode_most_single` reads only that. +const fn single_entry(symbol: u8, nb_bits: u32) -> Entry { + let entry = entry(symbol, 0, nb_bits); + entry | (entry >> 16) | ((symbol as u32) << 16) +} + +/// For each of `symbols`, `1 << run_log` entries that are `base` and the symbol times `unit`. +fn fill_runs(out: &mut [Entry], symbols: &[u8], run_log: u32, base: Entry, unit: Entry) { + #[inline(always)] + fn runs(out: &mut [Entry], symbols: &[u8], base: Entry, unit: Entry) { + let (out, _) = out.as_chunks_mut::(); + for (run, &symbol) in out.iter_mut().zip(symbols) { + *run = [base + u32::from(symbol) * unit; RUN]; + } + } + match run_log { + 0 => runs::<1>(out, symbols, base, unit), + 1 => runs::<2>(out, symbols, base, unit), + 2 => runs::<4>(out, symbols, base, unit), + 3 => runs::<8>(out, symbols, base, unit), + _ => { + for (run, &symbol) in out.chunks_exact_mut(1 << run_log).zip(symbols) { + run.fill(base + u32::from(symbol) * unit); + } + } + } +} + +/// The symbols of a code in the order of their codes, which is by increasing weight. +struct Sorted { + symbols: [u8; NB_SYMBOLS], + /// Where the symbols of each weight start in `symbols`. + class_start: [usize; NB_CLASSES], + /// Where their entries start in a table. + position: [usize; NB_CLASSES], + /// How many bits index that table. + log: u32, + table_log: u32, + /// The weight of the shortest code. + max_weight: usize, +} + +/// More than there are weights, and a power of 2. +const NB_CLASSES: usize = 16; + +impl Sorted { + fn new(code: &Code, log: u32) -> Result { + let mut class_start = [0usize; NB_CLASSES]; + let mut position = [0usize; NB_CLASSES]; + let mut nb_sorted = 0usize; + let mut filled = 0usize; + let mut max_weight = 1; + for (weight, &count) in code.rank_stats.iter().enumerate().skip(1) { + class_start.put(weight, nb_sorted)?; + position.put(weight, filled)?; + if count != 0 { + max_weight = weight; + } + nb_sorted += count as usize; + filled += ((count as usize) << (weight - 1)) << (log - code.table_log); + } + class_start.sub_mut(NB_RANKS..)?.fill(nb_sorted); + position.sub_mut(NB_RANKS..)?.fill(filled); + + // The symbols that have no code go after the others. + let mut symbols = [0u8; NB_SYMBOLS]; + let mut cursor = class_start; + cursor.put(0, nb_sorted)?; + let mut place = |symbol: usize, weight: u8| { + if let Some(at) = cursor.get_mut(usize::from(weight) % NB_CLASSES) { + if let Some(slot) = symbols.get_mut(*at) { + *slot = symbol as u8; + } + *at += 1; + } + }; + let (eights, others) = code + .weights + .sub(..code.nb_symbols as usize)? + .as_chunks::<8>(); + for (index, eight) in eights.iter().enumerate() { + if *eight != [0; 8] { + for (offset, &weight) in eight.iter().enumerate() { + place(8 * index + offset, weight); + } + } + } + for (offset, &weight) in others.iter().enumerate() { + place(8 * eights.len() + offset, weight); + } + Ok(Self { + symbols, + class_start, + position, + log, + table_log: code.table_log, + max_weight, + }) + } + + fn class(&self, weight: usize) -> Result<&[u8]> { + self.symbols + .sub(self.class_start.at(weight)?..self.class_start.at(weight + 1)?) + } + + fn nb_bits(&self, weight: usize) -> u32 { + self.table_log + 1 - weight as u32 + } + + fn shortest(&self) -> u32 { + self.nb_bits(self.max_weight) + } + + /// The lowest weight of a code of at most `room` bits. + fn min_weight(&self, room: u32) -> usize { + (self.table_log as usize + 1) + .saturating_sub(room as usize) + .max(1) + } + + /// Where the entries of the symbols of a weight start in a table that `room` bits index. + fn position(&self, weight: usize, room: u32) -> Result { + Ok(self.position.at(weight)? >> (self.log - room)) + } + + /// Fills a table that `room` bits index: each entry is `base`, which has `index` symbols, and the symbol that the + /// bits start with if they hold all of its code. `next` is what the bits after that add: the table for `n` bits is + /// there from `1 << n` on. + fn fill( + &self, + out: &mut [Entry], + room: u32, + base: Entry, + index: u32, + next: Option<&[Entry]>, + ) -> Result<()> { + let min_weight = self.min_weight(room); + out.sub_mut(..self.position(min_weight, room)?)?.fill(base); + for weight in min_weight..=self.max_weight { + let symbols = self.class(weight)?; + let room_after = room - self.nb_bits(weight); + let out = out.span_mut(self.position(weight, room)?, symbols.len() << room_after)?; + let base = base + entry(0, index, self.nb_bits(weight)); + match next { + Some(next) if room_after >= self.shortest() => { + let size = 1usize << room_after; + let next = next.span(size, size)?; + for (out, &symbol) in out.chunks_exact_mut(size).zip(symbols) { + let base = base + (u32::from(symbol) << (8 * index)); + for (entry, next) in out.iter_mut().zip(next) { + *entry = base + next; + } + } + } + _ => fill_runs(out, symbols, room_after, base, 1 << (8 * index)), + } + } + Ok(()) + } +} + +/// `HUF_DTable` +#[derive(Clone)] +pub(crate) struct HufDTable { + code: Code, + /// The kind of table that the C code has. + table_type: TableType, + /// How many bits `table` is indexed with. + log: u32, + /// Every entry of `table` is a [`single_entry`]. + single: bool, + /// [`Code::best_form`] for `FAST_MIN_LITERALS` and for `LARGE_MIN_LITERALS`, once it is asked for. + best_forms: [Option<(u32, bool)>; 2], + table: [Entry; TABLE_SIZE], +} + +impl HufDTable { + pub(crate) const fn new() -> Self { + Self { + code: Code { + weights: [0; NB_SYMBOLS], + rank_stats: [0; NB_RANKS], + nb_symbols: 0, + table_log: 0, + }, + table_type: TableType::X1, + log: 0, + single: false, + best_forms: [None; 2], + table: [0; TABLE_SIZE], + } + } + + /// Reads the description of a table at the start of `src`, and returns its size. The table is to decode + /// `nb_literals` with, from the rest of `src` if that is `four_streams`. + fn read( + &mut self, + src: &[u8], + table_type: TableType, + nb_literals: usize, + four_streams: bool, + spread: &mut Spread, + ) -> Result { + let Code { + weights, + rank_stats, + .. + } = &mut self.code; + let stats = huf_read_stats_wksp(weights, rank_stats, src, spread)?; + self.code.nb_symbols = stats.nb_symbols; + self.code.table_log = stats.table_log; + self.table_type = table_type; + self.best_forms = [None; 2]; + let nb_bytes = src.len().saturating_sub(stats.size); + let (log, single) = self.best_form(nb_literals, nb_bytes, four_streams); + self.build(log, single)?; + Ok(stats.size) + } + + /// [`Code::best_form`], which is for four streams. + fn best_form( + &mut self, + nb_literals: usize, + nb_bytes: usize, + four_streams: bool, + ) -> (u32, bool) { + if !four_streams { + return (self.code.best_log(nb_literals), false); + } + if nb_literals < FAST_MIN_LITERALS { + return self.code.best_form(nb_literals, nb_bytes); + } + let [fast, large] = &mut self.best_forms; + let best_form = if nb_literals < LARGE_MIN_LITERALS { + fast + } else { + large + }; + *best_form.get_or_insert_with(|| self.code.best_form(nb_literals, nb_bytes)) + } + + /// Makes the table fit to decode `nb_literals` of another block with, from `nb_bytes`. + pub(crate) fn prepare( + &mut self, + nb_literals: usize, + nb_bytes: usize, + four_streams: bool, + ) -> Result<()> { + let (log, single) = self.best_form(nb_literals, nb_bytes, four_streams); + let is_worth_it = match (self.single, single) { + (true, true) => false, + // Few literals do not make up for the time that it takes to fill a large table once more. + (true, false) => nb_literals >= FAST_MIN_LITERALS || self.log <= SINGLE_SMALL_LOG, + // The largest table is only made if its lookups yield enough. + (false, true) => nb_literals >= FAST_MIN_LITERALS && self.log <= FAST_LOG, + (false, false) => self.log < log, + }; + if is_worth_it { + self.build(log, single)?; + } + Ok(()) + } + + /// Fills `table` for lookups of `log` bits, which is at least as many as the longest code has. + fn build(&mut self, log: u32, single: bool) -> Result<()> { + self.log = log; + self.single = single; + let sorted = Sorted::new(&self.code, log)?; + // The tables of what a third symbol adds end with the one for this many bits. + let room = log.saturating_sub(2 * sorted.shortest()); + if single { + self.fill_single(&sorted) + } else if room < sorted.shortest() { + self.fill(&sorted, &mut []) + } else if room < 7 { + self.fill(&sorted, &mut [0; 1 << 7]) + } else { + self.fill(&sorted, &mut [0; TABLE_SIZE / 2]) + } + } + + fn fill_single(&mut self, sorted: &Sorted) -> Result<()> { + let log = sorted.log; + for weight in 1..=sorted.max_weight { + let symbols = sorted.class(weight)?; + let nb_bits = sorted.nb_bits(weight); + let room = log - nb_bits; + let entries = self + .table + .span_mut(sorted.position(weight, log)?, symbols.len() << room)?; + let base = single_entry(0, nb_bits); + let unit = single_entry(1, nb_bits) - base; + fill_runs(entries, symbols, room, base, unit); + } + Ok(()) + } + + fn fill(&mut self, sorted: &Sorted, thirds: &mut [Entry]) -> Result<()> { + let log = sorted.log; + // From how many bits on what a symbol leaves of a lookup holds another. + let min_room = sorted.shortest(); + for room in min_room..=log.saturating_sub(2 * min_room) { + sorted.fill(thirds.span_mut(1 << room, 1 << room)?, room, 0, 2, None)?; + } + for weight in 1..=sorted.max_weight { + let symbols = sorted.class(weight)?; + let nb_bits = sorted.nb_bits(weight); + let room = log - nb_bits; + let entries = self + .table + .span_mut(sorted.position(weight, log)?, symbols.len() << room)?; + if room < min_room { + fill_runs(entries, symbols, room, entry(0, 0, nb_bits), 1); + continue; + } + let Some((&first_symbol, other_symbols)) = symbols.split_first() else { + continue; + }; + let size = 1usize << room; + let (entries, other_entries) = entries.split_at_mut_checked(size).ok_or(BUG)?; + sorted.fill( + entries, + room, + entry(first_symbol, 0, nb_bits), + 1, + Some(thirds), + )?; + for (copy, &symbol) in other_entries.chunks_exact_mut(size).zip(other_symbols) { + let change = u32::from(symbol).wrapping_sub(u32::from(first_symbol)); + for (to, from) in copy.iter_mut().zip(&*entries) { + *to = from.wrapping_add(change); + } + } + } + Ok(()) + } + + /// One stream. Whether it is exactly `dst.len()` symbols. + fn decode_1x(&self, dst: &mut [u8], src: &[u8]) -> bool { + let Some(mut stream) = Stream::new(Part { src, out: dst }) else { + return false; + }; + stream.decode_most(&self.table, self.log); + self.decode_tails(&mut [stream]) + } + + /// Four streams after their sizes. Whether each is exactly its share of `dst.len()` symbols. + fn decode_4x(&self, dst: &mut [u8], src: &[u8]) -> bool { + let Some(parts) = Part::split(dst, src) else { + return false; + }; + let Some(mut streams) = Streams::new(parts) else { + return false; + }; + if streams.0.iter().all(|stream| stream.bits != 0) { + if self.single { + self.decode_most_single(&mut streams); + } else { + self.decode_most(&mut streams); + } + } + if !streams.0.iter().all(Stream::fits_tail) { + return self.decode_stragglers(streams); + } + self.decode_4_tails(&mut streams) + } + + #[inline(never)] + fn decode_most(&self, streams: &mut Streams<'_>) { + match self.log { + 8 => streams.decode_most::<8, 5>(&self.table), + 9 => streams.decode_most::<9, 5>(&self.table), + 10 => streams.decode_most::<10, 5>(&self.table), + 11 => streams.decode_most::<11, 5>(&self.table), + _ => streams.decode_most::<12, 4>(&self.table), + } + } + + /// The same with a table where every entry is one symbol. + #[inline(never)] + fn decode_most_single(&self, streams: &mut Streams<'_>) { + match self.log { + 8 => streams.decode_most_single::<8, 7>(&self.table), + 9 => streams.decode_most_single::<9, 6>(&self.table), + 10 => streams.decode_most_single::<10, 5>(&self.table), + _ => streams.decode_most_single::(&self.table), + } + } + + /// The same for streams of which some have much left. + #[inline(never)] + fn decode_stragglers(&self, streams: Streams<'_>) -> bool { + // Some streams get there before others, which go on four at a time all the same: with these. + static NO_STREAM: [u8; 256] = [0x55; 256]; + let mut no_output = [[0u8; 512]; 4]; + let mut streams: Streams<'_> = streams; + let mut decoding = Streams(no_output.each_mut().map(|out| Stream { + src: &NO_STREAM, + ip: 0, + bits: 0, + out, + op: 0, + })); + for (slot, stream) in decoding.0.iter_mut().zip(&mut streams.0) { + core::mem::swap(slot, stream); + } + // `streams[i]` is in `decoding`. + let mut left = [true; 4]; + loop { + for ((stream, done), left) in decoding.0.iter_mut().zip(&mut streams.0).zip(&mut left) { + if stream.has_most_left() { + continue; + } + if *left { + *left = false; + core::mem::swap(stream, done); + } + stream.ip = NO_STREAM.len() as u32 - 8; + stream.bits = 1; + stream.op = 0; + stream.has_most_left(); + } + if left == [false; 4] { + break; + } + self.decode_most(&mut decoding); + } + self.decode_4_tails(&mut streams) + } + + #[inline(never)] + fn decode_4_tails(&self, streams: &mut Streams<'_>) -> bool { + self.decode_tails(&mut streams.0) + } + + /// Decodes what is left of streams. Whether each turns out to be exactly as many symbols as its output takes. + #[inline(always)] + fn decode_tails(&self, streams: &mut [Stream<'_>; N]) -> bool { + let mut tails = [const { Tail::new() }; N]; + for (tail, stream) in tails.iter_mut().zip(&*streams) { + if !tail.set(stream) { + return false; + } + } + let shift = 64 - self.log; + loop { + let mut rounds = tails.each_ref().map(Tail::start_round); + for _ in 0..Tail::LOOKUPS { + for (tail, round) in tails.iter_mut().zip(&mut rounds) { + let index = (round.bits >> shift) as usize; + let entry = self.table.get(index % TABLE_SIZE).copied().unwrap_or(0); + round.bits <<= (entry >> 24) & 63; + if let Some(to) = tail.out.get_mut(round.op..round.op + 4) { + to.copy_from_slice(&entry.to_le_bytes()); + } + round.op += (entry >> (24 + COUNT_SHIFT)) as usize; + } + } + let mut more = false; + for (tail, round) in tails.iter_mut().zip(&rounds) { + more |= tail.end_round(round); + } + if !more { + break; + } + } + let mut exact = true; + for (tail, stream) in tails.iter().zip(streams) { + exact &= tail.get(stream, self.code.table_log); + } + exact + } +} + +/// What is left of a stream that is close to its start or whose output is close to its end, where decoding can go too +/// far: that is made up for afterwards. +struct Tail { + /// Zeroes, then the start of the stream. + src: [u8; Self::PADDING + Self::MAX_SRC], + /// How many bits are left to read of `src`. + position: u32, + out: [u8; Self::MAX_SYMBOLS + Self::ROUND_SYMBOLS], + /// How much of `out` is written. + op: u32, +} + +/// A few lookups between two reads of a stream. +struct Round { + /// What is left of 64 bits, from the highest bit on, then a bit set. + bits: u64, + /// How many of the 64 bits were read before. + skipped: u32, + /// How much of the output is written. + op: usize, +} + +impl Tail { + const LOOKUPS: usize = 4; + /// How many bytes a round can write. + const ROUND_SYMBOLS: usize = 3 * Self::LOOKUPS + 1; + /// No less than a round can read, and 8 bytes. + const PADDING: usize = 16; + const MAX_SRC: usize = 64; + /// No less than [`Stream::decode_most`] can leave of a stream that is valid, at a bit a symbol. + const MAX_SYMBOLS: usize = 128; + + const fn new() -> Self { + Self { + src: [0; _], + position: 0, + out: [0; _], + op: 0, + } + } + + /// Takes what is left of a stream, if it is not too much and the stream has an end mark. + fn set(&mut self, stream: &Stream<'_>) -> bool { + let Some(&last) = stream.src.last() else { + return false; + }; + if last == 0 || !stream.fits_tail() { + return false; + } + let end = stream.end(); + let src = self.src.get_mut(Self::PADDING..).unwrap_or_default(); + if let (Some(to), Some(from)) = ( + src.first_chunk_mut::<{ Self::MAX_SRC }>(), + stream.src.first_chunk::<{ Self::MAX_SRC }>(), + ) { + *to = *from; + } else if let (Some(to), Some(from)) = (src.get_mut(..end), stream.src.get(..end)) { + to.copy_from_slice(from); + } + let consumed = if stream.bits == 0 { + last.leading_zeros() + 1 + } else { + stream.bits.trailing_zeros() + }; + self.position = 8 * (Self::PADDING + end) as u32 - consumed; + true + } + + /// Whether neither the start of the stream nor the end of `out` is reached. + fn has_more(&self) -> bool { + self.position > 8 * Self::PADDING as u32 && self.op as usize <= Self::MAX_SYMBOLS + } + + #[inline(always)] + fn start_round(&self) -> Round { + let end = (self.position as usize).div_ceil(8); + let container = self + .src + .get(end.wrapping_sub(8)..end) + .and_then(|bytes| bytes.try_into().ok()) + .map_or(0, u64::from_le_bytes); + let skipped = self.position.wrapping_neg() & 7; + Round { + bits: (container | 1) << skipped, + skipped, + op: self.op as usize, + } + } + + /// Returns whether there is to be another round. + #[inline(always)] + fn end_round(&mut self, round: &Round) -> bool { + if !self.has_more() { + return false; + } + self.op = round.op as u32; + self.position -= round.bits.trailing_zeros() - round.skipped; + self.has_more() + } + + /// Completes the output of the stream. Whether the stream is exactly as many symbols as its output takes. + fn get(&self, stream: &mut Stream<'_>, table_log: u32) -> bool { + let out = stream.out.get_mut(stream.op as usize..).unwrap_or_default(); + let Some(symbols) = self.out.get(..out.len()) else { + return false; + }; + out.copy_from_slice(symbols); + // Zeroes are the longest code. + let nb_bits = (8 * Self::PADDING as u32).wrapping_sub(self.position); + let nb_symbols = self.op.wrapping_sub(out.len() as u32); + !self.has_more() && u64::from(nb_bits) == u64::from(nb_symbols) * u64::from(table_log) + } +} + +/// A stream, and where its symbols go. +struct Part<'a> { + src: &'a [u8], + out: &'a mut [u8], +} + +impl<'a> Part<'a> { + /// `None` if the sizes at the start of `src` are not those of four streams. + fn split(dst: &'a mut [u8], src: &'a [u8]) -> Option<[Self; 4]> { + let (sizes, src) = src.split_first_chunk::<6>()?; + let (src0, src) = src.split_at_checked(usize::from(read_le16(sizes, 0).ok()?))?; + let (src1, src) = src.split_at_checked(usize::from(read_le16(sizes, 2).ok()?))?; + let (src2, src3) = src.split_at_checked(usize::from(read_le16(sizes, 4).ok()?))?; + let segment = dst.len().div_ceil(4); + let (dst0, dst) = dst.split_at_mut_checked(segment)?; + let (dst1, dst) = dst.split_at_mut_checked(segment)?; + let (dst2, dst3) = dst.split_at_mut_checked(segment)?; + Some( + [(src0, dst0), (src1, dst1), (src2, dst2), (src3, dst3)] + .map(|(src, out)| Self { src, out }), + ) + } +} + +/// What two rounds of lookups between reloads can write, as far as the compiler can tell. +const WINDOW: usize = 32; + +/// Reads the 64 bits from where a stream is. Whether there are as many. +#[inline(always)] +fn reload(src: &[u8], ip: &mut u32, bits: &mut u64) -> bool { + let consumed = bits.trailing_zeros(); + // Wraps around before the start. + let from = ip.wrapping_sub(consumed >> 3); + let Some(Ok(container)) = src + .get(from as usize..from as usize + 8) + .map(<[u8; 8]>::try_from) + else { + return false; + }; + *ip = from; + *bits = (u64::from_le_bytes(container) | 1) << (consumed & 7); + true +} + +/// A stream that is being decoded. +struct Stream<'a> { + src: &'a [u8], + /// Where in `src` `bits` was read. + ip: u32, + /// What is left to read of 64 bits, from the highest bit on, then a bit set. 0 if the stream is shorter than that: + /// nothing is read of it yet. + bits: u64, + out: &'a mut [u8], + /// How much of `out` is written. + op: u32, +} + +impl<'a> Stream<'a> { + fn new(Part { src, out }: Part<'a>) -> Option { + let mut stream = Self { + src, + ip: 0, + bits: 0, + out, + op: 0, + }; + if let Some(container) = src.last_chunk() { + let container = u64::from_le_bytes(*container); + // The highest bit set marks the end of the stream. + if container >> 56 == 0 { + return None; + } + stream.ip = src.len() as u32 - 8; + stream.bits = (container | 1) << (container.leading_zeros() + 1); + } + Some(stream) + } + + /// Whether the stream is far enough from its start and its output from its end for `Streams::decode_most`, for + /// which it then is ready. + fn has_most_left(&mut self) -> bool { + self.bits != 0 + && self.op as usize + WINDOW <= self.out.len() + && reload(self.src, &mut self.ip, &mut self.bits) + } + + /// Where what is left of `src` ends. + fn end(&self) -> usize { + if self.bits == 0 { + self.src.len() + } else { + self.ip as usize + 8 + } + } + + fn fits_tail(&self) -> bool { + self.end() <= Tail::MAX_SRC && self.out.len() - self.op as usize <= Tail::MAX_SYMBOLS + } + + /// Decodes until the stream is close to its start, or its output to its end. + fn decode_most(&mut self, table: &[Entry; TABLE_SIZE], log: u32) { + const LOOKUPS: usize = 4; + /// How many bytes are read at a time. + const READ: usize = size_of::(); + const { assert!(8 * (READ - 1) + 63 <= Tail::MAX_SYMBOLS) }; + if self.bits == 0 { + return; + } + let shift = 64 - log; + // With nothing else to do in the meantime, what counts is how soon the next lookup can start. So more bits + // come from where the last ones came from, which is known ahead of time. + let consumed = self.bits.trailing_zeros(); + // Where the bytes after those that `bits` was read from end. + let mut next = self.ip as usize; + let Some(Ok(container)) = (self.src.get(next..next + READ)).map(<[u8; READ]>::try_from) + else { + return; + }; + // The next bits, from the highest on: `valid` of them, or somewhat more. + let mut bits = u64::from_le_bytes(container) << consumed; + let mut valid = 64 - consumed; + let mut op = self.op as usize; + while let Some(window) = (self.out.get_mut(op..op + 3 * LOOKUPS + 1)) + .and_then(|window| window.as_mut_array::<{ 3 * LOOKUPS + 1 }>()) + && next >= READ + && let Some(Ok(more)) = (self.src.get(next - READ..next)).map(<[u8; READ]>::try_from) + { + bits |= u64::from_le_bytes(more) >> (valid & 63); + let nb_bytes = (63 - (valid & 63)) >> 3; + next -= nb_bytes as usize; + valid += 8 * nb_bytes; + let mut done = 0; + for _ in 0..LOOKUPS { + let entry = table.get((bits >> shift) as usize).copied().unwrap_or(0); + let nb_bits = (entry >> 24) & 63; + bits <<= nb_bits; + valid = valid.wrapping_sub(nb_bits); + if let Some(to) = window.get_mut(done..done + 4) { + to.copy_from_slice(&entry.to_le_bytes()); + } + done += (entry >> (24 + COUNT_SHIFT)) as usize; + } + op += done; + } + let position = 8 * next + valid as usize; + let end = position.div_ceil(8).max(READ); + if let Some(Ok(container)) = (self.src.get(end - READ..end)).map(<[u8; READ]>::try_from) { + self.ip = (end - READ) as u32; + self.bits = (u64::from_le_bytes(container) | 1) << ((8 * end - position) & 63); + self.op = op as u32; + } + } +} + +struct Streams<'a>([Stream<'a>; 4]); + +impl<'a> Streams<'a> { + fn new([part0, part1, part2, part3]: [Part<'a>; 4]) -> Option { + Some(Self([ + Stream::new(part0)?, + Stream::new(part1)?, + Stream::new(part2)?, + Stream::new(part3)?, + ])) + } + + /// Decodes for as long as no stream is close to its start and no output to its end. `LOOKUPS` of `LOG` bits are at + /// most 55 bits. + #[inline(always)] + fn decode_most(&mut self, table: &[Entry; TABLE_SIZE]) { + #[inline(always)] + fn lookup( + table: &[Entry; TABLE_SIZE], + bits: &mut u64, + window: &mut [u8; WINDOW], + done: &mut usize, + ) { + let index = (*bits >> (64 - LOG)) as usize; + // The highest byte of the entry, read on its own where that is simple: the next lookup waits for it. + let nb_bits = if cfg!(target_endian = "little") { + bun_pod::bytes_of(table).get(4 * index + 3).copied() + } else { + table.get(index).map(|entry| (entry >> 24) as u8) + }; + if let (Some(entry), Some(nb_bits)) = (table.get(index), nb_bits) { + *bits <<= nb_bits & 63; + if let Some(to) = window.get_mut(*done..*done + 4) { + to.copy_from_slice(&entry.to_le_bytes()); + } + *done += (entry >> (24 + COUNT_SHIFT)) as usize; + } + } + + /// `LOOKUPS` in each stream. Not a loop: one that is not unrolled checks `done` at every turn. + #[inline(always)] + fn round( + table: &[Entry; TABLE_SIZE], + bits: [&mut u64; 4], + windows: &mut [&mut [u8; WINDOW]; 4], + done: &mut [usize; 4], + ) { + let [bits0, bits1, bits2, bits3] = bits; + let [window0, window1, window2, window3] = windows; + let [done0, done1, done2, done3] = done; + let mut lookups = || { + lookup::(table, bits0, window0, done0); + lookup::(table, bits1, window1, done1); + lookup::(table, bits2, window2, done2); + lookup::(table, bits3, window3, done3); + }; + lookups(); + lookups(); + lookups(); + lookups(); + if LOOKUPS > 4 { + lookups(); + } + } + + #[inline(always)] + fn window(out: &mut [u8], op: u32) -> Option<&mut [u8; WINDOW]> { + out.get_mut(op as usize..op as usize + WINDOW)? + .as_mut_array() + } + + const { assert!((LOOKUPS == 4 || LOOKUPS == 5) && LOG as usize * LOOKUPS <= 55) }; + let [s0, s1, s2, s3] = &mut self.0; + let (mut bits0, mut bits1, mut bits2, mut bits3) = (s0.bits, s1.bits, s2.bits, s3.bits); + let (mut ip0, mut ip1, mut ip2, mut ip3) = (s0.ip, s1.ip, s2.ip, s3.ip); + let (mut op0, mut op1, mut op2, mut op3) = (s0.op, s1.op, s2.op, s3.op); + loop { + let (Some(window0), Some(window1), Some(window2), Some(window3)) = ( + window(s0.out, op0), + window(s1.out, op1), + window(s2.out, op2), + window(s3.out, op3), + ) else { + break; + }; + let mut windows = [window0, window1, window2, window3]; + let mut done = [0; 4]; + round::( + table, + [&mut bits0, &mut bits1, &mut bits2, &mut bits3], + &mut windows, + &mut done, + ); + let mut more = reload(s0.src, &mut ip0, &mut bits0) + && reload(s1.src, &mut ip1, &mut bits1) + && reload(s2.src, &mut ip2, &mut bits2) + && reload(s3.src, &mut ip3, &mut bits3); + if more { + round::( + table, + [&mut bits0, &mut bits1, &mut bits2, &mut bits3], + &mut windows, + &mut done, + ); + more = reload(s0.src, &mut ip0, &mut bits0) + && reload(s1.src, &mut ip1, &mut bits1) + && reload(s2.src, &mut ip2, &mut bits2) + && reload(s3.src, &mut ip3, &mut bits3); + } + let [done0, done1, done2, done3] = done; + op0 += done0 as u32; + op1 += done1 as u32; + op2 += done2 as u32; + op3 += done3 as u32; + if !more { + break; + } + } + (s0.bits, s1.bits, s2.bits, s3.bits) = (bits0, bits1, bits2, bits3); + (s0.ip, s1.ip, s2.ip, s3.ip) = (ip0, ip1, ip2, ip3); + (s0.op, s1.op, s2.op, s3.op) = (op0, op1, op2, op3); + } + + /// The same with a table where every entry is one symbol. `LOOKUPS` of `LOG` bits are at most 56 bits. + #[inline(always)] + fn decode_most_single( + &mut self, + table: &[Entry; TABLE_SIZE], + ) { + #[inline(always)] + fn lookup( + table: &[Entry; TABLE_SIZE], + bits: &mut u64, + to: Option<&mut u8>, + ) { + let index = (*bits >> (64 - LOG)) as usize; + let middle = if cfg!(target_endian = "little") { + (bun_pod::bytes_of(table).get(4 * index + 1..)) + .and_then(|bytes| bytes.first_chunk()) + .map(|bytes| u16::from_le_bytes(*bytes)) + } else { + table.get(index).map(|entry| (entry >> 8) as u16) + }; + if let (Some(to), Some(middle)) = (to, middle) { + *bits <<= middle & 63; + *to = (middle >> 8) as u8; + } + } + + #[inline(always)] + fn rounds<'a, const LOOKUPS: usize>(stream: &'a mut Stream<'_>) -> &'a mut [[u8; LOOKUPS]] { + let left = stream.out.get_mut(stream.op as usize..).unwrap_or_default(); + left.as_chunks_mut().0 + } + + const { assert!(LOOKUPS <= 7 && LOG as usize * LOOKUPS <= 56) }; + let [s0, s1, s2, s3] = &mut self.0; + let (mut bits0, mut bits1, mut bits2, mut bits3) = (s0.bits, s1.bits, s2.bits, s3.bits); + let (mut ip0, mut ip1, mut ip2, mut ip3) = (s0.ip, s1.ip, s2.ip, s3.ip); + let (src0, src1, src2, src3) = (s0.src, s1.src, s2.src, s3.src); + let mut nb_rounds = 0; + for (((out0, out1), out2), out3) in rounds::(s0) + .iter_mut() + .zip(rounds::(s1)) + .zip(rounds::(s2)) + .zip(rounds::(s3)) + { + // A stream that is not reloaded yet can have as few as 55 bits. + if !(reload(src0, &mut ip0, &mut bits0) + && reload(src1, &mut ip1, &mut bits1) + && reload(src2, &mut ip2, &mut bits2) + && reload(src3, &mut ip3, &mut bits3)) + { + break; + } + let mut lookups = |index: usize| { + lookup::(table, &mut bits0, out0.get_mut(index)); + lookup::(table, &mut bits1, out1.get_mut(index)); + lookup::(table, &mut bits2, out2.get_mut(index)); + lookup::(table, &mut bits3, out3.get_mut(index)); + }; + lookups(0); + lookups(1); + lookups(2); + lookups(3); + lookups(4); + lookups(5); + lookups(6); + nb_rounds += 1; + } + (s0.bits, s1.bits, s2.bits, s3.bits) = (bits0, bits1, bits2, bits3); + (s0.ip, s1.ip, s2.ip, s3.ip) = (ip0, ip1, ip2, ip3); + for stream in &mut self.0 { + stream.op += nb_rounds * LOOKUPS as u32; + } + } +} + +/// `HUF_readDTableX2_wksp`. Returns the size of the table's description in `src`. +pub(crate) fn read_dtable_x2( + dtable: &mut HufDTable, + src: &[u8], + spread: &mut Spread, +) -> Result { + dtable.read(src, TableType::X2, usize::MAX, false, spread) +} + +/// `HUF_decompress1X_usingDTable` +pub(crate) fn decompress_1x_using_dtable( + dst: &mut [u8], + src: &[u8], + dtable: &HufDTable, +) -> Result { + if dtable.decode_1x(dst, src) { + return Ok(dst.len()); + } + huf_exact::decompress_1x(dst, src, &dtable.code, dtable.table_type) +} + +/// `HUF_decompress1X1_DCtx_wksp` +pub(crate) fn decompress_1x1( + dtable: &mut HufDTable, + dst: &mut [u8], + src: &[u8], + spread: &mut Spread, +) -> Result { + let h_size = dtable.read(src, TableType::X1, dst.len(), false, spread)?; + if h_size >= src.len() { + return Err(Error::SrcSizeWrong); + } + decompress_1x_using_dtable(dst, src.sub(h_size..)?, dtable) +} + +/// `HUF_decompress4X_usingDTable` +pub(crate) fn decompress_4x_using_dtable( + dst: &mut [u8], + src: &[u8], + dtable: &HufDTable, +) -> Result { + if dtable.decode_4x(dst, src) { + return Ok(dst.len()); + } + huf_exact::decompress_4x(dst, src, &dtable.code, dtable.table_type) +} + +/// `HUF_decompress4X_hufOnly_wksp` +pub(crate) fn decompress_4x_huf_only( + dtable: &mut HufDTable, + dst: &mut [u8], + src: &[u8], + spread: &mut Spread, +) -> Result { + if dst.is_empty() { + return Err(Error::DstSizeTooSmall); + } + if src.is_empty() { + return Err(Error::CorruptionDetected); + } + let table_type = if select_decoder_x2(dst.len(), src.len())? { + TableType::X2 + } else { + TableType::X1 + }; + let h_size = dtable.read(src, table_type, dst.len(), true, spread)?; + if h_size >= src.len() { + return Err(Error::SrcSizeWrong); + } + decompress_4x_using_dtable(dst, src.sub(h_size..)?, dtable) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::compress::huf::{ + HufCTable, build_ctable, compress_1x_using_ctable, compress_4x_using_ctable, write_ctable, + }; + + #[test] + fn a_table_is_made_to_suit_each_block() -> Result<()> { + let mut state = 5u32; + let mut random = move || { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + state >> 8 + }; + // Few enough for codes of 8 bits at most that are not all as long. The lowest have the shortest codes. + const MAX_SYMBOL: u32 = 199; + let counts: [u32; MAX_SYMBOL as usize + 1] = core::array::from_fn(|symbol| { + let rank = MAX_SYMBOL + 1 - symbol as u32; + rank * rank * rank / 256 + 1 + }); + // How many literals, by how much they are lowered, and whether they are in four streams. + let blocks = [ + (300, 0, true), + (300, 3, true), + (FAST_MIN_LITERALS, 0, true), + (300, 3, true), + (FAST_MIN_LITERALS, 0, false), + (300, 0, false), + (LARGE_MIN_LITERALS, 3, true), + (200, 3, false), + (LARGE_MIN_LITERALS, 0, true), + ]; + // Miri takes a minute for 10,000 literals. + let (blocks, longest_codes, firsts, max_damaged) = if cfg!(miri) { + (blocks.sub(..5)?, 11..=11, [0].as_slice(), 300) + } else { + ( + blocks.sub(..)?, + 8..=12, + [0, 6, 8].as_slice(), + FAST_MIN_LITERALS, + ) + }; + // From which kind of table to which one a block went on with the code of the block before. + let mut changes = [[0; 2]; 2]; + let mut dtable = HufDTable::new(); + for (max_nb_bits, &first) in + longest_codes.flat_map(|log| firsts.iter().map(move |first| (log, first))) + { + let mut ctable = HufCTable::new(); + let table_log = build_ctable(&mut ctable, &counts, MAX_SYMBOL, max_nb_bits)?; + assert_eq!(table_log, max_nb_bits as usize); + let mut description = [0; NB_SYMBOLS]; + let size = write_ctable(&mut description, &ctable, MAX_SYMBOL, max_nb_bits)?; + let description = description.sub(..size)?; + + for (index, &(nb_literals, shift, four_streams)) in + blocks.iter().skip(first).chain(blocks).enumerate() + { + let literals: Vec = (0..nb_literals) + .map(|_| { + (((random() & 255).min(random() & 255) * MAX_SYMBOL / 256) >> shift) as u8 + }) + .collect(); + let mut streams = vec![0; 2 * nb_literals]; + let size = if four_streams { + compress_4x_using_ctable(&mut streams, &literals, &ctable)? + } else { + compress_1x_using_ctable(&mut streams, &literals, &ctable)? + }; + streams.truncate(size); + if index == 0 { + let src = [description, &streams].concat(); + let spread = &mut Spread::new(); + let read = dtable.read(&src, TableType::X1, nb_literals, four_streams, spread); + assert_eq!(read, Ok(description.len())); + } else { + let was_single = dtable.single; + dtable.prepare(nb_literals, size, four_streams)?; + *changes + .at_mut(usize::from(was_single))? + .at_mut(usize::from(dtable.single))? += 1; + } + let mut decoded = vec![0; nb_literals]; + assert!(if four_streams { + dtable.decode_4x(&mut decoded, &streams) + } else { + dtable.decode_1x(&mut decoded, &streams) + }); + assert_eq!(decoded, literals); + + // Streams that do not end where they should, some much shorter than the others. + let nb_damaged = match four_streams && nb_literals <= max_damaged { + true if cfg!(miri) => 2, + true => 6, + false => 0, + }; + for _ in 0..nb_damaged { + let mut damaged = streams.clone(); + let at = random() as usize % if random() % 2 == 0 { 6 } else { size }; + *damaged.at_mut(at)? ^= 1 << (random() % 8); + let mut exact = vec![0; nb_literals]; + let result = + huf_exact::decompress_4x(&mut exact, &damaged, &dtable.code, TableType::X1); + assert_eq!( + decompress_4x_using_dtable(&mut decoded, &damaged, &dtable), + result + ); + assert!(result.is_err() || decoded == exact); + } + } + } + assert!(changes.as_flattened().iter().all(|&count| count > 0)); + Ok(()) + } + + #[test] + fn a_single_stream_of_short_codes_is_not_left_to_the_careful_way() -> Result<()> { + // Up to as many as the header of a block can tell of a single stream. + let sizes: &[usize] = if cfg!(miri) { + &[129, 255] + } else { + &Vec::from_iter(1..=1023) + }; + // The first symbol takes a bit. The last ones take 3 bits, or 11. + let codes: [&[u32]; 2] = [ + &[4000, 3, 2, 1], + &[1024, 512, 256, 128, 64, 32, 16, 8, 4, 2, 1, 1], + ]; + for counts in codes { + let max_symbol = counts.len() - 1; + let mut ctable = HufCTable::new(); + let table_log = build_ctable(&mut ctable, counts, max_symbol as u32, 11)?; + let mut description = [0; NB_SYMBOLS]; + let size = write_ctable( + &mut description, + &ctable, + max_symbol as u32, + table_log as u32, + )?; + let description = description.sub(..size)?; + let mut dtable = HufDTable::new(); + let read = dtable.read(description, TableType::X1, 255, false, &mut Spread::new()); + assert_eq!(read, Ok(description.len())); + + for (log, single) in [ + (dtable.log, false), + (dtable.log, true), + (FAST_LOG + 1, false), + ] { + dtable.build(log, single)?; + for (&nb_literals, rarity) in sizes + .iter() + .flat_map(|size| [(size, usize::MAX), (size, 50)]) + { + let literals: Vec = (1..=nb_literals) + .map(|n| { + if n % rarity == 0 { + (n % max_symbol) as u8 + 1 + } else { + 0 + } + }) + .collect(); + let mut stream = vec![0; 2 * nb_literals + 8]; + let size = compress_1x_using_ctable(&mut stream, &literals, &ctable)?; + stream.truncate(size); + let mut decoded = vec![0; nb_literals]; + assert!(dtable.decode_1x(&mut decoded, &stream), "{nb_literals}"); + assert_eq!(decoded, literals); + + if cfg!(miri) { + continue; + } + // Streams that do not end where they should. + let step = if nb_literals < 300 { 5 } else { 8 * size }; + for bit in (0..8 * size).step_by(step) { + let mut damaged = stream.clone(); + *damaged.at_mut(bit / 8)? ^= 1 << (bit % 8); + for table_type in [TableType::X1, TableType::X2] { + dtable.table_type = table_type; + let mut exact = vec![0; nb_literals]; + let result = huf_exact::decompress_1x( + &mut exact, + &damaged, + &dtable.code, + table_type, + ); + assert_eq!( + decompress_1x_using_dtable(&mut decoded, &damaged, &dtable), + result + ); + assert!(result.is_err() || decoded == exact); + } + } + } + } + } + Ok(()) + } +} diff --git a/src/zstd/decompress/huf_exact.rs b/src/zstd/decompress/huf_exact.rs new file mode 100644 index 000000000000..c5b9be77c3e3 --- /dev/null +++ b/src/zstd/decompress/huf_exact.rs @@ -0,0 +1,1022 @@ +//! `huf_decompress.c`: Huffman decoding of literals, one symbol (X1) or up to two symbols (X2) per table lookup, step by +//! step as the C code does it. What it makes of streams that do not end where they should depends on all of that. + +use super::huf::Code; +use crate::common::bitstream::{BitDStream, DStreamStatus}; +use crate::common::{HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_MAX}; +use crate::error::{Error, Result}; +use crate::util::{BUG, SliceExt, highbit32, read_le16, read_le64}; + +/// Tables of this log are decoded by the loops that keep four streams in registers. +const HUF_DECODER_FAST_TABLELOG: u32 = 11; + +const TABLE_SIZE: usize = 1 << HUF_TABLELOG_MAX; +const TABLE_MASK: usize = TABLE_SIZE - 1; +const NB_SYMBOLS: usize = HUF_SYMBOLVALUE_MAX as usize + 1; +const NB_RANKS: usize = HUF_TABLELOG_MAX as usize + 1; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(super) enum TableType { + X1, + X2, +} + +/// `HUF_DTable`, of capacity `ZSTD_HUFFDTABLE_CAPACITY_LOG`. +struct ExactTable { + table_log: u32, + /// - `u16`, `HUF_DEltX1`: the number of bits in the low byte, and the symbol above it. + /// - `u32`, `HUF_DEltX2`: two symbols, then the number of bits, then how many of the symbols count. + entries: [Entry; TABLE_SIZE], +} + +impl ExactTable { + fn new() -> Self { + Self { + table_log: 0, + entries: [Entry::default(); TABLE_SIZE], + } + } +} + +#[inline(always)] +fn entry_x1(dt: &[u16; TABLE_SIZE], index: u64) -> u16 { + dt.get(index as usize & TABLE_MASK).copied().unwrap_or(0) +} + +#[inline(always)] +fn entry_x2(dt: &[u32; TABLE_SIZE], index: u64) -> u32 { + dt.get(index as usize & TABLE_MASK).copied().unwrap_or(0) +} + +// ── the four streams ────────────────────────────────────────────────────── + +/// Where the four streams of `src` start, after the jump table, and where the last one ends. +struct Streams { + start: [usize; 4], + length: [usize; 4], +} + +impl Streams { + /// `src` has at least 10 bytes. + fn new(src: &[u8]) -> Result { + let length1 = usize::from(read_le16(src, 0)?); + let length2 = usize::from(read_le16(src, 2)?); + let length3 = usize::from(read_le16(src, 4)?); + let length4 = src.len().wrapping_sub(length1 + length2 + length3 + 6); + let start1 = 6; + let start2 = start1 + length1; + let start3 = start2 + length2; + let start4 = start3 + length3; + Ok(Self { + start: [start1, start2, start3, start4], + length: [length1, length2, length3, length4], + }) + } + + fn open<'a>(&self, src: &'a [u8]) -> Result<[BitDStream<'a>; 4]> { + let [s1, s2, s3, s4] = self.start; + let [l1, l2, l3, l4] = self.length; + Ok([ + BitDStream::new(src.span(s1, l1)?)?, + BitDStream::new(src.span(s2, l2)?)?, + BitDStream::new(src.span(s3, l3)?)?, + BitDStream::new(src.span(s4, l4)?)?, + ]) + } +} + +/// `HUF_DecompressFastArgs`, with positions in the input and the output for pointers. +struct FastArgs { + ip: [usize; 4], + op: [usize; 4], + bits: [u64; 4], + /// Where each stream starts. + iend: [usize; 4], +} + +/// `HUF_initFastDStream`: the lowest set bit marks how much of the 64 bits is consumed. +#[inline] +fn init_fast_dstream(src: &[u8], ip: usize) -> Result { + let value = read_le64(src, ip)?; + let last_byte = (value >> 56) as u32; + let bits_consumed = if last_byte != 0 { + 8 - highbit32(last_byte) + } else { + 0 + }; + Ok((value | 1) << bits_consumed) +} + +impl FastArgs { + /// `HUF_DecompressFastArgs_init`. `None` if the input has to be decoded by the other loops. + fn new(dst_size: usize, src: &[u8], table_log: u32) -> Result> { + if dst_size == 0 { + return Ok(None); + } + if src.len() < 10 { + return Err(Error::CorruptionDetected); + } + if table_log != HUF_DECODER_FAST_TABLELOG { + return Ok(None); + } + let streams = Streams::new(src)?; + if streams.length.iter().any(|&length| length < 8) { + return Ok(None); + } + let [_, _, _, length4] = streams.length; + if length4 > src.len() { + return Err(Error::CorruptionDetected); + } + let [_, start2, start3, start4] = streams.start; + let ip = [start2 - 8, start3 - 8, start4 - 8, src.len() - 8]; + let segment = dst_size.div_ceil(4); + let op = [0, segment, 2 * segment, 3 * segment]; + if 3 * segment >= dst_size { + return Ok(None); + } + let [ip0, ip1, ip2, ip3] = ip; + Ok(Some(Self { + ip, + op, + bits: [ + init_fast_dstream(src, ip0)?, + init_fast_dstream(src, ip1)?, + init_fast_dstream(src, ip2)?, + init_fast_dstream(src, ip3)?, + ], + iend: streams.start, + })) + } + + /// `HUF_initRemainingDStream` + fn remaining_dstream<'a>( + &self, + src: &'a [u8], + stream: usize, + segment_end: usize, + ) -> Result> { + if self.op.at(stream)? > segment_end { + return Err(Error::CorruptionDetected); + } + let ip = self.ip.at(stream)?; + if ip + 8 < self.iend.at(stream)? { + return Err(Error::CorruptionDetected); + } + BitDStream::resume(src, ip, self.bits.at(stream)?.trailing_zeros()) + } + + /// How many rounds of at most 7 bytes the first stream has input for. The others start above it. + #[inline] + fn input_rounds(&self) -> usize { + let [ip0, ..] = self.ip; + ip0 / 7 + } + + #[inline] + fn streams_are_ordered(&self) -> bool { + self.ip.is_sorted() + } +} + +/// What both fast loops do to a stream after a round. +#[inline(always)] +fn reload_fast(src: &[u8], ip: &mut usize, bits: &mut u64) -> Result<()> { + let ctz = bits.trailing_zeros(); + let nb_bits = ctz & 7; + let nb_bytes = (ctz >> 3) as usize; + *ip = ip.wrapping_sub(nb_bytes); + *bits = (read_le64(src, *ip)? | 1) << nb_bits; + Ok(()) +} + +/// Three parts of `segment_size` bytes, and the rest. +fn split_in_4(dst: &mut [u8], segment_size: usize) -> Result<[&mut [u8]; 4]> { + let (part0, rest) = dst.split_at_mut_checked(segment_size).ok_or(BUG)?; + let (part1, rest) = rest.split_at_mut_checked(segment_size).ok_or(BUG)?; + let (part2, part3) = rest.split_at_mut_checked(segment_size).ok_or(BUG)?; + Ok([part0, part1, part2, part3]) +} + +/// The end of each of the four parts of an output of `dst_size` bytes. +fn segment_ends(dst_size: usize) -> [usize; 4] { + let segment = dst_size.div_ceil(4); + [1, 2, 3, 4].map(|n| (n * segment).min(dst_size)) +} + +// ── X1 ──────────────────────────────────────────────────────────────────── + +/// `HUF_rescaleStats` +fn rescale_stats( + huff_weight: &mut [u8], + rank_val: &mut [u32; NB_RANKS], + table_log: u32, + target_table_log: u32, +) -> Result { + if table_log > target_table_log { + return Ok(table_log); + } + if table_log < target_table_log { + let scale = (target_table_log - table_log) as usize; + for weight in huff_weight { + if *weight != 0 { + *weight += scale as u8; + } + } + let target = target_table_log as usize; + rank_val.move_within(1, scale + 1, target - scale)?; + rank_val.span_mut(1, scale)?.fill(0); + } + Ok(target_table_log) +} + +/// `HUF_readDTableX1_wksp`, once the description of the table is read. +fn build_dtable_x1(dtable: &mut ExactTable, stats: &Code) -> Result<()> { + let mut rank_val = stats.rank_stats; + let mut rank_start = [0u32; NB_RANKS]; + let mut symbols = [0u8; NB_SYMBOLS]; + let mut huff_weight = stats.weights; + let weights = huff_weight.sub_mut(..stats.nb_symbols as usize)?; + + // An X1 table takes half the room of an X2 table, so its log may be one more than the capacity. + let max_table_log = HUF_TABLELOG_MAX + 1; + let target_table_log = max_table_log.min(HUF_DECODER_FAST_TABLELOG); + let table_log = rescale_stats(weights, &mut rank_val, stats.table_log, target_table_log)?; + if table_log > max_table_log { + return Err(Error::TableLogTooLarge); + } + dtable.table_log = table_log; + + // Sort the symbols by weight. + let mut next_rank_start = 0u32; + for (start, &count) in rank_start + .iter_mut() + .zip(&rank_val) + .take(table_log as usize + 1) + { + *start = next_rank_start; + next_rank_start += count; + } + for (symbol, &weight) in weights.iter().enumerate() { + let start = rank_start.at_mut(usize::from(weight))?; + symbols.put(*start as usize, symbol as u8)?; + *start += 1; + } + + let [rank0, ..] = rank_val; + let mut symbol = rank0 as usize; + let mut u_start = 0usize; + for w in 1..=table_log { + let symbol_count = rank_val.at(w as usize)? as usize; + let length = (1usize << w) >> 1; + let nb_bits = (table_log + 1 - w) as u16; + for &byte in symbols.span(symbol, symbol_count)? { + dtable + .entries + .span_mut(u_start, length)? + .fill((u16::from(byte) << 8) | nb_bits); + u_start += length; + } + symbol += symbol_count; + } + Ok(()) +} + +/// `HUF_decodeSymbolX1` +#[inline(always)] +fn decode_symbol_x1(bit_d: &mut BitDStream<'_>, dt: &[u16; TABLE_SIZE], dt_log: u32) -> u8 { + let entry = entry_x1(dt, bit_d.look_bits_fast(dt_log)); + bit_d.skip_bits(u32::from(entry & 0xFF)); + (entry >> 8) as u8 +} + +/// `HUF_decodeStreamX1`: fills `out`. +#[inline] +fn decode_stream_x1( + out: &mut [u8], + bit_d: &mut BitDStream<'_>, + dt: &[u16; TABLE_SIZE], + dt_log: u32, +) { + let end = out.len(); + let mut p = 0usize; + if end > 3 { + while (bit_d.reload() == DStreamStatus::Unfinished) & (p + 3 < end) { + if let Ok(four) = out.array_at_mut::<4>(p) { + for byte in four { + *byte = decode_symbol_x1(bit_d, dt, dt_log); + } + } + p += 4; + } + } else { + bit_d.reload(); + } + for byte in out.get_mut(p..).unwrap_or_default() { + *byte = decode_symbol_x1(bit_d, dt, dt_log); + } +} + +/// `HUF_decompress1X1_usingDTable_internal_body` +fn decompress_1x1_using_dtable( + dst: &mut [u8], + src: &[u8], + dtable: &ExactTable, +) -> Result { + let mut bit_d = BitDStream::new(src)?; + decode_stream_x1(dst, &mut bit_d, &dtable.entries, dtable.table_log); + if !bit_d.is_at_end() { + return Err(Error::CorruptionDetected); + } + Ok(dst.len()) +} + +/// `HUF_decompress4X1_usingDTable_internal_body` +fn decompress_4x1_body(dst: &mut [u8], src: &[u8], dtable: &ExactTable) -> Result { + if src.len() < 10 { + return Err(Error::CorruptionDetected); + } + if dst.len() < 6 { + return Err(Error::CorruptionDetected); + } + let dst_size = dst.len(); + let dt = &dtable.entries; + let dt_log = dtable.table_log; + let streams = Streams::new(src)?; + let [_, _, _, length4] = streams.length; + if length4 > src.len() { + return Err(Error::CorruptionDetected); + } + let segment_size = dst_size.div_ceil(4); + if 3 * segment_size > dst_size { + return Err(Error::CorruptionDetected); + } + let [mut bit_d1, mut bit_d2, mut bit_d3, mut bit_d4] = streams.open(src)?; + + let [out1, out2, out3, out4] = split_in_4(dst, segment_size)?; + // The four streams advance together, by `done`, and the last part is the shortest. + let mut done = 0usize; + if out4.len() >= 8 { + let mut end_signal = true; + while end_signal & (done + 3 < out4.len()) { + let four1 = out1.array_at_mut::<4>(done)?; + let four2 = out2.array_at_mut::<4>(done)?; + let four3 = out3.array_at_mut::<4>(done)?; + let four4 = out4.array_at_mut::<4>(done)?; + for (((byte1, byte2), byte3), byte4) in + four1.iter_mut().zip(four2).zip(four3).zip(four4) + { + *byte1 = decode_symbol_x1(&mut bit_d1, dt, dt_log); + *byte2 = decode_symbol_x1(&mut bit_d2, dt, dt_log); + *byte3 = decode_symbol_x1(&mut bit_d3, dt, dt_log); + *byte4 = decode_symbol_x1(&mut bit_d4, dt, dt_log); + } + done += 4; + end_signal &= bit_d1.reload_fast() == DStreamStatus::Unfinished; + end_signal &= bit_d2.reload_fast() == DStreamStatus::Unfinished; + end_signal &= bit_d3.reload_fast() == DStreamStatus::Unfinished; + end_signal &= bit_d4.reload_fast() == DStreamStatus::Unfinished; + } + } + + decode_stream_x1(out1.sub_mut(done..)?, &mut bit_d1, dt, dt_log); + decode_stream_x1(out2.sub_mut(done..)?, &mut bit_d2, dt, dt_log); + decode_stream_x1(out3.sub_mut(done..)?, &mut bit_d3, dt, dt_log); + decode_stream_x1(out4.sub_mut(done..)?, &mut bit_d4, dt, dt_log); + + if !(bit_d1.is_at_end() & bit_d2.is_at_end() & bit_d3.is_at_end() & bit_d4.is_at_end()) { + return Err(Error::CorruptionDetected); + } + Ok(dst_size) +} + +/// `HUF_decompress4X1_usingDTable_internal_fast_c_loop` +fn fast_loop_4x1( + args: &mut FastArgs, + dst: &mut [u8], + src: &[u8], + dt: &[u16; TABLE_SIZE], +) -> Result<()> { + let [_, segment_size, ..] = args.op; + let [part0, part1, part2, part3] = split_in_4(dst, segment_size)?; + // How much of each part is written. It is the same for all of them. + let mut done = 0usize; + + let [mut ip0, mut ip1, mut ip2, mut ip3] = args.ip; + let [mut bits0, mut bits1, mut bits2, mut bits3] = args.bits; + + #[inline(always)] + fn decode(bits: &mut u64, dt: &[u16; TABLE_SIZE]) -> u8 { + let entry = entry_x1(dt, *bits >> 53); + *bits <<= entry & 0x3F; + (entry >> 8) as u8 + } + + loop { + // Each round writes 5 bytes to each part, of which the last is the shortest, and reads at most 7 of each stream. + let rounds = ((part3.len() - done) / 5).min(ip0 / 7); + if rounds == 0 { + break; + } + if !(ip0 <= ip1 && ip1 <= ip2 && ip2 <= ip3) { + break; + } + let (out0, _) = part0.span_mut(done, rounds * 5)?.as_chunks_mut::<5>(); + let (out1, _) = part1.span_mut(done, rounds * 5)?.as_chunks_mut::<5>(); + let (out2, _) = part2.span_mut(done, rounds * 5)?.as_chunks_mut::<5>(); + let (out3, _) = part3.span_mut(done, rounds * 5)?.as_chunks_mut::<5>(); + for (((out0, out1), out2), out3) in out0.iter_mut().zip(out1).zip(out2).zip(out3) { + for (((byte0, byte1), byte2), byte3) in out0.iter_mut().zip(out1).zip(out2).zip(out3) { + *byte0 = decode(&mut bits0, dt); + *byte1 = decode(&mut bits1, dt); + *byte2 = decode(&mut bits2, dt); + *byte3 = decode(&mut bits3, dt); + } + reload_fast(src, &mut ip0, &mut bits0)?; + reload_fast(src, &mut ip1, &mut bits1)?; + reload_fast(src, &mut ip2, &mut bits2)?; + reload_fast(src, &mut ip3, &mut bits3)?; + } + done += rounds * 5; + } + + args.ip = [ip0, ip1, ip2, ip3]; + args.bits = [bits0, bits1, bits2, bits3]; + args.op = args.op.map(|start| start + done); + Ok(()) +} + +/// `HUF_decompress4X1_usingDTable_internal_fast`. `None` if the input has to be decoded by the other loop. +fn decompress_4x1_fast( + dst: &mut [u8], + src: &[u8], + dtable: &ExactTable, +) -> Result> { + let Some(mut args) = FastArgs::new(dst.len(), src, dtable.table_log)? else { + return Ok(None); + }; + fast_loop_4x1(&mut args, dst, src, &dtable.entries)?; + + for (stream, segment_end) in segment_ends(dst.len()).into_iter().enumerate() { + let mut bit_d = args.remaining_dstream(src, stream, segment_end)?; + let out = dst.sub_mut(args.op.at(stream)?..segment_end)?; + decode_stream_x1(out, &mut bit_d, &dtable.entries, HUF_DECODER_FAST_TABLELOG); + } + Ok(Some(dst.len())) +} + +/// `HUF_decompress4X1_usingDTable_internal` +fn decompress_4x1_using_dtable( + dst: &mut [u8], + src: &[u8], + dtable: &ExactTable, +) -> Result { + if let Some(size) = decompress_4x1_fast(dst, src, dtable)? { + return Ok(size); + } + decompress_4x1_body(dst, src, dtable) +} + +// ── X2 ──────────────────────────────────────────────────────────────────── + +type RankValCol = [u32; NB_RANKS]; + +/// `HUF_buildDEltX2U32` +#[inline(always)] +fn build_delt_x2(symbol: u32, nb_bits: u32, base_seq: u32, level: u32) -> u32 { + let seq = if level == 1 { + symbol + } else { + base_seq + (symbol << 8) + }; + seq + (nb_bits << 16) + (level << 24) +} + +/// `HUF_fillDTableX2ForWeight` +fn fill_dtable_x2_for_weight( + dtable_rank: &mut [u32], + symbols: &[u8], + nb_bits: u32, + table_log: u32, + base_seq: u32, + level: u32, +) -> Result<()> { + let length = 1usize << (table_log.wrapping_sub(nb_bits) & 0x1F); + let entries = dtable_rank.sub_mut(..length * symbols.len())?; + for (entries, &symbol) in entries.chunks_exact_mut(length).zip(symbols) { + entries.fill(build_delt_x2(u32::from(symbol), nb_bits, base_seq, level)); + } + Ok(()) +} + +/// `HUF_fillDTableX2Level2` +fn fill_dtable_x2_level2( + dtable: &mut [u32], + target_log: u32, + consumed_bits: u32, + rank_val: &RankValCol, + min_weight: usize, + max_weight1: usize, + sorted_symbols: &[u8; NB_SYMBOLS], + rank_start: &[u32; NB_RANKS + 2], + nb_bits_baseline: u32, + base_seq: u32, +) -> Result<()> { + // The codes that are too long for a second symbol to fit hold the first symbol alone. + if min_weight > 1 { + let length = 1usize << (target_log.wrapping_sub(consumed_bits) & 0x1F); + let skip_size = rank_val.at(min_weight)? as usize; + let filled = match length { + 2 | 4 => length, + _ => skip_size.next_multiple_of(8), + }; + let first_symbol = base_seq; + dtable + .sub_mut(..filled)? + .fill(build_delt_x2(first_symbol, consumed_bits, 0, 1)); + } + + for w in min_weight..max_weight1 { + let begin = rank_start.at(w)? as usize; + let end = rank_start.at(w + 1)? as usize; + let nb_bits = nb_bits_baseline - w as u32; + let total_bits = nb_bits + consumed_bits; + fill_dtable_x2_for_weight( + dtable.sub_mut(rank_val.at(w)? as usize..)?, + sorted_symbols.sub(begin..end)?, + total_bits, + target_log, + base_seq, + 2, + )?; + } + Ok(()) +} + +/// `HUF_fillDTableX2` +fn fill_dtable_x2( + dtable: &mut [u32; TABLE_SIZE], + target_log: u32, + sorted_list: &[u8; NB_SYMBOLS], + rank_start: &[u32; NB_RANKS + 2], + rank_val_origin: &[RankValCol; HUF_TABLELOG_MAX as usize], + max_weight: u32, + nb_bits_baseline: u32, +) -> Result<()> { + let [rank_val, ..] = rank_val_origin; + let scale_log = nb_bits_baseline as i32 - target_log as i32; + let min_bits = nb_bits_baseline - max_weight; + let w_end = max_weight as usize + 1; + + for w in 1..w_end { + let begin = rank_start.at(w)? as usize; + let end = rank_start.at(w + 1)? as usize; + let nb_bits = nb_bits_baseline - w as u32; + + if target_log.wrapping_sub(nb_bits) >= min_bits { + // Enough room for a second symbol. + let mut start = rank_val.at(w)? as usize; + let length = 1usize << (target_log.wrapping_sub(nb_bits) & 0x1F); + let min_weight = (nb_bits as i32 + scale_log).max(1) as usize; + for &symbol in sorted_list.sub(begin..end)? { + fill_dtable_x2_level2( + dtable.sub_mut(start..)?, + target_log, + nb_bits, + rank_val_origin.get(nb_bits as usize).ok_or(BUG)?, + min_weight, + w_end, + sorted_list, + rank_start, + nb_bits_baseline, + u32::from(symbol), + )?; + start += length; + } + } else { + fill_dtable_x2_for_weight( + dtable.sub_mut(rank_val.at(w)? as usize..)?, + sorted_list.sub(begin..end)?, + nb_bits, + target_log, + 0, + 1, + )?; + } + } + Ok(()) +} + +/// `HUF_readDTableX2_wksp`, once the description of the table is read. +fn build_dtable_x2(dtable: &mut ExactTable, stats: &Code) -> Result<()> { + let mut rank_val = [[0u32; NB_RANKS]; HUF_TABLELOG_MAX as usize]; + let rank_stats = stats.rank_stats; + // `rank_start0[w + 1]` is where the symbols of weight `w` start, until sorting moves it to where they end. + let mut rank_start0 = [0u32; NB_RANKS + 2]; + let mut sorted_symbol = [0u8; NB_SYMBOLS]; + let weight_list = stats.weights; + let mut max_table_log = HUF_TABLELOG_MAX; + let table_log = stats.table_log; + + if table_log > max_table_log { + return Err(Error::TableLogTooLarge); + } + if table_log <= HUF_DECODER_FAST_TABLELOG && max_table_log > HUF_DECODER_FAST_TABLELOG { + max_table_log = HUF_DECODER_FAST_TABLELOG; + } + + let mut max_w = table_log as usize; + while rank_stats.at(max_w)? == 0 { + max_w = max_w.checked_sub(1).ok_or(BUG)?; + } + + let mut next_rank_start = 0u32; + for w in 1..=max_w { + rank_start0.put(w + 1, next_rank_start)?; + next_rank_start += rank_stats.at(w)?; + } + // The symbols of weight 0 go to the end. + rank_start0.put(1, next_rank_start)?; + rank_start0.put(max_w + 2, next_rank_start)?; + + for (symbol, &weight) in weight_list + .iter() + .enumerate() + .take(stats.nb_symbols as usize) + { + let start = rank_start0.at_mut(usize::from(weight) + 1)?; + sorted_symbol.put(*start as usize, symbol as u8)?; + *start += 1; + } + rank_start0.put(1, 0)?; + + { + let rescale = (max_table_log - table_log) as i32 - 1; + let mut next_rank_val = 0u32; + let [rank_val0, ..] = &mut rank_val; + for w in 1..=max_w { + rank_val0.put(w, next_rank_val)?; + next_rank_val += rank_stats.at(w)? << (w as i32 + rescale); + } + let rank_val0 = *rank_val0; + let min_bits = table_log + 1 - max_w as u32; + for consumed in min_bits..(max_table_log - min_bits + 1) { + let rank_val_col = rank_val.get_mut(consumed as usize).ok_or(BUG)?; + for (shifted, &val) in rank_val_col.iter_mut().zip(&rank_val0).skip(1).take(max_w) { + *shifted = val >> consumed; + } + } + } + + fill_dtable_x2( + &mut dtable.entries, + max_table_log, + &sorted_symbol, + &rank_start0, + &rank_val, + max_w as u32, + table_log + 1, + )?; + + dtable.table_log = max_table_log; + Ok(()) +} + +/// `HUF_decodeSymbolX2`: writes two bytes at `pos` and returns how many of them count. +#[inline(always)] +fn decode_symbol_x2( + out: &mut [u8], + pos: usize, + bit_d: &mut BitDStream<'_>, + dt: &[u32; TABLE_SIZE], + dt_log: u32, +) -> Result { + let entry = entry_x2(dt, bit_d.look_bits_fast(dt_log)); + *out.array_at_mut(pos)? = (entry as u16).to_le_bytes(); + bit_d.skip_bits((entry >> 16) & 0xFF); + Ok((entry >> 24) as usize) +} + +/// `HUF_decodeLastSymbolX2` +#[inline(always)] +fn decode_last_symbol_x2( + out: &mut [u8], + pos: usize, + bit_d: &mut BitDStream<'_>, + dt: &[u32; TABLE_SIZE], + dt_log: u32, +) -> Result<()> { + let entry = entry_x2(dt, bit_d.look_bits_fast(dt_log)); + out.put(pos, entry as u8)?; + let nb_bits = (entry >> 16) & 0xFF; + if entry >> 24 == 1 { + bit_d.skip_bits(nb_bits); + } else if bit_d.bits_consumed < u64::BITS { + bit_d.skip_bits(nb_bits); + // The code of the second symbol, which is not wanted, may be cut off. + if bit_d.bits_consumed > u64::BITS { + bit_d.bits_consumed = u64::BITS; + } + } + Ok(()) +} + +/// `HUF_decodeStreamX2`: fills `out` and returns its length. +#[inline] +fn decode_stream_x2( + out: &mut [u8], + bit_d: &mut BitDStream<'_>, + dt: &[u32; TABLE_SIZE], + dt_log: u32, +) -> Result { + let end = out.len(); + let mut p = 0usize; + + if end >= 8 { + if dt_log <= 11 { + while (bit_d.reload() == DStreamStatus::Unfinished) & (p + 9 < end) { + for _ in 0..5 { + p += decode_symbol_x2(out, p, bit_d, dt, dt_log)?; + } + } + } else { + while (bit_d.reload() == DStreamStatus::Unfinished) & (p + 7 < end) { + for _ in 0..4 { + p += decode_symbol_x2(out, p, bit_d, dt, dt_log)?; + } + } + } + } else { + bit_d.reload(); + } + + if end - p >= 2 { + while (bit_d.reload() == DStreamStatus::Unfinished) & (p + 2 <= end) { + p += decode_symbol_x2(out, p, bit_d, dt, dt_log)?; + } + while p + 2 <= end { + p += decode_symbol_x2(out, p, bit_d, dt, dt_log)?; + } + } + + if p < end { + decode_last_symbol_x2(out, p, bit_d, dt, dt_log)?; + p += 1; + } + Ok(p) +} + +/// `HUF_decompress1X2_usingDTable_internal_body` +fn decompress_1x2_using_dtable( + dst: &mut [u8], + src: &[u8], + dtable: &ExactTable, +) -> Result { + let mut bit_d = BitDStream::new(src)?; + decode_stream_x2(dst, &mut bit_d, &dtable.entries, dtable.table_log)?; + if !bit_d.is_at_end() { + return Err(Error::CorruptionDetected); + } + Ok(dst.len()) +} + +/// `HUF_decompress4X2_usingDTable_internal_body` +fn decompress_4x2_body(dst: &mut [u8], src: &[u8], dtable: &ExactTable) -> Result { + if src.len() < 10 { + return Err(Error::CorruptionDetected); + } + if dst.len() < 6 { + return Err(Error::CorruptionDetected); + } + let oend = dst.len(); + let dt = &dtable.entries; + let dt_log = dtable.table_log; + let streams = Streams::new(src)?; + let [_, _, _, length4] = streams.length; + if length4 > src.len() { + return Err(Error::CorruptionDetected); + } + let segment_size = oend.div_ceil(4); + let op_start2 = segment_size; + let op_start3 = 2 * segment_size; + let op_start4 = 3 * segment_size; + if op_start4 > oend { + return Err(Error::CorruptionDetected); + } + let [mut bit_d1, mut bit_d2, mut bit_d3, mut bit_d4] = streams.open(src)?; + let (mut op1, mut op2, mut op3, mut op4) = (0, op_start2, op_start3, op_start4); + + // A damaged stream can run over the start of the next part, so the parts are not split here. + if oend - op4 >= 8 { + let mut end_signal = true; + while end_signal & (op4 + 7 < oend) { + for _ in 0..4 { + op1 += decode_symbol_x2(dst, op1, &mut bit_d1, dt, dt_log)?; + } + for _ in 0..4 { + op2 += decode_symbol_x2(dst, op2, &mut bit_d2, dt, dt_log)?; + } + end_signal &= bit_d1.reload_fast() == DStreamStatus::Unfinished; + end_signal &= bit_d2.reload_fast() == DStreamStatus::Unfinished; + for _ in 0..4 { + op3 += decode_symbol_x2(dst, op3, &mut bit_d3, dt, dt_log)?; + } + for _ in 0..4 { + op4 += decode_symbol_x2(dst, op4, &mut bit_d4, dt, dt_log)?; + } + end_signal &= bit_d3.reload_fast() == DStreamStatus::Unfinished; + end_signal &= bit_d4.reload_fast() == DStreamStatus::Unfinished; + } + } + + if op1 > op_start2 || op2 > op_start3 || op3 > op_start4 { + return Err(Error::CorruptionDetected); + } + + decode_stream_x2(dst.sub_mut(op1..op_start2)?, &mut bit_d1, dt, dt_log)?; + decode_stream_x2(dst.sub_mut(op2..op_start3)?, &mut bit_d2, dt, dt_log)?; + decode_stream_x2(dst.sub_mut(op3..op_start4)?, &mut bit_d3, dt, dt_log)?; + decode_stream_x2(dst.sub_mut(op4..oend)?, &mut bit_d4, dt, dt_log)?; + + if !(bit_d1.is_at_end() & bit_d2.is_at_end() & bit_d3.is_at_end() & bit_d4.is_at_end()) { + return Err(Error::CorruptionDetected); + } + Ok(oend) +} + +/// `HUF_decompress4X2_usingDTable_internal_fast_c_loop` +fn fast_loop_4x2( + args: &mut FastArgs, + dst: &mut [u8], + src: &[u8], + dt: &[u32; TABLE_SIZE], +) -> Result<()> { + let [_, end0, end1, end2] = args.op; + let oend = [end0, end1, end2, dst.len()]; + + #[inline(always)] + fn decode( + dst: &mut [u8], + op: &mut usize, + bits: &mut u64, + dt: &[u32; TABLE_SIZE], + ) -> Result<()> { + let entry = entry_x2(dt, *bits >> 53); + *dst.array_at_mut(*op)? = (entry as u16).to_le_bytes(); + *bits <<= (entry >> 16) & 0x3F; + *op += (entry >> 24) as usize; + Ok(()) + } + + loop { + // Each round writes at most 10 bytes to each part and reads at most 7 of each stream. + let mut rounds = args.input_rounds(); + for (&end, &op) in oend.iter().zip(&args.op) { + rounds = rounds.min((end - op) / 10); + } + if rounds == 0 { + break; + } + if !args.streams_are_ordered() { + break; + } + let [mut op0, mut op1, mut op2, mut op3] = args.op; + let [mut ip0, mut ip1, mut ip2, mut ip3] = args.ip; + let [mut bits0, mut bits1, mut bits2, mut bits3] = args.bits; + let olimit = op3 + rounds * 5; + loop { + for _ in 0..5 { + decode(dst, &mut op0, &mut bits0, dt)?; + decode(dst, &mut op1, &mut bits1, dt)?; + decode(dst, &mut op2, &mut bits2, dt)?; + decode(dst, &mut op3, &mut bits3, dt)?; + } + reload_fast(src, &mut ip0, &mut bits0)?; + reload_fast(src, &mut ip1, &mut bits1)?; + reload_fast(src, &mut ip2, &mut bits2)?; + reload_fast(src, &mut ip3, &mut bits3)?; + if op3 >= olimit { + break; + } + } + args.op = [op0, op1, op2, op3]; + args.ip = [ip0, ip1, ip2, ip3]; + args.bits = [bits0, bits1, bits2, bits3]; + } + Ok(()) +} + +/// `HUF_decompress4X2_usingDTable_internal_fast`. `None` if the input has to be decoded by the other loop. +fn decompress_4x2_fast( + dst: &mut [u8], + src: &[u8], + dtable: &ExactTable, +) -> Result> { + let Some(mut args) = FastArgs::new(dst.len(), src, dtable.table_log)? else { + return Ok(None); + }; + fast_loop_4x2(&mut args, dst, src, &dtable.entries)?; + + for (stream, segment_end) in segment_ends(dst.len()).into_iter().enumerate() { + let mut bit_d = args.remaining_dstream(src, stream, segment_end)?; + let out = dst.sub_mut(args.op.at(stream)?..segment_end)?; + decode_stream_x2(out, &mut bit_d, &dtable.entries, HUF_DECODER_FAST_TABLELOG)?; + } + Ok(Some(dst.len())) +} + +/// `HUF_decompress4X2_usingDTable_internal` +fn decompress_4x2_using_dtable( + dst: &mut [u8], + src: &[u8], + dtable: &ExactTable, +) -> Result { + if let Some(size) = decompress_4x2_fast(dst, src, dtable)? { + return Ok(size); + } + decompress_4x2_body(dst, src, dtable) +} + +// ── choosing between X1 and X2 ──────────────────────────────────────────── + +/// `algoTime`: for each sixteenth of compression ratio, the time to build a table and to decode 256 bytes, for X1 and X2. +#[rustfmt::skip] +const ALGO_TIME: [[(u32, u32); 2]; 16] = [ + [( 0, 0), ( 1, 1)], + [( 0, 0), ( 1, 1)], + [( 150, 216), ( 381, 119)], + [( 170, 205), ( 514, 112)], + [( 177, 199), ( 539, 110)], + [( 197, 194), ( 644, 107)], + [( 221, 192), ( 735, 107)], + [( 256, 189), ( 881, 106)], + [( 359, 188), (1167, 109)], + [( 582, 187), (1570, 114)], + [( 688, 187), (1712, 122)], + [( 825, 186), (1965, 136)], + [( 976, 185), (2131, 150)], + [(1180, 186), (2070, 175)], + [(1377, 185), (1731, 202)], + [(1412, 185), (1695, 202)], +]; + +/// `HUF_selectDecoder`: whether X2 is expected to be the faster one. `dst_size` is not 0. +pub(super) fn select_decoder_x2(dst_size: usize, src_size: usize) -> Result { + let q = if src_size >= dst_size { + 15 + } else { + src_size * 16 / dst_size + }; + let d256 = (dst_size >> 8) as u32; + let [(table0, decode0), (table1, decode1)] = ALGO_TIME.at(q)?; + let dtime0 = table0 + decode0 * d256; + let mut dtime1 = table1 + decode1 * d256; + // A small advantage to the algorithm that uses less memory. + dtime1 += dtime1 >> 5; + Ok(dtime1 < dtime0) +} + +/// `HUF_decompress1X_usingDTable` +#[cold] +#[inline(never)] +pub(super) fn decompress_1x( + dst: &mut [u8], + src: &[u8], + code: &Code, + table_type: TableType, +) -> Result { + match table_type { + TableType::X1 => { + let mut dtable = ExactTable::new(); + build_dtable_x1(&mut dtable, code)?; + decompress_1x1_using_dtable(dst, src, &dtable) + } + TableType::X2 => { + let mut dtable = ExactTable::new(); + build_dtable_x2(&mut dtable, code)?; + decompress_1x2_using_dtable(dst, src, &dtable) + } + } +} + +/// `HUF_decompress4X_usingDTable` +#[cold] +#[inline(never)] +pub(super) fn decompress_4x( + dst: &mut [u8], + src: &[u8], + code: &Code, + table_type: TableType, +) -> Result { + match table_type { + TableType::X1 => { + let mut dtable = ExactTable::new(); + build_dtable_x1(&mut dtable, code)?; + decompress_4x1_using_dtable(dst, src, &dtable) + } + TableType::X2 => { + let mut dtable = ExactTable::new(); + build_dtable_x2(&mut dtable, code)?; + decompress_4x2_using_dtable(dst, src, &dtable) + } + } +} diff --git a/src/zstd/decompress/mod.rs b/src/zstd/decompress/mod.rs new file mode 100644 index 000000000000..7259d5dffb11 --- /dev/null +++ b/src/zstd/decompress/mod.rs @@ -0,0 +1,1441 @@ +//! `zstd_decompress.c` and `zstd_ddict.c`: frames, dictionaries and streaming. + +mod block; +mod huf; +mod huf_exact; +mod seq_table; + +use std::borrow::Cow; + +use block::{ + BlockDecoder, BlockOutput, EntropyDTables, ExtDict, Growth, OutputBuffer, Streaming, + get_c_block_size, +}; +use seq_table::{LL_CODES, ML_CODES, OF_CODES, SeqTableSources, TableSource}; + +use crate::common::entropy::read_ncount; +use crate::common::xxhash::Xxh64; +use crate::common::{ + BlockType, LL_FSE_LOG, MAX_LL, MAX_ML, MAX_OFF, ML_FSE_LOG, OFF_FSE_LOG, REP_START_VALUE, + WILDCOPY_OVERLENGTH, ZSTD_BLOCKHEADERSIZE, ZSTD_BLOCKSIZE_MAX, ZSTD_CONTENTSIZE_ERROR, + ZSTD_CONTENTSIZE_UNKNOWN, ZSTD_DID_FIELD_SIZE, ZSTD_FCS_FIELD_SIZE, ZSTD_FRAMEHEADERSIZE_MAX, + ZSTD_FRAMEIDSIZE, ZSTD_MAGIC_DICTIONARY, ZSTD_MAGIC_SKIPPABLE_MASK, ZSTD_MAGIC_SKIPPABLE_START, + ZSTD_MAGICNUMBER, ZSTD_REP_NUM, ZSTD_SKIPPABLEHEADERSIZE, ZSTD_WINDOWLOG_ABSOLUTEMIN, + ZSTD_WINDOWLOG_LIMIT_DEFAULT, ZSTD_WINDOWLOG_MAX, ZSTD_WORKSPACETOOLARGE_FACTOR, + ZSTD_WORKSPACETOOLARGE_MAXDURATION, +}; +use crate::error::{Error, Result}; +use crate::util::{SliceExt, append, read_le16, read_le32, read_le64, try_copy}; +use crate::{InBuffer, OutBuffer}; + +/// `ZSTD_FRAMEHEADERSIZE_PREFIX`: what it takes to know the size of a frame header. +const FRAMEHEADERSIZE_PREFIX: usize = 5; +const FRAMEHEADERSIZE_MIN: usize = 6; + +const ZSTD_MAXWINDOWSIZE_DEFAULT: usize = (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) + 1; +const ZSTD_NO_FORWARD_PROGRESS_MAX: u32 = 16; + +/// `ZSTD_d_windowLogMax`, the only parameter of a [`DCtx`] that `node:zlib` can name. +pub const ZSTD_D_WINDOW_LOG_MAX: i32 = 100; + +// ── frame headers ───────────────────────────────────────────────────────── + +#[derive(Clone, Copy, PartialEq, Eq)] +enum FrameType { + Frame, + Skippable, +} + +/// `ZSTD_FrameHeader` +#[derive(Clone, Copy)] +struct FrameHeader { + /// [`ZSTD_CONTENTSIZE_UNKNOWN`] if the header does not say. For a skippable frame, the size of its content. + frame_content_size: u64, + window_size: u64, + block_size_max: usize, + frame_type: FrameType, + header_size: usize, + dict_id: u32, + checksum_flag: bool, +} + +impl FrameHeader { + const EMPTY: Self = Self { + frame_content_size: 0, + window_size: 0, + block_size_max: 0, + frame_type: FrameType::Frame, + header_size: 0, + dict_id: 0, + checksum_flag: false, + }; +} + +enum HeaderStatus { + Complete, + /// The header cannot be read from fewer bytes than this. + Needs(usize), +} + +#[inline] +fn is_skippable_magic(magic: u32) -> bool { + magic & ZSTD_MAGIC_SKIPPABLE_MASK == ZSTD_MAGIC_SKIPPABLE_START +} + +/// `ZSTD_frameHeaderSize_internal` +fn frame_header_size(src: &[u8]) -> Result { + if src.len() < FRAMEHEADERSIZE_PREFIX { + return Err(Error::SrcSizeWrong); + } + let fhd = src.at(FRAMEHEADERSIZE_PREFIX - 1)?; + let dict_id = usize::from(fhd & 3); + let single_segment = (fhd >> 5) & 1 != 0; + let fcs_id = usize::from(fhd >> 6); + Ok(FRAMEHEADERSIZE_PREFIX + + usize::from(!single_segment) + + ZSTD_DID_FIELD_SIZE.at(dict_id)? + + ZSTD_FCS_FIELD_SIZE.at(fcs_id)? + + usize::from(single_segment && fcs_id == 0)) +} + +/// `ZSTD_getFrameHeader`: sets `header` if the one at the start of `src` is complete. +fn get_frame_header(header: &mut FrameHeader, src: &[u8]) -> Result { + const MIN_INPUT_SIZE: usize = FRAMEHEADERSIZE_PREFIX; + + if src.len() < MIN_INPUT_SIZE { + if !src.is_empty() { + // Refuse what cannot become a magic number. + let known = src.sub(..src.len().min(4))?; + let mut hbuf = ZSTD_MAGICNUMBER.to_le_bytes(); + hbuf.write_at(0, known)?; + if u32::from_le_bytes(hbuf) != ZSTD_MAGICNUMBER { + hbuf = ZSTD_MAGIC_SKIPPABLE_START.to_le_bytes(); + hbuf.write_at(0, known)?; + if !is_skippable_magic(u32::from_le_bytes(hbuf)) { + return Err(Error::PrefixUnknown); + } + } + } + return Ok(HeaderStatus::Needs(MIN_INPUT_SIZE)); + } + + let magic = read_le32(src, 0)?; + if magic != ZSTD_MAGICNUMBER { + if is_skippable_magic(magic) { + if src.len() < ZSTD_SKIPPABLEHEADERSIZE { + return Ok(HeaderStatus::Needs(ZSTD_SKIPPABLEHEADERSIZE)); + } + *header = FrameHeader { + frame_type: FrameType::Skippable, + dict_id: magic - ZSTD_MAGIC_SKIPPABLE_START, + header_size: ZSTD_SKIPPABLEHEADERSIZE, + frame_content_size: u64::from(read_le32(src, ZSTD_FRAMEIDSIZE)?), + ..FrameHeader::EMPTY + }; + return Ok(HeaderStatus::Complete); + } + return Err(Error::PrefixUnknown); + } + + let fhsize = frame_header_size(src)?; + if src.len() < fhsize { + return Ok(HeaderStatus::Needs(fhsize)); + } + + let fhd_byte = src.at(MIN_INPUT_SIZE - 1)?; + let mut pos = MIN_INPUT_SIZE; + let dict_id_size_code = fhd_byte & 3; + let checksum_flag = (fhd_byte >> 2) & 1 != 0; + let single_segment = (fhd_byte >> 5) & 1 != 0; + let fcs_id = fhd_byte >> 6; + let mut window_size = 0u64; + let mut dict_id = 0u32; + let mut frame_content_size = ZSTD_CONTENTSIZE_UNKNOWN; + if fhd_byte & 0x08 != 0 { + return Err(Error::FrameParameterUnsupported); + } + + if !single_segment { + let wl_byte = src.at(pos)?; + pos += 1; + let window_log = u32::from(wl_byte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN; + if window_log > ZSTD_WINDOWLOG_MAX { + return Err(Error::FrameParameterWindowTooLarge); + } + window_size = 1u64 << window_log; + window_size += (window_size >> 3) * u64::from(wl_byte & 7); + } + match dict_id_size_code { + 1 => { + dict_id = u32::from(src.at(pos)?); + pos += 1; + } + 2 => { + dict_id = u32::from(read_le16(src, pos)?); + pos += 2; + } + 3 => { + dict_id = read_le32(src, pos)?; + pos += 4; + } + _ => {} + } + match fcs_id { + 1 => frame_content_size = u64::from(read_le16(src, pos)?) + 256, + 2 => frame_content_size = u64::from(read_le32(src, pos)?), + 3 => frame_content_size = read_le64(src, pos)?, + _ => { + if single_segment { + frame_content_size = u64::from(src.at(pos)?); + } + } + } + if single_segment { + window_size = frame_content_size; + } + + *header = FrameHeader { + frame_type: FrameType::Frame, + frame_content_size, + window_size, + block_size_max: window_size.min(ZSTD_BLOCKSIZE_MAX as u64) as usize, + header_size: fhsize, + dict_id, + checksum_flag, + }; + Ok(HeaderStatus::Complete) +} + +/// `ZSTD_getFrameContentSize`: [`ZSTD_CONTENTSIZE_UNKNOWN`] if the frame at the start of `src` does not say, and +/// [`ZSTD_CONTENTSIZE_ERROR`] if there is no whole frame header. +pub fn get_frame_content_size(src: &[u8]) -> u64 { + let mut header = FrameHeader::EMPTY; + match get_frame_header(&mut header, src) { + Ok(HeaderStatus::Complete) => match header.frame_type { + FrameType::Skippable => 0, + FrameType::Frame => header.frame_content_size, + }, + _ => ZSTD_CONTENTSIZE_ERROR, + } +} + +/// The size of the skippable frame whose header is at the start of `src`. +fn skippable_frame_size(src: &[u8]) -> Result { + if src.len() < ZSTD_SKIPPABLEHEADERSIZE { + return Err(Error::SrcSizeWrong); + } + let size_u32 = read_le32(src, ZSTD_FRAMEIDSIZE)?; + if size_u32 + .checked_add(ZSTD_SKIPPABLEHEADERSIZE as u32) + .is_none() + { + return Err(Error::FrameParameterUnsupported); + } + Ok(ZSTD_SKIPPABLEHEADERSIZE + size_u32 as usize) +} + +/// `readSkippableFrameSize` +fn read_skippable_frame_size(src: &[u8]) -> Result { + let skippable_size = skippable_frame_size(src)?; + if skippable_size > src.len() { + return Err(Error::SrcSizeWrong); + } + Ok(skippable_size) +} + +/// How far a frame goes. +pub enum FrameExtent { + /// It is this many bytes. + Whole(usize), + /// It goes on past what there is of it, and is at least this many bytes. + AtLeast(usize), +} + +/// How far the frame at the start of `src` goes, for when where it ends is not known in advance. +pub fn frame_extent(src: &[u8]) -> Result { + if src.len() >= ZSTD_SKIPPABLEHEADERSIZE && is_skippable_magic(read_le32(src, 0)?) { + let size = skippable_frame_size(src)?; + return Ok(if size <= src.len() { + FrameExtent::Whole(size) + } else { + FrameExtent::AtLeast(size) + }); + } + let mut header = FrameHeader::EMPTY; + if let HeaderStatus::Needs(size) = get_frame_header(&mut header, src)? { + return Ok(FrameExtent::AtLeast(size)); + } + let mut ip = header.header_size; + loop { + let Some(remaining) = src + .get(ip..) + .filter(|rest| rest.len() >= ZSTD_BLOCKHEADERSIZE) + else { + return Ok(FrameExtent::AtLeast(ip + ZSTD_BLOCKHEADERSIZE)); + }; + let (c_block_size, properties) = get_c_block_size(remaining)?; + ip += ZSTD_BLOCKHEADERSIZE + c_block_size; + if properties.last_block { + break; + } + } + if header.checksum_flag { + ip += 4; + } + Ok(if ip <= src.len() { + FrameExtent::Whole(ip) + } else { + FrameExtent::AtLeast(ip) + }) +} + +/// `ZSTD_findFrameCompressedSize`: how many bytes the frame at the start of `src` takes. +fn find_frame_compressed_size(src: &[u8]) -> Result { + match frame_extent(src)? { + FrameExtent::Whole(size) => Ok(size), + FrameExtent::AtLeast(_) => Err(Error::SrcSizeWrong), + } +} + +/// `ZSTD_findDecompressedSize`: the size of all the frames of `src` together, once decompressed. +/// [`ZSTD_CONTENTSIZE_UNKNOWN`] if one of them does not say, [`ZSTD_CONTENTSIZE_ERROR`] if `src` is not whole frames. +pub(crate) fn find_decompressed_size(mut src: &[u8]) -> u64 { + let mut total_dst_size = 0u64; + while src.len() >= FRAMEHEADERSIZE_PREFIX { + let frame_src_size = if read_le32(src, 0).is_ok_and(is_skippable_magic) { + read_skippable_frame_size(src) + } else { + let fcs = get_frame_content_size(src); + if fcs >= ZSTD_CONTENTSIZE_ERROR { + return fcs; + } + match total_dst_size.checked_add(fcs) { + Some(total) => total_dst_size = total, + None => return ZSTD_CONTENTSIZE_ERROR, + } + find_frame_compressed_size(src) + }; + match frame_src_size.ok().and_then(|size| src.get(size..)) { + Some(rest) => src = rest, + None => return ZSTD_CONTENTSIZE_ERROR, + } + } + if !src.is_empty() { + return ZSTD_CONTENTSIZE_ERROR; + } + total_dst_size +} + +// ── dictionaries ────────────────────────────────────────────────────────── + +/// `ZSTD_loadDEntropy`: reads the tables at the start of a dictionary, and returns the last offsets that it sets. +fn load_d_entropy(entropy: &mut EntropyDTables, dict: &[u8]) -> Result<[u32; ZSTD_REP_NUM]> { + const CORRUPTED: Error = Error::DictionaryCorrupted; + if dict.len() <= 8 { + return Err(CORRUPTED); + } + // The magic number and the ID. + let mut pos = 8usize; + + pos += huf::read_dtable_x2( + &mut entropy.huf_table, + dict.sub(pos..)?, + &mut entropy.workspace, + ) + .map_err(|_| CORRUPTED)?; + + { + let header = read_ncount(&mut entropy.workspace.counts, MAX_OFF, dict.sub(pos..)?) + .map_err(|_| CORRUPTED)?; + if header.max_symbol_value > MAX_OFF || header.table_log > OFF_FSE_LOG { + return Err(CORRUPTED); + } + entropy.seq.of.build( + header.max_symbol_value as usize + 1, + &OF_CODES, + header.table_log, + &mut entropy.workspace, + )?; + pos += header.size; + } + { + let header = read_ncount(&mut entropy.workspace.counts, MAX_ML, dict.sub(pos..)?) + .map_err(|_| CORRUPTED)?; + if header.max_symbol_value > MAX_ML || header.table_log > ML_FSE_LOG { + return Err(CORRUPTED); + } + entropy.seq.ml.build( + header.max_symbol_value as usize + 1, + &ML_CODES, + header.table_log, + &mut entropy.workspace, + )?; + pos += header.size; + } + { + let header = read_ncount(&mut entropy.workspace.counts, MAX_LL, dict.sub(pos..)?) + .map_err(|_| CORRUPTED)?; + if header.max_symbol_value > MAX_LL || header.table_log > LL_FSE_LOG { + return Err(CORRUPTED); + } + entropy.seq.ll.build( + header.max_symbol_value as usize + 1, + &LL_CODES, + header.table_log, + &mut entropy.workspace, + )?; + pos += header.size; + } + + if pos + 12 > dict.len() { + return Err(CORRUPTED); + } + let dict_content_size = dict.len() - (pos + 12); + let mut reps = [0; ZSTD_REP_NUM]; + for rep in &mut reps { + *rep = read_le32(dict, pos)?; + pos += 4; + if *rep == 0 || *rep as usize > dict_content_size { + return Err(CORRUPTED); + } + } + Ok(reps) +} + +/// `ZSTD_dictContentType_e` +#[derive(Clone, Copy, PartialEq, Eq)] +enum DictContentType { + /// A dictionary with tables if it starts with the magic number of one, else content to refer to. + Auto, + RawContent, +} + +/// `ZSTD_DDict`: a dictionary that is ready to decompress with. +pub struct DDict<'a> { + /// All of it. Matches can reach into the tables too. + content: Cow<'a, [u8]>, + /// The tables, if the dictionary has some. + entropy: Option>, + /// The last offsets that a frame starts with. + rep: [u32; ZSTD_REP_NUM], + dict_id: u32, +} + +impl<'a> DDict<'a> { + /// `ZSTD_initDDict_internal` + fn new(content: Cow<'a, [u8]>, content_type: DictContentType) -> Result { + let mut ddict = Self { + content, + entropy: None, + rep: REP_START_VALUE, + dict_id: 0, + }; + let dict = &*ddict.content; + if content_type == DictContentType::RawContent + || dict.len() < 8 + || read_le32(dict, 0)? != ZSTD_MAGIC_DICTIONARY + { + return Ok(ddict); + } + ddict.dict_id = read_le32(dict, ZSTD_FRAMEIDSIZE)?; + let mut entropy = Box::new(EntropyDTables::new()); + ddict.rep = load_d_entropy(&mut entropy, dict).map_err(|_| Error::DictionaryCorrupted)?; + ddict.entropy = Some(entropy); + Ok(ddict) + } + + /// `ZSTD_createDDict_byReference` + pub fn by_reference(dict: &'a [u8]) -> Result { + Self::new(Cow::Borrowed(dict), DictContentType::Auto) + } + + /// What `ZSTD_DCtx_refPrefix` makes of `prefix`: content to refer to, whatever it starts with. + pub(crate) fn prefix(prefix: &'a [u8]) -> Result { + Self::new(Cow::Borrowed(prefix), DictContentType::RawContent) + } + + /// The address one past the content, which stands for the dictionary where the C code compares pointers. + fn end_address(&self) -> usize { + self.content.as_ptr_range().end.addr() + } +} + +// ── the context ─────────────────────────────────────────────────────────── + +/// `ZSTD_dStage` +#[derive(Clone, Copy, PartialEq, Eq)] +enum DStage { + GetFrameHeaderSize, + DecodeBlockHeader, + DecompressBlock, + DecompressLastBlock, + CheckChecksum, + SkipFrame, +} + +/// `ZSTD_dStreamStage` +#[derive(Clone, Copy, PartialEq, Eq)] +enum DStreamStage { + Init, + /// `lh_size` bytes of the frame header are in `header_buffer`. + LoadHeader { + lh_size: usize, + }, + Read, + Load, + Flush, + /// The frame is given up. The C library leaves what a call does after one that has failed undefined. + Failed(Error), +} + +/// What precedes `prefix_start`. +#[derive(Clone, Copy)] +enum ExtKind { + /// The dictionary, or nothing. + Dict, + /// See [`ExtDict::Within`]. + Within { end: usize, len: usize }, +} + +/// A buffer whose size a frame header dictates. It takes up no more memory than has been used of it. +struct Buffer { + /// What has been used so far. There is capacity for `size`. + bytes: Vec, + size: usize, +} + +impl Buffer { + const fn new() -> Self { + Self { + bytes: Vec::new(), + size: 0, + } + } + + fn with_size(size: usize) -> Result { + let mut bytes = Vec::new(); + bytes + .try_reserve_exact(size) + .map_err(|_| Error::MemoryAllocation)?; + Ok(Self { bytes, size }) + } + + /// The `len` bytes from `start` on, which are within `size`. + fn span_mut(&mut self, start: usize, len: usize) -> Result<&mut [u8]> { + let end = start + len; + if self.bytes.len() < end && end <= self.size { + self.bytes.resize(end, 0); + } + self.bytes.span_mut(start, len) + } +} + +impl Default for Buffer { + fn default() -> Self { + Self::new() + } +} + +/// All of a `ZSTD_DCtx` but the dictionary it owns. +struct DState { + block: BlockDecoder, + + // `prefixStart`, `previousDstEnd`, `virtualStart` and `dictEnd`, as positions in the buffer that is written to. + prefix_start: usize, + /// `None`: nothing of this frame is written yet. + previous_dst_end: Option, + ext: ExtKind, + /// The address `dictEnd` would hold if it points to the end of a dictionary. 0 for `NULL`, 1 for anything else. + dict_end_address: usize, + + expected: usize, + f_params: FrameHeader, + decoded_size: u64, + b_type: BlockType, + stage: DStage, + xxh_state: Xxh64, + validate_checksum: bool, + rle_size: usize, + dict_id: u32, + + stream_stage: DStreamStage, + in_buff: Buffer, + in_pos: usize, + max_window_size: usize, + out_buff: Buffer, + out_start: usize, + out_end: usize, + /// The last byte of a frame's input is held back until its output is flushed. + hostage_byte: bool, + no_forward_progress: u32, + header_buffer: [u8; ZSTD_FRAMEHEADERSIZE_MAX], + oversized_duration: usize, +} + +/// `ZSTD_DCtx`, which is also `ZSTD_DStream`. +pub struct DCtx { + state: DState, + /// `ddictLocal` + ddict: Option>>, +} + +impl Default for DCtx { + fn default() -> Self { + Self::new() + } +} + +impl DCtx { + /// How much memory one takes, apart from buffers whose size depends on the frames. + pub const SIZE: usize = size_of::() + BlockDecoder::SCRATCH_SIZE; + + /// `ZSTD_createDCtx` + pub fn new() -> Self { + Self { + state: DState { + block: BlockDecoder::new(), + prefix_start: 0, + previous_dst_end: None, + ext: ExtKind::Dict, + dict_end_address: 0, + expected: 0, + f_params: FrameHeader::EMPTY, + decoded_size: 0, + b_type: BlockType::Reserved, + stage: DStage::GetFrameHeaderSize, + xxh_state: Xxh64::new(), + validate_checksum: false, + rle_size: 0, + dict_id: 0, + stream_stage: DStreamStage::Init, + in_buff: Buffer::new(), + in_pos: 0, + max_window_size: ZSTD_MAXWINDOWSIZE_DEFAULT, + out_buff: Buffer::new(), + out_start: 0, + out_end: 0, + hostage_byte: false, + no_forward_progress: 0, + header_buffer: [0; ZSTD_FRAMEHEADERSIZE_MAX], + oversized_duration: 0, + }, + ddict: None, + } + } + + /// `ZSTD_decompressDCtx`: decompresses all the frames of `src`, which has to end with one, to the start of `dst`. + /// Returns how many bytes that is. As the other functions that take whole frames, it gives up a stream that is under + /// way or has failed. + pub fn decompress(&mut self, dst: &mut [u8], src: &[u8]) -> Result { + let (size, dst) = (dst.len(), &mut OutputBuffer::Whole(dst)); + self.state + .decompress_frames(dst, size, src, self.ddict.as_deref()) + } + + /// The same to a new buffer of `capacity` bytes, little more of which is touched than is written. With `ddict`, if + /// there is one, and not with a dictionary that was loaded. + pub(crate) fn decompress_to_vec( + &mut self, + capacity: usize, + src: &[u8], + ddict: Option<&DDict<'_>>, + ) -> Result> { + let mut dst = Vec::new(); + dst.try_reserve_exact(capacity) + .map_err(|_| Error::MemoryAllocation)?; + let used = &mut OutputBuffer::Used(&mut dst); + let size = self.state.decompress_frames(used, capacity, src, ddict)?; + dst.truncate(size); + Ok(dst) + } + + /// `ZSTD_decompress_usingDDict` + pub fn decompress_using_ddict( + &mut self, + dst: &mut [u8], + src: &[u8], + ddict: &DDict<'_>, + ) -> Result { + let (size, dst) = (dst.len(), &mut OutputBuffer::Whole(dst)); + self.state.decompress_frames(dst, size, src, Some(ddict)) + } + + /// `ZSTD_DCtx_loadDictionary`: `dict` is copied, and used for every frame from now on. None if it is empty. + pub fn load_dictionary(&mut self, dict: &[u8]) -> Result<()> { + if self.state.stream_stage != DStreamStage::Init { + return Err(Error::StageWrong); + } + self.ddict = None; + if !dict.is_empty() { + // The C function has nothing else to report a dictionary that it cannot make sense of as. + let ddict = DDict::new(Cow::Owned(try_copy(dict)?), DictContentType::Auto) + .map_err(|_| Error::MemoryAllocation)?; + self.ddict = Some(Box::new(ddict)); + } + Ok(()) + } + + /// `ZSTD_DCtx_setParameter` + pub fn set_parameter(&mut self, param: i32, mut value: i32) -> Result<()> { + if self.state.stream_stage != DStreamStage::Init { + return Err(Error::StageWrong); + } + if param != ZSTD_D_WINDOW_LOG_MAX { + return Err(Error::ParameterUnsupported); + } + if value == 0 { + value = ZSTD_WINDOWLOG_LIMIT_DEFAULT as i32; + } + if value < ZSTD_WINDOWLOG_ABSOLUTEMIN as i32 || value > ZSTD_WINDOWLOG_MAX as i32 { + return Err(Error::ParameterOutOfBound); + } + self.state.max_window_size = 1usize << value; + Ok(()) + } + + /// `ZSTD_initDStream`: forgets the frame in progress and the dictionary. + pub fn init_stream(&mut self) { + self.state.reset_session(); + self.ddict = None; + } + + /// `ZSTD_decompressStream`: reads from `input` and writes to `output` as much as it can, and moves their positions. + /// Returns 0 when a frame is complete and all of it is written, else a hint of how much input to give next. + /// Once a frame has made it fail, it fails in the same way until the stream is given up. + pub fn decompress_stream( + &mut self, + output: &mut OutBuffer<'_>, + input: &mut InBuffer<'_>, + ) -> Result { + self.state + .decompress_stream(output, input, self.ddict.as_deref()) + } +} + +/// `ZSTD_copyRawBlock` +fn copy_raw_block(dst: &mut [u8], src: &[u8]) -> Result { + if src.len() > dst.len() { + return Err(Error::DstSizeTooSmall); + } + dst.write_at(0, src)?; + Ok(src.len()) +} + +/// `ZSTD_setRleBlock` +fn set_rle_block(dst: &mut [u8], byte: u8, regen_size: usize) -> Result { + let Some(dst) = dst.get_mut(..regen_size) else { + return Err(Error::DstSizeTooSmall); + }; + dst.fill(byte); + Ok(regen_size) +} + +/// `ZSTD_decodingBufferSize_internal` +fn decoding_buffer_size(window_size: u64, frame_content_size: u64, block_size_max: usize) -> usize { + let block_size = (window_size.min(ZSTD_BLOCKSIZE_MAX as u64) as usize).min(block_size_max); + let needed_rb_size = window_size + (block_size * 2) as u64 + (WILDCOPY_OVERLENGTH * 2) as u64; + frame_content_size.min(needed_rb_size) as usize +} + +impl DState { + /// `ZSTD_decompressBegin_usingDDict` + fn begin_using_ddict(&mut self, ddict: Option<&DDict<'_>>) { + if let Some(ddict) = ddict { + self.block.ddict_is_cold = self.dict_end_address != ddict.end_address(); + } + + self.expected = FRAMEHEADERSIZE_PREFIX; + self.stage = DStage::GetFrameHeaderSize; + self.decoded_size = 0; + self.previous_dst_end = None; + self.prefix_start = 0; + self.ext = ExtKind::Dict; + self.dict_end_address = 0; + self.block.lit_entropy = false; + self.block.fse_entropy = false; + self.dict_id = 0; + self.b_type = BlockType::Reserved; + self.block.rep = REP_START_VALUE; + self.block.seq_sources = SeqTableSources::all(TableSource::Local); + self.block.huf_source = TableSource::Local; + + // `ZSTD_copyDDictParameters` + if let Some(ddict) = ddict { + self.dict_id = ddict.dict_id; + self.dict_end_address = ddict.end_address(); + self.block.rep = ddict.rep; + if ddict.entropy.is_some() { + self.block.lit_entropy = true; + self.block.fse_entropy = true; + self.block.seq_sources = SeqTableSources::all(TableSource::Dict); + self.block.huf_source = TableSource::Dict; + } + } + } + + /// `ZSTD_checkContinuity`: `dst` is where the output continues, with room for `dst_size` bytes. + fn check_continuity(&mut self, dst: usize, dst_size: usize) { + if self.previous_dst_end != Some(dst) && dst_size > 0 { + if let Some(end) = self.previous_dst_end { + self.ext = ExtKind::Within { + end, + len: end - self.prefix_start, + }; + self.dict_end_address = 1; + } + self.prefix_start = dst; + self.previous_dst_end = Some(dst); + } + } + + /// `ZSTD_decodeFrameHeader`: `src` is all of a frame header. + fn decode_frame_header(&mut self, src: &[u8]) -> Result<()> { + if let HeaderStatus::Needs(_) = get_frame_header(&mut self.f_params, src)? { + return Err(Error::SrcSizeWrong); + } + if self.f_params.dict_id != 0 && self.dict_id != self.f_params.dict_id { + return Err(Error::DictionaryWrong); + } + self.validate_checksum = self.f_params.checksum_flag; + if self.validate_checksum { + self.xxh_state = Xxh64::new(); + } + Ok(()) + } + + /// One compressed block, to `buf` from `start` on. + fn decompress_block( + &mut self, + buf: OutputBuffer<'_>, + start: usize, + capacity: usize, + src: &[u8], + ddict: Option<&DDict<'_>>, + streaming: Streaming, + ) -> Result { + let ext_dict = match self.ext { + ExtKind::Dict => ExtDict::Dict(ddict.map_or(&[], |ddict| &*ddict.content)), + ExtKind::Within { end, len } => ExtDict::Within { + end, + len, + window: self.f_params.window_size as usize, + }, + }; + self.block.decompress_block( + BlockOutput { + buf, + prefix_start: self.prefix_start, + start, + capacity, + ext_dict, + }, + src, + ddict.and_then(|ddict| ddict.entropy.as_deref()), + self.f_params.block_size_max, + streaming, + ) + } + + /// `ZSTD_decompressFrame`: one frame, from the start of `src` to `dst` from `ostart` on. + /// Returns how much was written, and how much was read. + fn decompress_frame( + &mut self, + dst: &mut OutputBuffer<'_>, + dst_size: usize, + ostart: usize, + src: &[u8], + ddict: Option<&DDict<'_>>, + ) -> Result<(usize, usize)> { + let mut ip = 0usize; + let mut op = ostart; + + if src.len() < FRAMEHEADERSIZE_MIN + ZSTD_BLOCKHEADERSIZE { + return Err(Error::SrcSizeWrong); + } + { + let frame_header_size = frame_header_size(src.sub(..FRAMEHEADERSIZE_PREFIX)?)?; + if src.len() < frame_header_size + ZSTD_BLOCKHEADERSIZE { + return Err(Error::SrcSizeWrong); + } + self.decode_frame_header(src.sub(..frame_header_size)?)?; + ip += frame_header_size; + } + + loop { + let (c_block_size, properties) = get_c_block_size(src.sub(ip..)?)?; + ip += ZSTD_BLOCKHEADERSIZE; + if c_block_size > src.len() - ip { + return Err(Error::SrcSizeWrong); + } + let block = src.span(ip, c_block_size)?; + + let decoded_size = match properties.block_type { + BlockType::Compressed => { + let decoded_size = self.decompress_block( + dst.reborrow(), + op, + dst_size - op, + block, + ddict, + Streaming::No, + )?; + // Only a streaming decoder checks that in 1.5.7, where such a block can overwrite its own literals. + if decoded_size > self.f_params.block_size_max { + return Err(Error::CorruptionDetected); + } + decoded_size + } + BlockType::Raw => { + if block.len() > dst_size - op { + return Err(Error::DstSizeTooSmall); + } + dst.write(op, block, Growth::Append)?; + block.len() + } + BlockType::Rle => { + let size = properties.orig_size as usize; + if size > dst_size - op { + return Err(Error::DstSizeTooSmall); + } + dst.fill(op, block.at(0)?, size, Growth::Append)?; + size + } + BlockType::Reserved => return Err(Error::CorruptionDetected), + }; + if self.validate_checksum { + self.xxh_state.update(dst.bytes().span(op, decoded_size)?); + } + op += decoded_size; + ip += c_block_size; + if properties.last_block { + break; + } + } + + if self.f_params.frame_content_size != ZSTD_CONTENTSIZE_UNKNOWN + && (op - ostart) as u64 != self.f_params.frame_content_size + { + return Err(Error::CorruptionDetected); + } + if self.f_params.checksum_flag { + if src.len() - ip < 4 { + return Err(Error::ChecksumWrong); + } + if read_le32(src, ip)? != self.xxh_state.digest() as u32 { + return Err(Error::ChecksumWrong); + } + ip += 4; + } + Ok((op - ostart, ip)) + } + + /// `ZSTD_DCtx_reset` with `ZSTD_reset_session_only` + fn reset_session(&mut self) { + self.stream_stage = DStreamStage::Init; + self.no_forward_progress = 0; + } + + /// [`Self::decompress_multi_frame`] to the start of `dst`. It resets the stream, as `zstd.h` says of what starts a new + /// job: the C library does not, and goes on with a stream that was under way, with what this leaves. + fn decompress_frames( + &mut self, + dst: &mut OutputBuffer<'_>, + dst_size: usize, + src: &[u8], + ddict: Option<&DDict<'_>>, + ) -> Result { + self.reset_session(); + self.decompress_multi_frame(dst, dst_size, 0, src, ddict) + } + + /// `ZSTD_decompressMultiFrame`: all the frames of `src`, to `dst` from `dst_start` on. + fn decompress_multi_frame( + &mut self, + dst: &mut OutputBuffer<'_>, + dst_size: usize, + dst_start: usize, + mut src: &[u8], + ddict: Option<&DDict<'_>>, + ) -> Result { + let mut op = dst_start; + let mut more_than_1_frame = false; + + while src.len() >= FRAMEHEADERSIZE_PREFIX { + if is_skippable_magic(read_le32(src, 0)?) { + src = src.sub(read_skippable_frame_size(src)?..)?; + continue; + } + + self.begin_using_ddict(ddict); + self.check_continuity(op, dst_size - op); + + match self.decompress_frame(dst, dst_size, op, src, ddict) { + // More likely `src` is longer than its frames than damaged where a magic number is expected. + Err(Error::PrefixUnknown) if more_than_1_frame => { + return Err(Error::SrcSizeWrong); + } + Err(err) => return Err(err), + Ok((written, read)) => { + op += written; + src = src.sub(read..)?; + } + } + more_than_1_frame = true; + } + + if !src.is_empty() { + return Err(Error::SrcSizeWrong); + } + Ok(op - dst_start) + } + + /// `ZSTD_nextSrcSizeToDecompressWithInputSize` + fn next_src_size_to_decompress_with_input_size(&self, input_size: usize) -> usize { + if !matches!( + self.stage, + DStage::DecompressBlock | DStage::DecompressLastBlock + ) { + return self.expected; + } + if self.b_type != BlockType::Raw { + return self.expected; + } + input_size.min(self.expected).max(1) + } + + /// `ZSTD_decompressContinue`, for the stages that `ZSTD_decompressStream` leaves to it. The output goes to + /// `out_buff` from `out_start` on, where there is room for `dst_capacity` bytes. + fn decompress_continue( + &mut self, + dst_capacity: usize, + src: &[u8], + ddict: Option<&DDict<'_>>, + ) -> Result { + if src.len() != self.next_src_size_to_decompress_with_input_size(src.len()) { + return Err(Error::SrcSizeWrong); + } + let dst = self.out_start; + self.check_continuity(dst, dst_capacity); + + match self.stage { + DStage::DecodeBlockHeader => { + let (c_block_size, bp) = get_c_block_size(src)?; + if c_block_size > self.f_params.block_size_max { + return Err(Error::CorruptionDetected); + } + self.expected = c_block_size; + self.b_type = bp.block_type; + self.rle_size = bp.orig_size as usize; + if c_block_size != 0 { + self.stage = if bp.last_block { + DStage::DecompressLastBlock + } else { + DStage::DecompressBlock + }; + return Ok(0); + } + // An empty block. + if bp.last_block { + self.end_of_blocks(); + } else { + self.expected = ZSTD_BLOCKHEADERSIZE; + self.stage = DStage::DecodeBlockHeader; + } + Ok(0) + } + + DStage::DecompressLastBlock | DStage::DecompressBlock => { + let mut out_buff = core::mem::take(&mut self.out_buff); + let r_size = match self.b_type { + BlockType::Compressed => { + self.expected = 0; + self.decompress_block( + OutputBuffer::Used(&mut out_buff.bytes), + dst, + dst_capacity, + src, + ddict, + Streaming::Yes, + ) + } + BlockType::Raw => out_buff + .span_mut(dst, dst_capacity.min(src.len())) + .and_then(|dst| copy_raw_block(dst, src)) + .inspect(|&r_size| self.expected -= r_size), + BlockType::Rle => { + self.expected = 0; + out_buff + .span_mut(dst, dst_capacity.min(self.rle_size)) + .and_then(|dst| set_rle_block(dst, src.at(0)?, self.rle_size)) + } + BlockType::Reserved => Err(Error::CorruptionDetected), + }; + self.out_buff = out_buff; + let r_size = r_size?; + if r_size > self.f_params.block_size_max { + return Err(Error::CorruptionDetected); + } + self.decoded_size += r_size as u64; + if self.validate_checksum { + self.xxh_state + .update(self.out_buff.bytes.span(dst, r_size)?); + } + self.previous_dst_end = Some(dst + r_size); + + // Stay on the same stage until all of a raw block is in. + if self.expected > 0 { + return Ok(r_size); + } + + if self.stage == DStage::DecompressLastBlock { + if self.f_params.frame_content_size != ZSTD_CONTENTSIZE_UNKNOWN + && self.decoded_size != self.f_params.frame_content_size + { + return Err(Error::CorruptionDetected); + } + self.end_of_blocks(); + } else { + self.stage = DStage::DecodeBlockHeader; + self.expected = ZSTD_BLOCKHEADERSIZE; + } + Ok(r_size) + } + + DStage::CheckChecksum => { + if self.validate_checksum && read_le32(src, 0)? != self.xxh_state.digest() as u32 { + return Err(Error::ChecksumWrong); + } + self.expected = 0; + self.stage = DStage::GetFrameHeaderSize; + Ok(0) + } + + DStage::SkipFrame => { + self.expected = 0; + self.stage = DStage::GetFrameHeaderSize; + Ok(0) + } + + // `ZSTD_decompressStream` reads frame headers itself, and is done when it gets back here. + DStage::GetFrameHeaderSize => Err(Error::Generic), + } + } + + /// After the last block comes the checksum, if there is one. + fn end_of_blocks(&mut self) { + if self.f_params.checksum_flag { + self.expected = 4; + self.stage = DStage::CheckChecksum; + } else { + self.expected = 0; + self.stage = DStage::GetFrameHeaderSize; + } + } + + /// `ZSTD_decompressContinueStream` + fn decompress_continue_stream(&mut self, src: &[u8], ddict: Option<&DDict<'_>>) -> Result<()> { + let is_skip_frame = self.stage == DStage::SkipFrame; + let dst_size = if is_skip_frame { + 0 + } else { + self.out_buff.size - self.out_start + }; + let decoded_size = self.decompress_continue(dst_size, src, ddict)?; + if decoded_size == 0 && !is_skip_frame { + self.stream_stage = DStreamStage::Read; + } else { + self.out_end = self.out_start + decoded_size; + self.stream_stage = DStreamStage::Flush; + } + Ok(()) + } + + /// Buffers for the frame whose header is `f_params`. Those that are there stay, unless they are too small, or have + /// been much too large for many frames. + fn resize_buffers(&mut self) -> Result<()> { + let needed_in_buff_size = self.f_params.block_size_max.max(4); + let needed_out_buff_size = decoding_buffer_size( + self.f_params.window_size, + self.f_params.frame_content_size, + self.f_params.block_size_max, + ); + + // `ZSTD_DCtx_updateOversizedDuration` + if self.in_buff.size + self.out_buff.size + >= (needed_in_buff_size + needed_out_buff_size) * ZSTD_WORKSPACETOOLARGE_FACTOR + { + self.oversized_duration += 1; + } else { + self.oversized_duration = 0; + } + + let too_small = + self.in_buff.size < needed_in_buff_size || self.out_buff.size < needed_out_buff_size; + let too_large = self.oversized_duration >= ZSTD_WORKSPACETOOLARGE_MAXDURATION; + if too_small || too_large { + self.in_buff = Buffer::new(); + self.out_buff = Buffer::new(); + self.in_buff = Buffer::with_size(needed_in_buff_size)?; + self.out_buff = Buffer::with_size(needed_out_buff_size)?; + } + // Nothing of the frame before is there to be read. + self.in_buff.bytes.clear(); + self.out_buff.bytes.clear(); + Ok(()) + } + + /// `ZSTD_decompressStream` + fn decompress_stream( + &mut self, + output: &mut OutBuffer<'_>, + input: &mut InBuffer<'_>, + ddict: Option<&DDict<'_>>, + ) -> Result { + if input.pos > input.src.len() { + return Err(Error::SrcSizeWrong); + } + if output.pos > output.dst.len() { + return Err(Error::DstSizeTooSmall); + } + self.decompress_stream_on(output, input, ddict) + .inspect_err(|&error| { + // These leave all as it was. + let can_go_on = matches!( + error, + Error::NoForwardProgressDestFull | Error::NoForwardProgressInputEmpty + ); + if !can_go_on { + self.stream_stage = DStreamStage::Failed(error); + } + }) + } + + /// [`Self::decompress_stream`], but for what becomes of the stream if it fails. + fn decompress_stream_on( + &mut self, + output: &mut OutBuffer<'_>, + input: &mut InBuffer<'_>, + ddict: Option<&DDict<'_>>, + ) -> Result { + if let DStreamStage::Failed(error) = self.stream_stage { + return Err(error); + } + let src = input.src; + let istart = input.pos; + let iend = src.len(); + let mut ip = istart; + let ostart = output.pos; + let oend = output.dst.len(); + let mut op = ostart; + + 'work: loop { + if self.stream_stage == DStreamStage::Init { + self.stream_stage = DStreamStage::LoadHeader { lh_size: 0 }; + self.in_pos = 0; + self.out_start = 0; + self.out_end = 0; + self.hostage_byte = false; + } + + if let DStreamStage::LoadHeader { lh_size } = self.stream_stage { + match get_frame_header(&mut self.f_params, self.header_buffer.sub(..lh_size)?)? { + HeaderStatus::Complete => {} + HeaderStatus::Needs(h_size) => { + let to_load = h_size - lh_size; + let remaining_input = iend - ip; + if to_load > remaining_input { + self.header_buffer.write_at(lh_size, src.sub(ip..)?)?; + let lh_size = lh_size + remaining_input; + self.stream_stage = DStreamStage::LoadHeader { lh_size }; + input.pos = iend; + // Refuse what cannot become a frame header. + get_frame_header( + &mut self.f_params, + self.header_buffer.sub(..lh_size)?, + )?; + return Ok( + FRAMEHEADERSIZE_MIN.max(h_size) - lh_size + ZSTD_BLOCKHEADERSIZE + ); + } + self.header_buffer + .write_at(lh_size, src.span(ip, to_load)?)?; + self.stream_stage = DStreamStage::LoadHeader { lh_size: h_size }; + ip += to_load; + continue 'work; + } + } + + // A frame that is all there, with room for all of its output, needs no buffer, so the limit on windows + // does not apply to it, as in the C code. + if self.f_params.frame_content_size != ZSTD_CONTENTSIZE_UNKNOWN + && self.f_params.frame_type != FrameType::Skippable + && (oend - op) as u64 >= self.f_params.frame_content_size + { + if let Ok(c_size) = find_frame_compressed_size(src.sub(istart..)?) { + let decompressed_size = self.decompress_multi_frame( + &mut OutputBuffer::Whole(output.dst), + oend, + op, + src.span(istart, c_size)?, + ddict, + )?; + ip = istart + c_size; + op += decompressed_size; + self.expected = 0; + self.stream_stage = DStreamStage::Init; + break 'work; + } + } + + self.begin_using_ddict(ddict); + + if is_skippable_magic(read_le32(&self.header_buffer, 0)?) { + self.expected = read_le32(&self.header_buffer, ZSTD_FRAMEIDSIZE)? as usize; + self.stage = DStage::SkipFrame; + } else { + let header_buffer = self.header_buffer; + self.decode_frame_header(header_buffer.sub(..lh_size)?)?; + self.expected = ZSTD_BLOCKHEADERSIZE; + self.stage = DStage::DecodeBlockHeader; + } + + self.f_params.window_size = self + .f_params + .window_size + .max(1 << ZSTD_WINDOWLOG_ABSOLUTEMIN); + if self.f_params.window_size > self.max_window_size as u64 { + return Err(Error::FrameParameterWindowTooLarge); + } + + self.resize_buffers()?; + self.stream_stage = DStreamStage::Read; + } + + if self.stream_stage == DStreamStage::Read { + let needed_in_size = self.next_src_size_to_decompress_with_input_size(iend - ip); + if needed_in_size == 0 { + // The end of the frame. + self.stream_stage = DStreamStage::Init; + break 'work; + } + if iend - ip >= needed_in_size { + // Decode straight from the input. + self.decompress_continue_stream(src.span(ip, needed_in_size)?, ddict)?; + ip += needed_in_size; + continue 'work; + } + if ip == iend { + break 'work; + } + self.stream_stage = DStreamStage::Load; + } + + if self.stream_stage == DStreamStage::Load { + let needed_in_size = self.expected; + let to_load = needed_in_size - self.in_pos; + let loaded_size = to_load.min(iend - ip); + if self.stage != DStage::SkipFrame { + if to_load > self.in_buff.size - self.in_pos { + return Err(Error::CorruptionDetected); + } + self.in_buff.bytes.truncate(self.in_pos); + append(&mut self.in_buff.bytes, src.span(ip, loaded_size)?)?; + } + ip += loaded_size; + self.in_pos += loaded_size; + if loaded_size < to_load { + break 'work; + } + + self.in_pos = 0; + let in_buff = core::mem::take(&mut self.in_buff.bytes); + // Nothing of a skippable frame is read, or kept. + let loaded = in_buff.get(..needed_in_size).unwrap_or_default(); + let result = if self.stage == DStage::SkipFrame { + self.skip_frame_loaded() + } else { + self.decompress_continue_stream(loaded, ddict) + }; + self.in_buff.bytes = in_buff; + result?; + continue 'work; + } + + // `zdss_flush` + let to_flush_size = self.out_end - self.out_start; + let flushed_size = to_flush_size.min(oend - op); + output + .dst + .write_at(op, self.out_buff.bytes.span(self.out_start, flushed_size)?)?; + op += flushed_size; + self.out_start += flushed_size; + if flushed_size == to_flush_size { + self.stream_stage = DStreamStage::Read; + if (self.out_buff.size as u64) < self.f_params.frame_content_size + && self.out_start + self.f_params.block_size_max > self.out_buff.size + { + self.out_start = 0; + self.out_end = 0; + } + continue 'work; + } + // The output is full. + break 'work; + } + + input.pos = ip; + output.pos = op; + + if ip == istart && op == ostart { + self.no_forward_progress += 1; + if self.no_forward_progress >= ZSTD_NO_FORWARD_PROGRESS_MAX { + if op == oend { + return Err(Error::NoForwardProgressDestFull); + } + if ip == iend { + return Err(Error::NoForwardProgressInputEmpty); + } + } + } else { + self.no_forward_progress = 0; + } + + let mut next_src_size_hint = self.expected; + if next_src_size_hint == 0 { + // The frame is decoded. + if self.out_end == self.out_start { + // And flushed. + if self.hostage_byte { + if input.pos >= input.src.len() { + // The caller has to present the byte again for it to be released. + self.stream_stage = DStreamStage::Read; + return Ok(1); + } + input.pos += 1; + } + return Ok(0); + } + if !self.hostage_byte { + // So that the caller comes back to flush, even one that stops when its input is consumed. + input.pos = input.pos.wrapping_sub(1); + self.hostage_byte = true; + } + return Ok(1); + } + if self.stage == DStage::DecompressBlock { + // The header of the next block too. + next_src_size_hint += ZSTD_BLOCKHEADERSIZE; + } + next_src_size_hint -= self.in_pos; + Ok(next_src_size_hint) + } + + /// `ZSTD_decompressContinueStream` for the content of a skippable frame that came in several pieces. + fn skip_frame_loaded(&mut self) -> Result<()> { + self.expected = 0; + self.stage = DStage::GetFrameHeaderSize; + self.out_end = self.out_start; + self.stream_stage = DStreamStage::Flush; + Ok(()) + } +} diff --git a/src/zstd/decompress/seq_table.rs b/src/zstd/decompress/seq_table.rs new file mode 100644 index 000000000000..cd103ce656a9 --- /dev/null +++ b/src/zstd/decompress/seq_table.rs @@ -0,0 +1,554 @@ +//! The FSE tables that sequences are decoded with: `ZSTD_seqSymbol` and `ZSTD_buildFSETable`. + +use crate::common::fse_decompress::{MAX_SPREAD_SIZE, NextState, Spread, State}; +use crate::common::{ + LL_BITS, LL_DEFAULT_NORM_LOG, MAX_LL, MAX_ML, MAX_OFF, ML_BITS, ML_DEFAULT_NORM_LOG, + OF_DEFAULT_NORM_LOG, +}; +use crate::error::Result; +use crate::util::SliceExt; + +#[rustfmt::skip] +const LL_BASE: [u32; MAX_LL as usize + 1] = [ + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 18, 20, 22, 24, 28, 32, 40, + 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, + 0x2000, 0x4000, 0x8000, 0x10000, +]; + +/// What a match length code starts at, above `MINMATCH`. +#[rustfmt::skip] +const ML_BASE: [u32; MAX_ML as usize + 1] = [ + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, + 32, 34, 36, 38, 40, 44, 48, 56, + 64, 80, 96, 0x80, 0x100, 0x200, 0x400, 0x800, + 0x1000, 0x2000, 0x4000, 0x8000, 0x10000, +]; + +/// `ZSTD_seqSymbol` +#[derive(Clone, Copy)] +pub(crate) struct SeqSymbol { + pub(crate) next_state: u16, + pub(crate) nb_additional_bits: u8, + pub(crate) nb_bits: u8, + pub(crate) base_value: u32, +} + +/// How a table keeps a [`SeqSymbol`]. +pub(crate) trait Cell: Copy + 'static { + const EMPTY: Self; + /// [`NextState::at`] as `to_bits` of a cell where nothing else is set. + const NEXT_STATES: &'static [u64; 4 * MAX_TABLE_SIZE]; + /// The fields one after the other, from the lowest bits on. + fn from_bits(bits: u64) -> Self; + fn unpack(self) -> SeqSymbol; +} + +const MAX_TABLE_SIZE: usize = MAX_SPREAD_SIZE; + +/// A state of the table of literal lengths or of match lengths. +#[derive(Clone, Copy)] +pub(crate) struct LengthCell { + /// All of the base value but its bit 16, which is set if and only if there are 16 additional bits. + pub(crate) base_value: u16, + pub(crate) next_state: u16, + /// As many bits set as `nb_bits`. + pub(crate) state_mask: u16, + pub(crate) nb_additional_bits: u8, + pub(crate) nb_bits: u8, +} + +const fn l(next_state: u16, nb_additional_bits: u8, nb_bits: u8, base_value: u32) -> LengthCell { + LengthCell { + base_value: base_value as u16, + next_state, + state_mask: (1u16 << nb_bits) - 1, + nb_additional_bits, + nb_bits, + } +} + +impl LengthCell { + const fn to_bits(self) -> u64 { + self.base_value as u64 + | (self.next_state as u64) << 16 + | (self.state_mask as u64) << 32 + | (self.nb_additional_bits as u64) << 48 + | (self.nb_bits as u64) << 56 + } + + #[inline(always)] + pub(crate) fn base_value(self) -> u32 { + u32::from(self.base_value) + (u32::from(self.nb_additional_bits >> 4) << 16) + } +} + +impl Cell for LengthCell { + const EMPTY: Self = l(0, 0, 0, 0); + + const NEXT_STATES: &'static [u64; 4 * MAX_TABLE_SIZE] = &{ + let mut all = [0; 4 * MAX_TABLE_SIZE]; + let mut left: &mut [u64] = &mut all; + let mut index = 0; + while let [bits, others @ ..] = left { + let NextState { state, nb_bits } = NextState::at(index); + *bits = l(state, 0, nb_bits, 0).to_bits(); + index += 1; + left = others; + } + all + }; + + #[inline(always)] + fn from_bits(bits: u64) -> Self { + Self { + base_value: bits as u16, + next_state: (bits >> 16) as u16, + state_mask: (bits >> 32) as u16, + nb_additional_bits: (bits >> 48) as u8, + nb_bits: (bits >> 56) as u8, + } + } + + #[inline(always)] + fn unpack(self) -> SeqSymbol { + SeqSymbol { + next_state: self.next_state, + nb_additional_bits: self.nb_additional_bits, + nb_bits: self.nb_bits, + base_value: self.base_value(), + } + } +} + +/// A state of the table of offsets. +#[derive(Clone, Copy)] +pub(crate) struct OffsetCell { + /// 1 with `nb_additional_bits` zeros after it. With the additional bits in their place, 1 to 3 stand for one of the + /// last offsets, and what is above for an offset 3 less than it. + pub(crate) first_value: u32, + pub(crate) next_state: u8, + /// As many bits set as `nb_bits`. + pub(crate) state_mask: u8, + pub(crate) nb_additional_bits: u8, + pub(crate) nb_bits: u8, +} + +const fn o(next_state: u8, nb_additional_bits: u8, nb_bits: u8) -> OffsetCell { + OffsetCell { + first_value: 1 << nb_additional_bits, + next_state, + state_mask: ((1u16 << nb_bits) - 1) as u8, + nb_additional_bits, + nb_bits, + } +} + +impl OffsetCell { + const fn to_bits(self) -> u64 { + self.first_value as u64 + | (self.next_state as u64) << 32 + | (self.state_mask as u64) << 40 + | (self.nb_additional_bits as u64) << 48 + | (self.nb_bits as u64) << 56 + } +} + +impl Cell for OffsetCell { + const EMPTY: Self = o(0, 0, 0); + + const NEXT_STATES: &'static [u64; 4 * MAX_TABLE_SIZE] = &{ + let mut all = [0; 4 * MAX_TABLE_SIZE]; + let mut left: &mut [u64] = &mut all; + let mut index = 0; + while let [bits, others @ ..] = left { + let NextState { state, nb_bits } = NextState::at(index); + *bits = o(state as u8, 0, nb_bits).to_bits() - o(0, 0, 0).to_bits(); + index += 1; + left = others; + } + all + }; + + #[inline(always)] + fn from_bits(bits: u64) -> Self { + Self { + first_value: bits as u32, + next_state: (bits >> 32) as u8, + state_mask: (bits >> 40) as u8, + nb_additional_bits: (bits >> 48) as u8, + nb_bits: (bits >> 56) as u8, + } + } + + #[inline(always)] + fn unpack(self) -> SeqSymbol { + SeqSymbol { + next_state: u16::from(self.next_state), + nb_additional_bits: self.nb_additional_bits, + nb_bits: self.nb_bits, + base_value: if self.nb_additional_bits < 2 { + u32::from(self.nb_additional_bits) + } else { + self.first_value - 3 + }, + } + } +} + +/// What each code of a kind stands for: `to_bits` of a cell that leads to no state. There are fewer codes than that. +pub(crate) type Codes = [u64; 64]; + +const fn length_codes(mut base_values: &[u32], mut nb_additional_bits: &[u8]) -> Codes { + let mut codes = [0; 64]; + let mut left: &mut [u64] = &mut codes; + while let ( + [code, other_codes @ ..], + [base_value, other_values @ ..], + [more_bits, other_nb_bits @ ..], + ) = (left, base_values, nb_additional_bits) + { + *code = l(0, *more_bits, 0, *base_value).to_bits(); + (left, base_values, nb_additional_bits) = (other_codes, other_values, other_nb_bits); + } + codes +} + +const fn offset_codes() -> Codes { + let mut codes = [0; 64]; + let (mut left, _) = codes.split_at_mut(MAX_OFF as usize + 1); + let mut nb_additional_bits = 0; + while let [code, others @ ..] = left { + *code = o(0, nb_additional_bits, 0).to_bits(); + nb_additional_bits += 1; + left = others; + } + codes +} + +/// `nb_additional_bits` is at the same place in cells of both kinds. +const fn has_additional_bits(code: u64) -> bool { + (code >> 48) as u8 != 0 +} + +/// How many of 512 symbols are expected to come with additional bits in a table made from `counts`. +fn share_with_more_bits(counts: &[i16], codes: &Codes, table_log: u32) -> u32 { + (counts.iter().zip(codes)) + .filter(|&(_, &code)| has_additional_bits(code)) + .map(|(&count, _)| u32::from(count.unsigned_abs())) + .sum::() + << (9 - table_log.min(9)) +} + +pub(crate) static LL_CODES: Codes = length_codes(&LL_BASE, &LL_BITS); +pub(crate) static OF_CODES: Codes = offset_codes(); +pub(crate) static ML_CODES: Codes = length_codes(&ML_BASE, &ML_BITS); + +/// A table of at most `N` states, which is a power of 2. +#[derive(Clone)] +pub(crate) struct SeqTable { + pub(crate) table_log: u32, + /// How many of 512 symbols are expected to come with additional bits. + pub(crate) with_more_bits: u32, + pub(crate) table: [C; N], +} + +pub(crate) type LengthTable = SeqTable; +pub(crate) type OffsetTable = SeqTable; + +impl SeqTable { + pub(crate) const fn new() -> Self { + Self { + table_log: 0, + with_more_bits: 0, + table: [C::EMPTY; N], + } + } + + /// `with_more_bits` of the `symbols` have additional bits. + const fn predefined( + table_log: u32, + with_more_bits: u32, + symbols: [C; M], + ) -> Self { + let mut table = [C::EMPTY; N]; + let (head, _) = table.split_at_mut(M); + head.copy_from_slice(&symbols); + Self { + table_log, + with_more_bits: with_more_bits << (9 - table_log), + table, + } + } + + #[inline(always)] + pub(crate) fn get(&self, state: usize) -> SeqSymbol { + self.table + .get(state & (N - 1)) + .copied() + .unwrap_or(C::EMPTY) + .unpack() + } + + /// The states in use. + pub(crate) fn symbols(&self) -> &[C] { + self.table.get(..1 << self.table_log).unwrap_or_default() + } + + fn copy_from(&mut self, other: &Self) { + self.table_log = other.table_log; + self.with_more_bits = other.with_more_bits; + let symbols = other.symbols(); + if let Some(to) = self.table.get_mut(..symbols.len()) { + to.copy_from_slice(symbols); + } + } + + /// `ZSTD_buildSeqTable_rle` + pub(crate) fn build_rle(&mut self, code: u64) { + self.table_log = 0; + self.with_more_bits = u32::from(has_additional_bits(code)) << 9; + if let Some(cell) = self.table.first_mut() { + *cell = C::from_bits(code); + } + } + + /// `ZSTD_buildFSETable`: `table_log` is such that the table fits, and the first `nb_symbols` of `spread.counts` add + /// up to its size. + pub(crate) fn build( + &mut self, + nb_symbols: usize, + codes: &Codes, + table_log: u32, + spread: &mut Spread, + ) -> Result<()> { + let table_size = 1usize << table_log; + let table_decode = self.table.sub_mut(..table_size)?; + self.table_log = table_log; + self.with_more_bits = + share_with_more_bits(spread.counts.sub(..nb_symbols)?, codes, table_log); + let mut states = spread.states(nb_symbols, table_log)?; + for cell in table_decode { + let State { symbol, next } = states.next(); + let code = codes.get(usize::from(symbol) % 64).copied(); + let next_state = C::NEXT_STATES.get(usize::from(next) % (4 * MAX_TABLE_SIZE)); + *cell = C::from_bits(code.unwrap_or(0) | next_state.copied().unwrap_or(0)); + } + Ok(()) + } +} + +#[rustfmt::skip] +const LL_DEFAULT: [LengthCell; 64] = [ + l( 0, 0, 4, 0), l(16, 0, 4, 0), l(32, 0, 5, 1), l( 0, 0, 5, 3), + l( 0, 0, 5, 4), l( 0, 0, 5, 6), l( 0, 0, 5, 7), l( 0, 0, 5, 9), + l( 0, 0, 5, 10), l( 0, 0, 5, 12), l( 0, 0, 6, 14), l( 0, 1, 5, 16), + l( 0, 1, 5, 20), l( 0, 1, 5, 22), l( 0, 2, 5, 28), l( 0, 3, 5, 32), + l( 0, 4, 5, 48), l(32, 6, 5, 64), l( 0, 7, 5, 128), l( 0, 8, 6, 256), + l( 0, 10, 6, 1024), l( 0, 12, 6, 4096), l(32, 0, 4, 0), l( 0, 0, 4, 1), + l( 0, 0, 5, 2), l(32, 0, 5, 4), l( 0, 0, 5, 5), l(32, 0, 5, 7), + l( 0, 0, 5, 8), l(32, 0, 5, 10), l( 0, 0, 5, 11), l( 0, 0, 6, 13), + l(32, 1, 5, 16), l( 0, 1, 5, 18), l(32, 1, 5, 22), l( 0, 2, 5, 24), + l(32, 3, 5, 32), l( 0, 3, 5, 40), l( 0, 6, 4, 64), l(16, 6, 4, 64), + l(32, 7, 5, 128), l( 0, 9, 6, 512), l( 0, 11, 6, 2048), l(48, 0, 4, 0), + l(16, 0, 4, 1), l(32, 0, 5, 2), l(32, 0, 5, 3), l(32, 0, 5, 5), + l(32, 0, 5, 6), l(32, 0, 5, 8), l(32, 0, 5, 9), l(32, 0, 5, 11), + l(32, 0, 5, 12), l( 0, 0, 6, 15), l(32, 1, 5, 18), l(32, 1, 5, 20), + l(32, 2, 5, 24), l(32, 2, 5, 28), l(32, 3, 5, 40), l(32, 4, 5, 48), + l( 0, 16, 6, 65536), l( 0, 15, 6, 32768), l( 0, 14, 6, 16384), l( 0, 13, 6, 8192), +]; + +#[rustfmt::skip] +const OF_DEFAULT: [OffsetCell; 32] = [ + o( 0, 0, 5), o( 0, 6, 4), o( 0, 9, 5), o( 0, 15, 5), + o( 0, 21, 5), o( 0, 3, 5), o( 0, 7, 4), o( 0, 12, 5), + o( 0, 18, 5), o( 0, 23, 5), o( 0, 5, 5), o( 0, 8, 4), + o( 0, 14, 5), o( 0, 20, 5), o( 0, 2, 5), o(16, 7, 4), + o( 0, 11, 5), o( 0, 17, 5), o( 0, 22, 5), o( 0, 4, 5), + o(16, 8, 4), o( 0, 13, 5), o( 0, 19, 5), o( 0, 1, 5), + o(16, 6, 4), o( 0, 10, 5), o( 0, 16, 5), o( 0, 28, 5), + o( 0, 27, 5), o( 0, 26, 5), o( 0, 25, 5), o( 0, 24, 5), +]; + +#[rustfmt::skip] +const ML_DEFAULT: [LengthCell; 64] = [ + l( 0, 0, 6, 0), l( 0, 0, 4, 1), l(32, 0, 5, 2), l( 0, 0, 5, 3), + l( 0, 0, 5, 5), l( 0, 0, 5, 6), l( 0, 0, 5, 8), l( 0, 0, 6, 10), + l( 0, 0, 6, 13), l( 0, 0, 6, 16), l( 0, 0, 6, 19), l( 0, 0, 6, 22), + l( 0, 0, 6, 25), l( 0, 0, 6, 28), l( 0, 0, 6, 31), l( 0, 1, 6, 34), + l( 0, 1, 6, 38), l( 0, 2, 6, 44), l( 0, 3, 6, 56), l( 0, 4, 6, 80), + l( 0, 7, 6, 128), l( 0, 9, 6, 512), l(16, 0, 4, 1), l( 0, 0, 4, 2), + l(32, 0, 5, 3), l( 0, 0, 5, 4), l(32, 0, 5, 6), l( 0, 0, 5, 7), + l( 0, 0, 6, 9), l( 0, 0, 6, 12), l( 0, 0, 6, 15), l( 0, 0, 6, 18), + l( 0, 0, 6, 21), l( 0, 0, 6, 24), l( 0, 0, 6, 27), l( 0, 0, 6, 30), + l( 0, 1, 6, 32), l( 0, 1, 6, 36), l( 0, 2, 6, 40), l( 0, 3, 6, 48), + l( 0, 4, 6, 64), l( 0, 5, 6, 96), l( 0, 8, 6, 256), l(32, 0, 4, 1), + l(48, 0, 4, 1), l(16, 0, 4, 2), l(32, 0, 5, 4), l(32, 0, 5, 5), + l(32, 0, 5, 7), l(32, 0, 5, 8), l( 0, 0, 6, 11), l( 0, 0, 6, 14), + l( 0, 0, 6, 17), l( 0, 0, 6, 20), l( 0, 0, 6, 23), l( 0, 0, 6, 26), + l( 0, 0, 6, 29), l( 0, 16, 6, 65536), l( 0, 15, 6, 32768), l( 0, 14, 6, 16384), + l( 0, 13, 6, 8192), l( 0, 12, 6, 4096), l( 0, 11, 6, 2048), l( 0, 10, 6, 1024), +]; + +/// Which table `LLTptr`, `MLTptr`, `OFTptr` or `HUFptr` points to. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum TableSource { + /// The context's own. + Local, + /// The dictionary's. + Dict, + /// The one the format predefines. There is none for literals. + Predefined, +} + +/// The tables of literal lengths, offsets and match lengths. +#[derive(Clone)] +pub(crate) struct SeqTables { + pub(crate) ll: LengthTable, + pub(crate) of: OffsetTable, + pub(crate) ml: LengthTable, + /// [`ADDITIONAL`], where it takes no more to refer to it than to the tables. + additional: [Additional; 32], +} + +/// What to make of the additional bits of a length. +#[derive(Clone, Copy)] +struct Additional { + /// As many bits set as there are additional bits. + mask: u32, + /// The bit that [`LengthCell::base_value`] lacks. + bit_16: u32, +} + +/// For each number of additional bits. +const ADDITIONAL: [Additional; 32] = { + let mut all = [Additional { mask: 0, bit_16: 0 }; 32]; + let mut left: &mut [Additional] = &mut all; + let mut nb_bits = 0; + while let [additional, others @ ..] = left { + additional.mask = (1 << nb_bits) - 1; + additional.bit_16 = ((nb_bits == 16) as u32) << 16; + nb_bits += 1; + left = others; + } + all +}; + +static DEFAULT_TABLES: SeqTables = SeqTables { + ll: SeqTable::predefined(LL_DEFAULT_NORM_LOG, 32, LL_DEFAULT), + of: SeqTable::predefined(OF_DEFAULT_NORM_LOG, 31, OF_DEFAULT), + ml: SeqTable::predefined(ML_DEFAULT_NORM_LOG, 21, ML_DEFAULT), + additional: ADDITIONAL, +}; + +/// `LLTptr`, `OFTptr` and `MLTptr` +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) struct SeqTableSources { + pub(crate) ll: TableSource, + pub(crate) of: TableSource, + pub(crate) ml: TableSource, +} + +impl SeqTableSources { + pub(crate) const fn all(source: TableSource) -> Self { + Self { + ll: source, + of: source, + ml: source, + } + } +} + +impl SeqTables { + pub(crate) const fn new() -> Self { + Self { + ll: SeqTable::new(), + of: SeqTable::new(), + ml: SeqTable::new(), + additional: ADDITIONAL, + } + } + + /// What additional bits that are the lowest of `bits` make of the `base_value` of a [`LengthCell`]. + #[inline(always)] + pub(crate) fn length(&self, base_value: u16, nb_additional_bits: u8, bits: u64) -> u32 { + match self.additional.get(usize::from(nb_additional_bits) % 32) { + Some(additional) => { + u32::from(base_value) + additional.bit_16 + (bits as u32 & additional.mask) + } + None => 0, + } + } + + /// The three tables that `sources` designates, where they are. + pub(crate) fn pick<'a>( + &'a self, + sources: SeqTableSources, + dict: Option<&'a Self>, + ) -> SeqTableRefs<'a> { + let from = |source| match (source, dict) { + (TableSource::Predefined, _) => &DEFAULT_TABLES, + (TableSource::Dict, Some(dict)) => dict, + _ => self, + }; + SeqTableRefs { + ll: &from(sources.ll).ll, + of: &from(sources.of).of, + ml: &from(sources.ml).ml, + side_by_side: (sources == SeqTableSources::all(sources.ll)).then(|| from(sources.ll)), + } + } + + /// The same side by side. If they are not, those that are not in `self` are copied there, which `sources` then + /// says. + pub(crate) fn gather<'a>( + &'a mut self, + sources: &mut SeqTableSources, + dict: Option<&'a Self>, + ) -> SeqTableRefs<'a> { + let from = |source| match (source, dict) { + (TableSource::Predefined, _) => Some(&DEFAULT_TABLES), + (TableSource::Dict, Some(dict)) => Some(dict), + _ => None, + }; + if *sources == SeqTableSources::all(sources.ll) { + return self.pick(*sources, dict); + } + if let Some(other) = from(sources.ll) { + self.ll.copy_from(&other.ll); + } + if let Some(other) = from(sources.of) { + self.of.copy_from(&other.of); + } + if let Some(other) = from(sources.ml) { + self.ml.copy_from(&other.ml); + } + *sources = SeqTableSources::all(TableSource::Local); + self.pick(*sources, dict) + } +} + +/// Three tables to decode sequences with. +#[derive(Clone, Copy)] +pub(crate) struct SeqTableRefs<'a> { + pub(crate) ll: &'a LengthTable, + pub(crate) of: &'a OffsetTable, + pub(crate) ml: &'a LengthTable, + /// What the three are in, if that is one thing. + pub(crate) side_by_side: Option<&'a SeqTables>, +} + +impl SeqTableRefs<'_> { + /// Often enough for a branch on that to cost more than it saves: one that goes wrong costs many times that. + pub(crate) fn lengths_often_have_more_bits(&self) -> bool { + self.ll.with_more_bits + self.ml.with_more_bits > 512 / 7 + } + + pub(crate) fn matches_are_long(&self) -> bool { + self.ml.with_more_bits > 512 / 2 + } +} diff --git a/src/zstd/error.rs b/src/zstd/error.rs new file mode 100644 index 000000000000..82d11c4e9c6a --- /dev/null +++ b/src/zstd/error.rs @@ -0,0 +1,161 @@ +//! `zstd_errors.h` and `error_private.c`. + +use core::ffi::CStr; + +/// `ZSTD_ErrorCode`, without `ZSTD_error_no_error`. The values are part of `node:zlib`'s constants. +#[repr(u8)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[error("{}", self.message().to_str().unwrap_or_default())] +pub enum Error { + Generic = 1, + PrefixUnknown = 10, + VersionUnsupported = 12, + FrameParameterUnsupported = 14, + FrameParameterWindowTooLarge = 16, + CorruptionDetected = 20, + ChecksumWrong = 22, + LiteralsHeaderWrong = 24, + DictionaryCorrupted = 30, + DictionaryWrong = 32, + DictionaryCreationFailed = 34, + ParameterUnsupported = 40, + ParameterCombinationUnsupported = 41, + ParameterOutOfBound = 42, + TableLogTooLarge = 44, + MaxSymbolValueTooLarge = 46, + MaxSymbolValueTooSmall = 48, + CannotProduceUncompressedBlock = 49, + StabilityConditionNotRespected = 50, + StageWrong = 60, + InitMissing = 62, + MemoryAllocation = 64, + WorkSpaceTooSmall = 66, + DstSizeTooSmall = 70, + SrcSizeWrong = 72, + DstBufferNull = 74, + NoForwardProgressDestFull = 80, + NoForwardProgressInputEmpty = 82, +} + +pub type Result = core::result::Result; + +impl Error { + /// `ZSTD_getErrorString`. + pub const fn message(self) -> &'static CStr { + match self { + Self::Generic => c"Error (generic)", + Self::PrefixUnknown => c"Unknown frame descriptor", + Self::VersionUnsupported => c"Version not supported", + Self::FrameParameterUnsupported => c"Unsupported frame parameter", + Self::FrameParameterWindowTooLarge => c"Frame requires too much memory for decoding", + Self::CorruptionDetected => c"Data corruption detected", + Self::ChecksumWrong => c"Restored data doesn't match checksum", + Self::LiteralsHeaderWrong => { + c"Header of Literals' block doesn't respect format specification" + } + Self::ParameterUnsupported => c"Unsupported parameter", + Self::ParameterCombinationUnsupported => c"Unsupported combination of parameters", + Self::ParameterOutOfBound => c"Parameter is out of bound", + Self::InitMissing => c"Context should be init first", + Self::MemoryAllocation => c"Allocation error : not enough memory", + Self::WorkSpaceTooSmall => c"workSpace buffer is not large enough", + Self::StageWrong => c"Operation not authorized at current processing stage", + Self::TableLogTooLarge => c"tableLog requires too much memory : unsupported", + Self::MaxSymbolValueTooLarge => c"Unsupported max Symbol Value : too large", + Self::MaxSymbolValueTooSmall => c"Specified maxSymbolValue is too small", + Self::CannotProduceUncompressedBlock => { + c"This mode cannot generate an uncompressed block" + } + Self::StabilityConditionNotRespected => { + c"pledged buffer stability condition is not respected" + } + Self::DictionaryCorrupted => c"Dictionary is corrupted", + Self::DictionaryWrong => c"Dictionary mismatch", + Self::DictionaryCreationFailed => c"Cannot create Dictionary from provided samples", + Self::DstSizeTooSmall => c"Destination buffer is too small", + Self::SrcSizeWrong => c"Src size is incorrect", + Self::DstBufferNull => c"Operation on NULL destination buffer", + Self::NoForwardProgressDestFull => { + c"Operation made no progress over multiple calls, due to output buffer being full" + } + Self::NoForwardProgressInputEmpty => { + c"Operation made no progress over multiple calls, due to input being empty" + } + } + } + + /// The name of the constant in `zstd_errors.h`, which `node:zlib` reports as the error's `code`. + pub const fn name(self) -> &'static CStr { + match self { + Self::Generic => c"ZSTD_error_GENERIC", + Self::PrefixUnknown => c"ZSTD_error_prefix_unknown", + Self::VersionUnsupported => c"ZSTD_error_version_unsupported", + Self::FrameParameterUnsupported => c"ZSTD_error_frameParameter_unsupported", + Self::FrameParameterWindowTooLarge => c"ZSTD_error_frameParameter_windowTooLarge", + Self::CorruptionDetected => c"ZSTD_error_corruption_detected", + Self::ChecksumWrong => c"ZSTD_error_checksum_wrong", + Self::LiteralsHeaderWrong => c"ZSTD_error_literals_headerWrong", + Self::DictionaryCorrupted => c"ZSTD_error_dictionary_corrupted", + Self::DictionaryWrong => c"ZSTD_error_dictionary_wrong", + Self::DictionaryCreationFailed => c"ZSTD_error_dictionaryCreation_failed", + Self::ParameterUnsupported => c"ZSTD_error_parameter_unsupported", + Self::ParameterCombinationUnsupported => { + c"ZSTD_error_parameter_combination_unsupported" + } + Self::ParameterOutOfBound => c"ZSTD_error_parameter_outOfBound", + Self::TableLogTooLarge => c"ZSTD_error_tableLog_tooLarge", + Self::MaxSymbolValueTooLarge => c"ZSTD_error_maxSymbolValue_tooLarge", + Self::MaxSymbolValueTooSmall => c"ZSTD_error_maxSymbolValue_tooSmall", + // `node:zlib` has no constant for it. + Self::CannotProduceUncompressedBlock => c"ZSTD_error_GENERIC", + Self::StabilityConditionNotRespected => c"ZSTD_error_stabilityCondition_notRespected", + Self::StageWrong => c"ZSTD_error_stage_wrong", + Self::InitMissing => c"ZSTD_error_init_missing", + Self::MemoryAllocation => c"ZSTD_error_memory_allocation", + Self::WorkSpaceTooSmall => c"ZSTD_error_workSpace_tooSmall", + Self::DstSizeTooSmall => c"ZSTD_error_dstSize_tooSmall", + Self::SrcSizeWrong => c"ZSTD_error_srcSize_wrong", + Self::DstBufferNull => c"ZSTD_error_dstBuffer_null", + Self::NoForwardProgressDestFull => c"ZSTD_error_noForwardProgress_destFull", + Self::NoForwardProgressInputEmpty => c"ZSTD_error_noForwardProgress_inputEmpty", + } + } +} + +/// What the functions that allocate their output fail with. Users get to see the names. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error, strum::IntoStaticStr)] +#[error("{}", <&'static str>::from(self))] +pub enum ZstdError { + /// The output, or the decoder state the frame's window size dictates, could not be allocated. + OutOfMemory, + InvalidZstdData, + DecompressionFailed, + ZstdDecompressionError, + ShortRead, +} + +impl ZstdError { + /// The error for a failed decompression call; `other` is the non-allocation failure. + pub(crate) fn for_decompression(code: Error, other: ZstdError) -> ZstdError { + if code == Error::MemoryAllocation { + ZstdError::OutOfMemory + } else { + other + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn errors_show_what_c_and_users_know_them_by() { + assert_eq!( + Error::ChecksumWrong.to_string(), + "Restored data doesn't match checksum" + ); + assert_eq!(ZstdError::ShortRead.to_string(), "ShortRead"); + assert_eq!(<&'static str>::from(ZstdError::ShortRead), "ShortRead"); + } +} diff --git a/src/zstd/lib.rs b/src/zstd/lib.rs index b9fd810c9449..61c71dcd5710 100644 --- a/src/zstd/lib.rs +++ b/src/zstd/lib.rs @@ -1,333 +1,170 @@ +//! Zstandard (RFC 8878). A port of https://github.com/facebook/zstd v1.5.7 (LICENSE.md): the same bytes out for the +//! same bytes in, in both directions, and the same errors. Where that is not so, a comment tells what the C library does. +#![feature( + adt_const_params, + hint_prefetch, + mpmc_channel, + vec_push_within_capacity +)] +#![forbid(unsafe_code)] +// Frames come from anywhere and a panic aborts the process. +#![deny( + clippy::indexing_slicing, + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::unreachable +)] #![warn(unused_must_use)] -use core::ffi::{c_ulonglong, c_void}; - -use bun_core::ZStr; - -// ─── FFI bindings ───────────────────────────────────────────────────────── -// Externs stay in this crate per PORTING.md §FFI: "If your file has externs -// and isn't already *_sys, leave them in place". -#[allow(non_camel_case_types, non_upper_case_globals)] -pub mod c { - use core::ffi::{c_char, c_int, c_uint, c_ulonglong, c_void}; - - // `ZSTD_DStream` — opaque streaming-decompression context (Nomicon FFI pattern). - // - // `UnsafeCell` makes the type `!Freeze` so a `&ZSTD_DStream` does not assert - // immutability of the C-owned state (zstd mutates internally on every call). - bun_opaque::opaque_ffi! { - pub struct ZSTD_DStream; - /// `ZSTD_CCtx` — opaque streaming-compression context. - pub struct ZSTD_CCtx; - } - - /// `typedef ZSTD_DCtx ZSTD_DStream;` — same opaque object. - pub(crate) type ZSTD_DCtx = ZSTD_DStream; - - // C enums passed by value across FFI — model as `c_uint` (their declared - // underlying type) so callers can pass raw values without transmute. - pub(crate) type ZSTD_ErrorCode = c_uint; - pub(crate) type ZSTD_EndDirective = c_uint; - pub(crate) type ZSTD_ResetDirective = c_uint; - pub(crate) type ZSTD_cParameter = c_uint; - pub(crate) type ZSTD_dParameter = c_uint; - - // ZSTD_EndDirective - pub const ZSTD_e_continue: ZSTD_EndDirective = 0; - - // ZSTD_cParameter - pub const ZSTD_c_compressionLevel: ZSTD_cParameter = 100; - - pub const ZSTD_reset_session_and_parameters: ZSTD_ResetDirective = 3; - - // ZSTD_ErrorCode (zstd_errors.h) — only the public stable subset. - pub const ZSTD_error_no_error: ZSTD_ErrorCode = 0; - pub const ZSTD_error_GENERIC: ZSTD_ErrorCode = 1; - pub const ZSTD_error_prefix_unknown: ZSTD_ErrorCode = 10; - pub const ZSTD_error_version_unsupported: ZSTD_ErrorCode = 12; - pub const ZSTD_error_frameParameter_unsupported: ZSTD_ErrorCode = 14; - pub const ZSTD_error_frameParameter_windowTooLarge: ZSTD_ErrorCode = 16; - pub const ZSTD_error_corruption_detected: ZSTD_ErrorCode = 20; - pub const ZSTD_error_checksum_wrong: ZSTD_ErrorCode = 22; - pub const ZSTD_error_literals_headerWrong: ZSTD_ErrorCode = 24; - pub const ZSTD_error_dictionary_corrupted: ZSTD_ErrorCode = 30; - pub const ZSTD_error_dictionary_wrong: ZSTD_ErrorCode = 32; - pub const ZSTD_error_dictionaryCreation_failed: ZSTD_ErrorCode = 34; - pub const ZSTD_error_parameter_unsupported: ZSTD_ErrorCode = 40; - pub const ZSTD_error_parameter_combination_unsupported: ZSTD_ErrorCode = 41; - pub const ZSTD_error_parameter_outOfBound: ZSTD_ErrorCode = 42; - pub const ZSTD_error_tableLog_tooLarge: ZSTD_ErrorCode = 44; - pub const ZSTD_error_maxSymbolValue_tooLarge: ZSTD_ErrorCode = 46; - pub const ZSTD_error_maxSymbolValue_tooSmall: ZSTD_ErrorCode = 48; - pub const ZSTD_error_stabilityCondition_notRespected: ZSTD_ErrorCode = 50; - pub const ZSTD_error_stage_wrong: ZSTD_ErrorCode = 60; - pub const ZSTD_error_init_missing: ZSTD_ErrorCode = 62; - pub const ZSTD_error_memory_allocation: ZSTD_ErrorCode = 64; - pub const ZSTD_error_workSpace_tooSmall: ZSTD_ErrorCode = 66; - pub const ZSTD_error_dstSize_tooSmall: ZSTD_ErrorCode = 70; - pub const ZSTD_error_srcSize_wrong: ZSTD_ErrorCode = 72; - pub const ZSTD_error_dstBuffer_null: ZSTD_ErrorCode = 74; - pub const ZSTD_error_noForwardProgress_destFull: ZSTD_ErrorCode = 80; - pub const ZSTD_error_noForwardProgress_inputEmpty: ZSTD_ErrorCode = 82; - - #[repr(C)] - pub struct ZSTD_inBuffer { - pub src: *const c_void, - pub size: usize, - pub pos: usize, - } - #[repr(C)] - pub struct ZSTD_outBuffer { - pub dst: *mut c_void, - pub size: usize, - pub pos: usize, - } - - unsafe extern "C" { - pub(crate) fn ZSTD_compress( - dst: *mut c_void, - dst_capacity: usize, - src: *const c_void, - src_size: usize, - compression_level: c_int, - ) -> usize; - pub(crate) safe fn ZSTD_compressBound(src_size: usize) -> usize; - pub(crate) fn ZSTD_decompress( - dst: *mut c_void, - dst_capacity: usize, - src: *const c_void, - compressed_size: usize, - ) -> usize; - // Pure scalar fns — no preconditions. - pub safe fn ZSTD_isError(code: usize) -> c_uint; - pub(crate) safe fn ZSTD_getErrorName(code: usize) -> *const c_char; - pub(crate) safe fn ZSTD_defaultCLevel() -> c_int; - - pub(crate) safe fn ZSTD_createDStream() -> *mut ZSTD_DStream; - pub(crate) fn ZSTD_freeDStream(zds: *mut ZSTD_DStream) -> usize; - pub(crate) fn ZSTD_initDStream(zds: *mut ZSTD_DStream) -> usize; - pub fn ZSTD_decompressStream( - zds: *mut ZSTD_DStream, - output: *mut ZSTD_outBuffer, - input: *mut ZSTD_inBuffer, - ) -> usize; - - // ZSTD_findDecompressedSize() : - // `src` should point to the start of a series of ZSTD encoded and/or skippable frames - // `srcSize` must be the _exact_ size of this series - // (i.e. there should be a frame boundary at `src + srcSize`) - // @return : - decompressed size of all data in all successive frames - // - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN - // - if an error occurred: ZSTD_CONTENTSIZE_ERROR - // - // note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode. - // When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. - // In which case, it's necessary to use streaming mode to decompress data. - // note 2 : decompressed size is always present when compression is done with ZSTD_compress() - // note 3 : decompressed size can be very large (64-bits value), - // potentially larger than what local system can handle as a single memory segment. - // In which case, it's necessary to use streaming mode to decompress data. - // note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified. - // Always ensure result fits within application's authorized limits. - // Each application can set its own limits. - // note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to - // read each contained frame header. This is fast as most of the data is skipped, - // however it does mean that all frame data must be present and valid. */ - pub(crate) fn ZSTD_findDecompressedSize(src: *const c_void, src_size: usize) - -> c_ulonglong; - - // ── streaming-compress / advanced API (used by NativeZstd) ─────── - pub safe fn ZSTD_createCCtx() -> *mut ZSTD_CCtx; - pub fn ZSTD_freeCCtx(cctx: *mut ZSTD_CCtx) -> usize; - pub safe fn ZSTD_createDCtx() -> *mut ZSTD_DCtx; - pub fn ZSTD_freeDCtx(dctx: *mut ZSTD_DCtx) -> usize; - pub fn ZSTD_CCtx_setPledgedSrcSize( - cctx: *mut ZSTD_CCtx, - pledged_src_size: c_ulonglong, - ) -> usize; - /// Copies the dictionary into the context, so `dict` need not outlive - /// the call (unlike the `_byReference` variant). - pub fn ZSTD_CCtx_loadDictionary( - cctx: *mut ZSTD_CCtx, - dict: *const c_void, - dict_size: usize, - ) -> usize; - /// Copies the dictionary into the context; see `ZSTD_CCtx_loadDictionary`. - pub fn ZSTD_DCtx_loadDictionary( - dctx: *mut ZSTD_DCtx, - dict: *const c_void, - dict_size: usize, - ) -> usize; - /// References `prefix` as the raw content dictionary of the next frame only. It is not - /// copied, so it has to outlive the decompression of that frame. - pub(crate) fn ZSTD_DCtx_refPrefix( - dctx: *mut ZSTD_DCtx, - prefix: *const c_void, - prefix_size: usize, - ) -> usize; - pub(crate) fn ZSTD_decompressDCtx( - dctx: *mut ZSTD_DCtx, - dst: *mut c_void, - dst_capacity: usize, - src: *const c_void, - src_size: usize, - ) -> usize; - pub fn ZSTD_CCtx_setParameter( - cctx: *mut ZSTD_CCtx, - param: ZSTD_cParameter, - value: c_int, - ) -> usize; - pub fn ZSTD_DCtx_setParameter( - dctx: *mut ZSTD_DCtx, - param: ZSTD_dParameter, - value: c_int, - ) -> usize; - pub fn ZSTD_DCtx_reset(dctx: *mut ZSTD_DCtx, reset: ZSTD_ResetDirective) -> usize; - pub fn ZSTD_compressStream2( - cctx: *mut ZSTD_CCtx, - output: *mut ZSTD_outBuffer, - input: *mut ZSTD_inBuffer, - end_op: ZSTD_EndDirective, - ) -> usize; - pub safe fn ZSTD_getErrorCode(function_result: usize) -> ZSTD_ErrorCode; - pub safe fn ZSTD_getErrorString(code: ZSTD_ErrorCode) -> *const c_char; - } +mod common; +mod compress; +mod decompress; +mod error; +mod util; + +pub use common::ZSTD_CONTENTSIZE_ERROR; +use common::ZSTD_CONTENTSIZE_UNKNOWN; +use compress::cctx::{FrameCompressor, Output}; +pub use compress::internal::Strategy; +pub use compress::params::{ZSTD_CLEVEL_DEFAULT, c_param, compress_bound}; +use compress::stream::compress_to; +pub use compress::stream::{CCtx, EndDirective, compress}; +use decompress::find_decompressed_size; +pub use decompress::{ + DCtx, DDict, FrameExtent, ZSTD_D_WINDOW_LOG_MAX, frame_extent, get_frame_content_size, +}; +pub use error::{Error, Result, ZstdError}; + +// Callers share contexts between threads behind their own `unsafe impl`s, which should have nothing to vouch for here. +const _: () = { + const fn shareable() {} + shareable::(); + shareable::(); +}; + +/// `ZSTD_inBuffer` +pub struct InBuffer<'a> { + pub src: &'a [u8], + /// How much of `src` is read. + pub pos: usize, } -// ----------------------------------- - -pub enum Result { - Success(usize), - // Always assigned from ZSTD_getErrorName (static C string). - Err(&'static ZStr), +/// `ZSTD_outBuffer` +pub struct OutBuffer<'a> { + pub dst: &'a mut [u8], + /// How much of `dst` is written. + pub pos: usize, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, strum::IntoStaticStr)] -pub enum ZstdError { - /// The output, or the decoder state the frame's window size dictates, could not be allocated. - OutOfMemory, - InvalidZstdData, - DecompressionFailed, - ZstdFailedToCreateInstance, - ZstdDecompressionError, - ShortRead, +/// Output at the end of a `Vec`. The codecs write to initialized memory only: this zeroes little more than gets +/// written, however many calls that takes and however much capacity the `Vec` has to spare. +pub struct VecOutput<'a> { + vec: &'a mut Vec, + start: usize, + /// `vec[..filled]` is output, the rest of it zeros. The `Vec` is cut to it on drop. + filled: usize, } -bun_core::impl_tag_error!(ZstdError); +impl<'a> VecOutput<'a> { + /// Memory that is zeroed and not written stays resident with the output. + const MOST_STEPS_AT_ONCE: usize = 64; -impl ZstdError { - /// The error for a failed (`ZSTD_isError`) decompression call; `other` is the non-allocation failure. - fn for_decompression(rc: usize, other: ZstdError) -> ZstdError { - if c::ZSTD_getErrorCode(rc) == c::ZSTD_error_memory_allocation { - ZstdError::OutOfMemory - } else { - other + pub fn new(vec: &'a mut Vec) -> Self { + let filled = vec.len(); + Self { + vec, + start: filled, + filled, } } -} -/// ZSTD_compress() : -/// Compresses `src` content as a single zstd compressed frame into already allocated `dst`. -/// NOTE: Providing `dstCapacity >= ZSTD_compressBound(srcSize)` guarantees that zstd will have -/// enough space to successfully compress the data. -/// @return : compressed size written into `dst` (<= `dstCapacity), -/// or an error code if it fails (which can be tested using ZSTD_isError()). */ -// ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, -// const void* src, size_t srcSize, -// int compressionLevel); -pub fn compress(dest: &mut [u8], src: &[u8], level: Option) -> Result { - // SAFETY: dest/src are valid for their lengths; ZSTD_compress reads src and writes dest. - let result = unsafe { - c::ZSTD_compress( - dest.as_mut_ptr().cast::(), - dest.len(), - src.as_ptr().cast::(), - src.len(), - // Not redundant_closure: extern "C" fn items don't implement FnOnce - // (Fn* traits are only blanket-impl'd for the Rust ABI). - level.unwrap_or_else(|| c::ZSTD_defaultCLevel()), - ) - }; - if c::ZSTD_isError(result) != 0 { - // SAFETY: ZSTD_getErrorName returns a static NUL-terminated string. - return Result::Err(unsafe { ZStr::from_c_ptr(c::ZSTD_getErrorName(result)) }); + /// The length of the `Vec` up to the end of the output. + pub fn filled(&self) -> usize { + self.filled } - Result::Success(result) -} -pub fn compress_bound(src_size: usize) -> usize { - c::ZSTD_compressBound(src_size) + /// Room for one call of a codec: `step` bytes, or more of them after much output, but never past `limit`. + /// [`advance`](Self::advance) by what it wrote. + pub fn buffer(&mut self, step: usize, limit: usize) -> Result> { + let room = self.vec.len() - self.filled; + let budget = limit.saturating_sub(self.filled); + if room < step.min(budget) { + let most = step.saturating_mul(Self::MOST_STEPS_AT_ONCE); + let more = (self.filled - self.start).clamp(step, most).min(budget) - room; + self.vec + .try_reserve(more) + .map_err(|_| Error::MemoryAllocation)?; + self.vec.resize(self.vec.len() + more, 0); + } + let end = self.vec.len().min(limit); + Ok(OutBuffer { + dst: self.vec.get_mut(self.filled..end).unwrap_or_default(), + pos: 0, + }) + } + + pub fn advance(&mut self, written: usize) { + self.filled = (self.filled + written).min(self.vec.len()); + } } -/// [`compress`] into `out`'s spare capacity (append mode). On success -/// `out.len()` is advanced by the compressed size. Callers must ensure -/// `out.spare_capacity_mut().len() >= compress_bound(src.len())` to -/// guarantee success. -pub fn compress_append(out: &mut Vec, src: &[u8], level: Option) -> Result { - let spare = out.spare_capacity_mut(); - // SAFETY: spare/src are valid for their lengths; ZSTD_compress reads src - // and writes into spare. - let rc = unsafe { - c::ZSTD_compress( - spare.as_mut_ptr().cast::(), - spare.len(), - src.as_ptr().cast::(), - src.len(), - level.unwrap_or_else(|| c::ZSTD_defaultCLevel()), - ) - }; - if c::ZSTD_isError(rc) != 0 { - // SAFETY: ZSTD_getErrorName returns a static NUL-terminated string. - return Result::Err(unsafe { ZStr::from_c_ptr(c::ZSTD_getErrorName(rc)) }); +impl Drop for VecOutput<'_> { + fn drop(&mut self) { + self.vec.truncate(self.filled); } - // SAFETY: zstd has initialized `rc` bytes at the start of spare. - unsafe { bun_core::vec::commit_spare(out, rc) }; - Result::Success(rc) } -/// `ZSTD_isError` — true when a `size_t` return value (including from -/// [`compress_bound`], see zstd.h: "ZSTD_compressBound() itself can fail, if -/// `srcSize >= ZSTD_MAX_INPUT_SIZE`") is an error code rather than a size. -/// Error codes are near `usize::MAX`, so treating one as a size requests an -/// absurd allocation. -pub fn is_error(code: usize) -> bool { - c::ZSTD_isError(code) != 0 +// ----------------------------------- + +/// [`compress`] to the end of `out`, within its capacity: [`compress_bound`] more than its length is always enough. +/// Returns how many bytes are added. +pub fn compress_append(out: &mut Vec, src: &[u8], level: i32) -> Result { + let start = out.len(); + let room = (out.capacity() - start).min(compress_bound(src.len())); + let result = compress_to( + &mut FrameCompressor::new(), + &mut Output::Spare { + vec: out, + start, + end: start + room, + }, + src, + level, + ); + out.truncate(start + result.unwrap_or(0)); + result } -/// `ZSTD_decompress` into `out`'s spare capacity, which is the output bound; commits the bytes written. -fn decompress_append(out: &mut Vec, src: &[u8]) -> core::result::Result<(), ZstdError> { - let spare = out.spare_capacity_mut(); - // SAFETY: spare/src are valid for their lengths; ZSTD_decompress reads src - // and writes at most `spare.len()` bytes into spare. - let rc = unsafe { - c::ZSTD_decompress( - spare.as_mut_ptr().cast::(), - spare.len(), - src.as_ptr().cast::(), - src.len(), - ) - }; - if c::ZSTD_isError(rc) != 0 { - return Err(ZstdError::for_decompression( - rc, - ZstdError::DecompressionFailed, - )); +/// [`compress`] to a new buffer. +pub fn compress_alloc(src: &[u8], level: i32) -> Result> { + let mut out = Vec::new(); + out.try_reserve_exact(compress_bound(src.len())) + .map_err(|_| Error::MemoryAllocation)?; + compress_append(&mut out, src, level)?; + Ok(out) +} + +/// With that much room after its output, a decoder copies in chunks up to the end. +const OUTPUT_SLACK: usize = common::WILDCOPY_OVERLENGTH; + +/// How much room to give a decoder that is to write `size` bytes. +fn output_capacity(size: usize) -> usize { + // Not where it would be most of the buffer: to give that back can take a copy. + if size >= 2 * OUTPUT_SLACK { + size + OUTPUT_SLACK + } else { + size } - // SAFETY: zstd has initialized `rc` bytes at the start of spare. - unsafe { bun_core::vec::commit_spare(out, rc) }; - Ok(()) } /// Decompress data, automatically allocating the output buffer. -/// Returns owned slice that must be freed by the caller. /// Handles both frames with known and unknown content sizes. /// For safety, if the reported decompressed size exceeds 16MB, streaming decompression is used instead. /// Output allocations fail with [`ZstdError::OutOfMemory`] instead of aborting. -pub fn decompress_alloc(src: &[u8]) -> core::result::Result, ZstdError> { - let size = get_decompressed_size(src); +pub fn decompress_alloc(src: &[u8]) -> Result, ZstdError> { + let size = find_decompressed_size(src); - const ZSTD_CONTENTSIZE_UNKNOWN: usize = c_ulonglong::MAX as usize; // 0ULL - 1 - const ZSTD_CONTENTSIZE_ERROR: usize = (c_ulonglong::MAX - 1) as usize; // 0ULL - 2 const MAX_PREALLOCATE_SIZE: usize = 16 * 1024 * 1024; // 16MB safety limit if size == ZSTD_CONTENTSIZE_ERROR { @@ -337,7 +174,7 @@ pub fn decompress_alloc(src: &[u8]) -> core::result::Result, ZstdError> // Use streaming decompression if: // 1. Content size is unknown, OR // 2. Reported size exceeds safety limit (to prevent malicious inputs claiming huge sizes) - if size == ZSTD_CONTENTSIZE_UNKNOWN || size > MAX_PREALLOCATE_SIZE { + if size == ZSTD_CONTENTSIZE_UNKNOWN || size > MAX_PREALLOCATE_SIZE as u64 { let initial_capacity = if size == ZSTD_CONTENTSIZE_UNKNOWN { // A frame's output is rarely smaller than its input. src.len().clamp(STREAMING_OUTPUT_STEP, MAX_PREALLOCATE_SIZE) @@ -348,288 +185,68 @@ pub fn decompress_alloc(src: &[u8]) -> core::result::Result, ZstdError> let mut list: Vec = Vec::new(); list.try_reserve_exact(initial_capacity) .map_err(|_| ZstdError::OutOfMemory)?; - let mut reader = ZstdReaderArrayList::init(src, &mut list)?; - - reader.read_all(true)?; - drop(reader); + StreamingDecoder::new().decompress(src, &mut list, usize::MAX, true)?; return Ok(list); } // Fast path: size is known and within reasonable limits - let mut output: Vec = Vec::new(); - output - .try_reserve_exact(size) - .map_err(|_| ZstdError::OutOfMemory)?; - - decompress_append(&mut output, src)?; - Ok(output) + DCtx::new() + .decompress_to_vec(output_capacity(size as usize), src, None) + .map_err(|code| ZstdError::for_decompression(code, ZstdError::DecompressionFailed)) } /// Decompresses one frame that was compressed with `prefix` as its raw content dictionary, which is /// referenced and not copied. The size has to be in the header of the frame, and at most 16 MB. -pub fn decompress_alloc_with_prefix( - src: &[u8], - prefix: &[u8], -) -> core::result::Result, ZstdError> { - const MAX_SIZE: usize = 16 * 1024 * 1024; - let size = get_decompressed_size(src); +pub fn decompress_alloc_with_prefix(src: &[u8], prefix: &[u8]) -> Result, ZstdError> { + const MAX_SIZE: u64 = 16 * 1024 * 1024; + let size = find_decompressed_size(src); // Also if it is unknown or invalid, which are the two largest values. if size > MAX_SIZE { return Err(ZstdError::InvalidZstdData); } - let mut output: Vec = Vec::new(); - output - .try_reserve_exact(size) - .map_err(|_| ZstdError::OutOfMemory)?; - let dctx = c::ZSTD_createDCtx(); - if dctx.is_null() { - return Err(ZstdError::OutOfMemory); - } - let spare = output.spare_capacity_mut(); - // SAFETY: `dctx` is live until it is freed below. `prefix`, `src` and `spare` are valid for - // their lengths and outlive the calls, and at most `spare.len()` bytes are written to `spare`. - let rc = unsafe { - let rc = c::ZSTD_DCtx_refPrefix(dctx, prefix.as_ptr().cast::(), prefix.len()); - let rc = if c::ZSTD_isError(rc) != 0 { - rc - } else { - c::ZSTD_decompressDCtx( - dctx, - spare.as_mut_ptr().cast::(), - spare.len(), - src.as_ptr().cast::(), - src.len(), - ) - }; - c::ZSTD_freeDCtx(dctx); - rc - }; - if c::ZSTD_isError(rc) != 0 { - return Err(ZstdError::for_decompression( - rc, - ZstdError::DecompressionFailed, - )); - } - // SAFETY: zstd has initialized `rc` bytes at the start of spare. - unsafe { bun_core::vec::commit_spare(&mut output, rc) }; - Ok(output) + let capacity = output_capacity(size as usize); + DDict::prefix(prefix) + .and_then(|prefix| DCtx::new().decompress_to_vec(capacity, src, Some(&prefix))) + .map_err(|code| ZstdError::for_decompression(code, ZstdError::DecompressionFailed)) } -pub fn get_decompressed_size(src: &[u8]) -> usize { - // SAFETY: src is valid for src.len() bytes. - unsafe { c::ZSTD_findDecompressedSize(src.as_ptr().cast::(), src.len()) as usize } +#[derive(Clone, Copy, PartialEq, Eq)] +enum State { + Uninitialized, + Inflating, + End, + Error, } -pub use bun_core::compress::State; - -/// Minimum spare output capacity offered to `ZSTD_decompressStream` per call. +/// Minimum spare output capacity offered to the decoder per call. const STREAMING_OUTPUT_STEP: usize = 4096; -struct ZstdReaderArrayList<'a> { - pub(crate) input: &'a [u8], - // We operate on the caller's Vec directly via the `&mut` borrow. - pub(crate) list_ptr: &'a mut Vec, - // `list_allocator` / `allocator` params deleted — global mimalloc. - pub(crate) zstd: *mut c::ZSTD_DStream, - pub(crate) state: State, - pub(crate) total_in: usize, - /// Decompression-bomb guard: `read_all` errors instead of growing the - /// output past this many bytes. Defaults to unbounded. - pub(crate) max_output_size: usize, -} - -impl<'a> ZstdReaderArrayList<'a> { - // `pub const new = bun.TrivialNew(...)` → Box::new; no associated const needed. - - pub(crate) fn init( - input: &'a [u8], - list: &'a mut Vec, - ) -> core::result::Result>, ZstdError> { - Self::init_with_list_allocator(input, list) - } - - pub(crate) fn init_with_list_allocator( - input: &'a [u8], - list: &'a mut Vec, - // list_allocator / allocator params deleted (global mimalloc). - ) -> core::result::Result>, ZstdError> { - let zstd = c::ZSTD_createDStream(); - if zstd.is_null() { - return Err(ZstdError::ZstdFailedToCreateInstance); - } - // SAFETY: zstd is a freshly created non-null DStream. - let _ = unsafe { c::ZSTD_initDStream(zstd) }; - - Ok(Box::new(ZstdReaderArrayList { - input, - list_ptr: list, - zstd, - state: State::Uninitialized, - total_in: 0, - max_output_size: usize::MAX, - })) - } - - pub(crate) fn end(&mut self) { - if self.state != State::End { - // SAFETY: self.zstd was created by ZSTD_createDStream and has not been freed - // (guarded by state != End). - let _ = unsafe { c::ZSTD_freeDStream(self.zstd) }; - self.state = State::End; - } - } - - pub(crate) fn read_all(&mut self, is_done: bool) -> core::result::Result<(), ZstdError> { - if self.state == State::End || self.state == State::Error { - return Ok(()); - } - - while self.state == State::Uninitialized || self.state == State::Inflating { - let next_in = &self.input[self.total_in..]; - - // If we have no input to process - if next_in.is_empty() { - if is_done { - // If we're in the middle of inflating and stream is done, it's truncated - if self.state == State::Inflating { - self.state = State::Error; - return Err(ZstdError::ZstdDecompressionError); - } - // No more input and stream is done, we can end - self.end(); - } - return Ok(()); - } - - // Decompression-bomb guard: clamp the output space handed to a single - // ZSTD_decompressStream call so one call can never write past the cap. - let remaining_output = self.max_output_size.saturating_sub(self.list_ptr.len()); - if remaining_output == 0 { - self.state = State::Error; - return Err(ZstdError::ZstdDecompressionError); - } - - if self.list_ptr.try_reserve(STREAMING_OUTPUT_STEP).is_err() { - self.state = State::Error; - return Err(ZstdError::OutOfMemory); - } - let spare = self.list_ptr.spare_capacity_mut(); - let mut in_buf = c::ZSTD_inBuffer { - src: next_in.as_ptr().cast::(), - size: next_in.len(), - pos: 0, - }; - let mut out_buf = c::ZSTD_outBuffer { - dst: spare.as_mut_ptr().cast::(), - size: spare.len().min(remaining_output), - pos: 0, - }; - - // SAFETY: self.zstd is a valid DStream (state != End); in_buf/out_buf point - // into live slices with correct sizes. - let rc = - unsafe { c::ZSTD_decompressStream(self.zstd, &raw mut out_buf, &raw mut in_buf) }; - if c::ZSTD_isError(rc) != 0 { - self.state = State::Error; - return Err(ZstdError::for_decompression( - rc, - ZstdError::ZstdDecompressionError, - )); - } - - let bytes_written = out_buf.pos; - let bytes_read = in_buf.pos; - // SAFETY: ZSTD_decompressStream wrote exactly `bytes_written` initialized bytes - // into the spare capacity starting at the previous len. - unsafe { bun_core::vec::commit_spare(self.list_ptr, bytes_written) }; - self.total_in += bytes_read; - - if rc == 0 { - // Frame is complete - self.state = State::Uninitialized; // Reset state since frame is complete - - // Check if there's more input (multiple frames) - if self.total_in >= self.input.len() { - // We've consumed all available input - if is_done { - // No more data coming, we can end the stream - self.end(); - return Ok(()); - } - // Frame is complete and no more input available right now. - // Just return normally - the caller can provide more data later if they have it. - return Ok(()); - } - // More input available, reset for the next frame - // ZSTD_initDStream() safely resets the stream state without needing cleanup - // It's designed to be called multiple times on the same DStream object - // SAFETY: self.zstd is a valid DStream. - let _ = unsafe { c::ZSTD_initDStream(self.zstd) }; - continue; - } - - // If rc > 0, decompressor needs more data - if rc > 0 { - self.state = State::Inflating; - } - - if bytes_read == next_in.len() { - // We've consumed all available input - if bytes_written > 0 { - // We wrote some output, continue to see if we need more output space - continue; - } - - if is_done { - // Stream is truncated - we're at EOF but need more data - self.state = State::Error; - return Err(ZstdError::ZstdDecompressionError); - } - // Not at EOF - we can retry with more data - return Err(ZstdError::ShortRead); - } - } - Ok(()) - } -} - -impl Drop for ZstdReaderArrayList<'_> { - fn drop(&mut self) { - self.end(); - } -} - // ────────────────────────────────────────────────────────────────────────── // StreamingDecoder // ────────────────────────────────────────────────────────────────────────── -/// Streaming zstd decoder that owns only the `ZSTD_DStream`. Unlike -/// [`ZstdReaderArrayList`] it stores no `&'a [u8]` / `&'a mut Vec` -/// borrows — input and output are passed to [`decompress`](Self::decompress) -/// per call, so callers can hold the decoder across multiple body chunks -/// without lifetime erasure. +/// Streaming zstd decoder. Input and output are passed to [`decompress`](Self::decompress) +/// per call, so callers can hold the decoder across multiple body chunks. pub struct StreamingDecoder { - stream: core::ptr::NonNull, - pub(crate) state: State, - /// Decompression-bomb guard: `decompress` errors instead of growing the - /// output past this many bytes. Defaults to unbounded. - pub(crate) max_output_size: usize, - /// zstd filled its last window and may hold more. `max_output` can end a call there. + stream: DCtx, + state: State, + /// The decoder filled its last window and may hold more. `max_output` can end a call there. output_full: bool, } +impl Default for StreamingDecoder { + fn default() -> Self { + Self::new() + } +} + impl StreamingDecoder { - pub fn new() -> core::result::Result { - let stream = core::ptr::NonNull::new(c::ZSTD_createDStream()) - .ok_or(ZstdError::ZstdFailedToCreateInstance)?; - // SAFETY: stream is a freshly created non-null DStream. - let _ = unsafe { c::ZSTD_initDStream(stream.as_ptr()) }; - Ok(Self { - stream, + pub fn new() -> Self { + Self { + stream: DCtx::new(), state: State::Uninitialized, - max_output_size: usize::MAX, output_full: false, - }) + } } #[inline] @@ -637,7 +254,7 @@ impl StreamingDecoder { matches!(self.state, State::Inflating) } - /// Append decompressed bytes to `out` (growing in 4096-byte steps) until `input` is + /// Append decompressed bytes to `out` (growing in steps of 4096 bytes or more) until `input` is /// consumed or `out.len()` reaches `max_output`. Returns the input bytes consumed. /// Returns `ShortRead` when more input is required and `is_done` is false. pub fn decompress( @@ -646,17 +263,18 @@ impl StreamingDecoder { out: &mut Vec, max_output: usize, is_done: bool, - ) -> core::result::Result { + ) -> Result { if matches!(self.state, State::End | State::Error) { return Ok(input.len()); } - let mut total_in = 0usize; + let mut in_buf = InBuffer { src: input, pos: 0 }; + let mut out = VecOutput::new(out); while matches!(self.state, State::Uninitialized | State::Inflating) { - let next_in = &input[total_in..]; + let next_in_len = input.len() - in_buf.pos; - // Call zstd again with no input until it leaves the window short. - if next_in.is_empty() && !self.output_full { + // Call the decoder again with no input until it leaves the window short. + if next_in_len == 0 && !self.output_full { if is_done { if self.state == State::Inflating { self.state = State::Error; @@ -664,76 +282,53 @@ impl StreamingDecoder { } self.state = State::End; } - return Ok(total_in); + return Ok(in_buf.pos); } - if out.len() >= max_output { - return Ok(total_in); + if out.filled() >= max_output { + return Ok(in_buf.pos); } - let remaining_output = self.max_output_size.saturating_sub(out.len()); - if remaining_output == 0 { - self.state = State::Error; - return Err(ZstdError::ZstdDecompressionError); - } - - if out.try_reserve(STREAMING_OUTPUT_STEP).is_err() { + let Ok(mut out_buf) = out.buffer(STREAMING_OUTPUT_STEP, max_output) else { self.state = State::Error; return Err(ZstdError::OutOfMemory); - } - let budget = max_output - out.len(); - let spare = out.spare_capacity_mut(); - let mut in_buf = c::ZSTD_inBuffer { - src: next_in.as_ptr().cast::(), - size: next_in.len(), - pos: 0, }; - let mut out_buf = c::ZSTD_outBuffer { - dst: spare.as_mut_ptr().cast::(), - size: spare.len().min(remaining_output).min(budget), - pos: 0, - }; - - // SAFETY: stream is a valid DStream (not freed); in_buf/out_buf - // point into live slices with correct sizes. - let rc = unsafe { - c::ZSTD_decompressStream(self.stream.as_ptr(), &raw mut out_buf, &raw mut in_buf) + let read_before = in_buf.pos; + + let result = self.stream.decompress_stream(&mut out_buf, &mut in_buf); + let (bytes_written, window) = (out_buf.pos, out_buf.dst.len()); + out.advance(bytes_written); + let remaining = match result { + Ok(remaining) => remaining, + Err(code) => { + self.state = State::Error; + return Err(ZstdError::for_decompression( + code, + ZstdError::ZstdDecompressionError, + )); + } }; - if c::ZSTD_isError(rc) != 0 { - self.state = State::Error; - return Err(ZstdError::for_decompression( - rc, - ZstdError::ZstdDecompressionError, - )); - } - let bytes_written = out_buf.pos; - let bytes_read = in_buf.pos; - self.output_full = bytes_written == out_buf.size; - // SAFETY: zstd wrote exactly `bytes_written` initialized bytes into - // the spare capacity starting at the previous len. - unsafe { bun_core::vec::commit_spare(out, bytes_written) }; - total_in += bytes_read; + self.output_full = bytes_written == window; - if rc == 0 { + if remaining == 0 { // Frame complete, and fully flushed. self.state = State::Uninitialized; self.output_full = false; - if total_in >= input.len() { + if in_buf.pos >= input.len() { if is_done { self.state = State::End; } - return Ok(total_in); + return Ok(in_buf.pos); } // More input available — reinitialize for the next frame. - // SAFETY: stream is a valid DStream. - let _ = unsafe { c::ZSTD_initDStream(self.stream.as_ptr()) }; + self.stream.init_stream(); continue; } self.state = State::Inflating; - if bytes_read == next_in.len() { + if in_buf.pos - read_before == next_in_len { if self.output_full { continue; } @@ -744,14 +339,7 @@ impl StreamingDecoder { return Err(ZstdError::ShortRead); } } - Ok(total_in) - } -} - -impl Drop for StreamingDecoder { - fn drop(&mut self) { - // SAFETY: stream was created by ZSTD_createDStream; freed once here. - let _ = unsafe { c::ZSTD_freeDStream(self.stream.as_ptr()) }; + Ok(in_buf.pos) } } @@ -843,6 +431,7 @@ macro_rules! embed_compressed { /// Cold, shared body of [`embed_compressed!`]'s release arm. #[cold] #[inline(never)] +#[expect(clippy::expect_used, reason = "the build made the frame")] pub fn inflate_embedded(compressed: &'static [u8]) -> Box<[u8]> { decompress_alloc(compressed) .expect("embedded asset: invalid zstd frame") @@ -852,9 +441,528 @@ pub fn inflate_embedded(compressed: &'static [u8]) -> Box<[u8]> { /// [`inflate_embedded`] plus a trailing NUL byte. #[cold] #[inline(never)] +#[expect(clippy::expect_used, reason = "the build made the frame")] pub fn inflate_embedded_nul(compressed: &'static [u8]) -> Box<[u8]> { let mut inflated = decompress_alloc(compressed).expect("embedded asset: invalid zstd frame"); inflated.reserve_exact(1); inflated.push(0); inflated.into_boxed_slice() } + +#[cfg(test)] +mod tests { + use super::*; + use util::SliceExt; + + fn feed(stream: &mut DCtx, output: &mut OutBuffer<'_>, src: &[u8]) -> Result { + stream.decompress_stream(output, &mut InBuffer { src, pos: 0 }) + } + + #[test] + fn a_stream_that_has_failed_goes_on_failing() { + // 5 bytes are announced, and the last block has 3. + let frame = [ + 0x28, 0xB5, 0x2F, 0xFD, 0x20, 5, 0x19, 0, 0, b'a', b'b', b'c', + ]; + let valid = [&frame[..5], &[3], &frame[6..]].concat(); + for with_whole_frame in [false, true] { + let (mut stream, mut dst) = (DCtx::new(), [0; 64]); + let output = &mut OutBuffer { + dst: &mut dst, + pos: 0, + }; + assert_eq!(feed(&mut stream, output, &frame[..9]), Ok(3)); + for more in [&frame[9..], b"x", b"", &valid] { + let result = feed(&mut stream, output, more); + assert_eq!(result, Err(Error::CorruptionDetected)); + } + assert_eq!(stream.load_dictionary(&[]), Err(Error::StageWrong)); + if with_whole_frame { + assert_eq!(stream.decompress(&mut [0; 64], &valid), Ok(3)); + } else { + stream.init_stream(); + } + assert_eq!(feed(&mut stream, output, &valid), Ok(0)); + assert_eq!(output.pos, 3); + } + } + + #[test] + fn a_stream_goes_on_after_a_lack_of_progress_but_not_after_a_whole_frame() -> Result<()> { + let data = Vec::from_iter((0..300u32).map(|i| (i * i % 7) as u8)); + let frame = compress_alloc(&data, 3)?; + let (start, end) = frame.split_at(frame.len() - 5); + let (mut stream, mut dst) = (DCtx::new(), [0; 400]); + let output = &mut OutBuffer { + dst: &mut dst, + pos: 0, + }; + assert_eq!(feed(&mut stream, output, start), Ok(5)); + for _ in 0..15 { + assert_eq!(feed(&mut stream, output, &[]), Ok(5)); + } + let result = feed(&mut stream, output, &[]); + assert_eq!(result, Err(Error::NoForwardProgressInputEmpty)); + let mut input = InBuffer { src: end, pos: 0 }; + assert_eq!(stream.decompress_stream(output, &mut input), Ok(0)); + assert_eq!(input.pos, end.len()); + assert_eq!(*output.dst.sub(..output.pos)?, *data); + + assert_eq!(feed(&mut stream, output, start), Ok(5)); + assert_eq!(stream.decompress(&mut [0; 400], &frame), Ok(data.len())); + assert_eq!(feed(&mut stream, output, end), Err(Error::PrefixUnknown)); + Ok(()) + } + + #[test] + fn a_stream_that_has_failed_to_compress_goes_on_failing() -> Result<()> { + let data = Vec::from_iter((0..1500u32).map(|i| (i * i % 251) as u8)); + for nb_workers in [0, 1] { + let mut cctx = CCtx::new(); + cctx.set_parameter(c_param::COMPRESSION_LEVEL, 1)?; + cctx.set_parameter(c_param::NB_WORKERS, nb_workers)?; + let mut dst = vec![0; compress_bound(data.len())]; + let output = &mut OutBuffer { + dst: &mut dst, + pos: 0, + }; + let mut compress = |cctx: &mut CCtx, src, end| { + cctx.compress_stream2(output, &mut InBuffer { src, pos: 0 }, end) + }; + + // The magic number of a dictionary, and nothing that goes with it: no frame starts. + cctx.load_dictionary(&[0x37, 0xA4, 0x30, 0xEC, 1, 2, 3, 4, 5])?; + let result = compress(&mut cctx, &data, EndDirective::Continue); + assert_eq!(result, Err(Error::MemoryAllocation)); + cctx.load_dictionary(&[])?; + + // Enough for the workers to be used. + cctx.set_pledged_src_size(1 << 20)?; + compress(&mut cctx, &[], EndDirective::Continue)?; + for end in [EndDirective::End, EndDirective::End, EndDirective::Flush] { + let result = compress(&mut cctx, &data, end); + assert_eq!(result, Err(Error::SrcSizeWrong), "{nb_workers} workers"); + } + assert_eq!(cctx.set_pledged_src_size(1), Err(Error::StageWrong)); + } + Ok(()) + } + + #[test] + fn little_more_is_zeroed_than_is_written() { + const STEP: usize = 16; + const MOST: usize = STEP * VecOutput::MOST_STEPS_AT_ONCE; + let mut vec = Vec::new(); + let mut out = VecOutput::new(&mut vec); + while out.filled() < 4 * MOST { + let room = out + .buffer(STEP, usize::MAX) + .map_or(0, |buffer| buffer.dst.len()); + assert!((STEP..=MOST).contains(&room)); + out.advance(room); + } + } + + #[test] + #[cfg_attr(miri, ignore = "takes over 10 minutes")] + fn room_that_grows_gives_what_room_that_is_there_gives() -> Result<()> { + // Blocks that compress well, hardly, and not at all. + let mut state = 1u32; + let src: Vec = (0..400_000u32) + .map(|i| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + match i >> 17 { + 0 => (state % 5) as u8 * (i % 7) as u8, + 1 => (state >> 24) as u8, + _ => (state >> 24) as u8 | 1, + } + }) + .collect(); + for level in [1, 3, 6, 13, 16] { + let mut whole = vec![0; compress_bound(src.len())]; + let size = compress(&mut whole, &src, level)?; + assert!(compress_alloc(&src, level)? == *whole.sub(..size)?); + for capacity in [whole.len() + 1000, size, size - 1, size / 2, 0] { + let mut out = Vec::with_capacity(1 + capacity); + out.push(7); + let room = (out.capacity() - 1).min(whole.len()); + let appended = compress_append(&mut out, &src, level); + assert_eq!(appended, compress(whole.sub_mut(..room)?, &src, level)); + assert!(out.split_first() == Some((&7, whole.sub(..appended.unwrap_or(0))?))); + } + } + Ok(()) + } + + /// A frame with a block, or more, for each of `pieces`. + fn frame_of(cctx: &mut CCtx, pieces: &[&[u8]]) -> Result> { + let mut frame = vec![0; compress_bound(pieces.concat().len()) + 16 * pieces.len()]; + let mut output = OutBuffer { + dst: &mut frame, + pos: 0, + }; + for (index, src) in pieces.iter().enumerate() { + let end = if index + 1 == pieces.len() { + EndDirective::End + } else { + EndDirective::Flush + }; + assert_eq!( + cctx.compress_stream2(&mut output, &mut InBuffer { src, pos: 0 }, end), + Ok(0) + ); + } + let size = output.pos; + frame.truncate(size); + Ok(frame) + } + + #[test] + fn literals_that_start_and_end_like_noise_are_stored_whatever_the_room() { + // Long enough for the C library to judge literals without a match by their first and last 4 KB. + const LEN: usize = 41_000; + const STORED: usize = 4 + 1 + 2 + 3 + LEN; + let mut state = 13u32; + let src: Vec = (0..LEN) + .map(|i| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + let byte = (state >> 8) as u8; + if (4096..LEN - 4096).contains(&i) { + byte % 64 + } else { + byte + } + }) + .collect(); + let cases = [ + (STORED + 7, Ok(STORED)), + (STORED + 6, Ok(STORED)), + (STORED, Ok(STORED)), + (STORED - 1, Err(Error::DstSizeTooSmall)), + ]; + // Miri takes 15 seconds for a call. + for (capacity, expected) in cases.into_iter().take(if cfg!(miri) { 1 } else { 4 }) { + let mut dst = vec![0; capacity]; + assert_eq!(compress(&mut dst, &src, 1), expected); + let mut out = Vec::with_capacity(capacity); + assert_eq!(out.capacity(), capacity); + assert_eq!(compress_append(&mut out, &src, 1), expected); + if expected.is_ok() { + assert!(dst.get(..STORED) == Some(&*out)); + assert!(decompress_alloc(&out).as_ref() == Ok(&src)); + } + } + } + + #[test] + fn tables_that_change_size_give_the_same_frames() -> Result<()> { + let mut state = 11u32; + let src: Vec = (0..if cfg!(miri) { 300 } else { 6000 }) + .map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + (state >> 8) as u8 % 5 + }) + .collect(); + // The tables of the optimal parser are as long as a block. + let mut cctx = CCtx::new(); + cctx.set_parameter(c_param::COMPRESSION_LEVEL, 19)?; + let first = frame_of(&mut cctx, &[&src])?; + frame_of(&mut cctx, &[src.sub(..src.len() / 30)?])?; + assert_eq!(frame_of(&mut cctx, &[&src])?, first); + assert_eq!(decompress_alloc(&first), Ok(src)); + Ok(()) + } + + #[test] + fn long_matches_in_a_dictionary_are_appended() -> Result<()> { + let mut state = 7u32; + let mut noise = move |len: usize| -> Vec { + (0..len) + .map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + (state >> 8) as u8 + }) + .collect() + }; + // Miri takes long to load a dictionary. A block that is let to write 4 KB or less has that zeroed at once. + let unit = if cfg!(miri) { 1000 } else { 4000 }; + let dict = noise(3 * unit); + // A match in the dictionary, one from there into the output, and one in the output. + let first = [&noise(20), dict.sub(unit / 2..unit / 2 + unit)?].concat(); + let pieces = [ + &first, + &[&noise(20), dict.sub(100..100 + unit)?].concat(), + &[ + &noise(20), + dict.sub(3 * unit - unit / 2..)?, + first.sub(..unit / 2)?, + ] + .concat(), + &[&noise(20), first.sub(100..unit)?, &noise(30)].concat(), + ]; + let src = pieces.map(|piece| &**piece).concat(); + let mut cctx = CCtx::new(); + cctx.load_dictionary(&dict)?; + let frame = frame_of(&mut cctx, &pieces.map(|piece| &**piece))?; + assert!(frame.len() < 200); + + let ddict = DDict::by_reference(&dict)?; + let step = if cfg!(miri) { 997 } else { 1 }; + for capacity in + (0..src.len()) + .step_by(step) + .chain([src.len(), src.len() + 32, src.len() + 8192]) + { + let mut there = vec![0; capacity]; + let written = DCtx::new().decompress_using_ddict(&mut there, &frame, &ddict); + let appended = DCtx::new().decompress_to_vec(capacity, &frame, Some(&ddict)); + assert_eq!(appended.as_ref().map(Vec::len), written.as_ref().copied()); + assert_eq!( + appended.is_ok_and(|appended| appended == src), + capacity >= src.len() + ); + } + Ok(()) + } + + #[test] + fn matches_in_a_dictionary_are_found_from_every_frame_of_a_call() -> Result<()> { + let dict = Vec::from_iter((0..3000u32).map(|i| ((i * i % 251) ^ (i >> 3)) as u8)); + // Short and long, from all over the dictionary and from its end on into the output, with a literal between. + let mut src = Vec::new(); + for i in 0..if cfg!(miri) { 60 } else { 600 } { + // Where most are long, sequences are not executed in batches. + let len = if i % 11 == 0 { 70 } else { 3 + i * 7 % 45 }; + let at = if i % 5 == 0 { + dict.len() - len / 2 + } else { + i * 613 % (dict.len() - len) + }; + src.extend_from_slice(dict.sub(at..dict.len().min(at + len))?); + let beyond = (at + len).saturating_sub(dict.len()).min(src.len()); + let from_start = src.sub(..beyond)?.to_vec(); + src.extend_from_slice(&from_start); + src.push(i as u8); + } + let mut cctx = CCtx::new(); + cctx.load_dictionary(&dict)?; + let frame = frame_of(&mut cctx, &[&src])?; + assert!(frame.len() < src.len() / 3); + + let (frames, all) = (frame.repeat(3), src.repeat(3)); + let ddict = DDict::by_reference(&dict)?; + let mut there = vec![0; all.len()]; + let written = DCtx::new().decompress_using_ddict(&mut there, &frames, &ddict); + assert!(written == Ok(all.len()) && there == all); + let appended = DCtx::new().decompress_to_vec(all.len(), &frames, Some(&ddict))?; + assert!(appended == all); + Ok(()) + } + + const MAGIC: [u8; 4] = common::ZSTD_MAGICNUMBER.to_le_bytes(); + + fn le24(value: usize) -> [u8; 3] { + [value as u8, (value >> 8) as u8, (value >> 16) as u8] + } + + #[test] + fn output_does_not_catch_up_with_literals_that_would_be_in_its_buffer() { + let size = |more: usize, nb_seq: usize| (1 << 16) + more + 1027 * nb_seq; + // A last block: 64 KB and `more` times a byte, and sequences of 16 literals and 1027 bytes from 16 back. + let block = |more: usize, nb_seq: usize| { + // For each sequence 4 bits of the offset, 10 of the match length and one of the literal length. After the + // last, a bit set. + let mut bits = vec![0u8; 15 * nb_seq / 8 + 1]; + let of_offsets = (0..nb_seq).flat_map(|i| [15 * i + 11, 15 * i + 12]); + for bit in of_offsets.chain([15 * nb_seq]) { + if let Some(byte) = bits.get_mut(bit / 8) { + *byte |= 1 << (bit % 8); + } + } + let literals = le24(1 | 3 << 2 | ((1 << 16) + more) << 4); + // All three codes are one symbol. + let sequences = [nb_seq as u8, 0x54, 16, 4, 46]; + let header = le24(1 | 2 << 1 | (4 + sequences.len() + bits.len()) << 3); + [&header[..], &literals, b"a", &sequences, &bits].concat() + }; + let in_one_call = |more: usize, nb_seq: usize| { + let content_size = (size(more, nb_seq) as u32).to_le_bytes(); + let frame = [&MAGIC[..], &[0xA0], &content_size, &block(more, nb_seq)].concat(); + let mut dst = vec![0; size(more, nb_seq)]; + let written = DCtx::new().decompress(&mut dst, &frame)?; + dst.truncate(written); + Ok((written, dst == vec![b'a'; written])) + }; + // With a window of 128 KB, after 3 blocks of as many times the byte, which fill the buffer of the stream. + let as_a_stream = |more: usize, nb_seq: usize| { + let filler = [&le24(1 << 1 | 1 << 17 << 3)[..], b"a"].concat().repeat(3); + let frame = [&MAGIC[..], &[0, 7 << 3], &filler, &block(more, nb_seq)].concat(); + let mut dst = vec![0; 600_000]; + let mut output = OutBuffer { + dst: &mut dst, + pos: 0, + }; + assert_eq!(feed(&mut DCtx::new(), &mut output, &frame)?, 0); + let written = output.pos; + dst.truncate(written); + Ok((written - (3 << 17), dst == vec![b'a'; written])) + }; + // Where there is too little room for the C library to keep all the literals in it, it keeps the first `more`, + // which end 32 bytes before the 128 KB that a block can make. + let cases = [ + (500, 60, Ok((size(500, 60), true))), + // The 125th starts 6 bytes after where its literals do, and ends before where the last of them does. + (3698, 125, Err(Error::DstSizeTooSmall)), + (3698, 126, Err(Error::DstSizeTooSmall)), + (3698, 124, Err(Error::CorruptionDetected)), + (3698, 61, Err(Error::CorruptionDetected)), + (3698, 60, Ok((size(3698, 60), true))), + // Too few for a batch where the literals are elsewhere. + (50, 5, Ok((size(50, 5), true))), + ]; + for (more, nb_seq, expected) in cases.into_iter().take(if cfg!(miri) { 2 } else { 7 }) { + assert_eq!(in_one_call(more, nb_seq), expected, "{more}, {nb_seq}"); + if !cfg!(miri) { + assert_eq!(as_a_stream(more, nb_seq), expected, "{more}, {nb_seq}"); + } + } + } + + #[test] + fn literals_of_any_size_follow_literals_of_any_size() -> Result<()> { + let noise: Vec = (0..8_000u32) + .map(|i| (i.wrapping_mul(i) >> 5) as u8) + .collect(); + // The buffer for them gets a new allocation of 72 bytes, of 532 and of 1064. It then has 552 bytes, which 540 + // fit in but for what is read along with them, and room for 800. The last one is of 7032 bytes, few enough for + // the next decoder of the thread to start with it, empty. + let sizes = [40, 500, 520, 540, 800, 7_000, 9, 100]; + let mut dst = vec![0; 1 << 13]; + for stored in [[true, true], [false, false], [true, false]] { + let mut dctx = DCtx::new(); + for ((&size, stored), nth) in sizes.iter().zip(stored.into_iter().cycle()).zip(1..) { + let expected = if stored { + noise.span(100 * nth, size)?.to_vec() + } else { + vec![size as u8; size] + }; + let bytes = expected.sub(..if stored { size } else { 1 })?; + // A window of 128 KB, and a last block without a sequence. + let literals = le24(usize::from(!stored) | 3 << 2 | size << 4); + let header = le24(1 | 2 << 1 | (literals.len() + bytes.len() + 1) << 3); + let frame = [&MAGIC[..], &[0, 7 << 3], &header, &literals, bytes, &[0]].concat(); + dst.fill(0xAA); + let written = dctx.decompress(&mut dst, &frame)?; + assert!(dst.sub(..written)? == expected, "{size}, {stored}"); + } + } + Ok(()) + } + + #[test] + fn output_that_is_appended_is_what_output_that_is_there_is() -> Result<()> { + let mut state = 1u32; + let mut random = move || { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + state >> 8 + }; + let block = + |last: u32, kind: u32, size: u32| ((size << 3 | kind << 1 | last).to_le_bytes(), 3); + // Blocks that are stored and blocks of one byte, with a window of 1 KB. + let mut stored = vec![0x28, 0xB5, 0x2F, 0xFD, 0x00, 0x00]; + for (last, kind, size, content) in + [(0, 0, 300, 300), (0, 1, 500, 1), (0, 0, 0, 0), (1, 0, 9, 9)] + { + let (header, len) = block(last, kind, size); + stored.extend_from_slice(header.sub(..len)?); + stored.extend((0..content).map(|_| random() as u8)); + } + let frame = |src: &[u8]| compress_alloc(src, 3); + // Miri takes long for literals and sequences, and no time for long matches. A block that is let to write 4 KB or + // less has that zeroed at once. + let scale = if cfg!(miri) { 5 } else { 1 }; + let period: Vec = (0..700 / scale).map(|_| random() as u8).collect(); + let mut periods: Vec = period.iter().cycle().take(9000).copied().collect(); + periods.extend((0..40).map(|_| random() as u8)); + let few_sequences = frame(&periods)?; + // The same in blocks of 1 KB, which go on with the offsets of the blocks before. + let mut cctx = CCtx::new(); + cctx.set_parameter(c_param::WINDOW_LOG, 10)?; + let small_blocks = frame_of(&mut cctx, &[&periods])?; + let run = frame(&[b'r'; 9000])?; + let only_literals = frame( + &(0..4300) + .map(|_| (random() % 16 * (random() % 3)) as u8) + .collect::>(), + )?; + let words = [&b"one "[..], b"three ", b"four ", b"seven ", b"eleven "]; + let many_sequences = frame( + &(0..900 / scale) + .flat_map(|_| *words.get(random() as usize % 5).unwrap_or(&&b""[..])) + .copied() + .collect::>(), + )?; + + let same = |frames: &[u8], capacity: usize| { + let mut there = vec![0; capacity]; + let written = DCtx::new().decompress(&mut there, frames); + let appended = DCtx::new().decompress_to_vec(capacity, frames, None); + assert_eq!( + appended.as_ref().map(Vec::len), + written.as_ref().copied(), + "{capacity}" + ); + assert!( + appended.is_ok_and(|appended| Some(&*appended) == there.get(..appended.len())) + || written.is_err() + ); + written + }; + let orders = [ + [ + &many_sequences, + &stored, + &few_sequences, + &run, + &small_blocks, + &only_literals, + ], + [ + &only_literals, + &small_blocks, + &run, + &stored, + &many_sequences, + &few_sequences, + ], + ]; + for order in orders.iter().take(if cfg!(miri) { 1 } else { 2 }) { + let frames = order.map(|frame| &**frame).concat(); + let size = same(&frames, 1 << 16)?; + let (step, around) = if cfg!(miri) { (size, 1) } else { (7, 40) }; + for capacity in (0..size).step_by(step).chain(size - around..size + around) { + assert_eq!(same(&frames, capacity).is_ok(), capacity >= size); + } + let step = if cfg!(miri) { 8 * frames.len() / 3 } else { 1 }; + for bit in (0..8 * frames.len()).skip(step / 2).step_by(step) { + let mut damaged = frames.clone(); + if let Some(byte) = damaged.get_mut(bit / 8) { + *byte ^= 1 << (bit % 8); + } + let _ = same(&damaged, size + OUTPUT_SLACK); + let _ = same(&damaged, size + 8192); + } + } + Ok(()) + } +} diff --git a/src/zstd/util.rs b/src/zstd/util.rs new file mode 100644 index 000000000000..b03f1bb26085 --- /dev/null +++ b/src/zstd/util.rs @@ -0,0 +1,366 @@ +//! Slice access that reports a broken invariant instead of panicking, and `mem.h` / `bits.h`. + +use core::slice::SliceIndex; + +use crate::error::{Error, Result}; + +/// An index or a length that the algorithm keeps in range was out of range. No input should be able to cause it. +pub(crate) const BUG: Error = Error::Generic; + +pub(crate) trait SliceExt { + fn at(&self, index: usize) -> Result + where + T: Copy; + fn at_mut(&mut self, index: usize) -> Result<&mut T>; + fn put(&mut self, index: usize, value: T) -> Result<()>; + fn sub>(&self, range: I) -> Result<&[T]>; + fn sub_mut>(&mut self, range: I) -> Result<&mut [T]>; + /// The `len` elements from `start` on. + fn span(&self, start: usize, len: usize) -> Result<&[T]>; + fn span_mut(&mut self, start: usize, len: usize) -> Result<&mut [T]>; + /// The `N` elements from `start` on. + fn array_at(&self, start: usize) -> Result<&[T; N]>; + fn array_at_mut(&mut self, start: usize) -> Result<&mut [T; N]>; + /// `memmove(&self[dst], &self[src], len)`. + fn move_within(&mut self, src: usize, dst: usize, len: usize) -> Result<()> + where + T: Copy; + /// `memcpy(&self[start], from, from.len())`. + fn write_at(&mut self, start: usize, from: &[T]) -> Result<()> + where + T: Copy; +} + +impl SliceExt for [T] { + #[inline(always)] + fn at(&self, index: usize) -> Result + where + T: Copy, + { + self.get(index).copied().ok_or(BUG) + } + + #[inline(always)] + fn at_mut(&mut self, index: usize) -> Result<&mut T> { + self.get_mut(index).ok_or(BUG) + } + + #[inline(always)] + fn put(&mut self, index: usize, value: T) -> Result<()> { + *self.get_mut(index).ok_or(BUG)? = value; + Ok(()) + } + + #[inline(always)] + fn sub>(&self, range: I) -> Result<&[T]> { + self.get(range).ok_or(BUG) + } + + #[inline(always)] + fn sub_mut>(&mut self, range: I) -> Result<&mut [T]> { + self.get_mut(range).ok_or(BUG) + } + + #[inline(always)] + fn span(&self, start: usize, len: usize) -> Result<&[T]> { + self.get(start..).and_then(|s| s.get(..len)).ok_or(BUG) + } + + #[inline(always)] + fn span_mut(&mut self, start: usize, len: usize) -> Result<&mut [T]> { + self.get_mut(start..) + .and_then(|s| s.get_mut(..len)) + .ok_or(BUG) + } + + #[inline(always)] + fn array_at(&self, start: usize) -> Result<&[T; N]> { + self.get(start..).and_then(<[T]>::first_chunk).ok_or(BUG) + } + + #[inline(always)] + fn array_at_mut(&mut self, start: usize) -> Result<&mut [T; N]> { + self.get_mut(start..) + .and_then(<[T]>::first_chunk_mut) + .ok_or(BUG) + } + + #[inline(always)] + fn move_within(&mut self, src: usize, dst: usize, len: usize) -> Result<()> + where + T: Copy, + { + let src_end = src.checked_add(len).ok_or(BUG)?; + let dst_end = dst.checked_add(len).ok_or(BUG)?; + if src_end > self.len() || dst_end > self.len() { + return Err(BUG); + } + let size = size_of::(); + if dst >= src_end && is_slow_to_copy((dst - src) * size, len * size) { + move_in_parts(self, src, dst, len); + } else { + self.copy_within(src..src_end, dst); + } + Ok(()) + } + + #[inline(always)] + fn write_at(&mut self, start: usize, from: &[T]) -> Result<()> + where + T: Copy, + { + copy(self.span_mut(start, from.len())?, from) + } +} + +/// `to.copy_from_slice(from)` +#[inline(always)] +pub(crate) fn copy(to: &mut [T], from: &[T]) -> Result<()> { + if to.len() != from.len() { + return Err(BUG); + } + let distance = to.as_ptr().addr().wrapping_sub(from.as_ptr().addr()); + if is_slow_to_copy(distance, size_of_val(from)) { + copy_in_parts(to, from); + } else { + to.copy_from_slice(from); + } + Ok(()) +} + +/// glibc 2.39 copies more than 2112 bytes at once with `rep movsb`, which takes up to 25 times as long on Zen 4 +/// if the destination is a little further into its page than the source: +/// . +const MAX_COPY: usize = 2048; + +/// Whether `size` bytes are, at once, to `distance` bytes after where they are. +#[inline(always)] +fn is_slow_to_copy(distance: usize, size: usize) -> bool { + cfg!(all(target_arch = "x86_64", target_env = "gnu")) + && size > MAX_COPY + && distance.wrapping_sub(1) % 4096 < 127 +} + +/// `rep movsb` is also slow to write to pages for the first time, as an append to what is new from the system does. +const NEW_FROM_SYSTEM: usize = 1 << 19; +/// From there on glibc copies in another way on some processors, faster than in parts. +const REP_MOVSB_END: usize = 1 << 20; + +/// [`is_slow_to_copy`], to the end of `to`. +#[inline(always)] +fn is_slow_to_append(to: &Vec, distance: usize, size: usize) -> bool { + is_slow_to_copy(distance, size) + || cfg!(all(target_arch = "x86_64", target_env = "gnu")) + && (MAX_COPY + 1..REP_MOVSB_END).contains(&size) + && to.capacity() >= NEW_FROM_SYSTEM +} + +#[cold] +#[inline(never)] +fn copy_in_parts(to: &mut [T], from: &[T]) { + let part = (MAX_COPY / size_of::()).max(1); + for (to, from) in to.chunks_mut(part).zip(from.chunks(part)) { + to.copy_from_slice(from); + } +} + +/// `slice.copy_within(src..src + len, dst)`, which is in it and starts no earlier than the source ends. +#[cold] +#[inline(never)] +fn move_in_parts(slice: &mut [T], src: usize, dst: usize, len: usize) { + let part = (MAX_COPY / size_of::()).max(1); + for done in (0..len).step_by(part) { + let end = src + (done + part).min(len); + slice.copy_within(src + done..end, dst + done); + } +} + +/// `to.extend_from_slice(from)`, without a new allocation. +#[inline] +pub(crate) fn append(to: &mut Vec, from: &[u8]) -> Result<()> { + if from.len() > to.capacity() - to.len() { + return Err(BUG); + } + let distance = (to.as_ptr_range().end.addr()).wrapping_sub(from.as_ptr().addr()); + if is_slow_to_append(to, distance, from.len()) { + from.chunks(MAX_COPY) + .for_each(|part| to.extend_from_slice(part)); + } else { + to.extend_from_slice(from); + } + Ok(()) +} + +/// [`append`], in the place of what `to` has from `pos` on. +#[inline] +pub(crate) fn append_at(to: &mut Vec, pos: usize, from: &[u8]) -> Result<()> { + if pos > to.len() { + return Err(BUG); + } + to.truncate(pos); + append(to, from) +} + +/// `to.extend_from_within(start..start + len)`, without a new allocation. +#[inline] +pub(crate) fn append_within(to: &mut Vec, start: usize, len: usize) -> Result<()> { + let end = start + len; + if end > to.len() || len > to.capacity() - to.len() { + return Err(BUG); + } + if is_slow_to_append(to, to.len() - start, len) { + (start..end) + .step_by(MAX_COPY) + .for_each(|part| to.extend_from_within(part..(part + MAX_COPY).min(end))); + } else { + to.extend_from_within(start..end); + } + Ok(()) +} + +#[inline(always)] +pub(crate) fn read_le16(data: &[u8], pos: usize) -> Result { + Ok(u16::from_le_bytes(*data.array_at(pos)?)) +} + +#[inline(always)] +pub(crate) fn read_le24(data: &[u8], pos: usize) -> Result { + let [a, b, c] = *data.array_at(pos)?; + Ok(u32::from_le_bytes([a, b, c, 0])) +} + +#[inline(always)] +pub(crate) fn read_le32(data: &[u8], pos: usize) -> Result { + Ok(u32::from_le_bytes(*data.array_at(pos)?)) +} + +#[inline(always)] +pub(crate) fn read_le64(data: &[u8], pos: usize) -> Result { + Ok(u64::from_le_bytes(*data.array_at(pos)?)) +} + +#[inline(always)] +pub(crate) fn write_le16(data: &mut [u8], pos: usize, value: u16) -> Result<()> { + *data.array_at_mut(pos)? = value.to_le_bytes(); + Ok(()) +} + +#[inline(always)] +pub(crate) fn write_le24(data: &mut [u8], pos: usize, value: u32) -> Result<()> { + let [a, b, c, _] = value.to_le_bytes(); + *data.array_at_mut(pos)? = [a, b, c]; + Ok(()) +} + +#[inline(always)] +pub(crate) fn write_le32(data: &mut [u8], pos: usize, value: u32) -> Result<()> { + *data.array_at_mut(pos)? = value.to_le_bytes(); + Ok(()) +} + +#[inline(always)] +pub(crate) fn write_le64(data: &mut [u8], pos: usize, value: u64) -> Result<()> { + *data.array_at_mut(pos)? = value.to_le_bytes(); + Ok(()) +} + +/// `ZSTD_highbit32`: the position of the highest bit set. `value` is not 0. +#[inline(always)] +pub(crate) const fn highbit32(value: u32) -> u32 { + 31 - (value | 1).leading_zeros() +} + +/// `len` zeros, or [`Error::MemoryAllocation`]. An allocator that reuses memory has to clear all of it. +pub(crate) fn try_zeroed(len: usize) -> Result> { + bun_pod::try_zeroed_vec(len).ok_or(Error::MemoryAllocation) +} + +/// Gives `table` the length `len`. What it holds is 0, or if `clean` is false possibly what it held before. +pub(crate) fn reset_table( + table: &mut Vec, + len: usize, + clean: bool, +) -> Result<()> { + if len > table.capacity() { + // First, so that both are not allocated at once. + *table = Vec::new(); + *table = try_zeroed(len)?; + return Ok(()); + } + if clean { + table.clear(); + } + table.resize(len, T::default()); + Ok(()) +} + +/// Makes the capacity of `vec` at least `len`. A `vec` that has to grow is emptied first, so that nothing is copied and +/// both are not allocated at once. It gets twice its capacity, or `max` if that is less. +pub(crate) fn make_room(vec: &mut Vec, len: usize, max: usize) -> Result<()> { + if len <= vec.capacity() { + return Ok(()); + } + let capacity = len.max((2 * vec.capacity()).min(max)); + *vec = Vec::new(); + vec.try_reserve_exact(capacity) + .map_err(|_| Error::MemoryAllocation) +} + +/// A copy of `bytes`, or [`Error::MemoryAllocation`]. +pub(crate) fn try_copy(bytes: &[u8]) -> Result> { + let mut vec = Vec::new(); + vec.try_reserve_exact(bytes.len()) + .map_err(|_| Error::MemoryAllocation)?; + append(&mut vec, bytes)?; + Ok(vec) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn long_copies_are_whole_at_every_distance() -> Result<()> { + const PAGE: usize = 4096; + // Miri takes its time. The distances that are slow come first. + let step = if cfg!(miri) { 61 } else { 1 }; + let from: Vec = (0..2 * MAX_COPY + 77).map(|i| (i % 251) as u8).collect(); + let mut to = vec![0; 3 * PAGE + from.len() + 1]; + // As far into its page as `from`. + let base = PAGE + from.as_ptr().addr() % PAGE - to.as_ptr().addr() % PAGE; + for start in (base..base + PAGE).step_by(step) { + to.write_at(start, &from)?; + assert!(*to.span(start, from.len())? == *from); + assert_eq!((to.at(start - 1)?, to.at(start + from.len())?), (0, 0)); + to.fill(0); + + let mut vec = vec![0; start]; + vec.reserve_exact(from.len()); + let too_long = vec.capacity() - vec.len() + 1; + assert_eq!(append(&mut vec, to.sub(..too_long)?), Err(BUG)); + append(&mut vec, &from)?; + assert!(*vec.sub(start..)? == *from); + } + let mut within = from.repeat(3); + let len = MAX_COPY + 9; + for distance in (PAGE..PAGE + 200).chain(len..PAGE).step_by(step) { + within.truncate(3 * from.len()); + within.reserve(len); + append_within(&mut within, 3 * from.len() - distance, len)?; + assert!(within.span(3 * from.len() - distance, len)? == within.sub(3 * from.len()..)?); + + let expected = within.span(5, len)?.to_vec(); + within.move_within(5, 5 + distance, len)?; + assert!(*within.span(5 + distance, len)? == *expected); + within.clear(); + (0..3).for_each(|_| within.extend_from_slice(&from)); + } + + // To what is taken to be new from the system, whatever the distance. + let mut vec = Vec::with_capacity(NEW_FROM_SYSTEM); + append(&mut vec, &from)?; + append_within(&mut vec, 1, len)?; + assert!(*vec.sub(..from.len())? == *from && vec.sub(from.len()..)? == from.span(1, len)?); + Ok(()) + } +} diff --git a/test/internal/build-hardened-flags.test.ts b/test/internal/build-hardened-flags.test.ts index 4e5e0443fe6a..d92fb426ba03 100644 --- a/test/internal/build-hardened-flags.test.ts +++ b/test/internal/build-hardened-flags.test.ts @@ -4,6 +4,7 @@ import { existsSync } from "node:fs"; import { join } from "node:path"; import { resolveConfig, type Config, type PartialConfig, type Toolchain } from "../../scripts/build/config.ts"; +import { programDeps } from "../../scripts/build/deps/index.ts"; import { mimalloc } from "../../scripts/build/deps/mimalloc.ts"; import { webkit } from "../../scripts/build/deps/webkit.ts"; import { fetchCliPath } from "../../scripts/build/fetch-cli.ts"; @@ -318,7 +319,10 @@ test("the libraries that never call back into JavaScript keep their arrays on a expect(safeStackFlags(on, name)).toEqual(["-fsanitize=safe-stack"]); expect(safeStackFlags(off, name)).toEqual([]); } - for (const name of [...librariesThatCallBackIntoJavaScript, "mimalloc", "WebKit"] as const) + // The programs beside bun have no second stack. + const forOtherPrograms = programDeps.map(dep => dep.name); + expect(forOtherPrograms).toContain("zstd"); + for (const name of [...librariesThatCallBackIntoJavaScript, "mimalloc", "WebKit", ...forOtherPrograms] as const) expect(safeStackFlags(on, name)).toEqual([]); expect(everyFlagOf(on).cxx).toContain("-DBUN_SAFE_STACK=1"); expect(everyFlagOf(on).dependencyC).not.toContain("-fsanitize=safe-stack"); diff --git a/test/internal/build-local-webkit.test.ts b/test/internal/build-local-webkit.test.ts index d61faf2684f8..dd12702e436b 100644 --- a/test/internal/build-local-webkit.test.ts +++ b/test/internal/build-local-webkit.test.ts @@ -20,6 +20,7 @@ import { webkitTestAotPath, type WebKitAotFile, } from "../../scripts/build/deps/webkit.ts"; +import { zstd } from "../../scripts/build/deps/zstd.ts"; import { BuildError } from "../../scripts/build/error.ts"; import { Ninja, readManifest } from "../../scripts/build/ninja.ts"; import { getProfile } from "../../scripts/build/profiles.ts"; @@ -128,6 +129,7 @@ describe("which builds compile WebKit", () => { name === "testaot" ? ["zstd"] : undefined, ]), ); + expect(zstd.enabled!(cfg)).toBe(true); expect(webkit.provides(cfg).files).toEqual(expect.arrayContaining([objects("jsc"), objects("testaot")])); expect([webkitJSCPath(cfg), webkitTestAotPath(cfg)]).toEqual([join(buildDir, "jsc"), join(buildDir, "testaot")]); }); @@ -766,6 +768,24 @@ describe.skipIf(Bun.which(python) === null)("the graph of a checkout", () => { for (const source of program.sources) cxx(n, cfg, source, { flags: program.cxxflags }); } }); + + test("the sources of zstd are fetched for the programs that read the ICU of a prebuilt WebKit, and for nothing else", () => { + using local = checkOut({}, checkout); + const { buildDir, localDeps } = local.cfg; + const cfg = linux({ localDeps: `WebKit=${localDeps.WebKit}`, buildDir }); + const saved = process.env.BUN_ICU_ROOT; + try { + process.env.BUN_ICU_ROOT = join("/somewhere", "icu"); + expect(zstd.enabled!(cfg)).toBe(true); + // A prebuilt WebKit that is not fetched yet, whatever the cache of this machine has. + expect(zstd.enabled!(linux({ buildDir, cacheDir: join(buildDir, "cache") }))).toBe(false); + delete process.env.BUN_ICU_ROOT; + expect(zstd.enabled!(cfg)).toBe(false); + } finally { + if (saved === undefined) delete process.env.BUN_ICU_ROOT; + else process.env.BUN_ICU_ROOT = saved; + } + }); }); describe("DirectBuild groups and steps", () => { @@ -779,10 +799,10 @@ describe("DirectBuild groups and steps", () => { }); const root = String(dir); const buildDir = join(root, "build"); - const cfg = linux({ buildDir, asan: false, localDeps: `zstd=${join(root, "src")}` }); - const out = join(buildDir, "deps", "zstd"); + const cfg = linux({ buildDir, asan: false, localDeps: `zlib=${join(root, "src")}` }); + const out = join(buildDir, "deps", "zlib"); const dep: Dependency = { - name: "zstd", + name: "zlib", source: () => ({ kind: "github-archive", repo: "x/y", commit: "0" }), build: () => ({ kind: "direct", @@ -822,9 +842,9 @@ describe("DirectBuild groups and steps", () => { const rel = (p: string) => relative(buildDir, p); // The tool's object is not bun's; the generated header is order-only for consumers. - expect(resolved.objects.map(rel)).toEqual(["obj/vendor/zstd/a.cpp.o", "obj/vendor/zstd/b.c.o"]); + expect(resolved.objects.map(rel)).toEqual(["obj/vendor/zlib/a.cpp.o", "obj/vendor/zlib/b.c.o"]); expect(resolved.libs).toEqual(["/outside/libicu.a"]); - expect(resolved.generatedHeaders.map(rel)).toEqual(["deps/zstd/gen.h"]); + expect(resolved.generatedHeaders.map(rel)).toEqual(["deps/zlib/gen.h"]); expect(resolved.outputs).toEqual([]); const lines = n @@ -838,17 +858,17 @@ describe("DirectBuild groups and steps", () => { for (let i = at + 1; lines[i]?.startsWith(" "); i++) vars.push(lines[i]!.trim()); return { line: lines[at]!.replace(/ \| \S+(?=: )/, ""), vars: vars.join("\n") }; }; - expect(edge("deps/zstd/gen.h").line).toBe("build deps/zstd/gen.h: dep_gen deps/zstd/bin/tool"); - expect(edge("deps/zstd/gen.h").vars).toContain(`opts = --cwd=${out} --env=K=V --stdout=${join(out, "gen.h")}`); - expect(edge("deps/zstd/gen.h").vars).toContain("cmd = ruby gen.rb"); - expect(edge("deps/zstd/bin/tool").line).toStartWith("build deps/zstd/bin/tool: link obj/vendor/zstd/tool.cpp.o |"); - expect(edge("deps/zstd/.lib-ready").line).toBe("build deps/zstd/.lib-ready: phony deps/zstd/gen.h"); - expect(edge("pch/prefix.h.hxx.pch").line).toEndWith("|| pch/.dir deps/zstd/.lib-ready"); - expect(edge("obj/vendor/zstd/a.cpp.o").line).toContain(": cxx_pch "); - expect(edge("obj/vendor/zstd/a.cpp.o").line).toEndWith("|| obj/.dir deps/zstd/.lib-ready"); - expect(edge("obj/vendor/zstd/a.cpp.o").vars).toContain("-DLIB -std=c++23"); - expect(edge("obj/vendor/zstd/b.c.o").vars).toStartWith("cflags = -x c++ "); - expect(edge("lib").line).toBe("build lib: phony obj/vendor/zstd/a.cpp.o obj/vendor/zstd/b.c.o"); + expect(edge("deps/zlib/gen.h").line).toBe("build deps/zlib/gen.h: dep_gen deps/zlib/bin/tool"); + expect(edge("deps/zlib/gen.h").vars).toContain(`opts = --cwd=${out} --env=K=V --stdout=${join(out, "gen.h")}`); + expect(edge("deps/zlib/gen.h").vars).toContain("cmd = ruby gen.rb"); + expect(edge("deps/zlib/bin/tool").line).toStartWith("build deps/zlib/bin/tool: link obj/vendor/zlib/tool.cpp.o |"); + expect(edge("deps/zlib/.lib-ready").line).toBe("build deps/zlib/.lib-ready: phony deps/zlib/gen.h"); + expect(edge("pch/prefix.h.hxx.pch").line).toEndWith("|| pch/.dir deps/zlib/.lib-ready"); + expect(edge("obj/vendor/zlib/a.cpp.o").line).toContain(": cxx_pch "); + expect(edge("obj/vendor/zlib/a.cpp.o").line).toEndWith("|| obj/.dir deps/zlib/.lib-ready"); + expect(edge("obj/vendor/zlib/a.cpp.o").vars).toContain("-DLIB -std=c++23"); + expect(edge("obj/vendor/zlib/b.c.o").vars).toStartWith("cflags = -x c++ "); + expect(edge("lib").line).toBe("build lib: phony obj/vendor/zlib/a.cpp.o obj/vendor/zlib/b.c.o"); }); }); @@ -876,13 +896,13 @@ describe("Provides.files", () => { test("are only for a prebuilt source", () => { using g = graph(); const dep: Dependency = { - name: "zstd", + name: "zlib", source: () => ({ kind: "github-archive", repo: "x/y", commit: "0" }), build: () => ({ kind: "direct", sources: [] }), provides: () => ({ libs: [], includes: [], files: ["COPYING"] }), }; expect(() => resolveDep(g.n, g.cfg, dep, new Map())).toThrow( - "zstd: Provides.files is for a prebuilt source", + "zlib: Provides.files is for a prebuilt source", ); }); }); diff --git a/test/js/bun/util/zstd-frames-fixture.json b/test/js/bun/util/zstd-frames-fixture.json new file mode 100644 index 000000000000..e5b07fbe3c93 --- /dev/null +++ b/test/js/bun/util/zstd-frames-fixture.json @@ -0,0 +1,466 @@ +[ + { + "frame": "KLUv/QQmfAQAihAYARGwJekMZykICx3exGGUMUlyGgsACwANAHu32//Zdz27iv//9d/d+f/v4///ZgPrLJ/f713etje3e+IB//v/8ipeL/nf/j8aICCEEKRcH7YGTCGMw0GrcAdaEiTQ/TOYGxS1XSkQMCQgoxpAg/uIp8nleTaAZvbZQUHjJiXuNSIggPDACga/VVhmbgg8AgCbBFQABAAEAAQA8bd/Bf3a+wG7/24B3S7fC6iQKKTgjn0HECDBQ+wBEEgUQxRQvAz77zSBNlDa6dnlDd0QIOWvVwsFDPfZAgwCAA8JQAgABgAIAAcAb1P71/8/sWFu9jevUw9/X3fcaX4W+Td/z3ffBagQJsWe8NJlDxCIsR9wXrhwF9FZlv0M0ggYAAAAaAAAwEFeQd26rAikQUFeQRwAACGNACAAAAAnXwsAAAAJAADTJM6/Ig==", + "results": "803 1eb85d07,803 1eb85d07,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,612 808c5c76,ERR_ZSTD,803 1eb85d07,ERR_ZSTD,803 1eb85d07,ERR_ZSTD" + }, + { + "frame": "KLUv/QRIFAQAMskRHJAVFCTpn79ATN+tjW10TvMF24iNbnRO+/X4lgh5M5G3dVIgtY6gwB3d5EStexsL5BlAiBsyXBcOnGlOB2MSMyx6JyNHEggPqJBqZiZNdqnLARAcgYYPMMoEKZiZyq7WPwOpFSlX+fdJ0SO8eIBg91Bib8C+gcz7sfc1RLQuCwUBAIODBFuCyjKeTZYA1glO0EGTVCUsAgMAQAMSuBbkXp5EloX6Ww==", + "results": "430 b9de9245,430 b9de9245,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,430 b9de9245,ERR_ZSTD,430 b9de9245,ERR_ZSTD" + }, + { + "frame": "KLUv/QRTBRcANv+CjhEREREREREQfwB/AH8AkEAMir8VENRVlONP/bC9/OXrU9FrnA8VYM9zz4/NwjWeZxRLKoMtU/mVbG+V4Yu/JXvui7gMXaPVkGk5YkdwB1YQHTnup8Ag9Zh8amot540JRii1h+2GW3fXqAzIBUyEI+LPfKgO5uHWcF8ysBe3i4rRuqREbF1gdlsGOL9LGkb77tLHVOCWruEj0VfNBzxPllK9Ag9E0wzLnVwhow+D0wT4VUdifMCRNUKWdZNMB7S8aNMmNM99ABnRgnEEPLS0SPFAd9YOolxQa3jMuQnZZhxqK0hgQYp9VirQh7RQdLvrNFblslWncAhS6uXK7jszTcDzld4IkqCvOiXBlhCXF/fyfRmfZoYO/5pCrJBwSAdFbwQ7sRP83a09NPTi9yD73S7CXxIUAJvsLYybV4hqGq7WJuHBfp//vETum7/VIe/jMrGSOjOnaYgz3IKNmTUXzgJZjYfPr/MPdKgmrZyhoZAdW9weJRW9Cs06E/RXrQjgco+csP2R2OyMO0UfQHtPuu8NS713krOBPIec5zmqqc5rqEHkyrelTz8SKiS6/45MDLC8h2yEPZUqCyGW6Tk3t7G5zuk6g7S9dBIRTU0iXukXKQnQAcqinoQwx0/7kIelEtd5vtwuKQAKANk8yfn0yraP+4oMwvYpm8sFdh9aK/yOEex7LC57m55fAUsoMIiIDNXdAYKAGI2UJI0ZY4TQkM2ECcssbTGNCCdb+fgeOzZgM19HAXKgjkW2+GHx0Zm9jGAgShlHVNg0E4g3hvqH44XrUi483sFEliOVbR3Sd+6SzplnEhKPPNBaJmpLXG0DxMViz3lL7DoAsH5d0Xanerl6Z6CRadMgyDJK9ke8pfjmbmG7yqrOGK4RQddUiy6gIpVBcIiRFje8bPcSJDzH4HJbi+bDRRjvQw2iCg8RxR/HN7EJK5IRT3Li2YMFZBOsFVuePxuQkpJ7OoDNKyEn/KM=", + "results": "1833 456abcfa,1833 456abcfa,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,1833 456abcfa,ERR_ZSTD,1833 456abcfa,ERR_ZSTD" + }, + { + "frame": "KLUv/cQ2NgYAAAAAAADEBAAyzRyOERERERERERAp0lvBh+qowsqs4pI2OSjia7lDt4ChbxBpTwtjI7K/iuSnc5ZLRPjgAHLhO1PZEYyVu21nd0GN8UBKxIoPMJxQfNpwuf8esBxBADKD//7Vf4QNP5mTzTxLgKcyCzpRLDf8JaJkguwRzDgbA6gUgFYB76rAf/8T0FUAq6A/FMBWAbsq8M8/Ryg1Z8hLj70FAG4awCEAGoAVJEk6MxXTAX0DM0ZmGdR1zsKe4hJ6JAS+GQAbABgAfwvhgBD0fI/VGMlLasfuxsM6vH7WXMrbuH1QVFzM5lGixdYeJteDVhWa6hWiTLrctfCuC/9fXLcwa5KQPxH9ZtVqk/NulttbqiHOBtfrlGy8bknAiSwIyrfQi3ydELWuD/X2XskBCQhQAlLiH4r/q6gCpcYBA3jzhj4RC/0ILxAg9ErmZOQXnJsm6xnhpNUMEm+GOAw=", + "results": "1590 cf337341,1590 cf337341,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,257 7a917cae,ERR_ZSTD,1590 cf337341,ERR_ZSTD,1590 cf337341,ERR_ZSTD" + }, + { + "frame": "KLUv/QRHHAMABkoJBuAPxw4gyAYABgAGAP//////Af//3/e/D/2++///H////9/ODxOoYMpQzUSFxa2WAxAoY1TtAXCCgmbCLgcgQKTwRO04jcpK7Lh4bokE/naFVtgbEbzLMj1bbaAn8HwBwgUAwDAAAPjFwJAKHwgAAE8AAAAJAADv2oEcaw==", + "results": "482 4f2d578c,482 4f2d578c,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,482 4f2d578c,ERR_ZSTD,482 4f2d578c,ERR_ZSTD" + }, + { + "frame": "KLUv/URbNw8UEADSAwqOERERERERERBOqpJoHuEQhc7pIYoWGlDF8NyLAiQMukBUNLZwEhsQgPaokM8BIIgQmTG0A4EsyCRDNCMjSVJpDgtsQ6NoNPkEMiEXa0UVcEBq3JB3E8fYd8xPSye5q8R4vEgyiu6q0oGyr0mCneZqTofvCtKpd5bRbIMUN+oOuCXg5atl1JYY71YMOImTu2KZAj1OFzXtCJbUmsMQ5F6m00OF3wIeoKWh4+Ky5FJPA/2ikCjeUNwsaIJC1gydtWd8ESd32y+kYb1d+4wfLTJHJDQuBuZYQHan7CQ9aAQTJOQ27jXD9C+jKcpm/1av71nd/T3EILX2wjquqClP05vxC/a/4REKVjcP22g0YUSgD6I2RNz2AS/ccHLdQarZsDVDaRQsAqZWMKF2RG7gmTifDIXQ1RZr/YE/m5gWs6OCZxYQeJCPDnQuhzs5ZQy2FwdHFuj2l1ctYyEIE5yJ7RZ4DrJPhQhzA/Ant6fSeeG0YB/EgDXrv7bBm77x1lc/tHnvGAAUwkBq+10w1rjNO8WMpp9ry5Xdmlwguni91yheDHU+q3w0FGZApSOom+At7Dqv2itUSRcwdeUUIrdzEot4su4JrkR0oB60ZM9FYnYPyUKN7twjapAR31R5wVbhMHwMEQ0DWdOzJYMTqo+rdVBRNyVCcAk8RrfwJuQzGUcBJAcA/w9AIgAhACEAIQA2Yk67JwAf9geZeyIj/tU7HqTUMW4ionoPZyEXNKr9AwFwbmI73HzRbU/oGdlxOn18ejtBw/RKDsZJKmROijyapAGe7KFa3Z6ptuCIz+YA42HcFuL4jeNhIrbEfAJdkZq8NgFLS8vhcy7p4a32/L84mFnz3CwfeOVl3y3sbpnB8hbGEh+oUIIyJBKhJGmWAzBK7KQPQF7NNo0BfBgGXHOg5VWlAG/WECYrCWXU7LEpPnsfttxg5Xl5EyJWi9muRHjim4vOvMPLcJ8RrstGV5OtzOaRFXxICAJaoAIASOkvJlo7CjTq3QsLL2hd5LGzCYW+pzHAac8+GIGqSjNYvSr0bjVI3O/igcPo7/paCRLy3vBCxzyYThyHNYdIaTwUDc7JQNxBGh8XgZLHxBy4yvWRAAAAFQcAzhGAJQAigCVJ0gEgywzseK5lMiHAeYs8M7A7GzfNDOxwnmUyIeA0Mx0AGwAaAMsN5sonrb7+BSND17YEicfc1H+w4Zi8HzDAO0AGvzD9Frx/unVJDFx09iDQKD/COb6vmoJCvM0OAoHZWKUJ878r6FAdnM5p2707cu2/tnC/cAgrppY++BsSliy/wwvG9ixJnu8m/OYpD/zPsB4SKGDiU+8wwpT4n4lnoploghnbAdKDscRgvRKmlAu2wBeVS1zNg46YZ3R6RcmTBbRdr1dLMCuXYlS9vRBaU55q6+Rbymxy2mL16UE=", + "results": "4151 5343f820,4151 5343f820,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,2447 7791b47b,ERR_ZSTD,4151 5343f820,ERR_ZSTD,4151 5343f820,ERR_ZSTD" + }, + { + "frame": "KLUv/QRQJAQA4lwRBuAPWHFRRLf//7n/+//////+7/////f/v/+/9///////////z//fv///vv79///9/vX3///+1///9/////////////8DD6iw4iaYeCaaubQDEBghNGgPUIJR0oSJS9v6Z7j8/QA61PIVRkvkgwOlVOB1JShO/y+wyhLe9LL1W1QG2gMAVoUBABoGmAAO0GU4ACyFGIAiwMEW5wEFAAQABAD73P6+Af//zxf7499//X/PBwFUAAUTNHyF4Es=", + "results": "867 e531346a,867 e531346a,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,748 223bca28,ERR_ZSTD,867 e531346a,ERR_ZSTD,867 e531346a,ERR_ZSTD" + }, + { + "frame": "KLUv/UQKsggAAABcFwCOEsApAI4RERERERERECYAJgAmACZdtqp0ktFsBGe+HbTIXNN4OXSkUomDH8HkhI5omD4jYSizM5oB3rPO7K+09WVBeJvyIyn4QsZAu6mG5qGpRGTEhWJL931tOQLAMwHR8MhqZEMjEfc7dosMZPtbG5zwoNO3N7c7eeS3hdZ7vFyjjbypAWtfIjZXrpJRat1XYMItRT1pstruuF2KOU7Xi8Ik9cuiQLKajBkBgVGo8dXkBkEYYhCDEtwD8eGdMWIESLnZ0WtQLALlHskME8eMsokEh6IWr4RaE6OCCkP9dRgIWhoZLLRipBj4FgXVwlJbt7AoYhRAcA1VaEv5k7pJvBR1qJFcAaiCYYiNRxMRL0LwdfgWWMSctgjTokbg5+NVDGNlnkjDeQZoJoTwFP+QN4u/6VjqLKb+zDKE+s6RIdgbr53a+t2AqB3CQqOtQnKxhHKi7aBNCraH2fk8a6HeVQ2tBSINZg2BiovKV32VLNFCGeEMFgTjN4yzLGDKXlgffgTjaiR8Az7CXqqHQjYtQuIhlpCipgdbMdOmLD7mOgO1wkO3vErAgv8dteFwxs77nOM79lXmwiPaiGS2S+cUL01eVJE4iv7FckUu58j9XfnfjImklTwiDvDs4A2lg3jXpMhvmJVtuqTt5jygBcihEGgxwHjQY2DDjZJZLyLHb0WHndekey7i6dJH8tCSqH8s+IH4HREyhW6B6mwLdQLyBe00nEmA/6c3+15rcuRJd70yCrEU/wB7bYB58ThoWzD/UjDyC7j2bXMAyrZAhZC3xqKXefNYHNVohKUv5nQYMBXxcxQGInZO8WXBLJhLsSIj3nt0/mYEJeTJjh+kZqyotjMLNs0brp1M3B6KYApJQc9F8BW8fX7XsRu5vErq/4IXGphOTEFEJpHIo8oth1ZCpNCBkWcdvN7iG4FEmIEAlP2gNLSuVGwG4gl+mmNMlHMFsH4piAdtL+c0qxQOJ1+P7xBkaAwBeyxkrA4UBQBrCVQBFAAUABQAD6Mkpe4JCKY8+Am/TzRhy8GORAGYTnj5i/WIo20wdoi0WQhJTGYqAZN6X1SJIVBH21ubHKttlnMDz7EBGyP1qv7/7Bii4nD26To89DH6AROogMpYOBUWV6sBgCaYcNAeMMQIRjAyE8pMPJSWHyYPZHKQpwDreH1M+qagDWhDY9mj69NvdcEJ6lCqWErO0IgIB1jkV/owAumFAQCuCMAIAAjwOS8F+z8RAQUABQAFAP////8P/5///3D/////D//8//8JAVQTCiqJygFH/ROH", + "results": "2482 de10fc0a,2482 de10fc0a,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,2482 de10fc0a,ERR_ZSTD,2482 de10fc0a,ERR_ZSTD" + }, + { + "frame": "KLUv/QRReAYArsDi//amUBIzmYBRfcFzF215S1YTAbfl1zM+1O1R+/wbhsKYefpH2fpCdZF0KXQBoPdwe/vWn+Y1M+ulCOvg73yEpF1SkryUCinvz3YA6jkkDhHzF7VpJdIbLqwlPIwoB4x9w2rvLeNqE2Mxad1tqcUcme/rJLpUyMG6iT04SItb2MRxSoWvJ6Z79u0DRAwTizngCMeIxxLNudDuVyQL4Gaz/kQxv4iPNVgbxjqvZ6BBi29T8YGpFg750fIN9SJGcJ/32SkiLustpH50h+jpUQAAtiX8CGzoXoa/MFuMI7I=", + "results": "217 bc83152c,217 bc83152c,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,165 9420d09b,ERR_ZSTD,217 bc83152c,ERR_ZSTD,217 bc83152c,ERR_ZSTD" + }, + { + "frame": "KLUv/QQBUAAA0am8vb29vb29vXEAADDYqLIGvb29vYcGvb0rbFU6Zw==", + "results": "24 f4e73620,24 f4e73620,24 f4e73620,24 f4e73620,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,18 e5061c8,ERR_ZSTD,24 f4e73620,ERR_ZSTD,24 f4e73620,ERR_ZSTD" + }, + { + "frame": "KLUv/cRbBwAAAAAAAAAwAABo7WjtaL8AAAAJAADzJDjxww==", + "results": "7 6d8b4702,7 6d8b4702,frameParameter_unsupported,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,dstSize_tooSmall,ERR_ZSTD,checksum_wrong,ERR_ZSTD,6 952b8efb,ERR_ZSTD,7 6d8b4702,ERR_ZSTD,7 6d8b4702,ERR_ZSTD" + }, + { + "frame": "KLUv/QQffQYAShgIAh1wpSBJ0gH86/+8CgA9oao6/b+uwv6v0+u/XD2FwRgAGQAWAJXuEHSX8ayCj23TahaacwTcXdwW9q3KBnH8yndEoR16IHdcfcKD6Rhv8faZMgiEHirjHn2ttN///ZcNatFfxE6lCqKPCwI7ixJzn5o8aEVwCGN7z6f/5Gct88xTegMWoHBGAhoJJpwYGdIUGxAggWFo8AEnlPTQ9k8SWh1LgTDDy1ZPxgsrbT1y8GmgSxCACgXBbHa8OuIllB8cyV9fwYpAUcYfGAAGp1F/bQ==", + "results": "650 802d9d4f,650 802d9d4f,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,650 802d9d4f,ERR_ZSTD,650 802d9d4f,ERR_ZSTD" + }, + { + "frame": "KLUv/QRSEhoAACwKAG4owCYAE6AlhXT/P+s/+vd///0/tP0/UjkgACQAHgAV9S6onk+36jbzYfrK/9g8vudpVb8/75gLe+28/386T/1oufsRvOs1RINU6JbxfFB8HDK39+knbVf3tz2fIX1blfxSMq1fdu9/Na3b4Ow2anvPvr/+hftZX1W+8ewn/c7/X3uKT/uZIPv19mZ4/lfVV58Z92/q/pdctv1bePYv/xYviGEQZRFEiBKGCFKIqCgdURAGJAwpJVFJRIGSooKSSoY5SAVkHBOADh1yjZhkM0w+/iyBwro0gGU7V5IspJQkZR7NnyzhbMbhmiOGIufESq7+5ocblkJGDEF82ZbeVMjxSW5Pi6tLQa0zs4uU9xCykvYVRYNa3100Di3pG1S4yFcHmQSOLgURpyEy1BVAbN1ZeC5mK3FAiorkblAUs7YJVeiJiiikAgBCQgQL4BkOMxYAAAAIkZL/v//9fxkoIEocMdUHMNUDywigETvoKSgD6BpdNwt1XDVKoYAkfmlKdcQenVFqHzwmajkWIIgNQBERY5QOlvXrbFYwkg1VBQCvDcAJAAkACAAIAL/7/7r/f5/9Af9///93//cP////z/75+z/3//+//b/9BysAcgjSAeCJYeUBBuQECFxSIER6YACoz4DdgOOUAbwoAwAYENA1AIDGKQDCyYDdIIOAwfGQ4Cqmo79EgPcqSAFglg+4KWYgEIydAQhyAXcDtwMCzH3AKDcBtCdgE44HARjAqmRARZEB5ykHCwDDFxn4amdg4kjgQA9gUAE5Dmf0uHY=", + "results": "2631 ff0a9765,2631 ff0a9765,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,2266 a3492beb,ERR_ZSTD,2631 ff0a9765,ERR_ZSTD,2631 ff0a9765,ERR_ZSTD" + }, + { + "frame": "KLUv/QQCOgUA2bgFAAfZO5jyem3VUbGQGmiqd64U/AJ4O1amY54juJWoqSDc8SJFfO72Hy67rDxIzhAbeaiWeWs6dD2PQ3ii8+KVN6Yci7TPjED4Y1dDGN9fAmU7RS50Qbif7lpyEsyaDdos3hHCn4LEpz0qHmx7FZ6ZlZiDWlqmjqs+8W71cgNpN6zB1S9GXS+y7QUbV/oYgurMcRtlLmflI9Ta8MJqp/JLSsjBADeToQknCihIbltQEnqnikt2EAZzIvADAPD/Gi6j7Mhtp9Tu8WEV3ArnSGa3WI0Wg5OHoC1xTB/o06T3vKu+1Z1Y+rmeHpqtP4MOtqceq75fd9K8Pewuh4jyuIS64p7jb6+0gLKejb9E4b+f+t4G+SMi+2CfZdpi4LxztLQ6LV8sUSwOl4oJ+Y8yeCyVjynrac1UVGdeMAAAAFEAAFcZXjBXwtnZ2dn7Apho", + "results": "486 52e1be8a,486 52e1be8a,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,496 740d3515,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,409 e7c87075,ERR_ZSTD,486 52e1be8a,ERR_ZSTD,486 52e1be8a,ERR_ZSTD" + }, + { + "frame": "KLUv/QRKAgAAR+QGAHJYM44REREREREREN8bnIvc4C4jFAMDiKdfvl5oH7KeIsOjfWMF0bX+yDWkqiMUMunSX4AFQs4xaZedz6WKpRunoaEDuCBv3iWLqCNVkBxunM0BJ/HqqNCLrAQ5T++KQekrzYmUkNM3aAbsMPRJOphq4eZAkTwban8SvwUUwbmrHiLpD/J/mP/HkPj8SYrIzHw/bahbvHYNWfaYtJ3PVxoCJgLgikZaSbSyrPo1QO7ZBmkqhKyqMxvS08YS7cA3VIwg3eHN9dOqKAIGaDqMpB8DAHz5bqVTZ4I6gSsZBQCT6S/9Mka7Igt7A69+XHkhlfvUGaYqGF5j0l1MaBBAZx3DexUjeT8OeYjKTGbQpvDI9Z11WCVbQwr/0hS0Gbvce6LOoGWrjTBQPV9gMcMIDJ9IACe0G0+VFGnHmR4aLc24gER+gtbSqiM8AnH8hViY+Rq6oR9rMaxv3LMvKcwinY+6/bpnPjV6ObZyJjRbfsdPj5GzKY3T3Q9C60T73wMAlIDudLeZKA==", + "results": "790 c7c4b7be,790 c7c4b7be,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,693 2857d7d8,ERR_ZSTD,790 c7c4b7be,ERR_ZSTD,790 c7c4b7be,ERR_ZSTD" + }, + { + "frame": "KLUv/QQYfAcAAlM5N3AnSdIBcQXN24hxBZ3HiPEr6jxGjF8jymPE5VlEeYy4PIsojxE9YxFFAaJnLCJSQHvGYqqj3U0HyXiYHEy8HSaW9cA6BEz7LdJXi6Cizj35mPvPEC2l+KydEWsGSKSUjDtWlf80fC7ojHOM0jyE1DhnUQMf0LO8Ax1PexOrJA2GPEXo216sIHYMAIwN+s8K6uTLHUVgxp+It2eZfk4zvvWnOQZLxtLRs7LZXlpfXBZQoqaBZzeCgSlof8OsZ77zxZBB8WYgS7AsFxnjVYofXF/hWXoRpvCTbZeJypDF2oHciSsVIgFUGQAoNwdsAACRoQIkExjYfwVX2EAL7QEA6wKAAAcABgAHAKa2sidJ8HS3S8ZLueiYP5xnUpEDqxJlabWoA6AUoFYBP6sC+w8TgFXg/gEASkZj8EqEBEcMtA0=", + "results": "533 e253b797,533 e253b797,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,383 eb8559b4,ERR_ZSTD,533 e253b797,ERR_ZSTD,533 e253b797,ERR_ZSTD" + }, + { + "frame": "KLUv/QQKbAEAWgOMABDAJY0BD/zf9O7/4FUdejAcAwADAAMA6ek91yJ/72hPVb4FAVQBAgwEjQQAskUKFaClIOk/6tFqiKqgADCojr+uEKgaH29qd4buq97U5EY6v4zfbfKf9wEtoHAYnWoNECCBUUx7x93uCBDhFCRSMFzU8WSzaMgCgsB5GkgNthJQFSiegFk5gzaDtIFtAIMFCA3sBBQ5AIb/okMP3y2T61I3jesAECNYCt4F4A7g4gAXIISMw9Fgd0BowAQEH7VIK+Y=", + "results": "339 81975be8,339 81975be8,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,68 f6257a6c,ERR_ZSTD,339 81975be8,ERR_ZSTD,339 81975be8,ERR_ZSTD" + }, + { + "frame": "KLUv/QRWAAIAQD06Oj06Okw6Ojo6QDo9PT1DOkA6OkY9Ojo6RjpGOjo9PUZAPTpAPTo6OlI9OklDPT1AOjo6Ojo9QzpAPTo6OlwAACAT0y5tAVQABQYgGAIAQD06PVs9LltMLltMLltMPVtbxRBMW1tMPT1bWx9bTFtbW0xMEFsuLj1MTExMW1tMTC5MPVtbTFtbLltbEExbPT09Q/sBAHUiS2q5", + "results": "207 b8c220e0,207 b8c220e0,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,110 12f15dd1,ERR_ZSTD,207 b8c220e0,ERR_ZSTD,207 b8c220e0,ERR_ZSTD" + }, + { + "frame": "KLUv/QQztAIAhkQNGpClIEkH/3/p5/345U02GzcuxkSujZsEMT0BBAAFAAUA3Y5tLnnNOqMBG5eS9QjdY1EQcQgCqBSggAH/P/8BE1BgoH8U4P8JGPD//QdGbAfAAECsAACxxAKgFDBg/gMTcIEB/gEh4F8GAQLtAQAHwgMDAAIAAgBTYBzZHdU6/wERAAgTQVhxTK+QN1CUCYIDHOGFJQKhfjgjIhQtyGsuOAMnDlD/ulOAEYAisowx9Q==", + "results": "301 78a96b35,301 78a96b35,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,301 78a96b35,301 78a96b35,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,192 95dd61ec,ERR_ZSTD,301 78a96b35,ERR_ZSTD,301 78a96b35,ERR_ZSTD" + }, + { + "frame": "KLUv/YQWkAAAAOQAAEAh59YjHFuLNgMoExDrHxSAVqsCzz8vctHj5QrsAADpVgKoFOANGPD9BxNgYIB/FICAAf+//wFBDMAgQEQAAIHvAVQABAAadQAAkRsCIBNAYMB/RikHgyR6wqCV", + "results": "144 b2b4cdb7,144 b2b4cdb7,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,92 f95a4bbd,ERR_ZSTD,144 b2b4cdb7,ERR_ZSTD,144 b2b4cdb7,ERR_ZSTD" + }, + { + "frame": "KLUv/QQF/AkA7i2AHAAI8DmvA3L6qwcZABkAGQD7///f////f///f53v/+f73v/////7+///f/f//3v///v//7//9//////3//////+rD//v///3///////f/////+/v/+37/////wH/v////+/v///77//3///////9/////t9DqJEsZZghSJGZCUuGtOkgiohMnXmCOChK4RQFoiAlcMYJDDQTU8FM2EIH+pnU8OVXB7qzxxEVENwZvL2YSNz68TijhJYgFKVro7TkGQCIeP1QICHuYpdBiy76B7+L0fJTH/eAwCw8XnKYovxlYnBCriLZUf7iAclc/Gv7Za4q/ofTXiM4ufD5OMPLOF7EiyARHcREnRhgY55ji6Ljqwogr1kz2gIMG+2qHRFUtRiayfIlo4BgHCwgzqkgbXr1dGfwT7HAbfDPIqgAAMXFxUjFxcXFxcXFxejGTU6wSvMnRQAAAKEDAFOa1hBnOYo4Uy4F/LhG04toOIa2WK08dYYlrUL/v4xQ70FVPyguxiEMO+ajV8s1sb6NBIeIRzE93E6/p/zLjzEaHUx9yhYLGVvKrFlHcoKLlhgiTeNwyFsJcgMZp6pedUwFAbJPLfHieLNtcvslMY7DmJAXVZRW4A==", + "results": "1801 7eeff673,1801 7eeff673,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,1685 147f5c25,ERR_ZSTD,1801 7eeff673,ERR_ZSTD,1801 7eeff673,ERR_ZSTD" + }, + { + "frame": "KLUv/QQB/AEAJsQIDsAlHZtfqqqLx///dmQSAwAEAAQAM/9f37LlA2tqrqw9vw8HqCA3/z8HEDKm2QEgzQ4SXCDoYu6ZC6v4AAAAAAAAbAIAwkUMF6AlhXTzqY+qqi7W47v8/6+Gej1VVQMF2bVWjwgnLvOUdIZFhXBmuUdLGFQY8s7YPQOgFOBaBfyuCtw/E0BgYP4BY4k2VYEHIQEcAABhngBEAACJ9AFUAAYAfSgAAOzsDQ0NQAAAOzQ0hzTY6mchAAA0NIc0aJz17Q==", + "results": "249 cd614a9d,249 cd614a9d,corruption_detected,ERR_ZSTD,249 cd614a9d,249 cd614a9d,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,93 ad20e926,ERR_ZSTD,249 cd614a9d,ERR_ZSTD,249 cd614a9d,ERR_ZSTD" + }, + { + "frame": "KLUv/QRaLAMAtsUOHIClIEmGA9NCyAzMPUY6JsZ1cDGGJsZlPWiGXhcFAAUACABrZU3tPfvwy9iLZZKA3YOSwQGXbWiN1gILoEACJEgBRUSBGcYBEDoBuwdugHu2x2ncbuzyKlwL3RbnkVkAuBIAAAA4AABrKChfa2treAAAHTlra2tru+FJc1uDNlcVCAAACQAAAAAAAAAAAB0AAClWAFkISb0=", + "results": "182 22aa3495,182 22aa3495,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,161 843ded7d,ERR_ZSTD,182 22aa3495,ERR_ZSTD,182 22aa3495,ERR_ZSTD" + }, + { + "frame": "KLUv/YQn9QUAADwJAOYKExuAJUnSGNNBZwaqDRXLkAldM8KETIU7EWQGAAgKAAsACwD3K99F7G1W7mgI4f9OEw5by36M0AIV2DnvW2mJLQN/BdF7Q+6bl/xZFQJXqKCSjZLWMIaQqbnzchge03Egx0GQESjkBEYhNBVRngpLjQELUJnIL5zr3PNvXYSevLA2GwI5JYYvDpXi8XjwanyIijPiCkXoWiqoiWVyrPTtVwaHoCoybXK2ZUALxyubD7VJW2n61ygsaV2MYalCpR4N+oS1faqa4PCENP2EfdRvjXBGRwd5HIElDep6dJ4PAJqDSLJk8+g4oUxKeDTQ0Yh9y4QO2szk2wEngeXxWxQI3WOZq1WbcaHCCqVtxo124SCgT9wsd8+LDNmtsBhIlEReYj5n/gK6AgD+aAAAALd5+oP0vr6yvvu++wAAAAAAAAAAAAEAAAG6YPg=", + "results": "1525 4d7f4a3d,1525 4d7f4a3d,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,1525 4d7f4a3d,ERR_ZSTD,1525 4d7f4a3d,ERR_ZSTD" + }, + { + "frame": "KLUv/YQ/IwAAALwAAKnqAqgUYAMGfP8BE1DgPxTwP6kAGAIEHAAAIXcAGQAA5HSghFDoaA==", + "results": "35 bcb26766,35 bcb26766,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,28 b904d2e2,ERR_ZSTD,dstSize_tooSmall,ERR_ZSTD,28 b904d2e2,ERR_ZSTD,35 bcb26766,ERR_ZSTD,35 bcb26766,ERR_ZSTD" + }, + { + "frame": "KLUv/cRSBwIAAAAAAAC1BwCSBgoSsCWFDgxzSJksyI9hpcMoi6g8e+r24vdjf/fp91Xdxp2vAcvX25EBgIKIAJczBrA9gGPh5t2tpHQKUYuW7/Zo8cGqrbvLPFteC2Cc44fYdWBMu8Kib6Z9GGm77VKTuKzaaFtrJG4/rXgLFMsYUhMwD2nDGj/flgZN3AoaYnFe+hqzy+XGAwhQOuY6SAQ1iwJYmSZKWE/zqXYIp8FrfYacIrYXG2LiIp+3hkE9PN9npgpxtqfYC06uFtSBh9kRoNCwiQGuLbVWhhVE3epVKMHOAQa1LJDUJPClvgK7QtNVYAJApVMZlY5L3dY5zCgDdnNXdy0C+5cb", + "results": "519 1ebba85c,519 1ebba85c,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,519 1ebba85c,ERR_ZSTD,519 1ebba85c,ERR_ZSTD" + }, + { + "frame": "KLUv/YQL5AEAAJQDABICBxSwFaTQAQwLGrJrQxYGQpzkloqy65f8e3zImy4yqBDZAyCEITo+0JhJB/vYvbunKBm99dB6+/inZjV7YyY+cvSYGvvdMj801adQLva97Rb2UXxE/iJBg5FKpivIVOdA9DqgCfqaglbTMwZ2Sjr/AZsHAOldOqtp", + "results": "484 d8645593,484 d8645593,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,484 d8645593,484 d8645593,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,484 d8645593,ERR_ZSTD,484 d8645593,ERR_ZSTD" + }, + { + "frame": "KLUv/QQYKAAAVVVVVVUYAAChKTQcAAAp7wAAAAACAACBAAAAAAAAAAAAAQAA9DAtkw==", + "results": "13 b80be561,13 b80be561,40 9534eab5,ERR_ZSTD,34 7add1e50,ERR_ZSTD,8 bcd434b7,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,13 b80be561,13 b80be561,13 b80be561,ERR_ZSTD,13 b80be561,ERR_ZSTD,13 b80be561,ERR_ZSTD" + }, + { + "frame": "KLUv/YRIZgEAAN0CAIJKDhaQJUk6x7p7NTMmd2e6OyN3Yv9fnxUjmQffnhtZVW5ZX2sP73TOOtxg5//vxzbr69Fg//n7px2bOgQoExQgKGDwDxSgQIEC/t/AwP/9AxOQd7peGgoEFK4de/RA", + "results": "358 64a81c6f,358 64a81c6f,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,358 64a81c6f,ERR_ZSTD,358 64a81c6f,ERR_ZSTD" + }, + { + "frame": "KLUv/QRGEAAAtLTkAQD6A8gAjhEREREREREQCQAJAAkAQ1x5UELS6Y4BIpRMnpQfmwMBw9xV20MN6c0Ba20+IpXjdxMBVAYDAgohCwAEig0crwpQHCBqdzXunahoujTqUo6hXiLo+9ZD6f8EQuYfu8lVgC0vqYwjHqyqcnaJpQ/exUI3jOrYxYMcx95V+eFWEw75glhb0hpIhhCPjfj4MxNe2SzmspMZNLLdpsBHT3XsrGR24OBT95k8mo7Fh48if2K2Vs0AACUkUTuxaS6UeAavNwQytX2E33LChNTdsYCYdDqs96S105M0aaW1Jy1eGttqFkc5XyV0j0Vf7cFkb0o50F879m4rjbL4oRq5ytl0EUXPpzVrhJqXOmevsNzytB+t9ej9ZMFfC7IC+1pGGs1xAHR8ofbPQnUfYvuphiPF6y+uvs++SC77zK6Z2fpjdfZFk/RI9a9AWf9wm8K6gx1BynuMzFjdbIn0GZdECdCMwrhfDGpoSDEh3Q1CnR8y1wIROBAJGUrRt5EtgJ9nNqsY3EOSSenrImBbQKNsS/gy6x3a/jxG1pvrGkKnn6s7l1xm9FA=", + "results": "426 7eb103b5,426 7eb103b5,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,320 684422fb,ERR_ZSTD,426 7eb103b5,ERR_ZSTD,426 7eb103b5,ERR_ZSTD" + }, + { + "frame": "KLUv/QRXdQMAygosARmQpSBJB9s+MzMzHyrbdnd397MS2e7u7u4gDAALAAoAQyoMEFHY1uS9h83471qlVhspfNxpAyDfa4rfzKZJQf0Gb6a84ynKMVn9MQSIFCjgDRTwFxjoHxSggIGB///vHxwFqXbYmTGAWSDsaCSE", + "results": "231 e71c148b,231 e71c148b,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,231 e71c148b,ERR_ZSTD,231 e71c148b,ERR_ZSTD" + }, + { + "frame": "KLUv/YQTIwMAAMoQADEkAgACRw4bgCVJMhwrBvMSUwbYHoAZYGJglgUyA+DUxFQFV9ZwgDcxpU8O8OXWkI6PXW+VDJ76qPFlhM0FgQsBVBkHKjDoBEwBAJBYDZN/R9YmhlVnKLq+/pEbZRgFqHBO+A4QiLEfkFKGB9zhI31yo5ATIAAAz6bPMRAAAGYpAAAAKQAA4uIxMTFVtilZ", + "results": "803 f2f9258,803 f2f9258,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,748 447f9bc6,ERR_ZSTD,803 f2f9258,ERR_ZSTD,803 f2f9258,ERR_ZSTD" + }, + { + "frame": "KLUv/QRF1A0ANjsnCuDpAGgUDDoAgA4jACQAIgD7v/23+9bf9///t/N7//l99/3/f/97ff/939//9/f43e/2f/tz/Xvv++///3897Z533vn////P771/+/7/9+/9/vOd7+//A39+/////+/3/P9//////f19/N/z+/+7/7/2///313X8v32///97/ff/v//v//+/59+f/v/r7///Xc////H5/3//f/8Ga6jBmELSEESGKhgZGWnTASCMiBCR9bK4MIpyHEkqRhQjZVOSgkqxZuK3+T3QF23bHzqAS5lUAfEyLTu8SXdE6XRUmSgjQNy7WEwRxKpyJAVmlDo9oVv82RRDRdJXbvPrZK6iCuRNGaIHy4nWpVSkVgGMzN00krlsYzJGsbJqJxpUB8oL7UZnbJ3fVhds5eokcmSRlZW1b0TrQbPh0cGiKOBgc1r8OiZ85tMgc6g86k/i4VRGjbPTefylAHu93nz/pBxfqTLcTsITFQxBlFNWK2JVIS+xoKES6xq2zJMnUi8PhDUgcweUb7I+7kcxYk+RYEdY0vEnAVN1vjwY5ohu2kgUUwG2roUcq88Hu7T+JVsiCn8lJmglgF2hDGwCAFZEDY4REREREREREAoACgAKAJoyPc++q+tnDAG4HVDHlxsw3AMB2+sBZkphISVaAVyDb9pkSk0RAugTQGDg+wcUgIABP/8BBEgMgIEPcgAAAKQAADlpBaQQM/MQeLF3BwAHspAiXxYRAQAAtDSvFg==", + "results": "2172 a71f8964,2172 a71f8964,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,2172 a71f8964,2172 a71f8964,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,2172 a71f8964,ERR_ZSTD,2172 a71f8964,ERR_ZSTD" + }, + { + "frame": "KLUv/YQi5QAAAMwAAIEWB6DQVrA8BxBa9gE2sAFmcC6TugEvYAOkAgB6A5gAE7AlhQ4MCx1cMJwl7ujkYEw0FQ8DAAMAAwCLvzP7vQN/tQ3fZwMPoJCq6tJyMTBMMLN6RCB9EA5MSIhLfUAICADwBXAF2ATQgcxBcmAFqDKdAQCaA6AAFKAlSQf///////////8ooA4YH/xvAwAEAAMA5cd3o9qPBp83H33ZBQFUGAcKiQxyRki4", + "results": "229 375600f5,229 375600f5,229 375600f5,229 375600f5,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,159 aa630456,ERR_ZSTD,229 375600f5,ERR_ZSTD,229 375600f5,ERR_ZSTD" + }, + { + "frame": "KLUv/QQYaQAAtUj3gtAnqp3oJ6qd6CxvASw=", + "results": "13 ffe63e6d,13 ffe63e6d,prefix_unknown,ERR_ZSTD,17 9152ae84,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,10 14d028cb,ERR_ZSTD,13 ffe63e6d,ERR_ZSTD,13 ffe63e6d,ERR_ZSTD" + }, + { + "frame": "KLUv/URNJAAsAgC+BEAGAAbgDxACRgIDAAMAAwD//w/7/w///w///wcMgJACooAuMA+bAyAD0RsAsMtelpfR7mMCMC6FYZEkXGauv4xHya14AAAF0tLStdLS0sfS0tLS0oMAAAAAAAAAAAAAAAABAABFL0U3", + "results": "292 7c123ae8,292 7c123ae8,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,292 7c123ae8,292 7c123ae8,checksum_wrong,ERR_ZSTD,dstSize_tooSmall,ERR_ZSTD,281 c9a4702f,ERR_ZSTD,292 7c123ae8,ERR_ZSTD,292 7c123ae8,ERR_ZSTD" + }, + { + "frame": "KLUv/cQZWQEAAAAAAAAAAAC4BwCBeaSVQQm/cZpFUHjaFhupTDydM+wdNoxwuGQC4Xd4+lgG+QoGKJ/CQ893D6qITmLdA3DgDHMzIa+7mrBnjZ//98btkdbJURweSpYoyazCbj2U9yvojJAriey3Y/3j9U5ZUO/l5fPq8I7V1wc7N3CE1ee+JSnbmWhnlAN6A7YyFVk5IONk8wo0nh0pSuFwGcSkL8Ur/2y3z52gFlM9Ab1eXrIKJPv70IEZ59b+9AawltNprxaWOXqXJKkvSQowzh0tylTQEhHqzzIAmwAmt1wEADeznkGasgpvSEKv3dw9qvWc/x0S/x0S/x0S/x0S/x0SZPMKnh0phQIAlAKCYzLljRP2pLfre8bo70MuI+wXsvQNJiqnbGGAnGc0ZQdDWpHYyTPHrhOEsJJZbQCAZ8BBKK60ojfs+IFa87zhOYpibVh82DQfXW2XjQm0I420", + "results": "345 ee15bcf0,345 ee15bcf0,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,245 dec9c463,ERR_ZSTD,345 ee15bcf0,ERR_ZSTD,345 ee15bcf0,ERR_ZSTD" + }, + { + "frame": "KLUv/QQwJBUAbhzAIwAUcCVJkg6OB3vmL7xdQweY0P//ewEcABwAIAD7eY0m24A6g3k2eBgpf5ZUqC4+8L4dO7q2wF8B3aa2aKOipvtjLwP2kUVmDNl4GHjLH6jkbi8/EfFEmF26kBC5dxtDPNdjENgVcrpyKbtAVrT6P4A4RCIB2y6SnA89/P4OSvLHG1agJuTPNJTYz4iPIg9lBYDniHDQ0SQFSWuBMOqgKhpJkqQwzAHu1EdOEBRawRnBkFcXMkzoDhH9jQnI+JJCvNsFf2QfR5yLhAJJlsvMr8MCdKUQEY2WaYp68Ysga/mPlG1R2Y4ukTHodiAIHcnfSrQ2dbKUvLv9P2hvUpjw96WXJBD6DySCc8ng0qyglynZsbc89mZhTRbsa27Ka7wIBvhPOWkViW5pQCAGEshcr+i4l8NfuQSV3SDzjdpd5lEe5SDxBCrnWzgzSQ0hBpAvMhPHNA0qzVOW0MsXHcyBTEhYGhfg5gdMOnyTqImxOZFtYm0ACwglX6yXhMwWlNMwyROPENCIWeSJIbqXrRMNkIDywtMowhdiD8uusrwCn6j/ggT1U4VKAGkFC4jfBMyXYEDYbd5vD9WaLAHyuSQEC1VZj5dMveEq8wQQXQHhAQrhqPxpm7VvwltIu0ccaFWfD9MszFvfIbHZthrgifU0246NIQADUgMGvh356NsanJkSREi5DiAy25bda9btDnDFH8L6v7oDnQd36jxF/hFQ6AXmVQhReHOLJbw6a+KFT7+JQnlOudC61PLYnA6b3a1IW5c+aKK5FwujwfecfGU8nPUim0XARyrBTXtwM6JtDcAuHOx7uecHMGHx2xfEDExgj6tAB08WeczAJwGvIm27sFGySrcXsqNq8KSpBE8DqeE+RZy5QH8qKKJznIlZMNpuBOwGAI4RACsAI4ApSQcrm3QugEPbmLFuwMbGDnqW8ehZ50rnSm8c2oNHTxQBJAAfAB8AG4dJBw3hEsy4mEVxitBhLbyakZEVnKdIS1LW/X6TUGEwICSgdCDTraclhSvdNqQQbr7aCO11ms6pC8grcSubeDbr6jVqcrGvBXJoKrfqfR/kvMuk4CZKyD+I41TfQXAn9Myb0vbiWFv8XtOxJ031BV9AxecgfO2QSkz52uhJ04UaAg4oEJ6h8AGg9AELj7dzX9SzG2KFakExOe0HIoHsFF81xj0zhMIDf5cTMrEC1AIAxsUPjhEREREREREQDAAMAAwABdv0f0OSkEYtUCQUuc8Qlz8CIWzjmTARk0lXFocUfvWKorMTaBQ0q74m26o50Q8ZBSgQnpSCD/DucgfSsJcc+LYXuwGlRKMMjAAAuQICkBSAgAH9BwrNCbHsAATVAABwB6ns0fhRwVNKYRoTxqMDAH5moFTLwJ3cIgF65Zk=", + "results": "2582 9bb83f27,2582 9bb83f27,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,2048 f011a695,ERR_ZSTD,2582 9bb83f27,ERR_ZSTD,2582 9bb83f27,ERR_ZSTD" + }, + { + "frame": "KLUv/QQQogcAYCgAAJVepoXBAAAAWAAAiJpgkafpWXY8LsMJAADWy7/UDQ==", + "results": "261 575a038c,261 575a038c,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,265 9dbf5947,ERR_ZSTD,257 235170b8,ERR_ZSTD,261 575a038c,ERR_ZSTD,261 575a038c,ERR_ZSTD" + }, + { + "frame": "KLUv/UQERQlUFQCeQUCGACtgR5IkHYRFMwRBQBJWKGgQnLv5QaOjCNzNDxodxZLuBscMQbAkYqEZASBOfAB3AHwAK4NZCamVJkAejWHWNDIDZZ2/rQjGkOv1bX6VoaIlpQA4Q4u0sFd0IAwIwsjMICxaZyOi89/9F3x6Tb9NjU3LowjDCgyib2ogagmKq21sQFSk2SVIZGwnYQKszrDJM/4B/mxyxYaiuJBNZ918XqPyVIngnFCOhrT9Hvy0BWQQsb3LbPyNz6hqW+S9CzKocqDYwwoWcCy1Y1fv3Q8QG7WGjxotZZyC9WM/qsHm4IiDcSWLG7Fdc+U6Qkhl9I4a11sk2ob3EWkbdNOVYvOw9bHGyKDFwt7KP1uRyklRUqijuyid1NGO6Oy2VjdwKRCvfS4AMqVbq8u6WapLrBZj4FJDRAeGQWz2XC1H9sHaP3UNe/P3bJMnitH8BLQoAH/YEMu9ovrgZw5Azww6Gk82o3tDgKMRW2+iuszZzZQwswOiomIdpRMsdRkEQIbdFW3F49vpOuK6hEADNjFjGbuA5lTl2YqkEXCbJoU9YUzBRrJyGV09dovzTDFYvxhVOoT0lazzY1z3tMYVssz2ONcqSNlopRpTQYhAd1UBAsLqRK0Fq3U8+PQGJqH6KEzbdixvi0uabB1J6mobV5OCCyhGyxbZQRb6BkLgXgX/HTyPGryYgGiaNuPRpx3as27p8DY+/jBCgMvhS6eDAi+AURAFCYggCRFDCCEiVUge/omA0SMAfwEgo4Dq2LDLWDGwRQCrqwDoBQgr8GkiGuCCYvHAdEiiZSZIdwLiEDLRxg6wAYNuANQHYMERALgUqF0gLwWivQVyuuA5Puh3C3LSAJsZkM0NoI0GZRMCPpjARx8sqoKJRYmtnufAFqgWbwLHArcSsLsGwF0BjQoArkOALAAL4BkOADMQCANReQEpACcAKAD/7P9f/3//f9/f+3//e/v2/sv//pb//nd38V9///P9//+9+/+Pq4Lvb+////6/+/fXt3/3///++/1//195/v7nv///8O+//8//v/v////fDv3//vb+/+fu+02c/e///+/31v/X+/3/5+/f+f///z3f9/v///7v7qe/f/9rv2/+cvftv7/f//6+/373/7/+9d/f333/v31929yt////9/8/M6gBiRIUQpAAEQERECYQY6c7ECBhxkh8Ea1kkElVpjGFALrGuSdfdw0m9H8pQFBkSdoqWBhIfLYcUHudbh86KZHaAz7HrVP5LS1h9z5Fijio6tijGTxuPOZn2F7AKrX1Mb8hxyKKkXiVI9KJ+gnfwApxGMZ+NIS/kQYyhRBp0vpf2DRUKZHM8QUZNYuVoOSUmZWidTzYpmEfAh10zo0=", + "results": "2629 cd0b69ef,2629 cd0b69ef,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,1216 8d231c93,ERR_ZSTD,2629 cd0b69ef,ERR_ZSTD,2629 cd0b69ef,ERR_ZSTD" + }, + { + "frame": "KLUv/QROAAAACAAAzwgAADEIAABSAAAAAgAAdgAAAAIAAJkAAAAAAAACAAAZAQAAcDMKdQ==", + "results": "3 903d964a,3 903d964a,43 fa10c648,ERR_ZSTD,37 bbe9d677,ERR_ZSTD,31 5918ea3c,ERR_ZSTD,24 c752e337,ERR_ZSTD,corruption_detected,ERR_ZSTD,3 903d964a,3 903d964a,3 903d964a,ERR_ZSTD,3 903d964a,ERR_ZSTD,3 903d964a,ERR_ZSTD" + }, + { + "frame": "KLUv/QQYEBIAnyL1eOu2yY70mE7Mm0BDOWSu4y2HbA+cwnrd5M4TXq8VvLO60v6ppuWLck/mUNx9apaMgIvHjqaeq4/2nfXUFkR5j1vWkoVNjXCQVN8BW4WAZ6tZUBL1D80oblLjAePZqYK+eMT25lLB4bv+MAZ17baEefPL9QeFMufwvw+eK3dvBV7AKO9KprBLoWmSdI4tGyyJtK4NdVhr0m5EmV93yezg5UP2f30pAKwNWeoShUs88Mr0KLW8egHWAFGSgYFEPD+WekBWe07vsFk7OS6mGeCqBE+5sK2f77rvvWs+gW6p1Ff0YlVICRFonK6fHF7YzgK+AwQ3mJXpaREwAL0HRA3/N31tdosbGioDv7KeeHQBZdSvWhycQs0/HyzWjGLBUM/x792EXdkTwtnPxSDxOyruxGgGtJnNzoBxIJL7PncnZbAyYH2DBB93tm3yVGadANp8rQz8BRCdWcd4CHY9kSXpyd7twpK7bf4UoUjrF09csKq1oy2zHY/qTcUJ767Xk5HtPm6uUhSqmb8tsSw8FJr5qYyaeI5GCj8nrQM8wwJl1+zIQWDnq+4chGEVg1BV+AFcaKl0hy/ADGZS7mGjyAWAHp25OuGFkKfCpFlLnl64MjYTQdnkmyC6sCfAO7UDf3fCbRVUstEpERcw171vAKtX36cVq1NPnVDiBa8X/MncoIw5Cs6hVsGhaBxIqiX0MDWGRB//XXb0BeB1caWoAkiXTmUaw5d5QhbLkYkpDYWOp3yy+20lLzZM3DZ5XY8/4x8CCgCLFAsAOiBQAQjwOQ9U3P8kBhEAEgARAP////8/5///z//////P//8//M///8n/+f////////z/+f8B///8//////////n/z////x////8///////+f//+///n/H2agsTymGEJFQpJoWAMgSCikHB+XQAYNLgC2+gF9n+Y16zRiKBYGA/aJBAigQwQ4BgyXAbAtIPrDBioAmksQDeCvABTJAMAM+gtWNQxH4hCBqieQvwIclUC4AcLEgqqIoPkFzq/QBw/PErjHQa4KDAB0yQVtQIEnHgABFrZAtBSAWMBihXD3oNAGvAhgqwC8A6LSdXEAKCCgfAfqiAKiRGBvCsxIQPKI2BdkFsDcGPANA141HE6iPREsegAEMB+BLUaAKgF3fkA9AbrhgHsBbxIAgY8NqIDUMEDXga98YLcB3AOAvQOQLagQCE0piAF9AAy6MFMYKDqBPQCaFkAQA7tAgQWkohDgBDgYEAEA4xkPUuLBqVyElZWVlVLtaf30IAviyEEifnt5H/+VgW6VYAAAAFgAAHxhh2k49E/9hvStAAAA0gAAmRkAAPn5+aZzh8g=", + "results": "1822 237e37d1,1822 237e37d1,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,898 f33c6fdf,ERR_ZSTD,1822 237e37d1,ERR_ZSTD,1822 237e37d1,ERR_ZSTD" + }, + { + "frame": "KLUv/UQ7aQJUBAA+G4AZABHAJY0Bv2vQBaj6+nevWqjODBUAEgAUAD3P+0Xy9sf09/bO1f9bWq5a9PvuCO//+3//lTTr818ff7r8////AXdq73dfT2m+e7kvdv0nWkf//98Vt/fs/96f2Vjd98++tN+H995vLgSo1GpV4P+ffxMUIChQMOAPFIBWq/4BcBNlPRFWAWziBQBcVAMADgmAFwAekCdJB3nXZyIizpwgxDcxa3CCiIgzJwjxbXbiBH0FDQAPAA4AbvRTvChVic43yBxfKiKkQCnUGeJZR7eI9twNlI7rMGfx3tcOOWsVZg8VInmShQbY9UhlzBjjSP4KAVQWBShVEDgAAMbGxsbGxsYAAAAQAACzswAAAAAAAAAAAAAAAAAAAAEAAAju6x0=", + "results": "873 db4c47cf,873 db4c47cf,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,643 3ecaf6f0,ERR_ZSTD,873 db4c47cf,ERR_ZSTD,873 db4c47cf,ERR_ZSTD" + }, + { + "frame": "KLUv/QQhbAcAnhDADAAH8Dn6r8nKAwoACQAJAP2v/7/v//3/7wf39////////37/+/+///f//3/////3+////wdBiEGxdEZkRMSkhQOSeDzFMRRNcpSySglmoglqJtVqNUbFKVEiTWR9GrKMuGqMAFdQuPdcRzVFsuqidvV2SAvqiWVP8OyiPWPe62vELWfISOxggXgkQHKHJcNzaGhhS0yD98CgcQ/7u6lPNafhnUwC2Gh5S3lka4Sl7hRyuMjP0Cz7HorJjjIH5bppCuQx+ddPAEasQ5ppMWsJmZydQbflqrAy0Bc2uYdXC2biKDiWKplVTO4O7BIAYhovE8DWaNoGKdLa8GOMaRwaB0EQoAIb+7NC+d6SbSiBHAcbHoThTHP7zaruSYU5estj+JuV34jNEcZULC+eIJjNKbn44vXE1q6Pn6FNBR9nSGNtZidoSaHnwLm3EytW6yo9MUuxdVbozh06o42I19uewte1GSYENbegjbCazSO2CGNu1TGCsP7JeqjYWPQLJ2FXk5DaxH5ui6v5KNIVMrwtq/xm5Cvh7kcYTrj6mh496WdfQdQbwkYKvYERcSGAnYAhPauQJkmGzUFYoCE+lhZxIPINDJ8T1HJrkoKuN3lGDCiEJBcIm6kSFPjOVeDZzgYHYECLPHQEmLMC3ENeZHc3uyhAHA0M7JiJ1Vs8PDS3N4S9l/CdmbikDJgy02ATKNcrHEGFFyyN5PptAgleYm87OwnAVPnePttWlAvIlE/MgObY8qBPARKcg9gXhzPC8tYvSPy0LiBM++oJ4qD0HunnUIc50y1AiTIzqNw+AgjOqGy5CVZSzdoQiyEecQVZECsk1grRib2Lhplzf51AxYqTGS9JHg6TDfa83CosuaXE0vnFZcZu7pm3CjAi04gbU7UiA+fp0VgZqDI5TxTZ6u5pWHl/LAwZ5AYR4l3Ky/OtSffFA0IzYbgZAbIY+ZUAzEsGqSizksY4oiDig1BhBAFcjxfoHWgmJbVPeHez6R45QTQqKPeE1iALMH5BlSeiUmNgKhGYJ0PfCnmQy60lj+bEDV6rBuSGLJp6BSEHy9IAd0EA2Rey+Swxhi4B+Ur4KViw8TGD7nZ8gTRyBlSRByED9TJPICdSxIYyW0aMBAD6CCABHXAlSQrp/L9e/f98vfqywv/DcdXP9f/P/z/cvwkBCgAHAAoAC6EQy/TombUeDf9Vk16qu285CLozReshvZoFzVz7GUekpD4iE4hgTkAzQWnL5wEwzIwFU4HCgnN5gFiSgFQPwHzQATcQ0zq/o+UhFzuZwGSfMmddhIgCr0n7LMymQLBZRxggs0Gp2mUTYAAAnSmdnZ2dnZ2dKScNmAAA852dnaQqKCW2tp0qKykrnZ0kbUQAADFAAXQBAR7nAAAACQAAVF43o3k=", + "results": "3184 8c27883c,3184 8c27883c,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,1097 d2b56ac0,ERR_ZSTD,3184 8c27883c,ERR_ZSTD,3184 8c27883c,ERR_ZSTD" + }, + { + "frame": "KLUv/QQmfAIAdkIHDtClA2DEcSKqVgAAgNYCAgACAAIAv/D7R/8H7wcTACCApgCkCgAPYAQGrleAAgUwpoDXCRhQELlwLgNTgcczfBdYFwC7AIMDboAbAioFAJ7NAADjAgJz/s/9zd+fFwIIFMCAAfcfbwQwAIYI6Wi56g==", + "results": "333 d4e83dcf,333 d4e83dcf,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,269 fc78f411,ERR_ZSTD,333 d4e83dcf,ERR_ZSTD,333 d4e83dcf,ERR_ZSTD" + }, + { + "frame": "KLUv/QQsxAUAVs4VHHAlSZIO/B/2/7+reroQqNP/h/pVdVFV4fpfOkMOAAwADgDZUt+i08iR/uXLkhp0GdXF7nenfnemjtaiN0HiLjnpCH7s7PvLN8kDLUeE/F9Rc/9X/XEDLagxoUNKUG4CSjMuBzCKCCrt4FbtLmEFTMzSdM14vmbINz1WoCscCzKTZrb7UmR0CriuAEz2Ym1bgXDv6KmJjPUAi+LIRw6rZlxIJe7QygsQkAVQF8n5aEU4GcuoPAIAAMAEBwC+EcAnAI4RERERERERECQAJAAkAB+jRbFVV3G7VSyPTCSrjOrzv9L3/ApRJNVr7tDzBqMBEgBYHYXvIKop4GsNOi+/6bJusfI9+J6iHF9iMIjM25sU2UXvGz4xE9D9XeOXZRniWOI2jBfScyvxzOUwOPB7IO6Ak/492Vzmay4IEuPhU8cE2fxlkvxEdDy2gBLAnyFudI7mO8QVdNJtgzLoln4mARKoUMIxIdXEzITaFgMQHGOg6QOAUGs3BBJoT9kls+lC10sFz/fxWeX4f1zEsrjkk/1PrEIELjnRkhQ+B+wCAHaKDQ/AJR1v+qLnTcR98f//dyYJAAgABwDhu6vO34J+uwbJ797bn93zBj2/1/vO/H/bHppf+79/BA2EYFZoYTKLC8sNAHXkinV85UIzshjfbF23IiosVpVIIQgDLm0HAD4MAA0ADsAlHf/fgq66VHVedaIvCAAHAAgAf1//dXr3j/3v/9f///9z5a9v//f//ge//79/v927AVEoMEgkMnN80JhpMVsG8EPBUHQ4YA0c5y6RMbBqMY6OIh0bP8BAxYCSSHqBkyjy7rAQlr4xx9kb8h+3DuxxwJwoCsUBDIlGLHzQYcYAfANAkDAii2bZrbs2FsyibNbzGPtA7TG3h490jA59G1FM4p3+CZNQjCNpbVcSE5HOCFuzqcJI3+KsAsPnLGIJSEw2cgYcgAzbuZguMCRMLoNPu3oOZwEH0XzDadBgyA6Sz1S6zpVnBV5P+ao=", + "results": "1523 b75c9135,1523 b75c9135,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,993 8f26b8a,ERR_ZSTD,1523 b75c9135,ERR_ZSTD,1523 b75c9135,ERR_ZSTD" + }, + { + "frame": "KLUv/YQarAAAAFIDANIcAAC5YACEAAABNANgAHMV0CrgDwNgu+AlHAAAKb8AAAAAhQAAGRkFhBDzAQB3UASa6h6eCA+9/3s=", + "results": "172 9680b0ad,172 9680b0ad,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,149 684a5809,ERR_ZSTD,corruption_detected,ERR_ZSTD,172 9680b0ad,172 9680b0ad,154 1e27346a,ERR_ZSTD,172 9680b0ad,ERR_ZSTD,172 9680b0ad,ERR_ZSTD" + }, + { + "frame": "KLUv/QQ0xAgAUuI/H5AnSQceK3aOnWPnKFf77erZd9yRF+aFOY6shXhMhCKzbjQNi42lXDzFyPJHRvoXaghGxvkTXtE8e5rbqz2bOf3fGW51VjjOlmhonZRjl8nGzL5sVRjnKiQ+FY2ZR9Z3nhPc/6Hqpal8LOlFkWikq2Dkx9hkCpleNPrRehDnm265Sj0lIzpEsyJBdhD/vEcey7b/bgzD0JDo5EO9TOKJTEMkoWxPhERS1Z4ZZYxEcVsZrZVkC/G94uR5INZDl5KYtulxxEIHO3S5EKcM5y3p9WcLEhvpATxEIrtzTMpmj6VW+IxI6+eOjqe/uPecQD5HstHmiJgQIA9vk363lWgdBwByIGLyP7MnC4KNztrPET0J+Io5BkwDADoHwAATsBUkHadJkkRlHSZJ4sB+YxiyCwUABgAGAP9/93sD/6Gvj9oH3xP5T1vYe/7TuzcPqDDSBBfMZVaLATBMzEx7YAYas3XrnwHniISI4sURXNd8IwYJFrmwTROSYwslwUhF8N8DYLQIAP4EgA4AjhEREREREREQCwALAAsAD3wuRVlUr99tugGSjxtzqYW2vZ88AbyRL2pP87mvgrwBndaZsWsy+lPdFWcoIIiIRAr20JotB2QElsZOC5n5oWgox0qDOTJJpSMqGSuDwJrKAm9etxhbUv0qd8z3Bjb0DI4xj4UQg8EweCyrsCcyUBC8K5NDulWMoWqZtm1OmHolhsVPOa6B3AMBBf9e8CG8qQnJQMx1sTkzUe2Ayh2SybNbWmGZL9O/eM40EwSAmZ5axHZDNwANObc0M8clw16HhaBu4E6BPQBDocbxO00zGgTeQTArGDIk7BD2FSADIP312wMg8zgYpiSgxIIcgj+/WGEydB9Odu2A8BQbiwZMUhwhQgQAZa0EAMsN2AEcABwAHACjf8OnxwZcIybcvCWuDQ+0Axpq+5XUs2SfIPYWNyEfZhlGoC8H8cBMPERvXkiKeILTVG0EvMe1GqcqccFI2mVHEpg/8OFFHqYOCd5e940OlO7ysxzFyB6YJqIJeF1YjHs2+1N6qaAF15T3i87hE+ccBSgQOHEnuAeQYlJMpgeIL4mNL2Myhtqo2UFtiYmh2Q==", + "results": "1713 3a4d9898,1713 3a4d9898,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,830 8b8126a7,ERR_ZSTD,1713 3a4d9898,ERR_ZSTD,1713 3a4d9898,ERR_ZSTD" + }, + { + "frame": "KLUv/URRUgsCQACZpAYAzgsADQAOwCUdb7oKUBW/6kDV6bsIAAgACAD3+vf1/v2vA7d7z/9799sWay/b/23/2zv/t///5/81M4gRnbROXRBDoTEhGE06JyBEIhRIDEqZDlo7UzMDjmTR2tOvJGgoS7nametsAcF7xbkPyOatThfhyQeMyhQqOo/muXfyzQcgWAdZL5zIvWi+jTEYaZGcfxQZ4o6xynzWcbgw8JH9b8jPbtKrOUKhrNiprQTA4Ph71xCJcEKOjbTG5XFFIwTQoJPNR4ARKlLZocqpePgSySRH5SfFBQBuCUAQABmAJUnSATMQMjB3SIqzjKzVDKwQZA5Ahk7tCQAHAAkAq30uW6rjoN8B66+cvL/x3o8ZrfzVFbwyD//bvpc2dBIGLKghpXTOUbhu3TBK3NTtAYCMWanWzZPQGoJb2ea5DWgdLnuriwB9oFStEKOsKcGTRpJ1/XVqv+3OmRMce3L2lMscsH5I+pLaGdC/6AMC4LNHnuQHvx8tRj1XIl+Qzw2Q8QY7XjDwSepjsCFDgDhBaAgBov73tg==", + "results": "3154 e6208dce,3154 e6208dce,corruption_detected,ERR_ZSTD,3154 e6208dce,3154 e6208dce,3154 e6208dce,3154 e6208dce,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,2763 4a6a2f7d,ERR_ZSTD,3154 e6208dce,ERR_ZSTD,3154 e6208dce,ERR_ZSTD" + }, + { + "frame": "KLUv/QQK9BEAvmXAegAYoCWFdD+vCbLdtNkpuKrgBS94wQte8BhXeQBvAHMA/Xeol4c8Cnu9xfEAzJE40ITAQsBnwiphh+lROxnyug8Jo2/l4QxZzDu0rJpunmLEAVncdDieUDu0VDJUiKsdMgS0REwxAh0qDES+UKfcr8jQiocdWoW+pDbwYigVCrthGo/ZVGEMJ7OqXEwvqbBwPW1nCqlRbCj8A7+F8BR6c21YEdKWVxZCsKpkhzb8WP5JL83R7V/vOtY0751O/Gr0XNVzkogjHq0wIPdFQriNdoLhfTXSEDmkww6zOCyNJnRfW73OGJakdUCWmBKpMzaOLQoxBlSzMkw8ZzgWFuYU1ToykjBucgeABKlJKszFOO2G1oQn9N1VZWiIkZ+gTtWGXwOGnf+6lqFWpICE5gFmhYSHXRwIZ658DdkElD0xIeu0Y/EMx1iMp+ExwyyqZ8MNSQ/rRZgNV5nCyOiToR/HsBIGulDvG7ChqkN8q0Ov46NOJNXkyi0XQvoPhwO5MWM04jf0aka6yxUmr/MBb41agYVDKtRIYdFwbH4P0zf7OSsGpKpv+VBHtUEELqzehA5l7C6yXwyJzOJYYUQGomsKYw5tBcTjYWYJLerkeOfxFk/FEGBN+MrWfFi2aVsGbhULR+5wytC77CEwJfaKAReoQEIiEDESWySSQCwPEDCBiFCZDwCzmgOlF4IB4JOOGFAq/kXZl1SwsR35LHbEV3fDxjNgpIgIRJK2in+kt8nOUcN01Phgg4jZpzKCFhoxAJEAAAB9DQAaLygGjhEREREREREQXwBfAF8An7eRMDuUfcFKRxrpTZzEs7xMz6em4wQdbNy1KKgeD5JK5n5GThZ8Zbcg5SYtLtA9QLlCni7zcEvQuOno19B0JdZ92keq7kscUC0ha+rogcmS8RtcJ+gzn6NCNfk16hhZQBFmBn72IjckFHHu/tvV3htsoU2oAW8rpzyKoswuTK2Q4A/ZI6+3yB5yiy1hKGgkIVHkg2BiFTgDx0ayAkNtH7VbtGpquwCH0bSlnDuzAbJbkxO9G2X5YAGM1cEsEQAyOH3J23GaMIoMc++EbRU15/LAGCqru/wJ6xMfgyC36oXvmT4NmFJpz+tLYEbgOYz9mUOXAtlFZSo39NzdVpFCc1Y0yMwGd6klXQ/JxvFC5JIFnsseKg88W+1Kf7YY3nUqQIal/cEBd3lullQhDbF0arir7zfrAtbcfm9iq1eQhMfqz8p6RNSLqaB14jxNvLB7lY/tXSnRg7hG/Gc/nJzJUdbBTz52l4naICQT7LGHCS/v/L8DoMkX+r2HAQKoFOD/n4AB/QcT8BP4DxTAgAH//3//AX3+uGmcAGAAEPoBTbQ=", + "results": "4117 fd2dd5df,4117 fd2dd5df,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,3292 b626a551,ERR_ZSTD,4117 fd2dd5df,ERR_ZSTD,4117 fd2dd5df,ERR_ZSTD" + }, + { + "frame": "KLUv/QQrVQcAPgrAGACOERERERERERAVABUAFQA7qwdUoWR8KjNxyo2l1uuZv8SqBBqYS4WAmP75urQojFMZjMsqYZSxmh9NO8BlVfYzG1VyGjnKS6ABAb7SNB1Qx1spVVd0vrjrtqQke0r9m/oVYwE0oHDKEYkLaQwQIIIIlfZOeB8gu5G6GuCh5epfLNh7Sru42BAXpeJcwA4IHs7obmm+4oeVKfL8PtYg62c3ZChrTyGsWBvR7sgFiOmQ0ad6q8sVX6fczGa4n9gqCEzZWMKTQBr96bQcWMW/wBEhg0MnknJZCuKOtQZ+xTWgCFVvRTEBoPmlDw==", + "results": "858 59d36777,858 59d36777,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,858 59d36777,ERR_ZSTD,858 59d36777,ERR_ZSTD" + }, + { + "frame": "KLUv/QQThQIAoscRjhEREREREREQ8BE+5in2YPyQK68lx5gqvatHOj+8zW1k9by/MBxxeaJFmSPC5IzNYJ+A88nKpKuPVyRdqD/eAuSXyOQTrwEBVAYCCgREU5c2", + "results": "135 c0d6b541,135 c0d6b541,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,135 c0d6b541,ERR_ZSTD,135 c0d6b541,ERR_ZSTD" + }, + { + "frame": "KLUv/cRSrwIAAAAAAABtCABKCXgBH5AVJEkHl2u9zIVrYMyXuXBnTvu4KjIz+7iWOjCb64wPAA4ADQD+KDXfGNNwCk5BzYTzrAKu87i7Eo3H6B1SwYBUYutPO1e0k2U6h+9CgDILJRURrcwvvfESeFVkFVKgkFBukg4gypiauwMDpE0YWxI7na4WMfxeKiNEKoyhPQGL+Zy2Vc5DSMETmlGpXXCtkQzgHFxy+oOqtj54EHCuEVNKzERW8EjjSkFb/i6tFwHFNJkdQ5zRoWtJTRgW5gZhKpeYnRTy93zL34Daah+s19NgszH7hBZWDPsmAtES99TfDGCSK6snqDVgekAJoBVQ8alKwPCyf4gPoaCQQEUdxK/ypoaVNANAvYxCmyQ=", + "results": "687 3115c274,687 3115c274,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,687 3115c274,ERR_ZSTD,687 3115c274,ERR_ZSTD" + }, + { + "frame": "KLUv/cQA/gEAAAAAAAACBwDoHQUAkooXjhEREREREREQGLJY/gM8xohiyB3RKUprtreDFMZpg7SnbnQ9OwQ5zhysJP1HYvBp2M/a1ML5arooz0mNP6xFwe/mpt2q/AUX64gV25EyGoLSMYssRXShK20HFF6uHhmokEwRBUlqjwMgSECkKh/wzqoDlBBIcKzy2/v6oD2AWeaRUtdLSWM3eQMdVMDhsM/VDCS0CDBTjjzndoyKtaVUBunhyCc=", + "results": "510 802003ab,510 802003ab,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,224 b39b54b2,ERR_ZSTD,510 802003ab,ERR_ZSTD,510 802003ab,ERR_ZSTD" + }, + { + "frame": "KLUv/QQd/QMAxkYWJHAlSZIOHPzv+uEejvAI3dSABISuyH/+dw0NwiNcsxAJCF0RuA4ACgAMAKLhGDK1sirQCCEOAQMCUilBUVNdXgH+GhIMWoTHhGjVgZMcfLIUVV3TXVKJDQEKoEASRRRQoMQMmwMQOuzuZqGlAjEAJN7SBmEKDWmgdgD3AuCpcZE=", + "results": "153 2173cba4,153 2173cba4,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,153 2173cba4,ERR_ZSTD,153 2173cba4,ERR_ZSTD" + }, + { + "frame": "KLUv/YQ8BwAAADkAAF9fX19D+OdoDIQo", + "results": "7 5e21970f,7 5e21970f,prefix_unknown,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,5 514e8c54,ERR_ZSTD,7 5e21970f,ERR_ZSTD,7 5e21970f,ERR_ZSTD" + }, + { + "frame": "KLUv/QQHAAAADAMARoMIEMAljQGu///MnVNVdWF/BwEDAAIAAwDz7wF//p+/Ae+1AySosJTZYjAMiZ0PsB0DiDAccBkRhNaDj9QfhzhlvObcBFwapvQiXCLcIA+9UJVipRZSfJoHpT3QRWHCCqQFAHIMIiSQKelPF6+D5WgBWoAWoAVFmUdHnpI8JXlK8pSy6TKvZV4r0YFuYnNJgERDQw0jZCxaepdRZRPZyvVEbRSaPWfTyNlqWCdHOZrKn6VTtagVVr1lfQhvb2hH20EkmwyylASIELa8I7xu2Jp1Umla+FIUncCHCG/gQvgyfdG04dnjMooRYsYvhgEHqGCCFP9YnIUHEDSR0QcQ1oQlsvixoILlATrCzKIAudT+Dw+g0h1ABKAAAJdhlm8gVAUYyop724NilRMCObAbWgAAYgAAACkAANRmjyOorPqKdw==", + "results": "564 70b3584e,564 70b3584e,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,169 55ab6f5e,ERR_ZSTD,564 70b3584e,ERR_ZSTD,564 70b3584e,ERR_ZSTD" + }, + { + "frame": "KLUv/QQnEAAAiGQAAAAIAAAkAAAAAAAAAAAAAwAArp/3uNg=", + "results": "3 60ad3d79,3 60ad3d79,3 60ad3d79,3 60ad3d79,checksum_wrong,ERR_ZSTD,20 b3962dbe,ERR_ZSTD,corruption_detected,ERR_ZSTD,11 6607ce1d,ERR_ZSTD,3 60ad3d79,3 60ad3d79,3 60ad3d79,ERR_ZSTD,3 60ad3d79,ERR_ZSTD,3 60ad3d79,ERR_ZSTD" + }, + { + "frame": "KLUv/QQ+fAIAdkMGCuDpYAoAGBgEUQICAAIAAgD9//9//3//PxWowM6IuX4GEBiGlB5AEtmQHQ9grfvG1HkqiWkV2rk0wtz3nGv982U8u6oO2c6Wc4YbARgAABk1qXQEAD4GwBMAH4AnSTrokwmaU4hQITIyjAjZBzPAjaJ9GZHrGD+ZEFgKAAkACwD+ensmUNDV0oYSqj2Ld/ns0rwCImJfM9KHB8drxRBLBLBpSI3bhAm7BBKocErNxExtBxAowQgGtQMQWFNB1XJr1zeumXGBsxr1CwKZmkeufggcS4jOF59M4lpc4MVSAe5tOV3sAQBOA4AGAAvQpQNSAQAgpq2gNAIAAgACAN/v/z+9//8/BqjQLMai5ecQ7rzw8OIu/1z+Dhw64tmnVEF1zFcHHQQA9sYYKXAnSdIBhF+hRwTcVYCuSKxHUHXFFR02qA8ChV4QgK5IrBdwD2Phu0G9DgANAAsAB1cVCndF0V9tCJJ7FowTo15Q+DhCLIlpA3m1nWp7plfvt6moBz8zLQfFwYbjXuWfNTb4BogQMPGZoGUse9AsMMsdOkJAa2dtOYmErJASvlQ02JJ0", + "results": "721 24ac2395,721 24ac2395,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,574 df390fb8,ERR_ZSTD,721 24ac2395,ERR_ZSTD,721 24ac2395,ERR_ZSTD" + }, + { + "frame": "KLUv/cQ/yQAAAAAAAAD9AgDmgwyOERERERERERAJAAkACQDiUBNiuX6EzAFgpddWIJB0hAH+7JSsIc38UwHiRWc35qXTAQyoQAxFzvo7ECAnoLwDEMYcokvMMQ/BinVTMCNIe/4wrNjdd64VZN+KBGdT8xw=", + "results": "201 34b975c2,201 34b975c2,0 2cc5d05,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,201 34b975c2,ERR_ZSTD,201 34b975c2,ERR_ZSTD" + }, + { + "frame": "KLUv/QRU1AAAEXsIqIAfEEjK7BAopbJaBxIaJ8xczbcI6j/cAQDmAgcM0KUDSe4HAAAAAOC0AwACAAIA344D+z99f38dBqiATGSB5bF4EOa8B2ACYy0eb/RQZTishHOQWxIDAJQAAAAhAQBlXxyPFNgZkW/u7u7u7nvue3t7e5SUlJSUlGmTTSmQqSoCF4QF8EOp", + "results": "279 1fc320be,279 1fc320be,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,283 6544166b,ERR_ZSTD,checksum_wrong,ERR_ZSTD,246 1d1842ca,ERR_ZSTD,279 1fc320be,ERR_ZSTD,279 1fc320be,ERR_ZSTD" + }, + { + "frame": "KLUv/aQ1BAAAqSEA8mHhDidojuYWs1gr86wxaHjU2EpZsPDSM5H37Wrke2TdA2gmRdvK7QkkijsIboW13q2wC+Xo0yc42xsyWJ3bOV1OWTxR7of0cVXtTQ7QlQ6h6DnSypEwLLksAGqulJXodETdtSctRhWKuZwPGmiRs4ntvhh3qs5xLQXb12kRSglkSJcYSgpcHC3KHFX4AU9YrRKUIIrbhL9lnkaobZzgAFoXVsEXTxrVZ1Hj/0fGbWJso0FajJI6z1RAhkagf1ptN3SsWNHyJbk5fin9AQZUquUcvCz6jBVoFrRTkZypf8/XQMi2CI20HieKiFu08oPspQTi8M3Vb7IsTTcnpLueDnmoUKvNHFsOmMgb/QTmqVT14SOULYlFsxFN41M7gcOggKvlCM8otIVTZynFZG+flDFpXZt+uk6m1wswEWK5kpQJKTHCRS86U+ABAsDHD6/k+7MM/AdG+IVRZ6C6AX65XrMy+wBp1+IgRVvEm7s6yUB5X2HHiS+oZXbjVXmzlj4NIojc3Lli0Muo4Ant1YQYrPCcTmlxdDa7u34dxSpvfB/TyvjIICCJNZedaAPuV1c2nV3W4Rpz6xr5yhaOrVjEuzf+TKrn3CYR/7kUeAy29Q80qI104KVwsAutO+LBMw04O25MMO/93Rucal8DQfcF+k0GJTk97iVIs1C30CLR/pru2iK/cx5Lo4OrXG5X1ArNaqCRtCftqLBnjR49nC6FPu3O2Y5R2Lq9d37ZgsBnTlliEn094I4WcbUljgmDyCjXm9HYPyKuA3aRZWLj+vNhFToZ+XXgZdWPYTjb5FXzm9daEX+7yZ7gePz2KmUjQ4ZBul9eRmY/y0picILgHxPSfam22WcW4/kKPuhK0DMxQsqYIJWaCZr1dX5FmYxE+CIOqKikXAcSik5FkMx3W8UIYG0jXDSWPRD5wKbdsmitl9EdccMv14xsXszy+U0CMHPIsQAZu/F+JWzllKwyFz2P/onGA+fQ89A9Jk54LOas8wBZN+uTQgkK/R4mm3aE5YllFkMHaL00/1WzAQGQ5ElRa3L0TFj5Up/qbqR4FpqZf9AUsdWtGHwgNSOdN91dgqaLPtjIvake1YHup+BFcr/JWiFS/1WCgcUidXtNKd6MR/Yo9XxWr2GgwzDvUifVO7drRhRhHrhKhFVeYbIqLqwqmbxaDKP8y0t0C6UZueC+rDFUncAzLR0H6P4kt0Jtv8t7Q3bXxwNvHY3WmGGAP0eknd16BGuDFPKJzpgRfj+WeeylZKXpHiT2JrX7iC5PjM8zlH3Yg0QHUBwsCniXbmnmDNzopd+3oiMFIVe7i+FYuTULvTYyDf2XNUo9BLyWnv5yVOXoRWZ/+o+y+E8EyLdsjgc4CK613EDaN8JkavdU2aXRKwXx1lWdbjk010zt0J4AdRfmGqoq1ILVx5aeMjq75OoT7FjPdIbEEA==", + "results": "1077 46b59332,1077 46b59332,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,807 d1fb6ffa,ERR_ZSTD,1077 46b59332,ERR_ZSTD,1077 46b59332,ERR_ZSTD" + }, + { + "frame": "KLUv/QQudAUAqgPkACCAFRQkSQdzBvdEZwXosxJVJAzVrNTKpJhwAFSVCHbRAQQABgAEABPrLwM/C0AKBAtVm6GqPw8WAUAI4A8OJF9lhnnaY4CXmoHDwBYItCjjuhHDTL37MHi8y4utBzLMNeNtC2YYc+MqwBS9wC+jg/WwKaIJ4N2DmUwHaAIGgm8my6ohbUUQrXcImFW4NylqClgA5kEVRoGUyxOB4p2BoEOW0AEK5EAHOHcFrAEABgQLFaAVJOk/r0lVjRWh/v5fKxQ86P03KgMABQAEAPO+ehk7p2QBk32jFpsSTZ8BVAAFADawAQCSd9Ibzcr1Z6tjFmm51H80cL1ByXvA+hP7Ob8h0JviQLj/8QhbKAscS+p0b+TUE8ujQGGVPwikAAAQbkcCmBTwPwYU4A0Y4H/fHgAAAVEAAO70k8x/LuWb4kC2AenD", + "results": "412 bb073008,412 bb073008,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,412 302a58c0,ERR_ZSTD,checksum_wrong,ERR_ZSTD,328 fe21f2b1,ERR_ZSTD,412 bb073008,ERR_ZSTD,412 bb073008,ERR_ZSTD" + }, + { + "frame": "KLUv/YRDxgAAAIUDADoDrACOERERERERERAHAAcABwAEyIw9U5Uc3DXQU92FF6eeubzUSx/YX6DGXkgBJKjwtGY4MMYwlx6wnG2xBHWdhJOByh6SRYjWwPYaRnSyLbm8P4NYqCMykLtw7mEL23jMPTHSQVwmIoqSMM/w/AQ2MiPB", + "results": "198 2ad56024,198 2ad56024,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,198 2ad56024,ERR_ZSTD,198 2ad56024,ERR_ZSTD" + }, + { + "frame": "KLUv/QRaJAIAzgRABwAJ4OngkFJm8IFGAwAEAAMAf+9v9/1iB/fvX//vPgyokIJziRjZ3wEQOunuUOnqQgAqoFep3Qp30u22wSBTmThEAABpWgFUBAALAZwBAKhOPiXfdOLdY1xDRVKj4V7Y/oI2ZxYDqBRgWgW0KsA/E7CB/9QqYFYF/p9/A3YuF2IgJHwBAKOHBO/bz/bv///+//X++/////+/AQWokGJP/HN3EJ78kFKGB3HgA3nARDVuD0MBAAAA1AAAkXQHoNCW5eeQTIzZA9MHjFxG67jmFjABhAYiAQD3BQEAUdsEqNRqVWD/ARNAUKCgAME/1CpgVv0DgauhuV4dzgb/6HeS", + "results": "479 a75bb445,479 a75bb445,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,358 1a225456,ERR_ZSTD,479 a75bb445,ERR_ZSTD,479 a75bb445,ERR_ZSTD" + }, + { + "frame": "KLUv/YQU8AEAAJUGACLPII4RERERERERENLl1E16q7li5SQNG88rF9tVl/TT+DYRKspe0KmCNsvG5+e2eJ6WN9p2+PjLPno2LCsmDEVOmg4RA84/SJZ3kULYu9nAO1XNJOOpTVEnHqfMs9MQoRDeIvBLXF8cwCdbYDOJFzZlMKIV3fxZADcoM8g5ylnnFkG4kTIBFYhgxAQTTCQWJiksTYsBcGYkGAskKGW2UrfeATsLoLLHybh/bam3ZXmuE4yWGYiPmAle3yucUwtdN5AVVu8v+OJAYtiDJiqwMKJOBW8gEBo=", + "results": "496 205f0138,496 205f0138,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,496 205f0138,ERR_ZSTD,496 205f0138,ERR_ZSTD" + }, + { + "frame": "KLUv/QRBhQ4A9hQnIIAlSdIBc4IQBCEgRzmDSxzTWacjHVm24jTX7GiGqgReHQAeAB4AYlWm+aPVNKhfPxbvGfkr/l571ySu5ed7/e0iMgHqDvREUG+zf9/8XI+77/Z7BtgWOstHTQXNi/HXrQKbF8y22FuG9Yjt8u9tR+ha2V3vT/5cI3c4SAf8Xh17XF3fm/Um9EvCr+5I2PO9csHsL363j8MdnUsHgIKocNDNbJLWIIYYMlXoA5KAWJInScopJ0BVCjWVad+0g3G5Gv52yJTqRYHR/eAlmgCUSDprrF1PW5L3b2rgintAQKZ6ifxwAQhh1QvC012uC5eYE7e+WpKc0AKjYO7QiOHoFIGy3vQO6CG3TIOO//Omhn6/ijATwo90byRkRSuYSBXxGqhJDq7xezPsllzrCEQi4tE5+sOQ2wI0cr9oMi0ruTG4KMsdErA8rQtzwf9RT544KgTDeIkyhiPI/Mxmc4syiBE0q8RO9f/QJBLlWxLQDQBJtFmSZv1luz9bAyD3mqoi6IBR8ZJ1Scw0ylcHIhiSAZsVcs7Lqyv7JiikF8MPSGLVNepMcg0E0yA9npn6Kk7bZC9iWr0/11FsPn8q85uKmDGXvoS7wgN0n7ELif+LLCKE", + "results": "1549 daa98a6d,1549 daa98a6d,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,1549 daa98a6d,ERR_ZSTD,1549 daa98a6d,ERR_ZSTD" + }, + { + "frame": "KLUv/QRcnAEAxQo0EyAwSDBM9jBVqHDpw0PBhp5rdUBYRcRVXQCRAQYAqAEDEXBswVVgpbEcibpwAjATnQUAqhX8AAjwOR9zl7sDHAwADAAMAP//3/9////+fv/xf7/7////v/////3/Hv/3//7/v//+////D//93/3///f////vDiio0RBCzDCEmAmJiGGrATBEsFDiAzDQ0Ny2mDU1IEvK9L27nHnYPRQEvReJ4uUh1ypmtDLPVki6C5A+DWNsgm4HJ+STIq5Qe+mvtkKpA1SHpLQqnYZztxKOsFAoB6MAkasLWPEXRWpvZ+0c3Q9IhwMuAi21", + "results": "893 a57088dd,893 a57088dd,corruption_detected,ERR_ZSTD,893 a57088dd,893 a57088dd,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,893 a57088dd,893 a57088dd,250 32755193,ERR_ZSTD,893 a57088dd,ERR_ZSTD,893 a57088dd,ERR_ZSTD" + }, + { + "frame": "KLUv/QQJ/QUA/iFAJQAWkCVJhgP9/48R6u7WDLu75by728szNB4AHwAdAPXKy84/dn1sb+ptBCC6cLm5UZfAfTrh4y+n61+/AVtWH0pWudfr/JI6CUA555EQ2E7bec3v2WVpt0Z75wuL8/Xx2u8Bg3pXW7/Bogeyu/0mL6Lue1268n8LqvmKZR4Gv+o/83uyIz905zyrD4L3LKzjtSdla618BgOo1Crg/3dVgH8TwFUAq4A/FOBpFfCuCvz/8w8og1Z6RSAqQALZb+M/", + "results": "707 7144d18a,707 7144d18a,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,707 7144d18a,ERR_ZSTD,707 7144d18a,ERR_ZSTD" + }, + { + "frame": "KLUv/QRGvAIACgbEABSgpSDpP68peMGjvr/XBdlu+t6vKAYABQAFAE2zyzelMOvvs3ZCb2nz61jllL+LTwEJqBAsAUFBRZUotQ4QKAfVHhDzAVnCebIRkQxBszQkDAYBAAAABQEAl8MFBAAFAAQAneXqBENWeDkC/zUoDQV3hgkBVBYBIChBITIs", + "results": "222 1e1e8c73,222 1e1e8c73,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,129 d772798f,ERR_ZSTD,222 1e1e8c73,ERR_ZSTD,222 1e1e8c73,ERR_ZSTD" + }, + { + "frame": "KLUv/UQlhQDEBgD+BQAQABiwJYUOjcPAZjlMN8thlTYOq4RwmEqN2QUJAAgACACOS6LXrXUDDzxxYZ3+1ztzV1r1f8x9nU0D0/9KE9Jus1RFoODSJGkMIAgpZfY5ArAt4HcAKsD4ANQMOAngeEBcAKYAkzZgAWkKCIEdDoAIYIDZAKoamDHAA3xwyGNAP4A10AM9AJMBlAAxB1IIRA0kBPDAPcCcAXlHAIAVAIpAMqAClAHJgGsAM/BEAAW+DMAFFgCpAEqAC3QCogAagFagKuAUmAUEwAfgAyAEIIAYaAEAcBFEAAAICQFUAQYOXRMCAPelE2mR", + "results": "389 fddeb050,389 fddeb050,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,dstSize_tooSmall,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,389 fddeb050,ERR_ZSTD,389 fddeb050,ERR_ZSTD" + }, + { + "frame": "KLUv/cQgFQEAAAAAAAAFBwC0BmhrixRy2RqI4ngZQsE/YFRYgTnAVA/uvzneW1HNgVipE5w9TSQQljvdzro1aD0eiDA4fG4mdB7udUQc/a+Gw9flUebIPmA3Mv5dwFP0Id8XOll0NH3dB7xc/sZOiDBFSnWw0rmj/nWx/cb1MiAgiJGj9BMBACRNNOAw4LMuLQCDA8QGREug5IbGg5gBEACWwcUUcKdgZQT4NYAKAkABwBjI/5VssGSdISDiAaAAwJChmgdNAFgWgQAAGeAHTAEWCTwfsEvACK4zQBJgPqALYAQCocO1gMsAK2ABWIBZBCIvM1Y=", + "results": "277 3e0d973,277 3e0d973,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,277 3e0d973,ERR_ZSTD,277 3e0d973,ERR_ZSTD" + }, + { + "frame": "KLUv/cQ33AMAAAAAAAClDABKIWwEjhEREREREREQQwBDAEMA6Y0wnB8aDdLdvOKANehAPat56nLPhlCRyMqKDf8vhZ6yp+OGxLT1gxGJ54/389m7LH+KCiP0K4jxGFbIJfcqEO3GGBYMB+AKwMM1zqzC72dKyH1/b+gUtF5rlJu2y7RvoUxKp83UrzQ1I+EMfdKZMxrHu/Pcz8wrkbNNhiwizexWqijJExMchFSqIVznNEdlzd+a0MTTjfzHZrAsuoNUG3tz8rMQoMxBMwOYSkCA6iE05jgpTO3jYAcnku34X0IRi8tIBGwt7lURsLc6ZmxyLSOrSnCPpne2WIRlecog5h4DOujJgjiK3g5uG7lmohC9FL68dUU2ZF5gfvRTEOfzBXSzPg88G7Rq7P+tFiSooRg0igkNE5Rgookmwu0wiIwQzh5RDIVBo5hSZiyczGxpPRuqu0uugJxZchO7bc53JFLt29zbflRW7SJR03LtqE7AMnuSKG8Md0ZjTCY4UAbG0g5azVzM+qFe6WSTeJ0UYfOZRBlxTGFj7tc8X6nKk1dgAipgFb8=", + "results": "988 304a3bc4,988 304a3bc4,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,988 304a3bc4,ERR_ZSTD,988 304a3bc4,ERR_ZSTD" + }, + { + "frame": "KLUv/URHwgHlCwA+CIAPABagpSDp/7P/WXeNblZVAnhiV01U/S0YCgAHAAgAJ5XawhzyRlArCH+c041FN7e77gtWB7dvAnkQ+e2WLbUqgLQoMIjI0J0PwD1H8N8A2ouQ51qJfQbulgmjUOvVon1JBnQN20TgA0RpVgQsNd5AkwyaBYKU4NBQPiQCggf0EQokQgImsCiwzvTMHAHrwu5QIKrgF+IBjx4o1kZFrohjFgnIcAdUCVwHU0B6EdtBm1eIrcc6AWOVuZakCVSJWzBBmWnbIYAciCAuInE3MU4V6jyycGvAEVAQAlvg5mcg1uFo+OBG5gQWDn5hbJCYoPkS0jhuIdnQEWAHgHDANL6zQDtOYvCLD7oud+xBC/9g4uCTTQh23JEzmAYAMDinicAV8mEk+LAQQSm8ADUhwAXJHPIcLgXnSwzfIaaMk6gLHqACQoM03EkHgyXxYepLFpNaItvgUTmY7x2AHNZRPEoDISwLEgi/0GZFK4ZpCATguHEFTdAcZgDMKwB17k8=", + "results": "706 1a6e44b7,706 1a6e44b7,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,706 1a6e44b7,ERR_ZSTD,706 1a6e44b7,ERR_ZSTD" + }, + { + "frame": "KLUv/QQYDAYAkgsNE7AVJB0Mc5Ak12FqSWEHk5ObBgF9Wt97LK1/69j/f72/75fa/02z5///3sv3X+O/um7rA0mogA4ZaUEyjAEwxpBR5wPQ1E0He43Huhc7EvYMjHa239qfqoVRZch8FECWcb0qvZI3IdbpVqSPAIpZ5k2ST/FCvLmFqBqsE0FSz121vBEYOy8irRJ4qjbe2WJ/LZ60zP0T5o5YfLQun/oHzys7Ee3MzgLCuEeOBEd/lg6g0R4E9YxBBYBmyBBvATgBAPbLQUWvCzgLCzg4Cwu/+pALODhlOAv3yTj6h8bf72Rk3hYLCzg4OFwBAKPEA91Zzf3b5/Oz656D/HSzAQO4FKBWAf+3KtA/1CqAVf9dyqbMhYQkIAEoAADno2fNMQIAAH8AAAAAAAAAAAABAADY/JEy", + "results": "617 99704fbe,617 99704fbe,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,518 ed670b63,ERR_ZSTD,617 99704fbe,ERR_ZSTD,617 99704fbe,ERR_ZSTD" + }, + { + "frame": "KLUv/QRWlAoAIssYjhEREREREREQjKN3aWZM2gUfeGOcWHTh2SE03RS1feMTuBPjmFDir2n2sGbz/NkrALDzHh6pZCklkjgfgm8Yhhok2bEotI8XuhROVW6jwF2RkL60mjEDJ/+0deOBjm4ZhQYBgICoQWHMGvYOMIYQUmc3kFQRpwM/tFUyLZkXJVAlktH4paP3oKec6OKEwlPi05Ud2SCeGMfQwvLZFbnhaMlHa0dG76PvxkhZjeXYzCwuvtCIgErQ7aINqdFKAQUlN70Vm5TNt1wBGfXVOzguubvs8cqae9aJh+7/IDjsPPHyt2n429tmvFq7cdjMQFXC8qMFKw6xNkMTsQjK5rLL6hJDeuKXgPm7DBHb1Rs/x2fCmizQEmrcKUkIIGSbdHG/jeXpndxrUY75ZcUOsUYEo4jGxCD5vBsEcERLVhFtmZ1RZMWEZ8bDhe0cZ89d53kAAADEAgB2CA0SsCUZDpAlcS6JwzlJ1SQ5IeJcBgAHAAcA9+tf+fM6n6+jV5P/A/1V8930Swt/up76OukoCbhQSpDCyn97UBym9nO4QyLm2z1M6WBs7HBZ+jvC5WMRHQQAZscYJoAlSdIBPQdnnLVKzvQ1YKxd1rOsF+rjrFU4464BY+0FMNYu67kgDQANAA4AhXexjZfyt51l6hpjZdQaUaP5TE9Nj4tOzBsd7nOR3zYemHJVe90WASzVSgUxuhPIwkO6Y4IFBoiAQhNaYPksd1DThQMJGDG+hDIRpuYrR6RcFiuZtK1b", + "results": "1113 e322eab4,1113 e322eab4,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,775 e5bdec53,ERR_ZSTD,1113 e322eab4,ERR_ZSTD,1113 e322eab4,ERR_ZSTD" + }, + { + "frame": "KLUv/QQ/dAMA5ARVy4sm39sJgn5n6zATXSgc/+rb/EvGsMM9b84Qb3mDkurJ8H8RFoVS14HjP1Qus/xJZgk7Km5NV0ihLDfNnf0Jw8g7g63nn0KfnwsGulkEKBNAUFDA4B8U4P+fQAE/fwP7D9QAsUsOzJwBAkncBQCShQ2OERERERERERCFhd22osVIy+LyvhGYgJWEuZqAjAh25thaUV8Km4H1d0Ffk3NtpQ/dKwa4cxE2qKCYrcYwRiRkSh+BPGqgIiMzYmFRjSybAfhpnu/xu3f6eLL50M1eoSKy1mbabfn4QpPd0es8mJLeMKOmiCFJvW1+WGqwxVneFhToSKFEjKsNjO7oQsW2v+FmKouprFpKZF/wQDUMMkDPNpDQwcN9ufwAbPmOqHmKlQ1L6+AsoKuujAMAShIoARDAJY0BD/p/1rdgquoi7n+aDAAPAAwAv33dP/2xn6X///5zrf3Mz59p9E32cxWfvXwF+//97Xt+596f+q03/b217pW//eu7//cfBaiQ4n9PdOkOECZFKcMDEP5LJ7jL/xwBBcMGmmiOmTuyycQCAADvIgEAVrgAADB+Z+swfmfrMH5n6zB+Z+swfgeFUoVSGAAAra2tAAAAAAAAAAAACAAAiQIAAAcCAABrAAAAAQAAogIg/A==", + "results": "1508 9558be51,1508 9558be51,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,1036 9e023764,ERR_ZSTD,1508 9558be51,ERR_ZSTD,1508 9558be51,ERR_ZSTD" + }, + { + "frame": "KLUv/cQapwEAAAAAAABcAgCKBoAACPA5D7S4H8jEBQAEAAQAx//YfQT///8Hf//3H//H/z4RqJDQBBa37DYQGCunDpCvBuicOLv2zlX2DjGaZGgcixDf6zU+Za4AAACUAABY1J8GcyKlJGskyMQB9BMPAQEbBwDKU1WROA==", + "results": "423 544c3a61,423 544c3a61,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,423 544c3a61,423 544c3a61,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,163 38d24635,ERR_ZSTD,423 544c3a61,ERR_ZSTD,423 544c3a61,ERR_ZSTD" + }, + { + "frame": "KLUv/QQpxBQAKgZoASdwJ0nSAbz7YBI31+8JCnRedyjSobhDkU6E+2ASPVftpQgUXnco1g4LAAwACwCE+chdrCiVYcdnW+UIAc1kWiRT6rqNNJeLJffydOobN1ADNSFursdDfyUhHoFAqPF2BzEUQhBjFtMH0UAuVU2SDqQwV0SEpQwrrraRrJeYcz72GjjFbnTjamhH+zHKZJk/ILz3h/WMLRbIJKR6CMQgRXAVLz34v5f3Rk8ZRMxLxbu4qagAQONx+hJFwvy1mxESLKOov5OEFCWUm4EdkEZ6J91dD9lYwHcs/bwUNoYaY8iLKmRYBE0hVKihsIMAbRwHIQtpIILfA0RrAIXTQyWN91DPECxu2tOSwSYOAXkag1/Ou5lto5MW9kL5q/TIg3gsKFGDeD+jlUDCUGrvd+EjfwiEPLznVIc0SbDxh6nN+Hg2/770wgxbRESNtKlvoo+II0/zVMeelWGY+D4DXy//nD+BTL4OtQ1dUHKEMITJ3dU1Ag82KOdy49KpE/vreuRLw1KJFw8MTTVhoxqqmX4+/77qihMe1iwC/CaEw+ymQM6PLRtaCJxqsNoeMN1p828C43sJ3bVDAGrXtUE0BJINhkWWz+YDkb9cjUjqwCIgQOpuIrVAeL72OHqUg0WeOAhslQHjhxiLjAVat+6jZkoiZtQp18Kt5s7b3OYPbiLgdTIcemzfHFs3gwHjkT1vKZEhKjgPOFhleMK1zJ2cCcYCyumV7xTC7Oy38+ghD41dBEiHRzLZSgPy3JsmAahg1FLBQI8cxihiUt54W1eVEk79md8UGLDiEFciG5syAzo74ClOcQH1vyTtBLsZCz7RQuBbYxJa+yXbXYdlk+HYKj1UUNIMNakFo9CJbbdwMuzSeu3RIocABYUFADYFDhuQpSRJBw/efRGxmY8j5vz2y1T57sjEB85OdAYFAAUABQArTbq3E0/XxSW036PPBKoPGMNRnEYkNaiwEtpmMEYgREQsH5ASlqQxXJw9jptsG4jwJQJV+metN6lujd+AsOcuxT+r9725rVx+SSvZyguFzZapmyXOfdkg6j4pl1kGZevMWjoJWfI+USNgy2rcRRPDPUBBxu+2XsjUIsysO7ucfFo/2P3jVKoALUhKvOJhww==", + "results": "2120 562a36b0,2120 562a36b0,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,2120 562a36b0,2120 562a36b0,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,2120 562a36b0,ERR_ZSTD,2120 562a36b0,ERR_ZSTD" + }, + { + "frame": "KLUv/URE9AGhFwAsEr0JPKjRrgx8dS5LbDQPNYifjYnYE8LrLyouxky6x2/Pj3SXBcpQbEEYfkdW4dQyUek0id1WMYG9O7YxzSWQGglVk5sp+KSTqzHC+2+nwq9vJAH4V6ppo3RHHjLqLUgb8OOwWw31CM2IIYvE2DYK86i0s10NoMU5r3zFAy8g4hcznLp/Pvatw9xfALbXhIVGR8Bo5DSlKENs36sBXHDJheBiuT7+18ozgIjZU4JofWE2uDa4veZGb/8eTHaQaUELv4p8fjYfFU+ZIWKFo1LQlxGli4MY7Ovx5EUwJb0YPyfukPjTxoVmyiPbK/csQIidGJi1T01xd0gFxi257AMXjflafs9229EjUapNiRCY8/xVBNLw8Q2A24dDOpeuS/43xjPmHbN6nps6VDuhriMJXUgOvv7+Ysesz6Zq/PQ1+dZmTlMtIYpeDbKolBiJWJrJmgVudZS3vUT+9WWMQ59jVEN4YVFDoS3TDKv7Y0ajtqwN7l4ykgKG3cc3Omnut+QB8wxM9tCnrhW8yDCBPZWXRAk/4qZTmk7ycIrI3ADXZfpDAvj7JsKoE1yXSx0QlvOj1S6nLqtiWbLSHrEj7SEqfkwx8Xr1MFp1UnAa2So0PDduP5Y+yblYbtJA3PLiIfXmdXd1BOWye+RWZJOzaU2NJZjXsFxNEkZ3f6smCxu6NfX9V3X9xd5AFp+1dtzl+MC7j1Yc/+qgSvmQO5+quneHWseWcMZlWeDik5Z73RYULuKMpKm9PA7AQK+oKYPsks7pIN/kcPNcWf4SE9aPMRoUqrkZlqcpaukRZAIsMAxDSMOwt6LQq4aNZzg53mACZ/tN5hKk8etTFcz5EZyOGhRuLTQ+bLkU/IcDpwL8/EYJAKn30VElgSZa5bL35hRzxHp7VoasRlyHVlsh2NIP0YOQS8mkZnYjepmVWiXzJdXNTdZQhBqVSnbzUtXNC5xoOmuZ9PO+wX6S1cyRtMksQO4dUAqF95E1N+qGpNvGyM8HYoPDpwsF80pY", + "results": "756 3f16db57,756 3f16db57,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,567 77ea7019,ERR_ZSTD,756 3f16db57,ERR_ZSTD,756 3f16db57,ERR_ZSTD" + }, + { + "frame": "KLUv/QQKAAAAXAQArggADwAVcCVJkg6OtxfmEL68S77q/////3+ECgAHAAgAWTNjUeBy1C15Ade9/XobtA9brfzuhwm3MMF79e8fbgofqFCWBRIqTJLmEBhinEEfQBCb6oIqhSACVzDrD4Cxn5E2AEGx+YRovkZPv6CzUKyau5npwQmJD9TmOpf9blDkcLQMJlgsFJBzOSABAGCEx/AUNdnoZARuOa3Ownncdj+hjMuPpljLRd6eIpuLufiGzgAAAGkCAEMofAAu1/R7Z5aFHDNqvie4m7macpoZ3rti+vqtRlBYFLJYwhSPrXEKpYyA9sciHm1KGsMTr37e44KERv9f1sCH7/w7vC6Jne4aUo97qPHerw==", + "results": "418 fb164dc6,418 fb164dc6,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,352 f82bc7ad,ERR_ZSTD,418 fb164dc6,ERR_ZSTD,418 fb164dc6,ERR_ZSTD" + }, + { + "frame": "KLUv/YQqHgAAAFgAAM3Nzc3Nzc3Nzc3NAAAAmQAAjCUhEBpZ9x63ytaO/BjQDz7t/zvD1No=", + "results": "30 44cda6b8,30 44cda6b8,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,20 446543dc,ERR_ZSTD,30 44cda6b8,ERR_ZSTD,30 44cda6b8,ERR_ZSTD" + }, + { + "frame": "KLUv/cQFVAEAAAAAAAD6BADLJAIAMggKEbAl6SzKMAwokGSYw5W6MVUCf91075Z/nd3a+8z3351+337/b9r/HwSg1CrgWxX49x8TsMEBfzBXK6gTWW3tgA0BAAB4PiX1", + "results": "340 18c76d0e,340 18c76d0e,340 18c76d0e,340 18c76d0e,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,159 b3d3440,ERR_ZSTD,340 18c76d0e,ERR_ZSTD,340 18c76d0e,ERR_ZSTD" + }, + { + "frame": "KLUv/cQV6QEAAAAAAACcBQDmkBobkKUgSQfgfTkTzoQzoZ0wn3Nj7vMdMJudLQUIEgASABEAidDmQ+ziky3Yfw1EKQd2Q6R1o4zRjbFcmSt2q+KGDMGSX04VD6WprgMHp/OgzCB3lt+9CHmBVxaPEeksep3VMXiBmgpoEh8OGghgRNWEUlgorQEz8cBDO2lABU2z70/gPMz/9sJQvR4MdvSMAT479AyxswE2piyAYQGrgOCtOSpQhWr39J9hLahcgKrDAggAABEIAAASCAAA3hAAAJ61AAAACQAAkiISTzA=", + "results": "489 a2fdb50f,489 a2fdb50f,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,489 a2fdb50f,ERR_ZSTD,489 a2fdb50f,ERR_ZSTD" + }, + { + "frame": "KLUv/QQYRAAAgV8BVBAECCKIAwBCwFPiUhMOoXGLAqWga5B77tYfTS1wI2I2YwKgxgsX5XWTVjQxPcFg0uN6VEyhUY+HuBDJUqA3YysD1dh2ghc2A13X1yfRaOUKs3iea9pPOqeu/WIKo4378yYEEjE53vO3VYkCX1cbBNlXwHVRRTMeVHAAAOcK72TSo5Vc0lLLnr6njAAAeVIDiBQwgP909R8viQfkAggAAAAAAAAAAAABAADG5FcA", + "results": "179 19570d79,179 19570d79,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,143 431e5de1,ERR_ZSTD,179 19570d79,ERR_ZSTD,179 19570d79,ERR_ZSTD" + }, + { + "frame": "KLUv/eRuAAAAAAAAAEwAAJUDOgFUDgQCEYEBAFWK3nE9xapOQlCXxfJPJ5VkKWOHPAeq3zGiL5EpCfW7Ln6EvuEeMwcOlxzi+ZPvfi9o1jI=", + "results": "110 40d144d5,110 40d144d5,frameParameter_windowTooLarge,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,94 c9e03014,ERR_ZSTD,110 40d144d5,ERR_ZSTD,110 40d144d5,ERR_ZSTD" + }, + { + "frame": "KLUv/WRIAAAAAAAAANQDAD4GwBUAJIAlSdIBazNmyQW76AMKiIoMN/aeWxg4kBqoDggnzDLLLLPMnQ0ACgALAAORVRLoIN4FkwIzjwjb57FAqy3FxBonCkciMv0KphOR5QSS4RbIs1QNjWHcBagQJgUnuvxcOJCiGx4QftKJ98LlPQdseD6+soABAAAAegAAwEUCAGoF+ACOERERERERERAMAAwADAA6OREmWhVCaAYniAE+vYZS85i+XchXHwGxReYRs6DJT9arqQFFpuoVDIwVqe4HAQFUAwcLuZlX4x0=", + "results": "328 f709df2,328 f709df2,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,228 d09f618e,ERR_ZSTD,328 f709df2,ERR_ZSTD,328 f709df2,ERR_ZSTD" + }, + { + "frame": "KLUv/QQ6NA0Aos4SF6AVJBkOP/IH/eoKBZyAE3CozpUA8bkJ3eS6GkDN8Pe/+Td+Wbh22zd73QHisvkk59P9K0A9nUfXWTvN69pO3/75+A3BK1JLSrIBgJmAkJZx0hxuJYDqmrqJkJcIMiNAacQgMwBAhlYwUhLAb9i0QRDmBiISkasbC61CwA4UoEQ7K0w4cX/zUaJZSASLBrxg+G4iEM27vhKLGxfaA+jsUTQ2IEL7FnCDmGELoxF7EdBaIBiSdFxWAWDSgPcXvG/Af13A5trsEFYOsYRboJ8LxXEdXVDQgOUWdFux6jplIfDtpmzHmxjlsIHRit2x4AawdDhT1CCkdRVhmBIMQYH6B/AA+sdGhC/ZS4gB9ixQ4c5OOyLWBMIOhUEAYWsgF+B8JhsYL2slGBC3m2MsryxQgYPFEbsbbTRDQBd0YAVgHO2GhAEm9n6QYwYAfkOEGmBMTBMH8hcTg2TsamNAqkPRAIoFABwAiwKNAAV1gyUxgJ1mwAGuESMRDAQwamDGABgVOmowFXSNo4hBCAIgECKqYk5dHwX2DQAKqhkkB0QUQDvgAiZEAQCA+Z3GgHmnqgCKlz+LD3jsWgKoFPA/E0Bg4PkHFICAAf/zH6wVAAOARAsAcmhTjhEREREREREQHilx9ew2azfRrmH8JEt2Z5+ydRGwYW1kAKpsZa244TesGjc12Xe8n6LGaYTEeWbcUXX0wufM786oNAON/zyc0YD1+3tQRpqqcmJHKl3t4tu0liPq/ZJ/8h/zLHE5r2tUk91rVTNSxE32rUzMKdjN6AAZqM9EyPjky3aF4L+U5ALYJzowdGdxeaMJXpppcPhbXBSVZ7m/wukGkvb4Kt0vxNzkCfdFJ5nHKPA6hLIn5VTFCoc72W5VWIfqd9RL4Q8or+jadAGImACUCooubphjsDpQAh2UGa0xDMKH3mPot82xDRFauDc3XDbfb//GV+oDZnhe1BBmIa6RZivYpze15Sq+dpx+MtLW860VZI8/SQGuqIC13gpg3q4ggco0LCG6XfPK0UKo017rwreXf1NnCngjCn5k0aGlLsnTSGM9mQ5PUiccApgU4P83YAD/AQkU4AIG8B9wwrKTAAAChQUAppchF7AVFNIBDGcpWApCVigZ9yrcU+A0jvtLGQAdABoAVRDVQfmjJTFj7pKOWijppsKCnF1Ojer+r0XoVpVIDmH0sNI+aNRCkoZvRCd9KmVk3DMEkCwFpWVlLkKZ7EJug0NZuOHWDIoa9Zcd3D56s8HpE4T57FM3rFrjF4xd6EvRsE8PZsfbqQEHqNBCiz/LWeYQHtP2YEqsWVQ56/8OWZqhJWwn5T3T+khAbsV6EQZb+oxz", + "results": "2080 8668c61e,2080 8668c61e,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,925 9345e1b8,ERR_ZSTD,2080 8668c61e,ERR_ZSTD,2080 8668c61e,ERR_ZSTD" + }, + { + "frame": "KLUv/QRQ/AQA3g1AIACOERERERERERAdAB0AHQDBjEg+jO1tEg6ZzojGeAV9UJSi5rx18P4LOySyAT4eXIBxCRhHWCIwyb6tW4zqdNvnnWsnXzFJhwcBReIDF0IWrDlUUP9Is55hLsL3HFnVdGONlHKyYwG77zKWtwr8coGk804KR1aSQEnbSaRDWexcyxwGiGBM/Fo8lh5wg6UDZiiXoAMON9WUM8Lh3AAAEdgIqIAfEFCRRR+Qqmq7AwZE7c878a2gt8L+RAQADgeAGQAkkCnpuaBVSZwPAWOokuoRF2NDlcT5EHjAtCqJP2DGUCXVA9YBDwAOAA8A7VFdA6GX4DHtORI/v2Zup7/Rrzdl9MaSqQldghN0Hx10EhrzcOar33TepA/3HxmBizUt0Ts5iI0LM7IHoJACFVp+LXdgYkygD1CGJL7c8hQ5zrw9WwwhB3gAAMK8zY6MeGNBcn8xOHdM82gAAA8FCQEGa2vx+DwGLslMAAAQY3YBVAEBBwIIAAC4AAAAAAAAAQAAFmdXvA==", + "results": "497 7879869f,497 7879869f,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,296 ed1a706,ERR_ZSTD,497 7879869f,ERR_ZSTD,497 7879869f,ERR_ZSTD" + }, + { + "frame": "KLUv/QQunAcApArkYoHVPfGacpXFdyijjEvC8m8paAC0AnaLJfJCeF+bnI6aEaHZC5KgqGuNmbyat0Smf5fCMs8r9i91f0jJkKFmeuP96NrEvtKQsb1yrSUEj8LYEpWrKV0E4dcRANyXPL9a3hznABPhhVyHxYeTB7mF7OdZWqGSysp34PblP7NCS6GQw9UckTDRHfExYAdjwLLpjRopDAwOyZhAtta9cUne2sK1ZGMD1t3QDiGoET3jzFwwQYVarRtARBE0fADZMdBsahY9wLm9AXDsHP+CsJ8EN6ivXshjyige++y8AmYz9qVDZQTi557bb6rE3xZQhsnhFAMA4kULGZAlSRoD//8XxXV3Vya4qirjh5Wpm/3jKQEb/2d1vFe+3XUnz0J3b9NxO+U0EaDQaiZUau8GMEKjVN4BphCxcNGutgtJzEXW4KGIPnAg1EgYSAHUQ/wFjJ9M9GWAUgJKCQBuAAAARAAA6WcBVBIFB15qAAAg7AIAogMLF6CnkA7bbwMAAHiflt4GAADwPklqs08MuovXXOeaobW8+a4fRORBVu1Nr3gQiHDWTJhQHdAqmwd4i0uHq2G5AqFKWi2IMRyoJ6XIYR4QeIJbEMI/sSFUpVAnCAAAih0AACFgAP9LBPQ=", + "results": "946 77613f00,946 77613f00,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,753 b00c93be,ERR_ZSTD,946 77613f00,ERR_ZSTD,946 77613f00,ERR_ZSTD" + }, + { + "frame": "KLUv/UQ+3gPMAwDCQgkagCVJ0gFzFjSaCDahK3dhpoAZqCIhPSs1A2Ft5r5Q0Zbxp3YPIyAwCCET9dg46QAEJN+DaEZEKJUjFnsawJ1gUQwGP43loL2CtuBcmaXbyMzwVp7awOdgI4MDQEi+ewTtEmEDtd0IHNYe3QSoTjLY7QoGkKkrAAAAAAAAHAMAOgKIABKwpZAODAMJOOa0gikstCShogsCAAIAAgC3VbsPrT4NlQYXqJDWbRswSEgOPiCUiB2VpNAaP8QG5lE4LxT8xf5MVVes8A+Rlp21gzbtvjQHIpQx2iDXQ4l1WeQrYZ8MgxYA2QCspMs=", + "results": "1246 9619c71,1246 9619c71,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,1246 9619c71,1246 9619c71,290 e53bead5,ERR_ZSTD,1246 9619c71,ERR_ZSTD,1246 9619c71,ERR_ZSTD" + }, + { + "frame": "KLUv/YQqCQAAAEkAAENDQ0NDQ0NDQ8UpJk4=", + "results": "9 48ef29bd,9 48ef29bd,prefix_unknown,ERR_ZSTD,corruption_detected,ERR_ZSTD,dstSize_tooSmall,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,6 ec519e21,ERR_ZSTD,9 48ef29bd,ERR_ZSTD,9 48ef29bd,ERR_ZSTD" + }, + { + "frame": "KLUv/QQp3AQA5ogWjhEREREREREQEwATABMAoCJeD68+B5NdADL9vreceY/fAUB8WXlD+AjfLjQ4fyWiBwrsgQEHcq2joWpJLOwUTTaox47vmMEBc4cAIiCVqe6K4vUP/VfJcWcBGahwSkkFFUrSbkCKlKUOkJqJNrYDOM5BxzL/r7BNdU5UQA5Qw/U3IYOkXeq37c2HDOYVHEd7n9IuYbhQzQg5BQCaw6DmShMJHknKPNzgQk8xzJ1AXJWSxh4zKvhqHtQARqC/b4LwhxknzTtPtyFN95dWCfPPdoD7+S2meVYeAc9btdJ8bCcUfcRH7/FqQlTbvFGeLbb3zR9om1TRwZtveNcB0cvbk0L2ROFf7LeBzbGEIqdLZeMDj/k+RuVCub9zA3KXx4speNnt8aQ+A4eXPopkGEpt9vQcgZ5H6FfAIdO84l5xqhme+HPcOls=", + "results": "444 3f5168e1,444 3f5168e1,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,363 612befef,ERR_ZSTD,444 3f5168e1,ERR_ZSTD,444 3f5168e1,ERR_ZSTD" + }, + { + "frame": "KLUv/QRV5AIAceQ9iKBvkM8BNkD3WWR60E8AV8gheVyGEdlxhcfyMpLjFL6ekRcWFJeioZlybKtMDeMGdpNiqYcPJEAJ78RpVfA+VU1edLqqM3FjcxVgQ7Xzrp1dyFTpztsd537cAABIn+/AN5V9IlPaArgUMMD/FOA34H8SSwjABAhFAAAIswF0AQZOE1cEupE=", + "results": "266 8b2f35e0,266 8b2f35e0,corruption_detected,ERR_ZSTD,266 8b2f35e0,266 8b2f35e0,corruption_detected,ERR_ZSTD,266 8b2f35e0,266 8b2f35e0,srcSize_wrong,ERR_ZSTD,266 8b2f35e0,266 8b2f35e0,216 f2bf9995,ERR_ZSTD,266 8b2f35e0,ERR_ZSTD,266 8b2f35e0,ERR_ZSTD" + }, + { + "frame": "KLUv/QQA3AMAtggcJJAp6XmXKgZLMRbFGUTdNL0VrlQTKIRAIQSKM6TOIDzadWkCHxEAEgARAFeXi/o8putkL67TY+hq1/cPLLYMCm3jNWMF+7rsG3epncQWiO10m3Q0iaXPjfE7FLl2dxiutqw8QmJF353U7F5Lj3C7AQFUGgYsC5EoDAUARhcmHJAlBSl0x2+J2ExYN2PwoooPnHDmCO/Eji+L+IgdAB0AHgCzJNGl4KaUGVONJ8yuABOKShIFbRYFhaiw8oGqMj1DiD30GAGrTebICgR44kCgQ6GtW2MqoyIfZMAUDzn4MBpIkrBiEGTFU6B7QyA5IYJwssTXyn0XBRIFPSvwzUCoMByOAGnRdVA+Cky8ZSaKrKLQ6In3QAUBVBAGC9IcAQDzRAYbPBZPQzmTYlYRMvAMfdOQQuGBsQOBBcUkAVQRBxg2AfoJAMmMAgCGBAgN0BU6CwFA4wEA4AusBQMAAwADAPd9X6/u9///B1/93QmokFJQqfLbAxAocaU9ECgooPinosr/UOgB24Efa0am4+G7MFr/Sk8BpU/MYwJ8AQATCgb/d373//ff2/P+///u///vu+fvv9vf/wMCpBQw4L//E/AF/gMsgqyWSWAAEMAHAF3ZwxwkKgG6a96VjJ+YPiZxG08/d25inL56u72LHiWdwvr141wTXr+73oyjnauVbXkUiji9bjP7foY84KOTfY47mLWeKfpL7oSkfr8S3iGsaDxUWjYSgqskvHD8gmajBhG3BlbugQNq7TRqx2avztIuMZedWVqsQTcfqVjC8oVg8QPDlrUJrQbpFy+Uowl8R6JfLl50UiWEfwcsvgU+c47p2K3CFC92Iqe7yykqzNn283Vu7Fzkk8EZjyar9Rzv0KwRc0jxMI+PRFjT0obxhhPKmTJFmj9tATu3OcTBb4wRLvlrqsq8n3ktFiGMCNgibox0qSEJu8pBIAcAzqL6REe/ycnJZaWCKy+bx9queiy2PvYttRrEc69WsDANcOWU09+Wt2kDmnh6o5x6mnFWDKGWxXaP3B8VSgqRWokJV2+hskaM3y+q23w/UBLBWNGnsRQuwcyZ+cEG1Tz3bqysGNy+B8p3XZyY2meTnum70cAxrtTecd4SDJnNbxD0oexqhHXeaWKJWrQyeEOsEXPPf+ZrFlScw0zPWcnJyfBdtf54ePVTLgHVAVw3pL3s3gwA5Pdv4WZTLiLK+6FlHQOUd67F8y8orJSeHJrHRxN/w0AhX3yiw2smra2XLRYhycnJohAAFngAAN7oHxLcma24Pd5cc9g7gjkBAKmNI7P5zRYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFprK1eDxmSxY2Pm51d4=", + "results": "3570 9be53c42,3570 9be53c42,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,2796 1cee85e5,ERR_ZSTD,3570 9be53c42,ERR_ZSTD,3570 9be53c42,ERR_ZSTD" + }, + { + "frame": "KLUv/UQTegGUCAAuEIAkAI4RERERERERECEAIQAhAIIZ8ih+8XC70fyZ97+4MzLQQ9JpfJRDX09m8Coc0cpNFelkCalRxg2t081TntgrrpIYzcIvjGrsk1rbTCgMk5DGH9bzqICg/Hd31QIglJHrNz7jnvi/awpldCPWC570w8uWEUOdrs5ZT3mx/ECX4IWlEVTBdGSLmhcKUAdyD+p17oEXMqABKXWKBRRUkhTYHSDEGMTWDpiiSFWAiOujBBwDQY8RsECQJwXIwQZssJyABQGjEgAF4TLYZTCOQsYpGrY2hdqZs0oihQ8ZlEGYBuY55m9rQoYB2cGaBMxBiTBSPIDgPpgBgBIHfZAUKN1EY+lGaIUJwAHQQZlgAK4LgAAA/TUCsutNXY3imC4Z4cr/omAAAC10ZktjUBUyDAoLDAEFAMewWNopArPuxaR+UKwBspfAadYAFXeCR1ef6K+b3ApiUyPEHsXqIweJF7HZtUJ3xpevtxTQuYpTwWGlz1ngmtCfkfdTWUFJMbC9lUBX/22yTI2EhcEHF7dgze1idMNqMR8dixc1uevTtBF7kYlph/xMBp66Dt6K8I/1wrAxjRtcXDIBR8R/S1Gq4dt1c2WDwMtvI0/UEN1mG0NRoJirYcacO36h", + "results": "634 63c1e83f,634 63c1e83f,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,dstSize_tooSmall,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,516 912cf686,ERR_ZSTD,634 63c1e83f,ERR_ZSTD,634 63c1e83f,ERR_ZSTD" + }, + { + "frame": "KLUv/WS6AgAAAIUHAEJkGAvgaQwAo0AJaC0A1P/3/viev//99/9v//7/9//7///9/3/5/99/v/7+Pff/r9/ffff/P//5s9fWT//3///9N/ed53/9/b/5+m+/f/9f/tf9//d+//7799Pf+7//3/a//3wvqAEVU0gRgZCACJFIIFKUYyCEEMQU2QPhKEOKSUhBUUGOAb3MlFUDQVtB0F1pF3WJspfJuPomLtY8orO1s6UdM6E8OcRs6gU3zFxBsLYUtABT0f1TrRAZW1E1g07me4yJVcRDSwSA7oNpizD2qI4Ocir34EOYDqn2tPIsVtTIcp8c/H0kqIvLi3IXBgw9ibY=", + "results": "954 2ef0a76a,954 2ef0a76a,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,954 2ef0a76a,ERR_ZSTD,954 2ef0a76a,ERR_ZSTD" + }, + { + "frame": "KLUv/QRKAkAAlngMAA53SiMX+4ZHFd/U9lTVXbE2LaLWmjExxwxP4oF2B8+NqPixOI5Oo/qgpkwx3504U+lBUahtHCkPjcjPCFzIEw0oqLfDfA+o5DlTMQeblzfbwoJFaGC0PCB6O30MAWhvPMZuVDzCDrs1Ot1o1cixCXrB1lSaOLpgIJw7yHDRpc0yar10Bsa3022nNLXRWVH1OH9uD1o48p4bzYpO0FYRvgkm6JLXpjEvXarSPaf9i30hCduONYK9666QPOJHF/+JJbwbblFo4ao3HJ3BOB3OqeuTJHX7qkYbPJKOz3bVZRBHn59ACg0IwlOsAUstsGyoTyqwxynoLAxWY2EYrcVc1HwrdhI0Nv17mcRa3Vz0krUa7goOEAKZT5z92l4hTdtuCf/I6yMYG5u5g14sar7UaoUxRxEAiZcG2AlCd3bARZkxYZ3cVyyBrWs99GXAHK+YVb/hVkBR3xlzJ0Vy6pRkcnzYVrbbRiILisX465OHif+1lI3qrDx8sNOr8VXWY9Ft6xfkoYYfZ8I1NX0SN+Ze0DouAAl1DQBuZ0BNABHAJY0B/++r2Addnf5n/wtwA0oARwBIAP043+9zPlr7p3b6j+R3vsL5//lfB3m2927Xu2397Wtzq67BfS96/3ef91vP399b1vXxuv7vr31q//Ha/Tev36fV8H/966ubr3sO16/3vvize3wa977V/6/l6/OP7/9Pz/svX7Om9/xP7/O9XtL4/OT35b6/f+X/9W1+p//8n//mmV3///ZVX/3bf/fbv7OnbwTv9c/+7o8/P9zde8/Xvuv373vZnen9LK+3+77bdk756e6zelb7vPt3hc3f7e8y637nf///tyw/5av73f4p2f46/H/fX1v+2k/9+z26+/9e3f3/ePdV/f/1S59fd/+/fv3U/7XOXz6XV3f+94/+l33v+d/ueH//fn/r/uv/X/1c9ZuV//wu3hvXv7a7/iQoME6ckFwPMTQLjYApNcEEtcYGu3dwnBIA8P00EOvYHIU4YhcskUKS/yq9zMDq5+cX2+9iZKjZ9hY5FwPmaEJNiU7r2XJsMyipx58+htIuT+blrGu78bH12Ia1o2DBifELeKbPbrNG4uh8Ww2iisS5YVXaAhCgNg==", + "results": "5900 1171bfdf,5900 1171bfdf,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,3926 d5e9c65e,ERR_ZSTD,5900 1171bfdf,ERR_ZSTD,5900 1171bfdf,ERR_ZSTD" + }, + { + "frame": "KLUv/QQWIAwATv9A1IMxhFoQcteST+n6v5Q+jTwazih3M2WxzQcgFXmztmYaW+rPDyaIwYRTR27GNG+fcOulUOPpM2l210UVlULJbe94xKf/Q+ZlFbg5qDa/nas0A32mQxJf2qL7i/poWNRp6mCErmHLw0rirzNlJ94bEbgTHgkuEEzvOfROAFXS4eGTBinfvpweHg59qP//lob01ZQxnYyEuEniBVMCXK+yc8a873KnR8vrtCqNolO4oQxlhsOF/uSmO7RiLTQ1Dd38zpxgDLZWLTtB8b9m5wXz7VDCAfrCJk9X3HPiiNvigjADJLfSlaVp1WYOY4GG0ke7hjvHqwB51yOAvafaSBhQ5whiKFdaf1K9NSB9vtpMIgTaIqLxeasWXFG4wXIGs3fR6O3kS8Bur84oK6xMinuYdqYyq00TDRmw2iLCfzRH3RTi/LO8jp24hmxTFCoYcepDm0W69nlZdsK4LX5KRFGf25dXCVmAPmfqtaRtO6Zmy5jr6ESEbyp0hWbWA+DqJ0QyziszAMOBM1Sj", + "results": "2025 575c0567,2025 575c0567,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,294 1b560bc8,ERR_ZSTD,2025 575c0567,ERR_ZSTD,2025 575c0567,ERR_ZSTD" + }, + { + "frame": "KLUv/QRZ1AcACg4MAo4REREREREREB0AHQAdAJGeNBZffkFItQtY4ubHZ/KdMMsr5DbvkFTL80YBvbq8kOgTZprkizirfH1H6K/st13TEhGzsCwFKAERRs9w3y2r3Nw5Ksmvt52Z0pS6HVyFuLiaIknFAQ17OaiFTjCK1kcQHB0/q4kPhGYz8rB+neJ//t8BLYCxLAwNzdSEiWOsBngI1Lz8MyP4Ualdzmy5zA4MMYAX0HgASxqAZMOGDVtxaCwLRzR9wAcLaSz4HEByg4Uy4WgeYwweNyzr0omJ97gAB8Bil3YAvxIZdfPaISV2BqCY8JY6fddOpsCiYs680qU904blwmQCAJLGCBHAJY0B/+2qHn/Q7/rda0HNED21nzv++6+/r8j//v/+/fY/CaiQKr7MVLsDUOJSsA8QUFFh5esBSDjoCA6M1JWwjQEh3e3oHgooAABqAwwNC1UFALQHYPrAq638UQgtyFcyuj0obk20628mFMAe8ErAThOwD8efbzf2uBWLVnJhEtZfsWDW2lFcvyie8DuwTlM4DE4UwyuPfFNVwYIAZxfjZjejJQGFV8tmQWN9kTgJdx0Wii6t5bys9I3V96Z067vq9gR6qDK8SG8utAo16Y9uDaBAQoOQES3L0BxAUkSBmR3AAXd1G++2N3TDUtKBARWgD7g3pkx2RQkIncETBZFKzA==", + "results": "804 528c00ee,804 528c00ee,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,617 98c27afb,ERR_ZSTD,804 528c00ee,ERR_ZSTD,804 528c00ee,ERR_ZSTD" + }, + { + "frame": "KLUv/YRDVAcAADIJADDcBAB2cx4I8Dm/u3f6JgYbABoAGwD///z/8///8////////5//5////////8n//z////////////////9//v//////////////////////8/////xz/v//f/7/////////8///Of//P////////////////////8//////////A6gU4P+3VQCrAvwDE/BcWf0BdPUfVKAJo+yjA/LSI3wGADbTKo4RERERERERECcAJwAnAOSGAHdGnDyiyF0V3j5thIcU3Mx4F5V4gtECQ6Nst3P8GKlgGq3BHIdQP+6lv+XFZu9D6Wz8bvQaVODThTgE0xUxljqfyHd+FEzNw0qYGuRGtJPzkFCfaj0nUa+HD+u5fEDeEa9ADVHfpr1HPTC1cE4FFr5XFT2eUqp7p2tXEKhacfJkdxZ9j8+Uxxq5cGei7mm/br9GzcpHguCoAQooECixZR/wtgcAgBEl1ezmiqXlgUYc9jgMgpy8udQ4bS0EACLCBhSgJUkHP/uR/7/qf/fXVTVeVWNRDz/+d3u9PCaIEJeGAxGAJKmRANEENAvsXd+zBLyto6WnnOgUM4mwTy5oEVlOsG0Qw7LcFY9BHr7LIyUDZclq4Y6Qwg1mCXAai5rLW8xCKPkP0YAZohi4ApBWuCUsanURpQigVqCWnHiri8a2WAHWJimW", + "results": "1876 b64ec80b,1876 b64ec80b,checksum_wrong,ERR_ZSTD,1876 b64ec80b,1876 b64ec80b,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,1472 e5639bd6,ERR_ZSTD,1876 b64ec80b,ERR_ZSTD,1876 b64ec80b,ERR_ZSTD" + }, + { + "frame": "KLUv/cQrLAIAAAAAAAD8AgCaBfgAjhEREREREREQDAAMAAwAfj+dcnc36sLGdIkbywc3HacSpqeY/Z0UGKgYhWYEGTRWdqQRJ1+aZL1VtltbJwEEoBQogEAB/xsY6B8TMFdWAX8PIHMALoA2htotEHwCAHsEtAAKAAoACgBbw3bhfi1Cm/oBxWDFMP+E3XJWAdAd/0PSAbnORwERqJOHpUgeNB8FiHDie/kuHhBI8f9z4ufCZQ6IryHFEWHKGYgXTgEQAAAUCRwAACldAAAAAAAAAAsAAEkGvOwr", + "results": "556 70816d4e,556 70816d4e,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,242 58b68857,ERR_ZSTD,556 70816d4e,ERR_ZSTD,556 70816d4e,ERR_ZSTD" + }, + { + "frame": "KLUv/QQ+wgYA0qQCADoEzACOERERERERERAJAAkACQCOtsIFU5BNQhQvMSPDljWBKBN120JEsmI6oRlMfJqG/lJ/igEHqBAwgYlBkUVLBxB26QPADzqgBTrNC0Wf7fT7AuEBAG+arG/0VLCEOQd8Jaxbngd+94ehRAvQFptZfjgZrgPXHpEvFP+ZNesmZJ101+X6OgMuMwG+1yzT0TEyDcIcK+Y=", + "results": "365 13c09b09,365 13c09b09,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,365 13c09b09,365 13c09b09,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,328 3738326,ERR_ZSTD,365 13c09b09,ERR_ZSTD,365 13c09b09,ERR_ZSTD" + }, + { + "frame": "KLUv/QQLWRMAp3QqC48IE/ZuFUkcaUoZ7PmkBhek+eJyUqU+GGLKLdSDsR52dYVEibrYO42AeDJzuCWCLmjssQKJd0cf66oOn5H6wVHqgXOPmHh46BCEtpvlmrSla+v2SzUQEUoQUEbnsNN7mnOaaHtsNnww1R06DsXI7Ixl2IV4O3ZG12DRX2Irw+pJTxCJh5dnGbfdjQywJU2ZgZYn2F2e0pu2pm3nGTBK8Jq0mjiJxbgtXFqyq+n9xNpPzFAP+CJmqAzkaDR5RFYgtbYtYGl1NBNRWgHIyCEA1KX5a0aXzAle3aWi4QuuyGax6G0JUL5hOAuk6I8v82zXLFFMC8eEL9NAkn9gT2LwzyXMe2n6l2YIZMijKy9O7NO5nB63D/eo/x8WanDErs8ZF46ZZ9VOMUx3hOUPszktyFKUgTjdOaIKV1yB2NkJojMDDJfx9PC1dq/0/CPA1o9JxnXPUXLlxQ4sG4w1Y1EjtPmDBgc6+66Q/WNGWPfilbu6X54h9kAV47gWXuGFiUfM8R9V5PIIFTvcBiLxD2foyA4ACAtvO3T4zUUgpv4Mhh2hrcuVbK4U7O8eRN2FnUc9bE0UChTleYXIHGA3h2AHUbkhXuU24L1ejlM4259E+l7PK0BNfMo2A4K5x9ru/6SQZWmj6Hi/9n6/iz7bI5+b1CcVG6+hjuscdsfl19MLUdnLdiAg3lLiksw9AEP+AIZsA/F3isABLwe6KX7Ppv9rtxQXuQchVxuuRJffXGi5VMV7EvaIbKq0hpFnKcp6GrbSVf+hyhwvakEjrxvhPUMufssJIJdrwJTfyb3AxOlJJInnCPMI8AkUNmZqzsQ=", + "results": "619 303423ca,619 303423ca,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,465 afc3c79,ERR_ZSTD,619 303423ca,ERR_ZSTD,619 303423ca,ERR_ZSTD" + }, + { + "frame": "KLUv/cQBJAEAAAAAAAAAAAC0AAA5uQMoE9CV1R/UKqBVAf4BAGASQ16uvAUAFAjBV7oyFL1SNVcdfx6uajzfCOKpRTj/iAnQKPtgBMN4to8teKdNKyhk6WMmkEAJ/DhFTsakv+ui+PLGSAzvaC4hvxs52ioCYhR8Gd9F9X/vrtecciEQG6brlIRVnTYand1J4kpaHzK0jGUYraTf66ZULUER5uBzvecw09U82E6khpISqFDMTGCVFpbdahCmVDuAVE0t1g4Q+pFkxHoIG7Wlvt/R7E4zgHnbJ6rfwmQCgxJGshbERAAAYdkBVAAFFDAAAAABAAABZ0sf", + "results": "292 228895d0,292 228895d0,292 228895d0,292 228895d0,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,24 d65cde7a,ERR_ZSTD,292 228895d0,ERR_ZSTD,292 228895d0,ERR_ZSTD" + }, + { + "frame": "KLUv/cQ1PwIAAAAAAAAAAAC9BQBqGQwBBvA5SM7foQ4ADQAOAH/y//////////////8B///////////////n//8/n/8///////////8D///8////////z///fzioEaVjztAEE4nIbEFaDSCGIEOpHuAP4I++UFDVBy6zQCqeGz0sObzEVsOks54GugGmYTtQMB0z2gMmPJfT79LqjuGPxF0BjxZOh8xJPV2b+vuSSgwdaTIwPY2+wIv+vxPAObpTCoTHnll1lxxjRgGbmj5B", + "results": "575 ebde8e3e,575 ebde8e3e,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,575 ebde8e3e,575 ebde8e3e,corruption_detected,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,575 ebde8e3e,ERR_ZSTD,575 ebde8e3e,ERR_ZSTD" + }, + { + "frame": "KLUv/cRS6gMAAAAAAABEAwBosNgWMryd2hhDCpevtkKosA8whhhG7QFAe2Agz4FDT9HbdF2Ij0gTEN1BiCCxndR6ipKkXRgEuKAbR5sAFNH0Ndldvm/7TF1AU4h9MhLm59rQOBiIC75JzVxWrkSlDebkQBQtqiTxFqQIAIKdPY4REREREREREKuniaq7VFB9l1xAgmyJlnXYDc2wf4B8tu+X/4Y9b3SQAAL292jyDIRZM49Ubsav3QOkxdNV0WnE/9R5aMLbYqkzgiZ3YLQYNbu+zWUDGnbRPUlqea9q84GMSJ+rnJiNRl+NCg/hU0yyaYiPIe58NFdJz5hT73mF16UQL9wdehHoiMHauRxH8K801Xo1ZBR2oaEVc9MLsi/9miKqFIHuJWkrPefhvE2ezDeqrZ2kr/phrmPv+lRDspTvVMv56zPNpmUZUqgFwxA8KNYvRoms+my+mf51PUBz7TGoDBR5NfWUnVxMnF+MVOdRIR6oAQQoE/CwBvwBFDCgwMD7D4EAa0bATJ4lsADIU2QBACKIBgjwOf9CzztBAfv////99/+//////////9//ApQU4C9gwM9/CAKPywQgxAEAwPdxBdbtUX/fy8LbOm2uSYSjJzEnVpsHeA2ksFKQPICiiWbaAwARtuNRnaGagC78haD07IadrQPRAQC85QEOBwcPCg4MsLCwsDKooxnNyskvCozCZAsDHLcFBAgPRA3zGd4KODwLDk7+1gwLCe0CCQpR3wkKLHH6/Q==", + "results": "1002 666d59b1,1002 666d59b1,1002 666d59b1,1002 666d59b1,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,741 60b8501a,ERR_ZSTD,1002 666d59b1,ERR_ZSTD,1002 666d59b1,ERR_ZSTD" + }, + { + "frame": "KLUv/QRQ9AEAAoMJG4AlSTIcc4bE1BlIj4zouRgjgDIETdDvzkjcCZ99+5/fmrCJrQMEKBMga8AfFICrAf4BRG2IgB8W2Q4cAAA5LAA0AQBw5r/VXS/ShTr4Lk5hQWkFqBCjzPMQLn0QOtG7vAcqpYADGQWlE0QAAAifAVQABxuIIAAA0NDQBQEAALMC2Vo=", + "results": "171 3d2ec0b2,171 3d2ec0b2,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,73 9e7af546,ERR_ZSTD,171 3d2ec0b2,ERR_ZSTD,171 3d2ec0b2,ERR_ZSTD" + }, + { + "frame": "KLUv/QQMWAAAUHH5X5kSDxMFQpsCQACPKjcAbOUVAAKwYo4REREREREREPjtS6yt/u+zohG+n9k08Kv/7L/Y+2+qlHr5aZjajfyt8sMJhVfn0n0zQd6Oju9A5e2PqNWNb/WgJchdpZfPXGvwDX5SGfMQe9AsA6KrYIjVAZTId+mf7bKkmXdO8SyesIQny4VTtXQt5Jdu2L4v+pZFdvxRLu/4OngpBTdPEcf7L/CMXbCUehFqjDdulLe4VYTCAZ46SPJjgVb0IzegBUtUMCuMGMtcltXadNQbHNr41qViG6hajCRdFdGzZ3lHPMcgIiI56y3C51lM9vyplQemAK1zydjH3JmwgPfezcUiTCPFD8Z9k8YRcHwHVdNVS6k/HJmxlvds0Cr0KgkPwbjFBUPz2mNqvfGLiutdyxYUiaXxvnPJKPKhL0cAhpjBKvt89In5gFdP7jeoFcDRLvCEN/3Q4mrDxlED7ERDTNkYytObf1IKw+Hw+QeysP1yFp/ZAKQQuQNLKyUXy4DV/y3xTbQPmJVHmok9Gzbrnuu/DYp4WdTBPV14O9/JkfQBUgGAgSgwSigyUvnAlm0O8aQUuKSB6HD0nmIHWZGADkA4ORJHWFhJ9tJy5DvnKXme9C6YAEg75hNLIUwQhlfYGmDQB+cCYyyt6t4vWFLjSeiArG7qawaByuI88cbllGoVDR2XFK2BGD6TtY4qHEkRixzAjuMIuUEGrJmu7grkNrTBF64lZjyaOhpah3XGL4KldKQGkW9BRKxv4dF38/3pesYM+qd1oagfWPhPeLVi6POA9BO8oYWEKbwgiPZQlkuaejOE1LaBTt7g1LEGQmmEWZYoDBgBUvw/+1prOYgfZMfea0UTNERg/VIwFyEjBvrS1ylgV52MBJnRApNhEEms0lIUjyugql0KiEfumYEoMXGXMZpxrMXcjpvwlK4i2VigjJ+txRHy5ICUSRBvIvj3C4fTI3Bd", + "results": "5138 cf7af287,5138 cf7af287,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,dstSize_tooSmall,ERR_ZSTD,3824 9445837,ERR_ZSTD,5138 cf7af287,ERR_ZSTD,5138 cf7af287,ERR_ZSTD" + }, + { + "frame": "KLUv/QQ+sgsAHwAAADgBAF9F8mTnozQh7hvomZ5ahCMn+1j0AxWs7oPk1fIxlT+pQB8fHx8fHwgAANgAAAAIAABsuAAAHx8fHx8fHx0dHx8fHx0/qUAfHx8fHx0QAABEPgAAACkAADoXct2Aw/rpLg==", + "results": "445 cb41d271,445 cb41d271,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,430 bc19cb1c,ERR_ZSTD,445 cb41d271,ERR_ZSTD,445 cb41d271,ERR_ZSTD" + }, + { + "frame": "KLUv/QQ4SAAAyCvSNdqdTXLQxAEAegW0ABKwJekMK6jImOAUxiinmiQBy5UFAAUABQA7/3YHA5l7c6Mu+48zrwHn3LhPBQFUGQMJlAMSAQCocQEAmoc8fgwWWx0fCoHPjqB6iw345F4dssRifWPgwFuvANvSgr3NOABTrTH8nCylzyUUTMw=", + "results": "188 33992c7a,188 33992c7a,9 bb7810e,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,158 5a8470f1,ERR_ZSTD,188 33992c7a,ERR_ZSTD,188 33992c7a,ERR_ZSTD" + }, + { + "frame": "KLUv/YQVPAEAAMwDALZGFiCQKembDfs/SHc/9n9Odz/2f05p8d+dSVq8MxMkLbJbIg0ADAALAHelBiVJ1pl1IPTQTqtfzlQtWdoio5O5pdoHL6wpcE99s9eFz2y3YVJjVaTHlAm2kksRCIBQYkoFVdlqdwGT8gTcawHrEUIBYM9zoi5YFBjtAwAihhYnkCdJB+uIgaXzLr/ac34gtnTe5Vd7zg8oJui8y6/2nB9QTND9ai4Ex3s1HkXKR4t+ORXZ1V5cB5nSVd7Q4x2ybykpDIgZNNwsvqQAwWChlR0pM4lSN41umLUKiNCCqliZ9ntw1Dt9BGJLAeJgQTSGiAXaIBgyJp/rA5T+85I=", + "results": "316 607b1dee,316 607b1dee,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,174 5b061cb9,ERR_ZSTD,316 607b1dee,ERR_ZSTD,316 607b1dee,ERR_ZSTD" + }, + { + "frame": "KLUv/YQaBwsAABQDAGoFKAEgkCVJOvPK97LDx5zbkbGdlj52dtCZdrt+ubNXMLcjdcwJAAcACgBz8j+K6G3qggTzlk+Pq1Q3roEiHlQ6yQlkBoMWlWkbCEE7DQMoEzBXVn8UgID/ARZ4GoARwAQKVAwA2idEBY4REREREREREFEAUQBRAJLNGoZGALbJ1szKbRklazXxkM+TTdkVeRYTBmPJ77XVy+pvINRB0ohVbTlv7d8ScuG57Dvk08eX8fYyexUiQ7tuQjXKTKroPQYATLo7RFPVAVGXEV2J0ll6xQxITFpw1a9GXJ2vzgGOH8TwhvRBRyvICb+2vej9V2yJjJnI8rgufMsDgjbBJVPqJvccQtTmk/I+8rJ4xQ8s7pplmEdhQ4ohAZA0pbwX6Dv6oFWdywSquid0bpBESrflXKLZUY8hdpnjPDwSCffHwNxx7xJiTpLp/mdWFeh2nyDBenVkcC+DyXr90VyR58ofWoJzttkrwPdmASq8eUxAj9LpGWZBhqnCWznL4CHf2rE8w4ccAfwSh8+MYBd2lFjBse+R8srPW83dI0RCYlU5rgpUt94MS8ZiHdbivJtmqr5IIVlM2AZrvhQSiICCmZixeGYyWwdg9gENVvkaX1GJZFkuXaWY4X5Eid4RzXm+xE/nfD8mdrCoK9PVbEMxDZQLAFo0+AEH8Dlm/10jCBwAHAAcAPn/5/P/P//////5/////////+n8////P////w/z/P//P/+fP8//+f8///////////////Pn/z8/z//M////n/////+c//n/////8z///+c//3/m//9//v8//5z/D///P/P////////z/3/+5////wdgqJEgpBQjSIAQGSKhSTQcMEYkZKb2ATE9d0IlhWU7dvQL3W5dzED7kWD0VOKsYaJOLErLyzTI2SGQFtST3xYrv7+W3M+VlFJ7MUBMaYHAgddQC1hwXA0RxIGRPV2aRk7IoVpO2CRtVIIVnwu+A48NA6XeGTKafj1yxuLyOIGHTrxRXdjw0xtDX5IRt0Csq2WiCZZcSIjR7L0+SlekIXxzi6RniZYnateh3mD94axrACg1wEcB94uF+0zdQqYqnOhJ0QZxAQUys7r2IaHM4TtxejcsUd3QPNDBAbdsyMZt/6NZCig34EOFmerlFgknec41BgAeCoAYAI4REREREREREBUAFQAVAEaDPU8WUeeVcdX7fsH5ttXCabD/HuRnYPOcBaUF1olWSVIpgUDFwhfWGo1nd/KqX9/pGzVaLHBbw1Uy+0aJEhvcCL8vrKMsVxiXjD1Sp+3CYlQBIKARrXJKVYVa2PoGQEqsMR/CBKgDgVCRgHsUoDRpXLhoxNqLJXwEN9PlqZDyRURMPJ3QnUnENCym91BxtocelHTmkqEWOEP9EgwDoLkXkGtY08bKdBeTzKYDcFGdBwlrgSSS", + "results": "2823 af736cc,2823 af736cc,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,808 5c2cac83,ERR_ZSTD,2823 af736cc,ERR_ZSTD,2823 af736cc,ERR_ZSTD" + }, + { + "frame": "KLUv/YRa4AAAAIoAANndAgASQwMF4A94gVH///////8DLqiBPg7tZxAoYpDaA0B9OIIS78n2wQ4sNgrY1c+AZNAYBP5RaoFA9i3TvTpKxwoILM0F/XsTSbkxacK/S2KxBHCmZlA8FKpN2a4TZWeEWQ==", + "results": "224 2ee8b580,224 2ee8b580,17 c5258290,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,17 c5258290,ERR_ZSTD,224 2ee8b580,ERR_ZSTD,224 2ee8b580,ERR_ZSTD" + }, + { + "frame": "KLUv/QRPXA8AOg/UAArg6YCVigDggtEBCQAJAAkAzf+//Xf9//99v/+/v+337/8f/7P9//////0H///p//+/338VgNmIwFhNWgNwEhIlHa5l/ka8kbasESz6ujDpEaqrZNxfYUOCNgoPLSZd3ZLCgyZeg7Qg9IIC5CuErivSqZUg2LU5IZqN8PwiEkwI4QRiQlQQB2DjQMVjCuqEDAkWZJMRK3nVRJhAWAp0XWghEymCa7eBi0JDzYdWBhHqRhC2NmNpNorwGBAPT6iFTbaJdjdRUIIP7VqWRe79KqBNuat9gUs1chdCKU3y7vdf1luS8e4UUOqAMBy0TWbcvAZBnQQwvcZbwt7Tl/ArWCxBqpVTIv2a0sUjsYY1QEwL0Eb/CMuHTHudCeJ3XEzra42AyGrBQcUQ67UGbiEoJSgSSk6bncGnlhLca91Kjx/VmtXwbuq37WYQa55h2m2RVmh7Y6PBLUtGbYQZkbCFbXSjmSXJcNg6hXFWe7cL246FCa1HNpQNYWJk67TsNhbPIPa9CXfew5NTC7kLhGH7UBb4xVR4sMhM+yCtA6ZBFx550Mkon5n2kAUHJrmh/h0M9ByJCmIC4D54geFAbWB0RAOKZOGpEI4CbyVzldKiVFWBqjqdRa7o3KYAfEeFmVczKHLNzPlUnAy0AQBuB4AKAA7QZTgAAAhEBFWmAJDoAQUABQAGANv/1/4P7/X9+h+/7/vdoQH/62/+AwFUFwoPXCGFDACaL+wDGaAVFCSNAVZV16c2AOA9GQDwfg8AeC+tKt4yADkANgCZiICO2BBDLt/66o3Kxtv2JWW1QB6NeoaBCnDfRDZZhCrfhGfKF2LWoHqYjFV7Me58E/vMcxwYhB1oMTpBEeCYgaDfdiNir0LTJXZDJwIDI9hU7HMCLYMcCSCw7CCjIgIuRJOL22dT58FrMrcZ1GG3F8qLVFcHAI9qYBTb6oGwiIpgdtgG/H4MpkUbg3cEGDdHhdgjznyeEjJCH84mFAS4F7/9GCYkCwc+7gQHZhkSJgKKLKocYZQkSJAk3sDJFdjWIFhiNygaQlogVj3puIZ4BCuAc6l2jSK4bTAkqLGYESqBMUZohIZBFU7tATBIIBKOvQMxFCExyggMMcYIjAAZgTLGBBJMMLEnE24DFAKYJ1UnDL3TedBiwLGmIbYE1Ya5EnAf3CMbdcIIJEbYkT+jGivSNvw/WZqatknrGAYRILHaRp7iQyvtIwswv4b1vICBjBA64Yt1QaHUih22IYYgF5titYDF2v6UMo8lTNcFkg==", + "results": "3280 ef966a16,3280 ef966a16,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,corruption_detected,ERR_ZSTD,checksum_wrong,ERR_ZSTD,corruption_detected,ERR_ZSTD,1430 d208aa94,ERR_ZSTD,3280 ef966a16,ERR_ZSTD,3280 ef966a16,ERR_ZSTD" + }, + { + "frame": "KLUv/URQrgAAAABtBACaFTgBD8AlHf//////7QDxN/C0Gg4ADwAOAL9/9szu7/+/tr7//v8DN+v/857/87dX99bv9lMP1/2zs35+7//7733+/wvfvb7+//f/j6//dXc/FaiQJJpIJpSwappzMEYwzBkfwFjHcAGS14oGyyNtZf0rox9BJdlH/kPuAWb4VKR9bnSPlVFV4y9aeq6Z6imu", + "results": "430 26ba970a,430 26ba970a,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,checksum_wrong,ERR_ZSTD,430 26ba970a,430 26ba970a,checksum_wrong,ERR_ZSTD,0 2cc5d05,ERR_ZSTD,430 26ba970a,ERR_ZSTD,430 26ba970a,ERR_ZSTD" + } +] diff --git a/test/js/bun/util/zstd.test.ts b/test/js/bun/util/zstd.test.ts index 4af68cf27e9d..468251b5c682 100644 --- a/test/js/bun/util/zstd.test.ts +++ b/test/js/bun/util/zstd.test.ts @@ -13,6 +13,7 @@ import { bunEnv, bunExe, isASAN, isLinux, libcPathForDlopen, rss } from "harness import { readFileSync } from "node:fs"; import zlib from "node:zlib"; import path from "path"; +import frames from "./zstd-frames-fixture.json"; // A hand-written empty frame: magic, a descriptor without a content size (so it goes through the // streaming decoder), the window descriptor (the decoder allocates a window of 2^windowLog bytes before @@ -108,6 +109,75 @@ describe("Zstandard compression", async () => { ).toBeLessThan(10); } + describe("few literals whose codes are long", () => { + // How often each of how many symbols is there, out of 1536: a code of 5 bits for 48 times, of 9 bits for 3 times. + const upTo8Bits = [ + [48, 8], + [6, 192], + ]; + const upTo9Bits = [ + [48, 8], + [6, 140], + [3, 104], + ]; + let x = 1; + /** In a random order, but that each quarter, which is a Huffman stream of its own, starts with the longest codes. */ + function literals(mix: number[][]) { + const input: number[] = []; + let symbol = 0; + for (const [count, nbSymbols] of mix) { + for (const end = symbol + nbSymbols; symbol < end; symbol++) input.push(...Array(count).fill(symbol)); + } + const isRare = (other: number) => other >= symbol - mix.at(-1)![1]; + for (let i = input.length - 1; i > 0; i--) { + x ^= x << 13; + x ^= x >>> 17; + x ^= x << 5; + const j = (x >>> 0) % (i + 1); + [input[i], input[j]] = [input[j], input[i]]; + } + for (let start = 0; start < input.length; start += input.length / 4) { + for (let i = start, from = start + 8; i < start + 8; i++) { + while (!isRare(input[from])) from++; + if (!isRare(input[i])) [input[i], input[from]] = [input[from], input[i]]; + } + } + return Uint8Array.from(input); + } + + // Where a stream ends within a byte, which is what varies, is how many bits there are before more have to be read. + it.each([ + [8, upTo8Bits], + [9, upTo9Bits], + ])("of up to %i bits", (_, mix) => { + for (let round = 0; round < 40; round++) { + const input = literals(mix); + const frame = zstdCompressSync(input); + expect(frame.length).toBeLessThan(1500); + expect(Buffer.compare(zstdDecompressSync(frame), input)).toBe(0); + } + }); + + it("with a table that later blocks use again", () => { + // Blocks of a target size end with a match. + const input = Buffer.concat( + Array.from({ length: 4 }, () => literals(upTo8Bits)).flatMap(part => [part, part.subarray(0, 64)]), + ); + const ZSTD_c_targetCBlockSize = 130; + const frame = zlib.zstdCompressSync(input, { params: { [ZSTD_c_targetCBlockSize]: 1340 } }); + // The blocks are after the magic number, the descriptor and a content size of 2 bytes. + expect(frame[4]).toBe(0x60); + const literalsTypes: number[] = []; + for (let at = 7, header = 0; !(header & 1); at += 3 + (header >> 3)) { + header = frame.readUIntLE(at, 3); + literalsTypes.push(frame[at + 3] & 3); + } + const [compressed, treeless] = [2, 3]; + expect(literalsTypes).toEqual([compressed, treeless, treeless, treeless]); + expect(Buffer.compare(zstdDecompressSync(frame), input)).toBe(0); + }); + }); + // Test with known zstd-compressed data describe("zstd CLI compatibility", () => { for (const { name, compressed, original } of [ @@ -273,6 +343,261 @@ describe("Zstandard compression", async () => { } }); +describe("decompressing frames whose size is known", () => { + let x = 0x9e3779b9; + const noise = (length: number) => + Buffer.from( + Uint8Array.from({ length }, () => { + x ^= x << 13; + x ^= x >>> 17; + x ^= x << 5; + return x; + }), + ); + const repeated = (period: Buffer, length: number) => Buffer.alloc(length, period); + // Blocks that are stored, blocks of one byte, blocks of a few long matches, and blocks of many short ones. + const pieces = [ + noise(140_000), + Buffer.alloc(270_000, 7), + repeated(noise(1000), 140_000), + repeated(Buffer.from("one two three two one three "), 20_000), + repeated(noise(60_000), 200_000), + Buffer.from(noise(30_000).map(byte => 97 + (byte & 3) * ((byte >> 2) % 3))), + noise(9), + ]; + const whole = Buffer.concat(pieces); + + // "raw", "rle", or how many sequences a compressed block has. Or else how its literals are written, or how many it has. + function blocks(frame: Uint8Array, ofLiterals: boolean | "count" = false) { + const descriptor = frame[4]; + const isSingleSegment = (descriptor >> 5) & 1; + let at = 5 + (1 - isSingleSegment) + [0, 1, 2, 4][descriptor & 3] + [isSingleSegment, 2, 4, 8][descriptor >> 6]; + const kinds: (string | number)[] = []; + for (let isLast = 0; !isLast;) { + const header = frame[at] | (frame[at + 1] << 8) | (frame[at + 2] << 16); + at += 3; + isLast = header & 1; + const type = (header >> 1) & 3; + if (type === 2) { + const format = (frame[at] >> 2) & 3; + const bits = + frame[at] + + frame[at + 1] * 2 ** 8 + + frame[at + 2] * 2 ** 16 + + frame[at + 3] * 2 ** 24 + + frame[at + 4] * 2 ** 32; + let literals: number; + let count: number; + if ((frame[at] & 3) < 2) { + count = format === 1 ? (bits >> 4) & 0xfff : format === 3 ? (bits >> 4) & 0xfffff : frame[at] >> 3; + literals = [1, 2, 1, 3][format] + ((frame[at] & 3) === 0 ? count : 1); + } else { + const width = [10, 10, 14, 18][format]; + count = Math.floor(bits / 2 ** 4) % 2 ** width; + literals = [3, 3, 4, 5][format] + (Math.floor(bits / 2 ** (4 + width)) % 2 ** width); + } + const [first, second, third] = frame.subarray(at + literals); + kinds.push( + ofLiterals === "count" + ? count + : ofLiterals + ? ["raw", "rle", "code", "same code"][frame[at] & 3] + : first < 128 + ? first + : first < 255 + ? ((first - 128) << 8) + second + : second + (third << 8) + 0x7f00, + ); + } else { + kinds.push(["raw", "rle"][type]); + } + at += type === 1 ? 1 : header >> 3; + } + return kinds; + } + + it("the frames have blocks of these kinds", () => { + expect(pieces.map(piece => blocks(zstdCompressSync(piece, { level: 1 })))).toEqual([ + ["raw", "raw"], + [1, "rle", "rle"], + [1, 1], + [4], + [1, 1], + [3647], + ["raw"], + ]); + }); + + it.each([1, 3])("stretches of noise, runs and long matches in one frame of level %i", level => { + expect(zstdDecompressSync(zstdCompressSync(whole, { level })).equals(whole)).toBe(true); + }); + + it("the same in a frame each, whose blocks are let to write more or less", () => { + const frames = pieces.map((piece, i) => + zlib.zstdCompressSync(piece, { params: { [zlib.constants.ZSTD_c_windowLog]: [17, 10, 20, 12][i % 4] } }), + ); + expect(zstdDecompressSync(Buffer.concat(frames)).equals(whole)).toBe(true); + expect(zstdDecompressSync(Buffer.concat(frames.toReversed())).equals(Buffer.concat(pieces.toReversed()))).toBe( + true, + ); + }); + + it.each([4097, 4100, 4222, 8200])("long matches from %i bytes back, which are copied in parts", period => { + const data = repeated(noise(period), 40_000); + const frame = zstdCompressSync(data, { level: 3 }); + expect(zstdDecompressSync(frame).equals(data)).toBe(true); + expect(zlib.zstdDecompressSync(frame).equals(data)).toBe(true); + }); + + it.each([ + [0xffff, 0, ["raw", "raw", "raw", "code", "code", "code,same code,same code,same code"]], + [0xffffffff, 0, ["raw", "code", "code", "code", "code", "code,code,code,same code"]], + [0xffffffff, 2, ["code", "code", "code", "code", "code", "code,code,same code,same code"]], + ])( + "literals that take many bits: the least of the bytes of a number up to %i, shifted by %i", + (mask, shift, written) => { + let x = 0x9e3779b9; + const kinds = [300, 1500, 3000, 6000, 20_000, 400_000].map(length => { + const data = Buffer.from( + Uint8Array.from({ length }, () => { + x ^= x << 13; + x ^= x >>> 17; + x ^= x << 5; + const y = x | ~mask; + return Math.min(y & 255, (y >> 8) & 255, (y >> 16) & 255, y >>> 24) >> shift; + }), + ); + const frame = zstdCompressSync(data, { level: 1 }); + expect(zstdDecompressSync(frame).equals(data)).toBe(true); + expect(zlib.zstdDecompressSync(frame).equals(data)).toBe(true); + return blocks(frame, true).join(); + }); + expect(kinds).toEqual(written); + }, + ); + + it.each([1, 2, 8])("short matches from less than 16 bytes back, in one sequence out of %i", others => { + const data = noise(60_000); + // Where the first 24 bytes are a dictionary, these two reach into the part of it that matches are looked for in. + data.copyWithin(24, 9, 13); + data.copyWithin(29, 14, 18); + for (let at = 33, i = 0; at < data.length - 40; i++) { + at += 1 + (i % 7); + const offset = i % others === 0 || at < 320 ? 15 - (i % 15) : 16 + (i % 300); + for (const end = at + 3 + ((i + 2) % 31); at < end; at++) data[at] = data[at - offset]; + } + const frame = zstdCompressSync(data, { level: 19 }); + expect(blocks(frame)[0]).toBeGreaterThan(2_000); + expect(zstdDecompressSync(frame).equals(data)).toBe(true); + expect(zlib.zstdDecompressSync(frame).equals(data)).toBe(true); + + // After a dictionary, and in a buffer that is written to from its start again. + const params = { [zlib.constants.ZSTD_c_compressionLevel]: 19, [zlib.constants.ZSTD_c_windowLog]: 10 }; + const dictionary = data.subarray(0, 24); + const afterDictionary = zlib.zstdCompressSync(data.subarray(24), { params, dictionary }); + expect(blocks(afterDictionary)[0]).toBeGreaterThan(32); + expect(zlib.zstdDecompressSync(afterDictionary, { dictionary }).equals(data.subarray(24))).toBe(true); + const inSmallWindow = zlib.zstdCompressSync(data, { params }); + expect(blocks(inSmallWindow)[1]).toBeGreaterThan(32); + expect(zlib.zstdDecompressSync(inSmallWindow).equals(data)).toBe(true); + }); + + it("frames one after the other whose matches are in a dictionary", () => { + const dictionary = noise(20_000); + const piece = (i: number) => dictionary.subarray((i * 7919) % 19_000).subarray(0, 8 + (i % 9)); + const data = Buffer.concat(Array.from({ length: 1_500 }, (_, i) => piece(i))); + const frame = zlib.zstdCompressSync(data, { dictionary }); + expect(blocks(frame)[0]).toBeGreaterThan(1_000); + // There is room for all three, so each is decoded in one go to where the one before it ends. + const decoded = zlib.zstdDecompressSync(Buffer.concat([frame, frame, frame]), { dictionary, chunkSize: 1 << 17 }); + expect(decoded.equals(Buffer.concat([data, data, data]))).toBe(true); + }); + + // Then a sequence waits until 8 more are decoded, so that what it copies can be fetched by then. + it("long matches from more than 16 MB back", () => { + const c = zlib.constants; + const piece = (i: number) => pieces[0].subarray((i * 7919) % 139_000).subarray(0, 40 + ((i * 31) % 300)); + const data = Buffer.concat([ + pieces[0], + Buffer.alloc(17 << 20), + ...Array.from({ length: 4_000 }, (_, i) => piece(i)), + ]); + const params = { [c.ZSTD_c_windowLog]: 25, [c.ZSTD_c_enableLongDistanceMatching]: 1 }; + const frame = zlib.zstdCompressSync(data, { params }); + const sequences = blocks(frame).filter(block => typeof block === "number" && block > 300); + expect(new Set(sequences.map(count => Number(count) % 8)).size).toBeGreaterThan(2); + expect(zstdDecompressSync(frame).equals(data)).toBe(true); + // In one go to where it is wanted, and by way of the decoder's own buffer. + expect(zlib.zstdDecompressSync(frame, { chunkSize: data.length }).equals(data)).toBe(true); + expect(zlib.zstdDecompressSync(frame).equals(data)).toBe(true); + }); + + // The C library keeps some of them in the output buffer, unless that has room for all of them. + describe("blocks of more than 64 KB of literals", () => { + // Noise of 6 bits, and after every 6 to 30 bytes of it, up to `end`, 5 to 8 bytes from `nearest` to `furthest` back. + function literalsAndMatches(length: number, end: number, nearest: number, furthest: number) { + const data = Buffer.alloc(length); + for (let at = 0, x = 0x9e3779b9; at < length; at++) { + x ^= x << 13; + x ^= x >>> 17; + x ^= x << 5; + data[at] = x & 63; + } + for (let at = furthest, i = 0; at < end - 64; i++) { + at += 6 + (i % 25); + const from = at - nearest - ((i * 7919) % (furthest - nearest)); + data.copyWithin(at, from, from + 5 + (i % 4)); + at += 5 + (i % 4); + } + return data; + } + const params = { [zlib.constants.ZSTD_c_compressionLevel]: 6, [zlib.constants.ZSTD_c_windowLog]: 17 }; + + it.each([128_000, 40_000])("with matches in the first %i bytes", end => { + const data = literalsAndMatches(128_000, end, 16, 56); + const frame = zstdCompressSync(data, { level: 3 }); + expect(blocks(frame)).toEqual([expect.any(Number)]); + expect(blocks(frame)[0]).toBeGreaterThan(end / 40); + expect(blocks(frame, "count")[0]).toBeGreaterThan(1 << 16); + expect(zstdDecompressSync(frame).equals(data)).toBe(true); + expect(zlib.zstdDecompressSync(frame).equals(data)).toBe(true); + }); + + it("with matches in the older part of a buffer that is written to from its start again", () => { + const data = literalsAndMatches(900_000, 900_000, 70_000, 125_000); + const frame = zlib.zstdCompressSync(data, { params }); + expect(blocks(frame).length).toBe(7); + // The buffer holds 3 blocks. + for (const sequences of blocks(frame).slice(3)) expect(sequences).toBeGreaterThan(3_000); + for (const literals of blocks(frame, "count")) expect(literals).toBeGreaterThan(1 << 16); + expect(zlib.zstdDecompressSync(frame).equals(data)).toBe(true); + }); + + it("with matches in a dictionary", () => { + const all = literalsAndMatches(250_000, 250_000, 70_000, 125_000); + const [dictionary, data] = [all.subarray(0, 125_000), all.subarray(125_000)]; + const frame = zlib.zstdCompressSync(data, { params, dictionary }); + expect(blocks(frame)).toEqual([expect.any(Number)]); + expect(blocks(frame)[0]).toBeGreaterThan(3_000); + expect(blocks(frame, "count")[0]).toBeGreaterThan(1 << 16); + expect(zlib.zstdDecompressSync(frame, { dictionary }).equals(data)).toBe(true); + }); + }); + + it("a frame that makes more than it says is refused", () => { + for (const piece of pieces.filter(piece => piece.length >= 135_000)) { + const frame = zstdCompressSync(piece.subarray(0, 135_000), { level: 1 }); + // The content size, 4 bytes of it from 64 KB on, after the magic number and the descriptor. + expect(frame.readUInt32LE(5)).toBe(135_000); + for (const size of [134_999, 131_072, 70_000]) { + const lying = Buffer.from(frame); + lying.writeUInt32LE(size, 5); + expect(() => zstdDecompressSync(lying)).toThrow("Decompression failed: DecompressionFailed"); + } + } + }); +}); + // zstdDecompressSync sizes its output from the frame header when the content size is // present and at most 16 MiB. Anything else is decoded in a stream into a buffer that // starts at a guess (the input size, or the 16 MiB limit) and grows as needed. @@ -777,15 +1102,14 @@ describe("sync compression argument handling", () => { }, 60_000); }); -// x64 builds target nehalem, so zstd picks its BMI2 kernels at run time. It used to ask CPUID in -// every CCtx and DCtx init. CPUID is a VM exit under a hypervisor, about 2 us each, two per init. -// arch_prctl(ARCH_SET_CPUID, 0) makes CPUID raise SIGSEGV on the calling thread, so a context that -// still probes the CPU kills the child. The first context of each kind is allowed to probe. +// CPUID is a VM exit under a hypervisor, about 2 us each. The C library once asked it twice in every +// CCtx and DCtx init. arch_prctl(ARCH_SET_CPUID, 0) makes CPUID raise SIGSEGV on the calling thread, +// so a context that probes the CPU kills the child. The first round trip loads what else may probe. describe("zstd context setup", () => { const canFaultOnCpuid = isLinux && process.arch === "x64" && /^flags\s*:.*\bcpuid_fault\b/m.test(readFileSync("/proc/cpuinfo", "utf8")); - it.skipIf(!canFaultOnCpuid)("asks CPUID once, not for each context", async () => { + it.skipIf(!canFaultOnCpuid)("does not ask CPUID", async () => { const script = ` const { dlopen } = require("bun:ffi"); const zlib = require("node:zlib"); @@ -1055,3 +1379,1682 @@ describe.concurrent("Zstandard HTTP compression", () => { expect(echoed).toBe(testString); }); }); + +// The compressor is a port of zstd 1.5.7, and gives the same bytes. The expected values are from the C library. +describe("the output is that of zstd 1.5.7", () => { + const xorshift = (x: number) => () => { + x ^= x << 13; + x ^= x >>> 17; + x ^= x << 5; + return (x >>>= 0); + }; + + // Words, copies of earlier parts at all distances, noise and runs: something for every part of the compressor. + function sample(size: number, seed: number) { + const next = xorshift(seed >>> 0 || 1); + const words = [ + "function ", + "return ", + "const ", + "zstd", + "\n ", + "{", + "}", + "(", + ");", + "await ", + "=> ", + "length", + "0", + ].map(word => Buffer.from(word)); + // Room for the last piece to overrun. + const out = new Uint8Array(size + 2048); + let at = 0; + while (at < size) { + const kind = next() % 16; + if (kind < 7) { + const word = words[next() % words.length]; + out.set(word, at); + at += word.length; + } else if (kind < 11 && at > 8) { + const distance = 1 + (next() % Math.min(at, 1 << (next() % 20))); + const length = 3 + (next() % (1 << (next() % 10))); + if (distance >= length) out.copyWithin(at, at - distance, at - distance + length); + else for (let i = 0; i < length; i++) out[at + i] = out[at + i - distance]; + at += length; + } else if (kind < 15) { + for (let n = 1 + (next() % 24); n > 0; n--) out[at++] = next() >>> (next() % 8 ? 27 : 24); + } else { + const length = 1 + (next() % 300); + out.fill(next(), at, at + length); + at += length; + } + } + return out.subarray(0, size); + } + const digest = (bytes: Uint8Array) => `${bytes.length} ${Bun.hash.xxHash64(bytes).toString(16)}`; + const c = zlib.constants; + // More than one job of the least size there is, for the workers. + const corpus = sample(700_000, 1); + const large = corpus.subarray(0, 300_000); + const small = corpus.subarray(300_000, 303_000); + + // Not -1: node:zlib takes that for a parameter that is not set. + it.each([ + [-100, "167967 45693a3bc89d2838, 2112 a1f04bdd2c75d853"], + [-7, "65641 25bb487794d2f45f, 1081 6841140ea417de55"], + [-2, "55329 a7cd8da33a4cce8b, 954 cb3a52b86b5193c7"], + [1, "46828 2a61d18b13a9df88, 824 3a3b2b799ed70f7b"], + [2, "45933 2795a6f87171faac, 811 bd7ae68cb7d95862"], + [3, "45122 344d3c35e5ad0340, 810 e35cec5c3500c856"], + [4, "45093 7634242fc968e2c3, 822 6b45b45a5d921bfc"], + [5, "44424 daecc1abec496995, 810 4489026b2c29b8dc"], + [6, "42948 a879387b2601910d, 809 b15c4cf338e1d77a"], + [7, "42675 1f9ad80ad0656d09, 804 603c9d665714a2"], + [8, "42562 26911127c1e54142, 804 603c9d665714a2"], + [9, "42562 26911127c1e54142, 807 bc69857972f7737c"], + [10, "42353 d560f70047c278d, 807 bc69857972f7737c"], + [11, "42124 cc37ec9a45f8d164, 808 95559a23325a524d"], + [12, "42124 cc37ec9a45f8d164, 806 5984ef401fd2141"], + [13, "42364 fd1a09c03715fc5f, 807 6648f2fa58a397d4"], + [14, "42287 87ee023408ff3e30, 807 6648f2fa58a397d4"], + [15, "42233 109ef580b5685e85, 804 52073852c548e55e"], + [16, "42522 3293e655be1b0330, 807 6648f2fa58a397d4"], + [17, "41809 7cb274a8772cd4cc, 806 b609660f6b3d21d9"], + [18, "41253 d616eadc7ae1a5de, 804 52073852c548e55e"], + [19, "18189 44c06af9c0a7b97f, 804 52073852c548e55e"], + [20, "18125 7c0ef33ba30541a2, 804 52073852c548e55e"], + [21, "18125 7c0ef33ba30541a2, 804 52073852c548e55e"], + [22, "18125 e1b7fc6756c67456, 804 52073852c548e55e"], + ])("at level %i", (level, expected) => { + const compress = (data: Uint8Array) => + level > 0 + ? zstdCompressSync(data, { level }) + : zlib.zstdCompressSync(data, { params: { [c.ZSTD_c_compressionLevel]: level } }); + // The highest levels are slow. + const data = level < 19 ? large : large.subarray(0, 140_000); + expect(`${digest(compress(data))}, ${digest(compress(small))}`).toBe(expected); + }); + + it("where blocks that are stored are between blocks that are compressed", () => { + const noise = new Uint8Array(200_000); + for (let i = 0, x = 9; i < noise.length; i++) { + x ^= x << 13; + x ^= x >>> 17; + x ^= x << 5; + noise[i] = x; + } + const data = Buffer.concat([large.subarray(0, 150_000), noise, large.subarray(150_000)]); + expect([1, 3, 6, 13, 16].map(level => digest(zstdCompressSync(data, { level })))).toEqual([ + "248567 de53cc88abf776bf", + "245529 c13322b7116144a", + "245428 8fdcf68c52aa046d", + "244844 ad91ca42d335a19", + "244843 e1282705ef659832", + ]); + }); + + it("where blocks are nothing but literals", () => { + // The lesser of two bytes has no matches worth taking, and compresses. At negative levels literals are not compressed. + const data = new Uint8Array(300_000); + for (let i = 0, x = 5; i < data.length; i++) { + x ^= x << 13; + x ^= x >>> 17; + x ^= x << 5; + data[i] = Math.min(x & 255, (x >> 8) & 255); + } + const inOneCall = [1, 3, 9, 16].map(level => digest(zstdCompressSync(data, { level }))); + const streamed = [-5, 1, 3, 9, 16].map(level => + digest(zlib.zstdCompressSync(data, { params: { [c.ZSTD_c_compressionLevel]: level } })), + ); + expect(inOneCall).toEqual([ + "290920 e4a05b68a9febcae", + "290984 c04074b7a3113a0c", + "290920 e4a05b68a9febcae", + "290920 e4a05b68a9febcae", + ]); + expect(streamed).toEqual([ + "300018 3aa98e9a8e545874", + "290920 e4a05b68a9febcae", + "290936 bdaff5fb067b174b", + "290920 e4a05b68a9febcae", + "290920 e4a05b68a9febcae", + ]); + }); + + const level = (level: number) => ({ [c.ZSTD_c_compressionLevel]: level }); + const piece = (n: number, size: number) => corpus.subarray(500_000 + n * size, 500_000 + (n + 1) * size); + /** Records of a counter and the same `size` bytes. */ + const counted = (size: number, count: number) => + Buffer.concat( + Array.from({ length: count }, (_, i) => [Buffer.from(`${i}`.padStart(8, "0")), piece(0, size)]).flat(), + ); + + it("where records differ in a few fields", () => { + // What is before a position ends at a field, and may be the same again after it. + const flags: Uint8Array[] = []; + for (let i = 0, x = 7; i < 2_400; i++) { + x ^= x << 13; + x ^= x >>> 17; + x ^= x << 5; + flags.push(Buffer.from(`${x & 1}`), piece(i % 4, 80)); + } + const flagged = Buffer.concat(flags); + const dictionary = flagged.subarray(0, 30_000); + const cases: [Buffer, zlib.ZstdOptions][] = [ + // Hash chains. + [flagged.subarray(0, 16_000), { params: level(5) }], + [flagged.subarray(0, 16_000), { params: level(6) }], + [flagged.subarray(0, 16_000), { params: level(7) }], + [counted(100, 148), { params: level(8) }], + // Rows. From level 11 on, it is binary trees at these sizes. + [flagged, { params: level(6) }], + [flagged, { params: level(7) }], + [flagged, { params: level(9) }], + [flagged, { params: level(10) }], + [counted(100, 2_000), { params: level(10) }], + [counted(1_000, 200), { params: level(10) }], + [flagged, { params: { [c.ZSTD_c_strategy]: c.ZSTD_greedy } }], + // Rows of 64 that fill up. + [flagged, { params: { [c.ZSTD_c_strategy]: c.ZSTD_lazy2, [c.ZSTD_c_searchLog]: 6, [c.ZSTD_c_hashLog]: 12 } }], + // What is before the window is left behind, and what is in the dictionary is not: hash chains, then rows. + [flagged, { params: { ...level(8), [c.ZSTD_c_windowLog]: 12 } }], + [flagged, { params: { ...level(8), [c.ZSTD_c_windowLog]: 15 } }], + [flagged.subarray(30_000, 40_000), { params: { ...level(7), [c.ZSTD_c_windowLog]: 14 }, dictionary }], + [flagged.subarray(30_000, 40_000), { params: level(7), dictionary }], + ]; + expect(cases.map(([data, options]) => digest(zlib.zstdCompressSync(data, options)))).toMatchInlineSnapshot(` + [ + "249 f907d3f525b7aa81", + "243 de9085e2cdfac856", + "235 d670b708be78ef4c", + "225 854a885c1d8be4a5", + "1692 1de698fb782b56f9", + "1616 66239976b21d3320", + "1517 f00cc0a5a2e99559", + "1476 a94b01a6e016ad19", + "2460 ecb935f7ffd006fb", + "626 ee78b0ff57f42d3e", + "1497 d88328c87f134e60", + "1540 80ad275d9b5cea5a", + "2457 a55760737c90d6d0", + "1727 77731bfed55e726", + "104 61304d0540ed07df", + "115 f5c7d2c89a55ace1", + ] + `); + }); + + it("where the binary trees lead to what is before the current segment of the window", () => { + // Fields on both sides of 0x80, so that which record comes first depends on how bytes are compared. + const fields = (size: number, count: number) => { + const records = counted(size, count); + for (let i = 0, x = 5; i < records.length; i += 37) { + x ^= x << 13; + x ^= x >>> 17; + x ^= x << 5; + if (x & 3) records[i] = 0x78 + (x >>> 28); + } + return records; + }; + const short = fields(100, 800); + const long = fields(1_000, 80); + const window = (level_: number, windowLog: number) => ({ ...level(level_), [c.ZSTD_c_windowLog]: windowLog }); + const cases: [Buffer, zlib.ZstdOptions][] = []; + // After a dictionary. + for (const data of [short, long]) { + const dictionary = data.subarray(0, 30_000); + for (const n of [11, 13, 16]) cases.push([data.subarray(30_000, 80_000), { params: level(n), dictionary }]); + } + // In a buffer that is written to from its start again. + cases.push( + [short, { params: window(11, 14) }], + [short, { params: window(13, 14) }], + [long, { params: window(16, 15) }], + ); + expect(cases.map(([data, options]) => digest(zlib.zstdCompressSync(data, options)))).toMatchInlineSnapshot(` + [ + "3714 c51791f02b5ce6e2", + "3821 722f4103ea02c85", + "2886 7b33aadb1b4c6928", + "3000 f11a3c6cbaee134b", + "3238 8a4a2b286c782c0d", + "2254 d22a01e8a758fe5b", + "6664 47c6eea66da02b71", + "6845 9bca37c99d215a5d", + "3911 a4190024e0f41380", + ] + `); + }); + + /** + * Noise of so many bits. What follows the dictionary, which is the first 30,000 bytes, has pieces of it and of itself + * among the noise, three of the one for one of the other. + */ + const withPieces = (bits: number, shortest = 4, longest = 15, mostlyOfTheDictionary = true) => { + const next = xorshift(5); + const all = Buffer.alloc(42_000); + for (let at = 0; at < all.length;) { + for (let n = 4 + (next() % 24); n > 0; n--) all[at++] = next() >>> (32 - bits); + if (at < 30_000) continue; + const ofTheDictionary = next() % 4 !== 0 ? mostlyOfTheDictionary : !mostlyOfTheDictionary; + const from = ofTheDictionary ? next() % 29_000 : 30_000 + (next() % (at - 30_000 + 1)); + const length = shortest + (next() % (longest - shortest + 1)); + all.copyWithin(at, from, from + length); + at += length; + } + return all; + }; + + it("where the rows of a dictionary have candidates and those of the input have none", () => { + // Rows of 16 entries up to level 8, then of 32 and of 64. + const digests = (all: Buffer) => + [5, 6, 7, 9, 10].map(n => + digest(zlib.zstdCompressSync(all.subarray(30_000), { params: level(n), dictionary: all.subarray(0, 30_000) })), + ); + expect([6, 3].map(bits => digests(withPieces(bits)))).toEqual([ + [ + "7108 e6ce3932970435bf", + "7106 33de66f30821afb9", + "7105 94b0e055b688b475", + "7105 94b0e055b688b475", + "7105 94b0e055b688b475", + ], + [ + "4799 cea9496d4270b5a1", + "4733 b9472094002e88fd", + "4685 d7691fb52ce63095", + "4623 2532b3477d8a5f2d", + "4624 856abd3827ed1af8", + ], + ]); + }); + + it("after a dictionary that is indexed in several parts", () => { + // Rows take 65,536 positions at a time. The pieces are from all over it, then from where a part ends and from its end, + // whose last 8 positions are not indexed. 3 bits to a byte make for many candidates. + const next = xorshift(9); + const dictionary = Buffer.alloc(200_000); + for (let at = 0; at < dictionary.length; at++) dictionary[at] = next() >>> 29; + const fromAllOver = (i: number) => dictionary.subarray((i * 104_729) % 199_000).subarray(0, 8 + (i % 30)); + const atTheEndOfAPart = (i: number) => dictionary.subarray(65_536 * (1 + (i % 3)) + (i >> 2) - 9).subarray(0, 40); + const data = Buffer.concat([ + ...Array.from({ length: 400 }, (_, i) => fromAllOver(i)), + ...Array.from({ length: 48 }, (_, i) => atTheEndOfAPart(i)), + ...Array.from({ length: 6 }, (_, i) => dictionary.subarray(-7 - i)), + ]); + // Hash chains, which are indexed by as many bytes as the least length of a match, then rows. + const chains = { [c.ZSTD_c_windowLog]: 14, [c.ZSTD_c_chainLog]: 17, [c.ZSTD_c_hashLog]: 17 }; + const all = [ + ...[5, 8].map(n => ({ ...level(n), ...chains })), + ...[3, 6, 7].map(minMatch => ({ ...level(5), ...chains, [c.ZSTD_c_minMatch]: minMatch })), + ...[5, 8, 10].map(level), + ]; + expect(all.map(params => digest(zlib.zstdCompressSync(data, { params, dictionary })))).toEqual([ + "1781 f75b9689d167d764", + "3049 d39dde15a799187e", + "2458 6a9eaca5014ab4c7", + "1454 2f96fdade038991b", + "3566 626fb7b8cb1df2dd", + "1480 cf7dbfcd5f5f068f", + "3077 7c69c63d013fb421", + "1385 c99be48f15f658e8", + ]); + }); + + it("where there are literals before a match in the trees of a dictionary that is searched in place", () => { + // Few trees and few steps down them, so that it counts how many are left for the dictionary. + const crowded = (n: number) => ({ ...level(n), [c.ZSTD_c_hashLog]: 10, [c.ZSTD_c_searchLog]: 2 }); + // From level 16 on, the tables of the dictionary are copied for more than 8 KB. + const digests = (all: Buffer) => + [level(13), level(16), level(19), crowded(13), crowded(16)].map(params => + digest(zlib.zstdCompressSync(all.subarray(30_000, 38_000), { params, dictionary: all.subarray(0, 30_000) })), + ); + expect([withPieces(6, 3, 6, false), withPieces(5, 4, 15, false), withPieces(5, 3, 6, false)].map(digests)).toEqual([ + [ + "5719 406ca55e1f7a8e51", + "5723 638b7b1d34f5e7e4", + "5723 638b7b1d34f5e7e4", + "5775 e3985bd1b7f3cc06", + "5777 f4d41a4a3fe9753d", + ], + [ + "4036 957012d8fbbad463", + "4032 4a494ca0f71ad17d", + "4032 4a494ca0f71ad17d", + "4216 9e7fdaed4122c373", + "4205 a05cfe9758c76e6c", + ], + [ + "4895 2d4708a58b44a327", + "4897 1f4aad7c4526b8fd", + "4897 1f4aad7c4526b8fd", + "4919 5671cafb2cb68402", + "4917 92674bc1d6487eb", + ], + ]); + }); + + // Noise of 5 bits. After 30,000 bytes it has pieces from so far back, in turn, with one byte of each altered. + const withAlteredPieces = (distances: number[]) => { + const next = xorshift(5); + const all = Buffer.alloc(63_000); + for (let at = 0; at < 30_000; at++) all[at] = next() >>> 27; + for (let at = 30_000, n = 0; at < all.length; n++) { + for (let k = 1 + (next() % 12); k > 0; k--) all[at++] = next() >>> 27; + const from = at - distances[n % distances.length]; + all.copyWithin(at, from, from + 16); + all[at + 4 + (next() % 8)] ^= 1 + (next() % 3); + at += 16; + } + return all; + }; + + it("where there are literals while a repeat offset leads into a dictionary", () => { + const compress = (all: Buffer, n: number, from: number, size: number) => + zlib.zstdCompressSync(all.subarray(from, from + size), { params: level(n), dictionary: all.subarray(0, 30_000) }); + const digests = (all: Buffer, size: number) => [13, 16].map(n => digest(compress(all, n, 30_000, size))); + const fromThreePlaces = withAlteredPieces([25_000, 12_345, 50]); + const fromNearby = withAlteredPieces([20]); + expect([ + digests(fromThreePlaces, 8_000), + // The tables of the dictionary are copied for more than 8 KB from level 16 on, else for more than 32 KB. + digests(fromThreePlaces, 33_000), + // A piece goes on over the end of the dictionary. + digests(fromNearby, 9_000), + ]).toEqual([ + ["2519 24ce676ed4360f6b", "2482 b0633ed5d69d3d65"], + ["10359 10c42f22fc137208", "10027 1fd5fb810e81ee70"], + ["2689 ac0ddc2b63fa3cf0", "2593 665843657aba87c4"], + ]); + // The offsets that a frame starts with, 1, 4 and 8, lead into the dictionary for its first 8 bytes. + const starts = (all: Buffer) => + [13, 16].map(n => + digest(Buffer.concat(Array.from({ length: 8 }, (_, start) => compress(all, n, 30_000 + start, 300)))), + ); + expect([starts(fromThreePlaces), starts(fromNearby)]).toEqual([ + ["1093 e9f462ba5074daef", "1061 d826405e9ce9c5d2"], + ["1044 9e83e945e3585cf6", "1018 d57ef3f0c34b3189"], + ]); + }); + + it("where there are literals while a repeat offset leads before the place where the buffer of a stream starts again", () => { + // Of the distances, one is just out of a window of 16 KB and one just in it. + const all = withAlteredPieces([16_450, 16_300, 9_000]); + const window = (n: number, windowLog: number) => ({ params: { ...level(n), [c.ZSTD_c_windowLog]: windowLog } }); + expect( + [13, 14].map(windowLog => [13, 16, 19].map(n => digest(zlib.zstdCompressSync(all, window(n, windowLog))))), + ).toEqual([ + ["38845 7721ae005eb10446", "38929 da0285f338ad0055", "38929 da0285f338ad0055"], + ["32494 857cd1a443168536", "32293 4e37f66382f2b3a3", "32293 4e37f66382f2b3a3"], + ]); + }); + + it("where the kind of data changes within a block, for rows of 64 entries", () => { + const next = xorshift(11); + // Many positions start like another one and go on otherwise. + const fewSymbols = (size: number) => Uint8Array.from({ length: size }, () => 65 + (next() >>> 30)); + const noise = (size: number) => Uint8Array.from({ length: size }, () => next() >>> 24); + // What starts like another position goes on like it. + const records = (size: number, record: number) => counted(record, Math.ceil(size / record)).subarray(0, size); + const text = (size: number) => corpus.subarray(0, size); + const rowsOf64 = (strategy: number, hashLog: number) => ({ + [c.ZSTD_c_strategy]: strategy, + [c.ZSTD_c_searchLog]: 6, + [c.ZSTD_c_hashLog]: hashLog, + [c.ZSTD_c_windowLog]: 17, + }); + const cases: [Uint8Array[], Record][] = [ + [[fewSymbols(1 << 17)], level(10)], + [[text(1 << 17)], level(10)], + [[records(1 << 17, 100)], level(10)], + [[records(1 << 17, 600)], level(10)], + [[fewSymbols(80 << 10), records(48 << 10, 100)], level(10)], + [[fewSymbols(30_000), records(30_000, 100)], rowsOf64(c.ZSTD_lazy2, 12)], + [[records(30_000, 100), fewSymbols(30_000)], rowsOf64(c.ZSTD_lazy2, 12)], + [[fewSymbols(12_000), noise(30_000), fewSymbols(12_000)], rowsOf64(c.ZSTD_lazy2, 12)], + [[noise(20_000), text(30_000)], rowsOf64(c.ZSTD_lazy, 12)], + [[text(20_000), Buffer.alloc(20_000), text(20_000)], rowsOf64(c.ZSTD_lazy, 13)], + [[fewSymbols(20_000), records(20_000, 600), text(20_000)], rowsOf64(c.ZSTD_greedy, 12)], + // More than a block. + [[fewSymbols(125_000), records(40_000, 100), fewSymbols(20_000)], rowsOf64(c.ZSTD_lazy2, 14)], + ]; + expect(cases.map(([data, params]) => digest(zlib.zstdCompressSync(Buffer.concat(data), { params })))) + .toMatchInlineSnapshot(` + [ + "38882 e2cf1ebea8540fae", + "17398 f542e301793c6b24", + "1513 1e09820fa83c397", + "574 9c214dab588dfad4", + "26008 3a7c167a6f55827b", + "10030 3433cfd18a1e6b89", + "10066 f5047098fb382ae5", + "38583 6ec8f3e7937fbdfb", + "25171 39b72ef5650bbcd5", + "2868 4efe031f2810b3ce", + "9636 ec7bd23f67b5ae31", + "45730 6bb17dea79d5fff0", + ] + `); + }); + + it("where matches are far between, for the optimal parsers", () => { + const data = Buffer.alloc(100_000); + const next = xorshift(9); + const copy = (at: number, distance: number, length: number) => { + const end = Math.min(at + length, data.length); + while (at < end) data[at] = data[at++ - distance]; + return end; + }; + for (let at = 0, distance = 1; at < data.length;) { + if (at < 100 || next() % 300 !== 0) { + data[at++] = next() >> 8; + continue; + } + // From as far as the one before, from anywhere, or from one byte less far, which only counts right after a match. + const kind = next() % 4; + if (kind === 1 && distance > 1) distance--; + if (kind > 1) distance = 1 + (next() % Math.min(at, 1 << (3 + (next() % 15)))); + at = copy(at, distance, next() % 8 === 0 ? 300 : 3 + (next() % 12)); + if (next() % 2 === 0 && distance > 1) at = copy(at, distance - 1, 3 + (next() % 5)); + } + const smallTables = { [c.ZSTD_c_minMatch]: 4, [c.ZSTD_c_hashLog]: 8, [c.ZSTD_c_chainLog]: 8 }; + const cases: [Buffer | Uint8Array, Record][] = [ + [data, { [c.ZSTD_c_strategy]: c.ZSTD_btopt, [c.ZSTD_c_minMatch]: 3 }], + [data, { [c.ZSTD_c_compressionLevel]: 17 }], + [data, { [c.ZSTD_c_compressionLevel]: 19 }], + [data, { [c.ZSTD_c_strategy]: c.ZSTD_btultra, [c.ZSTD_c_minMatch]: 4 }], + [data, { [c.ZSTD_c_compressionLevel]: 19, [c.ZSTD_c_windowLog]: 12 }], + [data, { [c.ZSTD_c_compressionLevel]: 19, ...smallTables }], + [data, { [c.ZSTD_c_compressionLevel]: 19, ...smallTables, [c.ZSTD_c_enableLongDistanceMatching]: 1 }], + [large, { [c.ZSTD_c_strategy]: c.ZSTD_btopt, [c.ZSTD_c_windowLog]: 12 }], + ]; + expect(cases.map(([data, params]) => digest(zlib.zstdCompressSync(data, { params })))).toEqual([ + "89890 2e527bd77ff0248a", + "89813 167bd065007f9d2b", + "89828 46d69a88706923f0", + "89863 b3219cbfd91243e7", + "92097 87e40b8681136738", + "92667 ff08869afb33cf7b", + "90656 95d8a11469d222d9", + "57217 69a87c5b3e0a2b75", + ]); + const dictionary = data.subarray(0, 20_000); + const withDictionary = [3_000, 80_000].map(size => { + const params = { [c.ZSTD_c_compressionLevel]: 19 }; + const options = { dictionary, params, pledgedSrcSize: size }; + return digest(zlib.zstdCompressSync(data.subarray(20_000, 20_000 + size), options)); + }); + expect(withDictionary).toEqual(["2964 61e43e2a0511038c", "71053 b34a7f9afac77810"]); + }); + + it("with parameters", () => { + const cases: Record> = { + checksum: { [c.ZSTD_c_checksumFlag]: 1 }, + noContentSize: { [c.ZSTD_c_contentSizeFlag]: 0 }, + smallWindow: { [c.ZSTD_c_windowLog]: 12, [c.ZSTD_c_compressionLevel]: 7 }, + hashChain: { [c.ZSTD_c_strategy]: c.ZSTD_lazy2, [c.ZSTD_c_windowLog]: 14, [c.ZSTD_c_searchLog]: 6 }, + rows: { [c.ZSTD_c_strategy]: c.ZSTD_greedy, [c.ZSTD_c_searchLog]: 6, [c.ZSTD_c_minMatch]: 6 }, + binaryTree: { [c.ZSTD_c_strategy]: c.ZSTD_btlazy2, [c.ZSTD_c_chainLog]: 14, [c.ZSTD_c_hashLog]: 13 }, + optimal: { [c.ZSTD_c_strategy]: c.ZSTD_btopt, [c.ZSTD_c_targetLength]: 20, [c.ZSTD_c_minMatch]: 3 }, + longDistance: { [c.ZSTD_c_enableLongDistanceMatching]: 1, [c.ZSTD_c_windowLog]: 20 }, + longDistanceOptimal: { + [c.ZSTD_c_enableLongDistanceMatching]: 1, + [c.ZSTD_c_compressionLevel]: 16, + [c.ZSTD_c_ldmMinMatch]: 16, + [c.ZSTD_c_ldmHashLog]: 12, + [c.ZSTD_c_ldmBucketSizeLog]: 2, + [c.ZSTD_c_ldmHashRateLog]: 3, + }, + // ZSTD_c_targetCBlockSize + smallBlocks: { 130: 2000, [c.ZSTD_c_compressionLevel]: 5 }, + // ZSTD_c_format: ZSTD_f_zstd1_magicless + magicless: { 10: 1 }, + workers: { [c.ZSTD_c_nbWorkers]: 2, [c.ZSTD_c_jobSize]: 1, [c.ZSTD_c_checksumFlag]: 1 }, + workersOverlap: { + [c.ZSTD_c_nbWorkers]: 3, + [c.ZSTD_c_jobSize]: 1, + [c.ZSTD_c_overlapLog]: 9, + [c.ZSTD_c_windowLog]: 17, + }, + workersLongDistance: { + [c.ZSTD_c_nbWorkers]: 2, + [c.ZSTD_c_jobSize]: 1, + [c.ZSTD_c_enableLongDistanceMatching]: 1, + [c.ZSTD_c_windowLog]: 18, + }, + }; + const results: Record = {}; + for (const [name, params] of Object.entries(cases)) { + const data = name.startsWith("workers") ? corpus : large; + const compressed = zlib.zstdCompressSync(data, { params }); + results[name] = digest(compressed); + if (name !== "magicless") expect(zlib.zstdDecompressSync(compressed).equals(data)).toBe(true); + } + expect(results).toMatchInlineSnapshot(` + { + "binaryTree": "43844 a5c9eabbae2dd756", + "checksum": "44939 b9750b95ad15111d", + "hashChain": "48609 ad3f314838cea64", + "longDistance": "44645 1be0f74f50e4c2f1", + "longDistanceOptimal": "42035 826bf710d9511423", + "magicless": "44931 ef5be3a06c229e22", + "noContentSize": "44932 69582ab759972bd0", + "optimal": "42193 af8904ce81d257c5", + "rows": "43948 1c7f2cd5acfbb9eb", + "smallBlocks": "44603 b857325b812bea9c", + "smallWindow": "55906 6786884ad22d6b2d", + "workers": "108770 1df628af1e7dc077", + "workersLongDistance": "109211 732c0b64dad8574b", + "workersOverlap": "111826 bfab26c4004f255d", + } + `); + }); + + it("with every least length of a match, at the strategies that hash as many bytes", () => { + const dictionary = sample(20_000, 4); + const short = corpus.slice(400_000, 405_000); + short.set(dictionary.subarray(-600, -100), 1_000); + const results: Record = {}; + for (const [name, strategy] of [ + ["fast", c.ZSTD_fast], + ["dfast", c.ZSTD_dfast], + ] as const) { + for (const minMatch of [3, 4, 5, 6, 7]) { + const params = { [c.ZSTD_c_strategy]: strategy, [c.ZSTD_c_minMatch]: minMatch }; + const tables = (hashLog: number) => ({ [c.ZSTD_c_hashLog]: hashLog, [c.ZSTD_c_chainLog]: hashLog - 1 }); + const compressed = [ + zlib.zstdCompressSync(large, { params: { ...params, ...tables(16) } }), + zlib.zstdCompressSync(large, { params: { ...params, ...tables(10) } }), + zlib.zstdCompressSync(short, { params, dictionary, pledgedSrcSize: short.length }), + ]; + results[`${name}, ${minMatch}`] = compressed.map(digest).join(", "); + expect(zlib.zstdDecompressSync(compressed[0]).equals(large)).toBe(true); + expect(zlib.zstdDecompressSync(compressed[1]).equals(large)).toBe(true); + expect(zlib.zstdDecompressSync(compressed[2], { dictionary }).equals(short)).toBe(true); + } + } + expect(results).toMatchInlineSnapshot(` + { + "dfast, 3": "44487 25e0622d3e12bfbd, 48050 558295cad240a785, 1005 858a418a2405d05a", + "dfast, 4": "44487 25e0622d3e12bfbd, 48050 558295cad240a785, 1005 858a418a2405d05a", + "dfast, 5": "44922 5901d333a7330ff9, 48936 4abcce7cd20aaf96, 1028 6aaf30fe0e09dd18", + "dfast, 6": "45343 386c1fe73dd038a9, 49635 2fa5fd5697de4709, 1032 46565d1807a0eaa6", + "dfast, 7": "46545 39a718fcbe68cb38, 51913 6367517af191ced2, 1089 8dde7dc61d61525b", + "fast, 3": "45605 bb8d5e0a2a6d89bc, 48410 b143a25a8491bf48, 1013 c8b68aa258a78281", + "fast, 4": "45605 bb8d5e0a2a6d89bc, 48410 b143a25a8491bf48, 1013 c8b68aa258a78281", + "fast, 5": "45771 fe9b40f28db9fd9, 49021 56c10ab35e811607, 1036 c3e106a367a73897", + "fast, 6": "46011 8660df15f934f2e1, 49559 90160f0d372cf91, 1040 78faf9395332bbf", + "fast, 7": "46936 560777669662bd37, 52350 c186154a65c5f997, 1088 538a72fd960bac79", + } + `); + }); + + it("where a block is one match, which is too short for the optimal parser to take at once", () => { + const digests = (data: Buffer, windowLog: number) => + [16, 19, 22].map(level => { + const params = { + [c.ZSTD_c_compressionLevel]: level, + [c.ZSTD_c_windowLog]: windowLog, + [c.ZSTD_c_targetLength]: 4096, + }; + const compressed = zlib.zstdCompressSync(data, { params }); + expect(zlib.zstdDecompressSync(compressed).equals(data)).toBe(true); + return digest(compressed); + }); + const periodic = Buffer.alloc(5_000, corpus.subarray(0, 700)); + expect(digests(periodic, 10)).toEqual(Array(3).fill("163 571cb5ea10428fa7")); + expect(digests(periodic, 11)).toEqual(Array(3).fill("142 671efcd2063db130")); + expect(digests(Buffer.alloc(1_000), 10)).toEqual(Array(3).fill("19 bf47b44041ed0aa6")); + }); + + // Short input searches the tables of the dictionary, long input starts from a copy of them. + function withDictionary(dictionary: Uint8Array) { + const results: Record = {}; + for (const level of [-3, 1, 3, 5, 8, 13, 16, 19]) { + for (const size of [5_000, 150_000]) { + const data = corpus.slice(400_000, 400_000 + size); + data.set(dictionary.subarray(-600, -100), 1_000); + const options = { dictionary, params: { [c.ZSTD_c_compressionLevel]: level }, pledgedSrcSize: size }; + const compressed = zlib.zstdCompressSync(data, options); + results[`${level}, ${size}`] = digest(compressed); + expect(zlib.zstdDecompressSync(compressed, { dictionary }).equals(data)).toBe(true); + } + } + return results; + } + + it("with a dictionary that is only content", () => { + expect(withDictionary(sample(20_000, 4))).toMatchInlineSnapshot(` + { + "-3, 150000": "31691 a6cba8ed83f78cbe", + "-3, 5000": "1280 99e5c402823fdcad", + "1, 150000": "25367 4eed614674d852f5", + "1, 5000": "1042 c36ace4ccb9d37ab", + "13, 150000": "23771 c93a9f67de8b8c82", + "13, 5000": "988 6a7990046fc8b0de", + "16, 150000": "22992 c47f0966b096f185", + "16, 5000": "969 3be3d777b3ae4229", + "19, 150000": "22935 28de114f08f4bd22", + "19, 5000": "969 552f7abe774fa6ca", + "3, 150000": "24931 2feb8f5d9e046e54", + "3, 5000": "1028 6aaf30fe0e09dd18", + "5, 150000": "24343 c26404163f4b0830", + "5, 5000": "1002 6cf280704008a5eb", + "8, 150000": "23534 8b65e660bd8911a6", + "8, 5000": "973 f42818736cafcdfb", + } + `); + }); + + it("with a dictionary that has tables", () => { + // zstd --train --maxdict=1200, on 400 of these samples. + const dictionary = Buffer.from( + "N6Qw7CRPAAA5EKgmaQOIpOamUZ6YaYFWNK4FBYVp7RRHeEwRH2jGgTCmatsY1bd2V0loSVJyU2hzaXNTUlJIUminQwFAaTAYEstI" + + "FScAAATAhwSXlAeZRmKhICQIhcKgMCAQBgPCgCAYDAaAQWEgGBgOCWRJpPcCAAB0bWcj0zgoEAaDwXAwHA4MszhRamoRAAAAAQAA" + + "AAQAAAAIAAAACg1jb24XFBxYHlgKYXdhaXQgYXdhaXN0IAMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMD" + + "AwMDAwNubm5ubm5ubm5ubm5ubmxlbmd0aAgCHgZmdW5jdGlvbiBDDgQJFgCsIAMDAwMDAwMDKCgoKCgoPT09PT09PT0MDJMND8YY" + + "EgAKDAUNt5sCBQoIKN/f39/f39/f39/f39/f3/4aenN0ZHpzdGTBwcHBwcHBwcHBwcEwGwcJCAsfDROdHA8V6gMHGg89PT0wYXdh" + + "aXQgc3RkCA8MCwcIFhvWenN0ZAgPDAsHCAogIN/f3wkNBBcXBQ4WFRUbDQYbBBACCwILAgsCCwILY29uc3QgcmV0dXJuIGZ1bmN0" + + "aW9uID09PT09PT09PT09PT09PWF3YWl0IHQgdB4GHg0IBwMpO2F3YWl0ID0+IGZ1bmN0aW9uIGZ1bmN0aW9uIG5ubm5ubm5ubm5u" + + "bm5ubm5ubm5ubm5ubm5ubm5ubm57BvoUBvoUBvoCAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICw8PDw8PDw8PD" + + "w8PDw8PDw8PDw8PDw8PDw8MkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQk" + + "JCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQk" + + "JCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQ9PiBsZW5ndGhoowEX" + + "ImNvbnN0IB8IHRwHHgIYChUJBAgECAMYEg8bCxwDuQ4dDgYOjY2NC4IWBgcwKKCgoKB7KTswBgccAg8aEwIWDx4dGBUMFA8fCugK" + + "FxsfCB0cBx4CGAoADgt9ZnVuY3Rpb24gGRgLCBUbHAYaBwgQCxwQBQofGBgAAgMKgbcSHAkIHhwPGRwOcRsoPT4gMGNvbnN0IGZ1" + + "bmN0aW9uIBQOAiQWEAgUDRwUBBQIGwbP8MQYChIOBBQeHgsNDxALHMUXCBEaBADwxBgKEg4EFB4eCw0PEAscxRcIERoEAPDEGAoS" + + "DgRhd2FpdCBsZW5ndGhjb25zdCBsZW5ndGhsZW5ndGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGho" + + "aCh7e3t7e3t7e3t7e3t7e3t7e3t7e3toaGhoaGhoaGhokJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQ", + "base64", + ); + expect(withDictionary(dictionary)).toMatchInlineSnapshot(` + { + "-3, 150000": "31674 a6f9cb1f5deb45f", + "-3, 5000": "1255 86e003673a214f0a", + "1, 150000": "25439 b187159ef67d798f", + "1, 5000": "967 7ab6c959cbc9173", + "13, 150000": "23377 807244f3412a1248", + "13, 5000": "918 51a5e4bbad4678c8", + "16, 150000": "23360 7e0e03d462f05b47", + "16, 5000": "918 51a5e4bbad4678c8", + "19, 150000": "23224 dbc52271327892b3", + "19, 5000": "916 a6838a535dce670d", + "3, 150000": "24745 9a4a4d7d66087c52", + "3, 5000": "942 2f652f811a065d94", + "5, 150000": "24209 a726791cfb67d565", + "5, 5000": "926 f64c220e68e0c73c", + "8, 150000": "24000 7f2abf062f75ff72", + "8, 5000": "913 b12d4bfdabcdaf2b", + } + `); + + const compressed = zlib.zstdCompressSync(small, { dictionary }); + expect(() => zlib.zstdDecompressSync(compressed)).toThrow( + expect.objectContaining({ code: "ZSTD_error_dictionary_wrong", errno: c.ZSTD_error_dictionary_wrong }), + ); + const damaged = Buffer.from(dictionary); + damaged.fill(0xff, 8, 40); + expect(() => zlib.zstdCompressSync(small, { dictionary: damaged })).toThrow( + expect.objectContaining({ code: "ZSTD_error_memory_allocation", errno: c.ZSTD_error_memory_allocation }), + ); + }); + + async function stream(params: Record, flushes: boolean) { + const stream = zlib.createZstdCompress({ params }); + const chunks: Buffer[] = []; + stream.on("data", chunk => chunks.push(chunk)); + const { promise, resolve, reject } = Promise.withResolvers(); + stream.on("end", resolve).on("error", reject); + stream.on("close", () => reject(new Error("closed before the end"))); + for (let at = 0, n = 1; at < corpus.length; at += n, n = (n * 7) % 90_001) { + stream.write(corpus.subarray(at, at + n)); + if (flushes && n % 3 === 0) stream.flush(); + } + stream.end(); + await promise; + return Buffer.concat(chunks); + } + + it("in a stream with flushes", async () => { + const results: Record = {}; + for (const level of [1, 6, 12, 17]) { + results[level] = digest(await stream({ [c.ZSTD_c_compressionLevel]: level }, true)); + } + expect(results).toMatchInlineSnapshot(` + { + "1": "111500 f8b357a7535f4d59", + "12": "99899 c7c64eb5fefd6b8c", + "17": "99325 43b1ca1d605f6a2a", + "6": "101943 73ac52aed8b6fec6", + } + `); + }); + + it("in a stream with workers", async () => { + for (const flushes of [false, true]) { + const compressed = await stream({ [c.ZSTD_c_nbWorkers]: 1, [c.ZSTD_c_jobSize]: 1 }, flushes); + expect(zlib.zstdDecompressSync(compressed).equals(corpus)).toBe(true); + } + }); + + it("in a stream with workers, after others were given up in the middle of a frame", async () => { + const params = { [c.ZSTD_c_nbWorkers]: 2, [c.ZSTD_c_jobSize]: 1 }; + for (let i = 0; i < 4; i++) { + const abandoned = zlib.createZstdCompress({ params }); + abandoned.write(corpus); + abandoned.destroy(); + } + expect(zlib.zstdDecompressSync(await stream(params, false)).equals(corpus)).toBe(true); + }); + + async function flushed(params: Record, sizes: number[]) { + const stream = zlib.createZstdCompress({ params }); + const chunks: Buffer[] = []; + stream.on("data", chunk => chunks.push(chunk)); + const { promise, resolve, reject } = Promise.withResolvers(); + stream.on("end", resolve).on("error", reject); + stream.on("close", () => reject(new Error("closed before the end"))); + let at = 0; + for (const size of sizes) { + stream.write(corpus.subarray(at, at + size)); + stream.flush(); + at += size; + } + stream.end(); + await promise; + expect(zlib.zstdDecompressSync(Buffer.concat(chunks)).equals(corpus.subarray(0, at))).toBe(true); + } + + it("in a stream with workers, where a flush does not cost as much as a whole job", async () => { + // A job makes room for one long distance match for each ldmMinMatch bytes. For a whole job that is 192 MB here. + await flushed( + { + [c.ZSTD_c_nbWorkers]: 1, + [c.ZSTD_c_jobSize]: 64 << 20, + [c.ZSTD_c_enableLongDistanceMatching]: 1, + [c.ZSTD_c_ldmMinMatch]: 4, + }, + Array(200).fill(1000), + ); + }); + + it("in a stream with workers, where a larger job than those before does not clear the tables", async () => { + // The table is 256 MB. + await flushed( + { [c.ZSTD_c_nbWorkers]: 4, [c.ZSTD_c_strategy]: c.ZSTD_fast, [c.ZSTD_c_hashLog]: 26 }, + Array.from({ length: 300 }, (_, i) => i + 1), + ); + }); +}); + +// Frames from `decodecorpus`, zstd's own generator of anything the format allows, most of which no encoder makes, and +// what zstd 1.5.7 makes of each of them, whole and damaged in nine ways, through `node:zlib` and through `Bun`. +describe("frames that no encoder makes decode as with zstd 1.5.7", () => { + const digest = (output: Uint8Array) => `${output.length} ${Bun.hash.xxHash32(output).toString(16)}`; + const attempt = (decompress: (input: Buffer) => Uint8Array, input: Buffer) => { + try { + return digest(decompress(input)); + } catch (error: any) { + return error.code.replace("ZSTD_error_", ""); + } + }; + function* damaged(frame: Buffer) { + for (let k = 1; k <= 6; k++) { + const copy = Buffer.from(frame); + copy[Math.floor((frame.length * k) / 7)] ^= 1 << ((k * 3) % 8); + yield copy; + } + yield frame.subarray(0, Math.floor((frame.length * 3) / 4)); + yield frame.subarray(0, frame.length - 1); + yield Buffer.concat([frame, Buffer.from([0x28])]); + } + + it("whole and damaged", () => { + for (const { frame: base64, results } of frames) { + const frame = Buffer.from(base64, "base64"); + const actual: string[] = []; + for (const input of [frame, ...damaged(frame)]) { + actual.push(attempt(zlib.zstdDecompressSync, input), attempt(Bun.zstdDecompressSync, input)); + } + expect({ frame: base64, results: actual.join(",") }).toEqual({ frame: base64, results }); + } + }); +}); + +describe("bugs of zstd 1.5.7", () => { + // Fixed by facebook/zstd cef5a5611: `windowLog - ldmHashRateLog` wrapped around, which made for a table of 8 GiB. + it("a hash rate above the window does not make for the largest table of long distance matches", async () => { + const script = ` + const zlib = require("node:zlib"); + const c = zlib.constants; + const input = Buffer.alloc(1024, "long distance "); + const output = zlib.zstdCompressSync(input, { + params: { [c.ZSTD_c_enableLongDistanceMatching]: 1, [c.ZSTD_c_ldmHashRateLog]: 11 }, + }); + console.log(input.equals(zlib.zstdDecompressSync(output)), process.resourceUsage().maxRSS < 4 * 1024 * 1024); + `; + await using proc = Bun.spawn({ cmd: [bunExe(), "-e", script], env: bunEnv, stdout: "pipe", stderr: "pipe" }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + expect({ stdout, stderr: stderr.slice(0, 2000) }).toEqual({ stdout: "true true\n", stderr: "" }); + expect(exitCode).toBe(0); + }); + + // Only a streaming decoder checked that. Decoded in one call, such a block could overwrite literals it had yet to read. + it("a compressed block that makes more than 128 KB is an error", () => { + const frame = Buffer.from("28b52ffd8050c6ff030040000041424344454647487500008dfb1f7a02540103340000030058", "hex"); + expect(() => zlib.zstdDecompressSync(frame, { chunkSize: 262086 })).toThrow( + expect.objectContaining({ code: "ZSTD_error_corruption_detected" }), + ); + expect(() => Bun.zstdDecompressSync(frame)).toThrow(expect.objectContaining({ code: "ERR_ZSTD" })); + }); + + // No decoder checked that. A streaming one keeps little more than the window, and gave whatever was there by then. + it("an offset beyond the window is an error once a streaming decoder has let go of what it is for", () => { + // A window of 1 KB, 1 KB each of "a", "b" and "c", then "xyz" and 4 bytes from 1500 bytes back. + const frame = Buffer.from("28b52ffd00000220006102200062022000635d00001878797a0154030a01df05", "hex"); + expect(() => zlib.zstdDecompressSync(frame)).toThrow( + expect.objectContaining({ code: "ZSTD_error_corruption_detected" }), + ); + }); + + // With workers, only a frame of one job was checked. Any other came out with the wrong size in its header. + it.each([-1, 1])("with workers, input that is %i off the pledged size is an error", async difference => { + const c = zlib.constants; + const input = Buffer.alloc(3 << 20, "several jobs "); + const stream = zlib.createZstdCompress({ + pledgedSrcSize: input.length - difference, + params: { [c.ZSTD_c_nbWorkers]: 2, [c.ZSTD_c_jobSize]: 1 << 20 }, + }); + const { promise, resolve } = Promise.withResolvers(); + stream.on("error", resolve).on("end", resolve).on("close", resolve).resume(); + stream.write(input.subarray(0, 1000)); + stream.end(input.subarray(1000)); + expect(await promise).toMatchObject({ code: "ZSTD_error_srcSize_wrong" }); + }); +}); + +// Bugs that zstd has had, from its history up to October 2026. zstd 1.5.7 has none of them, and what is expected is what +// it gives: they are here because the port does not follow the C code line by line. So Bun 1.4.4 passes them too, but for +// bugs of its own: streams with workers lose their output, and a dictionary that a decoder cannot load goes unnoticed. +describe("bugs that zstd has had", () => { + const c = zlib.constants; + const hex = (digits: string) => Buffer.from(digits.replaceAll(" ", ""), "hex"); + const base64 = (text: string) => Buffer.from(text, "base64"); + const repeated = (digits: string, count: number) => Buffer.alloc((digits.length / 2) * count, hex(digits)); + const skippable = (size: number) => { + const frame = Buffer.alloc(8 + size, 7); + frame.writeUInt32LE(0x184d2a50, 0); + frame.writeUInt32LE(size, 4); + return frame; + }; + /** xorshift32 */ + function noise(length: number, seed: number, mask = 255) { + const bytes = Buffer.alloc(length); + for (let i = 0, x = seed >>> 0; i < length; i++) { + x ^= x << 13; + x ^= x >>> 17; + x ^= x << 5; + bytes[i] = x & mask; + } + return bytes; + } + function through(stream: zlib.ZstdCompress | zlib.ZstdDecompress, chunks: Iterable, flush = false) { + const { promise, resolve } = Promise.withResolvers(); + const output: Buffer[] = []; + stream.on("data", chunk => output.push(chunk)); + stream.on("error", (error: any) => resolve(error.code)); + stream.on("end", () => resolve(Buffer.concat(output))); + stream.on("close", () => resolve("closed")); + for (const chunk of chunks) { + stream.write(chunk); + if (flush) stream.flush(); + } + stream.end(); + return promise; + } + function* pieces(bytes: Buffer, size: number) { + for (let i = 0; i < bytes.length; i += size) yield bytes.subarray(i, i + size); + } + const attempt = (decompress: () => Uint8Array) => { + try { + return Buffer.from(decompress()); + } catch (error: any) { + if (typeof error.code !== "string") throw error; + return error.code as string; + } + }; + /** By `Bun.zstdDecompressSync`, by `zlib.zstdDecompressSync`, and by a stream that gets little at a time. */ + const decoded = async (frame: Buffer, options?: zlib.ZstdOptions) => [ + attempt(() => Bun.zstdDecompressSync(frame)), + attempt(() => zlib.zstdDecompressSync(frame, options)), + await through(zlib.createZstdDecompress(options), pieces(frame, frame.length > 3000 ? 1000 : 1)), + ]; + const hello = hex("28b52ffd200529000068656c6c6f"); + const corrupt = "ZSTD_error_corruption_detected"; + + it.each<[string, Buffer, Buffer | string]>([ + ["an empty compressed block", hex("28b52ffd00001500000000"), ""], + ["an empty raw block", hex("28b52ffd0000 010000"), ""], + ["an empty run", hex("28b52ffd0000 030000 09"), ""], + ["empty blocks between others", hex("28b52ffd0000 180000616263 000000 0200007a 190000646566"), "abcdef"], + [ + "a run for a first block", + hex("28b52ffda400001000" + "020010 00".repeat(7) + "030010 00 f13e16e1"), + Buffer.alloc(1 << 20), + ], + ["no sequences, said in 2 bytes", hex("28b52ffd00008500006848656c6c6f20576f726c64210a8000"), "Hello World!\n"], + ["no sequences", hex("28b52ffd0000 2d0000 1861626300"), "abc"], + [ + "a compressed block of 128 KB", + Buffer.concat([hex("28b52ffd0038050010ccff1f"), Buffer.alloc(131068), hex("00")]), + Buffer.alloc(131068), + ], + [ + "copies that go too far at the end of a stream's buffer", + hex("28b52ffd04004c00001061610100002a800544000008620100002a20045d00000003400000646027b0e00c6762cee0"), + Buffer.concat([Buffer.alloc(517, "a"), Buffer.alloc(543, "b"), Buffer.from("aaa")]), + ], + [ + "sequence sections of 3 and 4 bytes", + base64("KLUv/SQ8NQEA8IUIwsRwz9fAln5Ma6mLvMW22X9M8QWmVO+saZSJHANECgcAtASAQAqk"), + hex("8508c2c470cfd7c0967e".repeat(2) + "4c6ba98bbcc5b6d97f4c".repeat(2) + "f105a654efac6994891c".repeat(2)), + ], + [ + "skippable frames before, between and after frames", + hex("502a4d1803000000010203 28b52ffd0000190000616263 5f2a4d1800000000 28b52ffd0000190000646566"), + "abcdef", + ], + ["a skippable frame longer than any buffer", Buffer.concat([skippable(200_000), hello]), "hello"], + ["a window of 128 MB", hex("28b52ffd0088190000616263"), "abc"], + ["a match for the byte before", hex("28b52ffd8000040000004d00001c0000610150010040"), "aaaa"], + ["a match that overlaps itself", hex("28b52ffd80000c00000045000010616201006d0e0b"), "abababababab"], + [ + "a raw block of the size of the window", + Buffer.concat([hex("28b52ffd0000 012000"), Buffer.alloc(1024, "A")]), + Buffer.alloc(1024, "A"), + ], + ])("%s is decoded", async (_, frame, content) => { + const expected = Buffer.from(content); + expect(await decoded(frame)).toEqual([expected, expected, expected]); + }); + + it.each<[string, Buffer, string]>([ + ["an offset of 0", hex("28b52ffd0000450000080002002f430bae"), corrupt], + ["a repeat offset that comes to 0", hex("28b52ffd000040000061626364656667683d000000015400010003"), corrupt], + ["a Huffman state that is cut short", hex("28b52ffd000055000072800104207e1f02aa00"), corrupt], + [ + "bytes after no sequences, said in 2 bytes", + hex("28b52ffd00009500006848656c6c6f20576f726c64210a80000000"), + corrupt, + ], + ["a byte after no sequences", hex("28b52ffd0000 350000 186162630000"), corrupt], + ["less content than the header says", hex("28b52ffd200629000068656c6c6f"), corrupt], + ["more content than the header says", hex("28b52ffd200429000068656c6c6f"), corrupt], + [ + "a raw block larger than the window", + Buffer.concat([hex("28b52ffd0000 092000"), Buffer.alloc(1025, "A")]), + corrupt, + ], + [ + "the Huffman table of the block before, in the first", + hex("28b52ffd0000350000434000000000"), + "ZSTD_error_dictionary_corrupted", + ], + ["a window of 256 MB", hex("28b52ffd0090190000616263"), "ZSTD_error_frameParameter_windowTooLarge"], + ["the reserved bit", hex("28b52ffd0800190000616263"), "ZSTD_error_frameParameter_unsupported"], + ["the reserved bit of a frame in one segment", hex("28b52ffd2800010000"), "ZSTD_error_frameParameter_unsupported"], + ["the largest window there is a code for", hex("28b52ffd00ff010000"), "ZSTD_error_frameParameter_windowTooLarge"], + ["the sequence tables of the block before, in the first", hex("28b52ffd8000040000002d0000086101fc01"), corrupt], + ["a literal length code of 36", hex("28b52ffd800004000000350000086101402401"), corrupt], + ["an offset code of 32", hex("28b52ffd800004000000350000086101102001"), corrupt], + ["a match length code of 53", hex("28b52ffd800004000000350000086101043501"), corrupt], + [ + "4 streams for 5 literals", + hex("28b52ffd8000050000004d0000568001811100000000"), + "ZSTD_error_literals_headerWrong", + ], + [ + "Huffman weights that add up to more than 2^12", + hex("28b52ffd80001400000085000042010383cccc01010101010101010100"), + corrupt, + ], + ["the first repeat offset less 1, in the first sequence", hex("28b52ffd800003000000350000000150000181"), corrupt], + ["an offset of 2 after 1 byte", hex("28b52ffd8000040000003d0000096101440100b9"), corrupt], + ["a run of literals without its byte", hex("28b52ffd8000280000001d00008d0200"), corrupt], + ["compressed literals without a table", hex("28b52ffd80002800000025000002000000"), corrupt], + [ + // CVE-2026-116591 of another decoder, where it got through on top of what the block before had left. + "Huffman weights with a table of 2^15 states", + Buffer.concat([ + hex("28b52ffd003874060042b8320d1a00000090affaff030080ff7f"), + repeated("a594524a29", 37), + hex("a5941200ad5603eeffdfb31a1c1a00e0ff1f0fb19c5acf4de41d7bab995e0151a473d8b1f3394b67f6ab1aab1aab1a"), + ...[repeated("8c31c61863", 1365), hex("8c31"), repeated("638c31c618", 1365), hex("632c")], + ...[repeated("31c618638c", 1365), hex("3146"), repeated("31c618638c", 1365), hex("310600")], + ]), + corrupt, + ], + ])("%s is an error", async (_, frame, code) => { + expect(await decoded(frame)).toEqual(["ERR_ZSTD", code, code]); + }); + + it("what follows the last frame is refused or left alone", async () => { + const abc = Buffer.from("abc"); + const frame = hex("28b52ffd0000190000616263"); + const after = (rest: string) => decoded(Buffer.concat([frame, hex(rest)])); + expect(await after("ff")).toEqual(["ERR_ZSTD", abc, "ZSTD_error_prefix_unknown"]); + expect(await after("502a4d18ffffffff")).toEqual(["ERR_ZSTD", abc, abc]); + // 8 more than that wraps around. + expect(await decoded(hex("502a4d18f8ffffff"))).toEqual(["ERR_ZSTD", Buffer.alloc(0), Buffer.alloc(0)]); + }); + + it("what no frame starts with is refused before there are 4 bytes of it", async () => { + const after = async (first: string) => { + const stream = zlib.createZstdDecompress(); + const { promise, resolve } = Promise.withResolvers(); + stream.on("error", (error: any) => resolve(error.code)); + stream.write(hex(first), error => resolve((error as any)?.code ?? "more")); + try { + return await promise; + } finally { + stream.destroy(); + } + }; + expect(await Promise.all(["28", "28b52f", "50", "5f2a4d"].map(after))).toEqual(["more", "more", "more", "more"]); + const unknown = "ZSTD_error_prefix_unknown"; + expect(await Promise.all(["000000", "29", "28b530", "4f2a4d"].map(after))).toEqual(Array(4).fill(unknown)); + }); + + it("a first block cannot repeat the tables of the frame that was decoded before it", async () => { + const text = zlib.zstdCompressSync(noise(20_000, 18, 15)); + for (const [frame, code] of [ + [hex("28b52ffd800008000000350000834000010100"), "ZSTD_error_dictionary_corrupted"], + [hex("28b52ffd8000040000002d0000086101fc01"), corrupt], + ] as const) { + expect(Bun.zstdDecompressSync(text).length).toBe(20_000); + expect(zlib.zstdDecompressSync(text).length).toBe(20_000); + expect(await decoded(frame)).toEqual(["ERR_ZSTD", code, code]); + expect(await through(zlib.createZstdDecompress(), [text, frame])).toBe(code); + } + }); + + it.each([1000, 1017, 1023, 1024])( + "offsets up to the window are decoded after a stream's buffer has wrapped around: %i", + async distance => { + const input = Buffer.alloc(100_000, noise(distance, distance)); + for (const level of [5, 13, 19]) { + const params = { [c.ZSTD_c_windowLog]: 10, [c.ZSTD_c_compressionLevel]: level }; + const frame = zlib.zstdCompressSync(input, { params }); + expect(frame.length).toBeLessThan(5000); + expect(await through(zlib.createZstdDecompress(), pieces(frame, 1))).toEqual(input); + expect(await through(zlib.createZstdDecompress(), [frame])).toEqual(input); + } + }, + ); + + it("a larger window is decoded if allowed", async () => { + const [, sync, streamed] = await decoded(hex("28b52ffd0090190000616263"), { + params: { [c.ZSTD_d_windowLogMax]: 28 }, + }); + expect([sync, streamed]).toEqual([Buffer.from("abc"), Buffer.from("abc")]); + }); + + it("each bit of the checksum counts", async () => { + const frame = zlib.zstdCompressSync(noise(3000, 1, 15), { params: { [c.ZSTD_c_checksumFlag]: 1 } }); + for (let bit = 0; bit < 32; bit++) { + const damaged = Buffer.from(frame); + damaged[frame.length - 4 + (bit >> 3)] ^= 1 << (bit & 7); + expect(attempt(() => zlib.zstdDecompressSync(damaged))).toBe("ZSTD_error_checksum_wrong"); + } + }); + + it("a raw block comes out as it comes in", async () => { + const content = noise(100_000, 2); + const frame = zlib.zstdCompressSync(content); + const stream = zlib.createZstdDecompress(); + const { promise, resolve, reject } = Promise.withResolvers(); + stream.once("data", chunk => resolve(chunk.length)).on("error", reject); + stream.on("close", () => reject(new Error("closed before any data"))); + try { + stream.write(frame.subarray(0, 5000), () => stream.flush()); + expect(await promise).toBeWithin(4000, 5000); + } finally { + stream.destroy(); + } + }); + + const roundTrip = (input: Buffer, params: Record, dictionary?: Buffer) => { + const frame = zlib.zstdCompressSync(input, { params, dictionary }); + expect(zlib.zstdDecompressSync(frame, { dictionary }).equals(input)).toBe(true); + return frame; + }; + const levels = (from: number, to: number) => Array.from({ length: to - from + 1 }, (_, i) => from + i); + + // facebook/zstd#3517 + it("a block is split where 64 KB of literals come before a repeat offset", () => { + const input = noise(256 << 10, 2654435761, 127); + noise(32 << 10, 0xcafebabe, 3).copy(input); + const put = (at: number, values: number[]) => { + for (const skip of [0, 4]) values.forEach((value, i) => input.writeUInt32LE(value, (at + skip + i) * 4)); + }; + put(8 << 10, [0xffffffff, 0xeeeeeeee]); + put((24 << 10) + 2, [0xdddddddd, 0xcccccccc]); + put(32 << 10, [0xbbbbbbbb, 0xaaaaaaaa]); + expect(roundTrip(input, { [c.ZSTD_c_compressionLevel]: 19, [c.ZSTD_c_minMatch]: 7 }).length).toBe(210649); + }); + + // facebook/zstd#3212 + const ZSTD_c_targetCBlockSize = 130; + it("literals and matches of more than 64 KB go in blocks of a target size", () => { + const input = Buffer.alloc(270_000); + noise(70_000, 12345, 127).copy(input); + for (let i = 70_000; i < 200_000; i++) input[i] = input[i - 1000]; + noise(70_000, 777, 127).copy(input, 200_000); + for (const level of [1, 3, 7, 19]) { + roundTrip(input, { [c.ZSTD_c_compressionLevel]: level, [ZSTD_c_targetCBlockSize]: 1340 }); + } + }); + + it("runs are coded as runs, but for the first block", () => { + const twoRuns = Buffer.concat([Buffer.alloc((256 << 10) - 2, "B"), Buffer.alloc(100 << 10, "A")]); + expect(Buffer.from(Bun.zstdCompressSync(twoRuns, { level: 1 })).toString("hex")).toBe( + "28b52ffda0fe8f05005400001042420100fbff39c0025c0000184241410100faff391002f37f0c41", + ); + const runs = "02001000".repeat(6) + "03120a00"; + const zeros = (level: number) => + Buffer.from(Bun.zstdCompressSync(Buffer.alloc(1_000_000), { level })).toString("hex"); + expect(zeros(1)).toBe("28b52ffd804840420f005400001000000100fbff39c002" + runs); + expect(zeros(3)).toBe("28b52ffda040420f005400001000000100fbff39c002" + runs); + expect(zeros(19)).toBe("28b52ffda040420f004c000008000100fcff391002" + runs); + }); + + it("what does not compress is not split", () => { + const input = noise(256 << 10, 99); + for (let chunk = 0; chunk < 32; chunk++) { + for (let i = chunk * 8192; i < (chunk + 1) * 8192; i += 43) + input[i] = chunk % 2 ? input[i] & 0x80 : input[i] | 0x80; + } + for (const level of [1, 3, 5, 9, 13, 16, 19, 22]) { + expect(Bun.zstdCompressSync(input, { level }).length).toBe(input.length + 9 + 2 * 3); + } + }); + + it("literals that Huffman coding makes longer are left as they are", () => { + const input = base64( + "KLUv/QRQFQQAZoYeHZByBTMAgCylAQAAAAAQ3rSu67renkkEAACAAACAFgAXABYAPAbsctt3bakp+jCTL8TBUCAMBAHAB/qINH48GBJqFKKYQCuWaqejAYm/HkwYO8iHxTOx0WQulspEBGEVN1ttJot1X2V//vv1OxoLTidSkpxSsxzjcgEBAFsWJDvL4y4=", + ); + for (const level of levels(1, 22)) + expect(roundTrip(input, { [c.ZSTD_c_compressionLevel]: level }).length).toBe(152); + expect(Bun.zstdDecompressSync(input).length).toBe(142); + }); + + it("there is no empty block after input of the pledged size", async () => { + const block = Buffer.alloc(128 << 10, "a"); + const compressed = async (options: zlib.ZstdOptions) => + ((await through(zlib.createZstdCompress(options), [block])) as Buffer).toString("hex"); + expect(await compressed({ pledgedSrcSize: block.length })).toBe("28b52ffda0000002005500001061610100fbff39c002"); + expect(await compressed({})).toBe("28b52ffd00585400001061610100fbff39c002010000"); + }); + + it("tiny inputs", () => { + const compressed = (input: string | Buffer, level: number) => + Buffer.from(Bun.zstdCompressSync(input, { level })).toString("hex"); + for (const level of [1, 3, 19]) { + expect(compressed(Buffer.alloc(100), level)).toBe("28b52ffd206445000010000001003f012c"); + expect(compressed("", level)).toBe("28b52ffd2000010000"); + expect(compressed("a", level)).toBe("28b52ffd200109000061"); + } + const abab = Buffer.alloc(20, "ab"); + expect([1, 3, 19].map(level => compressed(abab, level))).toEqual([ + "28b52ffd20144500001061620100790e0b", + "28b52ffd20144500001061620100790e0b", + "28b52ffd20144d0000186162610100cd6e08", + ]); + for (const length of levels(0, 40)) { + for (const input of [Buffer.alloc(length), Buffer.from(levels(1, length)), Buffer.alloc(length, "ab")]) { + for (const level of [-5, 1, 3, 6, 13, 19, 22]) roundTrip(input, { [c.ZSTD_c_compressionLevel]: level }); + } + } + }); + + it("127, 128 and 129 sequences", () => { + const made = (total: number) => { + let x = 1; + const next = () => ((x = (Math.imul(x, 1103515245) + 12345) & 0x7fffffff), (x >> 16) & 255); + const bytes = Array.from({ length: 16 }, next); + while (bytes.length < total) { + const distance = 4 + (next() % 9); + for (let i = 0; i < 3; i++) bytes.push(bytes.at(-distance)!); + const byte = next(); + bytes.push(byte === bytes.at(-distance) ? (byte + 1) & 255 : byte); + } + const frame = roundTrip(Buffer.from(bytes), { [c.ZSTD_c_compressionLevel]: 19 }); + return `${frame.length} ${Bun.hash.xxHash64(frame).toString(16)}`; + }; + expect([532, 536, 540].map(made)).toEqual(["237 675545151ba98499", "239 18edd3092b7b0659", "241 bf78b419d4d9bdc1"]); + }); + + it("literals and matches of about 64 KB", () => { + for (const literals of [65_535, 65_536, 65_537]) { + for (const match of [65_538, 65_539]) { + const input = Buffer.concat([ + noise(literals + 10, literals), + noise(literals + 10, literals).subarray(0, match), + ]); + for (const level of [1, 3, 9, 19]) roundTrip(input, { [c.ZSTD_c_compressionLevel]: level }); + } + } + }); + + it("level 0 is level 3", () => { + const input = noise(20_000, 3, 7); + expect(roundTrip(input, { [c.ZSTD_c_compressionLevel]: 0 })).toEqual( + roundTrip(input, { [c.ZSTD_c_compressionLevel]: 3 }), + ); + }); + + it("the size of the output buffer does not show", () => { + for (const input of [Buffer.alloc(0), Buffer.from("a"), noise(200, 4, 7), noise(5000, 5, 7), noise(5000, 6)]) { + for (const level of [1, 3, 9, 19]) { + const params = { [c.ZSTD_c_compressionLevel]: level }; + expect(zlib.zstdCompressSync(input, { params, chunkSize: 64 })).toEqual(roundTrip(input, params)); + } + } + }); + + it("a match for the first position is found alike in one call and in a stream", () => { + const input = Buffer.from([...levels(0, 19), ...levels(0, 9)]); + for (const level of levels(1, 19)) { + expect(Buffer.from(Bun.zstdCompressSync(input, { level }))).toEqual( + roundTrip(input, { [c.ZSTD_c_compressionLevel]: level }), + ); + } + }); + + it("the first block that counts comes after blocks too small to", async () => { + const input = noise(5000, 7, 15); + const stream = zlib.createZstdCompress({ + pledgedSrcSize: input.length, + params: { [c.ZSTD_c_compressionLevel]: 22 }, + }); + const cuts = [0, 4, 9, 20, input.length]; + const chunks = cuts.slice(1).map((to, i) => input.subarray(cuts[i], to)); + expect(zlib.zstdDecompressSync((await through(stream, chunks, true)) as Buffer)).toEqual(input); + }); + + it("raw blocks between compressed ones", () => { + const input = Buffer.concat(levels(1, 3).flatMap(i => [noise(128 << 10, i, 7), noise(128 << 10, i + 10)])); + for (const level of [1, 3, 9, 19]) roundTrip(input, { [c.ZSTD_c_compressionLevel]: level }); + }); + + it("a block of literals with a match for its last bytes", () => { + const input = noise(128 << 10, 19); + input.copy(input, input.length - 16, 1000, 1016); + for (const level of [16, 19, 22]) + expect(roundTrip(input, { [c.ZSTD_c_compressionLevel]: level }).length).toBe(131084); + }); + + it("less input than the shortest long distance match", async () => { + const params = { [c.ZSTD_c_enableLongDistanceMatching]: 1, [c.ZSTD_c_ldmMinMatch]: 4096 }; + const frames = [0, 1, 7, 63, 64, 4095].flatMap(length => + [3, 19].map(level => roundTrip(noise(length, 21, 3), { ...params, [c.ZSTD_c_compressionLevel]: level })), + ); + expect(Bun.MD5.hash(Buffer.concat(frames), "hex")).toStartWith("033266ef"); + + const input = noise(300, 22, 3); + const frame = (await through(zlib.createZstdCompress({ params }), pieces(input, 5), true)) as Buffer; + expect(`${frame.length} ${Bun.MD5.hash(frame, "hex").slice(0, 8)}`).toBe("504 1f230d95"); + expect(zlib.zstdDecompressSync(frame)).toEqual(input); + }); + + it("the least and the most that parameters can be", () => { + const taken = (parameter: number, values: number[]) => + values.filter(value => typeof attempt(() => roundTrip(hello, { [parameter]: value })) !== "string"); + expect(taken(c.ZSTD_c_compressionLevel, [-200_000, -131_073, -131_072, 22, 23, 200_000])).toBeArrayOfSize(6); + expect(taken(c.ZSTD_c_windowLog, [9, 10, 31, 32])).toEqual([10, 31]); + expect(taken(c.ZSTD_c_enableLongDistanceMatching, [0, 1, 2, 3])).toEqual([0, 1, 2]); + expect(taken(c.ZSTD_c_ldmHashLog, [5, 6, 30, 31])).toEqual([6, 30]); + expect(taken(c.ZSTD_c_ldmMinMatch, [3, 4, 4096, 4097])).toEqual([4, 4096]); + expect(taken(c.ZSTD_c_ldmBucketSizeLog, [0, 1, 8, 9])).toEqual([0, 1, 8]); + expect(taken(c.ZSTD_c_ldmHashRateLog, [0, 25, 26])).toEqual([0, 25]); + }); + + describe("dictionaries", () => { + // `tests/dict-files/zero-weight-dict` + const zeroWeights = base64( + "N6Qw7P8g/wA2EBsBAS69U+gVvLDIxkg0+R20Td9JXrqkGW5ZfDAqCkIVpJJ/1SYRHsHu2M4pkpKSkpKSkjIF0wIgDAoHxYOiAYE+uwUEIMKpoI4m1OVZksMoCiFkjDGEECAAAEAAQGREAAAAABRwrVMUAAAATCGEjEHGDM3IDAAAAAEAAAAEAAAACAAAACAgjibU5VmSwygK", + ); + + it("whose Huffman table has symbols of weight 0", () => { + const skewed = noise(5000, 8).map((byte, i, all) => Math.min(byte, all[(i + 1) % all.length])); + for (const level of levels(1, 22)) { + for (const input of [zeroWeights, Buffer.from("0000000000000000000000000\n"), Buffer.from(skewed)]) { + roundTrip(input, { [c.ZSTD_c_compressionLevel]: level }, zeroWeights); + } + } + }); + + // A compressor only looks at its dictionary when it starts, and has one error for all that can go wrong with it. + const refused = ["ZSTD_error_memory_allocation", "ERR_ZLIB_INITIALIZATION_FAILED"]; + const taken = (dictionary: Buffer) => + [ + attempt(() => zlib.zstdCompressSync("hello", { dictionary })), + attempt(() => zlib.zstdDecompressSync(hello, { dictionary })), + ].map(result => (typeof result === "string" ? result : "ok")); + + it("whose repeat offsets reach no further than their content", () => { + const withOffset = (offset: number, at: number) => { + const dictionary = Buffer.from(zeroWeights); + dictionary.writeUInt32LE(offset, at); + return taken(dictionary); + }; + for (const at of [130, 134, 138]) { + expect([withOffset(11, at), withOffset(12, at), withOffset(0, at)]).toEqual([["ok", "ok"], refused, refused]); + } + }); + + it("that are short", () => { + const input = noise(300, 9, 7); + const plain = zlib.zstdCompressSync(input); + for (const length of levels(1, 20)) { + const raw = noise(length, length, 7); + const frame = roundTrip(input, {}, raw); + if (length < 8) expect(frame).toEqual(plain); + // With the magic number, 8 bytes and more have to be a whole dictionary. + hex("37a430ec").copy(raw); + expect(taken(raw)).toEqual(length < 8 ? ["ok", "ok"] : refused); + } + }); + + it("that do not make the window of a stream of unknown size smaller", async () => { + const chunks = [Buffer.from("abc"), Buffer.from("def")]; + const start = async (options: zlib.ZstdOptions) => + ((await through(zlib.createZstdCompress(options), chunks)) as Buffer).toString("hex", 0, 6); + const dictionary = noise(1000, 20, 7); + expect(await start({ dictionary })).toBe("28b52ffd0058"); + expect(await start({ dictionary, pledgedSrcSize: 6 })).toBe("28b52ffd2006"); + }); + + it("of which the tables have no room for all", () => { + const dictionary = noise(17 << 20, 24, 15); + const params = { [c.ZSTD_c_compressionLevel]: 1 }; + expect(roundTrip(dictionary.subarray(0, 10240), params, dictionary).length).toBe(5154); + expect(roundTrip(dictionary.subarray(-10240), params, dictionary).length).toBe(19); + }); + + it("that leave the window", async () => { + const dictionary = noise(10240, 10, 15); + const input = Buffer.alloc(10240); + for (let i = 0, x = 11; i < input.length; x = (x * 1103515245 + 12345) >>> 0) { + const length = 4 + ((x >>> 8) % 57); + const distance = [7, 300, 1000][(x >>> 16) % 3]; + const from = (x >>> 4) % (dictionary.length - 64); + for (let k = 0; k < length && i < input.length; k++, i++) { + input[i] = + x % 5 === 0 + ? (x >>> (k % 24)) & 15 + : x % 5 === 1 || i < distance + ? dictionary[from + k] + : input[i - distance]; + } + } + for (const strategy of levels(1, 9)) { + const params = { [c.ZSTD_c_windowLog]: 10, [c.ZSTD_c_strategy]: strategy, [c.ZSTD_c_checksumFlag]: 1 }; + const frame = (await through(zlib.createZstdCompress({ dictionary, params }), pieces(input, 512))) as Buffer; + const decoder = zlib.createZstdDecompress({ dictionary, params: { [c.ZSTD_d_windowLogMax]: 10 } }); + expect(await through(decoder, pieces(frame, 100))).toEqual(input); + } + }); + }); + + describe("workers", () => { + it("whose overlap starts like a dictionary", async () => { + const input = noise(5 << 20, 12, 7); + for (const job of levels(1, 4)) hex("37a430ec").copy(input, (job << 20) - (1 << 14)); + const params = { + [c.ZSTD_c_nbWorkers]: 2, + [c.ZSTD_c_jobSize]: 1 << 20, + [c.ZSTD_c_windowLog]: 17, + [c.ZSTD_c_overlapLog]: 6, + }; + const frame = (await through(zlib.createZstdCompress({ params }), pieces(input, 300_000))) as Buffer; + expect(zlib.zstdDecompressSync(frame).equals(input)).toBe(true); + }); + + it("with long distance matching and a tiny dictionary", () => { + const input = noise(600_000, 13, 7); + for (const length of [1, 8, 63]) { + roundTrip(input, { [c.ZSTD_c_nbWorkers]: 2, [c.ZSTD_c_enableLongDistanceMatching]: 1 }, noise(length, 14)); + } + }); + + // What Node 24 gives, with zstd 1.5.7. + it("make the same of input however it is cut", async () => { + const input = noise(3 << 20, 15, 7); + const options = { pledgedSrcSize: input.length, params: { [c.ZSTD_c_nbWorkers]: 2, [c.ZSTD_c_checksumFlag]: 1 } }; + const cut = async (size: number) => { + const frame = (await through(zlib.createZstdCompress(options), pieces(input, size))) as Buffer; + return `${frame.length} ${Bun.MD5.hash(frame, "hex").slice(0, 8)}`; + }; + expect(await cut(input.length)).toBe("1398674 d981d509"); + for (const size of [1 << 20, 64 << 10, 1000]) expect(await cut(size)).toBe("1398687 a871b693"); + }); + + it.each([ + [3, 0], + [3, 1], + [16, 0], + [16, 1], + ])("make the same twice: level %i, long distance matching %i", async (level, longDistance) => { + const input = noise(3 << 19, 23, 7); + const params = { + [c.ZSTD_c_nbWorkers]: 2, + [c.ZSTD_c_jobSize]: 512 << 10, + [c.ZSTD_c_compressionLevel]: level, + [c.ZSTD_c_enableLongDistanceMatching]: longDistance, + }; + const made = () => through(zlib.createZstdCompress({ params }), pieces(input, 100_000)) as Promise; + const [frame, again] = await Promise.all([made(), made()]); + expect(frame.equals(again)).toBe(true); + expect(zlib.zstdDecompressSync(frame).equals(input)).toBe(true); + }); + + it("with nothing left for the last job", async () => { + const input = noise(2 << 20, 16, 7); + const params = { [c.ZSTD_c_nbWorkers]: 2, [c.ZSTD_c_jobSize]: 512 << 10, [c.ZSTD_c_checksumFlag]: 1 }; + const dictionary = noise(1000, 17, 7); + const frame = (await through(zlib.createZstdCompress({ params, dictionary }), [input])) as Buffer; + expect(zlib.zstdDecompressSync(frame, { dictionary }).equals(input)).toBe(true); + }); + }); + + // Each input is one of many from its seed that gives other bytes once the check that it is for is taken out. + describe("the optimal parser", () => { + /** xorshift32 */ + class Input { + bytes: number[] = []; + #x: number; + constructor(seed: number) { + this.#x = Math.imul(seed, 2654435761) >>> 0; + } + next() { + this.#x ^= this.#x << 13; + this.#x ^= this.#x >>> 17; + this.#x ^= this.#x << 5; + return (this.#x >>>= 0); + } + range(from: number, to: number) { + return from + (this.next() % (to - from + 1)); + } + pick(values: number[]) { + return values[this.next() % values.length]; + } + some(length: number) { + return Array.from({ length }, () => this.next() & 255); + } + noise(length: number) { + return this.add(this.some(length)); + } + copy(offset: number, length: number) { + for (let i = 0; i < length; i++) this.bytes.push(this.bytes[this.bytes.length - offset]); + return this; + } + add(bytes: number[]) { + for (const byte of bytes) this.bytes.push(byte); + return this; + } + /** Many of these make one literal cheaper than none. */ + shortMatchAndLiteral() { + return this.copy(this.range(50, Math.min(2000, this.bytes.length)), this.range(5, 14)).noise(1); + } + } + const digest = (frame: Buffer) => `${frame.length} ${Bun.hash.xxHash32(frame).toString(16)}`; + + // The first block ends with a match in the dictionary, a little further back than the window is large. For the + // second block the dictionary is given up, and the same distance leads to the right bytes again. + it.each([ + [26, false, "1520 a9ba4dd5"], + [10, false, "2316 30a8306d"], + [26, true, "9125 e0ee4a07"], + ])( + "a repeat offset beyond the window is not used once the dictionary is given up: %i, %p", + async (seed, known, expected) => { + const input = new Input(seed); + const windowLog = input.range(10, 12); + const window = 1 << windowLog; + const offset = window + input.range(1, 200); + const dictionarySize = input.range(offset + 50, 4 * window); + input.noise(dictionarySize); + const first = input.next() % 2 < 1 ? input.range(window >> 2, window - 50) : window; + const [length, after] = [input.range(8, 40), input.range(0, 5)]; + input + .noise(first - length - after) + .copy(offset, length) + .noise(after); + for (let i = input.range(1, 6); i > 0; i--) input.noise(input.range(1, 250)).copy(offset, input.range(4, 40)); + // From 8 KB on the tables of the dictionary are copied, and it is the older segment. Else it is attached. + const more = known ? 8300 + input.range(0, 2999) : input.range(1, 3000); + while (input.bytes.length - dictionarySize < first + more) { + if (input.next() % 3 < 1) input.copy(offset, input.range(4, 40)); + else input.noise(input.range(1, 400)); + } + const dictionary = Buffer.from(input.bytes).subarray(0, dictionarySize); + const source = Buffer.from(input.bytes).subarray(dictionarySize); + const params = { + [c.ZSTD_c_strategy]: input.range(c.ZSTD_btopt, c.ZSTD_btultra2), + [c.ZSTD_c_windowLog]: windowLog, + [c.ZSTD_c_minMatch]: input.range(3, 5), + [c.ZSTD_c_searchLog]: input.range(2, 6), + }; + const stream = zlib.createZstdCompress({ + params, + dictionary, + pledgedSrcSize: known ? source.length : undefined, + }); + stream.write(source.subarray(0, first)); + stream.flush(); + const frame = (await through(stream, [source.subarray(first)])) as Buffer; + expect(digest(frame)).toBe(expected); + expect(zlib.zstdDecompressSync(frame, { dictionary }).equals(source)).toBe(true); + }, + ); + + // facebook/zstd b0e8580dc. The input ends with `q 6 x 1`. `q 5 x 2` is a little before, so that the repeat offset + // gives `x` and not what follows, and `x 1` is far before. + it("does not look at the byte after the block for a literal that replaces the end of a match", () => { + const input = new Input(68); + const x = input.some(input.range(8, 40)); + input + .noise(200) + .add(x) + .add([1]) + .noise(input.pick([300, 5000, 40000])); + for (let i = input.range(30, 600); i > 0; i--) input.shortMatchAndLiteral(); + const q = input.some(input.range(6, 20)); + for (let i = input.range(0, 29); i > 0; i--) { + const [a, b] = [input.some(input.range(6, 20)), input.some(input.range(6, 20))]; + for (const w of [3, 4]) + input + .add(a) + .add([w]) + .add(b) + .add([w + 10]); + } + input.add(q).add([5]).add(x).add([2]); + for (let i = input.range(0, 2); i > 0; i--) input.shortMatchAndLiteral(); + input.add(q).add([6]).add(x).add([1]); + const params = { + [c.ZSTD_c_strategy]: input.range(c.ZSTD_btopt, c.ZSTD_btultra2), + [c.ZSTD_c_windowLog]: input.range(19, 21), + [c.ZSTD_c_searchLog]: input.range(3, 7), + [c.ZSTD_c_minMatch]: input.range(3, 5), + [c.ZSTD_c_targetLength]: input.pick([48, 64, 200, 999]), + }; + expect(digest(roundTrip(Buffer.from(input.bytes), params))).toBe("41368 2e959c78"); + }); + + // facebook/zstd 22574d848. Matches for `text` are short and overlap, as both copies of it are spoilt every so often, + // at other places. Its last byte is only in a third, short copy. + it("has room for a series that goes on to the last position a match can lead to, and a literal", () => { + const input = new Input(255); + const size = input.pick([4095, 4095, 4095, 4095, 4094, 4096, 4093, 4097]); + const text = input.some(size); + const step = input.range(40, 300); + const spoilt = (first: number) => + text.map((byte, i) => (i === size - 1 || (i >= first && (i - first) % step === 0) ? byte ^ 0x55 : byte)); + const end = text.slice(size - input.range(5, 40)); + input.noise(100); + const far = input.next() % 2 < 1; + if (far) input.add(end).noise(input.pick([10, 1000, 20000])); + input + .add(spoilt(step >> 1)) + .noise(20) + .add(spoilt(step - 1)) + .noise(20); + for (let i = input.range(100, 900); i > 0; i--) input.shortMatchAndLiteral(); + if (!far) input.add(end); + input.noise(input.range(12, 60)).add(text).noise(input.range(20, 400)); + const params = { + [c.ZSTD_c_strategy]: input.range(c.ZSTD_btopt, c.ZSTD_btultra2), + [c.ZSTD_c_windowLog]: 20, + [c.ZSTD_c_searchLog]: input.range(3, 7), + [c.ZSTD_c_minMatch]: input.range(3, 5), + [c.ZSTD_c_targetLength]: input.pick([4095, 4096, 131072]), + }; + expect(digest(roundTrip(Buffer.from(input.bytes), params))).toBe("5917 9ffc7ec4"); + }); + + // facebook/zstd 4d8a2132d. No 4 bytes come twice before the last 9, which are a copy from far away that the small + // tables have lost by the position after its first, and half of the block is one byte, so that literals are cheap. + it("has a price for 128 KB of literals", () => { + const input = new Input(4); + const size = 128 << 10; + const last = [0, 0, 0, input.range(1, 254), 0, 0, 0, input.range(1, 254)]; + last.push(input.next() % 3 < 1 ? 0 : input.range(1, 254)); + const from = 4 * input.range(1, (size - 70000) >> 2); + const pairs = Array.from({ length: 254 * 254 }, (_, i) => [1 + Math.floor(i / 254), 1 + (i % 254)]); + for (let i = pairs.length - 1; i > 0; i--) { + const j = input.next() % (i + 1); + [pairs[i], pairs[j]] = [pairs[j], pairs[i]]; + } + for (const pair of pairs) { + if (input.bytes.length === from) input.add(last); + if (input.add([0, 0]).add(pair).bytes.length >= size) break; + } + input.bytes.length = size - last.length; + input.bytes[input.bytes.length - 1] ||= 7; + input.add(last); + const params = { + [c.ZSTD_c_strategy]: input.range(c.ZSTD_btopt, c.ZSTD_btultra2), + [c.ZSTD_c_minMatch]: input.range(4, 6), + [c.ZSTD_c_windowLog]: 18, + [c.ZSTD_c_targetLength]: input.pick([48, 64, 256, 999]), + [c.ZSTD_c_hashLog]: input.range(14, 19), + [c.ZSTD_c_chainLog]: input.range(6, 8), + [c.ZSTD_c_searchLog]: 1, + }; + expect(digest(roundTrip(Buffer.from(input.bytes), params))).toBe("81886 6893fa8c"); + }); + }); +}); diff --git a/test/js/node/process/process.test.js b/test/js/node/process/process.test.js index e3a2a7aca038..7e3cf0dbb0b2 100644 --- a/test/js/node/process/process.test.js +++ b/test/js/node/process/process.test.js @@ -776,7 +776,6 @@ it("process.versions", async () => { lolhtml: "lolhtml", ares: "cares", libdeflate: "libdeflate", - zstd: "zstd", lshpack: "lshpack", }; @@ -802,6 +801,9 @@ it("process.versions", async () => { expect(process.versions).toHaveProperty("sqlite"); expect(process.versions.sqlite).toMatch(/^3\.\d+\.\d+$/); + // The version of zstd that Bun's own implementation behaves like. + expect(process.versions.zstd).toBe("1.5.7"); + // The version of TypeScript that `bun check` behaves like. expect(process.versions.typescript).toMatch(/^\d+\.\d+\.\d+$/); }); diff --git a/test/js/node/zlib/zlib-estimated-size-gc.test.ts b/test/js/node/zlib/zlib-estimated-size-gc.test.ts index 4970f74b872e..36b74e8afa86 100644 --- a/test/js/node/zlib/zlib-estimated-size-gc.test.ts +++ b/test/js/node/zlib/zlib-estimated-size-gc.test.ts @@ -12,7 +12,7 @@ import zlib from "node:zlib"; // estimateShallowMemoryUsageOf(cell) == sizeof(cell) + estimated_size(). `min` // is a floor below the per-mode footprint the constructor caches (brotli encode -// 5143, brotli decode 855, zstd compress 5272, zstd decompress 95968). +// 5143, brotli decode 855, zstd compress about 15000, zstd decompress about 36000). function checkFootprintStableAcrossClose(create: () => any, min: number) { const engine = create(); engine.on("error", () => {}); @@ -34,11 +34,11 @@ test("brotli decompress: estimated size stays tied to construction mode across c }); test("zstd compress: estimated size stays tied to construction mode across close", () => { - checkFootprintStableAcrossClose(() => zlib.createZstdCompress(), 5000); + checkFootprintStableAcrossClose(() => zlib.createZstdCompress(), 12000); }); test("zstd decompress: estimated size stays tied to construction mode across close", () => { - checkFootprintStableAcrossClose(() => zlib.createZstdDecompress(), 90000); + checkFootprintStableAcrossClose(() => zlib.createZstdDecompress(), 30000); }); // GC-safety guard: drive a write so the JS thread enters with_mut, then force diff --git a/test/js/web/streams/compression.test.ts b/test/js/web/streams/compression.test.ts index a6229848f860..0579156ae455 100644 --- a/test/js/web/streams/compression.test.ts +++ b/test/js/web/streams/compression.test.ts @@ -347,6 +347,18 @@ describe("CompressionStream and DecompressionStream", () => { expect(out).toBe("payload"); }); + test("decompresses a zstd stream with many empty chunks in the middle of a frame", async () => { + const frame = zlib.zstdCompressSync(Buffer.from("a frame in two parts")); + const ds = new DecompressionStream("zstd"); + const writer = ds.writable.getWriter(); + const read = new Response(ds.readable).text(); + await writer.write(frame.subarray(0, 10)); + for (let i = 0; i < 20; i++) await writer.write(new Uint8Array(0)); + await writer.write(frame.subarray(10)); + await writer.close(); + expect(await read).toBe("a frame in two parts"); + }); + test("rejects trailing garbage after a zstd frame", async () => { const frame = zlib.zstdCompressSync(Buffer.from("hello")); const withJunk = Buffer.concat([frame, Buffer.from([0xde, 0xad, 0xbe, 0xef])]);