From 4c057a831e98d6d9e561d8db11ab7e76fade9053 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 19:30:32 +0000 Subject: [PATCH 01/33] Remove libarchive from the build Drop the dependency script, its six patches, the update workflow, the extern "C" bindings in bun_libarchive, process.versions.libarchive and the license, docs and leak-suppression entries. bun_libarchive and its consumers do not compile after this commit. The tar and gzip formats they need are reimplemented in Rust in the commits that follow. --- .github/workflows/update-libarchive.yml | 106 -- .gitignore | 1 - CLAUDE.md | 2 +- CONTRIBUTING.md | 8 - LICENSE.md | 1 - docs/project/contributing.mdx | 8 - docs/project/license.mdx | 1 - .../archive_string-codepage-cache.patch | 81 -- .../archive_write_add_filter_gzip.c.patch | 57 - patches/libarchive/gzip-verify-trailer.patch | 63 -- patches/libarchive/nonblocking-read.patch | 1008 ----------------- .../libarchive/select-registered-only.patch | 116 -- patches/libarchive/tar-npm-compat.patch | 175 --- scripts/build/CLAUDE.md | 2 +- scripts/build/binary-expectations.ts | 6 +- scripts/build/deps/README.md | 4 +- scripts/build/deps/cares.ts | 4 +- scripts/build/deps/index.ts | 7 +- scripts/build/deps/libarchive.ts | 330 ------ scripts/build/deps/zlib.ts | 2 +- scripts/build/deps/zstd.ts | 2 +- scripts/build/fetch-cli.ts | 2 +- scripts/build/source.ts | 13 +- src/jsc/bindings/BunProcess.cpp | 1 - .../bindings/workaround-missing-symbols.cpp | 2 +- src/libarchive/lib.rs | 890 --------------- test/js/node/process/process.test.js | 1 - test/leaksan.supp | 1 - 28 files changed, 19 insertions(+), 2875 deletions(-) delete mode 100644 .github/workflows/update-libarchive.yml delete mode 100644 patches/libarchive/archive_string-codepage-cache.patch delete mode 100644 patches/libarchive/archive_write_add_filter_gzip.c.patch delete mode 100644 patches/libarchive/gzip-verify-trailer.patch delete mode 100644 patches/libarchive/nonblocking-read.patch delete mode 100644 patches/libarchive/select-registered-only.patch delete mode 100644 patches/libarchive/tar-npm-compat.patch delete mode 100644 scripts/build/deps/libarchive.ts diff --git a/.github/workflows/update-libarchive.yml b/.github/workflows/update-libarchive.yml deleted file mode 100644 index 760911224b29..000000000000 --- a/.github/workflows/update-libarchive.yml +++ /dev/null @@ -1,106 +0,0 @@ -name: Update libarchive - -permissions: - contents: read - -on: - schedule: - - cron: "0 3 * * 0" - workflow_dispatch: - -jobs: - check-update: - if: github.repository == 'oven-sh/bun' - runs-on: ubuntu-latest - permissions: - contents: write - pull-requests: write - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Check libarchive version - id: check-version - run: | - set -euo pipefail - - DEP_FILE=scripts/build/deps/libarchive.ts - CURRENT_VERSION=$(sed -nE 's/^const LIBARCHIVE_COMMIT = "([0-9a-f]{40})";$/\1/p' "$DEP_FILE") - - if [ -z "$CURRENT_VERSION" ]; then - echo "Error: Could not find LIBARCHIVE_COMMIT in $DEP_FILE" - exit 1 - fi - - # Validate that it looks like a git hash - if ! [[ $CURRENT_VERSION =~ ^[0-9a-f]{40}$ ]]; then - echo "Error: Invalid git hash format in $DEP_FILE" - echo "Found: $CURRENT_VERSION" - echo "Expected: 40 character hexadecimal string" - exit 1 - fi - - echo "current=$CURRENT_VERSION" >> $GITHUB_OUTPUT - - LATEST_RELEASE=$(curl -sL https://api.github.com/repos/libarchive/libarchive/releases/latest) - if [ -z "$LATEST_RELEASE" ]; then - echo "Error: Failed to fetch latest release from GitHub API" - exit 1 - fi - - LATEST_TAG=$(echo "$LATEST_RELEASE" | jq -r '.tag_name') - if [ -z "$LATEST_TAG" ] || [ "$LATEST_TAG" = "null" ]; then - echo "Error: Could not extract tag name from GitHub API response" - exit 1 - fi - - LATEST_TAG_SHA=$(curl -sL "https://api.github.com/repos/libarchive/libarchive/git/refs/tags/$LATEST_TAG" | jq -r '.object.sha') - if [ -z "$LATEST_TAG_SHA" ] || [ "$LATEST_TAG_SHA" = "null" ]; then - echo "Error: Could not fetch SHA for tag $LATEST_TAG" - exit 1 - fi - LATEST_SHA=$(curl -sL "https://api.github.com/repos/libarchive/libarchive/git/tags/$LATEST_TAG_SHA" | jq -r '.object.sha') - if [ -z "$LATEST_SHA" ] || [ "$LATEST_SHA" = "null" ]; then - echo "Error: Could not fetch SHA for tag $LATEST_TAG @ $LATEST_TAG_SHA" - exit 1 - fi - - if ! [[ $LATEST_SHA =~ ^[0-9a-f]{40}$ ]]; then - echo "Error: Invalid SHA format received from GitHub" - echo "Found: $LATEST_SHA" - echo "Expected: 40 character hexadecimal string" - exit 1 - fi - - echo "latest=$LATEST_SHA" >> $GITHUB_OUTPUT - echo "tag=$LATEST_TAG" >> $GITHUB_OUTPUT - - - name: Update version if needed - if: success() && steps.check-version.outputs.current != steps.check-version.outputs.latest - env: - LATEST: ${{ steps.check-version.outputs.latest }} - run: | - set -euo pipefail - sed -i -E 's/^(const LIBARCHIVE_COMMIT = ")[0-9a-f]{40}(";)$/\1'"$LATEST"'\2/' scripts/build/deps/libarchive.ts - - - name: Create Pull Request - if: success() && steps.check-version.outputs.current != steps.check-version.outputs.latest - uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - add-paths: | - scripts/build/deps/libarchive.ts - commit-message: "deps: update libarchive to ${{ steps.check-version.outputs.tag }} (${{ steps.check-version.outputs.latest }})" - title: "deps: update libarchive to ${{ steps.check-version.outputs.tag }}" - delete-branch: true - branch: deps/update-libarchive - body: | - ## What does this PR do? - - Updates libarchive to version ${{ steps.check-version.outputs.tag }} - - Compare: https://github.com/libarchive/libarchive/compare/${{ steps.check-version.outputs.current }}...${{ steps.check-version.outputs.latest }} - - Auto-updated by [this workflow](https://github.com/oven-sh/bun/actions/workflows/update-libarchive.yml) diff --git a/.gitignore b/.gitignore index 6a7a508c79ed..4b09e2f7fd5a 100644 --- a/.gitignore +++ b/.gitignore @@ -153,7 +153,6 @@ codegen-for-zig-team.tar.gz /src/deps/boringssl /src/deps/brotli /src/deps/c*ares -/src/deps/libarchive /src/deps/libdeflate /src/deps/libuv /src/deps/lol*html diff --git a/CLAUDE.md b/CLAUDE.md index ffcddefc6811..fa3efe74eca0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -147,7 +147,7 @@ The Rust side is a Cargo workspace of ~200 crates rooted at `Cargo.toml`. The ke #### Vendored Dependencies (`vendor/`) -Third-party native libraries are vendored locally and can be read from disk (not git submodules): boringssl (TLS/crypto), brotli, cares (async DNS), hdrhistogram, highway (SIMD), libarchive (tar/zip), libdeflate, libuv (Windows event loop), lolhtml (HTML rewriter), lshpack (HTTP/2 HPACK), lsqpack + lsquic (HTTP/3), mimalloc (allocator), nodejs (headers), picohttpparser, rust-argon2 (argon2 for `Bun.password`, Rust), tinycc (FFI JIT, fork: oven-sh/tinycc), WebKit (JavaScriptCore), zlib (zlib-ng), zstd. Build configuration for these is in `scripts/build/deps/*.ts`. +Third-party native libraries are vendored locally and can be read from disk (not git submodules): boringssl (TLS/crypto), brotli, cares (async DNS), hdrhistogram, highway (SIMD), libdeflate, libuv (Windows event loop), lolhtml (HTML rewriter), lshpack (HTTP/2 HPACK), lsqpack + lsquic (HTTP/3), mimalloc (allocator), nodejs (headers), picohttpparser, rust-argon2 (argon2 for `Bun.password`, Rust), tinycc (FFI JIT, fork: oven-sh/tinycc), WebKit (JavaScriptCore), zlib (zlib-ng), zstd. Build configuration for these is in `scripts/build/deps/*.ts`. ### JavaScript Class Implementation (C++) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index af1aee0fcd4f..deed28d98cf8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -342,14 +342,6 @@ $ sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 $ sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100 ``` -### libarchive - -If you see an error on macOS when compiling `libarchive`, run: - -```bash -$ brew install pkg-config -``` - ### macOS `library not found for -lSystem` If you see this error when compiling, run: diff --git a/LICENSE.md b/LICENSE.md index 3e1b37f1c940..c45af52beec6 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -22,7 +22,6 @@ Bun statically links these libraries: |---------|---------| | [`boringssl`](https://boringssl.googlesource.com/boringssl/) | [several licenses](https://boringssl.googlesource.com/boringssl/+/refs/heads/master/LICENSE) | | [`brotli`](https://github.com/google/brotli) | MIT | -| [`libarchive`](https://github.com/libarchive/libarchive) | [several licenses](https://github.com/libarchive/libarchive/blob/master/COPYING) | | [`lol-html`](https://github.com/cloudflare/lol-html/tree/master/c-api) | BSD 3-Clause | | [`ls-hpack`](https://github.com/litespeedtech/ls-hpack) | MIT | | [`ls-qpack`](https://github.com/litespeedtech/ls-qpack) | MIT | diff --git a/docs/project/contributing.mdx b/docs/project/contributing.mdx index 60d34ed93ba9..2e55926c53d5 100644 --- a/docs/project/contributing.mdx +++ b/docs/project/contributing.mdx @@ -346,14 +346,6 @@ sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100 ``` -### libarchive - -If you see an error on macOS when compiling `libarchive`, run: - -```bash -brew install pkg-config -``` - ### macOS `library not found for -lSystem` If you see this error when compiling, run: diff --git a/docs/project/license.mdx b/docs/project/license.mdx index 67dc787c9943..ef492d283ded 100644 --- a/docs/project/license.mdx +++ b/docs/project/license.mdx @@ -27,7 +27,6 @@ Bun statically links these libraries: | ------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------- | | [`boringssl`](https://boringssl.googlesource.com/boringssl/) | [several licenses](https://boringssl.googlesource.com/boringssl/+/refs/heads/master/LICENSE) | | [`brotli`](https://github.com/google/brotli) | MIT | -| [`libarchive`](https://github.com/libarchive/libarchive) | [several licenses](https://github.com/libarchive/libarchive/blob/master/COPYING) | | [`lol-html`](https://github.com/cloudflare/lol-html/tree/master/c-api) | BSD 3-Clause | | [`ls-hpack`](https://github.com/litespeedtech/ls-hpack) | MIT (bundled xxhash is BSD 2-Clause) | | [`ls-qpack`](https://github.com/litespeedtech/ls-qpack) | MIT | diff --git a/patches/libarchive/archive_string-codepage-cache.patch b/patches/libarchive/archive_string-codepage-cache.patch deleted file mode 100644 index 0f12156bb63c..000000000000 --- a/patches/libarchive/archive_string-codepage-cache.patch +++ /dev/null @@ -1,81 +0,0 @@ ---- a/libarchive/archive_string.c -+++ b/libarchive/archive_string.c -@@ -1561,10 +1561,10 @@ make_codepage_from_charset(const char *charset) - } - - /* -- * Return ANSI Code Page of current locale set by setlocale(). -+ * Return ANSI Code Page of current locale set by setlocale() (uncached). - */ - static unsigned --get_current_codepage(void) -+get_current_codepage_uncached(void) - { - char *locale, *p; - unsigned cp; -@@ -1585,6 +1585,28 @@ get_current_codepage(void) - return (cp); - } - -+/* -+ * Bun: cache the codepage after the first call. The process never changes -+ * its locale at runtime, but `bun install` creates a charset converter per -+ * extracted archive on many worker threads at once, and hammering the UCRT's -+ * setlocale(LC_CTYPE, NULL) query from all of them concurrently is both -+ * wasted work and a needless trip through the CRT locale machinery (and its -+ * heap) on every archive. Compute once; later callers read the cached value. -+ */ -+static unsigned -+get_current_codepage(void) -+{ -+ static volatile LONG cached_cp = -1; -+ LONG cp = cached_cp; -+ -+ if (cp == -1) { -+ cp = (LONG)get_current_codepage_uncached(); -+ InterlockedCompareExchange(&cached_cp, cp, -1); -+ cp = cached_cp; -+ } -+ return ((unsigned)cp); -+} -+ - /* - * Translation table between Locale Name and ACP/OEMCP. - */ -@@ -1638,10 +1660,10 @@ static struct { - }; - - /* -- * Return OEM Code Page of current locale set by setlocale(). -+ * Return OEM Code Page of current locale set by setlocale() (uncached). - */ - static unsigned --get_current_oemcp(void) -+get_current_oemcp_uncached(void) - { - int i; - char *locale, *p; -@@ -1663,6 +1685,23 @@ get_current_oemcp(void) - } - return (GetOEMCP()); - } -+ -+/* -+ * Bun: cached for the same reason as get_current_codepage() above. -+ */ -+static unsigned -+get_current_oemcp(void) -+{ -+ static volatile LONG cached_ocp = -1; -+ LONG ocp = cached_ocp; -+ -+ if (ocp == -1) { -+ ocp = (LONG)get_current_oemcp_uncached(); -+ InterlockedCompareExchange(&cached_ocp, ocp, -1); -+ ocp = cached_ocp; -+ } -+ return ((unsigned)ocp); -+} - #else - - /* diff --git a/patches/libarchive/archive_write_add_filter_gzip.c.patch b/patches/libarchive/archive_write_add_filter_gzip.c.patch deleted file mode 100644 index 28b236d0e517..000000000000 --- a/patches/libarchive/archive_write_add_filter_gzip.c.patch +++ /dev/null @@ -1,57 +0,0 @@ ---- a/libarchive/archive_write_add_filter_gzip.c -+++ b/libarchive/archive_write_add_filter_gzip.c -@@ -60,6 +60,7 @@ struct gzip { - int compression_level; - int timestamp; - char *original_filename; -+ unsigned char os; - #ifdef HAVE_ZLIB_H - z_stream stream; - uint64_t total_in; -@@ -109,6 +110,7 @@ archive_write_add_filter_gzip(struct archive *a) - if (gzip == NULL) - goto memerr; - gzip->original_filename = NULL; -+ gzip->os = 3; /* default Unix */ - #ifdef HAVE_ZLIB_H - gzip->compression_level = Z_DEFAULT_COMPRESSION; - -@@ -184,6 +186,29 @@ archive_compressor_gzip_options(struct archive_write_filter *f, const char *key, - } - return (ARCHIVE_OK); - } -+ if (strcmp(key, "os") == 0) { -+ if (value == NULL) -+ return (ARCHIVE_WARN); -+ -+ if (strcmp(value, "FAT") == 0) gzip->os = 0; -+ else if (strcmp(value, "Amiga") == 0) gzip->os = 1; -+ else if (strcmp(value, "VMS") == 0 || strcmp(value, "OpenVMS") == 0) gzip->os = 2; -+ else if (strcmp(value, "Unix") == 0) gzip->os = 3; -+ else if (strcmp(value, "VM") == 0 || strcmp(value, "VM/CMS") == 0) gzip->os = 4; -+ else if (strcmp(value, "Atari TOS") == 0) gzip->os = 5; -+ else if (strcmp(value, "HPFS") == 0) gzip->os = 6; -+ else if (strcmp(value, "Macintosh") == 0) gzip->os = 7; -+ else if (strcmp(value, "Z-System") == 0) gzip->os = 8; -+ else if (strcmp(value, "CP/M") == 0) gzip->os = 9; -+ else if (strcmp(value, "TOPS-20") == 0) gzip->os = 10; -+ else if (strcmp(value, "NTFS") == 0) gzip->os = 11; -+ else if (strcmp(value, "QDOS") == 0) gzip->os = 12; -+ else if (strcmp(value, "Acorn RISCOS") == 0) gzip->os = 13; -+ else if (strcmp(value, "Unknown") == 0) gzip->os = 255; -+ else return (ARCHIVE_WARN); -+ -+ return (ARCHIVE_OK); -+ } - - /* Note: The "warn" return is just to inform the options - * supervisor that we didn't handle it. It will generate -@@ -244,7 +269,7 @@ archive_compressor_gzip_open(struct archive_write_filter *f) - } else { - gzip->compressed[8] = 0; - } -- gzip->compressed[9] = 3; /* OS=Unix */ -+ gzip->compressed[9] = gzip->os; - gzip->stream.next_out += 10; - gzip->stream.avail_out -= 10; - diff --git a/patches/libarchive/gzip-verify-trailer.patch b/patches/libarchive/gzip-verify-trailer.patch deleted file mode 100644 index c4e6bdb95df7..000000000000 --- a/patches/libarchive/gzip-verify-trailer.patch +++ /dev/null @@ -1,63 +0,0 @@ ---- a/libarchive/archive_read_support_filter_gzip.c -+++ b/libarchive/archive_read_support_filter_gzip.c -@@ -67,6 +67,11 @@ - * a gzip header). - */ - char trailer_pending; -+ /* -+ * BUN PATCH: uncompressed byte count of the current member, -+ * modulo 2^32, checked against the ISIZE trailer field. -+ */ -+ uint32_t member_isize; - }; - - /* Gzip Filter. */ -@@ -374,6 +379,7 @@ - - /* Initialize CRC accumulator. */ - gzip->crc = crc32(0L, NULL, 0); -+ gzip->member_isize = 0; - - /* - * BUN PATCH: Initialize compression library. next_in / avail_in -@@ -455,7 +461,18 @@ - if (p == NULL) - return (ARCHIVE_FATAL); - -- /* XXX TODO: Verify the length and CRC. */ -+ /* -+ * BUN PATCH: compare the trailer with what was inflated. Upstream -+ * has a TODO here, so a corrupted member decompressed without any -+ * error. -+ */ -+ if (archive_le32dec(p) != (uint32_t)gzip->crc || -+ archive_le32dec(p + 4) != gzip->member_isize) { -+ archive_set_error(&f->archive->archive, -+ ARCHIVE_ERRNO_MISC, -+ "gzip trailer mismatch: bad CRC32 or uncompressed size"); -+ return (ARCHIVE_FATAL); -+ } - - /* We've verified the trailer, so consume it now. */ - __archive_read_filter_consume(f->upstream, 8); -@@ -470,6 +487,8 @@ - struct gzip *gzip = f->data; - size_t decompressed; - ssize_t avail_in, max_in; -+ unsigned char *out_before; -+ size_t produced; - int ret; - - /* Empty our output buffer. */ -@@ -532,7 +551,11 @@ - gzip->stream.avail_in = (uInt)avail_in; - - /* Decompress and consume some of that data. */ -+ out_before = gzip->stream.next_out; - ret = inflate(&(gzip->stream), 0); -+ produced = gzip->stream.next_out - out_before; -+ gzip->crc = crc32(gzip->crc, out_before, (uInt)produced); -+ gzip->member_isize += (uint32_t)produced; - switch (ret) { - case Z_OK: /* Decompressor made some progress. */ - __archive_read_filter_consume(f->upstream, diff --git a/patches/libarchive/nonblocking-read.patch b/patches/libarchive/nonblocking-read.patch deleted file mode 100644 index 7fd118debbf9..000000000000 --- a/patches/libarchive/nonblocking-read.patch +++ /dev/null @@ -1,1008 +0,0 @@ ---- a/libarchive/archive_read.c -+++ b/libarchive/archive_read.c -@@ -643,33 +643,67 @@ _archive_read_next_header2(struct archive *_a, struct archive_entry *entry) - ARCHIVE_STATE_DATA_RECOVERY, - "archive_read_next_header"); - -- archive_entry_clear(entry); -- archive_clear_error(&a->archive); -- - /* -- * If client didn't consume entire data, skip any remainder -- * (This is especially important for GNU incremental directories.) -- * A header that failed to parse (DATA_RECOVERY) still needs the -- * same treatment: whatever of its body the format reader left -- * unconsumed must be skipped before we can read the next header. -+ * BUN PATCH: when resuming after an ARCHIVE_RETRY from a -+ * non-blocking source, the format reader has already -+ * populated `entry` from consumed extension headers (pax, -+ * GNU long name/link). Clearing it here would discard that -+ * work, and since no data has been read for the pending -+ * entry there is nothing to skip. Skip straight to the -+ * format reader which will pick up where it left off. - */ -- if (a->archive.state == ARCHIVE_STATE_DATA || -- a->archive.state == ARCHIVE_STATE_DATA_RECOVERY) { -- r1 = archive_read_data_skip(&a->archive); -- if (r1 == ARCHIVE_EOF) -- archive_set_error(&a->archive, EIO, -- "Premature end-of-file"); -- if (r1 == ARCHIVE_EOF || r1 == ARCHIVE_FATAL) { -- a->archive.state = ARCHIVE_STATE_FATAL; -- return (ARCHIVE_FATAL); -+ if (!a->read_header_in_progress) { -+ archive_entry_clear(entry); -+ archive_clear_error(&a->archive); -+ -+ /* -+ * If client didn't consume entire data, skip any remainder -+ * (This is especially important for GNU incremental directories.) -+ * A header that failed to parse (DATA_RECOVERY) still needs the -+ * same treatment: whatever of its body the format reader left -+ * unconsumed must be skipped before we can read the next header. -+ */ -+ if (a->archive.state == ARCHIVE_STATE_DATA || -+ a->archive.state == ARCHIVE_STATE_DATA_RECOVERY) { -+ r1 = archive_read_data_skip(&a->archive); -+ /* -+ * BUN PATCH: the skip may run out of buffered -+ * bytes when the source is non-blocking. The -+ * format's skip handler has already recorded how -+ * much remains, and archive_read_data_skip() -+ * leaves the state untouched on ARCHIVE_RETRY, so -+ * the next next_header() call will re-enter here -+ * and continue the skip. -+ */ -+ if (r1 == ARCHIVE_RETRY) -+ return (ARCHIVE_RETRY); -+ if (r1 == ARCHIVE_EOF) -+ archive_set_error(&a->archive, EIO, -+ "Premature end-of-file"); -+ if (r1 == ARCHIVE_EOF || r1 == ARCHIVE_FATAL) { -+ a->archive.state = ARCHIVE_STATE_FATAL; -+ return (ARCHIVE_FATAL); -+ } - } -- } - -- /* Record start-of-header offset in uncompressed stream. */ -- a->header_position = a->filter->position; -+ /* Record start-of-header offset in uncompressed stream. */ -+ a->header_position = a->filter->position; - -- ++_a->file_count; -+ ++_a->file_count; -+ } - r2 = (a->format->read_header)(a, entry); -+ /* -+ * BUN PATCH: the format reader sets read_header_in_progress -+ * itself when a non-blocking source yields mid-header (see -+ * `bun_retry` in the tar reader). Clear it on any terminal -+ * result so the next call runs the full reset above. An -+ * ARCHIVE_RETRY that did not set the flag — upstream tar's -+ * pre-existing damaged-block skip — therefore behaves exactly -+ * as upstream: the next call re-runs archive_entry_clear / -+ * archive_clear_error. -+ */ -+ if (r2 != ARCHIVE_RETRY) -+ a->read_header_in_progress = 0; - - /* - * EOF and FATAL are persistent at this layer. By -@@ -986,6 +1020,14 @@ archive_read_data_skip(struct archive *_a) - if (r == ARCHIVE_EOF) - r = ARCHIVE_OK; - -+ /* -+ * BUN PATCH: a non-blocking source ran dry mid-skip. Leave the -+ * state as-is so the next archive_read_next_header() re-enters -+ * the skip with whatever the format recorded as still owed. -+ */ -+ if (r == ARCHIVE_RETRY) -+ return (r); -+ - if (r == ARCHIVE_FATAL) - a->archive.state = ARCHIVE_STATE_FATAL; - else -@@ -1447,6 +1489,33 @@ __archive_read_filter_ahead(struct archive_read_filter *f, - } - bytes_read = (f->vtable->read)(f, - &f->client_buff); -+ /* -+ * BUN PATCH: non-blocking read support. -+ * -+ * A client reader that is being fed incrementally -+ * (e.g. tarball bytes arriving from the network -+ * during `bun install`) returns ARCHIVE_RETRY when -+ * no data is currently available. Unlike a real -+ * error this must not poison the filter: whatever -+ * has already been copied into `f->buffer` -+ * stays put so the next call with the same `min` -+ * resumes exactly where we left off, and the caller -+ * can yield its thread and try again once more -+ * input has arrived. -+ * -+ * Callers that do not opt in (pass avail == NULL or -+ * do not check for ARCHIVE_RETRY) will see NULL and -+ * treat it as truncated input, which matches the -+ * pre-patch behaviour for non-streaming sources. -+ */ -+ if (bytes_read == ARCHIVE_RETRY) { -+ f->client_total = f->client_avail = 0; -+ f->client_next = -+ f->client_buff = NULL; -+ if (avail != NULL) -+ *avail = ARCHIVE_RETRY; -+ return (NULL); -+ } - if (bytes_read < 0) { /* Read error. */ - f->client_total = f->client_avail = 0; - f->client_next = -@@ -1641,7 +1710,14 @@ advance_file_pointer(struct archive_read_filter *f, int64_t request) - bytes_read = (f->vtable->read)(f, &f->client_buff); - if (bytes_read < 0) { - f->client_buff = NULL; -- f->fatal = 1; -+ /* -+ * BUN PATCH: see the matching comment in -+ * __archive_read_filter_ahead. A non-blocking -+ * reader returning ARCHIVE_RETRY must not mark -+ * the filter fatal. -+ */ -+ if (bytes_read != ARCHIVE_RETRY) -+ f->fatal = 1; - return (bytes_read); - } - ---- a/libarchive/archive_read_private.h -+++ b/libarchive/archive_read_private.h -@@ -189,6 +189,18 @@ struct archive_read { - */ - int64_t entry_bytes_declared; - -+ /* -+ * BUN PATCH: set by the format's read_header callback when it -+ * returns ARCHIVE_RETRY from a non-blocking source and wants -+ * the next call to skip archive_entry_clear() so that -+ * attributes already populated by consumed extension headers -+ * (pax, GNU long name/link, etc.) survive the resume. Format -+ * readers that return ARCHIVE_RETRY without setting this — -+ * e.g. upstream tar's damaged-block skip — get the original -+ * reset-everything semantics on re-entry. -+ */ -+ int read_header_in_progress; -+ - /* Nodes and offsets of compressed data block */ - unsigned int data_start_node; - unsigned int data_end_node; ---- a/libarchive/archive_read_support_filter_gzip.c -+++ b/libarchive/archive_read_support_filter_gzip.c -@@ -60,6 +60,13 @@ struct gzip { - uint32_t mtime; - char *name; - char eof; /* True = found end of compressed data. */ -+ /* -+ * BUN PATCH: set after Z_STREAM_END so that a retry which -+ * interrupts consume_trailer() does not re-enter -+ * consume_header() (which would try to parse deflate bytes as -+ * a gzip header). -+ */ -+ char trailer_pending; - }; - - /* Gzip Filter. */ -@@ -140,8 +147,14 @@ peek_at_header(struct archive_read_filter *f, int *pbits, - * is all fixed layout. */ - len = 10; - p = __archive_read_filter_ahead(f, len, &avail); -+ /* -+ * BUN PATCH: propagate non-blocking retry. Nothing has been -+ * consumed yet, so calling peek_at_header again once more -+ * input is available will re-read the same header bytes from -+ * f->buffer. -+ */ - if (p == NULL) -- return (0); -+ return (avail == ARCHIVE_RETRY ? ARCHIVE_RETRY : 0); - /* We only support deflation- third byte must be 0x08. */ - if (memcmp(p, "\x1F\x8B\x08", 3) != 0) - return (0); -@@ -164,7 +177,7 @@ peek_at_header(struct archive_read_filter *f, int *pbits, - if (header_flags & 4) { - p = __archive_read_filter_ahead(f, len + 2, &avail); - if (p == NULL) -- return (0); -+ return (avail == ARCHIVE_RETRY ? ARCHIVE_RETRY : 0); - len += archive_le16dec(p + len); - len += 2; - } -@@ -184,7 +197,7 @@ peek_at_header(struct archive_read_filter *f, int *pbits, - len, &avail); - } - if (p == NULL) -- return (0); -+ return (avail == ARCHIVE_RETRY ? ARCHIVE_RETRY : 0); - } while (p[len - 1] != 0); - - #ifdef HAVE_ZLIB_H -@@ -208,7 +221,7 @@ peek_at_header(struct archive_read_filter *f, int *pbits, - len, &avail); - } - if (p == NULL) -- return (0); -+ return (avail == ARCHIVE_RETRY ? ARCHIVE_RETRY : 0); - } while (p[len - 1] != 0); - } - -@@ -216,7 +229,7 @@ peek_at_header(struct archive_read_filter *f, int *pbits, - if ((header_flags & 2)) { - p = __archive_read_filter_ahead(f, len + 2, &avail); - if (p == NULL) -- return (0); -+ return (avail == ARCHIVE_RETRY ? ARCHIVE_RETRY : 0); - #if 0 - int hcrc = ((int)p[len + 1] << 8) | (int)p[len]; - int crc = /* XXX TODO: Compute header CRC. */; -@@ -243,7 +256,16 @@ gzip_bidder_bid(struct archive_read_filter_bidder *b, - - (void)b; /* UNUSED */ - -- if (peek_at_header(f, &bits_checked, NULL)) -+ /* -+ * BUN PATCH: peek_at_header() may now return ARCHIVE_RETRY -+ * from a non-blocking source. Bidding has no retry channel, -+ * so only accept a positive header length as a match. -+ * Streaming callers (bun install) always deliver at least -+ * one HTTP chunk — well over the ~10-byte gzip header — -+ * before archive_read_open() is invoked, so this branch is -+ * defensive rather than expected. -+ */ -+ if (peek_at_header(f, &bits_checked, NULL) > 0) - return (bits_checked); - return (0); - } -@@ -333,12 +355,19 @@ static int - consume_header(struct archive_read_filter *f) - { - struct gzip *gzip = f->data; -- ssize_t avail, max_in; -- size_t len; -+ ssize_t len; - int ret; - - /* If this is a real header, consume it. */ - len = peek_at_header(f->upstream, NULL, gzip); -+ /* -+ * BUN PATCH: peek_at_header consumes nothing, so on -+ * ARCHIVE_RETRY the caller can simply try again later and the -+ * partial header bytes already copied into the upstream -+ * filter's buffer will still be there. -+ */ -+ if (len == ARCHIVE_RETRY) -+ return (ARCHIVE_RETRY); - if (len == 0) - return (ARCHIVE_EOF); - __archive_read_filter_consume(f->upstream, len); -@@ -346,22 +375,16 @@ consume_header(struct archive_read_filter *f) - /* Initialize CRC accumulator. */ - gzip->crc = crc32(0L, NULL, 0); - -- /* Initialize compression library. */ -- gzip->stream.next_in = (unsigned char *)(uintptr_t) -- __archive_read_filter_ahead(f->upstream, 1, &avail); -- if (avail < 0) { -- archive_set_error(&f->archive->archive, -- ARCHIVE_ERRNO_MISC, -- "Failed to read gzip input"); -- return (ARCHIVE_FATAL); -- } -- if (UINT_MAX >= SSIZE_MAX) -- max_in = SSIZE_MAX; -- else -- max_in = UINT_MAX; -- if (avail > max_in) -- avail = max_in; -- gzip->stream.avail_in = (uInt)avail; -+ /* -+ * BUN PATCH: Initialize compression library. next_in / avail_in -+ * are unused by inflateInit2 with a negative windowBits (raw -+ * deflate); the main read loop primes them before each -+ * inflate() call, so we do not need an extra read-ahead here -+ * that could itself return ARCHIVE_RETRY after the header has -+ * already been consumed. -+ */ -+ gzip->stream.next_in = NULL; -+ gzip->stream.avail_in = 0; - ret = inflateInit2(&(gzip->stream), - -15 /* Don't check for zlib header */); - -@@ -402,20 +425,33 @@ consume_trailer(struct archive_read_filter *f) - { - struct gzip *gzip = f->data; - const unsigned char *p; -- -- gzip->in_stream = 0; -- switch (inflateEnd(&(gzip->stream))) { -- case Z_OK: -- break; -- default: -- archive_set_error(&f->archive->archive, -- ARCHIVE_ERRNO_MISC, -- "Failed to clean up gzip decompressor"); -- return (ARCHIVE_FATAL); -+ ssize_t avail; -+ -+ /* -+ * BUN PATCH: inflateEnd()/in_stream=0 are only safe to run -+ * once. If a previous consume_trailer() attempt was -+ * interrupted by ARCHIVE_RETRY while reading the 8-byte -+ * footer, `trailer_pending` stays set and we skip straight to -+ * the read-ahead below. -+ */ -+ if (!gzip->trailer_pending) { -+ gzip->in_stream = 0; -+ switch (inflateEnd(&(gzip->stream))) { -+ case Z_OK: -+ break; -+ default: -+ archive_set_error(&f->archive->archive, -+ ARCHIVE_ERRNO_MISC, -+ "Failed to clean up gzip decompressor"); -+ return (ARCHIVE_FATAL); -+ } -+ gzip->trailer_pending = 1; - } - - /* GZip trailer is a fixed 8 byte structure. */ -- p = __archive_read_filter_ahead(f->upstream, 8, NULL); -+ p = __archive_read_filter_ahead(f->upstream, 8, &avail); -+ if (p == NULL && avail == ARCHIVE_RETRY) -+ return (ARCHIVE_RETRY); - if (p == NULL) - return (ARCHIVE_FATAL); - -@@ -423,6 +459,7 @@ consume_trailer(struct archive_read_filter *f) - - /* We've verified the trailer, so consume it now. */ - __archive_read_filter_consume(f->upstream, 8); -+ gzip->trailer_pending = 0; - - return (ARCHIVE_OK); - } -@@ -441,6 +478,18 @@ gzip_filter_read(struct archive_read_filter *f, const void **p) - - /* Try to fill the output buffer. */ - while (gzip->stream.avail_out > 0 && !gzip->eof) { -+ /* -+ * BUN PATCH: finish any pending trailer first. This can -+ * only be set when a previous gzip_filter_read() call hit -+ * ARCHIVE_RETRY inside consume_trailer(). -+ */ -+ if (gzip->trailer_pending) { -+ ret = consume_trailer(f); -+ if (ret == ARCHIVE_RETRY) -+ goto bun_retry; -+ if (ret < ARCHIVE_OK) -+ return (ret); -+ } - /* If we're not in a stream, read a header - * and initialize the decompression library. */ - if (!gzip->in_stream) { -@@ -449,6 +498,8 @@ gzip_filter_read(struct archive_read_filter *f, const void **p) - gzip->eof = 1; - break; - } -+ if (ret == ARCHIVE_RETRY) -+ goto bun_retry; - if (ret < ARCHIVE_OK) - return (ret); - } -@@ -459,6 +510,14 @@ gzip_filter_read(struct archive_read_filter *f, const void **p) - gzip->stream.next_in = (unsigned char *)(uintptr_t) - __archive_read_filter_ahead(f->upstream, 1, &avail_in); - if (gzip->stream.next_in == NULL) { -+ /* -+ * BUN PATCH: upstream has no bytes right now but -+ * isn't done. zlib's inflate state is untouched, -+ * so propagate the retry (or return whatever we -+ * have already decompressed this call). -+ */ -+ if (avail_in == ARCHIVE_RETRY) -+ goto bun_retry; - archive_set_error(&f->archive->archive, - ARCHIVE_ERRNO_MISC, - "truncated gzip input"); -@@ -485,6 +544,8 @@ gzip_filter_read(struct archive_read_filter *f, const void **p) - /* Consume the stream trailer; release the - * decompression library. */ - ret = consume_trailer(f); -+ if (ret == ARCHIVE_RETRY) -+ goto bun_retry; - if (ret < ARCHIVE_OK) - return (ret); - break; -@@ -504,6 +565,24 @@ gzip_filter_read(struct archive_read_filter *f, const void **p) - else - *p = gzip->out_block; - return (decompressed); -+ -+bun_retry: -+ /* -+ * BUN PATCH: upstream ran dry mid-call. If we managed to -+ * decompress anything before that happened, hand it back -+ * now; the tar layer will consume it and the next call will -+ * start with an empty out_block and immediately retry. -+ * Otherwise propagate ARCHIVE_RETRY so the tar layer (and -+ * ultimately the extract loop in TarballStream.rs) can yield -+ * this worker and reschedule when more bytes arrive. -+ */ -+ decompressed = gzip->stream.next_out - gzip->out_block; -+ if (decompressed > 0) { -+ *p = gzip->out_block; -+ return (decompressed); -+ } -+ *p = NULL; -+ return (ARCHIVE_RETRY); - } - - /* ---- a/libarchive/archive_read_support_format_tar.c -+++ b/libarchive/archive_read_support_format_tar.c -@@ -156,6 +156,23 @@ struct tar { - int read_concatenated_archives; - int default_inode; - int default_dev; -+ -+ /* -+ * BUN PATCH: resume state for non-blocking sources. -+ * -+ * `tar_read_header` walks a variable-length sequence of -+ * extension headers (pax 'x'/'g', GNU 'L'/'K', etc.) before -+ * the real ustar header. When the underlying reader returns -+ * ARCHIVE_RETRY part-way through that walk, we must remember -+ * which extensions have already been parsed so the next call -+ * picks up at the right header instead of re-running -+ * `tar_reset_header_state` and re-bidding bytes that were -+ * already consumed. -+ */ -+ int header_in_progress; -+ int32_t header_seen; -+ int header_eof_fatal; -+ int header_err; - }; - - /* Track which size fields were present in the headers */ -@@ -232,6 +249,8 @@ static int64_t tar_atol(const char *, size_t); - static int64_t tar_atol10(const char *, size_t); - static int64_t tar_atol256(const char *, size_t); - static int64_t tar_atol8(const char *, size_t); -+static int tar_consume_retryable(struct archive_read *, struct tar *, -+ int64_t); - static int tar_read_header(struct archive_read *, struct tar *, - struct archive_entry *, int64_t *); - static int tohex(int c); -@@ -474,6 +493,53 @@ archive_read_format_tar_options(struct archive_read *a, - return (ARCHIVE_WARN); - } - -+/* -+ * BUN PATCH: consume up to `request` bytes in a way that is safe to -+ * resume from a non-blocking source. Each iteration uses -+ * __archive_read_ahead(1) to discover how many bytes are currently -+ * buffered and then __archive_read_consume()s exactly that much, so -+ * the consume never has to invoke the client reader itself. On -+ * ARCHIVE_RETRY the remaining amount is written back into -+ * `tar->entry_bytes_remaining` / `tar->entry_padding` so the next -+ * call (via archive_read_next_header → read_data_skip, or another -+ * read_data_block) continues from the right offset. -+ */ -+static int -+tar_consume_retryable(struct archive_read *a, struct tar *tar, -+ int64_t request) -+{ -+ while (request > 0) { -+ ssize_t avail = 0; -+ const void *p = __archive_read_ahead(a, 1, &avail); -+ if (p == NULL) { -+ if (avail == ARCHIVE_RETRY) { -+ /* Persist what is still owed so the -+ * next resume asks for the remainder -+ * rather than the original total. */ -+ if (request >= tar->entry_padding) { -+ tar->entry_bytes_remaining = -+ request - tar->entry_padding; -+ } else { -+ tar->entry_bytes_remaining = 0; -+ tar->entry_padding = request; -+ } -+ return (ARCHIVE_RETRY); -+ } -+ archive_set_error(&a->archive, -+ ARCHIVE_ERRNO_MISC, -+ "Truncated tar archive" -+ " detected while skipping data"); -+ return (ARCHIVE_FATAL); -+ } -+ if (avail > request) -+ avail = (ssize_t)request; -+ if (__archive_read_consume(a, avail) != avail) -+ return (ARCHIVE_FATAL); -+ request -= avail; -+ } -+ return (ARCHIVE_OK); -+} -+ - /* utility function- this exists to centralize the logic of tracking - * how much unconsumed data we have floating around, and to consume - * anything outstanding since we're going to do read_aheads -@@ -530,32 +596,52 @@ archive_read_format_tar_read_header(struct archive_read *a, - size_t l; - int64_t unconsumed = 0; - -- /* Assign default device/inode values. */ -- archive_entry_set_dev(entry, 1 + tar->default_dev); /* Don't use zero. */ -- archive_entry_set_ino(entry, ++tar->default_inode); /* Don't use zero. */ -- /* Limit generated st_ino number to 16 bits. */ -- if (tar->default_inode >= 0xffff) { -- ++tar->default_dev; -- tar->default_inode = 0; -- } -- -- tar->entry_offset = 0; -- gnu_clear_sparse_list(tar); -- tar->size_fields = 0; /* We don't have any size info yet */ -- -- /* Setup default string conversion. */ -- tar->sconv = tar->opt_sconv; -- if (tar->sconv == NULL) { -- if (!tar->init_default_conversion) { -- tar->sconv_default = -- archive_string_default_conversion_for_read(&(a->archive)); -- tar->init_default_conversion = 1; -+ /* -+ * BUN PATCH: when resuming after an ARCHIVE_RETRY from a -+ * non-blocking source, skip the fresh-entry initialisation -+ * below. Extension headers parsed before the retry have -+ * already written into `entry` and `tar->entry_*`; repeating -+ * these resets would discard that work. -+ */ -+ if (!tar->header_in_progress) { -+ /* Assign default device/inode values. */ -+ archive_entry_set_dev(entry, 1 + tar->default_dev); /* Don't use zero. */ -+ archive_entry_set_ino(entry, ++tar->default_inode); /* Don't use zero. */ -+ /* Limit generated st_ino number to 16 bits. */ -+ if (tar->default_inode >= 0xffff) { -+ ++tar->default_dev; -+ tar->default_inode = 0; -+ } -+ -+ tar->entry_offset = 0; -+ gnu_clear_sparse_list(tar); -+ tar->size_fields = 0; /* We don't have any size info yet */ -+ -+ /* Setup default string conversion. */ -+ tar->sconv = tar->opt_sconv; -+ if (tar->sconv == NULL) { -+ if (!tar->init_default_conversion) { -+ tar->sconv_default = -+ archive_string_default_conversion_for_read(&(a->archive)); -+ tar->init_default_conversion = 1; -+ } -+ tar->sconv = tar->sconv_default; - } -- tar->sconv = tar->sconv_default; - } - - r = tar_read_header(a, tar, entry, &unconsumed); - -+ /* -+ * BUN PATCH: a non-blocking yield leaves `header_in_progress` -+ * set; in that case `*unconsumed == 0` (nothing pending that -+ * isn't already flushed) so the caller can yield and re-enter -+ * later. Upstream's damaged-block ARCHIVE_RETRY clears the -+ * flag and falls through to the original post-read handling -+ * below (which is what upstream did). -+ */ -+ if (r == ARCHIVE_RETRY && tar->header_in_progress) -+ return (ARCHIVE_RETRY); -+ - tar_flush_unconsumed(a, &unconsumed); - - /* -@@ -629,9 +715,20 @@ archive_read_format_tar_read_data(struct archive_read *a, - int64_t request = tar->entry_bytes_remaining + - tar->entry_padding; - -- if (__archive_read_consume(a, request) != request) -+ /* -+ * BUN PATCH: make the end-of-entry padding -+ * consume resumable. `tar_consume_retryable` -+ * decrements `entry_bytes_remaining` / -+ * `entry_padding` as bytes become available so a -+ * subsequent call picks up with the remainder. -+ */ -+ int cr = tar_consume_retryable(a, tar, request); -+ if (cr == ARCHIVE_RETRY) -+ return (ARCHIVE_RETRY); -+ if (cr != ARCHIVE_OK) - return (ARCHIVE_FATAL); - tar->entry_padding = 0; -+ tar->entry_bytes_remaining = 0; - *buff = NULL; - *size = 0; - *offset = tar->disk_size; -@@ -640,6 +737,15 @@ archive_read_format_tar_read_data(struct archive_read *a, - - *buff = __archive_read_ahead(a, 1, &bytes_read); - if (*buff == NULL) { -+ /* -+ * BUN PATCH: propagate non-blocking retry. Every -+ * counter we would have touched -+ * (`entry_bytes_remaining`, `sparse_list`, etc.) -+ * is still at its pre-call value, so the caller -+ * can yield and re-enter read_data_block later. -+ */ -+ if (bytes_read == ARCHIVE_RETRY) -+ return (ARCHIVE_RETRY); - archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, - "Truncated tar archive" - " detected while reading data"); -@@ -669,15 +775,26 @@ archive_read_format_tar_skip(struct archive_read *a) - { - struct tar *tar = a->format->data; - int64_t request; -+ int cr; - - request = tar->entry_bytes_remaining + tar->entry_padding + - tar->entry_bytes_unconsumed; - -- if (__archive_read_consume(a, request) != request) -+ /* -+ * BUN PATCH: use the retryable consume so a non-blocking -+ * source can yield mid-skip. Progress is recorded in -+ * `entry_bytes_remaining` / `entry_padding` / -+ * `entry_bytes_unconsumed`, so archive_read_next_header's -+ * implicit skip on re-entry resumes with the remainder. -+ */ -+ tar->entry_bytes_unconsumed = 0; -+ cr = tar_consume_retryable(a, tar, request); -+ if (cr == ARCHIVE_RETRY) -+ return (ARCHIVE_RETRY); -+ if (cr != ARCHIVE_OK) - return (ARCHIVE_FATAL); - - tar->entry_bytes_remaining = 0; -- tar->entry_bytes_unconsumed = 0; - tar->entry_padding = 0; - - /* Free the sparse list. */ -@@ -711,14 +828,14 @@ tar_read_header(struct archive_read *a, struct tar *tar, - struct archive_entry *entry, int64_t *unconsumed) - { - ssize_t bytes; -- int err = ARCHIVE_OK, err2; -- int eof_fatal = 0; /* EOF is okay at some points... */ -+ int err, err2; -+ int eof_fatal; /* EOF is okay at some points... */ - const char *h; - const struct archive_entry_header_ustar *header; - const struct archive_entry_header_gnutar *gnuheader; - - /* Bitmask of what header types we've seen. */ -- int32_t seen_headers = 0; -+ int32_t seen_headers; - static const int32_t seen_A_header = 1; - static const int32_t seen_g_header = 2; - static const int32_t seen_K_header = 4; -@@ -727,7 +844,24 @@ tar_read_header(struct archive_read *a, struct tar *tar, - static const int32_t seen_x_header = 32; /* Also X */ - static const int32_t seen_mac_metadata = 512; - -- tar_reset_header_state(tar); -+ /* -+ * BUN PATCH: `header_in_progress` persists the header loop's -+ * locals across an ARCHIVE_RETRY so a non-blocking source can -+ * resume mid-sequence. On a fresh entry we initialise them -+ * exactly as before; on resume we restore and skip the reset -+ * that would otherwise wipe pax/GNU long-name data already -+ * parsed into `tar->entry_*`. -+ */ -+ if (!tar->header_in_progress) { -+ tar_reset_header_state(tar); -+ tar->header_in_progress = 1; -+ tar->header_seen = 0; -+ tar->header_eof_fatal = 0; -+ tar->header_err = ARCHIVE_OK; -+ } -+ seen_headers = tar->header_seen; -+ eof_fatal = tar->header_eof_fatal; -+ err = tar->header_err; - - /* Ensure format is set. */ - if (a->archive.archive_format_name == NULL) { -@@ -735,6 +869,17 @@ tar_read_header(struct archive_read *a, struct tar *tar, - a->archive.archive_format_name = "tar"; - } - -+/* -+ * BUN PATCH: every terminal exit from this function (anything other -+ * than the non-blocking retry path) must clear `header_in_progress` -+ * so the next entry starts with fresh `seen_headers` / header state. -+ * The macro keeps the existing `return (X)` sites readable. -+ */ -+#define TAR_HEADER_RETURN(rc) do { \ -+ tar->header_in_progress = 0; \ -+ return (rc); \ -+ } while (0) -+ - /* - * TODO: Write global/default pax options into - * 'entry' struct here before overwriting with -@@ -747,11 +892,22 @@ tar_read_header(struct archive_read *a, struct tar *tar, - /* Find the next valid header record. */ - while (1) { - if (tar_flush_unconsumed(a, unconsumed) != ARCHIVE_OK) { -- return (ARCHIVE_FATAL); -+ TAR_HEADER_RETURN(ARCHIVE_FATAL); - } - - /* Read 512-byte header record */ - h = __archive_read_ahead(a, 512, &bytes); -+ if (h == NULL && bytes == ARCHIVE_RETRY) { -+ /* -+ * BUN PATCH: non-blocking source ran -+ * out mid-header. Nothing has been -+ * added to `*unconsumed` yet this -+ * iteration, so save loop state and -+ * let the caller try again once more -+ * input arrives. -+ */ -+ goto bun_retry; -+ } - if (bytes == 0) { /* EOF at a block boundary. */ - if (eof_fatal) { - /* We've read a special header already; -@@ -759,9 +915,9 @@ tar_read_header(struct archive_read *a, struct tar *tar, - * a premature EOF. */ - archive_set_error(&a->archive, EINVAL, - "Damaged tar archive (end-of-archive within a sequence of headers)"); -- return (ARCHIVE_FATAL); -+ TAR_HEADER_RETURN(ARCHIVE_FATAL); - } else { -- return (ARCHIVE_EOF); -+ TAR_HEADER_RETURN(ARCHIVE_EOF); - } - } - if (h == NULL) { /* Short block at EOF; this is bad. */ -@@ -769,7 +925,7 @@ tar_read_header(struct archive_read *a, struct tar *tar, - ARCHIVE_ERRNO_FILE_FORMAT, - "Truncated tar archive" - " detected while reading next header"); -- return (ARCHIVE_FATAL); -+ TAR_HEADER_RETURN(ARCHIVE_FATAL); - } - *unconsumed += 512; - -@@ -789,22 +945,35 @@ tar_read_header(struct archive_read *a, struct tar *tar, - __archive_read_consume(a, 512); - - archive_clear_error(&a->archive); -- return (ARCHIVE_EOF); -+ TAR_HEADER_RETURN(ARCHIVE_EOF); - } - - /* This is NOT a null block, so it must be a valid header. */ - if (!checksum(a, h)) { - if (tar_flush_unconsumed(a, unconsumed) != ARCHIVE_OK) { -- return (ARCHIVE_FATAL); -+ TAR_HEADER_RETURN(ARCHIVE_FATAL); - } - archive_set_error(&a->archive, EINVAL, - "Damaged tar archive (bad header checksum)"); - /* If we've read some critical information (pax headers, etc) - * and _then_ see a bad header, we can't really recover. */ - if (eof_fatal) { -- return (ARCHIVE_FATAL); -+ TAR_HEADER_RETURN(ARCHIVE_FATAL); - } else { -- return (ARCHIVE_RETRY); -+ /* -+ * BUN PATCH: upstream's -+ * "damaged block, skip and retry -+ * the next one" — NOT a -+ * non-blocking yield. The next -+ * call must re-run the full -+ * state reset (archive_entry_clear, -+ * tar_reset_header_state) exactly -+ * as upstream, so clear both -+ * in-progress flags instead of -+ * routing through bun_retry. -+ */ -+ a->read_header_in_progress = 0; -+ TAR_HEADER_RETURN(ARCHIVE_RETRY); - } - } - break; -@@ -812,12 +981,73 @@ tar_read_header(struct archive_read *a, struct tar *tar, - - /* Determine the format variant. */ - header = (const struct archive_entry_header_ustar *)h; -+ -+ /* -+ * BUN PATCH: extension headers ('A','g','K','L','V', -+ * 'x','X') are followed by a payload whose length is -+ * encoded in this block's `size` field. The handlers -+ * below flush `*unconsumed` (this 512-byte block) and -+ * then __archive_read_ahead() the payload; if that -+ * ahead were to return ARCHIVE_RETRY after the flush, -+ * the block would already be consumed and the resume -+ * would read payload bytes as a header. -+ * -+ * To keep the handlers unchanged, pre-buffer the -+ * header+payload here *before* the flush. On retry we -+ * roll back `*unconsumed` so nothing is consumed and -+ * the next attempt re-reads this exact block from the -+ * filter's buffer. -+ */ -+ switch (header->typeflag[0]) { -+ case 'A': case 'g': case 'K': case 'L': -+ case 'V': case 'X': case 'x': { -+ int64_t ext_size = tar_atol(header->size, -+ sizeof(header->size)); -+ /* Bound the prebuffer before rounding -+ * (tar_atol saturates to INT64_MAX on -+ * overflow, so adding 511 first would be UB). -+ * A corrupted stream can produce garbage that -+ * decodes as an extension header with a -+ * multi-GB `size`; retrying `ahead(512+huge)` -+ * every chunk would never make progress. Real -+ * pax/GNU extension payloads are path-length -+ * bounded (a few KB), so anything beyond this -+ * is handed to the handler below, which will -+ * fail cleanly on its own -+ * `__archive_read_ahead`. */ -+ if (ext_size < 0 || ext_size > (int64_t)(1u << 20)) -+ break; -+ int64_t ext_padded = (ext_size + 511) & ~511; -+ if (__archive_read_ahead(a, -+ (size_t)(512 + ext_padded), &bytes) == NULL && -+ bytes == ARCHIVE_RETRY) { -+ *unconsumed -= 512; -+ goto bun_retry; -+ } -+ /* Re-establish `h`: the filter may have -+ * memmoved its buffer to satisfy the larger -+ * request. A fatal error from the larger -+ * read (alloc failure, downstream I/O error) -+ * leaves `filter->fatal` set, so this read -+ * can return NULL even though the original -+ * 512 bytes are still buffered; bail cleanly -+ * rather than dereferencing `header` below. */ -+ h = __archive_read_ahead(a, 512, NULL); -+ if (h == NULL) -+ TAR_HEADER_RETURN(ARCHIVE_FATAL); -+ header = (const struct archive_entry_header_ustar *)h; -+ break; -+ } -+ default: -+ break; -+ } -+ - switch(header->typeflag[0]) { - case 'A': /* Solaris tar ACL */ - if (seen_headers & seen_A_header) { - archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, - "Redundant 'A' header"); -- return (ARCHIVE_FATAL); -+ TAR_HEADER_RETURN(ARCHIVE_FATAL); - } - seen_headers |= seen_A_header; - a->archive.archive_format = ARCHIVE_FORMAT_TAR_PAX_INTERCHANGE; -@@ -828,7 +1058,7 @@ tar_read_header(struct archive_read *a, struct tar *tar, - if (seen_headers & seen_g_header) { - archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, - "Redundant 'g' header"); -- return (ARCHIVE_FATAL); -+ TAR_HEADER_RETURN(ARCHIVE_FATAL); - } - seen_headers |= seen_g_header; - a->archive.archive_format = ARCHIVE_FORMAT_TAR_PAX_INTERCHANGE; -@@ -872,7 +1102,7 @@ tar_read_header(struct archive_read *a, struct tar *tar, - if (seen_headers & seen_x_header) { - archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, - "Redundant 'X'/'x' header"); -- return (ARCHIVE_FATAL); -+ TAR_HEADER_RETURN(ARCHIVE_FATAL); - } - seen_headers |= seen_x_header; - a->archive.archive_format = ARCHIVE_FORMAT_TAR_PAX_INTERCHANGE; -@@ -884,7 +1114,7 @@ tar_read_header(struct archive_read *a, struct tar *tar, - if (seen_headers & seen_x_header) { - archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, - "Redundant 'x' header"); -- return (ARCHIVE_FATAL); -+ TAR_HEADER_RETURN(ARCHIVE_FATAL); - } - seen_headers |= seen_x_header; - a->archive.archive_format = ARCHIVE_FORMAT_TAR_PAX_INTERCHANGE; -@@ -911,7 +1141,7 @@ tar_read_header(struct archive_read *a, struct tar *tar, - err = err_combine(err, err2); - /* We return warnings or success as-is. Anything else is fatal. */ - if (err < ARCHIVE_WARN) { -- return (ARCHIVE_FATAL); -+ TAR_HEADER_RETURN(ARCHIVE_FATAL); - } - /* Filename of the form `._filename` is an AppleDouble - * extension entry. The body is the macOS metadata blob; -@@ -930,7 +1160,7 @@ tar_read_header(struct archive_read *a, struct tar *tar, - && is_mac_metadata_entry(entry)) { - err2 = read_mac_metadata_blob(a, entry, unconsumed); - if (err2 < ARCHIVE_WARN) { -- return (ARCHIVE_FATAL); -+ TAR_HEADER_RETURN(ARCHIVE_FATAL); - } - err = err_combine(err, err2); - /* Note: Other headers can appear again. */ -@@ -945,7 +1175,7 @@ tar_read_header(struct archive_read *a, struct tar *tar, - if (tar->filetype != 'S' && tar->filetype != '0') { - archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, - "Non-regular file cannot be sparse"); -- return (ARCHIVE_WARN); -+ TAR_HEADER_RETURN(ARCHIVE_WARN); - } else if (tar->sparse_gnu_major == 0 && - tar->sparse_gnu_minor == 0) { - /* Sparse map already parsed from 'x' header */ -@@ -958,22 +1188,22 @@ tar_read_header(struct archive_read *a, struct tar *tar, - ssize_t bytes_read; - bytes_read = gnu_sparse_10_read(a, tar, unconsumed); - if (bytes_read < 0) -- return ((int)bytes_read); -+ TAR_HEADER_RETURN((int)bytes_read); - tar->entry_bytes_remaining -= bytes_read; - } else { - archive_set_error(&a->archive, - ARCHIVE_ERRNO_MISC, - "Unrecognized GNU sparse file format"); -- return (ARCHIVE_WARN); -+ TAR_HEADER_RETURN(ARCHIVE_WARN); - } - } -- return (err); -+ TAR_HEADER_RETURN(err); - } - - /* We're between headers ... */ - err = err_combine(err, err2); - if (err == ARCHIVE_FATAL) -- return (err); -+ TAR_HEADER_RETURN(err); - - /* The GNU volume header and the pax `g` global header - * are both allowed to be the only header in an -@@ -983,6 +1213,26 @@ tar_read_header(struct archive_read *a, struct tar *tar, - eof_fatal = 1; - } - } -+ -+bun_retry: -+ /* -+ * BUN PATCH: persist loop state so the next tar_read_header() -+ * call resumes exactly here. At this point `*unconsumed == 0` -+ * (either nothing was added this iteration, or we rolled the -+ * 512-byte header back above), so -+ * `archive_read_format_tar_read_header` can return -+ * ARCHIVE_RETRY without flushing. Setting -+ * `read_header_in_progress` here (rather than having -+ * archive_read_next_header2 set it on every RETRY) is what -+ * lets upstream's damaged-block RETRY keep its original -+ * reset-everything semantics. -+ */ -+ tar->header_seen = seen_headers; -+ tar->header_eof_fatal = eof_fatal; -+ tar->header_err = err; -+ a->read_header_in_progress = 1; -+ return (ARCHIVE_RETRY); -+#undef TAR_HEADER_RETURN - } - - /* diff --git a/patches/libarchive/select-registered-only.patch b/patches/libarchive/select-registered-only.patch deleted file mode 100644 index c0c0a7eed810..000000000000 --- a/patches/libarchive/select-registered-only.patch +++ /dev/null @@ -1,116 +0,0 @@ ---- a/libarchive/archive_read_append_filter.c -+++ b/libarchive/archive_read_append_filter.c -@@ -42,7 +42,12 @@ archive_read_append_filter(struct archive *_a, int code) - struct archive_read_filter *f; - struct archive_read *a = (struct archive_read *)_a; - -- r2 = (ARCHIVE_OK); -+ /* -+ * BUN PATCH: append only a filter the caller already registered. -+ * Upstream registers it here via archive_read_support_filter_(), -+ * which keeps every filter linked; bun only ever appends gzip. -+ */ -+ r1 = r2 = (ARCHIVE_OK); - switch (code) - { - case ARCHIVE_FILTER_NONE: -@@ -50,19 +55,15 @@ archive_read_append_filter(struct archive *_a, int code) - * NOTE: An initial "NONE" type filter is always set at the end of the - * filter chain. - */ -- r1 = (ARCHIVE_OK); - break; - case ARCHIVE_FILTER_GZIP: - str = "gzip"; -- r1 = archive_read_support_filter_gzip(_a); - break; - case ARCHIVE_FILTER_BZIP2: - str = "bzip2"; -- r1 = archive_read_support_filter_bzip2(_a); - break; - case ARCHIVE_FILTER_COMPRESS: - str = "compress (.Z)"; -- r1 = archive_read_support_filter_compress(_a); - break; - case ARCHIVE_FILTER_PROGRAM: - archive_set_error(&a->archive, ARCHIVE_ERRNO_PROGRAMMER, -@@ -70,43 +71,33 @@ archive_read_append_filter(struct archive *_a, int code) - return (ARCHIVE_FATAL); - case ARCHIVE_FILTER_LZMA: - str = "lzma"; -- r1 = archive_read_support_filter_lzma(_a); - break; - case ARCHIVE_FILTER_XZ: - str = "xz"; -- r1 = archive_read_support_filter_xz(_a); - break; - case ARCHIVE_FILTER_UU: - str = "uu"; -- r1 = archive_read_support_filter_uu(_a); - break; - case ARCHIVE_FILTER_RPM: - str = "rpm"; -- r1 = archive_read_support_filter_rpm(_a); - break; - case ARCHIVE_FILTER_LZ4: - str = "lz4"; -- r1 = archive_read_support_filter_lz4(_a); - break; - case ARCHIVE_FILTER_ZSTD: - str = "zstd"; -- r1 = archive_read_support_filter_zstd(_a); - break; - case ARCHIVE_FILTER_LZIP: - str = "lzip"; -- r1 = archive_read_support_filter_lzip(_a); - break; - case ARCHIVE_FILTER_LZOP: - str = "lzop"; -- r1 = archive_read_support_filter_lzop(_a); - break; - case ARCHIVE_FILTER_LRZIP: - str = "lrzip"; -- r1 = archive_read_support_filter_lrzip(_a); - break; - case ARCHIVE_FILTER_GRZIP: - str = "grzip"; -- r1 = archive_read_support_filter_grzip(_a); - break; - default: - archive_set_error(&a->archive, ARCHIVE_ERRNO_PROGRAMMER, -@@ -127,7 +118,7 @@ archive_read_append_filter(struct archive *_a, int code) - if (!b->name || strcmp(b->name, str)) - { - archive_set_error(&a->archive, ARCHIVE_ERRNO_PROGRAMMER, -- "Internal error: Unable to append filter"); -+ "Filter is not registered"); - return (ARCHIVE_FATAL); - } - ---- a/libarchive/archive_read_set_format.c -+++ b/libarchive/archive_read_set_format.c -@@ -40,9 +40,12 @@ archive_read_set_format(struct archive *_a, int code) - const char *str; - struct archive_read *a = (struct archive_read *)_a; - -- if ((r1 = archive_read_support_format_by_code(_a, code)) < (ARCHIVE_OK)) -- return r1; -- -+ /* -+ * BUN PATCH: select only among formats the caller already registered. -+ * Upstream first calls archive_read_support_format_by_code(), whose -+ * switch references every reader, so the whole reader set (7zip, rar, -+ * zip, ...) stays linked even though bun only ever registers tar. -+ */ - r1 = r2 = (ARCHIVE_OK); - if (a->format) - r2 = (ARCHIVE_WARN); -@@ -108,7 +111,7 @@ archive_read_set_format(struct archive *_a, int code) - if (!a->format->name || strcmp(a->format->name, str)) - { - archive_set_error(&a->archive, ARCHIVE_ERRNO_PROGRAMMER, -- "Internal error: Unable to set format"); -+ "Format is not registered"); - r1 = (ARCHIVE_FATAL); - } - diff --git a/patches/libarchive/tar-npm-compat.patch b/patches/libarchive/tar-npm-compat.patch deleted file mode 100644 index 962548b1b1f4..000000000000 --- a/patches/libarchive/tar-npm-compat.patch +++ /dev/null @@ -1,175 +0,0 @@ ---- a/libarchive/archive_read_support_format_tar.c -+++ b/libarchive/archive_read_support_format_tar.c -@@ -173,6 +173,16 @@ - int32_t header_seen; - int header_eof_fatal; - int header_err; -+ -+ /* -+ * BUN PATCH: `npm` is the reader option of that name: read a -+ * package the way npm's own tar reader does. With it `null_blocks` -+ * counts the zero blocks in a row where a header is due; two of them -+ * end the archive, which `past_end` remembers. -+ */ -+ int npm; -+ int null_blocks; -+ int past_end; - }; - - /* Track which size fields were present in the headers */ -@@ -485,6 +495,10 @@ - } else if (strcmp(key, "read_concatenated_archives") == 0) { - tar->read_concatenated_archives = (val != NULL && val[0] != 0); - return (ARCHIVE_OK); -+ } else if (strcmp(key, "npm") == 0) { -+ /* BUN PATCH */ -+ tar->npm = (val != NULL && val[0] != 0); -+ return (ARCHIVE_OK); - } - - /* Note: The "warn" return is just to inform the options -@@ -662,7 +676,13 @@ - } - } - -- if (r == ARCHIVE_OK && archive_entry_filetype(entry) == AE_IFREG) { -+ /* -+ * BUN PATCH: npm does this for types '0' and NUL alone, also behind -+ * a header that got a warning. -+ */ -+ if ((tar->npm ? (r == ARCHIVE_OK || r == ARCHIVE_WARN) && -+ (tar->filetype == '0' || tar->filetype == '\0') : -+ r == ARCHIVE_OK) && archive_entry_filetype(entry) == AE_IFREG) { - /* - * "Regular" entry with trailing '/' is really - * directory: This is needed for certain old tar -@@ -684,6 +704,19 @@ - } - } - } -+ -+ /* -+ * BUN PATCH: npm takes types '0', NUL and '7' for files and skips -+ * an entry whose pax path has a NUL in it. Anything else that got -+ * the file type of a regular file loses it. -+ */ -+ if (tar->npm && archive_entry_filetype(entry) == AE_IFREG && -+ ((tar->filetype != '0' && tar->filetype != '\0' && -+ tar->filetype != '7') || -+ (archive_strlen(&tar->entry_pathname) > 0 && -+ memchr(tar->entry_pathname.s, '\0', -+ archive_strlen(&tar->entry_pathname)) != NULL))) -+ archive_entry_set_filetype(entry, 0); - return (r); - } - -@@ -895,8 +928,13 @@ - TAR_HEADER_RETURN(ARCHIVE_FATAL); - } - -- /* Read 512-byte header record */ -- h = __archive_read_ahead(a, 512, &bytes); -+ /* -+ * Read 512-byte header record -+ * BUN PATCH: or whatever there is behind the end -+ * of the archive. -+ */ -+ h = __archive_read_ahead(a, tar->past_end ? 1 : 512, -+ &bytes); - if (h == NULL && bytes == ARCHIVE_RETRY) { - /* - * BUN PATCH: non-blocking source ran -@@ -909,7 +947,8 @@ - goto bun_retry; - } - if (bytes == 0) { /* EOF at a block boundary. */ -- if (eof_fatal) { -+ /* BUN PATCH: npm drops such a sequence. */ -+ if (eof_fatal && !tar->npm) { - /* We've read a special header already; - * if there's no regular header, then this is - * a premature EOF. */ -@@ -927,6 +966,10 @@ - " detected while reading next header"); - TAR_HEADER_RETURN(ARCHIVE_FATAL); - } -+ if (tar->past_end) { -+ *unconsumed += bytes; -+ continue; -+ } - *unconsumed += 512; - - if (h[0] == 0 && archive_block_is_null(h)) { -@@ -937,6 +980,18 @@ - continue; - } - -+ /* -+ * BUN PATCH: for npm two in a row end the -+ * archive, and nothing behind them is an entry. -+ * Read on to the end of the input, so that the -+ * filters get to check all of it. -+ */ -+ if (tar->npm) { -+ if (++tar->null_blocks == 2) -+ tar->past_end = 1; -+ continue; -+ } -+ - /* Try to consume a second all-null record, as well. */ - /* If we can't, that's okay. */ - tar_flush_unconsumed(a, unconsumed); -@@ -947,6 +1002,7 @@ - archive_clear_error(&a->archive); - TAR_HEADER_RETURN(ARCHIVE_EOF); - } -+ tar->null_blocks = 0; - - /* This is NOT a null block, so it must be a valid header. */ - if (!checksum(a, h)) { -@@ -1127,7 +1183,10 @@ - a->archive.archive_format = ARCHIVE_FORMAT_TAR_GNUTAR; - a->archive.archive_format_name = "GNU tar format"; - err2 = header_gnutar(a, tar, entry, h, unconsumed); -- } else if (memcmp(header->magic, "ustar", 5) == 0) { -+ } else if (memcmp(header->magic, "ustar", 5) == 0 && -+ /* BUN PATCH: npm wants the version too. */ -+ (!tar->npm || memcmp(gnuheader->magic, -+ "ustar\00000", 8) == 0)) { - if (a->archive.archive_format != ARCHIVE_FORMAT_TAR_PAX_INTERCHANGE) { - a->archive.archive_format = ARCHIVE_FORMAT_TAR_USTAR; - a->archive.archive_format_name = "POSIX ustar format"; -@@ -2619,6 +2678,15 @@ - const char *p; - int err = ARCHIVE_OK; - -+ /* -+ * BUN PATCH: npm reads none of the vendor keys that matter here. -+ * They all start with a capital letter. -+ */ -+ if (tar->npm && key[0] >= 'A' && key[0] <= 'Z') { -+ __archive_read_consume(a, value_length); -+ return (ARCHIVE_OK); -+ } -+ - switch (key[0]) { - case 'G': - /* GNU.* extensions */ -@@ -3353,14 +3421,15 @@ - archive_entry_set_ctime(entry, t, 0); - } - -- if (header->realsize[0] != 0) { -+ /* BUN PATCH: npm reads neither this field nor the next. */ -+ if (!tar->npm && header->realsize[0] != 0) { - /* Treat as a synonym for the pax GNU.sparse.realsize attr */ - tar->GNU_sparse_realsize - = tar_atol(header->realsize, sizeof(header->realsize)); - tar->size_fields |= TAR_SIZE_GNU_SPARSE_REALSIZE; - } - -- if (header->sparse[0].offset[0] != 0) { -+ if (!tar->npm && header->sparse[0].offset[0] != 0) { - if (gnu_sparse_old_read(a, tar, header, unconsumed) - != ARCHIVE_OK) - return (ARCHIVE_FATAL); diff --git a/scripts/build/CLAUDE.md b/scripts/build/CLAUDE.md index 4e7606462366..55f29955c068 100644 --- a/scripts/build/CLAUDE.md +++ b/scripts/build/CLAUDE.md @@ -22,7 +22,7 @@ This directory generates `build.ninja`. The scripts **describe** the build; ninj **Deps in our graph by default; native build systems when needed.** `BuildSpec` variants: -- `direct` — list the dep's sources explicitly; each becomes a first-class `cc`/`cxx` edge in our graph and the `.o`s go straight into bun's link. The default for the C/C++ deps (zlib, zstd, boringssl, libarchive, mimalloc, …). Skips a sub-process configure entirely and lets LTO see across the dep boundary. +- `direct` — list the dep's sources explicitly; each becomes a first-class `cc`/`cxx` edge in our graph and the `.o`s go straight into bun's link. The default for the C/C++ deps (zlib, zstd, boringssl, mimalloc, …). Skips a sub-process configure entirely and lets LTO see across the dep boundary. - `nested-cmake` — invoke the dep's own cmake configure + build as ninja edges. For deps whose build is too entangled to list by hand. Flags forwarded via `-DCMAKE_C_FLAGS`; cmake's own dependency tracking handles incrementality inside. - `cargo` — (rule kept for out-of-workspace cargo deps; none today — lolhtml and rust-argon2 are path dependencies compiled as units of bun's own Rust graph). - `prebuilt` — skip build entirely, download compiled `.a`/`.lib` (WebKit, nodejs-headers). diff --git a/scripts/build/binary-expectations.ts b/scripts/build/binary-expectations.ts index d256873a9324..1f7150a244f0 100644 --- a/scripts/build/binary-expectations.ts +++ b/scripts/build/binary-expectations.ts @@ -531,11 +531,7 @@ export function binaryExpectations(cfg: Config): BinaryExpectations { "ole32.dll", ], exact: true, - // bcrypt.dll: libarchive's archive_random.c / archive_digest.c call - // BCryptOpenAlgorithmProvider and BCryptGenRandom. A release link - // drops them as unreferenced (/OPT:REF); a debug link keeps every - // section, and the import with them. Never in a shipped binary. - allowed: [...sanitizerLibs, ...(cfg.debug ? ["bcrypt.dll"] : [])], + allowed: sanitizerLibs, }, minOSVersion: "6.0", forbiddenImports: forbiddenImports(cfg), diff --git a/scripts/build/deps/README.md b/scripts/build/deps/README.md index 0629ef5d8674..f566045e7104 100644 --- a/scripts/build/deps/README.md +++ b/scripts/build/deps/README.md @@ -99,7 +99,7 @@ export const mydep: Dependency = { patches: ["patches/mydep/fix-something.patch"], // Optional: deps whose SOURCE must be ready before this one builds - // (for -I cross-dep headers). See libarchive for an example. + // (for -I cross-dep headers). See libspng for an example. fetchDeps: ["zlib"], // How to build. `direct` lists sources explicitly; emitDirect compiles @@ -148,7 +148,7 @@ export const mydep: Dependency = { - **mimalloc.ts** — direct build, single unity TU compiled as C++ - **tinycc.ts** — direct build with a build-time codegen tool - **zlib.ts** — direct build with per-source SIMD `-m` flags + `.h.in` substitution -- **libarchive.ts** / **cares.ts** — direct build with hand-written per-target config.h +- **cares.ts** — direct build with hand-written per-target config.h - **boringssl.ts** — direct build with NASM assembly (win-x64) and a large gen/ manifest; `forbidUndefined` (with libuv.ts) keeps a dep that bun points at mimalloc from calling libc's allocator behind its back - **sqlite.ts** — direct build, in-tree source (lives in `src/`, not `vendor/`) - **libuv.ts** — `enabled: cfg => cfg.windows` for a platform-only dep diff --git a/scripts/build/deps/cares.ts b/scripts/build/deps/cares.ts index 38ac7cb52f4d..f3593b17b63f 100644 --- a/scripts/build/deps/cares.ts +++ b/scripts/build/deps/cares.ts @@ -138,8 +138,8 @@ function buildH(cfg: Config): string { } // ─────────────────────────────────────────────────────────────────────────── -// ares_config.h — private, included by every .c via ares_setup.h. Same shape -// as libarchive's: HAVE_* booleans + the recv/send prototype types c-ares +// ares_config.h — private, included by every .c via ares_setup.h. +// HAVE_* booleans + the recv/send prototype types c-ares // uses to cast its socket calls. // ─────────────────────────────────────────────────────────────────────────── diff --git a/scripts/build/deps/index.ts b/scripts/build/deps/index.ts index 4084e71f876c..9e0b2acb36d0 100644 --- a/scripts/build/deps/index.ts +++ b/scripts/build/deps/index.ts @@ -4,7 +4,7 @@ * Order matters for two reasons: * 1. `fetchDeps` relationships — a dep with fetchDeps must come AFTER the * deps it references, so the referenced .ref stamp node exists in ninja - * when we add order-only edges to it. (zlib before libarchive.) + * when we add order-only edges to it. (zlib before libspng.) * 2. Link order — when these libs hit the final link line, static linking * resolves left-to-right. Deps that PROVIDE symbols should come after * deps that USE them. This list becomes the link order. @@ -16,7 +16,6 @@ import { brotli } from "./brotli.ts"; import { cares } from "./cares.ts"; import { hdrhistogram } from "./hdrhistogram.ts"; import { highway } from "./highway.ts"; -import { libarchive } from "./libarchive.ts"; import { libdeflate } from "./libdeflate.ts"; import { libjpegTurbo } from "./libjpeg-turbo.ts"; import { libspng } from "./libspng.ts"; @@ -39,7 +38,7 @@ import { zstd } from "./zstd.ts"; /** * All deps in dependency-resolution + link order. * - * zlib FIRST — libarchive's fetchDeps references it. + * zlib FIRST — libspng's and lsquic's fetchDeps reference it. * brotli libs in internal dep order (common last on link line). * boringssl near the end — many things depend on crypto/ssl symbols. */ @@ -52,7 +51,6 @@ export const allDeps: readonly Dependency[] = [ zstd, brotli, libdeflate, - libarchive, // Image codecs — libspng has fetchDeps:["zlib"], so after zlib. libjpegTurbo, libspng, @@ -85,7 +83,6 @@ export { cares, hdrhistogram, highway, - libarchive, libdeflate, libjpegTurbo, libspng, diff --git a/scripts/build/deps/libarchive.ts b/scripts/build/deps/libarchive.ts deleted file mode 100644 index 2d8e0feccd22..000000000000 --- a/scripts/build/deps/libarchive.ts +++ /dev/null @@ -1,330 +0,0 @@ -/** - * libarchive — multi-format archive reader/writer. Bun uses it for tarball - * extraction during `bun install` (npm packages ship as tarballs) and - * `bun pm pack`. - * - * Minimal config: tar + gzip only. Every other codec backend (bz2/lzma/lz4/ - * zstd/openssl/iconv/...) is left out of config.h, so the corresponding - * `archive_*_support_*` calls compile to "format not supported" stubs. - * - * DirectBuild: cmake's configure step here was ~22s of try_compile probes - * to fill in config.h. We replace it with a static per-target header and - * compile the source list straight into our ninja graph. Side benefit: - * the cmake build was non-hermetic — it would detect host libacl/libmd - * and emit references bun never links. The hand-written config.h omits - * those entirely. - */ - -import type { Config } from "../config.ts"; -import type { Dependency } from "../source.ts"; -import { depBuildDir } from "../source.ts"; - -const LIBARCHIVE_COMMIT = "27cbc7827172698143e440801fc0ba39ccb4f1f5"; - -// The unconditional list from libarchive/CMakeLists.txt, minus the read -// formats/filters bun never registers (it only registers tar/gnutar/gzip, -// and select-registered-only.patch stops set_format/append_filter from -// pulling the rest in). Kept even though unused: format_empty/format_raw -// (archive_match.c), ppmd7 (write_set_format_7zip), filter_program -// (append_filter_program_signature) — dead code that other kept files -// still reference by name. -// prettier-ignore -const SOURCES = [ - "archive_acl", "archive_check_magic", "archive_cmdline", "archive_cryptor", - "archive_digest", "archive_entry", "archive_entry_copy_stat", - "archive_entry_link_resolver", "archive_entry_sparse", "archive_entry_stat", - "archive_entry_strmode", "archive_entry_xattr", "archive_hmac", "archive_match", - "archive_options", "archive_pack_dev", "archive_parse_date", "archive_pathmatch", - "archive_ppmd7", "archive_random", "archive_rb", "archive_read", - "archive_read_add_passphrase", "archive_read_append_filter", - "archive_read_data_into_fd", "archive_read_disk_entry_from_file", - "archive_read_disk_posix", "archive_read_disk_set_standard_lookup", - "archive_read_extract", "archive_read_extract2", "archive_read_open_fd", - "archive_read_open_file", "archive_read_open_filename", "archive_read_open_memory", - "archive_read_set_format", "archive_read_set_options", - "archive_read_support_filter_gzip", "archive_read_support_filter_program", - "archive_read_support_format_empty", "archive_read_support_format_raw", - "archive_read_support_format_tar", "archive_string", "archive_string_sprintf", - "archive_time", "archive_util", "archive_version_details", "archive_virtual", - "archive_write", "archive_write_disk_posix", "archive_write_disk_set_standard_lookup", - "archive_write_open_fd", "archive_write_open_file", "archive_write_open_filename", - "archive_write_open_memory", "archive_write_add_filter", - "archive_write_add_filter_b64encode", "archive_write_add_filter_by_name", - "archive_write_add_filter_bzip2", "archive_write_add_filter_compress", - "archive_write_add_filter_grzip", "archive_write_add_filter_gzip", - "archive_write_add_filter_lrzip", "archive_write_add_filter_lz4", - "archive_write_add_filter_lzop", "archive_write_add_filter_none", - "archive_write_add_filter_program", "archive_write_add_filter_uuencode", - "archive_write_add_filter_xz", "archive_write_add_filter_zstd", - "archive_write_set_format", "archive_write_set_format_7zip", - "archive_write_set_format_ar", "archive_write_set_format_by_name", - "archive_write_set_format_cpio", "archive_write_set_format_cpio_binary", - "archive_write_set_format_cpio_newc", "archive_write_set_format_cpio_odc", - "archive_write_set_format_filter_by_ext", "archive_write_set_format_gnutar", - "archive_write_set_format_iso9660", "archive_write_set_format_mtree", - "archive_write_set_format_pax", "archive_write_set_format_raw", - "archive_write_set_format_shar", "archive_write_set_format_ustar", - "archive_write_set_format_v7tar", "archive_write_set_format_warc", - "archive_write_set_format_xar", "archive_write_set_format_zip", - "archive_write_set_options", "archive_write_set_passphrase", "filter_fork_posix", - "xxhash", -]; - -const SOURCES_WIN = [ - "archive_entry_copy_bhfi", - "archive_read_disk_windows", - "archive_windows", - "archive_write_disk_windows", - "filter_fork_windows", -]; - -export const libarchive: Dependency = { - name: "libarchive", - versionMacro: "LIBARCHIVE", - - source: () => ({ - kind: "github-archive", - repo: "libarchive/libarchive", - commit: LIBARCHIVE_COMMIT, - }), - - patches: [ - "patches/libarchive/archive_write_add_filter_gzip.c.patch", - // archive_read_set_format/append_filter register by code through a - // switch over every reader/filter, which keeps them all linked. Bun - // registers tar/gnutar/gzip itself, so make those calls only select - // an already-registered one. - "patches/libarchive/select-registered-only.patch", - // Propagate ARCHIVE_RETRY from the client read callback up through - // the gzip filter and tar reader so the worker-thread extract loop - // in `bun install` can yield and resume as HTTP chunks arrive. See - // src/install/TarballStream.rs. - "patches/libarchive/nonblocking-read.patch", - // Windows: cache get_current_codepage()/get_current_oemcp() after the - // first call instead of querying setlocale(LC_CTYPE, NULL) once per - // created charset converter — `bun install` creates one per extracted - // archive across many worker threads at once, and the process never - // changes its locale, so the repeated trips through the UCRT locale - // machinery (and its heap) are pure overhead and needless cross-thread - // CRT-heap traffic. - "patches/libarchive/archive_string-codepage-cache.patch", - // Upstream's gzip read filter never checks the CRC32/ISIZE trailer. - // Verify it, so a corrupted tarball fails the extract instead of - // installing the corrupted bytes. - "patches/libarchive/gzip-verify-trailer.patch", - // Adds the tar reader option `npm`, which `bun install` sets: where tar - // readers differ in what they take for a file, for its name or for the end - // of the archive, do what npm's does. Without the option nothing changes. - "patches/libarchive/tar-npm-compat.patch", - ], - - // zlib-ng generates zlib.h during its own build; libarchive's gzip filter - // includes it. We don't link zlib here (bun's final link does), just need - // the header on -I. - fetchDeps: ["zlib"], - - build: cfg => ({ - kind: "direct", - sources: [...SOURCES, ...(cfg.windows ? SOURCES_WIN : [])].map(s => `libarchive/${s}.c`), - // zlib's build dir holds the generated zlib.h (subst'd from .in) that - // the gzip filter includes. Absolute path → emitDirect quotes it. - // android: archive.h does `#include ` under __ANDROID__; - // that header lives under contrib/android/include. - includes: ["libarchive", depBuildDir(cfg, "zlib"), ...(cfg.abi === "android" ? ["contrib/android/include"] : [])], - pic: true, - defines: { - HAVE_CONFIG_H: 1, - LIBARCHIVE_STATIC: 1, - // ELF-only; clang-cl doesn't support __attribute__((visibility)). - ...(!cfg.windows && { __LIBARCHIVE_ENABLE_VISIBILITY: true }), - // Upstream's CMakeLists sets this on MSVC; the DirectBuild conversion - // missed it (97× -Wdeprecated-declarations on localtime/strncpy/etc). - ...(cfg.windows && { _CRT_SECURE_NO_DEPRECATE: true }), - }, - cflags: [ - ...(cfg.windows ? [] : ["-fvisibility=hidden"]), - // Vendored C; const-discard casts in archive_options/_pack_dev/etc. - "-Wno-incompatible-pointer-types-discards-qualifiers", - ], - headers: { "config.h": configH(cfg) }, - }), - - provides: cfg => ({ - libs: [], - includes: cfg.abi === "android" ? ["libarchive", "contrib/android/include"] : ["libarchive"], - }), -}; - -// ─────────────────────────────────────────────────────────────────────────── -// config.h — replaces cmake's feature-detection pass -// ─────────────────────────────────────────────────────────────────────────── -// -// Mirrors the relevant subset of build/cmake/config.h.in for our four -// targets (linux-gnu, linux-musl, darwin, windows). Anything not defined -// here makes libarchive take its portable fallback. -// -// Deliberately omitted regardless of host: -// - HAVE_{BZLIB,LZMA,LZ4,ZSTD,OPENSSL,NETTLE,MBEDTLS,ICONV,LIBXML2,EXPAT, -// LIBB2}_H — codecs we don't ship. -// - HAVE_LIB{ACL,ATTR,MD} / ARCHIVE_ACL_* / ARCHIVE_CRYPTO_* — the cmake -// build would pick these up if dev headers happen to be installed, -// producing a non-hermetic binary. Bun never calls the ACL/digest entry -// points, so the extra code was dead-stripped anyway. -// -// If a libarchive bump adds a new HAVE_* check, the worst case is a missed -// optimization. The two cases that fail loudly are SIZEOF_LONG/WCHAR_T and -// the struct-stat-nsec field name — both pinned per-target below. - -const def1 = (names: string[]) => names.map(n => `#define ${n} 1`).join("\n"); - -// prettier-ignore -const ALWAYS = def1([ - "HAVE_INT16_T", "HAVE_INT32_T", "HAVE_INT64_T", "HAVE_INTMAX_T", - "HAVE_UINT8_T", "HAVE_UINT16_T", "HAVE_UINT32_T", "HAVE_UINT64_T", "HAVE_UINTMAX_T", - "HAVE_DECL_INT32_MAX", "HAVE_DECL_INT32_MIN", "HAVE_DECL_INT64_MAX", "HAVE_DECL_INT64_MIN", - "HAVE_DECL_INTMAX_MAX", "HAVE_DECL_INTMAX_MIN", "HAVE_DECL_SIZE_MAX", - "HAVE_DECL_UINT32_MAX", "HAVE_DECL_UINT64_MAX", "HAVE_DECL_UINTMAX_MAX", - "HAVE_CTYPE_H", "HAVE_ERRNO_H", "HAVE_FCNTL_H", "HAVE_LIMITS_H", "HAVE_LOCALE_H", - "HAVE_SIGNAL_H", "HAVE_STDARG_H", "HAVE_STDINT_H", "HAVE_STDIO_H", "HAVE_STDLIB_H", - "HAVE_STRING_H", "HAVE_TIME_H", "HAVE_WCHAR_H", "HAVE_WCTYPE_H", "HAVE_SYS_STAT_H", - "HAVE_SYS_TYPES_H", "HAVE_INTTYPES_H", - "HAVE_EILSEQ", "HAVE_WCHAR_T", "HAVE_FSTAT", "HAVE_GETPID", "HAVE_MEMMOVE", - "HAVE_MEMORY_H", "HAVE_MKDIR", "HAVE_SETLOCALE", "HAVE_STRCHR", "HAVE_STRDUP", - "HAVE_STRERROR", "HAVE_STRFTIME", "HAVE_STRNLEN", "HAVE_STRRCHR", "HAVE_TZSET", - "HAVE_VPRINTF", "HAVE_WCRTOMB", "HAVE_WCSCMP", "HAVE_WCSCPY", "HAVE_WCSLEN", - "HAVE_WCTOMB", "HAVE_WMEMCMP", "HAVE_WMEMCPY", "HAVE_WMEMMOVE", "HAVE_MBRTOWC", - "HAVE_ZLIB_H", -]); - -// POSIX: every non-Windows target (linux glibc, linux musl, darwin). -// prettier-ignore -const POSIX = def1([ - "HAVE_DECL_SSIZE_MAX", "HAVE_DECL_STRERROR_R", - "HAVE_DIRENT_H", "HAVE_DLFCN_H", "HAVE_FNMATCH_H", "HAVE_GRP_H", "HAVE_LANGINFO_H", - "HAVE_PATHS_H", "HAVE_POLL_H", "HAVE_PTHREAD_H", "HAVE_PWD_H", "HAVE_REGEX_H", - "HAVE_SPAWN_H", "HAVE_STRINGS_H", "HAVE_SYS_CDEFS_H", "HAVE_SYS_IOCTL_H", - "HAVE_SYS_MOUNT_H", "HAVE_SYS_PARAM_H", "HAVE_SYS_POLL_H", "HAVE_SYS_SELECT_H", - "HAVE_SYS_STATVFS_H", "HAVE_SYS_TIME_H", "HAVE_SYS_UTSNAME_H", "HAVE_SYS_WAIT_H", - "HAVE_UNISTD_H", "HAVE_UTIME_H", - "HAVE_CHOWN", "HAVE_CHROOT", "HAVE_CTIME_R", "HAVE_DIRFD", - "HAVE_FCHDIR", "HAVE_FCHMOD", "HAVE_FCHOWN", "HAVE_FCNTL", "HAVE_FDOPENDIR", - "HAVE_FNMATCH", "HAVE_FORK", "HAVE_FSEEKO", "HAVE_FSTATAT", "HAVE_FSTATVFS", - "HAVE_FTRUNCATE", "HAVE_FUTIMENS", "HAVE_FUTIMES", "HAVE_GETEGID", "HAVE_GETEUID", - "HAVE_GETGRGID_R", "HAVE_GETGRNAM_R", "HAVE_GETLINE", "HAVE_GETPWNAM_R", - "HAVE_GETPWUID_R", "HAVE_GMTIME_R", "HAVE_LCHOWN", "HAVE_LINK", "HAVE_LINKAT", - "HAVE_LOCALTIME_R", "HAVE_LSTAT", "HAVE_LUTIMES", "HAVE_MKFIFO", "HAVE_MKNOD", - "HAVE_MKSTEMP", "HAVE_NL_LANGINFO", "HAVE_OPENAT", "HAVE_PIPE", "HAVE_POLL", - "HAVE_POSIX_SPAWNP", "HAVE_READLINK", "HAVE_READLINKAT", "HAVE_SELECT", - "HAVE_SETENV", "HAVE_SIGACTION", "HAVE_STATVFS", "HAVE_STRERROR_R", - "HAVE_SYMLINK", "HAVE_SYSCONF", "HAVE_TCGETATTR", "HAVE_TCSETATTR", "HAVE_TIMEGM", - "HAVE_UNLINKAT", "HAVE_UNSETENV", "HAVE_UTIME", "HAVE_UTIMENSAT", "HAVE_UTIMES", - "HAVE_VFORK", - "HAVE_STRUCT_STAT_ST_BLKSIZE", "HAVE_STRUCT_TM_TM_GMTOFF", -]); - -// Linux-only. xattr via directly — modern glibc/musl ship it, -// no -lattr needed. -// prettier-ignore -const LINUX = def1([ - "HAVE_LINUX_FIEMAP_H", "HAVE_LINUX_FS_H", "HAVE_LINUX_MAGIC_H", "HAVE_LINUX_TYPES_H", - "HAVE_SYS_STATFS_H", "HAVE_SYS_SYSMACROS_H", "HAVE_SYS_VFS_H", "HAVE_SYS_XATTR_H", - "HAVE_STATFS", "HAVE_FSTATFS", "HAVE_FUTIMESAT", - "HAVE_FGETXATTR", "HAVE_FLISTXATTR", "HAVE_FSETXATTR", "HAVE_GETXATTR", - "HAVE_LGETXATTR", "HAVE_LISTXATTR", "HAVE_LLISTXATTR", "HAVE_LSETXATTR", - "HAVE_STRUCT_STAT_ST_MTIM_TV_NSEC", - "HAVE_WORKING_FS_IOC_GETFLAGS", - "MAJOR_IN_SYSMACROS", "ARCHIVE_XATTR_LINUX", -]); - -// FreeBSD: BSD-style stat (st_mtim, st_flags, st_birthtim), extattr_* xattr -// API, chflags family. No — extattr lives in . -// prettier-ignore -const FREEBSD = def1([ - "HAVE_ARC4RANDOM_BUF", - "HAVE_SYS_EXTATTR_H", "HAVE_SYS_MOUNT_H", - "HAVE_STATFS", "HAVE_FSTATFS", - "HAVE_LCHMOD", "HAVE_LCHFLAGS", "HAVE_CHFLAGS", "HAVE_FCHFLAGS", - "HAVE_EXTATTR_GET_FILE", "HAVE_EXTATTR_LIST_FILE", "HAVE_EXTATTR_SET_FD", - "HAVE_EXTATTR_SET_FILE", "HAVE_DECL_EXTATTR_NAMESPACE_USER", - "HAVE_STRUCT_STAT_ST_MTIM_TV_NSEC", "HAVE_STRUCT_STAT_ST_BIRTHTIM", - "HAVE_STRUCT_STAT_ST_FLAGS", - "HAVE_READPASSPHRASE", "HAVE_READPASSPHRASE_H", - "ARCHIVE_XATTR_FREEBSD", -]); - -// prettier-ignore -const DARWIN = def1([ - // arc4random_buf: BSD libc + glibc≥2.36 only. Bun's Linux CI targets - // older glibc, so Linux falls back to /dev/urandom. - "HAVE_ARC4RANDOM_BUF", - "HAVE_SYS_XATTR_H", "HAVE_COPYFILE_H", - "HAVE_FSTATFS", "HAVE_STATFS", "HAVE_LCHMOD", "HAVE_LCHFLAGS", "HAVE_CHFLAGS", - "HAVE_FCHFLAGS", - "HAVE_FGETXATTR", "HAVE_FLISTXATTR", "HAVE_FSETXATTR", "HAVE_GETXATTR", - "HAVE_LISTXATTR", "HAVE_SETXATTR", - "HAVE_STRUCT_STAT_ST_MTIMESPEC_TV_NSEC", "HAVE_STRUCT_STAT_ST_BIRTHTIME", - "HAVE_STRUCT_STAT_ST_BIRTHTIMESPEC_TV_NSEC", "HAVE_STRUCT_STAT_ST_FLAGS", - "HAVE_STRUCT_STATFS_F_IOSIZE", - "ARCHIVE_XATTR_DARWIN", -]); - -// Windows: clang-cl + UCRT. archive_windows.h supplies most POSIX shims; -// this declares what the UCRT actually has. No ARCHIVE_CRYPTO_*_WIN — we -// don't need digests for tar/gzip. (archive_random.c/archive_util.c call -// BCryptGenRandom regardless; bun's link line already carries bcrypt.lib.) -// prettier-ignore -const WINDOWS = def1([ - "HAVE_IO_H", "HAVE_DIRECT_H", "HAVE_PROCESS_H", "HAVE_SYS_UTIME_H", "HAVE_WINDOWS_H", - "HAVE__CTIME64_S", "HAVE__FSEEKI64", "HAVE__GET_TIMEZONE", "HAVE__GMTIME64_S", - "HAVE__LOCALTIME64_S", "HAVE__MKGMTIME64", - "HAVE_STRNCPY_S", "HAVE_WCSCPY_S", "HAVE_WCSNCPY_S", -]) + ` -/* POSIX type fallbacks — UCRT's doesn't define these. - Values match cmake's WIN32 branch (CMakeLists.txt CHECK_TYPE_SIZE block). */ -#define gid_t short -#define uid_t short -#define id_t short -#define mode_t unsigned short -#define pid_t int -#define ssize_t int64_t -`; - -function configH(cfg: Config): string { - const longSize = cfg.windows ? 4 : 8; - const wcharSize = cfg.windows ? 2 : 4; - const platform = cfg.windows ? WINDOWS : `${POSIX}\n${cfg.darwin ? DARWIN : cfg.freebsd ? FREEBSD : LINUX}`; - - // Feature-test macros must come before any system header. archive_platform.h - // includes config.h first, so defining them here is early enough. - const featureTest = cfg.windows - ? "#define NTDDI_VERSION 0x0A000000\n#define _WIN32_WINNT 0x0A00\n#define WINVER 0x0A00" - : "#define _GNU_SOURCE 1\n#define _DARWIN_C_SOURCE 1\n#define __EXTENSIONS__ 1"; - - return `/* Generated by scripts/build/deps/libarchive.ts for ${cfg.os}-${cfg.arch} */ -#define __LIBARCHIVE_CONFIG_H_INCLUDED 1 -${featureTest} - -#define SIZEOF_SHORT 2 -#define SIZEOF_INT 4 -#define SIZEOF_LONG ${longSize} -#define SIZEOF_LONG_LONG 8 -#define SIZEOF_UNSIGNED_SHORT 2 -#define SIZEOF_UNSIGNED 4 -#define SIZEOF_UNSIGNED_LONG ${longSize} -#define SIZEOF_UNSIGNED_LONG_LONG 8 -#define SIZEOF_WCHAR_T ${wcharSize} -#define ICONV_CONST - -#define LIBARCHIVE_VERSION_NUMBER "3008009" -#define LIBARCHIVE_VERSION_STRING "3.8.9" -#define BSDTAR_VERSION_STRING "3.8.9" -#define BSDCPIO_VERSION_STRING "3.8.9" -#define BSDCAT_VERSION_STRING "3.8.9" -#define BSDUNZIP_VERSION_STRING "3.8.9" -#define VERSION "3.8.9" - -${ALWAYS} - -${platform} -`; -} diff --git a/scripts/build/deps/zlib.ts b/scripts/build/deps/zlib.ts index 2ee3470f9502..03179d05a335 100644 --- a/scripts/build/deps/zlib.ts +++ b/scripts/build/deps/zlib.ts @@ -236,7 +236,7 @@ export const zlib: Dependency = { }, // The substituted zlib.h / zconf.h land in the build dir, not the source - // tree, so consumers (libarchive, bun's own bindings) include from there. + // tree, so consumers (libspng, bun's own bindings) include from there. provides: cfg => ({ libs: [], includes: [depBuildDir(cfg, "zlib")], diff --git a/scripts/build/deps/zstd.ts b/scripts/build/deps/zstd.ts index 803636dea2de..351bb62c9fc9 100644 --- a/scripts/build/deps/zstd.ts +++ b/scripts/build/deps/zstd.ts @@ -83,7 +83,7 @@ export const zstd: Dependency = { pic: true, // XXH_NAMESPACE must be a bare token prefix (xxhash pastes it onto // symbol names), not a string literal — DirectBuild.defines would - // wrap it in quotes. Avoids clashes with lshpack/libarchive's copies. + // wrap it in quotes. Avoids clashes with lshpack's copy. cflags: ["-DXXH_NAMESPACE=ZSTD_"], }; return spec; diff --git a/scripts/build/fetch-cli.ts b/scripts/build/fetch-cli.ts index 1763f555f7b1..1325d2c69270 100644 --- a/scripts/build/fetch-cli.ts +++ b/scripts/build/fetch-cli.ts @@ -70,7 +70,7 @@ async function main(): Promise { text = text.split(pairs[i]!).join(pairs[i + 1]!); } // dep_subst is restat=1 — preserve mtime when the rendered text is - // unchanged so depfile-tracked .o files (libarchive ← zlib.h) don't + // unchanged so depfile-tracked .o files (libspng ← zlib.h) don't // recompile after a no-op re-fetch. writeIfChanged(outPath, text); return; diff --git a/scripts/build/source.ts b/scripts/build/source.ts index ee3163e9f42e..14edc0e57d8a 100644 --- a/scripts/build/source.ts +++ b/scripts/build/source.ts @@ -343,7 +343,7 @@ export interface NestedCmakeBuild { * * Most deps don't need this (we link statically into a non-PIE executable), * but some build intermediate tools or have internal shared libs that - * require PIC. cares/highway/libarchive set it. + * require PIC. cares/highway set it. */ pic?: boolean; /** @@ -448,7 +448,6 @@ export type DepName = | "cares" | "hdrhistogram" | "highway" - | "libarchive" | "libdeflate" | "libjpeg-turbo" | "libspng" @@ -492,9 +491,9 @@ export interface Dependency { /** * Other deps that must be BUILT before this dep's configure runs. - * Used for header-level dependencies — e.g. libarchive needs zlib's + * Used for header-level dependencies — e.g. a dep that needs zlib's * headers at configure time (`check_include_file("zlib.h")`). zlib-ng - * generates `zlib.h` during its own cmake configure, so libarchive must + * generates `zlib.h` during its own cmake configure, so that dep must * wait for zlib's full build, not just its source fetch. * * Resolves to the named dep's build outputs (lib files for nested-cmake, @@ -901,7 +900,7 @@ export function resolveDep( // ─── Resolve fetchDeps → extra inputs on configure + build ─── // These are deps that must be BUILT before we configure (not link). - // E.g. libarchive's configure runs check_include_file("zlib.h"), and + // E.g. a dep's configure runs check_include_file("zlib.h"), and // zlib-ng generates zlib.h during its own cmake configure — so we depend // on zlib's lib output (which implies its configure ran). // @@ -1190,7 +1189,7 @@ function emitNestedCmake( // // NOT setting TRY_COMPILE_TARGET_TYPE=STATIC_LIBRARY: it stops // try_compile from linking, which makes check_function_exists and - // check_library_exists always succeed → libarchive "finds" fork, + // check_library_exists always succeed → a dep "finds" fork, // posix_spawnp, libmd on Windows. If llvm-mt is missing, better to // fail fast at "compiler works" than mis-detect every feature. args.push(`-DCMAKE_LINKER=${slash(cfg.ld)}`); @@ -1301,7 +1300,7 @@ function emitNestedCmake( inputs: [], // Configure re-runs if: source changed, cmake binary changed, preBuild // outputs changed. fetchDeps stamps are order-only — can't configure - // until cross-dep headers are on disk (libarchive's check_include_file + // until cross-dep headers are on disk (a check_include_file // for zlib.h runs at configure time). implicitInputs: [sourceStamp, cfg.cmake, ...preBuildOutputs], orderOnlyInputs: fetchDepStamps, diff --git a/src/jsc/bindings/BunProcess.cpp b/src/jsc/bindings/BunProcess.cpp index 1d0155876744..1ec1d33c241e 100644 --- a/src/jsc/bindings/BunProcess.cpp +++ b/src/jsc/bindings/BunProcess.cpp @@ -276,7 +276,6 @@ static JSValue constructVersions(VM& vm, JSObject* processObject) { "openssl", "1.1.0" }, // keep in sync with src/jsc/bindings/node/http/llhttp/README.md { "llhttp", "9.4.2" }, - { "libarchive", BUN_VERSION_LIBARCHIVE }, { "mimalloc", BUN_VERSION_MIMALLOC }, { "picohttpparser", BUN_VERSION_PICOHTTPPARSER }, { "uwebsockets", BUN_VERSION_UWS }, diff --git a/src/jsc/bindings/workaround-missing-symbols.cpp b/src/jsc/bindings/workaround-missing-symbols.cpp index 0ee12bafda18..0c75d8269a4f 100644 --- a/src/jsc/bindings/workaround-missing-symbols.cpp +++ b/src/jsc/bindings/workaround-missing-symbols.cpp @@ -356,7 +356,7 @@ extern "C" __attribute__((used)) char __libc_single_threaded = 0; // ─────────────────────────────────────────────────────────────────────────── // glibc 2.27–2.35 symbols pulled in by Rust std, host crt1.o, libgcc_eh, -// libarchive, and the ASAN runtime when building on a glibc ≥ 2.27 host. +// and the ASAN runtime when building on a glibc ≥ 2.27 host. // All have ABI-compatible older versions (or syscall fallbacks); pinning // each __wrap_X to the floor version keeps verneed ≤ 2.17 regardless of // build host. On a glibc 2.17 host these wraps are inert (the linker never diff --git a/src/libarchive/lib.rs b/src/libarchive/lib.rs index 556fbcc36cc9..1558a704a378 100644 --- a/src/libarchive/lib.rs +++ b/src/libarchive/lib.rs @@ -1,10 +1,4 @@ -// @link "../deps/libarchive.a" #![warn(unused_must_use)] -// ────────────────────────────────────────────────────────────────────────── -// Thin `extern "C"` wrappers over the libarchive C library live in `mod lib` -// below; higher-level extraction logic (`Archiver`, `BufferReadStream`) sits -// on top and uses `bun_sys` for I/O. -// ────────────────────────────────────────────────────────────────────────── use core::ptr; use bun_collections::StringArrayHashMap; @@ -19,143 +13,8 @@ use bun_wyhash::hash; pub mod error; pub use error::{Error, Result}; -// ────────────────────────────────────────────────────────────────────────── -// Local libarchive C-API surface. Thin safe(ish) wrappers over the raw -// `extern "C"` libarchive symbols. The opaque `Archive` / `Entry` types -// here are layout-compatible with libarchive's `struct archive` / -// `struct archive_entry` (zero-sized, `#[repr(C)]`, !Unpin). -// ────────────────────────────────────────────────────────────────────────── -#[allow(non_camel_case_types)] pub mod lib { use super::*; - use core::ffi::{c_char, c_int, c_long, c_uint, c_void}; - - pub type la_ssize_t = isize; - pub(crate) type la_int64_t = i64; - type time_t = isize; - - bun_opaque::opaque_ffi! { - /// Opaque libarchive `struct archive`. Always used behind `*mut Archive`. - /// Contains `UnsafeCell` so that `&Archive` does not assert immutability - /// (libarchive mutates through every call), making `&self -> *mut Self` - /// sound under Stacked Borrows. - pub struct Archive; - /// Opaque libarchive `struct archive_entry`. Always used behind `*mut Entry`. - /// Contains `UnsafeCell` for the same reason as `Archive` — the C side - /// mutates through getter/setter calls that take `&self` here. - pub struct Entry; - } - - #[repr(i32)] - #[derive(Copy, Clone, Eq, PartialEq, Debug)] - pub enum Result { - Eof = 1, - Ok = 0, - Retry = -10, - Warn = -20, - Failed = -25, - Fatal = -30, - } - - impl Result { - /// `Ok` or `Warn`: the call completed. libarchive returns `Warn` for - /// recoverable per-call issues while still producing a result, e.g. a - /// `read_next_header` that fell back to the raw pathname bytes. - #[inline] - pub fn succeeded(self) -> bool { - matches!(self, Result::Ok | Result::Warn) - } - } - - // ── raw libarchive C FFI ─────────────────────────────────────────────── - // Signatures match `vendor/libarchive/archive.h` exactly. `Result` is `#[repr(i32)]` - // so it is ABI-compatible with the C `int` return values. - unsafe extern "C" { - // read side - fn archive_read_new() -> *mut Archive; - fn archive_read_close(a: *mut Archive) -> Result; - fn archive_read_free(a: *mut Archive) -> Result; - fn archive_read_support_format_tar(a: *mut Archive) -> Result; - fn archive_read_support_format_gnutar(a: *mut Archive) -> Result; - fn archive_read_support_filter_gzip(a: *mut Archive) -> Result; - fn archive_read_set_options(a: *mut Archive, opts: *const c_char) -> Result; - fn archive_read_open_memory(a: *mut Archive, buf: *const c_void, size: usize) -> Result; - fn archive_read_next_header(a: *mut Archive, entry: *mut *mut Entry) -> Result; - fn archive_read_data(a: *mut Archive, buf: *mut c_void, size: usize) -> la_ssize_t; - fn archive_read_data_block( - a: *mut Archive, - buff: *mut *const c_void, - size: *mut usize, - offset: *mut la_int64_t, - ) -> Result; - fn archive_error_string(a: *mut Archive) -> *const c_char; - // streaming-read setup (used by TarballStream's resumable extractor) - pub fn archive_read_set_format(a: *mut Archive, code: c_int) -> c_int; - pub fn archive_read_append_filter(a: *mut Archive, code: c_int) -> c_int; - pub fn archive_read_open( - a: *mut Archive, - client_data: *mut c_void, - open: Option, - read: Option, - close: Option, - ) -> c_int; - - // write side - fn archive_write_new() -> *mut Archive; - fn archive_write_free(a: *mut Archive) -> Result; - fn archive_write_close(a: *mut Archive) -> Result; - fn archive_write_set_bytes_in_last_block(a: *mut Archive, bytes: c_int) -> Result; - fn archive_set_error(a: *mut Archive, err: c_int, fmt: *const c_char, ...); - fn archive_write_set_format_pax_restricted(a: *mut Archive) -> Result; - fn archive_write_add_filter_gzip(a: *mut Archive) -> Result; - fn archive_write_set_filter_option( - a: *mut Archive, - module: *const c_char, - option: *const c_char, - value: *const c_char, - ) -> Result; - fn archive_write_set_options(a: *mut Archive, opts: *const c_char) -> Result; - fn archive_write_header(a: *mut Archive, entry: *mut Entry) -> Result; - fn archive_write_data(a: *mut Archive, data: *const c_void, size: usize) -> la_ssize_t; - fn archive_write_finish_entry(a: *mut Archive) -> Result; - #[link_name = "archive_write_open2"] - fn archive_write_open2_raw( - a: *mut Archive, - client_data: *mut c_void, - open: Option, - write: Option, - close: Option, - free: Option, - ) -> c_int; - - // entry - fn archive_entry_new() -> *mut Entry; - fn archive_entry_new2(a: *mut Archive) -> *mut Entry; - fn archive_entry_free(e: *mut Entry); - fn archive_entry_clear(e: *mut Entry) -> *mut Entry; - fn archive_entry_pathname(e: *mut Entry) -> *const c_char; - #[cfg(windows)] - fn archive_entry_pathname_w(e: *mut Entry) -> *const u16; - fn archive_entry_symlink(e: *mut Entry) -> *const c_char; - fn archive_entry_perm(e: *mut Entry) -> bun_sys::Mode; - fn archive_entry_size(e: *mut Entry) -> la_int64_t; - fn archive_entry_filetype(e: *mut Entry) -> bun_sys::Mode; - fn archive_entry_mtime(e: *mut Entry) -> time_t; - fn archive_entry_set_pathname(e: *mut Entry, name: *const c_char); - fn archive_entry_set_pathname_utf8(e: *mut Entry, name: *const c_char); - fn archive_entry_set_size(e: *mut Entry, s: la_int64_t); - fn archive_entry_set_filetype(e: *mut Entry, t: c_uint); - fn archive_entry_set_perm(e: *mut Entry, p: bun_sys::Mode); - fn archive_entry_set_mtime(e: *mut Entry, secs: time_t, nsecs: c_long); - } - - /// One block from `archive_read_data_block`. `bytes` borrows libarchive's - /// internal buffer (valid until the next read call on the owning archive). - pub struct Block<'a> { - pub bytes: &'a [u8], - pub offset: i64, - pub result: Result, - } /// One per extraction. Once `pwrite` fails, later entries don't try it again. pub struct WriteStrategy { @@ -263,755 +122,6 @@ pub mod lib { Ok(()) } } - - impl Archive { - pub fn read_new() -> *mut Archive { - // SAFETY: FFI call with no preconditions. - let p = unsafe { archive_read_new() }; - // libarchive's `archive_read_new()` returns NULL on calloc failure. - // Every caller immediately dereferences the result (forming - // `&Archive`), so fail loudly here instead of invoking UB at the - // first accessor call. - assert!(!p.is_null(), "archive_read_new returned NULL (OOM)"); - p - } - pub fn read_close(&self) -> Result { - // SAFETY: self came from archive_read_new(). - unsafe { archive_read_close(self.as_mut_ptr()) } - } - pub fn read_free(&self) -> Result { - // SAFETY: self came from archive_read_new(); not used after this. - unsafe { archive_read_free(self.as_mut_ptr()) } - } - pub fn read_support_format_tar(&self) -> Result { - // SAFETY: self valid. - unsafe { archive_read_support_format_tar(self.as_mut_ptr()) } - } - pub fn read_support_format_gnutar(&self) -> Result { - // SAFETY: self valid. - unsafe { archive_read_support_format_gnutar(self.as_mut_ptr()) } - } - pub fn read_support_filter_gzip(&self) -> Result { - // SAFETY: self valid. - unsafe { archive_read_support_filter_gzip(self.as_mut_ptr()) } - } - pub fn read_set_options(&self, opts: &core::ffi::CStr) -> Result { - // SAFETY: self valid; opts is NUL-terminated. - unsafe { archive_read_set_options(self.as_mut_ptr(), opts.as_ptr()) } - } - pub fn read_open_memory(&self, buf: &[u8]) -> Result { - // SAFETY: self valid; buf outlives the archive (caller contract, - // see `BufferReadStream::buf` field comment). - unsafe { archive_read_open_memory(self.as_mut_ptr(), buf.as_ptr().cast(), buf.len()) } - } - pub fn read_next_header(&self, entry: &mut *mut Entry) -> Result { - // SAFETY: self valid; entry is a valid out-ptr. - unsafe { - archive_read_next_header(self.as_mut_ptr(), std::ptr::from_mut::<*mut Entry>(entry)) - } - } - pub fn read_data(&self, buf: &mut [u8]) -> isize { - // SAFETY: self valid; buf writable for buf.len(). - unsafe { archive_read_data(self.as_mut_ptr(), buf.as_mut_ptr().cast(), buf.len()) } - } - - /// Reads in steps, so that an untrusted `size` doesn't drive the allocation. Returns - /// `Ok(false)` on a read error. - pub fn read_data_to_vec( - &self, - size: usize, - out: &mut Vec, - ) -> core::result::Result { - let start = out.len(); - while out.len() - start < size { - let to_read = (size - (out.len() - start)).min(64 * 1024); - out.try_reserve(to_read).map_err(|_| bun_core::AllocError)?; - // SAFETY: `archive_read_data` only writes into the slice; the written prefix is committed below. - let dest = unsafe { &mut bun_core::vec::spare_bytes_mut(out)[..to_read] }; - let read = self.read_data(dest); - if read < 0 { - return Ok(false); - } - if read == 0 { - break; - } - // SAFETY: `archive_read_data` returns exactly the byte count it wrote (`<= to_read`). - unsafe { - bun_core::vec::commit_spare(out, usize::try_from(read).expect("int cast")) - }; - } - Ok(true) - } - - /// Calls `archive_read_data_block`. Returns `None` on EOF, and `*offset` is then the - /// entry's length. - pub fn next(&self, offset: &mut i64) -> Option> { - let mut buff: *const c_void = core::ptr::null(); - let mut size: usize = 0; - // SAFETY: self valid; out-ptrs are valid stack locations. - let r = unsafe { - archive_read_data_block(self.as_mut_ptr(), &raw mut buff, &raw mut size, offset) - }; - if r == Result::Eof { - return None; - } - if r != Result::Ok { - return Some(Block { - bytes: &[], - offset: *offset, - result: r, - }); - } - // SAFETY: on ARCHIVE_OK, libarchive guarantees buff[0..size] is - // readable until the next read call on this archive. - let bytes = unsafe { core::slice::from_raw_parts(buff.cast::(), size) }; - Some(Block { - bytes, - offset: *offset, - result: r, - }) - } - - pub fn read_data_into_fd(&self, fd: Fd, strategy: &mut WriteStrategy) -> Result { - let mut writer = EntryWriter::new(fd); - let mut offset: i64 = 0; - while let Some(block) = self.next(&mut offset) { - if block.result != Result::Ok { - return block.result; - } - if writer.write(strategy, block.offset, block.bytes).is_err() { - return Result::Failed; - } - } - match writer.finish(offset) { - Ok(()) => Result::Ok, - Err(_) => Result::Failed, - } - } - - // `self` must be a live archive handle from `archive_{read,write}_new()`. - // `Archive` is `opaque_ffi!`-backed (UnsafeCell), so `&self → *mut Self` - // is sound; libarchive never returns null from `*_new()`. - pub fn error_string(&self) -> &'static [u8] { - // SAFETY: `self` is a live archive handle. - let p = unsafe { archive_error_string(self.as_mut_ptr()) }; - if p.is_null() { - return b""; - } - // SAFETY: libarchive owns the error string for the lifetime of the - // archive; callers treat it as borrowed-until-next-call. The - // `'static` here is a lifetime erasure — the caller must not let - // the slice outlive the archive. - unsafe { ZStr::from_c_ptr(p) }.as_bytes() - } - - // ── write side ───────────────────────────────────────────────────── - pub fn write_new() -> *mut Archive { - // SAFETY: FFI call with no preconditions. - unsafe { archive_write_new() } - } - pub fn write_close(&self) -> Result { - // SAFETY: self valid. - unsafe { archive_write_close(self.as_mut_ptr()) } - } - /// Sets the padding unit for the last output block. The default pads it to 10 KiB, and 1 - /// disables padding. - pub fn write_set_bytes_in_last_block(&self, bytes: c_int) -> Result { - // SAFETY: self valid. - unsafe { archive_write_set_bytes_in_last_block(self.as_mut_ptr(), bytes) } - } - pub fn write_set_format_pax_restricted(&self) -> Result { - // SAFETY: self valid. - unsafe { archive_write_set_format_pax_restricted(self.as_mut_ptr()) } - } - pub fn write_add_filter_gzip(&self) -> Result { - // SAFETY: self valid. - unsafe { archive_write_add_filter_gzip(self.as_mut_ptr()) } - } - pub fn write_set_filter_option( - &self, - module: Option<&ZStr>, - option: &ZStr, - value: &ZStr, - ) -> Result { - // SAFETY: self valid; ZStr guarantees NUL-termination. - unsafe { - archive_write_set_filter_option( - self.as_mut_ptr(), - module.map_or(core::ptr::null(), |m| m.as_ptr().cast()), - option.as_ptr().cast(), - value.as_ptr().cast(), - ) - } - } - pub fn write_set_options(&self, opts: &ZStr) -> Result { - // SAFETY: self valid; ZStr guarantees NUL-termination. - unsafe { archive_write_set_options(self.as_mut_ptr(), opts.as_ptr().cast()) } - } - pub fn write_header(&self, entry: &Entry) -> Result { - // SAFETY: self valid; entry came from Entry::new()/read_next_header(). - // `Entry` has interior mutability so `&Entry -> *mut Entry` is sound. - unsafe { archive_write_header(self.as_mut_ptr(), entry.as_mut_ptr()) } - } - pub fn write_data(&self, data: &[u8]) -> isize { - // SAFETY: self valid; data readable for data.len(). - unsafe { archive_write_data(self.as_mut_ptr(), data.as_ptr().cast(), data.len()) } - } - pub fn write_finish_entry(&self) -> Result { - // SAFETY: self valid. - unsafe { archive_write_finish_entry(self.as_mut_ptr()) } - } - } - - impl Entry { - pub fn pathname(&self) -> &ZStr { - // SAFETY: self valid; returned string owned by libarchive for the - // lifetime of this entry. - unsafe { ZStr::from_c_ptr(archive_entry_pathname(self.as_mut_ptr())) } - } - #[cfg(windows)] - pub fn pathname_w(&self) -> &bun_core::WStr { - // SAFETY: self valid. - unsafe { bun_core::WStr::from_ptr(archive_entry_pathname_w(self.as_mut_ptr())) } - } - pub fn symlink(&self) -> &ZStr { - // SAFETY: self valid. - unsafe { ZStr::from_c_ptr(archive_entry_symlink(self.as_mut_ptr())) } - } - pub fn perm(&self) -> u32 { - // SAFETY: self valid. - unsafe { archive_entry_perm(self.as_mut_ptr()) as u32 } - } - pub fn size(&self) -> i64 { - // SAFETY: self valid. - unsafe { archive_entry_size(self.as_mut_ptr()) } - } - pub fn filetype(&self) -> u32 { - // SAFETY: self valid. - unsafe { archive_entry_filetype(self.as_mut_ptr()) as u32 } - } - pub fn mtime(&self) -> i64 { - // SAFETY: self valid. - unsafe { archive_entry_mtime(self.as_mut_ptr()) as i64 } - } - - // ── write side ───────────────────────────────────────────────────── - pub(crate) fn new() -> *mut Entry { - // SAFETY: FFI call with no preconditions. - unsafe { archive_entry_new() } - } - /// `archive_entry_new2(archive)` — ties the entry to the archive's - /// charset-conversion context (preferred over `new()` when an archive - /// is available). `archive` is a live handle from `read_new()`/`write_new()`. - pub fn new2(archive: &Archive) -> *mut Entry { - // SAFETY: `archive` is a live handle (opaque_ffi! `&self → *mut Self`). - unsafe { archive_entry_new2(archive.as_mut_ptr()) } - } - pub fn free(&self) { - // SAFETY: self came from Entry::new(); not used after this. - unsafe { archive_entry_free(self.as_mut_ptr()) } - } - pub fn clear(&self) -> *mut Entry { - // SAFETY: self valid. - unsafe { archive_entry_clear(self.as_mut_ptr()) } - } - /// Raw `archive_entry_set_pathname` — bytes are stored verbatim (no - /// charset conversion). - pub fn set_pathname(&self, name: &ZStr) { - // SAFETY: self valid; name is NUL-terminated. - unsafe { archive_entry_set_pathname(self.as_mut_ptr(), name.as_ptr()) } - } - pub fn set_pathname_utf8(&self, name: &ZStr) { - // SAFETY: self valid; name is NUL-terminated. - unsafe { archive_entry_set_pathname_utf8(self.as_mut_ptr(), name.as_ptr()) } - } - pub fn set_size(&self, s: i64) { - // SAFETY: self valid. - unsafe { archive_entry_set_size(self.as_mut_ptr(), s) } - } - pub fn set_filetype(&self, t: u32) { - // SAFETY: self valid. - unsafe { archive_entry_set_filetype(self.as_mut_ptr(), t as c_uint) } - } - pub fn set_perm(&self, p: u32) { - // SAFETY: self valid. - unsafe { archive_entry_set_perm(self.as_mut_ptr(), p as bun_sys::Mode) } - } - pub fn set_mtime(&self, secs: isize, nsecs: core::ffi::c_long) { - // SAFETY: self valid. - unsafe { archive_entry_set_mtime(self.as_mut_ptr(), secs as time_t, nsecs) } - } - } - - // ── RAII owners ──────────────────────────────────────────────────────── - // - // The raw `*mut Archive` / `*mut Entry` constructors above mirror the C - // API. These thin owners pair them with the matching `*_free` on `Drop` - // so callers stop hand-rolling `defer { (*archive).read_free() }`. - - /// Owns a `*mut Archive` opened with [`Archive::read_new`]; calls - /// `archive_read_free` on drop. Derefs to `&Archive`. - pub struct ReadArchive(core::ptr::NonNull); - impl ReadArchive { - #[inline] - pub fn new() -> Self { - Self( - core::ptr::NonNull::new(Archive::read_new()) - .expect("archive_read_new returned null"), - ) - } - } - impl core::ops::Deref for ReadArchive { - type Target = Archive; - #[inline] - fn deref(&self) -> &Archive { - // SAFETY: handle is live until Drop; libarchive owns the storage. - unsafe { self.0.as_ref() } - } - } - impl Drop for ReadArchive { - #[inline] - fn drop(&mut self) { - // SAFETY: handle came from archive_read_new() and is freed exactly once. - let _ = unsafe { archive_read_free(self.0.as_ptr()) }; - } - } - - /// Owns a `*mut Archive` opened with [`Archive::write_new`]; calls - /// `archive_write_free` on drop. Derefs to `&Archive`. - pub struct WriteArchive(core::ptr::NonNull); - impl WriteArchive { - #[inline] - pub fn new() -> Self { - Self( - core::ptr::NonNull::new(Archive::write_new()) - .expect("archive_write_new returned null"), - ) - } - } - impl core::ops::Deref for WriteArchive { - type Target = Archive; - #[inline] - fn deref(&self) -> &Archive { - // SAFETY: handle is live until Drop; libarchive owns the storage. - unsafe { self.0.as_ref() } - } - } - impl Drop for WriteArchive { - #[inline] - fn drop(&mut self) { - // SAFETY: handle came from archive_write_new() and is freed exactly once. - let _ = unsafe { archive_write_free(self.0.as_ptr()) }; - } - } - - /// Owns a `*mut Entry` created with [`Entry::new`] / [`Entry::new2`]; - /// calls `archive_entry_free` on drop. Derefs to `&Entry`. - pub struct OwnedEntry(core::ptr::NonNull); - impl OwnedEntry { - #[inline] - pub fn new() -> Self { - Self(core::ptr::NonNull::new(Entry::new()).expect("archive_entry_new returned null")) - } - } - impl core::ops::Deref for OwnedEntry { - type Target = Entry; - #[inline] - fn deref(&self) -> &Entry { - // SAFETY: handle is live until Drop; libarchive owns the storage. - unsafe { self.0.as_ref() } - } - } - impl Drop for OwnedEntry { - #[inline] - fn drop(&mut self) { - // SAFETY: handle came from archive_entry_new()/new2() and is freed exactly once. - unsafe { archive_entry_free(self.0.as_ptr()) }; - } - } - - // ── Archive::Iterator ────────────────────────────────────────────────── - // - // Thin streaming reader over a tar.gz blob: `init` opens the archive in - // memory, `next` yields one header at a time, `read_entry_data` slurps the - // current entry's payload, `close` tears down. Errors are surfaced as the - // libarchive `*mut Archive` plus a static message so callers can append - // `Archive::error_string`. - - /// Generic result type used by [`ArchiveIterator`]. - pub enum IteratorResult { - Err { - archive: *mut Archive, - message: &'static [u8], - }, - Result(T), - } - - impl IteratorResult { - #[inline] - pub(crate) fn init_err(arch: *mut Archive, msg: &'static [u8]) -> Self { - Self::Err { - message: msg, - archive: arch, - } - } - #[inline] - pub(crate) fn init_res(value: T) -> Self { - Self::Result(value) - } - } - - /// Iterates over the entries of an open archive, skipping entries whose - /// file kind has its bit set in `filter`. - pub struct ArchiveIterator { - pub archive: *mut Archive, - // A u16 bitmask over - // `bun_sys::FileKind` variants. - pub(crate) filter: u16, - } - - /// One entry returned from [`ArchiveIterator::next`]. - pub struct NextEntry { - pub entry: *mut Entry, - pub kind: bun_sys::FileKind, - } - - impl NextEntry { - /// The header `next()` just read; valid until the iterator advances. - pub fn entry(&self) -> &Entry { - // SAFETY: `entry` is the non-null header libarchive handed to `next()` and stays live until the next read. - unsafe { &*self.entry } - } - } - - impl ArchiveIterator { - /// Borrow the underlying libarchive handle. - /// - /// SAFETY (invariant): `self.archive` is set to a fresh non-null - /// handle by `Archive::read_new()` in [`init`] and remains valid - /// until `read_free()` in [`close`]. All `Archive` methods take - /// `&self` (FFI interior mutability), so a shared borrow suffices. - #[inline] - pub fn archive(&self) -> &Archive { - // SAFETY: see doc comment — non-null for the lifetime of `self`. - unsafe { &*self.archive } - } - - /// Reads the body of the entry `next()` just returned. - pub fn read_entry_data( - &mut self, - next: &NextEntry, - ) -> core::result::Result>, bun_core::OOM> { - next.read_entry_data(self.archive()) - } - - pub fn init(tarball_bytes: &[u8]) -> IteratorResult { - let archive = Archive::read_new(); - // SAFETY: archive_read_new() returns a non-null handle owned by libarchive. - let a = unsafe { &*archive }; - - match a.read_support_format_tar() { - Result::Failed | Result::Fatal | Result::Warn => { - return IteratorResult::init_err( - archive, - b"failed to enable tar format support", - ); - } - _ => {} - } - match a.read_support_format_gnutar() { - Result::Failed | Result::Fatal | Result::Warn => { - return IteratorResult::init_err( - archive, - b"failed to enable gnutar format support", - ); - } - _ => {} - } - match a.read_support_filter_gzip() { - Result::Failed | Result::Fatal | Result::Warn => { - return IteratorResult::init_err( - archive, - b"failed to enable support for gzip compression", - ); - } - _ => {} - } - match a.read_set_options(c"read_concatenated_archives") { - Result::Failed | Result::Fatal | Result::Warn => { - return IteratorResult::init_err( - archive, - b"failed to set option `read_concatenated_archives`", - ); - } - _ => {} - } - match a.read_open_memory(tarball_bytes) { - Result::Failed | Result::Fatal | Result::Warn => { - return IteratorResult::init_err(archive, b"failed to read tarball"); - } - _ => {} - } - - IteratorResult::init_res(Self { archive, filter: 0 }) - } - - pub fn next(&mut self) -> IteratorResult> { - let a = self.archive(); - let mut entry: *mut Entry = core::ptr::null_mut(); - loop { - return match a.read_next_header(&mut entry) { - Result::Retry => continue, - Result::Eof => IteratorResult::init_res(None), - // `Warn` still yields a fully populated entry; see `Result::succeeded`. - Result::Ok | Result::Warn => { - let kind = bun_sys::kind_from_mode( - Entry::opaque_ref(entry).filetype() as bun_sys::Mode - ); - if (self.filter & (1u16 << (kind as u8))) != 0 { - continue; - } - IteratorResult::init_res(Some(NextEntry { entry, kind })) - } - _ => IteratorResult::init_err(self.archive, b"failed to read archive header"), - }; - } - } - - /// Returns a `Result` the caller inspects, so this - /// cannot be `Drop`. Explicit-close per PORTING.md §Idiom map. - pub fn close(self) -> IteratorResult<()> { - let a = self.archive(); - match a.read_close() { - Result::Failed | Result::Fatal | Result::Warn => { - return IteratorResult::init_err(self.archive, b"failed to close archive read"); - } - _ => {} - } - match a.read_free() { - Result::Failed | Result::Fatal | Result::Warn => { - return IteratorResult::init_err(self.archive, b"failed to free archive read"); - } - _ => {} - } - IteratorResult::init_res(()) - } - } - - impl NextEntry { - /// Reads this entry's full data into a heap buffer. `archive` is the - /// live handle this `NextEntry` was yielded from. - pub fn read_entry_data( - &self, - archive: &Archive, - ) -> core::result::Result>, bun_core::OOM> { - // SAFETY: self.entry is the libarchive-owned entry from read_next_header. - let size = unsafe { (*self.entry).size() }; - let Ok(size) = usize::try_from(size) else { - return Ok(IteratorResult::init_err( - archive.as_mut_ptr(), - b"invalid archive entry size", - )); - }; - let mut buf: Vec = Vec::new(); - if !archive.read_data_to_vec(size, &mut buf)? { - return Ok(IteratorResult::init_err( - archive.as_mut_ptr(), - b"failed to read archive data", - )); - } - Ok(IteratorResult::init_res(buf.into_boxed_slice())) - } - } - - // ── write-open callback surface (libarchive `archive_write_open2`) ───── - type archive_open_callback = unsafe extern "C" fn(*mut Archive, *mut c_void) -> c_int; - pub type archive_read_callback = - unsafe extern "C" fn(*mut Archive, *mut c_void, *mut *const c_void) -> la_ssize_t; - type archive_write_callback = - unsafe extern "C" fn(*mut Archive, *mut c_void, *const c_void, usize) -> la_ssize_t; - type archive_close_callback = unsafe extern "C" fn(*mut Archive, *mut c_void) -> c_int; - type archive_free_callback = unsafe extern "C" fn(*mut Archive, *mut c_void) -> c_int; - - /// `a` is a live `archive_write_new()` handle. `client_data` is forwarded - /// opaquely to the callbacks (never dereferenced here); its lifetime must - /// outlast the registered callbacks. - #[allow(clippy::not_unsafe_ptr_arg_deref)] - pub fn archive_write_open2( - a: &Archive, - client_data: *mut c_void, - open: Option, - write: Option, - close: Option, - free: Option, - ) -> c_int { - // SAFETY: `a` is a live handle (`opaque_ffi!` `&self → *mut Self`); - // `client_data` is opaque to libarchive until a callback dereferences it. - unsafe { archive_write_open2_raw(a.as_mut_ptr(), client_data, open, write, close, free) } - } - - /// Growing memory buffer for archive writes with libarchive callbacks. - pub struct GrowingBuffer<'a> { - pub(crate) list: Vec, - pub(crate) had_error: bool, - /// When set, every write goes to this callback instead of `list`, so `list` stays empty. - pub(crate) forward_to: Option<&'a mut dyn FnMut(&[u8])>, - } - - impl<'a> GrowingBuffer<'a> { - pub fn init() -> Self { - GrowingBuffer { - list: Vec::new(), - had_error: false, - forward_to: None, - } - } - - pub fn forward_to(write: &'a mut dyn FnMut(&[u8])) -> Self { - GrowingBuffer { - forward_to: Some(write), - ..Self::init() - } - } - - pub fn to_owned_slice(&mut self) -> core::result::Result, bun_core::OOM> { - if self.had_error { - return Err(bun_core::AllocError); - } - Ok(core::mem::take(&mut self.list)) - } - - pub unsafe extern "C" fn open_callback( - _a: *mut Archive, - client_data: *mut c_void, - ) -> c_int { - // SAFETY: client_data is a *mut GrowingBuffer registered via archive_write_open2. - let this = unsafe { bun_core::callback_ctx::(client_data) }; - this.list.clear(); - this.had_error = false; - 0 - } - - pub unsafe extern "C" fn write_callback( - a: *mut Archive, - client_data: *mut c_void, - buff: *const c_void, - length: usize, - ) -> la_ssize_t { - // SAFETY: client_data is a *mut GrowingBuffer registered via archive_write_open2. - let this = unsafe { bun_core::callback_ctx::(client_data) }; - if buff.is_null() || length == 0 { - return 0; - } - // SAFETY: buff[0..length] is valid for reads per libarchive contract. - let data = unsafe { core::slice::from_raw_parts(buff.cast::(), length) }; - if let Some(write) = this.forward_to.as_deref_mut() { - write(data); - return la_ssize_t::try_from(length).expect("int cast"); - } - if this.list.try_reserve(length).is_err() { - this.had_error = true; - // libarchive sets no error of its own for a failed client write. - // SAFETY: `a` is the archive this callback was invoked for; the - // format string has no conversions, so no varargs are read. - unsafe { archive_set_error(a, bun_sys::E::ENOMEM as c_int, c"No memory".as_ptr()) }; - return -1; - } - this.list.extend_from_slice(data); - la_ssize_t::try_from(length).expect("int cast") - } - - pub unsafe extern "C" fn close_callback( - _a: *mut Archive, - _client_data: *mut c_void, - ) -> c_int { - 0 - } - } - - // ── Archive::Iterator ────────────────────────────────────────────────── - // Thin - // wrapper that opens a tarball from memory and yields one - // `IteratorEntry` per `next()`, used by `bun publish `. -} - -use lib::Archive; - -pub struct BufferReadStream { - buf: *const [u8], - - archive: *mut Archive, -} - -impl BufferReadStream { - /// Construct a stream over `buf`. - /// - /// # Safety - /// `buf` is type-erased to a raw `*const [u8]` (no lifetime parameter on - /// `BufferReadStream` — see field comment). The caller - /// **must** guarantee that the slice `buf` points to remains valid and - /// unmoved for the entire lifetime of the returned `BufferReadStream` - /// (including its `Drop`). Violating this makes [`buf()`], [`buf_left()`], - /// and [`open_read()`] dereference a dangling pointer (UB). - pub(crate) unsafe fn init(buf: &[u8]) -> Self { - // was an out-param constructor (`this.* = ...`) - Self { - buf: std::ptr::from_ref::<[u8]>(buf), - archive: Archive::read_new(), - } - } - - /// Borrow the underlying libarchive handle. - /// - /// SAFETY (invariant): `self.archive` is set to a fresh non-null handle by - /// `Archive::read_new()` in `init()` (asserted there) and remains valid - /// until `read_free()` in `Drop`. All `Archive` methods take `&self` - /// (FFI interior mutability), so a shared borrow is sufficient. - #[inline] - fn archive(&self) -> &Archive { - // SAFETY: see doc comment — non-null for the lifetime of `self`. - unsafe { &*self.archive } - } - - /// Borrow the input buffer. - /// - /// SAFETY (invariant): `self.buf` is a fat pointer captured from the - /// `&[u8]` passed to `init()`; the caller guarantees it outlives `self` - /// (see field comment). Never null, never mutated. - #[inline] - fn buf(&self) -> &[u8] { - // SAFETY: see doc comment — borrowed for `self`'s lifetime. - unsafe { &*self.buf } - } - - /// `npm` makes libarchive read the tarball the way npm's own tar reader does. See - /// patches/libarchive/tar-npm-compat.patch. - pub(crate) fn open_read(&mut self, npm: bool) -> lib::Result { - let archive = self.archive(); - - let _ = archive.read_support_format_tar(); - let _ = archive.read_support_format_gnutar(); - let _ = archive.read_support_filter_gzip(); - - // Ignore zeroed blocks in the archive, which occurs when multiple tar archives - // have been concatenated together. - // Without this option, only the contents of - // the first concatenated archive would be read. - let _ = archive.read_set_options(if npm { - c"npm" - } else { - c"read_concatenated_archives" - }); - - archive.read_open_memory(self.buf()) - } -} - -impl Drop for BufferReadStream { - fn drop(&mut self) { - let _ = self.archive().read_close(); - let _ = self.archive().read_free(); - } } /// `mkdirat` mode for a directory entry. `perm` is whatever the header diff --git a/test/js/node/process/process.test.js b/test/js/node/process/process.test.js index 781c8c37606b..9633bbbb89f4 100644 --- a/test/js/node/process/process.test.js +++ b/test/js/node/process/process.test.js @@ -754,7 +754,6 @@ it("process.versions", async () => { const depsDir = resolve(import.meta.dir, "../../../../scripts/build/deps"); const deps = { boringssl: "boringssl", - libarchive: "libarchive", mimalloc: "mimalloc", picohttpparser: "picohttpparser", zlib: "zlib", diff --git a/test/leaksan.supp b/test/leaksan.supp index 06f00aa8c43d..05f4a16120f2 100644 --- a/test/leaksan.supp +++ b/test/leaksan.supp @@ -102,7 +102,6 @@ leak:fse_run_loop leak:Zig::NapiClass_ConstructorFunction leak:runtime.webcore.fetch.FetchTasklet.toResponse leak:JSC::jsonProtoFuncStringify -leak:libarchive_sys.bindings.Archive.readNew leak:Zig::SourceProvider::create leak:fromErrorInstance From a8188b07da377a121325b6e506eaa31609bf5c14 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 19:55:07 +0000 Subject: [PATCH 02/33] Add bun_tar: a push-style tar reader in safe Rust Reads v7, ustar, pax, GNU long names and the four GNU/Solaris sparse formats without doing any I/O. The caller hands in whatever bytes it has and gets events back, so a source that runs dry is an ordinary state. Two dialects (General, Npm), the macOS AppleDouble rule and the charset rule are runtime options, so every platform's behaviour can be tested on any platform. ACL records are skipped for now. Nothing uses the crate yet. --- Cargo.lock | 8 + Cargo.toml | 2 + src/tar/Cargo.toml | 14 + src/tar/error.rs | 43 ++ src/tar/header.rs | 129 ++++ src/tar/lib.rs | 17 + src/tar/num.rs | 147 ++++ src/tar/reader.rs | 1674 ++++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 2034 insertions(+) create mode 100644 src/tar/Cargo.toml create mode 100644 src/tar/error.rs create mode 100644 src/tar/header.rs create mode 100644 src/tar/lib.rs create mode 100644 src/tar/num.rs create mode 100644 src/tar/reader.rs diff --git a/Cargo.lock b/Cargo.lock index 01b1822ab1e6..21d0f7b4ad9a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1910,6 +1910,14 @@ dependencies = [ "bun_sys", ] +[[package]] +name = "bun_tar" +version = "0.0.0" +dependencies = [ + "bun_core", + "thiserror", +] + [[package]] name = "bun_tcc_sys" version = "0.0.0" diff --git a/Cargo.toml b/Cargo.toml index 2f3934682524..8b30243d5a05 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,6 +44,7 @@ members = [ "src/sourcemap", "src/sql", "src/sys", + "src/tar", "src/threading", "src/valkey", "src/watcher", @@ -411,6 +412,7 @@ bun_semver = { path = "src/semver" } bun_sourcemap = { path = "src/sourcemap" } bun_sql = { path = "src/sql" } bun_sys = { path = "src/sys" } +bun_tar = { path = "src/tar" } bun_threading = { path = "src/threading" } bun_valkey = { path = "src/valkey" } bun_watcher = { path = "src/watcher" } diff --git a/src/tar/Cargo.toml b/src/tar/Cargo.toml new file mode 100644 index 000000000000..f64efb19cc5d --- /dev/null +++ b/src/tar/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "bun_tar" +version.workspace = true +edition.workspace = true + +[lib] +path = "lib.rs" + +[lints] +workspace = true + +[dependencies] +thiserror.workspace = true +bun_core.workspace = true diff --git a/src/tar/error.rs b/src/tar/error.rs new file mode 100644 index 000000000000..bd9c179945b3 --- /dev/null +++ b/src/tar/error.rs @@ -0,0 +1,43 @@ +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +pub enum Error { + #[error("Unrecognized archive format")] + UnrecognizedFormat, + #[error("Damaged tar archive (bad header checksum)")] + BadChecksum, + #[error("Damaged tar archive (end-of-archive within a sequence of headers)")] + EndWithinHeaders, + #[error("Truncated tar archive detected while reading next header")] + TruncatedHeader, + #[error("Truncated tar archive detected while reading data")] + TruncatedData, + #[error("Truncated archive detected while reading metadata")] + TruncatedMetadata, + #[error("Truncated input file")] + TruncatedInput, + #[error("Truncated tar archive detected while reading sparse file data")] + TruncatedSparse, + #[error("Redundant '{0}' header")] + RedundantHeader(char), + #[error("Special header has invalid size")] + InvalidSpecialHeaderSize, + #[error("Invalid empty pax extended header")] + EmptyPaxHeader, + #[error("Malformed pax attribute")] + MalformedPaxAttribute, + #[error("Tar size attribute overflow")] + PaxSizeOverflow, + #[error("Tar entry has negative size")] + NegativeSize, + #[error("Tar entry size overflow")] + SizeOverflow, + #[error("Malformed sparse map data")] + MalformedSparseMap, + #[error("Unreasonably large sparse map")] + SparseMapTooLarge, + #[error("Oversized AppleDouble extension")] + OversizedMacMetadata, + #[error("Out of memory")] + OutOfMemory, +} + +pub type Result = core::result::Result; diff --git a/src/tar/header.rs b/src/tar/header.rs new file mode 100644 index 000000000000..f97e2f6bd854 --- /dev/null +++ b/src/tar/header.rs @@ -0,0 +1,129 @@ +//! The 512-byte header block. + +use crate::num::parse_field; + +pub(crate) type Block = [u8; 512]; + +pub(crate) fn name(h: &Block) -> &[u8] { + &h[0..100] +} +pub(crate) fn mode(h: &Block) -> i64 { + parse_field(&h[100..108]) +} +pub(crate) fn size(h: &Block) -> i64 { + parse_field(&h[124..136]) +} +pub(crate) fn mtime(h: &Block) -> i64 { + parse_field(&h[136..148]) +} +pub(crate) fn typeflag(h: &Block) -> u8 { + h[156] +} +pub(crate) fn linkname(h: &Block) -> &[u8] { + &h[157..257] +} +/// `magic` and `version` together. +pub(crate) fn magic(h: &Block) -> &[u8] { + &h[257..265] +} +pub(crate) fn prefix(h: &Block) -> &[u8] { + &h[345..500] +} + +/// The GNU header reuses the `prefix` area. +pub(crate) mod gnu { + use super::Block; + + pub(crate) const MAGIC: &[u8; 8] = b"ustar \0"; + + pub(crate) fn sparse(h: &Block) -> &[u8] { + &h[386..482] + } + pub(crate) fn is_extended(h: &Block) -> bool { + h[482] != 0 + } + pub(crate) fn real_size(h: &Block) -> &[u8] { + &h[483..495] + } + /// A block of 21 more sparse entries. + pub(crate) fn extension_sparse(h: &Block) -> &[u8] { + &h[0..504] + } + pub(crate) fn extension_is_extended(h: &Block) -> bool { + h[504] != 0 + } +} + +pub(crate) fn is_null(h: &Block) -> bool { + h.iter().all(|&b| b == 0) +} + +/// Old BSD, Solaris and HP-UX tars summed signed bytes, so either sum is accepted. +pub(crate) fn checksum_ok(h: &Block) -> bool { + let field = &h[148..156]; + if !field.iter().all(|c| matches!(c, b' ' | 0 | b'0'..=b'7')) { + return false; + } + // Eight octal digits fit in 24 bits. + let expected = parse_field(field); + let mut unsigned: i64 = 8 * 32; + let mut signed: i64 = 8 * 32; + for &b in h[..148].iter().chain(&h[156..]) { + unsigned += i64::from(b); + signed += i64::from(b.cast_signed()); + } + expected == unsigned || expected == signed +} + +fn number_field_looks_valid(field: &[u8]) -> bool { + if matches!(field.first(), Some(128 | 255 | 0)) { + return true; + } + let rest = field.trim_ascii_start_spaces(); + let digits = rest + .iter() + .position(|c| !matches!(c, b'0'..=b'7')) + .unwrap_or(rest.len()); + rest.iter().skip(digits).all(|c| matches!(c, b' ' | 0)) +} + +trait TrimSpaces { + fn trim_ascii_start_spaces(&self) -> &[u8]; +} +impl TrimSpaces for [u8] { + fn trim_ascii_start_spaces(&self) -> &[u8] { + let n = self.iter().position(|&c| c != b' ').unwrap_or(self.len()); + self.get(n..).unwrap_or_default() + } +} + +/// How sure we are that `h` starts a tar archive, in bits verified. Zero means it does not. +pub(crate) fn bid(h: &Block) -> u32 { + if is_null(h) { + return 10; + } + if !checksum_ok(h) { + return 0; + } + let mut bid = 48; + if magic(h) == b"ustar\x0000" || magic(h) == gnu::MAGIC { + bid += 56; + } + if !matches!(typeflag(h), 0 | b'0'..=b'9' | b'A'..=b'Z' | b'a'..=b'z') { + return 0; + } + bid += 2; + let fields = [ + &h[100..108], + &h[108..116], + &h[116..124], + &h[136..148], + &h[124..136], + &h[329..337], + &h[337..345], + ]; + if !fields.into_iter().all(number_field_looks_valid) { + return 0; + } + bid +} diff --git a/src/tar/lib.rs b/src/tar/lib.rs new file mode 100644 index 000000000000..f3ea5e7422d3 --- /dev/null +++ b/src/tar/lib.rs @@ -0,0 +1,17 @@ +//! The tar format, read and written without any I/O. +//! +//! [`Reader`] is a push parser: the caller hands it whatever bytes it has and gets events back, so +//! a source that runs dry is an ordinary state. [`Dialect`] picks between the two ways Bun reads +//! an archive. +#![forbid(unsafe_code)] +#![warn(unused_must_use)] + +pub mod error; +mod header; +mod num; +pub mod reader; + +pub use error::{Error, Result}; +pub use reader::{Charset, Dialect, Entry, Event, Options, Reader}; + +pub const BLOCK_SIZE: usize = 512; diff --git a/src/tar/num.rs b/src/tar/num.rs new file mode 100644 index 000000000000..a57b97fed0e9 --- /dev/null +++ b/src/tar/num.rs @@ -0,0 +1,147 @@ +//! Number fields. Overflow saturates instead of failing, and a field ends at the first byte that +//! is not a digit. + +fn parse_base_n(field: &[u8], base: u8) -> i64 { + let mut bytes = field.iter().copied().peekable(); + while bytes.next_if(|c| matches!(c, b' ' | b'\t')).is_some() {} + let negative = bytes.next_if_eq(&b'-').is_some(); + let mut value: i64 = 0; + for c in bytes { + let digit = c.wrapping_sub(b'0'); + if digit >= base { + break; + } + let digit = if negative { + -i64::from(digit) + } else { + i64::from(digit) + }; + let Some(next) = value + .checked_mul(i64::from(base)) + .and_then(|v| v.checked_add(digit)) + else { + return if negative { i64::MIN } else { i64::MAX }; + }; + value = next; + } + value +} + +pub(crate) fn parse_decimal(field: &[u8]) -> i64 { + parse_base_n(field, 10) +} + +/// Big-endian two's complement whose top bit is the base-256 marker. +fn parse_base_256(field: &[u8]) -> i64 { + let Some((&first, mut rest)) = field.split_first() else { + return 0; + }; + let negative = first & 0x40 != 0; + let fill: u8 = if negative { 0xff } else { 0 }; + let saturated = if negative { i64::MIN } else { i64::MAX }; + let mut c = if negative { first | 0x80 } else { first & 0x7f }; + while rest.len() >= size_of::() { + if c != fill { + return saturated; + } + let Some((&next, tail)) = rest.split_first() else { + break; + }; + c = next; + rest = tail; + } + if (c ^ fill) & 0x80 != 0 { + return saturated; + } + let mut value: u64 = if negative { u64::MAX } else { 0 }; + value = (value << 8) | u64::from(c); + for &b in rest { + value = (value << 8) | u64::from(b); + } + value.cast_signed() +} + +/// A header field: octal, or base-256 when the top bit of the first byte is set. +pub(crate) fn parse_field(field: &[u8]) -> i64 { + match field.first() { + Some(first) if first & 0x80 != 0 => parse_base_256(field), + _ => parse_base_n(field, 8), + } +} + +/// `seconds[.fraction]`. `None` for anything else, or for seconds outside `i64`. +pub(crate) fn parse_pax_time(value: &[u8]) -> Option { + if value.is_empty() { + return Some(0); + } + let (negative, digits) = match value.split_first() { + Some((b'-', rest)) => (true, rest), + _ => (false, value), + }; + let int_len = digits + .iter() + .position(|c| !c.is_ascii_digit()) + .unwrap_or(digits.len()); + let (int, fraction) = digits.split_at(int_len); + let mut seconds: i64 = 0; + for c in int { + seconds = seconds + .checked_mul(10)? + .checked_add(i64::from(c.wrapping_sub(b'0')))?; + } + match fraction.split_first() { + None => {} + Some((b'.', rest)) if rest.iter().all(u8::is_ascii_digit) => {} + Some(_) => return None, + } + Some(if negative { -seconds } else { seconds }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn octal() { + assert_eq!(parse_field(b"0000644\0"), 0o644); + assert_eq!(parse_field(b" \t755 "), 0o755); + assert_eq!(parse_field(b"-17"), -0o17); + assert_eq!(parse_field(b"128"), 0o12); + assert_eq!(parse_field(b""), 0); + assert_eq!(parse_field(b"7777777777777777777777777"), i64::MAX); + assert_eq!(parse_field(b"-7777777777777777777777777"), i64::MIN); + } + + #[test] + fn base_256() { + assert_eq!(parse_field(&[0x80, 0, 0, 0, 0, 0, 0, 1]), 1); + assert_eq!(parse_field(&[0xff; 8]), -1); + assert_eq!(parse_field(&[0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0]), 256); + assert_eq!( + parse_field(&[0x80, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), + i64::MAX + ); + assert_eq!( + parse_field(&[0xff, 0xfe, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), + i64::MIN + ); + assert_eq!( + parse_field(&[0x80, 0, 0, 0, 0x80, 0, 0, 0, 0, 0, 0, 0]), + i64::MAX + ); + assert_eq!(parse_field(&[0x81]), 1); + assert_eq!(parse_field(&[0xc1]), -63); + } + + #[test] + fn pax_time() { + assert_eq!(parse_pax_time(b""), Some(0)); + assert_eq!(parse_pax_time(b"12"), Some(12)); + assert_eq!(parse_pax_time(b"-12.5"), Some(-12)); + assert_eq!(parse_pax_time(b"12."), Some(12)); + assert_eq!(parse_pax_time(b"12.0000000001234"), Some(12)); + assert_eq!(parse_pax_time(b"12.5x"), None); + assert_eq!(parse_pax_time(b"12x"), None); + assert_eq!(parse_pax_time(b"99999999999999999999"), None); + } +} diff --git a/src/tar/reader.rs b/src/tar/reader.rs new file mode 100644 index 000000000000..4d62d288454a --- /dev/null +++ b/src/tar/reader.rs @@ -0,0 +1,1674 @@ +use crate::error::{Error, Result}; +use crate::header::{self, Block, gnu}; +use crate::num::{parse_decimal, parse_field, parse_pax_time}; +use bun_core::strings; + +/// Where tar readers disagree about what counts as a file, as its name or as the end of the +/// archive, this picks whose answer to give. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Dialect { + /// Zero blocks never end the archive, so archives that were concatenated read as one. + General, + /// What npm's own tar reader does with a package. + Npm, +} + +/// Decides which pax strings count as convertible. An entry with one that is not carries +/// [`Entry::warned`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Charset { + /// POSIX in the "C" locale: only ASCII converts. + Ascii, + /// Windows, where names become UTF-16: valid UTF-8 converts. + Utf8, +} + +#[derive(Debug, Clone, Copy)] +pub struct Options { + pub dialect: Dialect, + pub charset: Charset, + /// Take an entry named `._name` for the AppleDouble metadata of the entry behind it and drop + /// it. macOS only. + pub mac_metadata: bool, + /// Refuse input whose first block does not look like tar. + pub detect: bool, +} + +pub const S_IFMT: u32 = 0o170000; +pub const S_IFREG: u32 = 0o100000; +pub const S_IFDIR: u32 = 0o040000; +pub const S_IFLNK: u32 = 0o120000; +pub const S_IFCHR: u32 = 0o020000; +pub const S_IFBLK: u32 = 0o060000; +pub const S_IFIFO: u32 = 0o010000; + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +enum LinkKind { + #[default] + None, + Hard, + Symbolic, +} + +#[derive(Debug, Default)] +pub struct Entry { + path: Vec, + path_is_utf8: bool, + link: Vec, + link_is_utf8: bool, + link_kind: LinkKind, + mode: u32, + perm_is_set: bool, + size: i64, + mtime: Option, + warned: bool, +} + +impl Entry { + /// Never holds a NUL. + pub fn path(&self) -> &[u8] { + &self.path + } + /// The path came from a pax record that [`Charset`] could convert. Otherwise its bytes have no + /// known encoding. + pub fn path_is_utf8(&self) -> bool { + self.path_is_utf8 + } + pub fn symlink(&self) -> Option<&[u8]> { + (self.link_kind == LinkKind::Symbolic).then_some(&self.link[..]) + } + pub fn hardlink(&self) -> Option<&[u8]> { + (self.link_kind == LinkKind::Hard).then_some(&self.link[..]) + } + pub fn link_is_utf8(&self) -> bool { + self.link_is_utf8 + } + /// One of the `S_IF*` constants, or whatever the header's mode field held for the few types + /// that say nothing about it. + pub fn filetype(&self) -> u32 { + self.mode & S_IFMT + } + /// Not masked to the permission bits: a base-256 mode field can hold more. + pub fn perm(&self) -> u32 { + self.mode & !S_IFMT + } + /// The length of the file on disk, which for a sparse entry is more than the archive stores. + pub fn size(&self) -> i64 { + self.size + } + pub fn mtime(&self) -> i64 { + self.mtime.unwrap_or(0) + } + /// Something about the headers was off, and the entry is a best effort. + pub fn warned(&self) -> bool { + self.warned + } +} + +#[derive(Debug)] +pub enum Event<'a> { + /// Everything given was consumed. Call again with more, or with `eof` set. + NeedInput, + Entry(&'a Entry), + /// `offset` only jumps for a sparse entry. + Data { + offset: i64, + bytes: &'a [u8], + }, + /// `size` is the length of the file, which may lie past the last `Data`. + EndOfEntry { + size: i64, + }, + EndOfArchive, +} + +/// Ordered so that the worse of two is the smaller one. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +enum Status { + /// Becomes an error once the entry's own header has been read. + Failed, + Warn, + Ok, +} + +/// Sticks across entries. It decides whether the size of a hardlink entry is believed. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Format { + Old, + Ustar, + Pax, + Gnu, +} + +#[derive(Debug, Clone, Copy)] +struct SparseBlock { + offset: i64, + remaining: i64, + hole: bool, +} + +mod seen { + pub(super) const SOLARIS_ACL: u32 = 1; + pub(super) const PAX_GLOBAL: u32 = 2; + pub(super) const LONG_LINK: u32 = 4; + pub(super) const LONG_NAME: u32 = 8; + pub(super) const VOLUME: u32 = 16; + pub(super) const PAX: u32 = 32; + pub(super) const MAC_METADATA: u32 = 512; +} + +mod size_field { + pub(super) const GNU_SPARSE_REALSIZE: u8 = 1; + pub(super) const GNU_SPARSE_SIZE: u8 = 2; + pub(super) const SCHILY_REALSIZE: u8 = 4; + pub(super) const PAX_SIZE: u8 = 8; +} + +const PATHNAME_LIMIT: i64 = 1024 * 1024; +const USER_NAME_LIMIT: i64 = 65536; +const SPARSE_MAP_LIMIT: i64 = 8 * 1024 * 1024; +const XATTR_LIMIT: i64 = 16 * 1024 * 1024; +const FFLAGS_LIMIT: i64 = 512; +const ENTRY_LIMIT: i64 = 0x0fff_ffff_ffff_ffff; +const PAX_BODY_LIMIT: i64 = 1024 * 1024 * 1024; +const PAX_LINE_LIMIT: i64 = 99_999_999; +/// Longer than every key that is matched in full. +const KEY_CAPACITY: usize = 32; + +fn padding_for(size: i64) -> i64 { + 0x1ff & size.wrapping_neg() +} + +/// Everything here was a C string once, and callers depend on names ending at the first NUL. +fn until_nul(bytes: &[u8]) -> &[u8] { + match strings::index_of_char_usize(bytes, 0) { + Some(end) => bytes.get(..end).unwrap_or_default(), + None => bytes, + } +} + +fn assign(dest: &mut Vec, bytes: &[u8]) -> Result<()> { + dest.clear(); + append(dest, bytes) +} + +fn append(dest: &mut Vec, bytes: &[u8]) -> Result<()> { + dest.try_reserve(bytes.len()) + .map_err(|_| Error::OutOfMemory)?; + dest.extend_from_slice(bytes); + Ok(()) +} + +#[derive(Debug, Clone, Copy)] +enum AfterSkip { + /// The body of a header that says something about the entry behind it. + BetweenHeaders(Status), + PaxNewline(Status), + MacMetadata, + SparseMap, + EndOfEntry, + NextEntry, +} + +#[derive(Debug, Clone, Copy)] +enum Number { + SparseOffset, + SparseNumBytes, + SparseSize, + SparseMajor, + SparseMinor, + SparseRealSize, + SchilyRealSize, + Size, + /// Only whether it parses matters. + Unused, +} + +#[derive(Debug, Clone, Copy)] +enum Text { + Path, + PathOverride, + LinkPath, + UserName, + GroupName, +} + +#[derive(Debug, Clone, Copy)] +enum Collect { + LongName, + LongLink, + Number(Number), + /// `true` for `mtime`. Of the others only whether they parse matters. + Time(bool), + Text(Text), + HeaderCharset, + SymlinkType, + SparseMap, + HolesData, +} + +#[derive(Debug, Clone, Copy)] +enum SparseMapField { + Count, + Offset, + Size, +} + +#[derive(Debug, Clone, Copy)] +enum State { + Detect, + NextEntry, + Header, + Skip(i64, AfterSkip), + Collect(usize, Collect), + PaxLength, + PaxKey, + PaxNewline(Status), + OldSparseExtension, + SparseMapLine(SparseMapField), + Emit, + Data, + /// Behind the end of an npm archive. The input is still read to its end so that whatever + /// decompresses it gets to check all of it. + Drain, + Done, + Failed(Error), +} + +enum Step { + Continue, + NeedInput, + Entry, + Data { + offset: i64, + start: usize, + len: usize, + }, + EndOfEntry, + EndOfArchive, +} + +pub struct Reader { + options: Options, + state: State, + + block: Block, + block_len: usize, + /// The entry's own header, kept while `block` takes the sparse extension blocks behind it. + main_header: Block, + collected: Vec, + + // One sequence of headers, which ends with the entry's own. + status: Status, + failure: Error, + eof_is_fatal: bool, + seen: u32, + pax_is_utf8: bool, + sparse_attributes_seen: bool, + pax_path: Vec, + pax_path_override: Vec, + pax_link: Vec, + user_name_converts: bool, + group_name_converts: bool, + + // One pax header. + pax_status: Status, + pax_left: i64, + pax_padding: i64, + pax_line_length: i64, + pax_line_pos: i64, + pax_key: [u8; KEY_CAPACITY], + /// Up to the first NUL, and possibly more than `pax_key` holds. + pax_key_len: usize, + pax_key_ended: bool, + pax_key_started: bool, + + // One entry. + entry: Entry, + size_fields: u8, + sparse: Vec, + sparse_head: usize, + sparse_map_left: i64, + sparse_map_entries: i64, + sparse_map_offset: i64, + line: [u8; 100], + line_len: usize, + + // Never reset, so a header that leaves one out inherits it from an earlier entry. + format: Format, + typeflag: u8, + sparse_major: i64, + sparse_minor: i64, + sparse_offset: i64, + sparse_num_bytes: i64, + gnu_sparse_size: i64, + gnu_sparse_real_size: i64, + schily_real_size: i64, + pax_size: i64, + disk_size: i64, + bytes_remaining: i64, + padding: i64, + null_blocks: u32, +} + +impl Reader { + pub fn new(options: Options) -> Self { + Self { + options, + state: if options.detect { + State::Detect + } else { + State::NextEntry + }, + block: [0; 512], + block_len: 0, + main_header: [0; 512], + collected: Vec::new(), + status: Status::Ok, + failure: Error::MalformedPaxAttribute, + eof_is_fatal: false, + seen: 0, + pax_is_utf8: true, + sparse_attributes_seen: false, + pax_path: Vec::new(), + pax_path_override: Vec::new(), + pax_link: Vec::new(), + user_name_converts: true, + group_name_converts: true, + pax_status: Status::Ok, + pax_left: 0, + pax_padding: 0, + pax_line_length: 0, + pax_line_pos: 0, + pax_key: [0; KEY_CAPACITY], + pax_key_len: 0, + pax_key_ended: false, + pax_key_started: false, + entry: Entry::default(), + size_fields: 0, + sparse: Vec::new(), + sparse_head: 0, + sparse_map_left: 0, + sparse_map_entries: 0, + sparse_map_offset: 0, + line: [0; 100], + line_len: 0, + format: Format::Old, + typeflag: 0, + sparse_major: 0, + sparse_minor: 0, + sparse_offset: 0, + sparse_num_bytes: 0, + gnu_sparse_size: 0, + gnu_sparse_real_size: 0, + schily_real_size: 0, + pax_size: 0, + disk_size: 0, + bytes_remaining: 0, + padding: 0, + null_blocks: 0, + } + } + + fn is_npm(&self) -> bool { + self.options.dialect == Dialect::Npm + } + + /// Returns how much of `input` was consumed along with the event. `eof` says that nothing + /// follows `input`. An error is final: every later call returns it again. + pub fn next<'a>(&'a mut self, input: &'a [u8], eof: bool) -> Result<(usize, Event<'a>)> { + let mut pos = 0usize; + loop { + let rest = input.get(pos..).unwrap_or_default(); + let mut used = 0usize; + let step = match self.step(rest, eof, &mut used) { + Ok(step) => step, + Err(error) => { + self.state = State::Failed(error); + return Err(error); + } + }; + let start_of_rest = pos; + pos = pos.saturating_add(used); + let event = match step { + Step::Continue => continue, + Step::NeedInput => Event::NeedInput, + Step::Entry => Event::Entry(&self.entry), + Step::Data { offset, start, len } => { + let from = start_of_rest.saturating_add(start); + Event::Data { + offset, + bytes: input + .get(from..from.saturating_add(len)) + .unwrap_or_default(), + } + } + Step::EndOfEntry => Event::EndOfEntry { + size: self.disk_size, + }, + Step::EndOfArchive => Event::EndOfArchive, + }; + return Ok((pos, event)); + } + } + + /// Drops what is left of the current entry's data. No `EndOfEntry` follows. + pub fn skip_entry(&mut self) { + if matches!(self.state, State::Data) { + self.state = State::NextEntry; + } + } + + /// `None` when `input` ran out first. + fn fill_block(&mut self, input: &[u8], used: &mut usize) -> Option<()> { + let want = self.block.len().saturating_sub(self.block_len); + let take = want.min(input.len()); + if let (Some(dest), Some(src)) = ( + self.block + .get_mut(self.block_len..self.block_len.saturating_add(take)), + input.get(..take), + ) { + dest.copy_from_slice(src); + } + self.block_len = self.block_len.saturating_add(take); + *used = take; + (self.block_len == self.block.len()).then_some(()) + } + + fn step(&mut self, input: &[u8], eof: bool, used: &mut usize) -> Result { + match self.state { + State::Failed(error) => Err(error), + State::Done => Ok(Step::EndOfArchive), + + State::Detect => { + if self.fill_block(input, used).is_none() { + if !eof { + return Ok(Step::NeedInput); + } + return Err(Error::UnrecognizedFormat); + } + if header::bid(&self.block) == 0 { + return Err(Error::UnrecognizedFormat); + } + // `block` stays full, so `Header` takes it from here. + self.state = State::NextEntry; + Ok(Step::Continue) + } + + State::NextEntry => { + let left = self.bytes_remaining.saturating_add(self.padding); + self.bytes_remaining = 0; + self.padding = 0; + self.state = State::Skip(left, AfterSkip::NextEntry); + Ok(Step::Continue) + } + + State::Header => { + if self.fill_block(input, used).is_none() { + if !eof { + return Ok(Step::NeedInput); + } + if self.block_len != 0 { + return Err(Error::TruncatedHeader); + } + if self.eof_is_fatal && !self.is_npm() { + return Err(Error::EndWithinHeaders); + } + self.state = State::Done; + return Ok(Step::EndOfArchive); + } + self.block_len = 0; + self.on_header()?; + Ok(Step::Continue) + } + + State::Skip(left, after) => { + let take = usize::try_from(left).unwrap_or(usize::MAX).min(input.len()); + *used = take; + let left = left.saturating_sub(i64::try_from(take).unwrap_or(i64::MAX)); + if left > 0 { + self.state = State::Skip(left, after); + if !eof { + return Ok(Step::NeedInput); + } + return Err(Error::TruncatedInput); + } + self.after_skip(after) + } + + State::Collect(left, what) => { + let take = left.min(input.len()); + append(&mut self.collected, input.get(..take).unwrap_or_default())?; + *used = take; + let left = left.saturating_sub(take); + if left > 0 { + self.state = State::Collect(left, what); + if !eof { + return Ok(Step::NeedInput); + } + return Err(Error::TruncatedMetadata); + } + self.on_collected(what)?; + Ok(Step::Continue) + } + + State::PaxLength => { + for &c in input { + if self.pax_line_pos >= self.pax_left { + break; + } + *used = used.saturating_add(1); + self.pax_line_pos = self.pax_line_pos.saturating_add(1); + match c { + b' ' => { + if self.pax_line_length > self.pax_left { + self.ignore_rest_of_pax(); + } else { + self.pax_key_len = 0; + self.pax_key_ended = false; + self.state = State::PaxKey; + } + return Ok(Step::Continue); + } + b'0'..=b'9' => { + self.pax_line_length = self + .pax_line_length + .saturating_mul(10) + .saturating_add(i64::from(c.wrapping_sub(b'0'))); + if self.pax_line_length > PAX_LINE_LIMIT { + self.ignore_rest_of_pax(); + return Ok(Step::Continue); + } + } + _ => { + self.ignore_rest_of_pax(); + return Ok(Step::Continue); + } + } + } + self.pax_prologue_ran_out(eof) + } + + State::PaxKey => { + for &c in input { + if self.pax_line_pos >= self.pax_left + || self.pax_line_pos >= self.pax_line_length + { + break; + } + *used = used.saturating_add(1); + self.pax_line_pos = self.pax_line_pos.saturating_add(1); + if c != b'=' { + if c == 0 { + self.pax_key_ended = true; + } + if !self.pax_key_ended { + if let Some(slot) = self.pax_key.get_mut(self.pax_key_len) { + *slot = c; + } + self.pax_key_len = self.pax_key_len.saturating_add(1); + } + // A key that starts with a NUL is still a key. + self.pax_key_started = true; + continue; + } + if !self.pax_key_started { + self.ignore_rest_of_pax(); + return Ok(Step::Continue); + } + self.on_pax_key()?; + return Ok(Step::Continue); + } + self.pax_prologue_ran_out(eof) + } + + State::PaxNewline(status) => { + self.pax_line_pos = 0; + if status == Status::Failed { + self.skip_rest_of_pax(Status::Failed); + return Ok(Step::Continue); + } + let Some(&c) = input.first() else { + if !eof { + return Ok(Step::NeedInput); + } + return Err(Error::TruncatedMetadata); + }; + self.pax_status = self.pax_status.min(status); + if c != b'\n' { + self.ignore_rest_of_pax(); + return Ok(Step::Continue); + } + *used = 1; + self.pax_left = self.pax_left.saturating_sub(1); + if self.pax_left > 0 { + self.begin_pax_line(); + } else { + self.finish_pax()?; + } + Ok(Step::Continue) + } + + State::OldSparseExtension => { + if self.fill_block(input, used).is_none() { + if !eof { + return Ok(Step::NeedInput); + } + return Err(Error::TruncatedSparse); + } + self.block_len = 0; + let block = self.block; + self.add_old_sparse_entries(gnu::extension_sparse(&block))?; + if !gnu::extension_is_extended(&block) { + self.header_common()?; + } + Ok(Step::Continue) + } + + State::SparseMapLine(field) => { + let limit = usize::try_from(self.sparse_map_left.min(100)).unwrap_or(0); + for &c in input { + *used = used.saturating_add(1); + if self.line_len >= limit { + return Err(Error::MalformedSparseMap); + } + if let Some(slot) = self.line.get_mut(self.line_len) { + *slot = c; + } + self.line_len = self.line_len.saturating_add(1); + if c == b'\n' { + self.on_sparse_map_line(field)?; + return Ok(Step::Continue); + } + } + if !eof { + return Ok(Step::NeedInput); + } + Err(Error::MalformedSparseMap) + } + + State::Emit => { + self.state = State::Data; + Ok(Step::Entry) + } + + State::Data => { + while self + .sparse + .get(self.sparse_head) + .is_some_and(|block| block.remaining == 0) + { + self.sparse_head = self.sparse_head.saturating_add(1); + } + let Some(block) = self + .sparse + .get_mut(self.sparse_head) + .filter(|_| self.bytes_remaining != 0) + else { + let left = self.bytes_remaining.saturating_add(self.padding); + self.bytes_remaining = 0; + self.padding = 0; + self.state = State::Skip(left, AfterSkip::EndOfEntry); + return Ok(Step::Continue); + }; + if input.is_empty() { + if !eof { + return Ok(Step::NeedInput); + } + return Err(Error::TruncatedData); + } + let len = i64::try_from(input.len()) + .unwrap_or(i64::MAX) + .min(self.bytes_remaining) + .min(block.remaining); + let offset = block.offset; + block.remaining = block.remaining.saturating_sub(len); + block.offset = block.offset.saturating_add(len); + self.bytes_remaining = self.bytes_remaining.saturating_sub(len); + let len = usize::try_from(len).unwrap_or(0); + *used = len; + if block.hole { + return Ok(Step::Continue); + } + Ok(Step::Data { + offset, + start: 0, + len, + }) + } + + State::Drain => { + *used = input.len(); + if !eof { + return Ok(Step::NeedInput); + } + self.state = State::Done; + Ok(Step::EndOfArchive) + } + } + } + + fn after_skip(&mut self, after: AfterSkip) -> Result { + match after { + AfterSkip::BetweenHeaders(status) => self.between_headers(status), + AfterSkip::PaxNewline(status) => self.state = State::PaxNewline(status), + AfterSkip::MacMetadata => { + self.entry.clear(); + self.seen = seen::MAC_METADATA; + self.reset_header_state(); + self.between_headers(Status::Ok); + } + AfterSkip::SparseMap => self.finish_header(self.status)?, + AfterSkip::EndOfEntry => { + self.state = State::NextEntry; + return Ok(Step::EndOfEntry); + } + AfterSkip::NextEntry => { + self.entry.clear(); + self.sparse.clear(); + self.sparse_head = 0; + self.size_fields = 0; + self.status = Status::Ok; + self.eof_is_fatal = false; + self.seen = 0; + self.reset_header_state(); + self.state = State::Header; + } + } + Ok(Step::Continue) + } + + fn reset_header_state(&mut self) { + self.pax_is_utf8 = true; + self.sparse_attributes_seen = false; + self.pax_path.clear(); + self.pax_path_override.clear(); + self.pax_link.clear(); + self.user_name_converts = true; + self.group_name_converts = true; + } + + fn fail_later(&mut self, error: Error) -> Status { + self.failure = error; + Status::Failed + } + + fn between_headers(&mut self, status: Status) { + self.status = self.status.min(status); + if self.seen & !(seen::VOLUME | seen::PAX_GLOBAL) != 0 { + self.eof_is_fatal = true; + } + self.state = State::Header; + } + + fn mark_seen(&mut self, bit: u32) -> bool { + let before = self.seen & bit != 0; + self.seen |= bit; + before + } + + fn on_header(&mut self) -> Result<()> { + let block = self.block; + if header::is_null(&block) { + if self.is_npm() { + self.null_blocks = self.null_blocks.saturating_add(1); + if self.null_blocks == 2 { + self.state = State::Drain; + } + } + return Ok(()); + } + self.null_blocks = 0; + + if !header::checksum_ok(&block) { + if self.eof_is_fatal { + return Err(Error::BadChecksum); + } + // Look for a header one block further on, with nothing carried over. + self.state = State::NextEntry; + return Ok(()); + } + + let size = header::size(&block); + match header::typeflag(&block) { + b'A' => { + if self.mark_seen(seen::SOLARIS_ACL) { + return Err(Error::RedundantHeader('A')); + } + self.format = Format::Pax; + self.begin_body(size, None) + } + b'g' => { + if self.mark_seen(seen::PAX_GLOBAL) { + return Err(Error::RedundantHeader('g')); + } + self.format = Format::Pax; + if !(0..=ENTRY_LIMIT).contains(&size) { + return Err(Error::InvalidSpecialHeaderSize); + } + if size == 0 { + return Err(Error::EmptyPaxHeader); + } + self.skip_body(size, Status::Ok); + Ok(()) + } + b'K' => { + if self.mark_seen(seen::LONG_LINK) { + self.status = self.status.min(Status::Warn); + } + self.format = Format::Gnu; + self.begin_body(size, Some(Collect::LongLink)) + } + b'L' => { + if self.mark_seen(seen::LONG_NAME) { + self.status = self.status.min(Status::Warn); + } + self.format = Format::Gnu; + self.begin_body(size, Some(Collect::LongName)) + } + b'V' => { + if self.mark_seen(seen::VOLUME) { + self.status = self.status.min(Status::Warn); + } + self.format = Format::Gnu; + if !(0..=PATHNAME_LIMIT).contains(&size) { + return Err(Error::InvalidSpecialHeaderSize); + } + self.skip_body(size, Status::Ok); + Ok(()) + } + flag @ (b'x' | b'X') => { + if self.mark_seen(seen::PAX) { + return Err(Error::RedundantHeader(char::from(flag))); + } + self.format = Format::Pax; + self.begin_pax(size) + } + _ => self.on_main_header(), + } + } + + fn skip_body(&mut self, size: i64, status: Status) { + self.state = State::Skip( + size.saturating_add(padding_for(size)), + AfterSkip::BetweenHeaders(status), + ); + } + + /// The body of a header whose size is not trusted past [`PATHNAME_LIMIT`]. + fn begin_body(&mut self, size: i64, collect: Option) -> Result<()> { + if !(0..=ENTRY_LIMIT).contains(&size) { + return Err(Error::InvalidSpecialHeaderSize); + } + if size > PATHNAME_LIMIT { + self.skip_body(size, Status::Warn); + return Ok(()); + } + let Some(collect) = collect else { + self.skip_body(size, Status::Ok); + return Ok(()); + }; + self.collected.clear(); + self.state = State::Collect(usize::try_from(size).unwrap_or(0), collect); + Ok(()) + } + + fn begin_pax(&mut self, size: i64) -> Result<()> { + if !(0..=ENTRY_LIMIT).contains(&size) { + return Err(Error::InvalidSpecialHeaderSize); + } + if size == 0 { + return Err(Error::EmptyPaxHeader); + } + if size > PAX_BODY_LIMIT { + self.skip_body(size, Status::Warn); + return Ok(()); + } + self.pax_status = Status::Ok; + self.pax_left = size; + self.pax_padding = padding_for(size); + self.begin_pax_line(); + Ok(()) + } + + fn begin_pax_line(&mut self) { + self.pax_line_length = 0; + self.pax_line_pos = 0; + self.pax_key_started = false; + self.state = State::PaxLength; + } + + /// One malformed record makes the rest of the header unreadable. What came before it counts. + fn ignore_rest_of_pax(&mut self) { + self.skip_rest_of_pax(Status::Warn); + } + + fn skip_rest_of_pax(&mut self, status: Status) { + let left = self + .pax_left + .saturating_sub(self.pax_line_pos) + .saturating_add(self.pax_padding); + self.state = State::Skip(left, AfterSkip::BetweenHeaders(status)); + } + + fn pax_prologue_ran_out(&mut self, eof: bool) -> Result { + let at_end = self.pax_line_pos >= self.pax_left + || (matches!(self.state, State::PaxKey) && self.pax_line_pos >= self.pax_line_length); + if at_end { + self.ignore_rest_of_pax(); + return Ok(Step::Continue); + } + if !eof { + return Ok(Step::NeedInput); + } + Err(Error::TruncatedMetadata) + } + + fn on_pax_key(&mut self) -> Result<()> { + self.pax_left = self.pax_left.saturating_sub(self.pax_line_pos); + let with_newline = self.pax_line_length.saturating_sub(self.pax_line_pos); + self.pax_line_pos = 0; + if with_newline == 0 { + self.ignore_rest_of_pax(); + return Ok(()); + } + let length = with_newline.saturating_sub(1); + self.pax_left = self.pax_left.saturating_sub(length); + + enum Value { + Skip(Status), + Collect(Collect), + } + use Value::{Collect as Take, Skip}; + + let number = |which: Number| -> Result { + if length > 64 { + return Err(Error::MalformedPaxAttribute); + } + Ok(Take(Collect::Number(which))) + }; + let time = |is_mtime: bool| -> Result { + if length > 128 { + return Err(Error::MalformedPaxAttribute); + } + Ok(Take(Collect::Time(is_mtime))) + }; + let warn_unless = |ok: bool| Skip(if ok { Status::Ok } else { Status::Warn }); + + let stored = self.pax_key_len.min(KEY_CAPACITY); + let key_buf = self.pax_key; + let key = key_buf.get(..stored).unwrap_or_default(); + let whole = self.pax_key_len <= KEY_CAPACITY; + let is = |name: &[u8]| whole && key == name; + + let value = if self.is_npm() && key.first().is_some_and(u8::is_ascii_uppercase) { + Skip(Status::Ok) + } else if let Some(sparse) = key.strip_prefix(b"GNU.sparse") { + if sparse.is_empty() || (sparse.len() > 1 && sparse.first() == Some(&b'.')) { + self.sparse_attributes_seen = true; + } + match sparse { + _ if !whole => Skip(Status::Ok), + b".numblocks" => { + self.sparse_offset = -1; + self.sparse_num_bytes = -1; + self.sparse_major = 0; + self.sparse_minor = 0; + Skip(Status::Ok) + } + b".offset" => number(Number::SparseOffset)?, + b".numbytes" => number(Number::SparseNumBytes)?, + b".size" => number(Number::SparseSize)?, + b".map" => { + self.sparse_major = 0; + self.sparse_minor = 1; + if length > SPARSE_MAP_LIMIT { + Skip(self.fail_later(Error::SparseMapTooLarge)) + } else { + Take(Collect::SparseMap) + } + } + b".major" => number(Number::SparseMajor)?, + b".minor" => number(Number::SparseMinor)?, + b".name" if length > PATHNAME_LIMIT => Skip(Status::Warn), + b".name" => Take(Collect::Text(Text::PathOverride)), + b".realsize" => number(Number::SparseRealSize)?, + _ => Skip(Status::Ok), + } + } else if is(b"LIBARCHIVE.creationtime") || is(b"atime") || is(b"ctime") { + time(false)? + } else if is(b"LIBARCHIVE.symlinktype") { + if length < 16 { + Take(Collect::SymlinkType) + } else { + Skip(Status::Warn) + } + } else if (key.starts_with(b"LIBARCHIVE.xattr.") && self.pax_key_len > 17) + || is(b"RHT.security.selinux") + { + warn_unless(length <= XATTR_LIMIT) + } else if is(b"SCHILY.devmajor") + || is(b"SCHILY.devminor") + || is(b"SCHILY.dev") + || is(b"SCHILY.ino") + || is(b"SCHILY.nlink") + || is(b"gid") + || is(b"uid") + { + number(Number::Unused)? + } else if is(b"SCHILY.fflags") { + warn_unless(length < FFLAGS_LIMIT) + } else if is(b"SCHILY.realsize") { + number(Number::SchilyRealSize)? + } else if key.starts_with(b"SCHILY.xattr.") && self.pax_key_len > 13 { + warn_unless(length < XATTR_LIMIT && self.pax_key_len.saturating_sub(13) <= 128) + } else if is(b"SUN.holesdata") { + if length < SPARSE_MAP_LIMIT { + Take(Collect::HolesData) + } else { + Skip(self.fail_later(Error::SparseMapTooLarge)) + } + } else if is(b"gname") || is(b"uname") { + if length > USER_NAME_LIMIT { + Skip(Status::Warn) + } else if is(b"gname") { + Take(Collect::Text(Text::GroupName)) + } else { + Take(Collect::Text(Text::UserName)) + } + } else if is(b"hdrcharset") { + if length < 64 { + Take(Collect::HeaderCharset) + } else { + Skip(Status::Warn) + } + } else if is(b"linkpath") || is(b"path") { + if length > PATHNAME_LIMIT { + Skip(Status::Warn) + } else if is(b"path") { + Take(Collect::Text(Text::Path)) + } else { + Take(Collect::Text(Text::LinkPath)) + } + } else if is(b"mtime") { + time(true)? + } else if is(b"size") { + number(Number::Size)? + } else { + Skip(Status::Ok) + }; + + match value { + Skip(status) => self.state = State::Skip(length, AfterSkip::PaxNewline(status)), + Take(what) => { + self.collected.clear(); + self.state = State::Collect(usize::try_from(length).unwrap_or(0), what); + } + } + Ok(()) + } + + /// `status` is what the value's own parser made of it. + fn pax_value_done(&mut self, status: Status) { + self.state = State::PaxNewline(status); + } + + fn converts(&self, text: &[u8]) -> bool { + let text = until_nul(text); + match self.options.charset { + Charset::Ascii => strings::first_non_ascii(text).is_none(), + Charset::Utf8 => strings::is_valid_utf8(text), + } + } + + fn finish_pax(&mut self) -> Result<()> { + let mut status = self.pax_status; + let utf8 = self.pax_is_utf8; + + let path = if self.pax_path_override.is_empty() { + &self.pax_path + } else { + &self.pax_path_override + }; + if !path.is_empty() { + let converts = !utf8 || self.converts(path); + assign(&mut self.entry.path, until_nul(path))?; + self.entry.path_is_utf8 = utf8 && converts; + if !converts { + status = Status::Warn; + } + } + if utf8 && !(self.user_name_converts && self.group_name_converts) { + status = Status::Warn; + } + if !self.pax_link.is_empty() { + let converts = !utf8 || self.converts(&self.pax_link); + assign(&mut self.entry.link, until_nul(&self.pax_link))?; + self.entry.link_is_utf8 = utf8 && converts; + if self.entry.link_kind == LinkKind::None { + self.entry.link_kind = LinkKind::Hard; + } + if !converts { + status = Status::Warn; + } + } + self.state = State::Skip( + self.pax_left.saturating_add(self.pax_padding), + AfterSkip::BetweenHeaders(status), + ); + Ok(()) + } + + fn on_collected(&mut self, what: Collect) -> Result<()> { + let collected = core::mem::take(&mut self.collected); + let result = self.use_collected(what, &collected); + self.collected = collected; + result + } + + fn use_collected(&mut self, what: Collect, value: &[u8]) -> Result<()> { + let body_padding = padding_for(i64::try_from(value.len()).unwrap_or(0)); + match what { + Collect::LongName => { + assign(&mut self.entry.path, until_nul(value))?; + self.entry.path_is_utf8 = false; + self.state = State::Skip(body_padding, AfterSkip::BetweenHeaders(Status::Ok)); + } + Collect::LongLink => { + assign(&mut self.entry.link, until_nul(value))?; + self.entry.link_is_utf8 = false; + if self.entry.link_kind == LinkKind::None { + self.entry.link_kind = LinkKind::Hard; + } + self.state = State::Skip(body_padding, AfterSkip::BetweenHeaders(Status::Ok)); + } + Collect::Number(which) => { + let parsed = parse_decimal(value); + if parsed < 0 || parsed == i64::MAX { + if matches!(which, Number::Size) { + return Err(Error::PaxSizeOverflow); + } + self.pax_value_done(Status::Warn); + return Ok(()); + } + self.on_pax_number(which, parsed)?; + self.pax_value_done(Status::Ok); + } + Collect::Time(is_mtime) => match parse_pax_time(value) { + Some(seconds) => { + if is_mtime { + self.entry.mtime = Some(seconds); + } + self.pax_value_done(Status::Ok); + } + None => self.pax_value_done(Status::Warn), + }, + Collect::Text(which) => { + match which { + Text::Path => assign(&mut self.pax_path, value)?, + Text::PathOverride => assign(&mut self.pax_path_override, value)?, + Text::LinkPath => assign(&mut self.pax_link, value)?, + Text::UserName => self.user_name_converts = self.converts(value), + Text::GroupName => self.group_name_converts = self.converts(value), + } + self.pax_value_done(Status::Ok); + } + Collect::HeaderCharset => { + let status = match value { + b"BINARY" => { + self.pax_is_utf8 = false; + Status::Ok + } + b"ISO-IR 10646 2000 UTF-8" => { + self.pax_is_utf8 = true; + Status::Ok + } + _ => Status::Warn, + }; + self.pax_value_done(status); + } + Collect::SymlinkType => { + self.pax_value_done(if matches!(value, b"file" | b"dir") { + Status::Ok + } else { + Status::Warn + }); + } + Collect::SparseMap => { + let status = if self.parse_sparse_map_0_1(value)? { + Status::Ok + } else { + Status::Warn + }; + self.pax_value_done(status); + } + Collect::HolesData => { + let status = if self.parse_holes_data(value)? { + Status::Ok + } else { + Status::Warn + }; + self.pax_value_done(status); + } + } + Ok(()) + } + + fn on_pax_number(&mut self, which: Number, value: i64) -> Result<()> { + match which { + Number::SparseOffset => { + self.sparse_offset = value; + if self.sparse_num_bytes != -1 { + self.add_pending_sparse_block()?; + } + } + Number::SparseNumBytes => { + self.sparse_num_bytes = value; + if self.sparse_offset != -1 { + self.add_pending_sparse_block()?; + } + } + Number::SparseSize => { + self.gnu_sparse_size = value; + self.size_fields |= size_field::GNU_SPARSE_SIZE; + } + Number::SparseMajor if value <= 10 => self.sparse_major = value, + Number::SparseMinor if value <= 10 => self.sparse_minor = value, + Number::SparseMajor | Number::SparseMinor | Number::Unused => {} + Number::SparseRealSize => { + self.gnu_sparse_real_size = value; + self.size_fields |= size_field::GNU_SPARSE_REALSIZE; + } + Number::SchilyRealSize => { + self.schily_real_size = value; + self.size_fields |= size_field::SCHILY_REALSIZE; + } + Number::Size => { + self.pax_size = value; + self.size_fields |= size_field::PAX_SIZE; + } + } + Ok(()) + } + + fn add_pending_sparse_block(&mut self) -> Result<()> { + if !self.add_sparse_block(self.sparse_offset, self.sparse_num_bytes, false)? { + return Err(Error::MalformedSparseMap); + } + self.sparse_offset = -1; + self.sparse_num_bytes = -1; + Ok(()) + } + + /// `false` for a block that cannot be. An empty one takes its place, so the list is no longer + /// empty and the entry does not fall back to one block that covers all of it. + fn add_sparse_block(&mut self, offset: i64, remaining: i64, hole: bool) -> Result { + self.sparse.try_reserve(1).map_err(|_| Error::OutOfMemory)?; + let valid = remaining >= 0 && offset >= 0 && offset.checked_add(remaining).is_some(); + self.sparse.push(if valid { + SparseBlock { + offset, + remaining, + hole, + } + } else { + SparseBlock { + offset: 0, + remaining: 0, + hole: false, + } + }); + Ok(valid) + } + + /// `offset,size,offset,size,...` + fn parse_sparse_map_0_1(&mut self, map: &[u8]) -> Result { + let mut offset: Option = None; + for field in strings::split(map, b",") { + if !field.iter().all(u8::is_ascii_digit) { + return Ok(false); + } + let value = parse_decimal(field); + match offset.take() { + None => offset = Some(value), + Some(offset) => { + if !self.add_sparse_block(offset, value, false)? { + return Ok(false); + } + } + } + } + Ok(true) + } + + /// ` end end end ...`, where the ranges between the ends alternate between hole and data. + fn parse_holes_data(&mut self, value: &[u8]) -> Result { + let Some(mut rest) = value.strip_prefix(b" ") else { + return Ok(false); + }; + let mut end: i64 = 0; + let mut hole = true; + loop { + let len = strings::index_of_char_usize(rest, b' ').unwrap_or(rest.len()); + let (field, tail) = rest.split_at(len); + if !field.iter().all(u8::is_ascii_digit) { + return Ok(false); + } + let start = end; + end = parse_decimal(field); + if start < end && !self.add_sparse_block(start, end.saturating_sub(start), hole)? { + return Err(Error::MalformedSparseMap); + } + let Some(tail) = tail.strip_prefix(b" ") else { + return Ok(true); + }; + rest = tail; + hole = !hole; + } + } + + fn add_old_sparse_entries(&mut self, entries: &[u8]) -> Result<()> { + for entry in entries.as_chunks::<24>().0 { + let (offset, num_bytes) = entry.split_at(12); + if offset.first() == Some(&0) { + break; + } + if !self.add_sparse_block(parse_field(offset), parse_field(num_bytes), false)? { + return Err(Error::MalformedSparseMap); + } + } + Ok(()) + } + + fn on_main_header(&mut self) -> Result<()> { + self.main_header = self.block; + let h = self.main_header; + let magic = header::magic(&h); + let npm = self.is_npm(); + + if magic == gnu::MAGIC { + self.format = Format::Gnu; + if self.entry.path.is_empty() { + assign(&mut self.entry.path, until_nul(header::name(&h)))?; + } + if !npm { + if gnu::real_size(&h).first() != Some(&0) { + self.gnu_sparse_real_size = parse_field(gnu::real_size(&h)); + self.size_fields |= size_field::GNU_SPARSE_REALSIZE; + } + if gnu::sparse(&h).first() != Some(&0) { + self.add_old_sparse_entries(gnu::sparse(&h))?; + if gnu::is_extended(&h) { + self.state = State::OldSparseExtension; + return Ok(()); + } + } + } + } else if magic.starts_with(b"ustar") && (!npm || magic == b"ustar\x0000") { + if self.format != Format::Pax { + self.format = Format::Ustar; + } + if self.entry.path.is_empty() { + let prefix = until_nul(header::prefix(&h)); + assign(&mut self.entry.path, prefix)?; + if !prefix.is_empty() && prefix.last() != Some(&b'/') { + append(&mut self.entry.path, b"/")?; + } + append(&mut self.entry.path, until_nul(header::name(&h)))?; + } + } else { + self.format = Format::Old; + if self.entry.path.is_empty() { + assign(&mut self.entry.path, until_nul(header::name(&h)))?; + } + } + self.header_common() + } + + fn header_common(&mut self) -> Result<()> { + let h = self.main_header; + // Truncated the way a 32-bit `mode_t` is. + let mode = header::mode(&h) as u32; + self.entry.mode = if self.entry.perm_is_set { + (mode & S_IFMT) | self.entry.perm() + } else { + mode + }; + if self.entry.mtime.is_none() { + self.entry.mtime = Some(header::mtime(&h)); + } + + let header_size = header::size(&h); + let has = |field: u8| self.size_fields & field != 0; + + self.disk_size = if has(size_field::GNU_SPARSE_REALSIZE) { + self.gnu_sparse_real_size + } else if has(size_field::GNU_SPARSE_SIZE) && self.sparse_major == 0 { + self.gnu_sparse_size + } else if has(size_field::SCHILY_REALSIZE) { + self.schily_real_size + } else if has(size_field::PAX_SIZE) { + self.pax_size + } else { + header_size + }; + if self.disk_size < 0 { + return Err(Error::NegativeSize); + } + if self.disk_size > ENTRY_LIMIT { + return Err(Error::SizeOverflow); + } + self.entry.size = self.disk_size; + + self.bytes_remaining = if has(size_field::GNU_SPARSE_SIZE) && self.sparse_major == 1 { + self.gnu_sparse_size + } else if has(size_field::PAX_SIZE) { + self.pax_size + } else { + header_size + }; + if self.bytes_remaining < 0 { + self.bytes_remaining = 0; + return Err(Error::NegativeSize); + } + if self.bytes_remaining > ENTRY_LIMIT { + self.bytes_remaining = 0; + return Err(Error::SizeOverflow); + } + + self.typeflag = header::typeflag(&h); + let mut no_body = |entry: &mut Entry, filetype: u32| { + entry.set_filetype(filetype); + entry.size = 0; + self.bytes_remaining = 0; + }; + match self.typeflag { + b'1' => { + self.entry.link_kind = LinkKind::Hard; + if self.entry.link.is_empty() { + assign(&mut self.entry.link, until_nul(header::linkname(&h)))?; + } + if self.entry.size > 0 { + self.entry.set_filetype(S_IFREG); + // Only pax lets a hardlink entry carry a body. Whether a ustar archive is pax + // cannot be known, so the header itself is asked how well-formed it is. + let ignore_size = match self.format { + Format::Pax => false, + Format::Old | Format::Gnu => true, + Format::Ustar => header::bid(&h) > 50, + }; + if ignore_size { + self.entry.size = 0; + self.bytes_remaining = 0; + } + } + } + b'2' => { + self.entry.link_kind = LinkKind::Symbolic; + if self.entry.link.is_empty() { + assign(&mut self.entry.link, until_nul(header::linkname(&h)))?; + } + no_body(&mut self.entry, S_IFLNK); + } + b'3' => no_body(&mut self.entry, S_IFCHR), + b'4' => no_body(&mut self.entry, S_IFBLK), + b'5' => no_body(&mut self.entry, S_IFDIR), + b'6' => no_body(&mut self.entry, S_IFIFO), + b'D' => self.entry.set_filetype(S_IFDIR), + b'M' => {} + _ => self.entry.set_filetype(S_IFREG), + } + self.padding = padding_for(self.bytes_remaining); + self.after_main_header() + } + + fn after_main_header(&mut self) -> Result<()> { + if self.status == Status::Failed { + return Err(self.failure); + } + + if self.options.mac_metadata + && self.seen & seen::MAC_METADATA == 0 + && is_mac_metadata_name(&self.entry.path) + { + let size = self.entry.size; + if size > XATTR_LIMIT { + return Err(Error::OversizedMacMetadata); + } + self.state = State::Skip( + size.saturating_add(padding_for(size)), + AfterSkip::MacMetadata, + ); + return Ok(()); + } + + if self.sparse_attributes_seen { + if !matches!(self.typeflag, b'S' | b'0') { + return self.finish_header(Status::Warn); + } + match (self.sparse_major, self.sparse_minor) { + (0, 0 | 1) => {} + (1, 0) => { + self.sparse.clear(); + self.sparse_head = 0; + self.sparse_map_left = self.bytes_remaining; + self.line_len = 0; + self.state = State::SparseMapLine(SparseMapField::Count); + return Ok(()); + } + _ => return self.finish_header(Status::Warn), + } + } + self.finish_header(self.status) + } + + /// Format 1.0 keeps the map in front of the data: a count, then that many pairs of offset and + /// size, each a decimal number on a line of its own. + fn on_sparse_map_line(&mut self, field: SparseMapField) -> Result<()> { + let line_buf = self.line; + let line = line_buf.get(..self.line_len).unwrap_or_default(); + self.sparse_map_left = self + .sparse_map_left + .saturating_sub(i64::try_from(line.len()).unwrap_or(0)); + self.line_len = 0; + if line.first() == Some(&b'#') { + return Ok(()); + } + let digits = line.strip_suffix(b"\n").unwrap_or(line); + if !digits.iter().all(u8::is_ascii_digit) { + return Err(Error::MalformedSparseMap); + } + let value = parse_decimal(digits); + + match field { + SparseMapField::Count => self.sparse_map_entries = value, + SparseMapField::Offset => { + self.sparse_map_offset = value; + self.state = State::SparseMapLine(SparseMapField::Size); + return Ok(()); + } + SparseMapField::Size => { + if !self.add_sparse_block(self.sparse_map_offset, value, false)? { + return Err(Error::MalformedSparseMap); + } + } + } + if self.sparse_map_entries > 0 { + self.sparse_map_entries = self.sparse_map_entries.saturating_sub(1); + self.state = State::SparseMapLine(SparseMapField::Offset); + return Ok(()); + } + + let read = self.bytes_remaining.saturating_sub(self.sparse_map_left); + let to_skip = padding_for(read); + if to_skip > self.sparse_map_left { + return Err(Error::MalformedSparseMap); + } + self.bytes_remaining = self + .bytes_remaining + .saturating_sub(read.saturating_add(to_skip)); + self.state = State::Skip(to_skip, AfterSkip::SparseMap); + Ok(()) + } + + fn finish_header(&mut self, status: Status) -> Result<()> { + if self.sparse.is_empty() { + self.add_sparse_block(0, self.bytes_remaining, false)?; + } + + let plain_file = matches!(self.typeflag, b'0' | 0); + let may_be_directory = if self.is_npm() { + plain_file + } else { + status == Status::Ok + }; + if may_be_directory + && self.entry.filetype() == S_IFREG + && self.entry.path.last() == Some(&b'/') + { + self.entry.set_filetype(S_IFDIR); + self.bytes_remaining = 0; + self.padding = 0; + } + + if self.is_npm() + && self.entry.filetype() == S_IFREG + && (!(plain_file || self.typeflag == b'7') || strings::contains_char(&self.pax_path, 0)) + { + self.entry.set_filetype(0); + } + + self.entry.warned = status != Status::Ok; + self.state = State::Emit; + Ok(()) + } +} + +impl Entry { + fn clear(&mut self) { + self.path.clear(); + self.link.clear(); + *self = Entry { + path: core::mem::take(&mut self.path), + link: core::mem::take(&mut self.link), + ..Entry::default() + }; + } + + fn set_filetype(&mut self, filetype: u32) { + self.mode = (self.mode & !S_IFMT) | filetype; + } +} + +/// The last component is `._` and at least one more character. +fn is_mac_metadata_name(path: &[u8]) -> bool { + let mut name = path; + let mut rest = path; + while let Some(slash) = strings::index_of_char_usize(rest, b'/') { + rest = rest.get(slash.saturating_add(1)..).unwrap_or_default(); + if !rest.is_empty() { + name = rest; + } + } + name.len() > 2 && name.starts_with(b"._") +} From b4898abbcaaf24566cbed056d869fa56afdaccba Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 20:27:53 +0000 Subject: [PATCH 03/33] bun_tar: take permissions from the header, never from ACL records libarchive marked the permissions as set while parsing an ACL record, before it had read the header's mode. An entry with an NFSv4 ACL came out with mode 000: Bun.Archive.extract() made the directory unusable and then failed to write into it, and files lost their exec bit. GNU tar and node-tar ignore these records. --- src/tar/reader.rs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/tar/reader.rs b/src/tar/reader.rs index 4d62d288454a..527dbbf4844a 100644 --- a/src/tar/reader.rs +++ b/src/tar/reader.rs @@ -58,7 +58,6 @@ pub struct Entry { link_is_utf8: bool, link_kind: LinkKind, mode: u32, - perm_is_set: bool, size: i64, mtime: Option, warned: bool, @@ -836,6 +835,8 @@ impl Reader { return Err(Error::RedundantHeader('A')); } self.format = Format::Pax; + // A Solaris ACL. Permissions come from the header's mode alone, as in GNU tar and + // node-tar, so neither this nor a `SCHILY.acl.*` record is read. self.begin_body(size, None) } b'g' => { @@ -1428,12 +1429,7 @@ impl Reader { fn header_common(&mut self) -> Result<()> { let h = self.main_header; // Truncated the way a 32-bit `mode_t` is. - let mode = header::mode(&h) as u32; - self.entry.mode = if self.entry.perm_is_set { - (mode & S_IFMT) | self.entry.perm() - } else { - mode - }; + self.entry.mode = header::mode(&h) as u32; if self.entry.mtime.is_none() { self.entry.mtime = Some(header::mtime(&h)); } From cc129c591241e6b45497e1ed4c8fd529c08d8633 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 20:37:38 +0000 Subject: [PATCH 04/33] bun_tar: gzip decoding and a decoder that stacks it under the tar reader The gzip container is parsed here: header flags, several members, whatever follows the last member, and the CRC-32 and size of each member. DEFLATE stays with zlib, through bun_zlib's streaming wrapper. --- Cargo.lock | 1 + src/tar/Cargo.toml | 1 + src/tar/decoder.rs | 248 ++++++++++++++++++++++++++++++++++++ src/tar/error.rs | 6 + src/tar/gzip.rs | 304 +++++++++++++++++++++++++++++++++++++++++++++ src/tar/lib.rs | 3 + src/tar/reader.rs | 67 ++++++---- src/zlib/lib.rs | 30 +++++ 8 files changed, 638 insertions(+), 22 deletions(-) create mode 100644 src/tar/decoder.rs create mode 100644 src/tar/gzip.rs diff --git a/Cargo.lock b/Cargo.lock index 21d0f7b4ad9a..879cb4988525 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1915,6 +1915,7 @@ name = "bun_tar" version = "0.0.0" dependencies = [ "bun_core", + "bun_zlib", "thiserror", ] diff --git a/src/tar/Cargo.toml b/src/tar/Cargo.toml index f64efb19cc5d..782a6d298fc2 100644 --- a/src/tar/Cargo.toml +++ b/src/tar/Cargo.toml @@ -12,3 +12,4 @@ workspace = true [dependencies] thiserror.workspace = true bun_core.workspace = true +bun_zlib.workspace = true diff --git a/src/tar/decoder.rs b/src/tar/decoder.rs new file mode 100644 index 000000000000..6f7162154d6a --- /dev/null +++ b/src/tar/decoder.rs @@ -0,0 +1,248 @@ +//! A tar reader with the gzip layers in front of it. + +use crate::error::{Error, Result}; +use crate::gzip::{self, Progress}; +use crate::reader::{Entry, Event, Options, Poll, Reader}; + +const BLOCK: usize = 64 * 1024; +/// More wrappings than this are not looked for. +const MAX_LAYERS: usize = 25; + +struct Layer { + gzip: gzip::Decoder, + out: Vec, + pos: usize, + done: bool, +} + +impl Layer { + fn new() -> Result { + let mut out = Vec::new(); + out.try_reserve_exact(BLOCK) + .map_err(|_| Error::OutOfMemory)?; + Ok(Self { + gzip: gzip::Decoder::new()?, + out, + pos: 0, + done: false, + }) + } + + fn unread(&self) -> &[u8] { + self.out.get(self.pos..).unwrap_or_default() + } +} + +/// Takes compressed or plain bytes as they come and gives tar events back. +pub struct Decoder { + /// The first one reads the caller's input, and each next one the output of the one before. + layers: Vec, + tar: Reader, +} + +impl Decoder { + /// `gzip` says whether the input is compressed. Nothing is detected, so this works however + /// little of the input is there yet. + pub fn new(options: Options, gzip: bool) -> Result { + let mut layers = Vec::new(); + if gzip { + layers.push(Layer::new()?); + } + Ok(Self { + layers, + tar: Reader::new(options), + }) + } + + pub fn entry(&self) -> &Entry { + self.tar.entry() + } + + /// See [`Reader::skip_entry`]. + pub fn skip_entry(&mut self) { + self.tar.skip_entry(); + } + + /// Refills layer `index`, whose output has all been read. Returns how much of `input` was + /// consumed, and whether the layer now has output or has ended. Otherwise it needs input. + fn refill(&mut self, index: usize, input: &[u8], eof: bool) -> Result<(usize, bool)> { + let mut used_total = 0usize; + loop { + let (below, rest) = self.layers.split_at_mut(index); + let Some(layer) = rest.first_mut() else { + return Ok((used_total, true)); + }; + layer.out.clear(); + layer.pos = 0; + let (source, source_eof) = match below.last() { + Some(lower) => (lower.unread(), lower.done), + None => (input.get(used_total..).unwrap_or_default(), eof), + }; + let (used, progress) = layer + .gzip + .decode(source, source_eof, &mut layer.out, BLOCK)?; + layer.done = progress == Progress::Done; + let ready = layer.done || !layer.out.is_empty(); + match below.last_mut() { + Some(lower) => lower.pos = lower.pos.saturating_add(used), + None => used_total = used_total.saturating_add(used), + } + if ready { + return Ok((used_total, true)); + } + // The layer wants more than its source has. + if index == 0 { + return Ok((used_total, false)); + } + let (used, ready) = self.refill( + index.saturating_sub(1), + input.get(used_total..).unwrap_or_default(), + eof, + )?; + used_total = used_total.saturating_add(used); + if !ready { + return Ok((used_total, false)); + } + } + } + + /// Runs every layer to its end, so that each one gets to check all of its input. An inner + /// layer stops at whatever follows its last member, which leaves the ones around it unfinished. + /// Returns how much of `input` was consumed, and `false` if more is needed. + fn drain(&mut self, input: &[u8], eof: bool) -> Result<(usize, bool)> { + let mut used_total = 0usize; + for index in (0..self.layers.len()).rev() { + while self.layers.get(index).is_some_and(|layer| !layer.done) { + let (used, ready) = + self.refill(index, input.get(used_total..).unwrap_or_default(), eof)?; + used_total = used_total.saturating_add(used); + if !ready { + return Ok((used_total, false)); + } + } + } + Ok((used_total, true)) + } + + /// Like [`Reader::next`], with `input` being what the outermost layer reads. + pub fn next<'a>(&'a mut self, input: &'a [u8], eof: bool) -> Result<(usize, Event<'a>)> { + let Some(top) = self.layers.len().checked_sub(1) else { + return self.tar.next(input, eof); + }; + let mut used_total = 0usize; + let poll = loop { + let Some(layer) = self.layers.get_mut(top) else { + break Poll::EndOfArchive; + }; + let base = layer.pos; + let (used, poll) = self.tar.poll(layer.unread(), layer.done)?; + layer.pos = layer.pos.saturating_add(used); + match poll { + Poll::NeedInput => {} + Poll::EndOfArchive => { + let (used, finished) = + self.drain(input.get(used_total..).unwrap_or_default(), eof)?; + used_total = used_total.saturating_add(used); + break if finished { + Poll::EndOfArchive + } else { + Poll::NeedInput + }; + } + Poll::Data { offset, start, len } => { + break Poll::Data { + offset, + start: base.saturating_add(start), + len, + }; + } + other => break other, + } + let (used, ready) = + self.refill(top, input.get(used_total..).unwrap_or_default(), eof)?; + used_total = used_total.saturating_add(used); + if !ready { + break Poll::NeedInput; + } + }; + let event = match poll { + Poll::NeedInput => Event::NeedInput, + Poll::Entry => Event::Entry(self.tar.entry()), + Poll::Data { offset, start, len } => Event::Data { + offset, + bytes: self + .layers + .get(top) + .and_then(|layer| layer.out.get(start..start.saturating_add(len))) + .unwrap_or_default(), + }, + Poll::EndOfEntry { size } => Event::EndOfEntry { size }, + Poll::EndOfArchive => Event::EndOfArchive, + }; + Ok((used_total, event)) + } +} + +/// Reads an archive that is all in memory, and works out by itself how it is wrapped. +pub struct MemoryDecoder<'a> { + data: &'a [u8], + pos: usize, + decoder: Decoder, + /// From looking at what a layer holds. It is reported in place of the first event. + failed: Option, +} + +impl<'a> MemoryDecoder<'a> { + pub fn new(data: &'a [u8], options: Options) -> Result { + let mut this = Self { + data, + pos: 0, + decoder: Decoder::new(options, false)?, + failed: None, + }; + while this.decoder.layers.len() < MAX_LAYERS { + let is_gzip = match this.decoder.layers.last() { + None => gzip::starts_with_header(data), + // One block is all that is looked at, which a header only outgrows with a name or + // a comment of that size. + Some(layer) => gzip::starts_with_header(layer.unread()), + }; + if !is_gzip { + break; + } + this.decoder.layers.push(Layer::new()?); + let top = this.decoder.layers.len().saturating_sub(1); + match this + .decoder + .refill(top, data.get(this.pos..).unwrap_or_default(), true) + { + Ok((used, _)) => this.pos = this.pos.saturating_add(used), + Err(error) => { + this.failed = Some(error); + break; + } + } + } + Ok(this) + } + + pub fn entry(&self) -> &Entry { + self.decoder.entry() + } + + pub fn skip_entry(&mut self) { + self.decoder.skip_entry(); + } + + /// Never returns [`Event::NeedInput`]. + pub fn next(&mut self) -> Result> { + if let Some(error) = self.failed { + return Err(error); + } + let (used, event) = self + .decoder + .next(self.data.get(self.pos..).unwrap_or_default(), true)?; + self.pos = self.pos.saturating_add(used); + Ok(event) + } +} diff --git a/src/tar/error.rs b/src/tar/error.rs index bd9c179945b3..e66cc5513e90 100644 --- a/src/tar/error.rs +++ b/src/tar/error.rs @@ -36,6 +36,12 @@ pub enum Error { SparseMapTooLarge, #[error("Oversized AppleDouble extension")] OversizedMacMetadata, + #[error("truncated gzip input")] + TruncatedGzip, + #[error("gzip decompression failed")] + GzipDecompressionFailed, + #[error("gzip trailer mismatch: bad CRC32 or uncompressed size")] + GzipTrailerMismatch, #[error("Out of memory")] OutOfMemory, } diff --git a/src/tar/gzip.rs b/src/tar/gzip.rs new file mode 100644 index 000000000000..b2b95b7cbd0f --- /dev/null +++ b/src/tar/gzip.rs @@ -0,0 +1,304 @@ +//! The gzip container (RFC 1952). DEFLATE itself is zlib's. + +use crate::error::{Error, Result}; +use bun_core::strings; +use bun_zlib::{InflateDecoder, ReturnCode, crc32_bytes}; + +mod flag { + pub(super) const HEADER_CRC: u8 = 2; + pub(super) const EXTRA: u8 = 4; + pub(super) const NAME: u8 = 8; + pub(super) const COMMENT: u8 = 16; + pub(super) const RESERVED: u8 = 0xe0; +} + +const RAW_DEFLATE: core::ffi::c_int = -15; + +fn fixed_header_flags(fixed: &[u8; 10]) -> Option { + let flags = fixed[3]; + (fixed.starts_with(&[0x1f, 0x8b, 8]) && flags & flag::RESERVED == 0).then_some(flags) +} + +/// Whether `data`, which is all there is, starts with a whole gzip header. +pub fn starts_with_header(data: &[u8]) -> bool { + let Some((fixed, mut rest)) = data.split_first_chunk::<10>() else { + return false; + }; + let Some(flags) = fixed_header_flags(fixed) else { + return false; + }; + if flags & flag::EXTRA != 0 { + let Some((len, tail)) = rest.split_first_chunk::<2>() else { + return false; + }; + // That the field itself is all there only matters if something is read behind it. + rest = tail + .get(usize::from(u16::from_le_bytes(*len))..) + .unwrap_or_default(); + if rest.is_empty() && flags & (flag::NAME | flag::COMMENT | flag::HEADER_CRC) == 0 { + return true; + } + } + for text in [flag::NAME, flag::COMMENT] { + if flags & text != 0 { + let Some(end) = strings::index_of_char_usize(rest, 0) else { + return false; + }; + rest = rest.get(end.saturating_add(1)..).unwrap_or_default(); + } + } + flags & flag::HEADER_CRC == 0 || rest.len() >= 2 +} + +/// In the order they appear in. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +enum Field { + Extra, + Name, + Comment, + HeaderCrc, + Body, +} + +#[derive(Debug, Clone, Copy)] +enum State { + Fixed, + ExtraLength, + Extra(u16), + Name, + Comment, + HeaderCrc, + Body, + Trailer, + Done, + Failed(Error), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Progress { + /// All of the input was consumed and more is wanted. + NeedInput, + /// `out` has reached `limit`. + OutputFull, + /// The last member has ended. What follows it, if anything, is not gzip and is left alone. + Done, +} + +/// Decompresses a sequence of gzip members, checking the CRC-32 and the size of each. +pub struct Decoder { + state: State, + inflate: InflateDecoder, + flags: u8, + small: [u8; 10], + small_len: usize, + crc: u32, + /// Modulo 2^32, like the field it is compared with. + size: u32, +} + +impl Decoder { + pub fn new() -> Result { + Ok(Self { + state: State::Fixed, + inflate: InflateDecoder::new(RAW_DEFLATE).map_err(|_| Error::OutOfMemory)?, + flags: 0, + small: [0; 10], + small_len: 0, + crc: 0, + size: 0, + }) + } + + /// Gathers `want` bytes in `small`. `false` when `input` ran out first. + fn gather(&mut self, want: usize, input: &[u8], used: &mut usize) -> bool { + let take = want.saturating_sub(self.small_len).min(input.len()); + if let (Some(dest), Some(src)) = ( + self.small + .get_mut(self.small_len..self.small_len.saturating_add(take)), + input.get(..take), + ) { + dest.copy_from_slice(src); + } + self.small_len = self.small_len.saturating_add(take); + *used = take; + if self.small_len < want { + return false; + } + self.small_len = 0; + true + } + + /// Moves to the first optional field at or behind `from` that the flags announce. + fn next_header_field(&mut self, from: Field) { + let has = |flag: u8| self.flags & flag != 0; + self.state = if from <= Field::Extra && has(flag::EXTRA) { + State::ExtraLength + } else if from <= Field::Name && has(flag::NAME) { + State::Name + } else if from <= Field::Comment && has(flag::COMMENT) { + State::Comment + } else if from <= Field::HeaderCrc && has(flag::HEADER_CRC) { + State::HeaderCrc + } else { + self.crc = 0; + self.size = 0; + State::Body + }; + } + + /// Appends to `out` until it holds `limit` bytes. Returns how much of `input` was consumed. + /// What was decompressed before an error is handed over first, and the error comes from the + /// next call. From then on every call returns it. + pub fn decode( + &mut self, + input: &[u8], + eof: bool, + out: &mut Vec, + limit: usize, + ) -> Result<(usize, Progress)> { + let before = out.len(); + let mut pos = 0usize; + loop { + let rest = input.get(pos..).unwrap_or_default(); + let mut used = 0usize; + let step = self.step(rest, eof, out, limit, &mut used); + pos = pos.saturating_add(used); + match step { + Ok(None) => {} + Ok(Some(progress)) => return Ok((pos, progress)), + Err(error) => { + self.state = State::Failed(error); + if out.len() == before { + return Err(error); + } + return Ok((pos, Progress::OutputFull)); + } + } + } + } + + fn step( + &mut self, + input: &[u8], + eof: bool, + out: &mut Vec, + limit: usize, + used: &mut usize, + ) -> Result> { + // A header that the input ends in the middle of is not a header, so the stream ended with + // the member before it. + let starved = |this: &mut Self| { + if eof { + this.state = State::Done; + Some(Progress::Done) + } else { + Some(Progress::NeedInput) + } + }; + match self.state { + State::Failed(error) => Err(error), + State::Done => Ok(Some(Progress::Done)), + + State::Fixed => { + if !self.gather(10, input, used) { + return Ok(starved(self)); + } + let Some(flags) = fixed_header_flags(&self.small) else { + self.state = State::Done; + return Ok(Some(Progress::Done)); + }; + self.flags = flags; + self.next_header_field(Field::Extra); + Ok(None) + } + State::ExtraLength => { + if !self.gather(2, input, used) { + return Ok(starved(self)); + } + self.state = State::Extra(u16::from_le_bytes([self.small[0], self.small[1]])); + Ok(None) + } + State::Extra(left) => { + let take = usize::from(left).min(input.len()); + *used = take; + let left = left.saturating_sub(u16::try_from(take).unwrap_or(u16::MAX)); + if left > 0 { + self.state = State::Extra(left); + if eof && self.flags & (flag::NAME | flag::COMMENT | flag::HEADER_CRC) == 0 { + return Err(Error::TruncatedGzip); + } + return Ok(starved(self)); + } + self.next_header_field(Field::Name); + Ok(None) + } + State::Name | State::Comment => { + let Some(end) = strings::index_of_char_usize(input, 0) else { + *used = input.len(); + return Ok(starved(self)); + }; + *used = end.saturating_add(1); + self.next_header_field(if matches!(self.state, State::Name) { + Field::Comment + } else { + Field::HeaderCrc + }); + Ok(None) + } + State::HeaderCrc => { + if !self.gather(2, input, used) { + return Ok(starved(self)); + } + self.next_header_field(Field::Body); + Ok(None) + } + + State::Body => { + let room = limit.saturating_sub(out.len()); + if room == 0 { + return Ok(Some(Progress::OutputFull)); + } + if input.is_empty() { + if eof { + return Err(Error::TruncatedGzip); + } + return Ok(Some(Progress::NeedInput)); + } + let start = out.len(); + let (consumed, code) = self.inflate.step_limited(input, out, room); + *used = consumed; + let produced = out.get(start..).unwrap_or_default(); + self.crc = crc32_bytes(self.crc, produced); + self.size = self.size.wrapping_add(produced.len() as u32); + match code { + ReturnCode::Ok => Ok(None), + ReturnCode::StreamEnd => { + if self.inflate.reset() != ReturnCode::Ok { + return Err(Error::GzipDecompressionFailed); + } + self.state = State::Trailer; + Ok(None) + } + ReturnCode::MemError => Err(Error::OutOfMemory), + _ => Err(Error::GzipDecompressionFailed), + } + } + State::Trailer => { + if !self.gather(8, input, used) { + if eof { + return Err(Error::TruncatedGzip); + } + return Ok(Some(Progress::NeedInput)); + } + let [c0, c1, c2, c3, s0, s1, s2, s3, ..] = self.small; + if u32::from_le_bytes([c0, c1, c2, c3]) != self.crc + || u32::from_le_bytes([s0, s1, s2, s3]) != self.size + { + return Err(Error::GzipTrailerMismatch); + } + self.state = State::Fixed; + Ok(None) + } + } + } +} diff --git a/src/tar/lib.rs b/src/tar/lib.rs index f3ea5e7422d3..fddbf2d9068f 100644 --- a/src/tar/lib.rs +++ b/src/tar/lib.rs @@ -6,11 +6,14 @@ #![forbid(unsafe_code)] #![warn(unused_must_use)] +pub mod decoder; pub mod error; +pub mod gzip; mod header; mod num; pub mod reader; +pub use decoder::{Decoder, MemoryDecoder}; pub use error::{Error, Result}; pub use reader::{Charset, Dialect, Entry, Event, Options, Reader}; diff --git a/src/tar/reader.rs b/src/tar/reader.rs index 527dbbf4844a..23c5347e6111 100644 --- a/src/tar/reader.rs +++ b/src/tar/reader.rs @@ -276,6 +276,17 @@ enum State { enum Step { Continue, + NeedInput, + Entry, + Data { offset: i64, len: usize }, + EndOfEntry, + EndOfArchive, +} + +/// [`Event`] without the borrows. `Data` is `input[start..start + len]`, and the entry is +/// [`Reader::entry`]. +#[derive(Debug, Clone, Copy)] +pub(crate) enum Poll { NeedInput, Entry, Data { @@ -283,7 +294,9 @@ enum Step { start: usize, len: usize, }, - EndOfEntry, + EndOfEntry { + size: i64, + }, EndOfArchive, } @@ -416,6 +429,28 @@ impl Reader { /// Returns how much of `input` was consumed along with the event. `eof` says that nothing /// follows `input`. An error is final: every later call returns it again. pub fn next<'a>(&'a mut self, input: &'a [u8], eof: bool) -> Result<(usize, Event<'a>)> { + let (used, poll) = self.poll(input, eof)?; + let event = match poll { + Poll::NeedInput => Event::NeedInput, + Poll::Entry => Event::Entry(&self.entry), + Poll::Data { offset, start, len } => Event::Data { + offset, + bytes: input + .get(start..start.saturating_add(len)) + .unwrap_or_default(), + }, + Poll::EndOfEntry { size } => Event::EndOfEntry { size }, + Poll::EndOfArchive => Event::EndOfArchive, + }; + Ok((used, event)) + } + + /// The entry of the last [`Event::Entry`]. + pub fn entry(&self) -> &Entry { + &self.entry + } + + pub(crate) fn poll(&mut self, input: &[u8], eof: bool) -> Result<(usize, Poll)> { let mut pos = 0usize; loop { let rest = input.get(pos..).unwrap_or_default(); @@ -427,27 +462,19 @@ impl Reader { return Err(error); } }; - let start_of_rest = pos; + let start = pos; pos = pos.saturating_add(used); - let event = match step { + let poll = match step { Step::Continue => continue, - Step::NeedInput => Event::NeedInput, - Step::Entry => Event::Entry(&self.entry), - Step::Data { offset, start, len } => { - let from = start_of_rest.saturating_add(start); - Event::Data { - offset, - bytes: input - .get(from..from.saturating_add(len)) - .unwrap_or_default(), - } - } - Step::EndOfEntry => Event::EndOfEntry { + Step::NeedInput => Poll::NeedInput, + Step::Entry => Poll::Entry, + Step::Data { offset, len } => Poll::Data { offset, start, len }, + Step::EndOfEntry => Poll::EndOfEntry { size: self.disk_size, }, - Step::EndOfArchive => Event::EndOfArchive, + Step::EndOfArchive => Poll::EndOfArchive, }; - return Ok((pos, event)); + return Ok((pos, poll)); } } @@ -729,11 +756,7 @@ impl Reader { if block.hole { return Ok(Step::Continue); } - Ok(Step::Data { - offset, - start: 0, - len, - }) + Ok(Step::Data { offset, len }) } State::Drain => { diff --git a/src/zlib/lib.rs b/src/zlib/lib.rs index eb2465dd5767..b2dcc8ec3905 100644 --- a/src/zlib/lib.rs +++ b/src/zlib/lib.rs @@ -914,6 +914,18 @@ impl DeflateEncoder { deflate, ) } + + /// Like [`step`](Self::step), but `deflate()` sees exactly `limit` bytes of output space, + /// whatever `out`'s capacity is. At level 0 that space decides where a stored block ends. + pub fn step_limited( + &mut self, + input: &[u8], + out: &mut Vec, + limit: usize, + flush: FlushValue, + ) -> (usize, ReturnCode) { + step(&mut self.strm, input, out, limit, limit, flush, deflate) + } } impl Drop for DeflateEncoder { @@ -1002,6 +1014,24 @@ impl InflateDecoder { ) } + /// Like [`step`](Self::step), but appends at most `limit` bytes to `out`. + pub fn step_limited( + &mut self, + input: &[u8], + out: &mut Vec, + limit: usize, + ) -> (usize, ReturnCode) { + step( + &mut self.strm, + input, + out, + limit, + limit, + FlushValue::NoFlush, + inflate, + ) + } + /// Append decompressed output to `out` (growing by 4096-byte steps, capped at /// `max_output_size`) until `input` is consumed or `out.len()` reaches `max_output`. /// Returns the input bytes consumed. Returns `ShortRead` when more input is required and From f6553402b7b52fc15ebc30859f9d2c34674c4f59 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 20:44:50 +0000 Subject: [PATCH 05/33] bun_tar: writer, gzip encoder and the name composition Windows applies The writer emits restricted pax for regular files, and the encoder drives deflate with the same calls and the same output window as before, so that `bun pm pack` and Bun.Archive keep producing the bytes they produce today. On Windows a UTF-8 name has its characters composed on the way to UTF-16. That is not NFC: nothing is decomposed first, and the data is Unicode 6.0. --- src/tar/compose.rs | 304 +++++++++ src/tar/compose/tables.rs | 1228 +++++++++++++++++++++++++++++++++++++ src/tar/error.rs | 2 + src/tar/gzip.rs | 87 +++ src/tar/lib.rs | 2 + src/tar/reader.rs | 7 +- src/tar/writer.rs | 463 ++++++++++++++ 7 files changed, 2092 insertions(+), 1 deletion(-) create mode 100644 src/tar/compose.rs create mode 100644 src/tar/compose/tables.rs create mode 100644 src/tar/writer.rs diff --git a/src/tar/compose.rs b/src/tar/compose.rs new file mode 100644 index 000000000000..4fbd3d081c7c --- /dev/null +++ b/src/tar/compose.rs @@ -0,0 +1,304 @@ +//! What Windows does to a UTF-8 name on its way to UTF-16: adjacent characters that have a +//! composed form are replaced by it. +//! +//! This is not NFC. Nothing is decomposed first, so U+212B ANGSTROM SIGN stays, and the data is +//! that of Unicode 6.0. File names that exist on disk were made this way, so neither may change. + +mod tables; + +use crate::error::{Error, Result}; +use bun_core::strings; +use tables::{COMBINING_CLASSES, COMPOSING_BLOCKS, COMPOSITIONS}; + +/// More combining marks than this on one character make the name count as unconvertible. +const MAX_MARKS: usize = 10; + +mod hangul { + pub(super) const S_BASE: u32 = 0xAC00; + pub(super) const L_BASE: u32 = 0x1100; + pub(super) const V_BASE: u32 = 0x1161; + pub(super) const T_BASE: u32 = 0x11A7; + pub(super) const L_COUNT: u32 = 19; + pub(super) const V_COUNT: u32 = 21; + pub(super) const T_COUNT: u32 = 28; + pub(super) const S_COUNT: u32 = L_COUNT * V_COUNT * T_COUNT; +} + +fn compose_pair(first: u32, second: u32) -> Option { + let index = COMPOSITIONS + .binary_search_by_key(&(first, second), |&(a, b, _)| (a, b)) + .ok()?; + COMPOSITIONS.get(index).map(|&(_, _, composed)| composed) +} + +fn in_ranges(ranges: &[T], value: u32, bounds: impl Fn(&T) -> (u32, u32)) -> Option<&T> { + let index = ranges.partition_point(|range| bounds(range).1 < value); + ranges.get(index).filter(|range| bounds(range).0 <= value) +} + +fn combining_class(c: u32) -> u8 { + in_ranges(&COMBINING_CLASSES, c, |&(first, last, _)| (first, last)) + .map_or(0, |&(_, _, class)| class) +} + +fn may_compose(c: u32) -> bool { + in_ranges(&COMPOSING_BLOCKS, c >> 8, |&range| range).is_some() +} + +/// UTF-8, except that a surrogate pair written as two three-byte sequences (CESU-8) is also taken. +/// `None` for anything else. +fn decode(bytes: &[u8]) -> Result>> { + let mut out = Vec::new(); + out.try_reserve(bytes.len()) + .map_err(|_| Error::OutOfMemory)?; + let mut rest = bytes; + let continuation = |b: u8| (b & 0xc0 == 0x80).then_some(u32::from(b & 0x3f)); + let three = |a: u8, b: u8, c: u8| { + let value = (u32::from(a & 0x0f) << 12) | (continuation(b)? << 6) | continuation(c)?; + (value >= 0x800).then_some(value) + }; + while let Some((&first, tail)) = rest.split_first() { + let (value, tail) = match (first, tail) { + (0..=0x7f, tail) => (u32::from(first), tail), + (0xc2..=0xdf, &[b, ref tail @ ..]) => { + let Some(b) = continuation(b) else { + return Ok(None); + }; + ((u32::from(first & 0x1f) << 6) | b, tail) + } + (0xe0..=0xef, &[b, c, ref tail @ ..]) => match three(first, b, c) { + Some(high @ 0xd800..=0xdbff) => { + let &[d @ 0xe0..=0xef, e, f, ref tail @ ..] = tail else { + return Ok(None); + }; + let Some(low @ 0xdc00..=0xdfff) = three(d, e, f) else { + return Ok(None); + }; + (0x10000 + ((high - 0xd800) << 10) + (low - 0xdc00), tail) + } + Some(0xdc00..=0xdfff) | None => return Ok(None), + Some(value) => (value, tail), + }, + (0xf0..=0xf4, &[b, c, d, ref tail @ ..]) => { + let (Some(b), Some(c), Some(d)) = + (continuation(b), continuation(c), continuation(d)) + else { + return Ok(None); + }; + let value = (u32::from(first & 0x07) << 18) | (b << 12) | (c << 6) | d; + if !(0x10000..=0x10ffff).contains(&value) { + return Ok(None); + } + (value, tail) + } + _ => return Ok(None), + }; + out.push(value); + rest = tail; + } + Ok(Some(out)) +} + +/// A run of combining marks behind one character, each of a higher class than the one before. +struct Marks { + marks: [u32; MAX_MARKS], + count: usize, + last_class: u8, + /// The class of the character that ended the run, unless the input did. + stopper: Option, +} + +impl Marks { + /// Takes marks from `chars[*read..]` for as long as the classes keep rising. `None` past + /// [`MAX_MARKS`]. + fn collect(&mut self, chars: &[u32], read: &mut usize) -> Option<()> { + loop { + let slot = self.marks.get_mut(self.count)?; + let Some(&c) = chars.get(*read) else { + self.stopper = None; + return Some(()); + }; + let class = combining_class(c); + if self.last_class >= class && self.last_class != 228 && class != 228 { + self.stopper = Some(class); + return Some(()); + } + *slot = c; + *read = read.saturating_add(1); + self.last_class = class; + self.count = self.count.saturating_add(1); + } + } + + fn remove(&mut self, index: usize) { + self.marks + .copy_within(index.saturating_add(1)..self.count, index); + self.count = self.count.saturating_sub(1); + } + + fn as_slice(&self) -> &[u32] { + self.marks.get(..self.count).unwrap_or_default() + } +} + +/// `None` past [`MAX_MARKS`]. `out` has room for all of `chars`. +fn compose_chars(chars: &[u32], out: &mut Vec) -> Option<()> { + let mut read = 0usize; + 'starter: while let Some(&first) = chars.get(read) { + let mut current = first; + read = read.saturating_add(1); + loop { + let Some(&next) = chars.get(read) else { + out.push(current); + break 'starter; + }; + read = read.saturating_add(1); + + if !may_compose(next) { + out.push(current); + current = next; + continue; + } + let l_index = current.wrapping_sub(hangul::L_BASE); + let s_index = current.wrapping_sub(hangul::S_BASE); + if l_index < hangul::L_COUNT { + let v_index = next.wrapping_sub(hangul::V_BASE); + if v_index < hangul::V_COUNT { + current = + hangul::S_BASE + (l_index * hangul::V_COUNT + v_index) * hangul::T_COUNT; + } else { + out.push(current); + current = next; + } + continue; + } + if s_index < hangul::S_COUNT && s_index % hangul::T_COUNT == 0 { + let t_index = next.wrapping_sub(hangul::T_BASE); + if 0 < t_index && t_index < hangul::T_COUNT { + current += t_index; + } else { + out.push(current); + current = next; + } + continue; + } + if let Some(composed) = compose_pair(current, next) { + current = composed; + continue; + } + let class = combining_class(next); + if class == 0 { + out.push(current); + current = next; + continue; + } + + // `next` is a mark that does not combine with `current`. One behind it still may, as + // long as no mark in between has the same class or a higher one. + let mut marks = Marks { + marks: [next; MAX_MARKS], + count: 1, + last_class: class, + stopper: None, + }; + marks.collect(chars, &mut read)?; + let mut i = 1usize; + while let Some(&mark) = marks.as_slice().get(i) { + let Some(composed) = compose_pair(current, mark) else { + i = i.saturating_add(1); + continue; + }; + current = composed; + marks.remove(i); + // The mark that is gone may have been what kept the next one out. + if marks.count > 0 && i == marks.count && marks.stopper == Some(marks.last_class) { + marks.last_class = marks + .as_slice() + .last() + .map_or(0, |&mark| combining_class(mark)); + marks.collect(chars, &mut read)?; + } + i = 0; + } + + out.push(current); + out.extend_from_slice(marks.as_slice()); + if marks.stopper == Some(marks.last_class) { + let mut last_class = marks.last_class; + while let Some(&c) = chars.get(read) { + let class = combining_class(c); + if last_class > class { + break; + } + read = read.saturating_add(1); + last_class = class; + out.push(c); + } + } + continue 'starter; + } + } + Some(()) +} + +/// The characters of `name`, composed. `None` if it does not convert, in which case Windows takes +/// each byte of it for one UTF-16 unit. +pub fn compose(name: &[u8]) -> Result>> { + let Some(chars) = decode(name)? else { + return Ok(None); + }; + let mut out = Vec::new(); + out.try_reserve(chars.len()) + .map_err(|_| Error::OutOfMemory)?; + Ok(compose_chars(&chars, &mut out).map(|()| out)) +} + +/// Whether [`compose`] would return something. +pub fn converts(name: &[u8]) -> bool { + strings::first_non_ascii(name).is_none() || matches!(compose(name), Ok(Some(_))) +} + +fn push_utf8(out: &mut Vec, c: u32) { + let c = char::from_u32(c).unwrap_or(char::REPLACEMENT_CHARACTER); + out.extend_from_slice(c.encode_utf8(&mut [0; 4]).as_bytes()); +} + +/// The name an entry gets when it is written on Windows, or `None` if `name` does not convert. +pub(crate) fn windows_name(name: &[u8]) -> Result>> { + let mut out = Vec::new(); + out.try_reserve(name.len()) + .map_err(|_| Error::OutOfMemory)?; + if strings::first_non_ascii(name).is_none() { + out.extend_from_slice(name); + } else { + let Some(chars) = compose(name)? else { + return Ok(None); + }; + for c in chars { + push_utf8(&mut out, c); + } + } + for b in &mut out { + if *b == b'\\' { + *b = b'/'; + } + } + Ok(Some(out)) +} + +/// Appends `name` as Windows sees it. `is_utf8` is [`crate::Entry::path_is_utf8`]. +pub fn to_utf16(name: &[u8], is_utf8: bool, out: &mut Vec) -> Result<()> { + out.try_reserve(name.len()) + .map_err(|_| Error::OutOfMemory)?; + if is_utf8 && strings::first_non_ascii(name).is_some() { + if let Some(chars) = compose(name)? { + for c in chars { + let c = char::from_u32(c).unwrap_or(char::REPLACEMENT_CHARACTER); + out.extend_from_slice(c.encode_utf16(&mut [0; 2])); + } + return Ok(()); + } + } + out.extend(name.iter().map(|&b| u16::from(b))); + Ok(()) +} diff --git a/src/tar/compose/tables.rs b/src/tar/compose/tables.rs new file mode 100644 index 000000000000..325a78e5eb62 --- /dev/null +++ b/src/tar/compose/tables.rs @@ -0,0 +1,1228 @@ +//! Unicode 6.0 data for [`super`]. Generated: each table is what its comment says, taken from the +//! Unicode Character Database. + +/// `(first, second, composed)` for every character whose canonical decomposition is a pair and +/// that is not a composition exclusion, sorted. Hangul syllables are computed instead. +pub(super) static COMPOSITIONS: [(u32, u32, u32); 931] = [ + (0x003C, 0x0338, 0x226E), + (0x003D, 0x0338, 0x2260), + (0x003E, 0x0338, 0x226F), + (0x0041, 0x0300, 0x00C0), + (0x0041, 0x0301, 0x00C1), + (0x0041, 0x0302, 0x00C2), + (0x0041, 0x0303, 0x00C3), + (0x0041, 0x0304, 0x0100), + (0x0041, 0x0306, 0x0102), + (0x0041, 0x0307, 0x0226), + (0x0041, 0x0308, 0x00C4), + (0x0041, 0x0309, 0x1EA2), + (0x0041, 0x030A, 0x00C5), + (0x0041, 0x030C, 0x01CD), + (0x0041, 0x030F, 0x0200), + (0x0041, 0x0311, 0x0202), + (0x0041, 0x0323, 0x1EA0), + (0x0041, 0x0325, 0x1E00), + (0x0041, 0x0328, 0x0104), + (0x0042, 0x0307, 0x1E02), + (0x0042, 0x0323, 0x1E04), + (0x0042, 0x0331, 0x1E06), + (0x0043, 0x0301, 0x0106), + (0x0043, 0x0302, 0x0108), + (0x0043, 0x0307, 0x010A), + (0x0043, 0x030C, 0x010C), + (0x0043, 0x0327, 0x00C7), + (0x0044, 0x0307, 0x1E0A), + (0x0044, 0x030C, 0x010E), + (0x0044, 0x0323, 0x1E0C), + (0x0044, 0x0327, 0x1E10), + (0x0044, 0x032D, 0x1E12), + (0x0044, 0x0331, 0x1E0E), + (0x0045, 0x0300, 0x00C8), + (0x0045, 0x0301, 0x00C9), + (0x0045, 0x0302, 0x00CA), + (0x0045, 0x0303, 0x1EBC), + (0x0045, 0x0304, 0x0112), + (0x0045, 0x0306, 0x0114), + (0x0045, 0x0307, 0x0116), + (0x0045, 0x0308, 0x00CB), + (0x0045, 0x0309, 0x1EBA), + (0x0045, 0x030C, 0x011A), + (0x0045, 0x030F, 0x0204), + (0x0045, 0x0311, 0x0206), + (0x0045, 0x0323, 0x1EB8), + (0x0045, 0x0327, 0x0228), + (0x0045, 0x0328, 0x0118), + (0x0045, 0x032D, 0x1E18), + (0x0045, 0x0330, 0x1E1A), + (0x0046, 0x0307, 0x1E1E), + (0x0047, 0x0301, 0x01F4), + (0x0047, 0x0302, 0x011C), + (0x0047, 0x0304, 0x1E20), + (0x0047, 0x0306, 0x011E), + (0x0047, 0x0307, 0x0120), + (0x0047, 0x030C, 0x01E6), + (0x0047, 0x0327, 0x0122), + (0x0048, 0x0302, 0x0124), + (0x0048, 0x0307, 0x1E22), + (0x0048, 0x0308, 0x1E26), + (0x0048, 0x030C, 0x021E), + (0x0048, 0x0323, 0x1E24), + (0x0048, 0x0327, 0x1E28), + (0x0048, 0x032E, 0x1E2A), + (0x0049, 0x0300, 0x00CC), + (0x0049, 0x0301, 0x00CD), + (0x0049, 0x0302, 0x00CE), + (0x0049, 0x0303, 0x0128), + (0x0049, 0x0304, 0x012A), + (0x0049, 0x0306, 0x012C), + (0x0049, 0x0307, 0x0130), + (0x0049, 0x0308, 0x00CF), + (0x0049, 0x0309, 0x1EC8), + (0x0049, 0x030C, 0x01CF), + (0x0049, 0x030F, 0x0208), + (0x0049, 0x0311, 0x020A), + (0x0049, 0x0323, 0x1ECA), + (0x0049, 0x0328, 0x012E), + (0x0049, 0x0330, 0x1E2C), + (0x004A, 0x0302, 0x0134), + (0x004B, 0x0301, 0x1E30), + (0x004B, 0x030C, 0x01E8), + (0x004B, 0x0323, 0x1E32), + (0x004B, 0x0327, 0x0136), + (0x004B, 0x0331, 0x1E34), + (0x004C, 0x0301, 0x0139), + (0x004C, 0x030C, 0x013D), + (0x004C, 0x0323, 0x1E36), + (0x004C, 0x0327, 0x013B), + (0x004C, 0x032D, 0x1E3C), + (0x004C, 0x0331, 0x1E3A), + (0x004D, 0x0301, 0x1E3E), + (0x004D, 0x0307, 0x1E40), + (0x004D, 0x0323, 0x1E42), + (0x004E, 0x0300, 0x01F8), + (0x004E, 0x0301, 0x0143), + (0x004E, 0x0303, 0x00D1), + (0x004E, 0x0307, 0x1E44), + (0x004E, 0x030C, 0x0147), + (0x004E, 0x0323, 0x1E46), + (0x004E, 0x0327, 0x0145), + (0x004E, 0x032D, 0x1E4A), + (0x004E, 0x0331, 0x1E48), + (0x004F, 0x0300, 0x00D2), + (0x004F, 0x0301, 0x00D3), + (0x004F, 0x0302, 0x00D4), + (0x004F, 0x0303, 0x00D5), + (0x004F, 0x0304, 0x014C), + (0x004F, 0x0306, 0x014E), + (0x004F, 0x0307, 0x022E), + (0x004F, 0x0308, 0x00D6), + (0x004F, 0x0309, 0x1ECE), + (0x004F, 0x030B, 0x0150), + (0x004F, 0x030C, 0x01D1), + (0x004F, 0x030F, 0x020C), + (0x004F, 0x0311, 0x020E), + (0x004F, 0x031B, 0x01A0), + (0x004F, 0x0323, 0x1ECC), + (0x004F, 0x0328, 0x01EA), + (0x0050, 0x0301, 0x1E54), + (0x0050, 0x0307, 0x1E56), + (0x0052, 0x0301, 0x0154), + (0x0052, 0x0307, 0x1E58), + (0x0052, 0x030C, 0x0158), + (0x0052, 0x030F, 0x0210), + (0x0052, 0x0311, 0x0212), + (0x0052, 0x0323, 0x1E5A), + (0x0052, 0x0327, 0x0156), + (0x0052, 0x0331, 0x1E5E), + (0x0053, 0x0301, 0x015A), + (0x0053, 0x0302, 0x015C), + (0x0053, 0x0307, 0x1E60), + (0x0053, 0x030C, 0x0160), + (0x0053, 0x0323, 0x1E62), + (0x0053, 0x0326, 0x0218), + (0x0053, 0x0327, 0x015E), + (0x0054, 0x0307, 0x1E6A), + (0x0054, 0x030C, 0x0164), + (0x0054, 0x0323, 0x1E6C), + (0x0054, 0x0326, 0x021A), + (0x0054, 0x0327, 0x0162), + (0x0054, 0x032D, 0x1E70), + (0x0054, 0x0331, 0x1E6E), + (0x0055, 0x0300, 0x00D9), + (0x0055, 0x0301, 0x00DA), + (0x0055, 0x0302, 0x00DB), + (0x0055, 0x0303, 0x0168), + (0x0055, 0x0304, 0x016A), + (0x0055, 0x0306, 0x016C), + (0x0055, 0x0308, 0x00DC), + (0x0055, 0x0309, 0x1EE6), + (0x0055, 0x030A, 0x016E), + (0x0055, 0x030B, 0x0170), + (0x0055, 0x030C, 0x01D3), + (0x0055, 0x030F, 0x0214), + (0x0055, 0x0311, 0x0216), + (0x0055, 0x031B, 0x01AF), + (0x0055, 0x0323, 0x1EE4), + (0x0055, 0x0324, 0x1E72), + (0x0055, 0x0328, 0x0172), + (0x0055, 0x032D, 0x1E76), + (0x0055, 0x0330, 0x1E74), + (0x0056, 0x0303, 0x1E7C), + (0x0056, 0x0323, 0x1E7E), + (0x0057, 0x0300, 0x1E80), + (0x0057, 0x0301, 0x1E82), + (0x0057, 0x0302, 0x0174), + (0x0057, 0x0307, 0x1E86), + (0x0057, 0x0308, 0x1E84), + (0x0057, 0x0323, 0x1E88), + (0x0058, 0x0307, 0x1E8A), + (0x0058, 0x0308, 0x1E8C), + (0x0059, 0x0300, 0x1EF2), + (0x0059, 0x0301, 0x00DD), + (0x0059, 0x0302, 0x0176), + (0x0059, 0x0303, 0x1EF8), + (0x0059, 0x0304, 0x0232), + (0x0059, 0x0307, 0x1E8E), + (0x0059, 0x0308, 0x0178), + (0x0059, 0x0309, 0x1EF6), + (0x0059, 0x0323, 0x1EF4), + (0x005A, 0x0301, 0x0179), + (0x005A, 0x0302, 0x1E90), + (0x005A, 0x0307, 0x017B), + (0x005A, 0x030C, 0x017D), + (0x005A, 0x0323, 0x1E92), + (0x005A, 0x0331, 0x1E94), + (0x0061, 0x0300, 0x00E0), + (0x0061, 0x0301, 0x00E1), + (0x0061, 0x0302, 0x00E2), + (0x0061, 0x0303, 0x00E3), + (0x0061, 0x0304, 0x0101), + (0x0061, 0x0306, 0x0103), + (0x0061, 0x0307, 0x0227), + (0x0061, 0x0308, 0x00E4), + (0x0061, 0x0309, 0x1EA3), + (0x0061, 0x030A, 0x00E5), + (0x0061, 0x030C, 0x01CE), + (0x0061, 0x030F, 0x0201), + (0x0061, 0x0311, 0x0203), + (0x0061, 0x0323, 0x1EA1), + (0x0061, 0x0325, 0x1E01), + (0x0061, 0x0328, 0x0105), + (0x0062, 0x0307, 0x1E03), + (0x0062, 0x0323, 0x1E05), + (0x0062, 0x0331, 0x1E07), + (0x0063, 0x0301, 0x0107), + (0x0063, 0x0302, 0x0109), + (0x0063, 0x0307, 0x010B), + (0x0063, 0x030C, 0x010D), + (0x0063, 0x0327, 0x00E7), + (0x0064, 0x0307, 0x1E0B), + (0x0064, 0x030C, 0x010F), + (0x0064, 0x0323, 0x1E0D), + (0x0064, 0x0327, 0x1E11), + (0x0064, 0x032D, 0x1E13), + (0x0064, 0x0331, 0x1E0F), + (0x0065, 0x0300, 0x00E8), + (0x0065, 0x0301, 0x00E9), + (0x0065, 0x0302, 0x00EA), + (0x0065, 0x0303, 0x1EBD), + (0x0065, 0x0304, 0x0113), + (0x0065, 0x0306, 0x0115), + (0x0065, 0x0307, 0x0117), + (0x0065, 0x0308, 0x00EB), + (0x0065, 0x0309, 0x1EBB), + (0x0065, 0x030C, 0x011B), + (0x0065, 0x030F, 0x0205), + (0x0065, 0x0311, 0x0207), + (0x0065, 0x0323, 0x1EB9), + (0x0065, 0x0327, 0x0229), + (0x0065, 0x0328, 0x0119), + (0x0065, 0x032D, 0x1E19), + (0x0065, 0x0330, 0x1E1B), + (0x0066, 0x0307, 0x1E1F), + (0x0067, 0x0301, 0x01F5), + (0x0067, 0x0302, 0x011D), + (0x0067, 0x0304, 0x1E21), + (0x0067, 0x0306, 0x011F), + (0x0067, 0x0307, 0x0121), + (0x0067, 0x030C, 0x01E7), + (0x0067, 0x0327, 0x0123), + (0x0068, 0x0302, 0x0125), + (0x0068, 0x0307, 0x1E23), + (0x0068, 0x0308, 0x1E27), + (0x0068, 0x030C, 0x021F), + (0x0068, 0x0323, 0x1E25), + (0x0068, 0x0327, 0x1E29), + (0x0068, 0x032E, 0x1E2B), + (0x0068, 0x0331, 0x1E96), + (0x0069, 0x0300, 0x00EC), + (0x0069, 0x0301, 0x00ED), + (0x0069, 0x0302, 0x00EE), + (0x0069, 0x0303, 0x0129), + (0x0069, 0x0304, 0x012B), + (0x0069, 0x0306, 0x012D), + (0x0069, 0x0308, 0x00EF), + (0x0069, 0x0309, 0x1EC9), + (0x0069, 0x030C, 0x01D0), + (0x0069, 0x030F, 0x0209), + (0x0069, 0x0311, 0x020B), + (0x0069, 0x0323, 0x1ECB), + (0x0069, 0x0328, 0x012F), + (0x0069, 0x0330, 0x1E2D), + (0x006A, 0x0302, 0x0135), + (0x006A, 0x030C, 0x01F0), + (0x006B, 0x0301, 0x1E31), + (0x006B, 0x030C, 0x01E9), + (0x006B, 0x0323, 0x1E33), + (0x006B, 0x0327, 0x0137), + (0x006B, 0x0331, 0x1E35), + (0x006C, 0x0301, 0x013A), + (0x006C, 0x030C, 0x013E), + (0x006C, 0x0323, 0x1E37), + (0x006C, 0x0327, 0x013C), + (0x006C, 0x032D, 0x1E3D), + (0x006C, 0x0331, 0x1E3B), + (0x006D, 0x0301, 0x1E3F), + (0x006D, 0x0307, 0x1E41), + (0x006D, 0x0323, 0x1E43), + (0x006E, 0x0300, 0x01F9), + (0x006E, 0x0301, 0x0144), + (0x006E, 0x0303, 0x00F1), + (0x006E, 0x0307, 0x1E45), + (0x006E, 0x030C, 0x0148), + (0x006E, 0x0323, 0x1E47), + (0x006E, 0x0327, 0x0146), + (0x006E, 0x032D, 0x1E4B), + (0x006E, 0x0331, 0x1E49), + (0x006F, 0x0300, 0x00F2), + (0x006F, 0x0301, 0x00F3), + (0x006F, 0x0302, 0x00F4), + (0x006F, 0x0303, 0x00F5), + (0x006F, 0x0304, 0x014D), + (0x006F, 0x0306, 0x014F), + (0x006F, 0x0307, 0x022F), + (0x006F, 0x0308, 0x00F6), + (0x006F, 0x0309, 0x1ECF), + (0x006F, 0x030B, 0x0151), + (0x006F, 0x030C, 0x01D2), + (0x006F, 0x030F, 0x020D), + (0x006F, 0x0311, 0x020F), + (0x006F, 0x031B, 0x01A1), + (0x006F, 0x0323, 0x1ECD), + (0x006F, 0x0328, 0x01EB), + (0x0070, 0x0301, 0x1E55), + (0x0070, 0x0307, 0x1E57), + (0x0072, 0x0301, 0x0155), + (0x0072, 0x0307, 0x1E59), + (0x0072, 0x030C, 0x0159), + (0x0072, 0x030F, 0x0211), + (0x0072, 0x0311, 0x0213), + (0x0072, 0x0323, 0x1E5B), + (0x0072, 0x0327, 0x0157), + (0x0072, 0x0331, 0x1E5F), + (0x0073, 0x0301, 0x015B), + (0x0073, 0x0302, 0x015D), + (0x0073, 0x0307, 0x1E61), + (0x0073, 0x030C, 0x0161), + (0x0073, 0x0323, 0x1E63), + (0x0073, 0x0326, 0x0219), + (0x0073, 0x0327, 0x015F), + (0x0074, 0x0307, 0x1E6B), + (0x0074, 0x0308, 0x1E97), + (0x0074, 0x030C, 0x0165), + (0x0074, 0x0323, 0x1E6D), + (0x0074, 0x0326, 0x021B), + (0x0074, 0x0327, 0x0163), + (0x0074, 0x032D, 0x1E71), + (0x0074, 0x0331, 0x1E6F), + (0x0075, 0x0300, 0x00F9), + (0x0075, 0x0301, 0x00FA), + (0x0075, 0x0302, 0x00FB), + (0x0075, 0x0303, 0x0169), + (0x0075, 0x0304, 0x016B), + (0x0075, 0x0306, 0x016D), + (0x0075, 0x0308, 0x00FC), + (0x0075, 0x0309, 0x1EE7), + (0x0075, 0x030A, 0x016F), + (0x0075, 0x030B, 0x0171), + (0x0075, 0x030C, 0x01D4), + (0x0075, 0x030F, 0x0215), + (0x0075, 0x0311, 0x0217), + (0x0075, 0x031B, 0x01B0), + (0x0075, 0x0323, 0x1EE5), + (0x0075, 0x0324, 0x1E73), + (0x0075, 0x0328, 0x0173), + (0x0075, 0x032D, 0x1E77), + (0x0075, 0x0330, 0x1E75), + (0x0076, 0x0303, 0x1E7D), + (0x0076, 0x0323, 0x1E7F), + (0x0077, 0x0300, 0x1E81), + (0x0077, 0x0301, 0x1E83), + (0x0077, 0x0302, 0x0175), + (0x0077, 0x0307, 0x1E87), + (0x0077, 0x0308, 0x1E85), + (0x0077, 0x030A, 0x1E98), + (0x0077, 0x0323, 0x1E89), + (0x0078, 0x0307, 0x1E8B), + (0x0078, 0x0308, 0x1E8D), + (0x0079, 0x0300, 0x1EF3), + (0x0079, 0x0301, 0x00FD), + (0x0079, 0x0302, 0x0177), + (0x0079, 0x0303, 0x1EF9), + (0x0079, 0x0304, 0x0233), + (0x0079, 0x0307, 0x1E8F), + (0x0079, 0x0308, 0x00FF), + (0x0079, 0x0309, 0x1EF7), + (0x0079, 0x030A, 0x1E99), + (0x0079, 0x0323, 0x1EF5), + (0x007A, 0x0301, 0x017A), + (0x007A, 0x0302, 0x1E91), + (0x007A, 0x0307, 0x017C), + (0x007A, 0x030C, 0x017E), + (0x007A, 0x0323, 0x1E93), + (0x007A, 0x0331, 0x1E95), + (0x00A8, 0x0300, 0x1FED), + (0x00A8, 0x0301, 0x0385), + (0x00A8, 0x0342, 0x1FC1), + (0x00C2, 0x0300, 0x1EA6), + (0x00C2, 0x0301, 0x1EA4), + (0x00C2, 0x0303, 0x1EAA), + (0x00C2, 0x0309, 0x1EA8), + (0x00C4, 0x0304, 0x01DE), + (0x00C5, 0x0301, 0x01FA), + (0x00C6, 0x0301, 0x01FC), + (0x00C6, 0x0304, 0x01E2), + (0x00C7, 0x0301, 0x1E08), + (0x00CA, 0x0300, 0x1EC0), + (0x00CA, 0x0301, 0x1EBE), + (0x00CA, 0x0303, 0x1EC4), + (0x00CA, 0x0309, 0x1EC2), + (0x00CF, 0x0301, 0x1E2E), + (0x00D4, 0x0300, 0x1ED2), + (0x00D4, 0x0301, 0x1ED0), + (0x00D4, 0x0303, 0x1ED6), + (0x00D4, 0x0309, 0x1ED4), + (0x00D5, 0x0301, 0x1E4C), + (0x00D5, 0x0304, 0x022C), + (0x00D5, 0x0308, 0x1E4E), + (0x00D6, 0x0304, 0x022A), + (0x00D8, 0x0301, 0x01FE), + (0x00DC, 0x0300, 0x01DB), + (0x00DC, 0x0301, 0x01D7), + (0x00DC, 0x0304, 0x01D5), + (0x00DC, 0x030C, 0x01D9), + (0x00E2, 0x0300, 0x1EA7), + (0x00E2, 0x0301, 0x1EA5), + (0x00E2, 0x0303, 0x1EAB), + (0x00E2, 0x0309, 0x1EA9), + (0x00E4, 0x0304, 0x01DF), + (0x00E5, 0x0301, 0x01FB), + (0x00E6, 0x0301, 0x01FD), + (0x00E6, 0x0304, 0x01E3), + (0x00E7, 0x0301, 0x1E09), + (0x00EA, 0x0300, 0x1EC1), + (0x00EA, 0x0301, 0x1EBF), + (0x00EA, 0x0303, 0x1EC5), + (0x00EA, 0x0309, 0x1EC3), + (0x00EF, 0x0301, 0x1E2F), + (0x00F4, 0x0300, 0x1ED3), + (0x00F4, 0x0301, 0x1ED1), + (0x00F4, 0x0303, 0x1ED7), + (0x00F4, 0x0309, 0x1ED5), + (0x00F5, 0x0301, 0x1E4D), + (0x00F5, 0x0304, 0x022D), + (0x00F5, 0x0308, 0x1E4F), + (0x00F6, 0x0304, 0x022B), + (0x00F8, 0x0301, 0x01FF), + (0x00FC, 0x0300, 0x01DC), + (0x00FC, 0x0301, 0x01D8), + (0x00FC, 0x0304, 0x01D6), + (0x00FC, 0x030C, 0x01DA), + (0x0102, 0x0300, 0x1EB0), + (0x0102, 0x0301, 0x1EAE), + (0x0102, 0x0303, 0x1EB4), + (0x0102, 0x0309, 0x1EB2), + (0x0103, 0x0300, 0x1EB1), + (0x0103, 0x0301, 0x1EAF), + (0x0103, 0x0303, 0x1EB5), + (0x0103, 0x0309, 0x1EB3), + (0x0112, 0x0300, 0x1E14), + (0x0112, 0x0301, 0x1E16), + (0x0113, 0x0300, 0x1E15), + (0x0113, 0x0301, 0x1E17), + (0x014C, 0x0300, 0x1E50), + (0x014C, 0x0301, 0x1E52), + (0x014D, 0x0300, 0x1E51), + (0x014D, 0x0301, 0x1E53), + (0x015A, 0x0307, 0x1E64), + (0x015B, 0x0307, 0x1E65), + (0x0160, 0x0307, 0x1E66), + (0x0161, 0x0307, 0x1E67), + (0x0168, 0x0301, 0x1E78), + (0x0169, 0x0301, 0x1E79), + (0x016A, 0x0308, 0x1E7A), + (0x016B, 0x0308, 0x1E7B), + (0x017F, 0x0307, 0x1E9B), + (0x01A0, 0x0300, 0x1EDC), + (0x01A0, 0x0301, 0x1EDA), + (0x01A0, 0x0303, 0x1EE0), + (0x01A0, 0x0309, 0x1EDE), + (0x01A0, 0x0323, 0x1EE2), + (0x01A1, 0x0300, 0x1EDD), + (0x01A1, 0x0301, 0x1EDB), + (0x01A1, 0x0303, 0x1EE1), + (0x01A1, 0x0309, 0x1EDF), + (0x01A1, 0x0323, 0x1EE3), + (0x01AF, 0x0300, 0x1EEA), + (0x01AF, 0x0301, 0x1EE8), + (0x01AF, 0x0303, 0x1EEE), + (0x01AF, 0x0309, 0x1EEC), + (0x01AF, 0x0323, 0x1EF0), + (0x01B0, 0x0300, 0x1EEB), + (0x01B0, 0x0301, 0x1EE9), + (0x01B0, 0x0303, 0x1EEF), + (0x01B0, 0x0309, 0x1EED), + (0x01B0, 0x0323, 0x1EF1), + (0x01B7, 0x030C, 0x01EE), + (0x01EA, 0x0304, 0x01EC), + (0x01EB, 0x0304, 0x01ED), + (0x0226, 0x0304, 0x01E0), + (0x0227, 0x0304, 0x01E1), + (0x0228, 0x0306, 0x1E1C), + (0x0229, 0x0306, 0x1E1D), + (0x022E, 0x0304, 0x0230), + (0x022F, 0x0304, 0x0231), + (0x0292, 0x030C, 0x01EF), + (0x0391, 0x0300, 0x1FBA), + (0x0391, 0x0301, 0x0386), + (0x0391, 0x0304, 0x1FB9), + (0x0391, 0x0306, 0x1FB8), + (0x0391, 0x0313, 0x1F08), + (0x0391, 0x0314, 0x1F09), + (0x0391, 0x0345, 0x1FBC), + (0x0395, 0x0300, 0x1FC8), + (0x0395, 0x0301, 0x0388), + (0x0395, 0x0313, 0x1F18), + (0x0395, 0x0314, 0x1F19), + (0x0397, 0x0300, 0x1FCA), + (0x0397, 0x0301, 0x0389), + (0x0397, 0x0313, 0x1F28), + (0x0397, 0x0314, 0x1F29), + (0x0397, 0x0345, 0x1FCC), + (0x0399, 0x0300, 0x1FDA), + (0x0399, 0x0301, 0x038A), + (0x0399, 0x0304, 0x1FD9), + (0x0399, 0x0306, 0x1FD8), + (0x0399, 0x0308, 0x03AA), + (0x0399, 0x0313, 0x1F38), + (0x0399, 0x0314, 0x1F39), + (0x039F, 0x0300, 0x1FF8), + (0x039F, 0x0301, 0x038C), + (0x039F, 0x0313, 0x1F48), + (0x039F, 0x0314, 0x1F49), + (0x03A1, 0x0314, 0x1FEC), + (0x03A5, 0x0300, 0x1FEA), + (0x03A5, 0x0301, 0x038E), + (0x03A5, 0x0304, 0x1FE9), + (0x03A5, 0x0306, 0x1FE8), + (0x03A5, 0x0308, 0x03AB), + (0x03A5, 0x0314, 0x1F59), + (0x03A9, 0x0300, 0x1FFA), + (0x03A9, 0x0301, 0x038F), + (0x03A9, 0x0313, 0x1F68), + (0x03A9, 0x0314, 0x1F69), + (0x03A9, 0x0345, 0x1FFC), + (0x03AC, 0x0345, 0x1FB4), + (0x03AE, 0x0345, 0x1FC4), + (0x03B1, 0x0300, 0x1F70), + (0x03B1, 0x0301, 0x03AC), + (0x03B1, 0x0304, 0x1FB1), + (0x03B1, 0x0306, 0x1FB0), + (0x03B1, 0x0313, 0x1F00), + (0x03B1, 0x0314, 0x1F01), + (0x03B1, 0x0342, 0x1FB6), + (0x03B1, 0x0345, 0x1FB3), + (0x03B5, 0x0300, 0x1F72), + (0x03B5, 0x0301, 0x03AD), + (0x03B5, 0x0313, 0x1F10), + (0x03B5, 0x0314, 0x1F11), + (0x03B7, 0x0300, 0x1F74), + (0x03B7, 0x0301, 0x03AE), + (0x03B7, 0x0313, 0x1F20), + (0x03B7, 0x0314, 0x1F21), + (0x03B7, 0x0342, 0x1FC6), + (0x03B7, 0x0345, 0x1FC3), + (0x03B9, 0x0300, 0x1F76), + (0x03B9, 0x0301, 0x03AF), + (0x03B9, 0x0304, 0x1FD1), + (0x03B9, 0x0306, 0x1FD0), + (0x03B9, 0x0308, 0x03CA), + (0x03B9, 0x0313, 0x1F30), + (0x03B9, 0x0314, 0x1F31), + (0x03B9, 0x0342, 0x1FD6), + (0x03BF, 0x0300, 0x1F78), + (0x03BF, 0x0301, 0x03CC), + (0x03BF, 0x0313, 0x1F40), + (0x03BF, 0x0314, 0x1F41), + (0x03C1, 0x0313, 0x1FE4), + (0x03C1, 0x0314, 0x1FE5), + (0x03C5, 0x0300, 0x1F7A), + (0x03C5, 0x0301, 0x03CD), + (0x03C5, 0x0304, 0x1FE1), + (0x03C5, 0x0306, 0x1FE0), + (0x03C5, 0x0308, 0x03CB), + (0x03C5, 0x0313, 0x1F50), + (0x03C5, 0x0314, 0x1F51), + (0x03C5, 0x0342, 0x1FE6), + (0x03C9, 0x0300, 0x1F7C), + (0x03C9, 0x0301, 0x03CE), + (0x03C9, 0x0313, 0x1F60), + (0x03C9, 0x0314, 0x1F61), + (0x03C9, 0x0342, 0x1FF6), + (0x03C9, 0x0345, 0x1FF3), + (0x03CA, 0x0300, 0x1FD2), + (0x03CA, 0x0301, 0x0390), + (0x03CA, 0x0342, 0x1FD7), + (0x03CB, 0x0300, 0x1FE2), + (0x03CB, 0x0301, 0x03B0), + (0x03CB, 0x0342, 0x1FE7), + (0x03CE, 0x0345, 0x1FF4), + (0x03D2, 0x0301, 0x03D3), + (0x03D2, 0x0308, 0x03D4), + (0x0406, 0x0308, 0x0407), + (0x0410, 0x0306, 0x04D0), + (0x0410, 0x0308, 0x04D2), + (0x0413, 0x0301, 0x0403), + (0x0415, 0x0300, 0x0400), + (0x0415, 0x0306, 0x04D6), + (0x0415, 0x0308, 0x0401), + (0x0416, 0x0306, 0x04C1), + (0x0416, 0x0308, 0x04DC), + (0x0417, 0x0308, 0x04DE), + (0x0418, 0x0300, 0x040D), + (0x0418, 0x0304, 0x04E2), + (0x0418, 0x0306, 0x0419), + (0x0418, 0x0308, 0x04E4), + (0x041A, 0x0301, 0x040C), + (0x041E, 0x0308, 0x04E6), + (0x0423, 0x0304, 0x04EE), + (0x0423, 0x0306, 0x040E), + (0x0423, 0x0308, 0x04F0), + (0x0423, 0x030B, 0x04F2), + (0x0427, 0x0308, 0x04F4), + (0x042B, 0x0308, 0x04F8), + (0x042D, 0x0308, 0x04EC), + (0x0430, 0x0306, 0x04D1), + (0x0430, 0x0308, 0x04D3), + (0x0433, 0x0301, 0x0453), + (0x0435, 0x0300, 0x0450), + (0x0435, 0x0306, 0x04D7), + (0x0435, 0x0308, 0x0451), + (0x0436, 0x0306, 0x04C2), + (0x0436, 0x0308, 0x04DD), + (0x0437, 0x0308, 0x04DF), + (0x0438, 0x0300, 0x045D), + (0x0438, 0x0304, 0x04E3), + (0x0438, 0x0306, 0x0439), + (0x0438, 0x0308, 0x04E5), + (0x043A, 0x0301, 0x045C), + (0x043E, 0x0308, 0x04E7), + (0x0443, 0x0304, 0x04EF), + (0x0443, 0x0306, 0x045E), + (0x0443, 0x0308, 0x04F1), + (0x0443, 0x030B, 0x04F3), + (0x0447, 0x0308, 0x04F5), + (0x044B, 0x0308, 0x04F9), + (0x044D, 0x0308, 0x04ED), + (0x0456, 0x0308, 0x0457), + (0x0474, 0x030F, 0x0476), + (0x0475, 0x030F, 0x0477), + (0x04D8, 0x0308, 0x04DA), + (0x04D9, 0x0308, 0x04DB), + (0x04E8, 0x0308, 0x04EA), + (0x04E9, 0x0308, 0x04EB), + (0x0627, 0x0653, 0x0622), + (0x0627, 0x0654, 0x0623), + (0x0627, 0x0655, 0x0625), + (0x0648, 0x0654, 0x0624), + (0x064A, 0x0654, 0x0626), + (0x06C1, 0x0654, 0x06C2), + (0x06D2, 0x0654, 0x06D3), + (0x06D5, 0x0654, 0x06C0), + (0x0928, 0x093C, 0x0929), + (0x0930, 0x093C, 0x0931), + (0x0933, 0x093C, 0x0934), + (0x09C7, 0x09BE, 0x09CB), + (0x09C7, 0x09D7, 0x09CC), + (0x0B47, 0x0B3E, 0x0B4B), + (0x0B47, 0x0B56, 0x0B48), + (0x0B47, 0x0B57, 0x0B4C), + (0x0B92, 0x0BD7, 0x0B94), + (0x0BC6, 0x0BBE, 0x0BCA), + (0x0BC6, 0x0BD7, 0x0BCC), + (0x0BC7, 0x0BBE, 0x0BCB), + (0x0C46, 0x0C56, 0x0C48), + (0x0CBF, 0x0CD5, 0x0CC0), + (0x0CC6, 0x0CC2, 0x0CCA), + (0x0CC6, 0x0CD5, 0x0CC7), + (0x0CC6, 0x0CD6, 0x0CC8), + (0x0CCA, 0x0CD5, 0x0CCB), + (0x0D46, 0x0D3E, 0x0D4A), + (0x0D46, 0x0D57, 0x0D4C), + (0x0D47, 0x0D3E, 0x0D4B), + (0x0DD9, 0x0DCA, 0x0DDA), + (0x0DD9, 0x0DCF, 0x0DDC), + (0x0DD9, 0x0DDF, 0x0DDE), + (0x0DDC, 0x0DCA, 0x0DDD), + (0x1025, 0x102E, 0x1026), + (0x1B05, 0x1B35, 0x1B06), + (0x1B07, 0x1B35, 0x1B08), + (0x1B09, 0x1B35, 0x1B0A), + (0x1B0B, 0x1B35, 0x1B0C), + (0x1B0D, 0x1B35, 0x1B0E), + (0x1B11, 0x1B35, 0x1B12), + (0x1B3A, 0x1B35, 0x1B3B), + (0x1B3C, 0x1B35, 0x1B3D), + (0x1B3E, 0x1B35, 0x1B40), + (0x1B3F, 0x1B35, 0x1B41), + (0x1B42, 0x1B35, 0x1B43), + (0x1E36, 0x0304, 0x1E38), + (0x1E37, 0x0304, 0x1E39), + (0x1E5A, 0x0304, 0x1E5C), + (0x1E5B, 0x0304, 0x1E5D), + (0x1E62, 0x0307, 0x1E68), + (0x1E63, 0x0307, 0x1E69), + (0x1EA0, 0x0302, 0x1EAC), + (0x1EA0, 0x0306, 0x1EB6), + (0x1EA1, 0x0302, 0x1EAD), + (0x1EA1, 0x0306, 0x1EB7), + (0x1EB8, 0x0302, 0x1EC6), + (0x1EB9, 0x0302, 0x1EC7), + (0x1ECC, 0x0302, 0x1ED8), + (0x1ECD, 0x0302, 0x1ED9), + (0x1F00, 0x0300, 0x1F02), + (0x1F00, 0x0301, 0x1F04), + (0x1F00, 0x0342, 0x1F06), + (0x1F00, 0x0345, 0x1F80), + (0x1F01, 0x0300, 0x1F03), + (0x1F01, 0x0301, 0x1F05), + (0x1F01, 0x0342, 0x1F07), + (0x1F01, 0x0345, 0x1F81), + (0x1F02, 0x0345, 0x1F82), + (0x1F03, 0x0345, 0x1F83), + (0x1F04, 0x0345, 0x1F84), + (0x1F05, 0x0345, 0x1F85), + (0x1F06, 0x0345, 0x1F86), + (0x1F07, 0x0345, 0x1F87), + (0x1F08, 0x0300, 0x1F0A), + (0x1F08, 0x0301, 0x1F0C), + (0x1F08, 0x0342, 0x1F0E), + (0x1F08, 0x0345, 0x1F88), + (0x1F09, 0x0300, 0x1F0B), + (0x1F09, 0x0301, 0x1F0D), + (0x1F09, 0x0342, 0x1F0F), + (0x1F09, 0x0345, 0x1F89), + (0x1F0A, 0x0345, 0x1F8A), + (0x1F0B, 0x0345, 0x1F8B), + (0x1F0C, 0x0345, 0x1F8C), + (0x1F0D, 0x0345, 0x1F8D), + (0x1F0E, 0x0345, 0x1F8E), + (0x1F0F, 0x0345, 0x1F8F), + (0x1F10, 0x0300, 0x1F12), + (0x1F10, 0x0301, 0x1F14), + (0x1F11, 0x0300, 0x1F13), + (0x1F11, 0x0301, 0x1F15), + (0x1F18, 0x0300, 0x1F1A), + (0x1F18, 0x0301, 0x1F1C), + (0x1F19, 0x0300, 0x1F1B), + (0x1F19, 0x0301, 0x1F1D), + (0x1F20, 0x0300, 0x1F22), + (0x1F20, 0x0301, 0x1F24), + (0x1F20, 0x0342, 0x1F26), + (0x1F20, 0x0345, 0x1F90), + (0x1F21, 0x0300, 0x1F23), + (0x1F21, 0x0301, 0x1F25), + (0x1F21, 0x0342, 0x1F27), + (0x1F21, 0x0345, 0x1F91), + (0x1F22, 0x0345, 0x1F92), + (0x1F23, 0x0345, 0x1F93), + (0x1F24, 0x0345, 0x1F94), + (0x1F25, 0x0345, 0x1F95), + (0x1F26, 0x0345, 0x1F96), + (0x1F27, 0x0345, 0x1F97), + (0x1F28, 0x0300, 0x1F2A), + (0x1F28, 0x0301, 0x1F2C), + (0x1F28, 0x0342, 0x1F2E), + (0x1F28, 0x0345, 0x1F98), + (0x1F29, 0x0300, 0x1F2B), + (0x1F29, 0x0301, 0x1F2D), + (0x1F29, 0x0342, 0x1F2F), + (0x1F29, 0x0345, 0x1F99), + (0x1F2A, 0x0345, 0x1F9A), + (0x1F2B, 0x0345, 0x1F9B), + (0x1F2C, 0x0345, 0x1F9C), + (0x1F2D, 0x0345, 0x1F9D), + (0x1F2E, 0x0345, 0x1F9E), + (0x1F2F, 0x0345, 0x1F9F), + (0x1F30, 0x0300, 0x1F32), + (0x1F30, 0x0301, 0x1F34), + (0x1F30, 0x0342, 0x1F36), + (0x1F31, 0x0300, 0x1F33), + (0x1F31, 0x0301, 0x1F35), + (0x1F31, 0x0342, 0x1F37), + (0x1F38, 0x0300, 0x1F3A), + (0x1F38, 0x0301, 0x1F3C), + (0x1F38, 0x0342, 0x1F3E), + (0x1F39, 0x0300, 0x1F3B), + (0x1F39, 0x0301, 0x1F3D), + (0x1F39, 0x0342, 0x1F3F), + (0x1F40, 0x0300, 0x1F42), + (0x1F40, 0x0301, 0x1F44), + (0x1F41, 0x0300, 0x1F43), + (0x1F41, 0x0301, 0x1F45), + (0x1F48, 0x0300, 0x1F4A), + (0x1F48, 0x0301, 0x1F4C), + (0x1F49, 0x0300, 0x1F4B), + (0x1F49, 0x0301, 0x1F4D), + (0x1F50, 0x0300, 0x1F52), + (0x1F50, 0x0301, 0x1F54), + (0x1F50, 0x0342, 0x1F56), + (0x1F51, 0x0300, 0x1F53), + (0x1F51, 0x0301, 0x1F55), + (0x1F51, 0x0342, 0x1F57), + (0x1F59, 0x0300, 0x1F5B), + (0x1F59, 0x0301, 0x1F5D), + (0x1F59, 0x0342, 0x1F5F), + (0x1F60, 0x0300, 0x1F62), + (0x1F60, 0x0301, 0x1F64), + (0x1F60, 0x0342, 0x1F66), + (0x1F60, 0x0345, 0x1FA0), + (0x1F61, 0x0300, 0x1F63), + (0x1F61, 0x0301, 0x1F65), + (0x1F61, 0x0342, 0x1F67), + (0x1F61, 0x0345, 0x1FA1), + (0x1F62, 0x0345, 0x1FA2), + (0x1F63, 0x0345, 0x1FA3), + (0x1F64, 0x0345, 0x1FA4), + (0x1F65, 0x0345, 0x1FA5), + (0x1F66, 0x0345, 0x1FA6), + (0x1F67, 0x0345, 0x1FA7), + (0x1F68, 0x0300, 0x1F6A), + (0x1F68, 0x0301, 0x1F6C), + (0x1F68, 0x0342, 0x1F6E), + (0x1F68, 0x0345, 0x1FA8), + (0x1F69, 0x0300, 0x1F6B), + (0x1F69, 0x0301, 0x1F6D), + (0x1F69, 0x0342, 0x1F6F), + (0x1F69, 0x0345, 0x1FA9), + (0x1F6A, 0x0345, 0x1FAA), + (0x1F6B, 0x0345, 0x1FAB), + (0x1F6C, 0x0345, 0x1FAC), + (0x1F6D, 0x0345, 0x1FAD), + (0x1F6E, 0x0345, 0x1FAE), + (0x1F6F, 0x0345, 0x1FAF), + (0x1F70, 0x0345, 0x1FB2), + (0x1F74, 0x0345, 0x1FC2), + (0x1F7C, 0x0345, 0x1FF2), + (0x1FB6, 0x0345, 0x1FB7), + (0x1FBF, 0x0300, 0x1FCD), + (0x1FBF, 0x0301, 0x1FCE), + (0x1FBF, 0x0342, 0x1FCF), + (0x1FC6, 0x0345, 0x1FC7), + (0x1FF6, 0x0345, 0x1FF7), + (0x1FFE, 0x0300, 0x1FDD), + (0x1FFE, 0x0301, 0x1FDE), + (0x1FFE, 0x0342, 0x1FDF), + (0x2190, 0x0338, 0x219A), + (0x2192, 0x0338, 0x219B), + (0x2194, 0x0338, 0x21AE), + (0x21D0, 0x0338, 0x21CD), + (0x21D2, 0x0338, 0x21CF), + (0x21D4, 0x0338, 0x21CE), + (0x2203, 0x0338, 0x2204), + (0x2208, 0x0338, 0x2209), + (0x220B, 0x0338, 0x220C), + (0x2223, 0x0338, 0x2224), + (0x2225, 0x0338, 0x2226), + (0x223C, 0x0338, 0x2241), + (0x2243, 0x0338, 0x2244), + (0x2245, 0x0338, 0x2247), + (0x2248, 0x0338, 0x2249), + (0x224D, 0x0338, 0x226D), + (0x2261, 0x0338, 0x2262), + (0x2264, 0x0338, 0x2270), + (0x2265, 0x0338, 0x2271), + (0x2272, 0x0338, 0x2274), + (0x2273, 0x0338, 0x2275), + (0x2276, 0x0338, 0x2278), + (0x2277, 0x0338, 0x2279), + (0x227A, 0x0338, 0x2280), + (0x227B, 0x0338, 0x2281), + (0x227C, 0x0338, 0x22E0), + (0x227D, 0x0338, 0x22E1), + (0x2282, 0x0338, 0x2284), + (0x2283, 0x0338, 0x2285), + (0x2286, 0x0338, 0x2288), + (0x2287, 0x0338, 0x2289), + (0x2291, 0x0338, 0x22E2), + (0x2292, 0x0338, 0x22E3), + (0x22A2, 0x0338, 0x22AC), + (0x22A8, 0x0338, 0x22AD), + (0x22A9, 0x0338, 0x22AE), + (0x22AB, 0x0338, 0x22AF), + (0x22B2, 0x0338, 0x22EA), + (0x22B3, 0x0338, 0x22EB), + (0x22B4, 0x0338, 0x22EC), + (0x22B5, 0x0338, 0x22ED), + (0x3046, 0x3099, 0x3094), + (0x304B, 0x3099, 0x304C), + (0x304D, 0x3099, 0x304E), + (0x304F, 0x3099, 0x3050), + (0x3051, 0x3099, 0x3052), + (0x3053, 0x3099, 0x3054), + (0x3055, 0x3099, 0x3056), + (0x3057, 0x3099, 0x3058), + (0x3059, 0x3099, 0x305A), + (0x305B, 0x3099, 0x305C), + (0x305D, 0x3099, 0x305E), + (0x305F, 0x3099, 0x3060), + (0x3061, 0x3099, 0x3062), + (0x3064, 0x3099, 0x3065), + (0x3066, 0x3099, 0x3067), + (0x3068, 0x3099, 0x3069), + (0x306F, 0x3099, 0x3070), + (0x306F, 0x309A, 0x3071), + (0x3072, 0x3099, 0x3073), + (0x3072, 0x309A, 0x3074), + (0x3075, 0x3099, 0x3076), + (0x3075, 0x309A, 0x3077), + (0x3078, 0x3099, 0x3079), + (0x3078, 0x309A, 0x307A), + (0x307B, 0x3099, 0x307C), + (0x307B, 0x309A, 0x307D), + (0x309D, 0x3099, 0x309E), + (0x30A6, 0x3099, 0x30F4), + (0x30AB, 0x3099, 0x30AC), + (0x30AD, 0x3099, 0x30AE), + (0x30AF, 0x3099, 0x30B0), + (0x30B1, 0x3099, 0x30B2), + (0x30B3, 0x3099, 0x30B4), + (0x30B5, 0x3099, 0x30B6), + (0x30B7, 0x3099, 0x30B8), + (0x30B9, 0x3099, 0x30BA), + (0x30BB, 0x3099, 0x30BC), + (0x30BD, 0x3099, 0x30BE), + (0x30BF, 0x3099, 0x30C0), + (0x30C1, 0x3099, 0x30C2), + (0x30C4, 0x3099, 0x30C5), + (0x30C6, 0x3099, 0x30C7), + (0x30C8, 0x3099, 0x30C9), + (0x30CF, 0x3099, 0x30D0), + (0x30CF, 0x309A, 0x30D1), + (0x30D2, 0x3099, 0x30D3), + (0x30D2, 0x309A, 0x30D4), + (0x30D5, 0x3099, 0x30D6), + (0x30D5, 0x309A, 0x30D7), + (0x30D8, 0x3099, 0x30D9), + (0x30D8, 0x309A, 0x30DA), + (0x30DB, 0x3099, 0x30DC), + (0x30DB, 0x309A, 0x30DD), + (0x30EF, 0x3099, 0x30F7), + (0x30F0, 0x3099, 0x30F8), + (0x30F1, 0x3099, 0x30F9), + (0x30F2, 0x3099, 0x30FA), + (0x30FD, 0x3099, 0x30FE), + (0x11099, 0x110BA, 0x1109A), + (0x1109B, 0x110BA, 0x1109C), + (0x110A5, 0x110BA, 0x110AB), +]; + +/// `(first, last, class)` for every run of characters that share a non-zero canonical combining +/// class, sorted. +pub(super) static COMBINING_CLASSES: [(u32, u32, u8); 266] = [ + (0x0300, 0x0314, 230), + (0x0315, 0x0315, 232), + (0x0316, 0x0319, 220), + (0x031A, 0x031A, 232), + (0x031B, 0x031B, 216), + (0x031C, 0x0320, 220), + (0x0321, 0x0322, 202), + (0x0323, 0x0326, 220), + (0x0327, 0x0328, 202), + (0x0329, 0x0333, 220), + (0x0334, 0x0338, 1), + (0x0339, 0x033C, 220), + (0x033D, 0x0344, 230), + (0x0345, 0x0345, 240), + (0x0346, 0x0346, 230), + (0x0347, 0x0349, 220), + (0x034A, 0x034C, 230), + (0x034D, 0x034E, 220), + (0x0350, 0x0352, 230), + (0x0353, 0x0356, 220), + (0x0357, 0x0357, 230), + (0x0358, 0x0358, 232), + (0x0359, 0x035A, 220), + (0x035B, 0x035B, 230), + (0x035C, 0x035C, 233), + (0x035D, 0x035E, 234), + (0x035F, 0x035F, 233), + (0x0360, 0x0361, 234), + (0x0362, 0x0362, 233), + (0x0363, 0x036F, 230), + (0x0483, 0x0487, 230), + (0x0591, 0x0591, 220), + (0x0592, 0x0595, 230), + (0x0596, 0x0596, 220), + (0x0597, 0x0599, 230), + (0x059A, 0x059A, 222), + (0x059B, 0x059B, 220), + (0x059C, 0x05A1, 230), + (0x05A2, 0x05A7, 220), + (0x05A8, 0x05A9, 230), + (0x05AA, 0x05AA, 220), + (0x05AB, 0x05AC, 230), + (0x05AD, 0x05AD, 222), + (0x05AE, 0x05AE, 228), + (0x05AF, 0x05AF, 230), + (0x05B0, 0x05B0, 10), + (0x05B1, 0x05B1, 11), + (0x05B2, 0x05B2, 12), + (0x05B3, 0x05B3, 13), + (0x05B4, 0x05B4, 14), + (0x05B5, 0x05B5, 15), + (0x05B6, 0x05B6, 16), + (0x05B7, 0x05B7, 17), + (0x05B8, 0x05B8, 18), + (0x05B9, 0x05BA, 19), + (0x05BB, 0x05BB, 20), + (0x05BC, 0x05BC, 21), + (0x05BD, 0x05BD, 22), + (0x05BF, 0x05BF, 23), + (0x05C1, 0x05C1, 24), + (0x05C2, 0x05C2, 25), + (0x05C4, 0x05C4, 230), + (0x05C5, 0x05C5, 220), + (0x05C7, 0x05C7, 18), + (0x0610, 0x0617, 230), + (0x0618, 0x0618, 30), + (0x0619, 0x0619, 31), + (0x061A, 0x061A, 32), + (0x064B, 0x064B, 27), + (0x064C, 0x064C, 28), + (0x064D, 0x064D, 29), + (0x064E, 0x064E, 30), + (0x064F, 0x064F, 31), + (0x0650, 0x0650, 32), + (0x0651, 0x0651, 33), + (0x0652, 0x0652, 34), + (0x0653, 0x0654, 230), + (0x0655, 0x0656, 220), + (0x0657, 0x065B, 230), + (0x065C, 0x065C, 220), + (0x065D, 0x065E, 230), + (0x065F, 0x065F, 220), + (0x0670, 0x0670, 35), + (0x06D6, 0x06DC, 230), + (0x06DF, 0x06E2, 230), + (0x06E3, 0x06E3, 220), + (0x06E4, 0x06E4, 230), + (0x06E7, 0x06E8, 230), + (0x06EA, 0x06EA, 220), + (0x06EB, 0x06EC, 230), + (0x06ED, 0x06ED, 220), + (0x0711, 0x0711, 36), + (0x0730, 0x0730, 230), + (0x0731, 0x0731, 220), + (0x0732, 0x0733, 230), + (0x0734, 0x0734, 220), + (0x0735, 0x0736, 230), + (0x0737, 0x0739, 220), + (0x073A, 0x073A, 230), + (0x073B, 0x073C, 220), + (0x073D, 0x073D, 230), + (0x073E, 0x073E, 220), + (0x073F, 0x0741, 230), + (0x0742, 0x0742, 220), + (0x0743, 0x0743, 230), + (0x0744, 0x0744, 220), + (0x0745, 0x0745, 230), + (0x0746, 0x0746, 220), + (0x0747, 0x0747, 230), + (0x0748, 0x0748, 220), + (0x0749, 0x074A, 230), + (0x07EB, 0x07F1, 230), + (0x07F2, 0x07F2, 220), + (0x07F3, 0x07F3, 230), + (0x0816, 0x0819, 230), + (0x081B, 0x0823, 230), + (0x0825, 0x0827, 230), + (0x0829, 0x082D, 230), + (0x0859, 0x085B, 220), + (0x093C, 0x093C, 7), + (0x094D, 0x094D, 9), + (0x0951, 0x0951, 230), + (0x0952, 0x0952, 220), + (0x0953, 0x0954, 230), + (0x09BC, 0x09BC, 7), + (0x09CD, 0x09CD, 9), + (0x0A3C, 0x0A3C, 7), + (0x0A4D, 0x0A4D, 9), + (0x0ABC, 0x0ABC, 7), + (0x0ACD, 0x0ACD, 9), + (0x0B3C, 0x0B3C, 7), + (0x0B4D, 0x0B4D, 9), + (0x0BCD, 0x0BCD, 9), + (0x0C4D, 0x0C4D, 9), + (0x0C55, 0x0C55, 84), + (0x0C56, 0x0C56, 91), + (0x0CBC, 0x0CBC, 7), + (0x0CCD, 0x0CCD, 9), + (0x0D4D, 0x0D4D, 9), + (0x0DCA, 0x0DCA, 9), + (0x0E38, 0x0E39, 103), + (0x0E3A, 0x0E3A, 9), + (0x0E48, 0x0E4B, 107), + (0x0EB8, 0x0EB9, 118), + (0x0EC8, 0x0ECB, 122), + (0x0F18, 0x0F19, 220), + (0x0F35, 0x0F35, 220), + (0x0F37, 0x0F37, 220), + (0x0F39, 0x0F39, 216), + (0x0F71, 0x0F71, 129), + (0x0F72, 0x0F72, 130), + (0x0F74, 0x0F74, 132), + (0x0F7A, 0x0F7D, 130), + (0x0F80, 0x0F80, 130), + (0x0F82, 0x0F83, 230), + (0x0F84, 0x0F84, 9), + (0x0F86, 0x0F87, 230), + (0x0FC6, 0x0FC6, 220), + (0x1037, 0x1037, 7), + (0x1039, 0x103A, 9), + (0x108D, 0x108D, 220), + (0x135D, 0x135F, 230), + (0x1714, 0x1714, 9), + (0x1734, 0x1734, 9), + (0x17D2, 0x17D2, 9), + (0x17DD, 0x17DD, 230), + (0x18A9, 0x18A9, 228), + (0x1939, 0x1939, 222), + (0x193A, 0x193A, 230), + (0x193B, 0x193B, 220), + (0x1A17, 0x1A17, 230), + (0x1A18, 0x1A18, 220), + (0x1A60, 0x1A60, 9), + (0x1A75, 0x1A7C, 230), + (0x1A7F, 0x1A7F, 220), + (0x1B34, 0x1B34, 7), + (0x1B44, 0x1B44, 9), + (0x1B6B, 0x1B6B, 230), + (0x1B6C, 0x1B6C, 220), + (0x1B6D, 0x1B73, 230), + (0x1BAA, 0x1BAA, 9), + (0x1BE6, 0x1BE6, 7), + (0x1BF2, 0x1BF3, 9), + (0x1C37, 0x1C37, 7), + (0x1CD0, 0x1CD2, 230), + (0x1CD4, 0x1CD4, 1), + (0x1CD5, 0x1CD9, 220), + (0x1CDA, 0x1CDB, 230), + (0x1CDC, 0x1CDF, 220), + (0x1CE0, 0x1CE0, 230), + (0x1CE2, 0x1CE8, 1), + (0x1CED, 0x1CED, 220), + (0x1DC0, 0x1DC1, 230), + (0x1DC2, 0x1DC2, 220), + (0x1DC3, 0x1DC9, 230), + (0x1DCA, 0x1DCA, 220), + (0x1DCB, 0x1DCC, 230), + (0x1DCD, 0x1DCD, 234), + (0x1DCE, 0x1DCE, 214), + (0x1DCF, 0x1DCF, 220), + (0x1DD0, 0x1DD0, 202), + (0x1DD1, 0x1DE6, 230), + (0x1DFC, 0x1DFC, 233), + (0x1DFD, 0x1DFD, 220), + (0x1DFE, 0x1DFE, 230), + (0x1DFF, 0x1DFF, 220), + (0x20D0, 0x20D1, 230), + (0x20D2, 0x20D3, 1), + (0x20D4, 0x20D7, 230), + (0x20D8, 0x20DA, 1), + (0x20DB, 0x20DC, 230), + (0x20E1, 0x20E1, 230), + (0x20E5, 0x20E6, 1), + (0x20E7, 0x20E7, 230), + (0x20E8, 0x20E8, 220), + (0x20E9, 0x20E9, 230), + (0x20EA, 0x20EB, 1), + (0x20EC, 0x20EF, 220), + (0x20F0, 0x20F0, 230), + (0x2CEF, 0x2CF1, 230), + (0x2D7F, 0x2D7F, 9), + (0x2DE0, 0x2DFF, 230), + (0x302A, 0x302A, 218), + (0x302B, 0x302B, 228), + (0x302C, 0x302C, 232), + (0x302D, 0x302D, 222), + (0x302E, 0x302F, 224), + (0x3099, 0x309A, 8), + (0xA66F, 0xA66F, 230), + (0xA67C, 0xA67D, 230), + (0xA6F0, 0xA6F1, 230), + (0xA806, 0xA806, 9), + (0xA8C4, 0xA8C4, 9), + (0xA8E0, 0xA8F1, 230), + (0xA92B, 0xA92D, 220), + (0xA953, 0xA953, 9), + (0xA9B3, 0xA9B3, 7), + (0xA9C0, 0xA9C0, 9), + (0xAAB0, 0xAAB0, 230), + (0xAAB2, 0xAAB3, 230), + (0xAAB4, 0xAAB4, 220), + (0xAAB7, 0xAAB8, 230), + (0xAABE, 0xAABF, 230), + (0xAAC1, 0xAAC1, 230), + (0xABED, 0xABED, 9), + (0xFB1E, 0xFB1E, 26), + (0xFE20, 0xFE26, 230), + (0x101FD, 0x101FD, 220), + (0x10A0D, 0x10A0D, 220), + (0x10A0F, 0x10A0F, 230), + (0x10A38, 0x10A38, 230), + (0x10A39, 0x10A39, 1), + (0x10A3A, 0x10A3A, 220), + (0x10A3F, 0x10A3F, 9), + (0x11046, 0x11046, 9), + (0x110B9, 0x110B9, 9), + (0x110BA, 0x110BA, 7), + (0x1D165, 0x1D166, 216), + (0x1D167, 0x1D169, 1), + (0x1D16D, 0x1D16D, 226), + (0x1D16E, 0x1D172, 216), + (0x1D17B, 0x1D182, 220), + (0x1D185, 0x1D189, 230), + (0x1D18A, 0x1D18B, 220), + (0x1D1AA, 0x1D1AD, 230), + (0x1D242, 0x1D244, 230), +]; + +/// `(first, last)` for every run of 256-character blocks that hold a character from +/// [`COMBINING_CLASSES`] or a Hangul jamo or syllable, sorted. A block is a code point `>> 8`. +pub(super) static COMPOSING_BLOCKS: [(u32, u32); 14] = [ + (0x03, 0x11), + (0x13, 0x13), + (0x17, 0x1D), + (0x20, 0x20), + (0x2C, 0x2D), + (0x30, 0x30), + (0xA6, 0xA6), + (0xA8, 0xD7), + (0xFB, 0xFB), + (0xFE, 0xFE), + (0x101, 0x101), + (0x10A, 0x10A), + (0x110, 0x110), + (0x1D1, 0x1D2), +]; diff --git a/src/tar/error.rs b/src/tar/error.rs index e66cc5513e90..dae325a54e35 100644 --- a/src/tar/error.rs +++ b/src/tar/error.rs @@ -40,6 +40,8 @@ pub enum Error { TruncatedGzip, #[error("gzip decompression failed")] GzipDecompressionFailed, + #[error("GZip compression failed")] + GzipCompressionFailed, #[error("gzip trailer mismatch: bad CRC32 or uncompressed size")] GzipTrailerMismatch, #[error("Out of memory")] diff --git a/src/tar/gzip.rs b/src/tar/gzip.rs index b2b95b7cbd0f..f9d3e9459905 100644 --- a/src/tar/gzip.rs +++ b/src/tar/gzip.rs @@ -302,3 +302,90 @@ impl Decoder { } } } + +/// Compresses into one gzip member with no name and no time in its header, so that the same input +/// always gives the same bytes. +pub struct Encoder { + out: O, + deflate: bun_zlib::DeflateEncoder, + buffer: Vec, + crc: u32, + size: u32, +} + +/// `deflate()` is shown what is left of a buffer of this size. At level 0 that decides where a +/// stored block ends, so it is part of what keeps the output stable. +const ENCODER_BUFFER: usize = 61440; +const OS_UNKNOWN: u8 = 255; + +impl Encoder { + /// `level` is 0 to 9. + pub fn new(out: O, level: u8) -> Result { + let mut buffer = Vec::new(); + buffer + .try_reserve_exact(ENCODER_BUFFER) + .map_err(|_| Error::OutOfMemory)?; + let extra_flags = match level { + 9 => 2, + 1 => 4, + _ => 0, + }; + buffer.extend_from_slice(&[0x1f, 0x8b, 8, 0, 0, 0, 0, 0, extra_flags, OS_UNKNOWN]); + Ok(Self { + out, + deflate: bun_zlib::DeflateEncoder::new(i32::from(level), RAW_DEFLATE, 8, 0) + .map_err(|_| Error::OutOfMemory)?, + buffer, + crc: 0, + size: 0, + }) + } + + fn drive(&mut self, mut input: &[u8], finishing: bool) -> Result<()> { + loop { + if self.buffer.len() >= ENCODER_BUFFER { + self.out.write(&self.buffer)?; + self.buffer.clear(); + } + if !finishing && input.is_empty() { + return Ok(()); + } + let room = ENCODER_BUFFER.saturating_sub(self.buffer.len()); + let (used, code) = self.deflate.step_limited( + input, + &mut self.buffer, + room, + if finishing { + bun_zlib::FlushValue::Finish + } else { + bun_zlib::FlushValue::NoFlush + }, + ); + input = input.get(used..).unwrap_or_default(); + match code { + ReturnCode::Ok => {} + ReturnCode::StreamEnd => return Ok(()), + ReturnCode::MemError => return Err(Error::OutOfMemory), + _ => return Err(Error::GzipCompressionFailed), + } + } + } + + pub fn finish(mut self) -> Result { + self.drive(&[], true)?; + self.out.write(&self.buffer)?; + let mut trailer = [0u8; 8]; + trailer[..4].copy_from_slice(&self.crc.to_le_bytes()); + trailer[4..].copy_from_slice(&self.size.to_le_bytes()); + self.out.write(&trailer)?; + Ok(self.out) + } +} + +impl crate::writer::Output for Encoder { + fn write(&mut self, bytes: &[u8]) -> Result<()> { + self.crc = crc32_bytes(self.crc, bytes); + self.size = self.size.wrapping_add(bytes.len() as u32); + self.drive(bytes, false) + } +} diff --git a/src/tar/lib.rs b/src/tar/lib.rs index fddbf2d9068f..2faad65b754c 100644 --- a/src/tar/lib.rs +++ b/src/tar/lib.rs @@ -6,12 +6,14 @@ #![forbid(unsafe_code)] #![warn(unused_must_use)] +pub mod compose; pub mod decoder; pub mod error; pub mod gzip; mod header; mod num; pub mod reader; +pub mod writer; pub use decoder::{Decoder, MemoryDecoder}; pub use error::{Error, Result}; diff --git a/src/tar/reader.rs b/src/tar/reader.rs index 23c5347e6111..1a445ba26219 100644 --- a/src/tar/reader.rs +++ b/src/tar/reader.rs @@ -1141,7 +1141,7 @@ impl Reader { let text = until_nul(text); match self.options.charset { Charset::Ascii => strings::first_non_ascii(text).is_none(), - Charset::Utf8 => strings::is_valid_utf8(text), + Charset::Utf8 => crate::compose::converts(text), } } @@ -1413,6 +1413,7 @@ impl Reader { if magic == gnu::MAGIC { self.format = Format::Gnu; if self.entry.path.is_empty() { + self.entry.path_is_utf8 = false; assign(&mut self.entry.path, until_nul(header::name(&h)))?; } if !npm { @@ -1433,6 +1434,7 @@ impl Reader { self.format = Format::Ustar; } if self.entry.path.is_empty() { + self.entry.path_is_utf8 = false; let prefix = until_nul(header::prefix(&h)); assign(&mut self.entry.path, prefix)?; if !prefix.is_empty() && prefix.last() != Some(&b'/') { @@ -1443,6 +1445,7 @@ impl Reader { } else { self.format = Format::Old; if self.entry.path.is_empty() { + self.entry.path_is_utf8 = false; assign(&mut self.entry.path, until_nul(header::name(&h)))?; } } @@ -1505,6 +1508,7 @@ impl Reader { b'1' => { self.entry.link_kind = LinkKind::Hard; if self.entry.link.is_empty() { + self.entry.link_is_utf8 = false; assign(&mut self.entry.link, until_nul(header::linkname(&h)))?; } if self.entry.size > 0 { @@ -1525,6 +1529,7 @@ impl Reader { b'2' => { self.entry.link_kind = LinkKind::Symbolic; if self.entry.link.is_empty() { + self.entry.link_is_utf8 = false; assign(&mut self.entry.link, until_nul(header::linkname(&h)))?; } no_body(&mut self.entry, S_IFLNK); diff --git a/src/tar/writer.rs b/src/tar/writer.rs new file mode 100644 index 000000000000..2ac2ef38f274 --- /dev/null +++ b/src/tar/writer.rs @@ -0,0 +1,463 @@ +//! Writes "restricted pax": plain ustar, with a pax header in front of the entries whose name +//! ustar cannot hold. Only regular files, which is all Bun archives. + +use crate::error::{Error, Result}; +use crate::header::Block; +use bun_core::strings; + +/// Where the bytes go. The size of each write is part of the contract: behind a compressor it +/// decides what the compressed bytes are. +pub trait Output { + fn write(&mut self, bytes: &[u8]) -> Result<()>; +} + +impl Output for Vec { + fn write(&mut self, bytes: &[u8]) -> Result<()> { + self.try_reserve(bytes.len()) + .map_err(|_| Error::OutOfMemory)?; + self.extend_from_slice(bytes); + Ok(()) + } +} + +impl Output for F { + fn write(&mut self, bytes: &[u8]) -> Result<()> { + self(bytes); + Ok(()) + } +} + +/// How a name that is not ASCII is recorded. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Names { + /// POSIX: the bytes as they are, marked `hdrcharset=BINARY`. + Binary, + /// Windows: UTF-8 with its characters composed, and `\` turned into `/`. + Utf8, +} + +#[derive(Debug, Clone, Copy)] +pub struct File<'a> { + /// Ends at its first NUL. + pub path: &'a [u8], + pub size: u64, + pub perm: u32, + pub mtime: i64, +} + +const RECORD_SIZE: u64 = 10240; +/// The last time that fits in eleven octal digits. It is written through a pax record all the +/// same. +const USTAR_MTIME_MAX: i64 = (1 << 33) - 1; +const USTAR_SIZE_END: u64 = 1 << 33; +static ZEROS: [u8; 1024] = [0; 1024]; + +pub struct Writer { + out: O, + names: Names, + written: u64, + /// Of the current entry: the data still owed, and the zeros behind it. + remaining: u64, + padding: u64, +} + +impl Writer { + pub fn new(out: O, names: Names) -> Self { + Self { + out, + names, + written: 0, + remaining: 0, + padding: 0, + } + } + + fn put(&mut self, bytes: &[u8]) -> Result<()> { + self.written = self.written.saturating_add(bytes.len() as u64); + self.out.write(bytes) + } + + fn put_zeros(&mut self, mut count: u64) -> Result<()> { + while count > 0 { + let take = count.min(ZEROS.len() as u64); + self.put(ZEROS.get(..take as usize).unwrap_or_default())?; + count = count.saturating_sub(take); + } + Ok(()) + } + + /// Starts an entry, after finishing the one before it. + pub fn header(&mut self, file: &File<'_>) -> Result<()> { + self.finish_entry()?; + + let given = match strings::index_of_char_usize(file.path, 0) { + Some(end) => file.path.get(..end).unwrap_or_default(), + None => file.path, + }; + let converted; + let (path, binary) = match self.names { + Names::Binary => (given, strings::first_non_ascii(given).is_some()), + Names::Utf8 => match crate::compose::windows_name(given)? { + Some(name) => { + converted = name; + (&converted[..], false) + } + None => (given, true), + }, + }; + + let mut pax: Vec = Vec::new(); + if binary { + add_record(&mut pax, b"hdrcharset", b"BINARY")?; + } + let shortened; + let mut name = path; + if strings::first_non_ascii(path).is_some() || !fits_in_ustar(path) { + add_record(&mut pax, b"path", path)?; + shortened = ustar_entry_name(path, None); + name = &shortened; + } + let mtime_fits = (0..USTAR_MTIME_MAX).contains(&file.mtime); + if !mtime_fits { + let mut digits = itoa_buffer(); + add_record(&mut pax, b"mtime", format_int(&mut digits, file.mtime))?; + } + if file.size >= USTAR_SIZE_END { + let mut digits = itoa_buffer(); + let size = i64::try_from(file.size).unwrap_or(i64::MAX); + add_record(&mut pax, b"size", format_int(&mut digits, size))?; + } + + if !pax.is_empty() { + let block = format_header(&Fields { + name: &pax_attribute_name(name), + perm: file.perm & 0o777, + size: pax.len() as u64, + mtime: file.mtime.clamp(0, USTAR_MTIME_MAX), + typeflag: b'x', + strict: true, + }); + self.put(&block)?; + self.put(&pax)?; + self.put_zeros(padding_for(pax.len() as u64))?; + } + + let block = format_header(&Fields { + name, + perm: file.perm, + size: file.size, + mtime: file.mtime, + typeflag: b'0', + strict: false, + }); + self.put(&block)?; + self.remaining = file.size; + self.padding = padding_for(file.size); + Ok(()) + } + + /// Returns how much was taken. What goes past the size in the header is dropped. + pub fn data(&mut self, bytes: &[u8]) -> Result { + let take = usize::try_from(self.remaining) + .unwrap_or(usize::MAX) + .min(bytes.len()); + if take > 0 { + self.put(bytes.get(..take).unwrap_or_default())?; + self.remaining = self.remaining.saturating_sub(take as u64); + } + Ok(take) + } + + /// Data that was announced and never written is made up with zeros. + pub fn finish_entry(&mut self) -> Result<()> { + let zeros = self.remaining.saturating_add(self.padding); + self.remaining = 0; + self.padding = 0; + self.put_zeros(zeros) + } + + /// Ends the archive. `pad_to_record` fills the last 10 KiB record up, the way tar writes to a + /// tape. + pub fn finish(mut self, pad_to_record: bool) -> Result { + self.finish_entry()?; + self.put_zeros(1024)?; + if pad_to_record { + let over = self.written % RECORD_SIZE; + if over != 0 { + self.put_zeros(RECORD_SIZE.saturating_sub(over))?; + } + } + Ok(self.out) + } +} + +fn padding_for(size: u64) -> u64 { + 0x1ff & size.wrapping_neg() +} + +fn itoa_buffer() -> [u8; 20] { + [0; 20] +} + +fn format_int(buffer: &mut [u8; 20], value: i64) -> &[u8] { + let mut at = buffer.len(); + let mut left = value.unsigned_abs(); + loop { + at = at.saturating_sub(1); + if let Some(slot) = buffer.get_mut(at) { + *slot = b'0'.wrapping_add((left % 10) as u8); + } + left /= 10; + if left == 0 { + break; + } + } + if value < 0 { + at = at.saturating_sub(1); + if let Some(slot) = buffer.get_mut(at) { + *slot = b'-'; + } + } + buffer.get(at..).unwrap_or_default() +} + +/// ` =\n`, where the length counts its own digits too. +fn add_record(pax: &mut Vec, key: &[u8], value: &[u8]) -> Result<()> { + let without_length = key.len().saturating_add(value.len()).saturating_add(3); + let mut digits = 0usize; + let mut next_ten = 1usize; + let mut left = without_length; + while left > 0 { + left /= 10; + digits = digits.saturating_add(1); + next_ten = next_ten.saturating_mul(10); + } + if without_length.saturating_add(digits) >= next_ten { + digits = digits.saturating_add(1); + } + let length = without_length.saturating_add(digits); + pax.try_reserve(length).map_err(|_| Error::OutOfMemory)?; + let mut buffer = itoa_buffer(); + pax.extend_from_slice(format_int( + &mut buffer, + i64::try_from(length).unwrap_or(i64::MAX), + )); + pax.push(b' '); + pax.extend_from_slice(key); + pax.push(b'='); + pax.extend_from_slice(value); + pax.push(b'\n'); + Ok(()) +} + +fn find_slash(path: &[u8], from: usize) -> Option { + let tail = path.get(from..)?; + Some(from.saturating_add(strings::index_of_char_usize(tail, b'/')?)) +} + +/// Where to cut a name longer than 100 bytes into `prefix` and `name`: at the first `/` that +/// leaves a name of at most 100. The prefix may not be empty. +fn split_point(path: &[u8]) -> Option { + let mut at = find_slash(path, path.len().saturating_sub(101))?; + if at == 0 { + at = find_slash(path, 1)?; + } + (at.saturating_add(1) < path.len() && at <= 155).then_some(at) +} + +fn fits_in_ustar(path: &[u8]) -> bool { + path.len() <= 100 || split_point(path).is_some() +} + +fn byte_at(path: &[u8], index: usize) -> u8 { + path.get(index).copied().unwrap_or(0) +} + +/// A stand-in for a name ustar cannot hold, for readers that do not know pax. It keeps as many +/// of the leading directories as fit and the start of the file name. `insert` goes in as one more +/// directory in front of the file name. +fn ustar_entry_name(src: &[u8], insert: Option<&[u8]>) -> Vec { + let insert_length = insert.map_or(0, |name| name.len().saturating_add(2)); + if src.len() < 100 && insert.is_none() { + return src.to_vec(); + } + + let mut need_slash = false; + let mut filename_end = src.len(); + loop { + if filename_end > 0 && byte_at(src, filename_end.wrapping_sub(1)) == b'/' { + filename_end = filename_end.saturating_sub(1); + } else if filename_end > 1 + && byte_at(src, filename_end.wrapping_sub(1)) == b'.' + && byte_at(src, filename_end.wrapping_sub(2)) == b'/' + { + filename_end = filename_end.saturating_sub(2); + } else { + break; + } + need_slash = true; + } + let mut dest = Vec::new(); + if filename_end == 0 { + dest.extend_from_slice(insert.unwrap_or_default()); + dest.push(b'/'); + return dest; + } + + let mut suffix_length = if need_slash { 97usize } else { 98 }; + let mut filename = filename_end.saturating_sub(1); + while filename > 0 && byte_at(src, filename) != b'/' { + filename = filename.saturating_sub(1); + } + if byte_at(src, filename) == b'/' && filename < filename_end.saturating_sub(1) { + filename = filename.saturating_add(1); + } + suffix_length = suffix_length.saturating_sub(insert_length); + filename_end = filename_end.min(filename.saturating_add(suffix_length)); + suffix_length = suffix_length.saturating_sub(filename_end.saturating_sub(filename)); + + // Backs up to a `/`, and then takes it along. + let directory_end = |start: usize, limit: usize| { + let mut end = limit.min(filename).max(start); + while end > start && byte_at(src, end) != b'/' { + end = end.saturating_sub(1); + } + if end < filename && byte_at(src, end) == b'/' { + end = end.saturating_add(1); + } + end + }; + let prefix_end = directory_end(0, 154); + let suffix_end = directory_end(prefix_end, prefix_end.saturating_add(suffix_length)); + + dest.extend_from_slice(src.get(..suffix_end).unwrap_or_default()); + if let Some(insert) = insert { + dest.extend_from_slice(insert); + dest.push(b'/'); + } + dest.extend_from_slice(src.get(filename..filename_end).unwrap_or_default()); + if need_slash { + dest.push(b'/'); + } + dest +} + +/// The name of the pax header's own entry: `dir/PaxHeader/file` for `dir/file`. +fn pax_attribute_name(src: &[u8]) -> Vec { + if src.is_empty() { + return b"PaxHeader/blank".to_vec(); + } + let mut end = src.len(); + loop { + let slash = end > 0 && byte_at(src, end.wrapping_sub(1)) == b'/'; + let slash_dot = end > 1 + && byte_at(src, end.wrapping_sub(1)) == b'.' + && byte_at(src, end.wrapping_sub(2)) == b'/'; + if !slash && !slash_dot { + break; + } + end = end.saturating_sub(1); + } + if end == 0 { + return b"/PaxHeader/rootdir".to_vec(); + } + if end == 1 && src.first() == Some(&b'.') { + return b"PaxHeader/currentdir".to_vec(); + } + ustar_entry_name(src.get(..end).unwrap_or_default(), Some(b"PaxHeader")) +} + +struct Fields<'a> { + name: &'a [u8], + perm: u32, + size: u64, + mtime: i64, + typeflag: u8, + /// Keep every number in octal with its terminator, whatever it takes. + strict: bool, +} + +fn copy_into(block: &mut Block, at: usize, bytes: &[u8]) { + if let Some(dest) = block.get_mut(at..at.saturating_add(bytes.len())) { + dest.copy_from_slice(bytes); + } +} + +/// A value too large for the field fills it with sevens. +fn format_octal(mut value: i64, field: &mut [u8]) { + if value < 0 { + field.fill(b'0'); + return; + } + for slot in field.iter_mut().rev() { + *slot = b'0'.wrapping_add((value & 7) as u8); + value >>= 3; + } + if value != 0 { + field.fill(b'7'); + } +} + +/// `digits` is the field without its terminator and `field` all of it. A number that needs the +/// room overwrites the terminator, and base-256 takes whatever still does not fit. +fn format_number(value: i64, field: &mut [u8], digits: usize, strict: bool) { + if strict { + return format_octal(value, field.get_mut(..digits).unwrap_or_default()); + } + if value >= 0 { + for width in digits..=field.len() { + if width >= 21 || value < 1i64 << width.saturating_mul(3) { + return format_octal(value, field.get_mut(..width).unwrap_or_default()); + } + } + } + let mut left = value; + for slot in field.iter_mut().rev() { + *slot = (left & 0xff) as u8; + left >>= 8; + } + if let Some(first) = field.first_mut() { + *first |= 0x80; + } +} + +fn format_header(fields: &Fields<'_>) -> Block { + let mut h: Block = [0; 512]; + for (at, digits) in [(100, 6), (108, 6), (116, 6), (329, 6), (337, 6)] { + copy_into(&mut h, at, b"000000 ".get(..digits + 1).unwrap_or_default()); + } + copy_into(&mut h, 124, b"00000000000 "); + copy_into(&mut h, 136, b"00000000000 "); + copy_into(&mut h, 148, b" "); + copy_into(&mut h, 257, b"ustar\x0000"); + + if fields.name.len() <= 100 { + copy_into(&mut h, 0, fields.name); + } else if let Some(at) = split_point(fields.name) { + copy_into(&mut h, 345, fields.name.get(..at).unwrap_or_default()); + copy_into( + &mut h, + 0, + fields.name.get(at.saturating_add(1)..).unwrap_or_default(), + ); + } + format_number( + i64::from(fields.perm & 0o7777), + &mut h[100..108], + 6, + fields.strict, + ); + format_number( + i64::try_from(fields.size).unwrap_or(i64::MAX), + &mut h[124..136], + 11, + fields.strict, + ); + format_number(fields.mtime, &mut h[136..147], 11, fields.strict); + h[156] = fields.typeflag; + + let checksum: u32 = h.iter().map(|&b| u32::from(b)).sum(); + h[154] = 0; + format_octal(i64::from(checksum), &mut h[148..154]); + h +} From 15966b0fb53426dd673f115c343046ae9c4130b7 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 21:02:08 +0000 Subject: [PATCH 06/33] Read and write archives with bun_tar Bun.Archive, bun install (buffered and streaming), bun pm pack, bun publish, bun pm diff and bun create now go through bun_tar. src/libarchive becomes src/archive (bun_archive) and keeps only the extraction policy. --- Cargo.lock | 40 +- Cargo.toml | 4 +- src/{libarchive => archive}/Cargo.toml | 4 +- src/{libarchive => archive}/error.rs | 0 src/{libarchive => archive}/lib.rs | 267 ++++----- src/bun_core/env_var.rs | 2 +- src/bun_core/lib.rs | 2 +- src/bun_core/util.rs | 8 +- src/install/Cargo.toml | 2 +- src/install/NetworkTask.rs | 8 +- src/install/TarballStream.rs | 506 +++++------------- src/install/cache_pack.rs | 10 +- src/install/error.rs | 14 +- src/install/extract_tarball.rs | 8 +- src/opaque/lib.rs | 2 +- src/paths/component_iterator.rs | 2 +- src/runtime/Cargo.toml | 2 +- src/runtime/api/Archive.rs | 263 +++------ src/runtime/cli/create_command.rs | 6 +- src/runtime/cli/pack_command.rs | 365 ++++--------- src/runtime/cli/pm_diff_command.rs | 70 +-- src/runtime/cli/publish_command.rs | 59 +- src/runtime/error.rs | 8 +- src/standalone_graph/Cargo.toml | 2 +- src/standalone_graph/StandaloneModuleGraph.rs | 4 +- src/tar/decoder.rs | 49 ++ src/tar/error.rs | 2 + src/tar/reader.rs | 16 + 28 files changed, 634 insertions(+), 1091 deletions(-) rename src/{libarchive => archive}/Cargo.toml (88%) rename src/{libarchive => archive}/error.rs (100%) rename src/{libarchive => archive}/lib.rs (78%) diff --git a/Cargo.lock b/Cargo.lock index 879cb4988525..55b9b51eb1e3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -154,6 +154,23 @@ dependencies = [ "libc", ] +[[package]] +name = "bun_archive" +version = "0.0.0" +dependencies = [ + "bstr", + "bun_alloc", + "bun_collections", + "bun_core", + "bun_errno", + "bun_paths", + "bun_sys", + "bun_tar", + "bun_wyhash", + "scopeguard", + "thiserror", +] + [[package]] name = "bun_ast" version = "0.0.0" @@ -720,6 +737,7 @@ dependencies = [ "bstr", "bun_alloc", "bun_analytics", + "bun_archive", "bun_ast", "bun_base64", "bun_boringssl_sys", @@ -739,7 +757,6 @@ dependencies = [ "bun_io", "bun_js_parser", "bun_js_printer", - "bun_libarchive", "bun_libdeflate_sys", "bun_options_types", "bun_parsers", @@ -957,23 +974,6 @@ dependencies = [ "syn", ] -[[package]] -name = "bun_libarchive" -version = "0.0.0" -dependencies = [ - "bstr", - "bun_alloc", - "bun_collections", - "bun_core", - "bun_errno", - "bun_opaque", - "bun_paths", - "bun_sys", - "bun_wyhash", - "scopeguard", - "thiserror", -] - [[package]] name = "bun_libdeflate_sys" version = "0.0.0" @@ -1466,6 +1466,7 @@ dependencies = [ "bstr", "bun_alloc", "bun_analytics", + "bun_archive", "bun_ast", "bun_ast_jsc", "bun_base64", @@ -1502,7 +1503,6 @@ dependencies = [ "bun_js_parser_jsc", "bun_js_printer", "bun_jsc", - "bun_libarchive", "bun_libdeflate_sys", "bun_libuv_sys", "bun_lint_driver", @@ -1850,6 +1850,7 @@ dependencies = [ "bitflags", "bstr", "bun_alloc", + "bun_archive", "bun_ast", "bun_bundler", "bun_collections", @@ -1858,7 +1859,6 @@ dependencies = [ "bun_errno", "bun_exe_format", "bun_http", - "bun_libarchive", "bun_opaque", "bun_options_types", "bun_parsers", diff --git a/Cargo.toml b/Cargo.toml index 8b30243d5a05..6fe2d8c86d3d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ members = [ "src/install/windows-shim", "src/parsers", "src/io", - "src/libarchive", + "src/archive", "src/md", "src/paths", "src/resolver", @@ -383,7 +383,7 @@ bun_http = { path = "src/http" } bun_install = { path = "src/install" } bun_parsers = { path = "src/parsers" } bun_io = { path = "src/io" } -bun_libarchive = { path = "src/libarchive" } +bun_archive = { path = "src/archive" } bun_md = { path = "src/md" } bun_paths = { path = "src/paths" } bun_resolver = { path = "src/resolver" } diff --git a/src/libarchive/Cargo.toml b/src/archive/Cargo.toml similarity index 88% rename from src/libarchive/Cargo.toml rename to src/archive/Cargo.toml index ab298977e303..8374ce5032cc 100644 --- a/src/libarchive/Cargo.toml +++ b/src/archive/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "bun_libarchive" +name = "bun_archive" version.workspace = true edition.workspace = true @@ -11,7 +11,7 @@ workspace = true [dependencies] thiserror.workspace = true -bun_opaque.workspace = true +bun_tar.workspace = true bstr.workspace = true scopeguard.workspace = true bun_core.workspace = true diff --git a/src/libarchive/error.rs b/src/archive/error.rs similarity index 100% rename from src/libarchive/error.rs rename to src/archive/error.rs diff --git a/src/libarchive/lib.rs b/src/archive/lib.rs similarity index 78% rename from src/libarchive/lib.rs rename to src/archive/lib.rs index 1558a704a378..fda7407f1c81 100644 --- a/src/libarchive/lib.rs +++ b/src/archive/lib.rs @@ -1,8 +1,8 @@ +//! Extracts tar archives to disk. The format itself is `bun_tar`'s. #![warn(unused_must_use)] -use core::ptr; use bun_collections::StringArrayHashMap; -use bun_core::{MutableString, slice_to_nul, strings}; +use bun_core::{MutableString, strings}; use bun_core::{Output, ZStr, slice_as_bytes}; #[cfg(unix)] use bun_paths::PathBuffer; @@ -13,6 +13,64 @@ use bun_wyhash::hash; pub mod error; pub use error::{Error, Result}; +pub use bun_tar as tar; +use bun_tar::{Dialect, Entry, Event, MemoryDecoder, Options}; + +/// The entry's path the way the file system takes it. `buf` is scratch space for Windows. +#[cfg(not(windows))] +pub fn os_path<'a>(entry: &'a Entry, _buf: &'a mut Vec) -> Result<&'a [OSPathChar]> { + Ok(entry.path()) +} +#[cfg(windows)] +pub fn os_path<'a>(entry: &'a Entry, buf: &'a mut Vec) -> Result<&'a [OSPathChar]> { + buf.clear(); + bun_tar::compose::to_utf16(entry.path(), entry.path_is_utf8(), buf) + .map_err(|_| Error::Alloc(bun_alloc::AllocError))?; + Ok(buf) +} + +pub fn entry_kind(entry: &Entry) -> bun_sys::FileKind { + bun_sys::kind_from_mode(entry.filetype() as bun_sys::Mode) +} + +/// A reader for an archive that is all in memory, gzipped or not. +pub fn open_memory(bytes: &[u8], npm: bool) -> Result> { + let dialect = if npm { Dialect::Npm } else { Dialect::General }; + MemoryDecoder::new(bytes, Options::for_host(dialect, true)) + .map_err(|_| Error::Alloc(bun_alloc::AllocError)) +} + +/// Writes the contents of the current entry to `fd`. +pub fn read_data_into_fd( + reader: &mut MemoryDecoder<'_>, + fd: Fd, + strategy: &mut lib::WriteStrategy, +) -> core::result::Result<(), ReadDataError> { + let mut writer = lib::EntryWriter::new(fd); + loop { + match reader.next().map_err(ReadDataError::Archive)? { + Event::Data { offset, bytes } => writer + .write(strategy, offset, bytes) + .map_err(ReadDataError::Sys)?, + Event::EndOfEntry { size } => return writer.finish(size).map_err(ReadDataError::Sys), + Event::NeedInput | Event::Entry(_) | Event::EndOfArchive => return Ok(()), + } + } +} + +fn log_archive_error(path: &[OSPathChar], err: bun_tar::Error) { + Output::err( + "archive error", + "extracting {}: {}", + (bun_core::fmt::fmt_os_path(path, Default::default()), err), + ); +} + +pub enum ReadDataError { + Archive(bun_tar::Error), + Sys(bun_sys::Error), +} + pub mod lib { use super::*; @@ -89,7 +147,7 @@ pub mod lib { Err(_) => { strategy.pwrite = false; bun_core::debug_warn!( - "libarchive: falling back to write() after pwrite() failure", + "archive: falling back to write() after pwrite() failure", ); } } @@ -111,8 +169,7 @@ pub mod lib { Ok(()) } - /// `end` is the offset that libarchive returned with `ARCHIVE_EOF`, which is the length of - /// the file. + /// `end` is the length of the file, which the last write need not have reached. pub fn finish(&mut self, end: i64) -> bun_sys::Maybe<()> { if end > self.end { self.before_hole(); @@ -181,11 +238,10 @@ pub fn first_path_component(path: &[OSPathChar]) -> &[OSPathChar] { #[cfg(unix)] pub fn is_symlink_target_safe( symlink_path: &[u8], - link_target: &ZStr, + link_target_bytes: &[u8], symlink_join_buf: &mut Option, ) -> bool { // Absolute symlink targets are never safe - they could point anywhere - let link_target_bytes = link_target.as_bytes(); if link_target_bytes.is_empty() || link_target_bytes[0] == b'/' { return false; } @@ -453,12 +509,7 @@ impl Archiver { // whatever its size. leaves_root: &mut dyn FnMut(&[u8]) -> bun_sys::Result, ) -> crate::Result<()> { - let mut entry: *mut lib::Entry = ptr::null_mut(); - - // SAFETY: `file_buffer` outlives `stream` (stack-local, dropped at fn exit). - let mut stream = unsafe { BufferReadStream::init(file_buffer) }; - let _ = stream.open_read(false); - let archive = stream.archive; + let mut reader = open_memory(file_buffer, false)?; // Uses the bun_sys directory-fd helpers (open_dir_absolute / open_dir_at). let dir: Fd = 'brk: { @@ -482,28 +533,20 @@ impl Archiver { let mut normalized_buf = bun_paths::path_buffer_pool::get(); 'loop_: loop { - // SAFETY: archive valid for stream lifetime - let r = unsafe { (*archive).read_next_header(&mut entry) }; - - match r { - lib::Result::Eof => break 'loop_, - lib::Result::Retry => continue 'loop_, - lib::Result::Failed | lib::Result::Fatal => { + match reader.next_entry() { + Ok(None) => break 'loop_, + Err(_) => { return Err(crate::Error::Fail); } - _ => { - // do not use the utf8 name there - // it will require us to pull in libiconv - // though we should probably validate the utf8 here nonetheless - // SAFETY: entry was just populated by read_next_header + Ok(Some(entry)) => { #[cfg(not(windows))] - let pathname_bytes = lib::Entry::opaque_ref(entry).pathname().as_bytes(); + let pathname_bytes = entry.path(); // The name `extract_to_dir` writes to. #[cfg(windows)] - let pathname_utf8 = strings::to_utf8_list_with_type( - Vec::new(), - lib::Entry::opaque_ref(entry).pathname_w().as_slice(), - )?; + let pathname_utf8 = { + let mut wide = Vec::new(); + strings::to_utf8_list_with_type(Vec::new(), os_path(entry, &mut wide)?)? + }; #[cfg(windows)] let pathname_bytes: &[u8] = &pathname_utf8; @@ -540,8 +583,7 @@ impl Archiver { } } - // pathname = sliceTo(remaining[..len :0], 0) - let pathname = slice_to_nul(remaining); + let pathname = remaining; if pathname.is_empty() || pathname.len() >= normalized_buf.len() { continue 'loop_; } @@ -561,9 +603,7 @@ impl Archiver { let dirname = strings::trim(bun_paths::dirname_simple(pathname), SEP_STR.as_bytes()); - // SAFETY: entry valid - let size: usize = - usize::try_from(lib::Entry::opaque_ref(entry).size().max(0)).unwrap(); + let size: usize = usize::try_from(entry.size().max(0)).unwrap(); let in_the_way = leaves_root(pathname)? || (size > 0 && match bun_sys::openat_a(dir, pathname, bun_sys::O::WRONLY, 0) { @@ -614,12 +654,8 @@ impl Archiver { appender: &mut A, options: ExtractOptions, ) -> crate::Result { - let mut entry: *mut lib::Entry = ptr::null_mut(); - - // SAFETY: `file_buffer` outlives `stream` (stack-local, dropped at fn exit). - let mut stream = unsafe { BufferReadStream::init(file_buffer) }; - let _ = stream.open_read(options.npm); - let archive = stream.archive; + let mut reader = open_memory(file_buffer, options.npm)?; + let mut os_path_buf: Vec = Vec::new(); let mut count: u32 = 0; let dir_fd = dir; @@ -636,34 +672,17 @@ impl Archiver { let mut write_strategy = lib::WriteStrategy::default(); 'loop_: loop { - // SAFETY: archive valid for stream lifetime - let r = unsafe { (*archive).read_next_header(&mut entry) }; - - match r { - lib::Result::Eof => break 'loop_, - lib::Result::Retry => continue 'loop_, - lib::Result::Failed | lib::Result::Fatal => { + match reader.next_entry() { + Ok(None) => break 'loop_, + Err(_) => { return Err(crate::Error::Fail); } - _ => { - // TODO: - // Due to path separator replacement and other copies that happen internally, libarchive changes the - // storage type of paths on windows to wide character strings. Using `archive_entry_pathname` or `archive_entry_pathname_utf8` - // on an wide character string will return null if there are non-ascii characters. - // (this can be seen by installing @fastify/send, which has a path "@fastify\send\test\fixtures\snow ☃") - // - // Ideally, we find a way to tell libarchive to not convert the strings to wide characters and also to not - // replace path separators. We can do both of these with our own normalization and utf8/utf16 string conversion code. - // SAFETY: entry was just populated by read_next_header - #[cfg(windows)] - let pathname_z = lib::Entry::opaque_ref(entry).pathname_w(); - // SAFETY: entry was just populated by read_next_header - #[cfg(not(windows))] - let pathname_z = lib::Entry::opaque_ref(entry).pathname(); + Ok(Some(entry)) => { + let pathname_z = os_path(entry, &mut os_path_buf)?; if A::HAS_ON_FIRST_DIRECTORY_NAME { if appender.needs_first_dirname() { - let root = first_path_component(&pathname_z[..]); + let root = first_path_component(pathname_z); #[cfg(windows)] { let result = strings::to_utf8_list_with_type(Vec::new(), root)?; @@ -677,8 +696,7 @@ impl Archiver { } } - // SAFETY: entry valid - let kind = bun_sys::kind_from_mode(lib::Entry::opaque_ref(entry).filetype()); + let kind = entry_kind(entry); if options.npm { // - ignore entries other than files (`true` can only be returned if type is file) @@ -696,10 +714,7 @@ impl Archiver { } // strip and normalize the path - // `pathname_z` is `&ZStr` on POSIX (`as_bytes() → &[u8]`) - // and `&WStr` on Windows (`as_slice() → &[u16]`); both - // deref to `&[OSPathChar]`. - let pathname_slice: &[OSPathChar] = &pathname_z[..]; + let pathname_slice: &[OSPathChar] = pathname_z; let mut remaining: &[OSPathChar] = pathname_slice; { let sep: OSPathChar = b'/' as OSPathChar; @@ -730,9 +745,6 @@ impl Archiver { } } } - // pathname = rest.ptr[0..rest.len :0] (NUL is at original buffer end) - // SAFETY: `remaining` is a tail slice of `pathname_z`, which is NUL-terminated - // at its original `.len()`; therefore `remaining[remaining.len()] == 0`. let pathname: &[OSPathChar] = remaining; if pathname.len() >= normalized_buf.len() { @@ -826,8 +838,7 @@ impl Archiver { } #[cfg(not(windows))] { - // SAFETY: entry valid - let mode = directory_mode(lib::Entry::opaque_ref(entry).perm()); + let mode = directory_mode(entry.perm() as bun_sys::Mode); // SAFETY: normalized_buf[path_slice.len()] == 0 (written above), // so path_slice is a NUL-terminated [:0]u8. let path_z: &ZStr = unsafe { @@ -854,8 +865,7 @@ impl Archiver { } } bun_sys::FileKind::SymLink => { - // SAFETY: entry valid - let link_target = lib::Entry::opaque_ref(entry).symlink(); + let link_target = entry.symlink().unwrap_or_default(); #[cfg(unix)] { // Validate that the symlink target doesn't escape the extraction directory. @@ -876,15 +886,13 @@ impl Archiver { Default::default(), ) ), - bun_core::fmt::escape_control_chars( - link_target.as_bytes() - ), + bun_core::fmt::escape_control_chars(link_target), ); } continue; } deferred_symlinks - .push(DeferredSymlink::new(path_slice, link_target.as_bytes())); + .push(DeferredSymlink::new(path_slice, link_target)); } #[cfg(not(unix))] { @@ -899,11 +907,8 @@ impl Archiver { // // we simplify and turn it into `entry.mode || 0o666` because we aren't accepting a umask or fmask option. #[cfg(not(windows))] - let mode: bun_sys::Mode = bun_sys::Mode::try_from( - // SAFETY: entry valid - (lib::Entry::opaque_ref(entry).perm() & 0o777) | 0o666, - ) - .unwrap(); + let mode: bun_sys::Mode = + ((entry.perm() & 0o777) | 0o666) as bun_sys::Mode; let flags = bun_sys::O::WRONLY | bun_sys::O::CREAT | bun_sys::O::TRUNC; @@ -963,10 +968,7 @@ impl Archiver { } }; - // SAFETY: entry valid - let size: usize = - usize::try_from(lib::Entry::opaque_ref(entry).size().max(0)) - .unwrap(); + let size: usize = usize::try_from(entry.size().max(0)).unwrap(); if size > 0 { if let Some(ctx_) = ctx.as_deref_mut() { @@ -979,30 +981,14 @@ impl Archiver { for plucker_ in ctx_.pluckers.iter_mut() { if plucker_.filename_hash == h { plucker_.contents.list.clear(); - // SAFETY: archive valid - let read_ok = unsafe { &*archive }.read_data_to_vec( - size, - &mut plucker_.contents.list, - )?; - if !read_ok { + if let Err(err) = + reader.read_data(size, &mut plucker_.contents.list) + { + if err == bun_tar::Error::OutOfMemory { + return Err(bun_alloc::AllocError.into()); + } if options.log { - // SAFETY: `archive` is the live - // `read_new()` handle this - // extraction loop is iterating. - let archive_error = slice_to_nul( - unsafe { &*archive }.error_string(), - ); - Output::err( - "libarchive error", - "extracting {}: {}", - ( - bun_core::fmt::fmt_os_path( - path_slice, - Default::default(), - ), - bstr::BStr::new(archive_error), - ), - ); + log_archive_error(path_slice, err); } return Err(crate::Error::Fail); } @@ -1013,55 +999,14 @@ impl Archiver { } } } - let mut retries_remaining: u8 = 5; - - 'possibly_retry: while retries_remaining != 0 { - // SAFETY: archive valid - match unsafe { - (*archive).read_data_into_fd(output.fd, &mut write_strategy) - } { - lib::Result::Eof => break 'loop_, - lib::Result::Ok => break 'possibly_retry, - lib::Result::Retry => { - if options.log { - Output::err( - "libarchive error", - "extracting {}, retry {} / {}", - ( - bun_core::fmt::fmt_os_path( - path_slice, - Default::default(), - ), - retries_remaining, - 5, - ), - ); - } - } - _ => { - if options.log { - // SAFETY: `archive` is the live - // `read_new()` handle this - // extraction loop is iterating. - let archive_error = slice_to_nul( - unsafe { &*archive }.error_string(), - ); - Output::err( - "libarchive error", - "extracting {}: {}", - ( - bun_core::fmt::fmt_os_path( - path_slice, - Default::default(), - ), - bstr::BStr::new(archive_error), - ), - ); - } - return Err(crate::Error::Fail); - } + if let Err(err) = + read_data_into_fd(&mut reader, output.fd, &mut write_strategy) + { + if let (true, ReadDataError::Archive(err)) = (options.log, err) + { + log_archive_error(path_slice, err); } - retries_remaining -= 1; + return Err(crate::Error::Fail); } } } diff --git a/src/bun_core/env_var.rs b/src/bun_core/env_var.rs index 78e86cefe1c9..0227e811986c 100644 --- a/src/bun_core/env_var.rs +++ b/src/bun_core/env_var.rs @@ -118,7 +118,7 @@ new!(pub BUN_INSTALL: string, "BUN_INSTALL", {}); new!(pub BUN_INSTALL_BIN: string, "BUN_INSTALL_BIN", {}); new!(pub BUN_INSTALL_GLOBAL_DIR: string, "BUN_INSTALL_GLOBAL_DIR", {}); // Minimum response `Content-Length` (in bytes) for `bun install` to -// stream a tarball directly into libarchive instead of buffering the +// stream a tarball directly into the extractor instead of buffering the // whole body first. Smaller tarballs stay on the buffered path where // the fixed overhead of the resumable state machine isn't worth it. new!(pub BUN_INSTALL_STREAMING_MIN_SIZE: unsigned, "BUN_INSTALL_STREAMING_MIN_SIZE", { default: 2 * 1024 * 1024 }); diff --git a/src/bun_core/lib.rs b/src/bun_core/lib.rs index e7ce6cf189c6..92e4b87b8daa 100644 --- a/src/bun_core/lib.rs +++ b/src/bun_core/lib.rs @@ -700,7 +700,7 @@ pub const unsafe fn container_of(field: *const F, offset: usize) -> *mut P /// Recover a typed `&mut T` from a C-callback's opaque user-data pointer. /// /// This is the canonical spelling for the ubiquitous trampoline pattern where -/// a C library (libarchive, c-ares, uWS, libuv, lol-html, BoringSSL, …) round- +/// a C library (c-ares, uWS, libuv, lol-html, BoringSSL, …) round- /// trips a Rust object through a `void *user_data` slot and hands it back to /// an `extern "C" fn` thunk. Earlier ports open-coded this as /// `unsafe { &mut *ctx.cast::() }` at every site; centralising it here diff --git a/src/bun_core/util.rs b/src/bun_core/util.rs index 2ceeb1857e42..a7ce75bc3c25 100644 --- a/src/bun_core/util.rs +++ b/src/bun_core/util.rs @@ -195,7 +195,7 @@ impl ZStr { } /// Borrow a NUL-terminated FFI C string as `&ZStr`, or `EMPTY` if `p` is /// null. Single audited funnel for the `strlen`-then-`from_raw` shape that - /// previously appeared as ad-hoc local helpers in libarchive / uSockets / + /// previously appeared as ad-hoc local helpers in uSockets / /// HTTPCertError. Returns `&'a ZStr` so the *caller* picks the lifetime. /// /// # Safety @@ -469,8 +469,8 @@ impl WStr { } /// Borrow a NUL-terminated FFI wide string as `&WStr`, or [`EMPTY`] if /// `p` is null. UTF-16 mirror of [`ZStr::from_c_ptr`]; single audited - /// funnel for the `wcslen`-then-`from_raw` shape at libarchive `_w` - /// accessors and Windows path-API ingestion points. + /// funnel for the `wcslen`-then-`from_raw` shape at Windows path-API + /// ingestion points. /// /// # Safety /// If non-null, `p` must point to a NUL-terminated u16 sequence readable @@ -1528,7 +1528,7 @@ pub mod fd { } // ─── FileKind / Mode / kind_from_mode (from bun_sys) ────────────────────── -// Pure S_IFMT arithmetic, no syscalls (libarchive_sys req). +// Pure S_IFMT arithmetic, no syscalls. pub type Mode = u32; // POSIX `mode_t` /// `stat` mode-flag constants and predicates. diff --git a/src/install/Cargo.toml b/src/install/Cargo.toml index a8a45c65c7e7..feaca157d586 100644 --- a/src/install/Cargo.toml +++ b/src/install/Cargo.toml @@ -46,7 +46,7 @@ bun_parsers.workspace = true bun_io.workspace = true bun_js_parser.workspace = true bun_js_printer.workspace = true -bun_libarchive.workspace = true +bun_archive.workspace = true bun_libdeflate_sys.workspace = true bun_ast.workspace = true bun_options_types.workspace = true diff --git a/src/install/NetworkTask.rs b/src/install/NetworkTask.rs index 4c58812b56f1..2b643f56fe24 100644 --- a/src/install/NetworkTask.rs +++ b/src/install/NetworkTask.rs @@ -72,7 +72,7 @@ pub struct NetworkTask { pub(crate) next: bun_threading::Link, /// Producer/consumer buffer that feeds tarball bytes from the HTTP thread - /// to a worker running libarchive. `None` when streaming extraction is + /// to a worker that extracts them. `None` when streaming extraction is /// disabled or this task is not a tarball download. pub(crate) tarball_stream: Option>, /// Extract `Task` pre-created on the main thread so the HTTP thread can @@ -949,10 +949,10 @@ impl NetworkTask { // Tell the HTTP client to invoke `notify` for every body chunk // instead of buffering the whole response. `notify` pushes each // chunk into `tarball_stream`, which schedules a drain task on - // `thread_pool`; the drain task calls into libarchive until it - // reports ARCHIVE_RETRY (out of input), then returns so the + // `thread_pool`; the drain task runs the decoder until it is + // out of input, then returns so the // worker can be reused for other install work. The next chunk - // reschedules it and libarchive — whose state lives on the heap + // reschedules it and the decoder — whose state lives on the heap // — resumes exactly where it stopped. // // The stream itself is created by the caller (see diff --git a/src/install/TarballStream.rs b/src/install/TarballStream.rs index eb913f976b33..237010a31be9 100644 --- a/src/install/TarballStream.rs +++ b/src/install/TarballStream.rs @@ -2,31 +2,27 @@ //! //! The HTTP thread hands each body chunk to `on_chunk`, which appends to a //! small pending buffer and (if not already running) schedules -//! `drain_task` on `PackageManager.thread_pool`. The drain task calls into -//! libarchive to gunzip and untar whatever is available, writing files as -//! their data arrives, until libarchive asks for more compressed bytes -//! than are currently buffered. At that point the read callback returns -//! `ARCHIVE_RETRY`, libarchive propagates it (see the BUN PATCHes in -//! `vendor/libarchive`), and the drain task returns — the worker is -//! released. The next HTTP chunk reschedules the drain task, which calls -//! back into libarchive and resumes exactly where it left off because the -//! `struct archive *`, the gzip inflate state, the partially-read tar -//! header and the open output `bun.FD` all live on the heap in this -//! struct. +//! `drain_task` on `PackageManager.thread_pool`. The drain task feeds +//! whatever is available to a `bun_tar::Decoder`, writing files as their +//! data arrives, until the decoder has consumed everything that is +//! buffered. At that point the drain task returns and the worker is +//! released. The next HTTP chunk reschedules the drain task, which resumes +//! exactly where it left off because the decoder and the open output +//! `bun.FD` live on the heap in this struct. //! //! This lets `bun install` overlap download and extraction on the normal //! resolve thread pool without ever parking a worker on a condvar, and //! without holding the full compressed or decompressed tarball in memory. -use core::ffi::{c_int, c_void}; use core::mem::ManuallyDrop; use core::sync::atomic::{AtomicBool, Ordering}; +use bun_archive::lib; +use bun_archive::tar::{self, Event}; use bun_collections::VecExt; #[cfg(windows)] use bun_core::strings; use bun_core::{self, Output, ZBox, env_var, fmt as bun_fmt}; -use bun_libarchive::lib; use bun_paths::resolve_path::{self, platform}; use bun_paths::{self, OSPathChar, OSPathSliceZ}; #[cfg(not(windows))] @@ -48,17 +44,6 @@ type Task = crate::package_manager_task::Task<'static>; type OSPathZ<'a> = &'a OSPathSliceZ; type OSPathZMut<'a> = &'a mut OSPathSliceZ; -#[derive(Clone, Copy, PartialEq, Eq)] -enum Phase { - /// Call `archive_read_next_header` next. - WantHeader, - /// Currently writing the body of `out`; call - /// `archive_read_data_block` next. - WantData, - /// `archive_read_next_header` returned EOF; we are done. - Done, -} - // `extract_task` / `package_manager` are raw pointers, not // `&'a mut` / `&'a`. This struct is heap-allocated (`heap::alloc`), // crosses threads via `drain_task`, and self-destroys in `finish()`, so a @@ -72,15 +57,15 @@ pub struct TarballStream { mutex: Mutex, /// Compressed .tgz bytes that have arrived from the HTTP thread but have - /// not yet been consumed by libarchive. + /// not yet been moved to `reading`. pending: Vec, /// True once the HTTP thread has delivered the final chunk (or an error). closed: bool, - /// Set if the HTTP request failed mid-stream. Unless libarchive still + /// Set if the HTTP request failed mid-stream. Unless the decoder still /// reached end-of-archive and the digest verifies, this — not - /// libarchive's truncation error — is the failure, and `finish()` hands + /// the decoder's truncation error — is the failure, and `finish()` hands /// the NetworkTask back to be retried as a failed download. http_err: Option, @@ -97,22 +82,15 @@ pub struct TarballStream { // --------------------------------------------------------------------- // Drain-side state (touched only by one drain task at a time) // --------------------------------------------------------------------- - /// Bytes currently being consumed by libarchive. Populated by swapping - /// with `pending` under the mutex so the HTTP thread can keep appending - /// while libarchive decompresses without the lock held. libarchive's - /// read callback hands out `reading[read_pos..]` and advances - /// `read_pos`; the slice must remain valid until the next callback, so - /// we only recycle this buffer on the *following* swap. + /// Bytes currently being decoded. Populated by swapping with `pending` + /// under the mutex so the HTTP thread can keep appending while the + /// decoder works without the lock held. reading: Vec, read_pos: usize, - archive_holds_reading: bool, - archive: Option, - - /// Where we are in the per-entry state machine between drain - /// invocations. libarchive preserves everything else (filter buffers, - /// zlib stream, tar header progress) on its own heap. - phase: Phase, + decoder: Option, + /// The decoder reached the end of the archive. + done: bool, /// Output file for the entry currently being written. `None` while /// between entries or when the current entry is being skipped. @@ -138,13 +116,13 @@ pub struct TarballStream { npm_mode: bool, bytes_received: usize, - /// Compressed bytes handed to libarchive by the read callback so far. + /// Compressed bytes the decoder has consumed so far. bytes_consumed: usize, /// `Content-Length` of the response, when the server sent one. pub(crate) content_length: Option, entry_count: u32, fail: Option, - /// libarchive's error string for `fail == Some(Fail)`. + /// The decoder's message for `fail == Some(Fail)`. fail_detail: Vec, invalid_name: bool, @@ -164,7 +142,7 @@ pub struct TarballStream { impl TarballStream { /// Minimum Content-Length for which the streaming path is used. Below - /// this the whole body is buffered as before; the resumable libarchive + /// this the whole body is buffered as before; the resumable /// state machine is only worth its per-chunk overhead for tarballs that /// would otherwise consume a noticeable amount of memory. pub(crate) fn min_size() -> usize { @@ -250,9 +228,8 @@ impl TarballStream { draining: AtomicBool::new(false), reading: Vec::new(), read_pos: 0, - archive_holds_reading: false, - archive: None, - phase: Phase::WantHeader, + decoder: None, + done: false, out: None, write_strategy: lib::WriteStrategy::default(), dest: None, @@ -279,7 +256,7 @@ impl TarballStream { } /// Called from the HTTP thread for each response-body chunk. Returns - /// without touching the filesystem or libarchive; actual processing is + /// without touching the filesystem or the decoder; actual processing is /// deferred to `drain` on a worker so the HTTP event loop stays /// responsive. /// @@ -347,9 +324,9 @@ impl TarballStream { } } - /// Pull whatever compressed bytes are available into libarchive, writing - /// entries to disk, until libarchive reports `ARCHIVE_RETRY` (out of - /// input — yield) or a terminal state (EOF / error — finish). + /// Feed whatever compressed bytes are available to the decoder, writing + /// entries to disk, until it is out of input (yield) or in a terminal + /// state (EOF / error — finish). /// /// # Safety /// `this` must be the live pointer returned by `init()`. Runs on a @@ -366,8 +343,8 @@ impl TarballStream { // free `*this`; each `return` after it touches nothing. unsafe { loop { - if (*this).fail.is_none() && (*this).phase != Phase::Done { - // Only pull bytes into `reading` while libarchive is still + if (*this).fail.is_none() && !(*this).done { + // Only pull bytes into `reading` while the decoder is still // going to consume them. After EOF/failure `step()` is // never called again, so appending here would let // `reading` grow by one HTTP chunk per wakeup for the @@ -376,7 +353,7 @@ impl TarballStream { if let Err(err) = Self::step(this) { // If the body was cut short by a transport error, - // that is the failure; libarchive's complaint about + // that is the failure; the decoder's complaint about // the truncated input is just its symptom. (*this).mutex.lock(); let http_err = (*this).http_err; @@ -388,11 +365,11 @@ impl TarballStream { (*this).close_output_file(); } - if (*this).fail.is_none() && (*this).phase != Phase::Done { + if (*this).fail.is_none() && !(*this).done { if more { continue; } - // libarchive consumed everything we had. Yield the + // The decoder consumed everything we had. Yield the // worker until the HTTP thread delivers the next // chunk. (*this).mutex.lock(); @@ -408,23 +385,22 @@ impl TarballStream { } } - // Terminal: archive finished or extraction failed. libarchive + // Terminal: archive finished or extraction failed. The decoder // will not be called again, so `reading` is dead — drop it // now rather than carrying its capacity until `finish()`. - // `reading` is drain-local (only the read callback touches - // it, and that runs inside `step()`), so this needs no lock. + // `reading` is drain-local, so this needs no lock. (*this).reading = Vec::new(); (*this).read_pos = 0; (*this).mutex.lock(); - // Hash any bytes that arrived after libarchive hit + // Hash any bytes that arrived after the decoder hit // end-of-archive so the integrity digest covers the full // response (tar zero-padding, gzip footer). Skip this once // an error is recorded — the digest won't be checked anyway. if (*this).fail.is_none() && !(*this).pending.is_empty() { (*this).hasher.update(&(*this).pending); } - // After EOF/failure we stop feeding libarchive but must keep + // After EOF/failure we stop feeding the decoder but must keep // consuming (and discarding) chunks until the HTTP thread // closes the stream; freeing ourselves earlier would let the // next `notify` dereference a dead pointer. @@ -454,14 +430,12 @@ impl TarballStream { } // unsafe } - /// Move any bytes still sitting in `pending` into `reading` so the read - /// callback can hand them to libarchive. Returns true if new bytes were - /// added or the stream is now closed. + /// Move any bytes still sitting in `pending` into `reading` for `step()`. + /// Returns true if new bytes were added or the stream is now closed. /// /// # Safety - /// `this` must be live. Called both from `drain()` and re-entrantly - /// from inside libarchive's read callback (while `step()` is on the - /// stack), so this must NOT materialise `&mut TarballStream` — all + /// `this` must be live. The HTTP thread may be inside `on_chunk()`, so + /// this must NOT materialise `&mut TarballStream` — all /// access is via raw-ptr field projection. Producer fields (`pending`/`closed`) /// are synchronised by `mutex`; drain-side fields (`reading`/ /// `read_pos`/`hasher`) are owned by the single active drain task. @@ -476,7 +450,7 @@ impl TarballStream { return closed; } - // Hash before libarchive sees the bytes so integrity covers exactly + // Hash before the decoder sees the bytes so integrity covers exactly // what came off the socket. (*this).hasher.update(&(*this).pending); @@ -487,13 +461,7 @@ impl TarballStream { core::mem::swap(&mut (*this).reading, &mut (*this).pending); (*this).read_pos = 0; } else { - // libarchive still holds a slice into `reading` (the read - // callback contract keeps the last-returned buffer valid until - // the next call). Appending would realloc and invalidate that - // slice, so instead shift the unconsumed tail down and append - // in place — the callback is not running concurrently with us - // (single drain at a time) and will be re-primed with the new - // base on its next invocation. + // Keep the unconsumed tail in front of the new bytes. let read_pos = (*this).read_pos; (*this).reading.drain_front(read_pos); (*this).read_pos = 0; @@ -505,186 +473,94 @@ impl TarballStream { } // unsafe } - /// Run libarchive until it needs more input (`Retry`) or hits a - /// terminal state. All libarchive state persists on the heap, so - /// returning from here and re-entering later is safe. + /// Run the decoder until it needs more input or hits a terminal state. /// /// # Safety - /// `this` must be live. Takes `*mut Self` (not `&mut self`) because - /// `open_archive()` hands `this` to libarchive as client_data and the - /// read callback dereferences it across MULTIPLE `step()` invocations. - /// A `&mut self` receiver would mint a fresh Unique tag on each call, - /// popping the SharedRW tag the stored client_data pointer carries and - /// leaving the callback with dead provenance (Stacked Borrows UB). - /// Threading the Box-rooted `*mut Self` from `drain()` keeps one - /// provenance alive for the lifetime of the archive. + /// `this` must be live, and the caller the one active drain task. The + /// HTTP thread may be inside `on_chunk()`, so the producer fields are + /// only read under `mutex`. unsafe fn step(this: *mut Self) -> crate::Result<()> { - // SAFETY: see fn-level # Safety — raw-ptr field projection only; no - // `&mut TarballStream` is held across any libarchive call (which may - // re-enter `archive_read_callback` and access `*this` via the same - // provenance). Transient `&mut *this` for `open_destination` / - // `begin_entry` / `write_block` / `finish_output_file` is sound: - // those do not call into libarchive. - unsafe { - if (*this).archive.is_none() { - let head: &[u8] = &(*this).reading; - let gzip = head.starts_with(&[0x1f, 0x8b]); - if head.len() < 2 { - // Telling gzip from tar takes two bytes. Wait for more unless the body ended. - (*this).mutex.lock(); - let closed = (*this).closed; - (*this).mutex.unlock(); - if !closed { + // SAFETY: see fn-level # Safety. Everything but `mutex`, `pending` + // and `closed` is drain-local. + let (mut decoder, eof, reading) = unsafe { + (*this).mutex.lock(); + // Nothing follows `reading` once the body has ended and all of it was taken. + let eof = (*this).closed && (*this).pending.is_empty(); + (*this).mutex.unlock(); + + let decoder = match (*this).decoder.take() { + Some(decoder) => decoder, + None => { + let head: &[u8] = &(*this).reading; + if head.len() < 2 && !eof { + // Telling gzip from tar takes two bytes. return Ok(()); } + // Plain tar is what a .tgz sent with `Content-Encoding: gzip` arrives as, + // because the HTTP client has already decompressed it. + let gzip = head.starts_with(&[0x1f, 0x8b]); + let dialect = if (*this).npm_mode { + tar::Dialect::Npm + } else { + tar::Dialect::General + }; + tar::Decoder::new(tar::Options::for_host(dialect, false), gzip) + .map_err(|_| crate::Error::Alloc(bun_alloc::AllocError))? } - Self::open_archive(this, gzip, (*this).npm_mode)?; - } + }; if (*this).dest.is_none() { (*this).open_destination()?; } + // Both are moved out so that events, which borrow them, can be handed to `&mut self` + // methods. + (decoder, eof, core::mem::take(&mut (*this).reading)) + }; + // SAFETY: the methods called on it touch drain-local fields only. + let result = unsafe { (*this).run_decoder(&mut decoder, &reading, eof) }; + // SAFETY: see above. + unsafe { + (*this).reading = reading; + (*this).decoder = Some(decoder); + } + result + } - // `archive` points to a libarchive heap allocation disjoint from - // `*this`; holding `&lib::Archive` across the loop does not - // alias any access to `*this`. - let archive: &lib::Archive = (*this).archive.as_deref().unwrap(); - - loop { - match (*this).phase { - Phase::Done => return Ok(()), - Phase::WantHeader => { - let mut entry: *mut lib::Entry = core::ptr::null_mut(); - match archive.read_next_header(&mut entry) { - lib::Result::Retry if (*this).archive_holds_reading => continue, - lib::Result::Retry => return Ok(()), - lib::Result::Eof => { - // A body that is not an archive also ends here, because the tar - // reader skips every block that is not a valid header. - if (*this).entry_count == 0 { - (*this).fail_detail = b"archive is empty".to_vec(); - return Err(crate::Error::Fail); - } - (*this).phase = Phase::Done; - return Ok(()); - } - lib::Result::Ok | lib::Result::Warn => { - // libarchive returned OK/WARN with a valid entry - // pointer owned by `archive`; it stays valid until - // the next `read_next_header`. No other Rust - // reference to it exists. - (*this).begin_entry(&mut *entry)?; - } - lib::Result::Failed | lib::Result::Fatal => { - (*this).fail_detail = archive.error_string().to_vec(); - return Err(crate::Error::Fail); - } - } - } - Phase::WantData => { - let mut offset: i64 = 0; - let Some(block) = archive.next(&mut offset) else { - // This is the end of the entry's data, and `offset` is its length. - (*this).finish_output_file(offset)?; - (*this).phase = Phase::WantHeader; - continue; - }; - match block.result { - lib::Result::Retry if !(*this).archive_holds_reading => return Ok(()), - lib::Result::Ok | lib::Result::Warn => { - (*this).write_block(&block)?; - } - _ => { - (*this).fail_detail = archive.error_string().to_vec(); - return Err(crate::Error::Fail); - } - } + fn run_decoder( + &mut self, + decoder: &mut tar::Decoder, + reading: &[u8], + eof: bool, + ) -> crate::Result<()> { + loop { + let (used, event) = match decoder.next(&reading[self.read_pos..], eof) { + Ok(next) => next, + Err(tar::Error::OutOfMemory) => { + return Err(crate::Error::Alloc(bun_alloc::AllocError)); + } + Err(err) => { + self.fail_detail = err.to_string().into_bytes(); + return Err(crate::Error::Fail); + } + }; + self.read_pos += used; + self.bytes_consumed += used; + match event { + Event::NeedInput => return Ok(()), + Event::Entry(entry) => self.begin_entry(entry)?, + Event::Data { offset, bytes } => self.write_block(offset, bytes)?, + Event::EndOfEntry { size } => self.finish_output_file(size)?, + Event::EndOfArchive => { + // A body that is not an archive also ends here, because the tar + // reader skips every block that is not a valid header. + if self.entry_count == 0 { + self.fail_detail = b"archive is empty".to_vec(); + return Err(crate::Error::Fail); } + self.done = true; + return Ok(()); } } - } // unsafe - } - - /// # Safety - /// `this` must be live and rooted at the Box allocation (i.e. the - /// pointer threaded from `drain_callback` → `drain` → `step`, NOT a - /// `&mut self as *mut Self` reborrow). libarchive stores `this` as - /// client_data and `archive_read_callback` dereferences it across the - /// lifetime of the archive — see `step()` # Safety for the provenance - /// requirement. - unsafe fn open_archive(this: *mut Self, gzip: bool, npm: bool) -> crate::Result<()> { - let archive = lib::ReadArchive::new(); - // Bypass bidding entirely. `gzip` is false for a .tgz sent with `Content-Encoding: gzip`, - // because the HTTP client has already decompressed it to a plain tar. - // With patches/libarchive/select-registered-only.patch, - // archive_read_append_filter / archive_read_set_format only select a - // filter/format that was already registered, so register gzip and tar - // first. Tar must also be registered before read_set_options (so the - // option has a slot) and set_format must come after it: libarchive's - // archive_set_format_option() clobbers `a->format` while dispatching, - // and losing the selected format means archive_read_open1() falls - // back to bidding, which fails when the first HTTP chunk is short. - // ARCHIVE_FILTER_GZIP = 1, ARCHIVE_FORMAT_TAR = 0x30000. - if gzip { - let _ = archive.read_support_filter_gzip(); - // SAFETY: archive is a valid non-null handle from read_new(); FFI call has no other preconditions. - if unsafe { lib::archive_read_append_filter(archive.as_mut_ptr(), 1) } != 0 { - return Err(crate::Error::Fail); - } - } - let _ = archive.read_support_format_tar(); - // patches/libarchive/tar-npm-compat.patch adds the `npm` option. - let _ = archive.read_set_options(if npm { - c"npm" - } else { - c"read_concatenated_archives" - }); - // SAFETY: archive is a valid non-null handle from read_new(); FFI call has no other preconditions. - if unsafe { lib::archive_read_set_format(archive.as_mut_ptr(), 0x30000) } != 0 { - return Err(crate::Error::Fail); - } - - // SAFETY: archive is a valid handle; `this` outlives the archive - // (dropped below on failure, otherwise as a field of `*this`). See - // fn-level # Safety for why client_data must be the Box-rooted - // `this` and not a `&mut self`-derived pointer. - let rc_raw: c_int = unsafe { - lib::archive_read_open( - archive.as_mut_ptr(), - this.cast::(), - None, - Some(archive_read_callback), - None, - ) - }; - // PORTING.md §Forbidden: `transmute::` is UB for any value not - // declared as a discriminant. Map known ARCHIVE_* codes explicitly and treat - // anything else as Fatal. - let rc: lib::Result = match rc_raw { - x if x == lib::Result::Ok as c_int => lib::Result::Ok, - x if x == lib::Result::Eof as c_int => lib::Result::Eof, - x if x == lib::Result::Retry as c_int => lib::Result::Retry, - x if x == lib::Result::Warn as c_int => lib::Result::Warn, - x if x == lib::Result::Failed as c_int => lib::Result::Failed, - _ => lib::Result::Fatal, - }; - match rc { - lib::Result::Ok | lib::Result::Warn => {} - lib::Result::Retry => { - // open() runs the filter bidder which we bypassed, but the - // client open path may still probe; treat as transient. - // SAFETY: see fn-level # Safety — raw-ptr field write. - unsafe { (*this).archive = Some(archive) }; - return Ok(()); - } - _ => { - // SAFETY: see fn-level # Safety — raw-ptr field write. - unsafe { (*this).fail_detail = archive.error_string().to_vec() }; - return Err(crate::Error::Fail); - } } - // SAFETY: see fn-level # Safety — raw-ptr field write. - unsafe { (*this).archive = Some(archive) }; - Ok(()) } fn open_destination(&mut self) -> crate::Result<()> { @@ -738,27 +614,25 @@ impl TarballStream { } } - fn write_block(&mut self, block: &lib::Block<'_>) -> crate::Result<()> { + fn write_block(&mut self, offset: i64, bytes: &[u8]) -> crate::Result<()> { let Some(out) = self.out.as_mut() else { return Ok(()); }; - out.write(&mut self.write_strategy, block.offset, block.bytes) + out.write(&mut self.write_strategy, offset, bytes) .map_err(|e| e.to_zig_err().into()) } - /// Process one entry header returned by `read_next_header`. Opens the - /// output file (or creates the directory) and transitions to - /// `WantData` so the next `step()` iteration starts pulling its body. - fn begin_entry(&mut self, entry: &mut lib::Entry) -> crate::Result<()> { - #[cfg(windows)] - let pathname: OSPathZ = entry.pathname_w(); - #[cfg(not(windows))] - let pathname: OSPathZ = entry.pathname(); + /// Process one entry header. Opens the output file (or creates the + /// directory). With `out` left empty, the entry's data is dropped. + fn begin_entry(&mut self, entry: &tar::Entry) -> crate::Result<()> { + self.out = None; + let mut os_path_buf: Vec = Vec::new(); + let pathname: &[OSPathChar] = bun_archive::os_path(entry, &mut os_path_buf)?; if self.want_first_dirname { self.want_first_dirname = false; // GitHub's first entry is the folder `repo-sha/`. A mirror's may be `repo-sha/file`. - let root = bun_libarchive::first_path_component(&pathname[..]); + let root = bun_archive::first_path_component(&pathname[..]); #[cfg(windows)] { let result = strings::to_utf8_list_with_type(Vec::new(), root)?; @@ -777,13 +651,11 @@ impl TarballStream { } } - let kind = bun_sys::kind_from_mode(entry.filetype() as Mode); + let kind = bun_archive::entry_kind(entry); if self.npm_mode && kind != FileKind::File { - // npm tarballs only contain files; matching the libarchive path - // in Archiver.extractToDir we skip everything else. - self.phase = Phase::WantData; - self.out = None; + // npm tarballs only contain files; like Archiver.extractToDir + // we skip everything else. return Ok(()); } @@ -794,8 +666,6 @@ impl TarballStream { .split(|c| *c == ('/' as OSPathChar)) .filter(|s| !s.is_empty()); if tokenizer.next().is_none() { - self.phase = Phase::WantData; - self.out = None; return Ok(()); } // tokenizeScalar.rest() — need byte offset of remainder, not just @@ -812,12 +682,10 @@ impl TarballStream { "Skipping entry with a path longer than the maximum path length: {}\n", bun_fmt::EscapeControlChars(bun_fmt::fmt_os_path(rest, Default::default())), ); - self.phase = Phase::WantData; - self.out = None; return Ok(()); } let normalized = if self.npm_mode { - bun_libarchive::normalize_npm_entry_path(rest, &mut norm_buf[..]) + bun_archive::normalize_npm_entry_path(rest, &mut norm_buf[..]) } else { resolve_path::normalize_buf_t::(rest, &mut norm_buf[..]) }; @@ -827,8 +695,6 @@ impl TarballStream { let path: OSPathZMut = unsafe { OSPathSliceZ::from_raw_mut(norm_buf.as_mut_ptr(), norm_len) }; if path.is_empty() || (path.len() == 1 && path[0] == ('.' as OSPathChar)) { - self.phase = Phase::WantData; - self.out = None; return Ok(()); } // `normalize_buf_t` collapses interior `..` but leaves a leading `..` @@ -841,15 +707,11 @@ impl TarballStream { && path[1] == ('.' as OSPathChar) && (path.len() == 2 || path[2] == bun_paths::SEP as OSPathChar) { - self.phase = Phase::WantData; - self.out = None; return Ok(()); } #[cfg(windows)] { if bun_paths::is_absolute_windows_wtf16(&path[..]) { - self.phase = Phase::WantData; - self.out = None; return Ok(()); } if self.npm_mode { @@ -866,8 +728,6 @@ impl TarballStream { match kind { FileKind::Directory => { make_directory(entry, dest, path, path_slice); - self.phase = Phase::WantData; - self.out = None; } FileKind::File => { #[cfg(windows)] @@ -879,12 +739,8 @@ impl TarballStream { let fd = open_output_file(dest, path, path_slice, mode)?; self.entry_count += 1; self.out = Some(lib::EntryWriter::new(fd)); - self.phase = Phase::WantData; - } - _ => { - self.phase = Phase::WantData; - self.out = None; } + _ => {} } Ok(()) } @@ -1062,7 +918,7 @@ impl TarballStream { if !tarball.skip_verify && tarball.integrity.tag.is_supported() { if !self.hasher.verify() { if let Some(http_err) = http_err { - // libarchive found the end-of-archive marker but the + // The decoder found the end-of-archive marker but the // body still ended early (gzip trailer, tar padding): // the same failed download as above. (*task).err = Some(http_err); @@ -1189,98 +1045,6 @@ fn drain_callback(task: *mut thread_pool::Task) { unsafe { TarballStream::drain(this) }; } -/// libarchive client read callback. Returns whatever compressed bytes -/// are currently buffered in `reading`; if none, returns `ARCHIVE_RETRY` -/// (when more data is still expected) so libarchive unwinds with a -/// resumable status, or `0` (EOF) once the HTTP response is complete. -/// -/// Safe-fn: only ever invoked by libarchive itself (never from Rust), with -/// `ctx`/`out_buffer` it threaded through from `open_archive()`. Every -/// pointer dereference inside carries its own SAFETY justification grounded -/// in that setup, so there is no caller-side precondition to encode in the -/// signature. The fn-pointer still coerces to the binding's expected type. -extern "C" fn archive_read_callback( - _a: *mut lib::Archive, - ctx: *mut c_void, - out_buffer: *mut *const c_void, -) -> lib::la_ssize_t { - // SAFETY: `ctx` is the Box-rooted `*mut TarballStream` threaded from - // `drain()` → `step()` → `open_archive()`; libarchive passes it back - // unchanged. `step()`/`open_archive()` take `*mut Self` (not - // `&mut self`) precisely so this pointer's provenance survives every - // re-entry — see `step()` # Safety. We keep `this` as a raw pointer and - // access fields through it directly; no `&mut TarballStream` is live anywhere on the - // call stack while libarchive runs. All fields touched here (`reading`, - // `read_pos`, `mutex`, `pending`, `closed`, `hasher`) are drain-side / - // mutex-guarded and are not accessed by `step()` across the FFI call - // boundary. - let this: *mut TarballStream = ctx.cast::(); - - // SAFETY: `this` is valid (see above); `reading`/`read_pos` are owned by - // the single active drain task. - unsafe { - // Explicit `&` on the `Vec` field (no implicit autoref through the - // raw-ptr deref) for `Index::index`. - let remaining = &(&(*this).reading)[(*this).read_pos..]; - if !remaining.is_empty() { - *out_buffer = remaining.as_ptr().cast(); - (*this).read_pos = (*this).reading.len(); - (*this).bytes_consumed += remaining.len(); - (*this).archive_holds_reading = true; - return lib::la_ssize_t::try_from(remaining.len()).expect("int cast"); - } - (*this).archive_holds_reading = false; - } - - // No data left in `reading`. Check for more under the lock — - // libarchive may have called us more than once for a single - // `step()` (e.g. gzip header + first deflate block), and `on_chunk` - // might have landed a fresh chunk in the meantime. - // SAFETY: `mutex`/`pending`/`closed` accessed via raw ptr; producer side - // is synchronised by the mutex itself. - let (has_pending, closed) = unsafe { - (*this).mutex.lock(); - let r = (!(*this).pending.is_empty(), (*this).closed); - (*this).mutex.unlock(); - r - }; - - if has_pending { - // Pull the new bytes into `reading` and retry the read. We are - // the only consumer of `reading`/`read_pos`, and `take_pending` - // only touches producer state under the same mutex. - // SAFETY: `take_pending` takes `*mut Self` and accesses fields via - // raw-ptr projection, never forming `&mut TarballStream`; `step()` - // holds no `&mut TarballStream` across the libarchive call that - // re-entered us. - unsafe { - let _ = TarballStream::take_pending(this); - // Explicit `&` on the `Vec` field (no implicit autoref through - // the raw-ptr deref) for `Index::index`. - let again = &(&(*this).reading)[(*this).read_pos..]; - if !again.is_empty() { - *out_buffer = again.as_ptr().cast(); - (*this).read_pos = (*this).reading.len(); - (*this).bytes_consumed += again.len(); - (*this).archive_holds_reading = true; - return lib::la_ssize_t::try_from(again.len()).expect("int cast"); - } - } - } - - if closed { - // SAFETY: out_buffer is a valid out-param; ptr is unused when len==0. - unsafe { *out_buffer = this.cast() }; - return 0; - } - - // Tell libarchive to unwind with a resumable status. The BUN PATCHes - // in vendor/libarchive make every layer (filter_ahead → gzip → tar) - // preserve its state and propagate ARCHIVE_RETRY to our `step()` - // loop, which then returns so this worker can be reused. - lib::Result::Retry as lib::la_ssize_t -} - fn open_output_file( dest_fd: Fd, path: OSPathZ, @@ -1325,7 +1089,7 @@ fn open_output_file( } } -fn make_directory(entry: &mut lib::Entry, dest_fd: Fd, path: OSPathZ, path_slice: &[OSPathChar]) { +fn make_directory(entry: &tar::Entry, dest_fd: Fd, path: OSPathZ, path_slice: &[OSPathChar]) { #[cfg(windows)] { let _ = bun_sys::make_path::make_path::(Dir::borrow(&dest_fd), &path[..]); @@ -1333,7 +1097,11 @@ fn make_directory(entry: &mut lib::Entry, dest_fd: Fd, path: OSPathZ, path_slice } #[cfg(not(windows))] { - match bun_sys::mkdirat_z(dest_fd, path, bun_libarchive::directory_mode(entry.perm())) { + match bun_sys::mkdirat_z( + dest_fd, + path, + bun_archive::directory_mode(entry.perm() as Mode), + ) { Ok(()) => {} Err(e) => match e.get_errno() { bun_sys::E::EEXIST | bun_sys::E::ENOTDIR => {} diff --git a/src/install/cache_pack.rs b/src/install/cache_pack.rs index 97db80fb8c99..2b60ecf3da5d 100644 --- a/src/install/cache_pack.rs +++ b/src/install/cache_pack.rs @@ -498,7 +498,7 @@ fn pack_entry( let len = sys::readlinkat(entry.dir, entry.basename, &mut target_buf[..])?; let target = ZStr::from_buf(&target_buf[..], len); if len > MAX_PACK_PATH_LEN - || !bun_libarchive::is_symlink_target_safe(rel, target, &mut join_buf) + || !bun_archive::is_symlink_target_safe(rel, target, &mut join_buf) { return Err(refuse( "cache entry has symlink that may point outside it", @@ -749,7 +749,7 @@ fn unpack_package( let mut target: Vec = Vec::new(); // Symlinks are created last, so that no path can go through one while files are being written. #[cfg(unix)] - let mut links: Vec = Vec::new(); + let mut links: Vec = Vec::new(); #[cfg(unix)] let mut join_buf = None; @@ -804,7 +804,7 @@ fn unpack_package( { let target = ZStr::from_slice_with_nul(&target); if strings::contains_char(target.as_bytes(), 0) - || !bun_libarchive::is_symlink_target_safe( + || !bun_archive::is_symlink_target_safe( rel.as_bytes(), target, &mut join_buf, @@ -812,7 +812,7 @@ fn unpack_package( { return Err(r.refuse("unsafe symlink target", rel.as_bytes())); } - links.push(bun_libarchive::DeferredSymlink::new( + links.push(bun_archive::DeferredSymlink::new( rel.as_bytes(), target.as_bytes(), )); @@ -829,7 +829,7 @@ fn unpack_package( } #[cfg(unix)] - if bun_libarchive::create_deferred_symlinks(dir.fd(), &links, false) as usize != links.len() { + if bun_archive::create_deferred_symlinks(dir.fd(), &links, false) as usize != links.len() { return Err(r.refuse("symlink collides with another entry", name.as_bytes())); } // Windows requires the directory to be closed before it's renamed. diff --git a/src/install/error.rs b/src/install/error.rs index 6501a1d207fb..7c77823fd7d0 100644 --- a/src/install/error.rs +++ b/src/install/error.rs @@ -396,14 +396,14 @@ impl From for Error { } } -impl From for Error { - fn from(e: bun_libarchive::Error) -> Self { +impl From for Error { + fn from(e: bun_archive::Error) -> Self { match e { - bun_libarchive::Error::Fail => Self::Fail, - bun_libarchive::Error::Sys(s) => Self::Sys(s), - bun_libarchive::Error::Alloc(a) => Self::Alloc(a), - bun_libarchive::Error::MakeLibUvOwned(_) => Self::SystemFdQuotaExceeded, - bun_libarchive::Error::Paths(p) => Self::Paths(p), + bun_archive::Error::Fail => Self::Fail, + bun_archive::Error::Sys(s) => Self::Sys(s), + bun_archive::Error::Alloc(a) => Self::Alloc(a), + bun_archive::Error::MakeLibUvOwned(_) => Self::SystemFdQuotaExceeded, + bun_archive::Error::Paths(p) => Self::Paths(p), } } } diff --git a/src/install/extract_tarball.rs b/src/install/extract_tarball.rs index 8d377c76dd3f..e095baf20cfd 100644 --- a/src/install/extract_tarball.rs +++ b/src/install/extract_tarball.rs @@ -9,13 +9,13 @@ use bun_paths::{self as path, PathBuffer}; use bun_semver::Version; use bun_sys::{self as sys, Dir, Fd}; +use bun_archive::{ArchiveAppender, ExtractOptions}; use bun_install::install::{self as Install, DependencyID, ExtractData}; use bun_install::integrity::Integrity; use bun_install::npm::{self as Npm}; use bun_install::package_manager_real::PackageManager; use bun_install::package_manager_real::directories; use bun_install::resolution::{Resolution, Tag as ResolutionTag}; -use bun_libarchive::{ArchiveAppender, ExtractOptions}; use bun_resolver::fs::FileSystem; use crate::cache_publish::{self, StagedEntry}; @@ -291,7 +291,7 @@ impl ExtractTarball { } }; - use bun_libarchive::Archiver; + use bun_archive::Archiver; let mut zlib_pool = Npm::Registry::BodyPool::get(); zlib_pool.reset(); // `defer Npm.Registry.BodyPool.release(zlib_pool)` → PoolGuard's Drop releases. @@ -302,7 +302,7 @@ impl ExtractTarball { 0 }; - // libarchive gunzips on the fly (`BufferReadStream::open_read`), + // `extract_to_dir` gunzips on the fly, // so hand it the compressed bytes and never buffer the full tar. // Small tarballs still try libdeflate first for speed; the gzip // ISIZE trailer (size mod 2^32) is only trusted when small. @@ -329,7 +329,7 @@ impl ExtractTarball { libdeflate::Encoding::Gzip, ); // libdeflate only reads the first gzip member, so a tarball with more - // than one member is left to libarchive. + // than one member is left to `extract_to_dir`. if result.status == libdeflate::Status::Success && result.read == tgz_bytes.len() { diff --git a/src/opaque/lib.rs b/src/opaque/lib.rs index 246284c5fb2a..e63b38750d22 100644 --- a/src/opaque/lib.rs +++ b/src/opaque/lib.rs @@ -339,7 +339,7 @@ pub mod ffi { /// excluding) the first NUL. Single audited funnel for the hand-rolled /// `while *p.add(n) != 0 { n += 1 }` loop that appeared at every /// `LPCWSTR` / `char16_t*` ingestion point (Windows path APIs, N-API - /// `napi_create_string_utf16`, libarchive `_w` accessors, env-block + /// `napi_create_string_utf16`, env-block /// scan). Adds a `debug_assert!(!p.is_null())` — same precondition as /// `CStr::from_ptr`. /// diff --git a/src/paths/component_iterator.rs b/src/paths/component_iterator.rs index 48b7632b2ff7..a76d4085be71 100644 --- a/src/paths/component_iterator.rs +++ b/src/paths/component_iterator.rs @@ -6,7 +6,7 @@ //! This is the single source of truth for "split a path into prefix slices and //! never yield the Windows root (`C:`, `C:\`, `\\server\share\`) as a component". //! Replaces the four hand-rolled copies in `bun_sys` (posix+windows), -//! `bun_libarchive` (u16) and `bun` (`make_path`, which already called the +//! `bun_archive` (u16) and `bun` (`make_path`, which already called the //! `component_iterator` free fn below before it existed). use crate::PathChar; diff --git a/src/runtime/Cargo.toml b/src/runtime/Cargo.toml index a1e652b8d6f3..6dd7d8045cec 100644 --- a/src/runtime/Cargo.toml +++ b/src/runtime/Cargo.toml @@ -70,7 +70,7 @@ bun_format_conformance.workspace = true bun_sema_baselines.workspace = true bun_js_printer.workspace = true bun_jsc.workspace = true -bun_libarchive.workspace = true +bun_archive.workspace = true bun_libdeflate_sys.workspace = true bun_ast.workspace = true bun_ast_jsc.workspace = true diff --git a/src/runtime/api/Archive.rs b/src/runtime/api/Archive.rs index cf70e4fff35b..725978a8dfb6 100644 --- a/src/runtime/api/Archive.rs +++ b/src/runtime/api/Archive.rs @@ -1,22 +1,18 @@ -//! `Bun.Archive` — tar/tgz pack + extract over libarchive. - -use std::ffi::CString; +//! `Bun.Archive` — tar/tgz pack + extract. use crate::webcore::Blob; use crate::webcore::BlobExt as _; use crate::webcore::blob::Store; -use bun_core::{self, EncodedSlice, Output, Utf8Bytes, ZBox, strings}; +use bun_archive::tar; +use bun_core::{self, Output, Utf8Bytes, ZBox, strings}; use bun_glob as glob; use bun_jsc::bindgen::{JsLocal, JsScope}; use bun_jsc::{self as jsc, JSGlobalObject, JSMap, JSPromise, JSPromiseStrong, JSValue, JsResult}; -use bun_jsc::{EncodedSliceJsc as _, StringJsc as _, SysErrorJsc as _}; -use bun_libarchive as libarchive; +use bun_jsc::{StringJsc as _, SysErrorJsc as _}; use bun_ptr::RefPtr; use bun_sys::{self, Fd, FdDirExt as _, FdExt as _, Mode}; -/// libarchive `AE_IFREG` (== `S_IFREG`). The Rust `bun_libarchive::lib` port -/// does not yet expose `FileType`, so mirror the constant locally. -const FILETYPE_REGULAR: u32 = 0o100000; +const FILETYPE_REGULAR: u32 = tar::reader::S_IFREG; /// Compression options for the archive #[derive(Clone, Copy, Default)] @@ -94,40 +90,25 @@ impl Archive { } } -/// Configure archive for reading tar/tar.gz -fn configure_archive_reader(archive: &libarchive::lib::Archive) { - let _ = archive.read_support_format_tar(); - let _ = archive.read_support_format_gnutar(); - let _ = archive.read_support_filter_gzip(); - let _ = archive.read_set_options(c"read_concatenated_archives"); -} - -/// Entry pathname as owned UTF-8 bytes. libarchive on Windows keeps a -/// charset-converted name (every pax `path=`) only in the wide-string slot; -/// `archive_entry_pathname` lossily narrows that through the "C" locale. +/// Entry pathname as owned UTF-8 bytes, from the UTF-16 name the entry has on Windows. #[cfg(windows)] -fn entry_pathname_utf8(entry: &libarchive::lib::Entry) -> Result, bun_alloc::AllocError> { - bun_core::strings::to_utf8_list_with_type(Vec::new(), entry.pathname_w().as_slice()) +fn entry_pathname_utf8(entry: &tar::Entry) -> Result, bun_alloc::AllocError> { + let mut wide = Vec::new(); + let path = bun_archive::os_path(entry, &mut wide).map_err(|_| bun_alloc::AllocError)?; + bun_core::strings::to_utf8_list_with_type(Vec::new(), path) } /// Count the number of files in an archive fn count_files_in_archive(data: &[u8]) -> u32 { - use libarchive::lib; - let archive = lib::ReadArchive::new(); - configure_archive_reader(&archive); - - if archive.read_open_memory(data) != lib::Result::Ok { + let Ok(mut reader) = bun_archive::open_memory(data, false) else { return 0; - } - + }; let mut count: u32 = 0; - let mut entry: *mut lib::Entry = core::ptr::null_mut(); - while archive.read_next_header(&mut entry).succeeded() { - if lib::Entry::opaque_ref(entry).filetype() == FILETYPE_REGULAR { + while let Ok(Some(entry)) = reader.next_entry() { + if entry.filetype() == FILETYPE_REGULAR { count += 1; } } - count } @@ -250,45 +231,28 @@ fn blob_from_js(value: JSValue) -> Option<&'static Blob> { /// Shared helper that builds tarball bytes from a JS object fn build_tarball_from_object(global: &JSGlobalObject, obj: JSValue) -> JsResult> { - use libarchive::lib; - let Some(js_obj) = obj.get_object() else { return Err(global.throw_invalid_arguments(format_args!("Expected an object"))); }; - // Set up archive first - let mut growing_buffer = lib::GrowingBuffer::init(); - // errdefer growing_buffer.deinit() — handled by Drop on Vec - - let archive = lib::WriteArchive::new(); - let archive_ref: &lib::Archive = &archive; - - if archive_ref.write_set_format_pax_restricted() != lib::Result::Ok { - return Err(global.throw_invalid_arguments(format_args!( - "Failed to create tarball: ArchiveFormatError" - ))); - } - - // `archive` is a live `archive_write_new()` handle (see `Archive::write_new` - // above); `growing_buffer` is stack-local and outlives all callback invocations - // (the archive is closed before this fn returns). - let open_rc = lib::archive_write_open2( - &archive, - (&raw mut growing_buffer).cast(), - Some(lib::GrowingBuffer::open_callback), - Some(lib::GrowingBuffer::write_callback), - Some(lib::GrowingBuffer::close_callback), - None, - ); - if open_rc != 0 { - return Err(global - .throw_invalid_arguments(format_args!("Failed to create tarball: ArchiveOpenError"))); - } - - let entry = lib::OwnedEntry::new(); - let entry_ref: &lib::Entry = &entry; + let names = if cfg!(windows) { + tar::writer::Names::Utf8 + } else { + tar::writer::Names::Binary + }; + let mut writer = tar::writer::Writer::new(Vec::::new(), names); + let fail = |what: &str, err: tar::Error| { + global.throw_invalid_arguments(format_args!( + "Failed to create tarball: {}", + if err == tar::Error::OutOfMemory { + "OutOfMemory" + } else { + what + } + )) + }; - let now_secs: isize = isize::try_from(bun_core::time::milli_timestamp() / 1000).unwrap_or(0); + let now_secs: i64 = bun_core::time::milli_timestamp() / 1000; // Iterate over object properties and write directly to archive let iter = jsc::JSPropertyIterator::init( @@ -305,61 +269,32 @@ fn build_tarball_from_object(global: &JSGlobalObject, obj: JSValue) -> JsResult< continue; } - // Get the key as a null-terminated string let key_slice = key.to_utf8(); - let key_str = ZBox::from_vec_with_nul(key_slice.slice().to_vec()); // Get data - use view for Blob/ArrayBuffer, convert for strings let mut array_buffer = None; let data_slice = get_entry_data(global, value, &mut array_buffer)?; - - // Write entry to archive let data = data_slice.slice(); - let _ = entry_ref.clear(); - // Same platform split as `pack_command::add_archive_entry`: the process - // locale is always "C", so libarchive's locale-keyed pax writer is only - // lossless with raw bytes on POSIX and with the UTF-8 form on Windows. - #[cfg(windows)] - entry_ref.set_pathname_utf8(key_str.as_zstr()); - #[cfg(not(windows))] - entry_ref.set_pathname(key_str.as_zstr()); - entry_ref.set_size(i64::try_from(data.len()).expect("int cast")); - entry_ref.set_filetype(FILETYPE_REGULAR); - entry_ref.set_perm(0o644); - entry_ref.set_mtime(now_secs, 0); - - // `Warn` means the header was still written (libarchive fell back to a - // per-entry binary hdrcharset for a name its locale machinery could not - // convert); only `Failed`/`Fatal` mean no header was produced. - if !archive_ref.write_header(entry_ref).succeeded() { - return Err(global.throw_invalid_arguments(format_args!( - "Failed to create tarball: ArchiveHeaderError" - ))); - } - if archive_ref.write_data(data) < 0 { - return Err(global.throw_invalid_arguments(format_args!( - "Failed to create tarball: ArchiveWriteError" - ))); - } - if archive_ref.write_finish_entry() != lib::Result::Ok { - return Err(global.throw_invalid_arguments(format_args!( - "Failed to create tarball: ArchiveFinishEntryError" - ))); - } - } - if archive_ref.write_close() != lib::Result::Ok { - return Err(global - .throw_invalid_arguments(format_args!("Failed to create tarball: ArchiveCloseError"))); + writer + .header(&tar::writer::File { + path: key_slice.slice(), + size: data.len() as u64, + perm: 0o644, + mtime: now_secs, + }) + .map_err(|err| fail("ArchiveHeaderError", err))?; + writer + .data(data) + .map_err(|err| fail("ArchiveWriteError", err))?; + writer + .finish_entry() + .map_err(|err| fail("ArchiveFinishEntryError", err))?; } - match growing_buffer.to_owned_slice() { - Ok(v) => Ok(v), - Err(_) => { - Err(global - .throw_invalid_arguments(format_args!("Failed to create tarball: OutOfMemory"))) - } - } + writer + .finish(true) + .map_err(|err| fail("ArchiveCloseError", err)) } fn get_entry_data<'a>( @@ -717,12 +652,12 @@ impl ExtractContext { } // Otherwise use the fast path without filtering - let count = match libarchive::Archiver::extract_to_disk( + let count = match bun_archive::Archiver::extract_to_disk( self.store.shared_view(), &self.path, None, &mut (), - libarchive::ExtractOptions { + bun_archive::ExtractOptions { depth_to_skip: 0, close_handles: true, log: false, @@ -969,7 +904,7 @@ enum FilesError { enum FilesResult { Success(FileEntryList), - LibarchiveErr(CString), + ArchiveErr(tar::Error), Err(FilesError), } @@ -982,44 +917,29 @@ pub(crate) struct FilesContext { } impl FilesContext { - fn read_error(archive: &libarchive::lib::Archive) -> FilesResult { - let err_str = archive.error_string(); - match CString::new(err_str) { - Ok(err) if !err_str.is_empty() => FilesResult::LibarchiveErr(err), - _ => FilesResult::Err(FilesError::ReadError), - } - } - fn do_run(&mut self) -> Result { - use libarchive::lib; - let archive = lib::ReadArchive::new(); - configure_archive_reader(&archive); - - if archive.read_open_memory(self.store.shared_view()) != lib::Result::Ok { - return Ok(Self::read_error(&archive)); - } + let read_error = |err: tar::Error| match err { + tar::Error::OutOfMemory => Err(bun_alloc::AllocError), + err => Ok(FilesResult::ArchiveErr(err)), + }; + let mut reader = bun_archive::open_memory(self.store.shared_view(), false) + .map_err(|_| bun_alloc::AllocError)?; let mut entries: FileEntryList = Vec::new(); - // errdefer freeEntries(&entries) — handled by Drop on `entries` - let mut entry: *mut lib::Entry = core::ptr::null_mut(); loop { - match archive.read_next_header(&mut entry) { - lib::Result::Eof => break, - lib::Result::Retry => continue, - lib::Result::Failed | lib::Result::Fatal => return Ok(Self::read_error(&archive)), - lib::Result::Ok | lib::Result::Warn => {} - } - let entry_ref = lib::Entry::opaque_ref(entry); + let entry_ref = match reader.next_entry() { + Ok(None) => break, + Ok(Some(entry)) => entry, + Err(err) => return read_error(err), + }; if entry_ref.filetype() != FILETYPE_REGULAR { continue; } - // POSIX: the raw header/pax bytes; the locale-converting - // `archive_entry_pathname_utf8` returns NULL for every non-ASCII - // name in the "C" locale, which would key the Map by "". + // POSIX: the raw header/pax bytes. #[cfg(not(windows))] - let pathname = entry_ref.pathname().as_bytes(); + let pathname = entry_ref.path(); #[cfg(windows)] let pathname_owned = entry_pathname_utf8(entry_ref)?; #[cfg(windows)] @@ -1034,14 +954,13 @@ impl FilesContext { let size: usize = usize::try_from(entry_ref.size().max(0)).expect("int cast"); let mtime: i64 = entry_ref.mtime(); + let path_copy: Box<[u8]> = Box::from(pathname); + let mut data: Vec = Vec::new(); - if !archive.read_data_to_vec(size, &mut data)? { - return Ok(Self::read_error(&archive)); + if let Err(err) = reader.read_data(size, &mut data) { + return read_error(err); } - let path_copy: Box<[u8]> = Box::from(pathname); - // errdefer free(path_copy) — handled by Drop - entries.push(FileEntry { path: path_copy, data, @@ -1089,8 +1008,8 @@ impl TaskContext for FilesContext { Ok(PromiseResult::Resolve(map)) } - FilesResult::LibarchiveErr(err_msg) => Ok(PromiseResult::Reject( - EncodedSlice::utf8(err_msg.to_bytes()).to_error_instance(global), + FilesResult::ArchiveErr(err) => Ok(PromiseResult::Reject( + global.create_error_instance(format_args!("{err}")), )), FilesResult::Err(e) => Ok(PromiseResult::Reject( global.create_error_instance(format_args!("{}", <&'static str>::from(&*e))), @@ -1247,13 +1166,8 @@ fn extract_to_disk_filtered( root: &[u8], glob_patterns: Option<&[GlobPattern]>, ) -> crate::Result { - use libarchive::lib; - let archive = lib::ReadArchive::new(); - configure_archive_reader(&archive); - - if archive.read_open_memory(file_buffer) != lib::Result::Ok { - return Err(crate::Error::ReadError); - } + use bun_archive::lib; + let mut reader = bun_archive::open_memory(file_buffer, false)?; // Open/create target directory using bun.sys let cwd = Fd::cwd(); @@ -1279,28 +1193,22 @@ fn extract_to_disk_filtered( let _dir_close = bun_sys::CloseOnDrop::new(dir_fd); let mut count: u32 = 0; - let mut entry: *mut lib::Entry = core::ptr::null_mut(); let mut write_strategy = lib::WriteStrategy::default(); loop { - match archive.read_next_header(&mut entry) { - lib::Result::Eof => break, - lib::Result::Retry => continue, - lib::Result::Failed | lib::Result::Fatal => return Err(crate::Error::ReadError), - lib::Result::Ok | lib::Result::Warn => {} - } - let entry_ref = lib::Entry::opaque_ref(entry); + let entry_ref = match reader.next_entry() { + Ok(None) => break, + Ok(Some(entry)) => entry, + Err(_) => return Err(crate::Error::ReadError), + }; // Same platform split as `FilesContext::do_run`; see `entry_pathname_utf8`. #[cfg(not(windows))] - let raw_pathname_z = entry_ref.pathname(); + let raw_pathname = entry_ref.path(); #[cfg(windows)] - let raw_pathname_zbox = ZBox::from_vec_with_nul( - entry_pathname_utf8(entry_ref) - .map_err(|_| crate::Error::Alloc(bun_alloc::AllocError))?, - ); + let raw_pathname_owned = entry_pathname_utf8(entry_ref) + .map_err(|_| crate::Error::Alloc(bun_alloc::AllocError))?; #[cfg(windows)] - let raw_pathname_z = raw_pathname_zbox.as_zstr(); - let raw_pathname = raw_pathname_z.as_bytes(); + let raw_pathname: &[u8] = &raw_pathname_owned; let mut normalized_buf = bun_paths::path_buffer_pool::get(); if raw_pathname.len() >= normalized_buf.len() { @@ -1325,8 +1233,7 @@ fn extract_to_disk_filtered( } } - let filetype = entry_ref.filetype(); - let kind = bun_sys::kind_from_mode(filetype); + let kind = bun_archive::entry_kind(entry_ref); match kind { bun_sys::FileKind::Directory => { @@ -1373,7 +1280,8 @@ fn extract_to_disk_filtered( }; let write_success = size == 0 - || archive.read_data_into_fd(file_fd, &mut write_strategy) == lib::Result::Ok; + || bun_archive::read_data_into_fd(&mut reader, file_fd, &mut write_strategy) + .is_ok(); let _ = file_fd.close(); if !write_success { @@ -1384,7 +1292,8 @@ fn extract_to_disk_filtered( count += 1; } bun_sys::FileKind::SymLink => { - let link_target_z = entry_ref.symlink(); + let link_target = ZBox::from_bytes(entry_ref.symlink().unwrap_or_default()); + let link_target_z = link_target.as_zstr(); // Validate symlink target is also safe if !is_safe_path(link_target_z.as_bytes()) { continue; diff --git a/src/runtime/cli/create_command.rs b/src/runtime/cli/create_command.rs index 9d53acf0954b..ecd97d3afce5 100644 --- a/src/runtime/cli/create_command.rs +++ b/src/runtime/cli/create_command.rs @@ -4,6 +4,7 @@ use std::io::Write as _; use crate::api::bun_process::Status; use crate::api::bun_process::sync as spawn_sync; +use bun_archive::{Archiver, archiver}; use bun_clap as clap; use bun_core::Progress::{Node as ProgressNode, Progress}; use bun_core::{Global, Output, pretty_error, pretty_errorln}; @@ -11,7 +12,6 @@ use bun_core::{MutableString, strings}; use bun_dotenv as DotEnv; use bun_http as HTTP; use bun_js_printer as JSPrinter; -use bun_libarchive::{Archiver, archiver}; use bun_parsers::json as JSON; use bun_paths::{OSPathSlice, PathBuffer}; use bun_resolver::fs; @@ -549,8 +549,8 @@ impl CreateCommand { struct OverwriteListAppender { buf: Vec, } - impl bun_libarchive::ArchiveAppender for OverwriteListAppender { - fn append(&mut self, path: &[u8]) -> Result<&[u8], bun_libarchive::Error> { + impl bun_archive::ArchiveAppender for OverwriteListAppender { + fn append(&mut self, path: &[u8]) -> Result<&[u8], bun_archive::Error> { self.buf.clear(); self.buf.extend_from_slice(path); Ok(&self.buf) diff --git a/src/runtime/cli/pack_command.rs b/src/runtime/cli/pack_command.rs index 8a0deea22074..645ff710825e 100644 --- a/src/runtime/cli/pack_command.rs +++ b/src/runtime/cli/pack_command.rs @@ -5,6 +5,7 @@ use std::io::Write as _; use crate::cli::Command; use crate::cli::publish_command as Publish; use bun_alloc::AllocError; +use bun_archive::tar; use bun_ast::{E, Expr, ExprData}; use bun_collections::StringHashMap; use bun_core::{Global, Output, Progress, fmt as bun_fmt}; @@ -14,10 +15,6 @@ use bun_install::package_manager::LogLevel; use bun_install::package_manager::workspace_manifests::WorkspaceManifests; use bun_install::package_manager::workspace_package_json_cache as WorkspacePackageJSONCache; use bun_js_printer as js_printer; -use bun_libarchive::lib::{ - Archive, Entry as ArchiveEntry, GrowingBuffer, Result as ArchiveResult, WriteArchive, - archive_write_open2, -}; use bun_parsers::json as JSON; use bun_paths::{self as path, SEP_STR}; use bun_ptr::cow_slice::CowSlice; @@ -1662,18 +1659,7 @@ fn fail_file(err: impl bun_core::output::ErrName, action: &str, path: &[u8]) -> Global::crash(); } -fn expect_archive_ok(archive: &Archive, status: ArchiveResult, action: &str) { - if matches!( - status, - ArchiveResult::Failed | ArchiveResult::Fatal | ArchiveResult::Warn - ) { - Output::err_generic( - "failed to {}: {}", - (action, bstr::BStr::new(archive.error_string())), - ); - Global::crash(); - } -} +type TarballWriter<'a> = tar::writer::Writer>; /// `Some` exactly when `for_publish`. #[inline(never)] @@ -1894,53 +1880,18 @@ pub(crate) fn pack( })); } - let archive = WriteArchive::new(); - - expect_archive_ok( - &archive, - archive.write_set_format_pax_restricted(), - "set archive format", - ); - expect_archive_ok( - &archive, - archive.write_add_filter_gzip(), - "set archive compression to gzip", - ); - // default is 9 // https://github.com/npm/cli/blob/ec105f400281a5bfd17885de1ea3d54d0c231b27/node_modules/pacote/lib/util/tar-create-options.js#L12 - let compression_level: &[u8] = ctx.manager.options.pack_gzip_level.unwrap_or(b"9"); - match archive.write_set_filter_option( - None, - bun_core::zstr!("compression-level"), - &ZBox::from_bytes(compression_level), - ) { - ArchiveResult::Failed | ArchiveResult::Fatal | ArchiveResult::Warn => { + let compression_level: u8 = match ctx.manager.options.pack_gzip_level.unwrap_or(b"9") { + &[digit @ b'0'..=b'9'] => digit - b'0', + level => { Output::err_generic( "compression level must be between 0 and 9, received {}", - format_args!("{}", bstr::BStr::new(compression_level)), + format_args!("{}", bstr::BStr::new(level)), ); Global::crash(); } - _ => {} - } - - expect_archive_ok( - &archive, - archive.write_set_filter_option(None, bun_core::zstr!("os"), bun_core::zstr!("Unknown")), - "set os to `Unknown`", - ); - expect_archive_ok( - &archive, - archive.write_set_options(bun_core::zstr!("gzip:!timestamp")), - "unset gzip timestamp option", - ); - // Same as `archive_write_open_filename` set for a file, so the output stays byte-identical. - expect_archive_ok( - &archive, - archive.write_set_bytes_in_last_block(1), - "set archive block padding", - ); + }; // `bun publish` uploads the bytes from memory. Only `bun pm pack` writes a file. let mut dest_buf = bun_paths::path_buffer_pool::get(); @@ -1978,34 +1929,25 @@ pub(crate) fn pack( fail_write_tarball(err, dest); } }; - let mut tarball_buffer = GrowingBuffer::forward_to(&mut write_tarball); - // Rebound here so that it is dropped before `tarball_buffer`. Closing the archive, which its - // `Drop` also does, writes into that buffer. - let archive = archive; - - if archive_write_open2( - &archive, - (&raw mut tarball_buffer).cast(), - Some(GrowingBuffer::open_callback), - Some(GrowingBuffer::write_callback), - Some(GrowingBuffer::close_callback), - None, - ) != 0 - { - Output::err_generic( - "failed to open archive for writing: {}", - format_args!("{}", bstr::BStr::new(archive.error_string())), - ); - Global::crash(); - } + let names = if cfg!(windows) { + tar::writer::Names::Utf8 + } else { + tar::writer::Names::Binary + }; + let mut writer: TarballWriter<'_> = tar::writer::Writer::new( + tar::gzip::Encoder::new( + &mut write_tarball as &mut dyn FnMut(&[u8]), + compression_level, + ) + .unwrap_or_else(|err| archive_write_failed(err)), + names, + ); // append removed items from `pack_queue` with their file size let mut pack_list: PackList = Vec::new(); let mut read_buf = vec![0u8; 512 * 1024]; - let mut entry = ArchiveEntry::new2(&archive); - { let mut progress = Progress::Progress::default(); let mut node: Option<&mut Progress::Node> = None; @@ -2016,7 +1958,7 @@ pub(crate) fn pack( node = Some(started); } - entry = archive_package_json(&mut ctx, &archive, entry, &root_dir, &edited_package_json); + archive_package_json(&mut ctx, &mut writer, &root_dir, &edited_package_json); if let Some(node) = node.as_mut() { node.complete_one(); } @@ -2046,14 +1988,13 @@ pub(crate) fn pack( size: usize::try_from(stat.st_size).expect("int cast"), }); - entry = add_archive_entry( + add_archive_entry( &mut ctx, fd, &stat, &item.path, &mut read_buf, - &archive, - entry, + &mut writer, &bins, None, ); @@ -2071,14 +2012,13 @@ pub(crate) fn pack( .stat() .unwrap_or_else(|err| fail_file(err, "stat", path)); - entry = add_archive_entry( + add_archive_entry( &mut ctx, file.handle, &stat, &item.path, &mut read_buf, - &archive, - entry, + &mut writer, &bins, item.edited.as_deref(), ); @@ -2093,17 +2033,11 @@ pub(crate) fn pack( } } - ArchiveEntry::opaque_ref(entry).free(); - - match archive.write_close() { - ArchiveResult::Failed | ArchiveResult::Fatal | ArchiveResult::Warn => { - archive_write_failed(&archive); - } - _ => {} + // The archive ends right behind its two end-of-archive blocks, with no padding to a full record. + if let Err(err) = writer.finish(false).and_then(|encoder| encoder.finish()) { + archive_write_failed(err); } - drop(archive); - let mut shasum: [u8; sha::SHA1::DIGEST] = [0; sha::SHA1::DIGEST]; sha1.r#final(&mut shasum); @@ -2441,20 +2375,16 @@ fn write_tarball_in_place(abs_tarball_dest: &ZStr, tarball_bytes: &[u8], is_type } #[cold] -fn archive_write_failed(archive: &Archive) -> ! { - Output::err_generic( - "failed to write tarball: {}", - format_args!("{}", bstr::BStr::new(archive.error_string())), - ); +fn archive_write_failed(err: tar::Error) -> ! { + Output::err_generic("failed to write tarball: {}", format_args!("{err}")); Global::crash(); } -/// `archive_write_data` returns the byte count, or a negative status when the write fails. -fn write_archive_data(archive: &Archive, data: &[u8]) -> usize { - match usize::try_from(archive.write_data(data)) { - Ok(written) => written, - Err(_) => archive_write_failed(archive), - } +/// Returns how much was taken, which stops at the size in the entry's header. +fn write_archive_data(writer: &mut TarballWriter<'_>, data: &[u8]) -> usize { + writer + .data(data) + .unwrap_or_else(|err| archive_write_failed(err)) } fn stat_package_json(root_dir: &Dir) -> bun_sys::Stat { @@ -2468,39 +2398,24 @@ fn stat_package_json(root_dir: &Dir) -> bun_sys::Stat { fn archive_package_json( ctx: &mut Context<'_>, - archive: &Archive, - entry: *mut ArchiveEntry, + writer: &mut TarballWriter<'_>, root_dir: &Dir, edited_package_json: &[u8], -) -> *mut ArchiveEntry { - // `entry` is the same pointer after `.clear()`. - let entry = ArchiveEntry::opaque_ref(entry); +) { let stat = stat_package_json(root_dir); - entry.set_pathname(bun_core::zstr!("package/package.json")); - entry.set_size(i64::try_from(edited_package_json.len()).expect("int cast")); - // https://github.com/libarchive/libarchive/blob/898dc8319355b7e985f68a9819f182aaed61b53a/libarchive/archive_entry.h#L185 - entry.set_filetype(0o100000); - entry.set_perm(stat.st_mode as bun_sys::Mode); - // '1985-10-26T08:15:00.000Z' - // https://github.com/npm/cli/blob/ec105f400281a5bfd17885de1ea3d54d0c231b27/node_modules/pacote/lib/util/tar-create-options.js#L28 - entry.set_mtime(499162500, 0); - - match archive.write_header(entry) { - ArchiveResult::Fatal => archive_write_failed(archive), - ArchiveResult::Failed | ArchiveResult::Warn => { - Output::err_generic( - "failed to write tarball header: {}", - format_args!("{}", bstr::BStr::new(archive.error_string())), - ); - Global::crash(); - } - _ => {} - } - - ctx.stats.unpacked_size += write_archive_data(archive, edited_package_json); + writer + .header(&tar::writer::File { + path: b"package/package.json", + size: edited_package_json.len() as u64, + perm: stat.st_mode as u32, + // '1985-10-26T08:15:00.000Z' + // https://github.com/npm/cli/blob/ec105f400281a5bfd17885de1ea3d54d0c231b27/node_modules/pacote/lib/util/tar-create-options.js#L28 + mtime: 499162500, + }) + .unwrap_or_else(|err| archive_write_failed(err)); - entry.clear() + ctx.stats.unpacked_size += write_archive_data(writer, edited_package_json); } fn add_archive_entry( @@ -2509,52 +2424,33 @@ fn add_archive_entry( stat: &bun_sys::Stat, filename: &ZStr, read_buf: &mut [u8], - archive: &Archive, - entry: *mut ArchiveEntry, + writer: &mut TarballWriter<'_>, bins: &[BinInfo], // If set, these bytes are archived instead of the contents of `file`. edited: Option<&[u8]>, -) -> *mut ArchiveEntry { - // `entry` is the same pointer after `.clear()`. - let entry = ArchiveEntry::opaque_ref(entry); - let pathname = [b"package/".as_slice(), filename.as_bytes_with_nul()].concat(); - let pathname = ZBox::from_vec_with_nul(pathname); - #[cfg(windows)] - entry.set_pathname_utf8(&pathname); - #[cfg(not(windows))] - entry.set_pathname(&pathname); - - entry.set_size(edited.map_or(stat.st_size as i64, |edited| edited.len() as i64)); - - // https://github.com/libarchive/libarchive/blob/898dc8319355b7e985f68a9819f182aaed61b53a/libarchive/archive_entry.h#L185 - entry.set_filetype(0o100000); +) { + let pathname = [b"package/".as_slice(), filename.as_bytes()].concat(); - let mut perm: bun_sys::Mode = stat.st_mode as bun_sys::Mode; + let mut perm = stat.st_mode as u32; // https://github.com/npm/cli/blob/ec105f400281a5bfd17885de1ea3d54d0c231b27/node_modules/pacote/lib/util/tar-create-options.js#L20 if is_package_bin(bins, filename.as_bytes()) { perm |= 0o111; } - entry.set_perm(perm); - // '1985-10-26T08:15:00.000Z' - // https://github.com/npm/cli/blob/ec105f400281a5bfd17885de1ea3d54d0c231b27/node_modules/pacote/lib/util/tar-create-options.js#L28 - entry.set_mtime(499162500, 0); - - match archive.write_header(entry) { - ArchiveResult::Fatal => archive_write_failed(archive), - ArchiveResult::Failed => { - Output::err_generic( - "failed to write tarball header: {}", - format_args!("{}", bstr::BStr::new(archive.error_string())), - ); - Global::crash(); - } - _ => {} - } + writer + .header(&tar::writer::File { + path: &pathname, + size: edited.map_or(stat.st_size as u64, |edited| edited.len() as u64), + perm, + // '1985-10-26T08:15:00.000Z' + // https://github.com/npm/cli/blob/ec105f400281a5bfd17885de1ea3d54d0c231b27/node_modules/pacote/lib/util/tar-create-options.js#L28 + mtime: 499162500, + }) + .unwrap_or_else(|err| archive_write_failed(err)); if let Some(edited) = edited { - ctx.stats.unpacked_size += write_archive_data(archive, edited); - return entry.clear(); + ctx.stats.unpacked_size += write_archive_data(writer, edited); + return; } let mut written: usize = 0; @@ -2566,12 +2462,12 @@ fn add_archive_entry( }; // The size of each write reaches the compressor, so it is part of what keeps the bytes stable. for chunk in read_buf[..read].chunks(8192) { - written += write_archive_data(archive, chunk); + written += write_archive_data(writer, chunk); } } ctx.stats.unpacked_size += written; - // libarchive drops bytes past the size in the header, and pads an entry that is too short with + // The writer drops bytes past the size in the header, and pads an entry that is too short with // zeros, so a file that shrank has to be caught here. if (written as i64) < stat.st_size as i64 { Output::err_generic( @@ -2580,8 +2476,6 @@ fn add_archive_entry( ); Global::crash(); } - - entry.clear() } /// What replaces a `workspace:` or `catalog:` specifier in the tarball's package.json. @@ -3213,100 +3107,55 @@ pub(crate) mod bindings { } let mut entries_info: Vec = Vec::new(); - let archive = Archive::opaque_ref(Archive::read_new()); - let check = |status: ArchiveResult, action: &str| -> JsResult<()> { - match status { - ArchiveResult::Failed | ArchiveResult::Fatal | ArchiveResult::Warn => Err(global - .throw(format_args!( - "failed to {}: {}", - action, - bstr::BStr::new(archive.error_string()) - ))), - _ => Ok(()), - } - }; - - check(archive.read_support_format_tar(), "support tar")?; - check(archive.read_support_format_gnutar(), "support gnutar")?; - check( - archive.read_support_filter_gzip(), - "support gzip compression", - )?; - check( - archive.read_set_options(c"read_concatenated_archives"), - "set read_concatenated_archives option", - )?; - check(archive.read_open_memory(&tarball), "open archive in memory")?; - - let mut archive_entry: *mut ArchiveEntry = core::ptr::null_mut(); - let mut header_status = archive.read_next_header(&mut archive_entry); - + let mut reader = bun_core::handle_oom( + bun_archive::open_memory(&tarball, false).map_err(|_| bun_alloc::AllocError), + ); let mut read_buf: Vec = Vec::new(); - while header_status != ArchiveResult::Eof { - match header_status { - ArchiveResult::Eof => unreachable!(), - ArchiveResult::Retry => { - header_status = archive.read_next_header(&mut archive_entry); - continue; + loop { + let entry = match reader.next_entry() { + Ok(None) => break, + Ok(Some(entry)) => entry, + Err(err) => { + return Err(global.throw(format_args!("failed to read archive header: {err}"))); } - ArchiveResult::Failed | ArchiveResult::Fatal => { + }; + #[cfg(windows)] + let pathname_string = { + let mut wide = Vec::new(); + let pathname_w = bun_core::handle_oom( + bun_archive::os_path(entry, &mut wide).map_err(|_| bun_alloc::AllocError), + ); + let result = + bun_core::handle_oom(strings::to_utf8_list_with_type(Vec::new(), pathname_w)); + BunString::clone_utf8(&result) + }; + #[cfg(not(windows))] + let pathname_string = BunString::clone_utf8(entry.path()); + + let kind = bun_archive::entry_kind(entry); + let size: usize = usize::try_from(entry.size()).expect("int cast"); + let mut entry_info = EntryInfo { + pathname: pathname_string, + kind, + perm: entry.perm() as bun_sys::Mode, + contents: None, + }; + + if kind == bun_sys::FileKind::File { + if let Err(err) = reader.read_data(size, &mut read_buf) { + let pathname_utf8 = entry_info.pathname.to_utf8(); return Err(global.throw(format_args!( - "failed to read archive header: {}", - bstr::BStr::new(archive.error_string()), + "failed to read archive entry \"{}\": {err}", + bstr::BStr::new(pathname_utf8.slice()), ))); } - _ => { - let archive_entry_ref = ArchiveEntry::opaque_mut(archive_entry); - #[cfg(windows)] - let pathname_string = { - let pathname_w = archive_entry_ref.pathname_w(); - let result = bun_core::handle_oom(strings::to_utf8_list_with_type( - Vec::new(), - pathname_w, - )); - BunString::clone_utf8(&result) - }; - #[cfg(not(windows))] - let pathname_string = BunString::clone_utf8(archive_entry_ref.pathname()); - - let kind = bun_sys::kind_from_mode(archive_entry_ref.filetype()); - let perm = archive_entry_ref.perm(); - - let mut entry_info = EntryInfo { - pathname: pathname_string, - kind, - perm, - contents: None, - }; - - if kind == bun_sys::FileKind::File { - let size: usize = - usize::try_from(archive_entry_ref.size()).expect("int cast"); - read_buf.resize(size, 0); - - let read = archive.read_data(&mut read_buf); - if read < 0 { - let pathname_utf8 = entry_info.pathname.to_utf8(); - return Err(global.throw(format_args!( - "failed to read archive entry \"{}\": {}", - bstr::BStr::new(pathname_utf8.slice()), - bstr::BStr::new(archive.error_string()), - ))); - } - read_buf.truncate(usize::try_from(read).expect("int cast")); - entry_info.contents = Some(BunString::clone_utf8(&read_buf)); - read_buf.clear(); - } - - entries_info.push(entry_info); - } + entry_info.contents = Some(BunString::clone_utf8(&read_buf)); + read_buf.clear(); } - header_status = archive.read_next_header(&mut archive_entry); - } - check(archive.read_close(), "close read archive")?; - check(archive.read_free(), "close read archive")?; + entries_info.push(entry_info); + } let entries = JSArray::create_empty(global, entries_info.len())?; diff --git a/src/runtime/cli/pm_diff_command.rs b/src/runtime/cli/pm_diff_command.rs index 087d77f5a1cc..913465f44c3b 100644 --- a/src/runtime/cli/pm_diff_command.rs +++ b/src/runtime/cli/pm_diff_command.rs @@ -5,6 +5,7 @@ use std::io::Write as _; use bstr::BStr; use bun_alloc::Arena as Bump; +use bun_archive::tar; use bun_collections::VecExt as _; use bun_core::MutableString; use bun_core::fmt as bun_fmt; @@ -19,7 +20,6 @@ use bun_install::npm::{self, PackageManifest}; use bun_install::package_manager::options::Do; use bun_install::package_manager::workspace_manifests::WorkspaceManifests; use bun_install::{PackageManager, resolution}; -use bun_libarchive::lib::{ArchiveIterator, IteratorResult as ArchiveIterResult}; use bun_sys::{Fd, FdExt as _, dir_iterator as DirIterator}; use bun_url::URL; @@ -531,36 +531,35 @@ fn materialize( } fn read_tarball_into(bytes: &[u8], tree: &mut Tree) -> Result<(), crate::Error> { - let mut iter = match ArchiveIterator::init(bytes) { - ArchiveIterResult::Result(it) => it, - ArchiveIterResult::Err { message, .. } => { - Status::clear(); - Output::err_generic("{}: {}", (BStr::new(&tree.label), BStr::new(message))); - Global::exit(1); - } - }; + let mut reader = bun_archive::open_memory(bytes, false)?; loop { - let next = match iter.next() { - ArchiveIterResult::Result(Some(n)) => n, - ArchiveIterResult::Result(None) => break, - ArchiveIterResult::Err { message, .. } => { + let entry = match reader.next_entry() { + Ok(Some(entry)) => entry, + Ok(None) => break, + Err(err) => { Status::clear(); - Output::err_generic("{}: {}", (BStr::new(&tree.label), BStr::new(message))); + let message = match err { + tar::Error::UnrecognizedFormat => "failed to read tarball", + _ => "failed to read archive header", + }; + Output::err_generic("{}: {}", (BStr::new(&tree.label), message)); Global::exit(1); } }; - if next.kind != bun_sys::FileKind::File { + if bun_archive::entry_kind(entry) != bun_sys::FileKind::File { continue; } - let perm = next.entry().perm() & 0o777; - // libarchive returns NULL from the narrow accessor for non-ASCII names on Windows; go through UTF-16 there. + let perm = (entry.perm() & 0o777) as bun_sys::Mode; + let size = usize::try_from(entry.size().max(0)).expect("int cast"); #[cfg(windows)] - let wide_path = - strings::to_utf8_list_with_type(Vec::new(), next.entry().pathname_w().as_slice())?; + let wide_path = { + let mut wide = Vec::new(); + strings::to_utf8_list_with_type(Vec::new(), bun_archive::os_path(entry, &mut wide)?)? + }; #[cfg(windows)] let path = wide_path.as_slice(); #[cfg(not(windows))] - let path = next.entry().pathname().as_bytes(); + let path = entry.path(); // npm tarballs root everything under one folder (usually `package/`). let rel = match strings::index_of_char(path, b'/') { Some(i) => &path[i as usize + 1..], @@ -569,21 +568,26 @@ fn read_tarball_into(bytes: &[u8], tree: &mut Tree) -> Result<(), crate::Error> if rel.is_empty() { continue; } - let data = match iter.read_entry_data(&next)? { - ArchiveIterResult::Result(d) => d, - ArchiveIterResult::Err { message, .. } => { - Status::clear(); - Output::err_generic( - "{}: {}: {}", - (BStr::new(&tree.label), BStr::new(rel), BStr::new(message)), - ); - Global::exit(1); + let rel = rel.to_vec(); + let mut data = Vec::new(); + if let Err(err) = reader.read_data(size, &mut data) { + if err == tar::Error::OutOfMemory { + return Err(bun_alloc::AllocError.into()); } - }; - tree.modes.insert(rel.to_vec(), perm); - tree.files.insert(rel.to_vec(), data.into_vec()); + Status::clear(); + Output::err_generic( + "{}: {}: {}", + ( + BStr::new(&tree.label), + BStr::new(&rel), + "failed to read archive data", + ), + ); + Global::exit(1); + } + tree.modes.insert(rel.clone(), perm); + tree.files.insert(rel, data); } - let _ = iter.close(); Ok(()) } diff --git a/src/runtime/cli/publish_command.rs b/src/runtime/cli/publish_command.rs index fedb790237ec..ce86aad4298d 100644 --- a/src/runtime/cli/publish_command.rs +++ b/src/runtime/cli/publish_command.rs @@ -3,6 +3,7 @@ use std::io::Write as _; use crate::Error; use bun_alloc::AllocError; +use bun_archive::tar; use bun_ast::{E, Expr, G}; use bun_core::MutableString; use bun_core::fmt as bun_fmt; @@ -12,7 +13,6 @@ use bun_http as http; use bun_install::package_manager::options::PublishTarget; use bun_install::package_manager::workspace_manifests; use bun_install::{self as install, Npm, PackageManager, Subcommand}; -use bun_libarchive::lib::{Archive, ArchiveIterator, IteratorResult as ArchiveIterResult}; use bun_parsers::json as json_mod; use bun_paths as path; use bun_paths::resolve_path::{join_abs_string_buf_checked, normalize_buf_z_spill}; @@ -71,20 +71,15 @@ fn is_readme_filename>(name: &[T]) -> bool { && (name.len() == README.len() || name[README.len()].into() == u32::from(b'.')) } -fn archive_ok(result: ArchiveIterResult) -> T { - match result { - ArchiveIterResult::Result(value) => value, - ArchiveIterResult::Err { archive, message } => { - Output::err_generic( - "{}: {}", - ( - bstr::BStr::new(message), - bstr::BStr::new(Archive::opaque_ref(archive).error_string()), - ), - ); - Global::crash(); - } - } +fn archive_ok(result: Result, message: &str) -> T { + result.unwrap_or_else(|err| { + let message = match err { + tar::Error::UnrecognizedFormat => "failed to read tarball", + _ => message, + }; + Output::err_generic("{}: {}", (message, err)); + Global::crash(); + }) } /// `E::String` erases the lifetime of its bytes, so these have to outlive the printing of the AST. @@ -172,25 +167,24 @@ impl Context { let mut maybe_readme: Option = None; let mut files: Vec> = Vec::new(); - let mut iter = archive_ok(ArchiveIterator::init(&tarball_bytes)); + let mut reader = + bun_archive::open_memory(&tarball_bytes, false).map_err(|_| PackError::OutOfMemory)?; + let mut os_path_buf: Vec = Vec::new(); let mut unpacked_size: usize = 0; let mut total_files: usize = 0; Output::print(format_args!("\n")); - while let Some(next) = archive_ok(iter.next()) { - // SAFETY: `next.entry` is valid until the next `iter.next()` call. - let entry = unsafe { &*next.entry }; - #[cfg(windows)] - let pathname: &[OSPathChar] = entry.pathname_w().as_slice(); - #[cfg(not(windows))] - let pathname: &[OSPathChar] = entry.pathname().as_bytes(); + while let Some(entry) = archive_ok(reader.next_entry(), "failed to read archive header") { + let pathname: &[OSPathChar] = bun_archive::os_path(entry, &mut os_path_buf) + .map_err(|_| PackError::OutOfMemory)?; + let kind = bun_archive::entry_kind(entry); let size = usize::try_from(entry.size().max(0)).expect("int cast"); unpacked_size += size; - total_files += usize::from(next.kind == FileKind::File); + total_files += usize::from(kind == FileKind::File); // this is option `strip: 1` (npm expects a `package/` prefix for all paths) let stripped = (pathname.iter().position(|&c| bun_paths::is_sep_any_t(c))) @@ -214,7 +208,7 @@ impl Context { ); let Some(filename) = stripped else { continue }; - if next.kind != FileKind::File { + if kind != FileKind::File { continue; } #[cfg(not(windows))] @@ -228,19 +222,26 @@ impl Context { if maybe_package_json_contents.is_none() && strings::eql_case_insensitive_t(filename, b"package.json") { - maybe_package_json_contents = Some(archive_ok(iter.read_entry_data(&next)?)); + let mut contents = Vec::new(); + archive_ok( + reader.read_data(size, &mut contents), + "failed to read archive data", + ); + maybe_package_json_contents = Some(contents.into_boxed_slice()); } else if maybe_readme.is_none() && is_readme_filename(filename) { - let contents = archive_ok(iter.read_entry_data(&next)?).into_vec(); #[cfg(not(windows))] let filename: Vec = filename.to_vec(); #[cfg(windows)] let filename: Vec = strings::to_utf8_alloc(filename); + let mut contents = Vec::new(); + archive_ok( + reader.read_data(size, &mut contents), + "failed to read archive data", + ); maybe_readme = Some(ReadmeInfo { filename, contents }); } } - archive_ok(iter.close()); - let Some(package_json_contents) = maybe_package_json_contents else { Output::err_generic( "failed to find package.json in tarball '{}'", diff --git a/src/runtime/error.rs b/src/runtime/error.rs index 254c6d9d673a..64924d2bb73b 100644 --- a/src/runtime/error.rs +++ b/src/runtime/error.rs @@ -274,11 +274,11 @@ impl From for Error { } } -impl From for Error { - fn from(e: bun_libarchive::Error) -> Self { +impl From for Error { + fn from(e: bun_archive::Error) -> Self { match e { - bun_libarchive::Error::Sys(s) => Self::Sys(s), - bun_libarchive::Error::Alloc(a) => Self::Alloc(a), + bun_archive::Error::Sys(s) => Self::Sys(s), + bun_archive::Error::Alloc(a) => Self::Alloc(a), _ => Self::Unexpected, } } diff --git a/src/standalone_graph/Cargo.toml b/src/standalone_graph/Cargo.toml index 7ade11b6adfd..bbf9eb90b516 100644 --- a/src/standalone_graph/Cargo.toml +++ b/src/standalone_graph/Cargo.toml @@ -27,7 +27,7 @@ bun_dotenv.workspace = true bun_exe_format.workspace = true bun_http.workspace = true bun_parsers.workspace = true -bun_libarchive.workspace = true +bun_archive.workspace = true bun_ast.workspace = true bun_options_types.workspace = true bun_paths.workspace = true diff --git a/src/standalone_graph/StandaloneModuleGraph.rs b/src/standalone_graph/StandaloneModuleGraph.rs index 91c73cb92201..2fd2ec3c3a22 100644 --- a/src/standalone_graph/StandaloneModuleGraph.rs +++ b/src/standalone_graph/StandaloneModuleGraph.rs @@ -2743,12 +2743,12 @@ pub(crate) fn download_to_path( scopeguard::defer! { let _ = bun_sys::Dir::cwd().delete_tree(tempdir_name.as_bytes()); } - let extract_res = bun_libarchive::Archiver::extract_to_dir( + let extract_res = bun_archive::Archiver::extract_to_dir( tarball_bytes.as_slice(), tmpdir.fd(), None, &mut (), - bun_libarchive::ExtractOptions { + bun_archive::ExtractOptions { // "package/bin" depth_to_skip: 2, ..Default::default() diff --git a/src/tar/decoder.rs b/src/tar/decoder.rs index 6f7162154d6a..f219be1dc4df 100644 --- a/src/tar/decoder.rs +++ b/src/tar/decoder.rs @@ -234,6 +234,55 @@ impl<'a> MemoryDecoder<'a> { self.decoder.skip_entry(); } + /// Moves to the next entry, past whatever is left of the current one. `None` at the end of the + /// archive. + pub fn next_entry(&mut self) -> Result> { + self.skip_entry(); + loop { + match self.next()? { + Event::Entry(_) => break, + Event::EndOfArchive => return Ok(None), + Event::NeedInput | Event::Data { .. } | Event::EndOfEntry { .. } => {} + } + } + Ok(Some(self.entry())) + } + + /// Appends the first `size` bytes of the current entry's contents to `out`, or all of them if + /// there are fewer. Holes read as zeros. + pub fn read_data(&mut self, size: usize, out: &mut Vec) -> Result<()> { + let start = out.len(); + let zeros_up_to = |out: &mut Vec, offset: i64| -> Result<()> { + let have = out.len().saturating_sub(start); + let offset = usize::try_from(offset).map_err(|_| Error::OutOfOrderSparse)?; + if offset < have { + return Err(Error::OutOfOrderSparse); + } + let zeros = offset.min(size).saturating_sub(have); + out.try_reserve(zeros).map_err(|_| Error::OutOfMemory)?; + out.resize(out.len().saturating_add(zeros), 0); + Ok(()) + }; + while out.len().saturating_sub(start) < size { + match self.next()? { + Event::Data { offset, bytes } => { + zeros_up_to(out, offset)?; + let room = size.saturating_sub(out.len().saturating_sub(start)); + let bytes = bytes.get(..room.min(bytes.len())).unwrap_or_default(); + out.try_reserve(bytes.len()) + .map_err(|_| Error::OutOfMemory)?; + out.extend_from_slice(bytes); + } + Event::EndOfEntry { size: end } => { + zeros_up_to(out, end)?; + break; + } + Event::NeedInput | Event::Entry(_) | Event::EndOfArchive => break, + } + } + Ok(()) + } + /// Never returns [`Event::NeedInput`]. pub fn next(&mut self) -> Result> { if let Some(error) = self.failed { diff --git a/src/tar/error.rs b/src/tar/error.rs index dae325a54e35..185c3dad4e0b 100644 --- a/src/tar/error.rs +++ b/src/tar/error.rs @@ -32,6 +32,8 @@ pub enum Error { SizeOverflow, #[error("Malformed sparse map data")] MalformedSparseMap, + #[error("Encountered out-of-order sparse blocks")] + OutOfOrderSparse, #[error("Unreasonably large sparse map")] SparseMapTooLarge, #[error("Oversized AppleDouble extension")] diff --git a/src/tar/reader.rs b/src/tar/reader.rs index 1a445ba26219..8f96da29dd95 100644 --- a/src/tar/reader.rs +++ b/src/tar/reader.rs @@ -34,6 +34,22 @@ pub struct Options { pub detect: bool, } +impl Options { + /// What the platform this was compiled for has always done. + pub fn for_host(dialect: Dialect, detect: bool) -> Self { + Self { + dialect, + charset: if cfg!(windows) { + Charset::Utf8 + } else { + Charset::Ascii + }, + mac_metadata: cfg!(target_os = "macos"), + detect, + } + } +} + pub const S_IFMT: u32 = 0o170000; pub const S_IFREG: u32 = 0o100000; pub const S_IFDIR: u32 = 0o040000; From 1e71af1621e036e1adb3b33e6e5f2f3de666b914 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 21:52:01 +0000 Subject: [PATCH 07/33] Add golden tests for archives and word errors as before The expected results were recorded with the last build that used libarchive, through Bun.Archive, bun install and bun pm pack. --- src/archive/lib.rs | 4 +- src/tar/compose.rs | 7 +- src/tar/decoder.rs | 9 +- src/tar/error.rs | 24 +- src/tar/num.rs | 49 ---- src/tar/reader.rs | 45 ++- src/tar/writer.rs | 4 +- test/cli/install/GHSA-pfwx-36v6-832x.test.ts | 2 +- .../bun-install-streaming-extract.test.ts | 73 ++--- test/cli/install/bun-pack.test.ts | 53 +++- .../bun-install-tarball-golden.test.ts | 50 ++++ test/cli/install/tarball-golden/cases.json.gz | Bin 0 -> 127787 bytes test/cli/install/tarball-golden/install.ts | 96 +++++++ test/cli/install/tarball-golden/record.ts | 28 ++ .../__snapshots__/archive-golden.test.ts.snap | 270 ++++++++++++++++++ .../bun/archive-golden/archive-golden.test.ts | 211 ++++++++++++++ test/js/bun/archive-golden/cases.json.gz | Bin 0 -> 434776 bytes test/js/bun/archive-golden/observe.ts | 95 ++++++ test/js/bun/archive-golden/record.ts | 29 ++ test/js/bun/archive.test.ts | 35 ++- 20 files changed, 956 insertions(+), 128 deletions(-) create mode 100644 test/cli/install/tarball-golden/bun-install-tarball-golden.test.ts create mode 100644 test/cli/install/tarball-golden/cases.json.gz create mode 100644 test/cli/install/tarball-golden/install.ts create mode 100644 test/cli/install/tarball-golden/record.ts create mode 100644 test/js/bun/archive-golden/__snapshots__/archive-golden.test.ts.snap create mode 100644 test/js/bun/archive-golden/archive-golden.test.ts create mode 100644 test/js/bun/archive-golden/cases.json.gz create mode 100644 test/js/bun/archive-golden/observe.ts create mode 100644 test/js/bun/archive-golden/record.ts diff --git a/src/archive/lib.rs b/src/archive/lib.rs index fda7407f1c81..63ffa420329c 100644 --- a/src/archive/lib.rs +++ b/src/archive/lib.rs @@ -2,8 +2,10 @@ #![warn(unused_must_use)] use bun_collections::StringArrayHashMap; +#[cfg(unix)] +use bun_core::ZStr; use bun_core::{MutableString, strings}; -use bun_core::{Output, ZStr, slice_as_bytes}; +use bun_core::{Output, slice_as_bytes}; #[cfg(unix)] use bun_paths::PathBuffer; use bun_paths::{OSPathChar, SEP, SEP_STR}; diff --git a/src/tar/compose.rs b/src/tar/compose.rs index 4fbd3d081c7c..1a5064472cce 100644 --- a/src/tar/compose.rs +++ b/src/tar/compose.rs @@ -7,7 +7,6 @@ mod tables; use crate::error::{Error, Result}; -use bun_core::strings; use tables::{COMBINING_CLASSES, COMPOSING_BLOCKS, COMPOSITIONS}; /// More combining marks than this on one character make the name count as unconvertible. @@ -255,7 +254,7 @@ pub fn compose(name: &[u8]) -> Result>> { /// Whether [`compose`] would return something. pub fn converts(name: &[u8]) -> bool { - strings::first_non_ascii(name).is_none() || matches!(compose(name), Ok(Some(_))) + name.is_ascii() || matches!(compose(name), Ok(Some(_))) } fn push_utf8(out: &mut Vec, c: u32) { @@ -268,7 +267,7 @@ pub(crate) fn windows_name(name: &[u8]) -> Result>> { let mut out = Vec::new(); out.try_reserve(name.len()) .map_err(|_| Error::OutOfMemory)?; - if strings::first_non_ascii(name).is_none() { + if name.is_ascii() { out.extend_from_slice(name); } else { let Some(chars) = compose(name)? else { @@ -290,7 +289,7 @@ pub(crate) fn windows_name(name: &[u8]) -> Result>> { pub fn to_utf16(name: &[u8], is_utf8: bool, out: &mut Vec) -> Result<()> { out.try_reserve(name.len()) .map_err(|_| Error::OutOfMemory)?; - if is_utf8 && strings::first_non_ascii(name).is_some() { + if is_utf8 && !name.is_ascii() { if let Some(chars) = compose(name)? { for c in chars { let c = char::from_u32(c).unwrap_or(char::REPLACEMENT_CHARACTER); diff --git a/src/tar/decoder.rs b/src/tar/decoder.rs index f219be1dc4df..dcd3ea2ffc89 100644 --- a/src/tar/decoder.rs +++ b/src/tar/decoder.rs @@ -135,7 +135,14 @@ impl Decoder { break Poll::EndOfArchive; }; let base = layer.pos; - let (used, poll) = self.tar.poll(layer.unread(), layer.done)?; + let (used, poll) = match self.tar.poll(layer.unread(), layer.done) { + // Damaged compression explains it better than what came out of it not being tar. + Err(Error::UnrecognizedFormat) if eof => { + self.drain(input.get(used_total..).unwrap_or_default(), eof)?; + return Err(Error::UnrecognizedFormat); + } + other => other?, + }; layer.pos = layer.pos.saturating_add(used); match poll { Poll::NeedInput => {} diff --git a/src/tar/error.rs b/src/tar/error.rs index 185c3dad4e0b..7408e5981f90 100644 --- a/src/tar/error.rs +++ b/src/tar/error.rs @@ -12,24 +12,38 @@ pub enum Error { TruncatedData, #[error("Truncated archive detected while reading metadata")] TruncatedMetadata, - #[error("Truncated input file")] - TruncatedInput, + #[error("Truncated tar archive detected while reading pax attribute name")] + TruncatedPaxName, + #[error("Truncated archive detected while reading macOS metadata")] + TruncatedMacMetadata, + #[error("Truncated tar archive detected while skipping data")] + TruncatedSkip, + #[error("Truncated input file (needed {needed} bytes, only {available} available)")] + TruncatedInput { needed: i64, available: i64 }, #[error("Truncated tar archive detected while reading sparse file data")] TruncatedSparse, #[error("Redundant '{0}' header")] - RedundantHeader(char), - #[error("Special header has invalid size")] - InvalidSpecialHeaderSize, + RedundantHeader(&'static str), + #[error("Special header has invalid size: {0}")] + InvalidSpecialHeaderSize(i64), + #[error("pax extension header has invalid size: {0}")] + InvalidPaxHeaderSize(i64), #[error("Invalid empty pax extended header")] EmptyPaxHeader, + #[error("Invalid empty pax global extended header")] + EmptyPaxGlobalHeader, #[error("Malformed pax attribute")] MalformedPaxAttribute, #[error("Tar size attribute overflow")] PaxSizeOverflow, + #[error("Tar entry has negative file size")] + NegativeFileSize, #[error("Tar entry has negative size")] NegativeSize, #[error("Tar entry size overflow")] SizeOverflow, + #[error("Line too long")] + LineTooLong, #[error("Malformed sparse map data")] MalformedSparseMap, #[error("Encountered out-of-order sparse blocks")] diff --git a/src/tar/num.rs b/src/tar/num.rs index a57b97fed0e9..6fba0daeab3c 100644 --- a/src/tar/num.rs +++ b/src/tar/num.rs @@ -96,52 +96,3 @@ pub(crate) fn parse_pax_time(value: &[u8]) -> Option { } Some(if negative { -seconds } else { seconds }) } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn octal() { - assert_eq!(parse_field(b"0000644\0"), 0o644); - assert_eq!(parse_field(b" \t755 "), 0o755); - assert_eq!(parse_field(b"-17"), -0o17); - assert_eq!(parse_field(b"128"), 0o12); - assert_eq!(parse_field(b""), 0); - assert_eq!(parse_field(b"7777777777777777777777777"), i64::MAX); - assert_eq!(parse_field(b"-7777777777777777777777777"), i64::MIN); - } - - #[test] - fn base_256() { - assert_eq!(parse_field(&[0x80, 0, 0, 0, 0, 0, 0, 1]), 1); - assert_eq!(parse_field(&[0xff; 8]), -1); - assert_eq!(parse_field(&[0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0]), 256); - assert_eq!( - parse_field(&[0x80, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), - i64::MAX - ); - assert_eq!( - parse_field(&[0xff, 0xfe, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), - i64::MIN - ); - assert_eq!( - parse_field(&[0x80, 0, 0, 0, 0x80, 0, 0, 0, 0, 0, 0, 0]), - i64::MAX - ); - assert_eq!(parse_field(&[0x81]), 1); - assert_eq!(parse_field(&[0xc1]), -63); - } - - #[test] - fn pax_time() { - assert_eq!(parse_pax_time(b""), Some(0)); - assert_eq!(parse_pax_time(b"12"), Some(12)); - assert_eq!(parse_pax_time(b"-12.5"), Some(-12)); - assert_eq!(parse_pax_time(b"12."), Some(12)); - assert_eq!(parse_pax_time(b"12.0000000001234"), Some(12)); - assert_eq!(parse_pax_time(b"12.5x"), None); - assert_eq!(parse_pax_time(b"12x"), None); - assert_eq!(parse_pax_time(b"99999999999999999999"), None); - } -} diff --git a/src/tar/reader.rs b/src/tar/reader.rs index 8f96da29dd95..94280f12737a 100644 --- a/src/tar/reader.rs +++ b/src/tar/reader.rs @@ -375,6 +375,8 @@ pub struct Reader { pax_size: i64, disk_size: i64, bytes_remaining: i64, + /// How much of the current `State::Skip` is behind us. + skipped: i64, padding: i64, null_blocks: u32, } @@ -433,6 +435,7 @@ impl Reader { pax_size: 0, disk_size: 0, bytes_remaining: 0, + skipped: 0, padding: 0, null_blocks: 0, } @@ -567,14 +570,24 @@ impl Reader { State::Skip(left, after) => { let take = usize::try_from(left).unwrap_or(usize::MAX).min(input.len()); *used = take; - let left = left.saturating_sub(i64::try_from(take).unwrap_or(i64::MAX)); + let taken = i64::try_from(take).unwrap_or(i64::MAX); + let left = left.saturating_sub(taken); + self.skipped = self.skipped.saturating_add(taken); if left > 0 { self.state = State::Skip(left, after); if !eof { return Ok(Step::NeedInput); } - return Err(Error::TruncatedInput); + return Err(match after { + AfterSkip::EndOfEntry | AfterSkip::NextEntry => Error::TruncatedSkip, + AfterSkip::MacMetadata => Error::TruncatedMacMetadata, + _ => Error::TruncatedInput { + needed: left.saturating_add(self.skipped), + available: self.skipped, + }, + }); } + self.skipped = 0; self.after_skip(after) } @@ -712,7 +725,7 @@ impl Reader { for &c in input { *used = used.saturating_add(1); if self.line_len >= limit { - return Err(Error::MalformedSparseMap); + return Err(Error::LineTooLong); } if let Some(slot) = self.line.get_mut(self.line_len) { *slot = c; @@ -871,7 +884,7 @@ impl Reader { match header::typeflag(&block) { b'A' => { if self.mark_seen(seen::SOLARIS_ACL) { - return Err(Error::RedundantHeader('A')); + return Err(Error::RedundantHeader("A")); } self.format = Format::Pax; // A Solaris ACL. Permissions come from the header's mode alone, as in GNU tar and @@ -880,14 +893,14 @@ impl Reader { } b'g' => { if self.mark_seen(seen::PAX_GLOBAL) { - return Err(Error::RedundantHeader('g')); + return Err(Error::RedundantHeader("g")); } self.format = Format::Pax; if !(0..=ENTRY_LIMIT).contains(&size) { - return Err(Error::InvalidSpecialHeaderSize); + return Err(Error::InvalidSpecialHeaderSize(size)); } if size == 0 { - return Err(Error::EmptyPaxHeader); + return Err(Error::EmptyPaxGlobalHeader); } self.skip_body(size, Status::Ok); Ok(()) @@ -912,14 +925,18 @@ impl Reader { } self.format = Format::Gnu; if !(0..=PATHNAME_LIMIT).contains(&size) { - return Err(Error::InvalidSpecialHeaderSize); + return Err(Error::InvalidSpecialHeaderSize(size)); } self.skip_body(size, Status::Ok); Ok(()) } flag @ (b'x' | b'X') => { if self.mark_seen(seen::PAX) { - return Err(Error::RedundantHeader(char::from(flag))); + return Err(Error::RedundantHeader(if flag == b'x' { + "x" + } else { + "X'/'x" + })); } self.format = Format::Pax; self.begin_pax(size) @@ -938,7 +955,7 @@ impl Reader { /// The body of a header whose size is not trusted past [`PATHNAME_LIMIT`]. fn begin_body(&mut self, size: i64, collect: Option) -> Result<()> { if !(0..=ENTRY_LIMIT).contains(&size) { - return Err(Error::InvalidSpecialHeaderSize); + return Err(Error::InvalidSpecialHeaderSize(size)); } if size > PATHNAME_LIMIT { self.skip_body(size, Status::Warn); @@ -955,7 +972,7 @@ impl Reader { fn begin_pax(&mut self, size: i64) -> Result<()> { if !(0..=ENTRY_LIMIT).contains(&size) { - return Err(Error::InvalidSpecialHeaderSize); + return Err(Error::InvalidPaxHeaderSize(size)); } if size == 0 { return Err(Error::EmptyPaxHeader); @@ -1001,7 +1018,7 @@ impl Reader { if !eof { return Ok(Step::NeedInput); } - Err(Error::TruncatedMetadata) + Err(Error::TruncatedPaxName) } fn on_pax_key(&mut self) -> Result<()> { @@ -1156,7 +1173,7 @@ impl Reader { fn converts(&self, text: &[u8]) -> bool { let text = until_nul(text); match self.options.charset { - Charset::Ascii => strings::first_non_ascii(text).is_none(), + Charset::Ascii => text.is_ascii(), Charset::Utf8 => crate::compose::converts(text), } } @@ -1491,7 +1508,7 @@ impl Reader { header_size }; if self.disk_size < 0 { - return Err(Error::NegativeSize); + return Err(Error::NegativeFileSize); } if self.disk_size > ENTRY_LIMIT { return Err(Error::SizeOverflow); diff --git a/src/tar/writer.rs b/src/tar/writer.rs index 2ac2ef38f274..3ecba7a8b1a0 100644 --- a/src/tar/writer.rs +++ b/src/tar/writer.rs @@ -96,7 +96,7 @@ impl Writer { }; let converted; let (path, binary) = match self.names { - Names::Binary => (given, strings::first_non_ascii(given).is_some()), + Names::Binary => (given, !given.is_ascii()), Names::Utf8 => match crate::compose::windows_name(given)? { Some(name) => { converted = name; @@ -112,7 +112,7 @@ impl Writer { } let shortened; let mut name = path; - if strings::first_non_ascii(path).is_some() || !fits_in_ustar(path) { + if !path.is_ascii() || !fits_in_ustar(path) { add_record(&mut pax, b"path", path)?; shortened = ustar_entry_name(path, None); name = &shortened; diff --git a/test/cli/install/GHSA-pfwx-36v6-832x.test.ts b/test/cli/install/GHSA-pfwx-36v6-832x.test.ts index c9ca2a047e33..27fe140b6e6c 100644 --- a/test/cli/install/GHSA-pfwx-36v6-832x.test.ts +++ b/test/cli/install/GHSA-pfwx-36v6-832x.test.ts @@ -73,7 +73,7 @@ async function lockedEntry(dir: string) { } // Tarballs smaller than BUN_INSTALL_STREAMING_MIN_SIZE (2 MiB by default) are extracted -// once the whole body has been buffered; larger ones are streamed into libarchive while +// once the whole body has been buffered; larger ones are streamed into the extractor while // they download. The two extractors hash and verify the tarball independently, so every // test runs against both. `--verbose` output names the extractor that ran. const variants = [ diff --git a/test/cli/install/bun-install-streaming-extract.test.ts b/test/cli/install/bun-install-streaming-extract.test.ts index d254ef2f066f..f6f7059bc903 100644 --- a/test/cli/install/bun-install-streaming-extract.test.ts +++ b/test/cli/install/bun-install-streaming-extract.test.ts @@ -1,9 +1,9 @@ // Verifies that `bun install` can extract a tarball while it is still // downloading. A local registry drip-feeds the .tgz body in small // chunks so the HTTP thread delivers multiple progress callbacks; the -// streaming extractor (TarballStream.zig + the ARCHIVE_RETRY patches in -// vendor/libarchive) must reassemble them into the same on-disk layout -// the buffered extractor would produce. +// streaming extractor (TarballStream.rs on top of bun_tar's push decoder) +// must reassemble them into the same on-disk layout the buffered +// extractor would produce. import { describe, expect, setDefaultTimeout, test } from "bun:test"; import { bunEnv, bunExe, isMacOS, isWindows, readdirSorted, runCommandMaxRSS, tempDir } from "harness"; @@ -68,8 +68,8 @@ function pad512(len: number): Buffer { function tarFile(name: string, body: Buffer): Buffer[] { // ustar stores at most 100 bytes of name; longer paths need a pax - // 'x' record. npm's `tar` uses pax, so this exercises the resumable - // `tar_read_header` path in the libarchive patch. + // 'x' record. npm's `tar` uses pax, so this exercises a header that + // arrives in pieces. if (name.length > 100) { // Build the pax record so that the declared length includes the // length field itself. Iterate because adding digits to the length @@ -133,7 +133,7 @@ function makeEntries(): Entry[] { { path: "index.js", body: Buffer.from("module.exports = 'ok';\n") }, { // > 100 chars → pax extended header → exercises the resumable - // header path in the libarchive patch. + // header path. path: "very/deeply/nested/directory/structure/that/exceeds/the/one/hundred/byte/ustar/limit/long-name-file.txt", body: Buffer.from("long path ok\n"), }, @@ -540,7 +540,7 @@ describe("streaming tarball extraction", () => { test("streaming rejects a tarball whose integrity does not match", async () => { // Serve the valid tarball but advertise the integrity of a // *different* blob. Extraction will stream to completion (so we - // exercise the full ARCHIVE_RETRY path through libarchive), the + // exercise running out of input all the way through), the // incremental hasher produces the real SHA-512, and `finish()` // must notice the mismatch before the temp tree is promoted into // the cache. @@ -569,7 +569,7 @@ describe("streaming tarball extraction", () => { // A connection that dies part-way through the body is a failed download, // whichever extractor is consuming it: report the transport error, retry // like any other failed download, and never hand the truncated body to - // libarchive (streaming) or the integrity check (buffered). + // the decoder (streaming) or the integrity check (buffered). describe.each([ ["streaming", {}], ["buffered", { BUN_FEATURE_FLAG_DISABLE_STREAMING_INSTALL: "1" }], @@ -638,9 +638,9 @@ describe("streaming tarball extraction", () => { }); // A body that arrives in full but is not a valid archive is an extraction - // failure: not retried, and the message carries libarchive's reason and + // failure: not retried, and the message carries the decoder's reason and // the offset it gave up at. - test.concurrent("streaming reports libarchive's error for a corrupt tarball", async () => { + test.concurrent("streaming reports the decoder's error for a corrupt tarball", async () => { const corrupt = Buffer.from(tgz); corrupt.fill(0x55, corrupt.length >> 1, (corrupt.length >> 1) + 4096); const corruptIntegrity = "sha512-" + createHash("sha512").update(corrupt).digest("base64"); @@ -837,25 +837,17 @@ describe("streaming tarball extraction", () => { }); // ------------------------------------------------------------------- -// Regression: the nonblocking-read patch routed upstream libarchive's -// pre-existing damaged-block ARCHIVE_RETRY through the same `bun_retry` -// path as a non-blocking yield, so `seen_headers` / entry state leaked -// across the retry. A second pax 'g' global header after the damaged -// block would then trip "Redundant 'g' header" → ARCHIVE_FATAL even -// though upstream libarchive (and a `tar` CLI) accepts this layout. +// A block with a bad checksum is skipped, and what was seen before it is +// forgotten. If it were not, a second pax 'g' global header after the +// damaged block would trip "Redundant 'g' header", though a `tar` CLI +// accepts this layout. // // This test goes through the buffered extractor only: local `file:` -// tarballs are read fully into memory by PackageManagerTask.readAndExtract -// and handed to Archiver.extractToDir, which loops on readNextHeader with -// `.retry => continue`. The streaming reader is never involved, so any -// behaviour change here is the libarchive patch leaking into the shared -// buffered codepath. +// tarballs are read fully into memory. // ------------------------------------------------------------------- test("buffered extract: damaged-block retry resets header state (upstream semantics)", async () => { - // One pax 'g' extended-header payload. libarchive's header_pax_global - // just skips it, but parsing it sets `seen_headers |= seen_g_header`; - // seeing a second one without an intervening state reset is what - // triggers the "Redundant 'g' header" FATAL. + // One pax 'g' extended-header payload. It is skipped, but remembered: + // a second one before the entry they belong to is refused. const pax = Buffer.from("16 comment=test\n", "utf8"); expect(pax.length).toBe(16); const paxEntry = () => [tarHeader("pax_global_header", pax.length, "g"), pax, pad512(pax.length)]; @@ -889,10 +881,8 @@ test("buffered extract: damaged-block retry resets header state (upstream semant const { stderr, exitCode } = await runInstall(String(dir)); - // With the broken patch the second 'g' header trips - // "Redundant 'g' header" → ARCHIVE_FATAL inside libarchive; the Zig - // extract loop surfaces that as `error.Fail` → "Fail extracting - // tarball". With upstream semantics restored the damaged block is + // If the first 'g' header were remembered across the damaged block, the + // second one would surface as "Fail extracting tarball". The damaged block is // skipped, state is fully reset, and the file following the second // 'g' header is extracted normally. expect(stderr).not.toContain("Fail extracting tarball"); @@ -905,13 +895,13 @@ test("buffered extract: damaged-block retry resets header state (upstream semant // ------------------------------------------------------------------- // Buffered extract: the decompressed tar is never materialised in -// memory. libarchive gunzips on the fly, so a highly compressible .tgz +// memory. It is gunzipped on the fly, so a highly compressible .tgz // installs without an RSS spike of roughly its decompressed size. // Covers `file:` dependencies, which always take the buffered path. // ------------------------------------------------------------------- test("buffered extract does not hold the decompressed local tarball in memory", async () => { // 256 MiB of zeros: above the 64 MB gzip ISIZE cutoff that gates the - // libdeflate fast path, so libarchive (not libdeflate) decompresses, + // libdeflate fast path, so zlib (not libdeflate) decompresses, // and a multiple of 512 so the tar entry needs no trailing pad block. // The old path inflated this into a ~256 MB Vec before extraction, // which shows up directly in the child's maxRSS. @@ -975,7 +965,7 @@ test("buffered extract does not hold the decompressed local tarball in memory", // decompressed tar in memory. With the old pre-decompress path the // install's maxRSS was well over 3x PAYLOAD_SIZE (Vec growth // reallocations): ~780 MB release, ~1 GB debug+ASAN. Streaming - // through libarchive it stays at baseline (~40 MB release, ~240 MB + // through zlib it stays at baseline (~40 MB release, ~240 MB // debug+ASAN, reported as the spawner's ~320 MB floor), so the // midpoint gives wide margin both ways without needing to branch on // build type. @@ -1038,7 +1028,7 @@ test("streaming extract skips a damaged header block and extracts the entries af }); // Unlike registry tarballs, a `github:` tarball has its directory entries -// created, with the mode libarchive reports for them. A GNU base-256 mode +// created, with the mode the tar reader reports for them. A GNU base-256 mode // field can set bits far above the twelve mode bits; both extractors must // drop those instead of aborting on them. test.concurrent.each([ @@ -1552,11 +1542,11 @@ function readFiles(root: string): Record { } // Installs `tgz` as `stream-pkg`, `installs` times with one cache, and returns what each install left. -// "libdeflate" and "libarchive": a `file:` tarball, which the buffered extractor decompresses that way. +// "libdeflate" and "zlib": a `file:` tarball, which the buffered extractor decompresses that way. // "buffered" and "streaming": downloaded, from the registry or as a URL dependency, and extracted that way. // `files`: also what `readFiles` finds in the installed package. // `gzipEncoded`: the server compresses `tgz` and says so with `Content-Encoding: gzip`. -type ExtractPath = "libdeflate" | "libarchive" | "buffered" | "streaming"; +type ExtractPath = "libdeflate" | "zlib" | "buffered" | "streaming"; async function installStreamPkg( path: ExtractPath, tgz: Buffer, @@ -1605,12 +1595,12 @@ async function installStreamPkg( BUN_TMPDIR: tmp, BUN_INSTALL_STREAMING_MIN_SIZE: "1", BUN_INSTALL_STREAMING_DRAIN_THRESHOLD: "1", - ...(path === "libarchive" ? { BUN_FEATURE_FLAG_NO_LIBDEFLATE: "1" } : {}), + ...(path === "zlib" ? { BUN_FEATURE_FLAG_NO_LIBDEFLATE: "1" } : {}), ...(path === "buffered" ? { BUN_FEATURE_FLAG_DISABLE_STREAMING_INSTALL: "1" } : {}), }); results.push({ // What another thread writes to the verbose log can come anywhere in this line. - error: stderr.replace(/\[stream-pkg\] Extract .*\n(libarchive error: .*\n)?/g, "").match(/^error: .*$/m)?.[0], + error: stderr.replace(/\[stream-pkg\] Extract .*\n(archive error: .*\n)?/g, "").match(/^error: .*$/m)?.[0], streamed: stderr.includes("Streamed "), installed: existsSync(join(String(dir), "node_modules", "stream-pkg")), cached: readdirSync(join(String(dir), ".cache")).filter(name => /^(stream-pkg@|@T@)/.test(name)).length, @@ -1630,7 +1620,7 @@ async function installStreamPkg( // install, a registry without `dist.integrity`, `--no-verify`) that // trailer is the only thing that catches a damaged tarball. All three // ways to decompress one must check it: libdeflate in memory and -// libarchive's gzip filter in the buffered extractor, and that filter +// bun_tar's gzip decoder in the buffered extractor, and that decoder // again in the streaming extractor. // ------------------------------------------------------------------- describe.concurrent("gzip trailer", () => { @@ -1670,7 +1660,7 @@ describe.concurrent("gzip trailer", () => { // Too long for the attempt in memory. ["a wrong ISIZE", changed(good, good.length - 4, (b, i) => b.writeUInt32LE(100 * 1024 * 1024, i))], ]; - const paths = ["libdeflate", "libarchive", "streaming"] as const; + const paths = ["libdeflate", "zlib", "streaming"] as const; const install = async (path: (typeof paths)[number], tgz: Buffer) => (await installStreamPkg(path, tgz))[0]; test.each(paths)("a valid tarball installs (%s)", async path => { @@ -1690,10 +1680,7 @@ describe.concurrent("gzip trailer", () => { expect(await install(path, tgz)).toEqual({ error: path === "streaming" - ? // libarchive's tar reader overwrites the message of the filter that failed. - expect.stringMatching( - /^error: Fail extracting tarball for "stream-pkg": (gzip trailer mismatch|Truncated tar archive)/, - ) + ? expect.stringMatching(/^error: Fail extracting tarball for "stream-pkg": gzip trailer mismatch/) : "error: Fail extracting tarball from stream-pkg", streamed: false, installed: false, diff --git a/test/cli/install/bun-pack.test.ts b/test/cli/install/bun-pack.test.ts index 81778503351c..7b28a7c5c498 100644 --- a/test/cli/install/bun-pack.test.ts +++ b/test/cli/install/bun-pack.test.ts @@ -109,9 +109,8 @@ test.concurrent("basic", async () => { expect(tarballEntries(join(dir, "pack-basic-1.2.3.tgz"))).toEqual(["package/package.json", "package/index.js"]); }); -// The archive ends right after the two end-of-archive blocks. libarchive's default for a custom -// output sink would pad it to a full 10 KiB record instead, which would change the size and shasum -// of every tarball bun produces. +// The archive ends right after the two end-of-archive blocks. Padding it to a full 10 KiB record, +// as Bun.Archive does, would change the size and shasum of every tarball bun produces. test.concurrent("the archive is not padded to a full tar record", async () => { using dir = tempDir("pack-unpadded", { "package.json": JSON.stringify({ name: "pack-unpadded", version: "1.0.0" }), @@ -125,6 +124,50 @@ test.concurrent("the archive is not padded to a full tar record", async () => { expect(tar.byteLength).toBe(6 * 512); }); +// A package packed twice must give the same tarball, whichever version of bun packs it. What the +// compressor makes of the archive depends on the CPU, so that part is compared decompressed. +// The hashes are from Bun 1.4.4. +test.concurrent.skipIf(isWindows).each([ + [[], "1f8b08000000000002ff"], + [["--gzip-level", "0"], "1f8b08000000000000ff"], + [["--gzip-level", "1"], "1f8b08000000000004ff"], + [["--gzip-level", "6"], "1f8b08000000000000ff"], +])("the bytes of a tarball do not change %j", async (args, gzipHeader) => { + const n = (count: number) => Buffer.alloc(count, "n").toString(); + using dir = tempDir("pack-bytes", { + "package.json": JSON.stringify({ name: "@scope/pack-bytes", version: "1.2.3", bin: { x: "bin/x.js" } }), + "index.js": "module.exports = 1;", + "empty.txt": "", + "bin/x.js": "#!/usr/bin/env node\n", + "run.sh": "#!/bin/sh\n", + "café.txt": "not ASCII", + [n(92) + ".js"]: "the name fits once package/ is in front of it", + [n(93) + ".js"]: "the name does not fit", + [n(200) + ".js"]: "the name does not fit and cannot be split", + [`${n(50)}/${n(50)}/${n(50)}/split.js`]: "the name fits once it is split", + [`${n(150)}/${n(100)}/no-split.js`]: "the name cannot be split", + "blocks/511": n(511), + "blocks/512": n(512), + "blocks/513": n(513), + "blocks/100000": n(100000), + }); + await chmod(join(String(dir), "run.sh"), 0o755); + + await pack(dir, bunEnv, ...args); + + const tgz = await file(join(dir, "scope-pack-bytes-1.2.3.tgz")).bytes(); + const tar = gunzipSync(tgz); + expect({ + gzipHeader: Buffer.from(tgz.subarray(0, 10)).toString("hex"), + length: tar.byteLength, + sha256: new Bun.CryptoHasher("sha256").update(tar).digest("hex"), + }).toEqual({ + gzipHeader, + length: 119296, + sha256: "f77c6cf6bbb5acff151d905cc601904310c7eebdfa00bc346b8a8c7465f19294", + }); +}); + // pack only reads package.json, so a file that is readable but not writable (a read-only // checkout, a file owned by another user) must not stop it. test.concurrent.skipIf(isWindows)("read-only package.json", async () => { @@ -657,7 +700,7 @@ describe.concurrent("flags", () => { }); // Every write(2) to /dev/full fails with ENOSPC, the same as a tarball destination on a full disk. - // "an entry": enough incompressible data that libarchive flushes a block while it writes index.js. + // "an entry": enough incompressible data that a block is flushed while it writes index.js. // "the end of the archive": small enough that the only write(2) happens when the archive is closed. test.skipIf(!isLinux).each([ ["an entry", () => `// ${randomBytes(128 * 1024).toString("base64")}`], @@ -5389,7 +5432,7 @@ test.concurrent("unicode", async () => { // names into JS strings, which maps the bytes this test is about onto U+FFFD. // Names are returned latin1-decoded (one char per stored byte) and collected // from every place the archive spells a name: the ustar header `name` field -// and, when libarchive emits one, the `path` record of the pax header. +// and, when there is one, the `path` record of the pax header. function rawTarballMembers(tgz: Uint8Array) { const tar = Buffer.from(Bun.gunzipSync(tgz)); const members: { names: Set; contents: string }[] = []; diff --git a/test/cli/install/tarball-golden/bun-install-tarball-golden.test.ts b/test/cli/install/tarball-golden/bun-install-tarball-golden.test.ts new file mode 100644 index 000000000000..a9f50c761eed --- /dev/null +++ b/test/cli/install/tarball-golden/bun-install-tarball-golden.test.ts @@ -0,0 +1,50 @@ +// Tarballs of every shape the tar reader knows about, well formed and damaged, with what +// `bun install` made of each one when it was recorded. See record.ts. +import { describe, expect, test } from "bun:test"; +import { bunExe, isLinux, isMacOS, isWindows } from "harness"; +import { join } from "node:path"; +import { install, type Case } from "./install"; + +const all: Case[] = JSON.parse( + Buffer.from(Bun.gunzipSync(await Bun.file(join(import.meta.dir, "cases.json.gz")).bytes())).toString(), +); +const cases = all + .map((c, id) => ({ ...c, id })) + // Recorded on Linux. Names outside ASCII are decoded differently on Windows, and neither its file + // systems nor those of macOS take every name. + .filter(c => isLinux || ((isMacOS ? c.mac : c.win) && !/[^\x20-\x7e\n]|[:*?<>|\\]|^link /m.test(c.tree ?? ""))); +// Windows has no permission bits. +const comparable = (tree: string | null | undefined) => (isWindows ? tree?.replace(/^(\w+ ".*") \d+/gm, "$1") : tree); + +describe.each([ + ["while they download", true], + ["from memory", false], +])("tarballs extracted %s", (_, streaming) => { + test.concurrent( + "are installed the way it was recorded", + async () => { + const accepted = cases.filter(c => c.tree !== null); + expect(accepted.length).toBeGreaterThan(200); + const { trees, exitCode, stderr } = await install(bunExe(), accepted, streaming); + const different = accepted + .filter(c => comparable(trees.get(c.id)) !== comparable(c.tree)) + .map(c => `case ${c.id}\n--- expected\n${c.tree}--- actual\n${trees.get(c.id)}`); + expect(different.slice(0, 5).join("\n")).toBe(""); + expect(stderr).not.toContain("error:"); + expect(exitCode).toBe(0); + }, + 60_000, + ); + + test.concurrent( + "are refused if they were when it was recorded", + async () => { + const refused = cases.filter(c => c.tree === null); + expect(refused.length).toBeGreaterThan(50); + const { trees, exitCode } = await install(bunExe(), refused, streaming); + expect(refused.filter(c => trees.get(c.id) !== null).map(c => c.id)).toEqual([]); + expect(exitCode).toBe(1); + }, + 60_000, + ); +}); diff --git a/test/cli/install/tarball-golden/cases.json.gz b/test/cli/install/tarball-golden/cases.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..e8ce5cfdc24696b2c632370ea8e8a164d2b50e0a GIT binary patch literal 127787 zcmZ^~b#NR_@Fggg#g<3RU@%R#w)_m#>H;K0*BVKtZ2%X*(61tZ_bQZmyNQd|^Q=TZ8{ZihvONnFPk8LPbtg z)RG8CrA$WsLt4Qp_BL~GZ=85MV54izb$H97Xwk@&2a}mi(0G8qmJs>+WSqSRa;e7;C^^UhKsusp$x9OHGJX z__0U#15egtLFuFZ8@9t{D2I3l6Z=}hd}oBDVWP#4ygx}IUKB*;Pg^sPRDYD|bd|AP zx-A(9P51Z3N0^Bz1m`#5VqN<-@@8QKw3q9bDajHp$NS-B@VU+n=>RI{XP492n>K!t z7Lh}bb2v?&F4`oZ?>WoS_sL3P|qLOxS9>eTh1w`HlWwSL(My+kjON5Yf3O!4mY%D(z{Pi8!hz zW~rUi%Xz>kV|p_i0%u*1U+_-z@;jB!uA z?%bpg&UlKMFOU3~oCl*51OO9|Pk)jQd>lT!e)D3|^7?rvL|FoAz7YS?lNFl4Kj4M& zp0f^ow+Els(AH*f{3&+{BB+W+sHy;KNg#oUVIyvoF;^0qx0P{?K8J4C^KLoIsI`US zU%z_SK#eHD`bzDk&wlIlt@#v+%gIti_Qkb`)>Ul)&$Y6N5}&Q5#HO*o$t%_3Hl;Qv zTOYtzf$N!bedePZ^>FTeX+NdKepv`u2fg}O!=&I|#gNWLL~3unmV~M|oJ1d|>S-2z zCURR+(yXdR+-UlwHYBWW1TU!i&t)Imj1JpHCBSwRWKpI+pZwi#Ib)GZAr8gB$DFmG z;`BNcxxn(@WSw$w@zQx#jj0Mv6ki()$>7z;vmllOhvR-3yHjqaAm82jjU)Nl=3ddU z#^P?^EyrXmhHJN=B<6H|eE5W@$v6ElB@w~e#sYK`;D!dooX8f)9tFB~?Yu74;WtZ7 z1dokwOUH&p6e&x#vP9Nk`*sQ0x{kXl%qeFgrz)Mhwo49wB{WlH@=WCbnK@6$ly@w2 zaS+Y!SrQ#=__s%!iyxAebs=$?qMcEvq3npY=hS$(Eo>GZ`)a@lmb~I~^jvLx6R?f3 zKACBuBxPGd8e<>Q(%4sD4KMjt*ANlf*dE-Q#T0I8j198Qf?t`vT7nk~!%i7eUZ)}* z->2J^K|QZX-9?*t6z^C(_9BK{7|}BwbGTx# z`tHl-`xiBL;%Aj^ZdmFUoi`qh@MURM!_UgMbi<496|D&W(I-udao}wNx@4*pjL#_j#Mx9n= zl$gA$I*#=|j`}_jsw`!AZq_I;?O!22WzMoM;l+i#-XN_rJ7>jaa5mV`fm*ie%~;0} zA5TU1D`JLRkT(C;N=y!|z~j5ra`zyrhFNih)1u4nv9H=!8ei)1gqA-q1%zy|lwxZI zS51xvL`*@g>uZ!;_f!4eTswS|-MCXUNYs&xi0jn{7e#fRdSHRjh%R(^$_${?n* zaYr`+iH8}dEUoBed~Y60c^Jxo63oAGx`}?NBWc3@1qrKfecTc$nbJLHFFSem&-~wg z@8>37(-2u`cCs?hu64ce&hGLOP;7S-kg8!Q2|#41o?j@F#`a&qK%vnT!##ih#hy?X zG(9|K?s5OxFn9jQJThoJC|DqVr1Eb+T+v)d^P;(;qtibdk~}dJjBIC)@XvdMZkQ=t zRQLNJ2m(2BcP;ENg#g}aJJ@zYLFtfvQco%{ITW$J{|0#4xD~>(@sy58GuX&)a5)%A zG6^~M;(Tv~CBAOO?)3N(wT;Z{f-a!1L-(V@YbMCt=`e?<ECeCeV;UB1QB z2Kw$U9E;i@MLD`b3%xu-Gdyzatp%3=d{QbZ6b_CECr*v)l0gP3o8CDJ<$6%x;Wd#Q zgzlP(yznkBnNCw04(mT&=EUz`W_^2v*m$~l4M`TmV>X)b)Y5TOq$_kyJM_WZw=Yi7&3ijo2!VKP^jxznT0w)Oz=Gw;0~*4QPqeUl zEF6165csohjO}CL%A6ffh`0yrbz))$lR`qyRfapo-jHT~%+~fo2yv;7Wz=ljis*Mx z(4c|niGKX3;DS*fqBbIRs*0}uP*D&u8vFP7z#9{FZzMozASET;Q8eR~%XM-bV?iFP zOL2P-f@@$E$E8`s!@H<^6lFd#bzndAS=YvOz(2~#AJpUl(=_lhjoPGs& zvL;sXcIV!{4bJS_N!7k&(G!JVBXEGr!WQC?X_hUz%>LY!@BwG~@=g@$;Bi9xLC@X8SljjGaiahl{hQAjAfoy2KiL!7AsdLvevsW zIMQw%vTy24&;WyxE3Jc$m$p+>Tg5ef5c?5vs{9+lYV2DL5s;fYO2$;1QAI_&C>l5B zjroWSt4()Je5~_%wk3CCpfjevB{QrJWnhy@q%$zo75>tJKRR?g4)b>YXJnR7OeZz| zK|j{p{R&pS@bSS(XO7eFT5kPCRwJrZPG;uK%3tMXVmQF%Vg6zK?v zgN>2RH~z}s4y8l&$5M{*cwU`gMX`iprNOjtf66L73ZYl?Ic!6>Z#snOxNI8+nVq9ZSi2-3GSF*b#$UT&E+d3>jcI5Ub?$eMLs!z8lxB z6GcC8-G^?|qrg$bAb33 z$Y7GEL$$#3N_yNGUC_oEExdcH9TzhcT~<&xeOASEF)To8k>lu3iOmv6-B#iiF z7lp$~AjC;1Drb5&yMrRq~tZn~@GcqSg-q+SOtmG`rm5au}+S6gX@UI%gQ z)I;r79Aj>U?&gP*#-(_*@@gFB3eBu*r++9zO9B@S_}}&$fB}DqD@ki%^=B6=fJm;xUS2mbDE}d z20j)g0Q3LLK@dQ$g8>SNH!SN<;rl+BMq8b`x9&pyESI>-(%Q<)4vg%6)ORE(9%|K< z>)ZRIx#g8j!OQ^!9=0RtjKT_ihl3I{r4@|(nm|B#wmk7=OR`(JTx5!?cbd*7VnC?4 zDyRiDZ?jOblyx?VMdYkBYVG>hITivF`Rg#J0qilM^*+hhGr# zBIJ(YCwL4!(GU#QlYW4jky&JOxC zTP3(L{NQs}6h1~)he1w?J3#3At5vyWkq&!lI2;_^B~TtAXB2N19sJvLzlB&JL4(r{vG#k4!kvLSX^HDG|&4h2x&5*agobUq?nI#n5!5e2dEF_k{7gYZ*|w zIe6_g6Xp{54g&SvZ3T-Pevml^XIE(|x2{$NZF2Xy?KHWqte%w73uJzo^7qld2%Oip zL|^%QN|i79ibZr^szvBp_N|WvDvO2ZG(jZeF!glXIj?!afoJli(<{ycq*$i9p&9x$ z^{EQ!XTCtPR`k2iM2U%Hye6M7flUY2H@BMB1%pvpk@#qkwJiU&)T8i8Ky=A(UI86s z$+Rhg@EM%Q8Ly4^*+@k;vQSV`@fkb}R?XXjcEWB$x1d5G^zye7Mr>FquGJe+sXJ(L z{rh&qj8;QmvT9qcw+>}kKY=Zohxo~Z(64=Zex85!D_bCr3i!;fxr3l89A!&tUcYuse(J{ zMFg)dLTt(S_pa8LR!AW&cYs3U@1n)kCJpNkg4T1JvB=k;Askh@IlZ8B){}|i;Wzg> znLPlTOjcuJ(*uvec`H3Hep|BwXUS5WAgY*HvX8AG^E6)&zu{iTG&m^|b@?vk0EjCU zlAAjH2o9A{9pgy{H&C8~$y32wE_H~md`2zMuDV7ou+u{PV8iQYc+g;ni#GxyIdk`@eBG!t{5tm?4c|UKmyssu%p)%~vs_Vbp#{;p!kJs? z``b^U{w?aH@FS!otpe$TcGE;sAlJHJ0$r|K8X6kpS=_2`Mn9XebMmIKJ+4%IW$!8Si4J?h zLsvsKb>6oA#=ETojpyI}mA(J93r`b3=r#LF7)Cy0ofu+-Icii-ChPm^t0d^&gVr7b z3%_}p^HUABo{YfZ?Zyusilh^iU+e-V7LDx6KX;X6Cro3S1oIXT$1H@tr~Otpl?5Pp z128ASzh(;0u%E?NxeJ=qX@s!Ka#_1yN^HK^oA0F4#P)~$kcYrU8MPFawEl`KA5up6 zpJn}0&CWYM#@il(w+D>rhYVVoJ8dQUjgq5d#G&Tu*bLRAxdIX0bJsZx+2vMs&JE5g zJ&m#6skEz_oRU7Cy}i(_n*R0mJFpGYhI;f;VVhLjf(T=d4`VQ9801<~R}$!U(#iC) z+6AAwjf5V2g%@F4Kl#NlgPCM;VHGmN?PxMiSgBu%I+Zl7Sm^f)+Q&W`ixRXn>wNVX zsL=WH7;#{0KB|8tqZ%E-xfT`7;jgX=KqW-IA210j#qaU8Xy4rT|W4VU`kPnSJ z>!C=_5%r>&Tdbwa^86HI(v~_OtAoTpy%$?)cws~P-x&NbiBsJn?GatQuim(JoQDDHy^) z43&m4-dGH)!~IN#0Rah?pcc2sG$kJ9g8U);`TfsN<}ohGR1KZLwMOc*F|O~q&S;!y zg6g&4kawXo@OLoUE71L8`x$I8E#%$T)3s}bU?%(oDR2$=5*+V93d8ffO5}xYmR##c8(y%&3+HccKyl&tmFCI z*C&5a7y5o?s{t37%prLUhajRv0#}TL5$bmg2b<3&0Cr0SGa)W1`}l9Bzu)bcJu0_r z5_#8HeJ(g~kPpC_1fv?TfaI0hflKGW4wMR-UMb=Kca;gk8w8%(zD$rbN~!PR7Y-*X zsjmEP3<(rY{Ji(cQEZOOf^BlnVur+5u`~=rH#K-ku&u%-aSMz-qbr z+U+*|jG2T`lAPu+9x7f&7Y8vBs2J{9^QA5zUNc3kp`|<5D>XIsl-3?INpu#m&6&DN zCYqW7@w{Hr0Ra*Sp^L$usXfeooCUi@cE9x>*h-4*v ze9iaCo>Fe0Um0Sv>0^94F9JMl$di4svHp@>T@WfeM*jUu#g*qLQNg5tS^H_0EEPFt zsbX=0t>@smCBk~S_@dmI6TDdLOL)ZVo4a^y^w8a$woHVqs&;)Gm)lNdBHWv6(-8G~ z>_e5fbyKp&1w}`*6Z@@8F2O)UyDfvGKXH}}gVq>>hhy$^fXJpoe_G2?UPYf=!>+cL z-XwFo%ChOnrK2tm3KQr{hutvrNb~{wuPiy!)u5{31$wO44DZRPP*6O(z+wiD6H(Xn zM&N#4kZnUJ#{#PvPGLVwmi$dV`PH1ikRAILdov+e#csOew2yt+*44E($Wd^V*S&j4 z551>Z=S#r&O?zwTck0h4y?o6b{7$t9ndhwOHw%3tBtKNv5bmsbp;kXf)34#qhHLO>z6O9c$`#(;{-TsXN-oX9%IH!5ma$=;w zj{?{@i{(;pfVWHBZujJGgf+3AP=kd*OU_Vx^<%jTRE(Do@5w-{#c0)TXSry*t>ctg z^9$|mV>vM#Zaf`Y+6)5nsaI$>vbji|4B{jIyP9S%hsaYKlBUbKoGH#?MBXW-w}xP? z>Hz%ka`%3+9aEBIT`?H=7P zHCZ01q&+f|=0~PE@6&b!VB_DhXb#h)h{u(8*{OngTe0JL3?yYM8)p5h~X#5X!tiL zO%P_(sNhS;aX#r`SEpS{cxzwUQs-fX{&Y4fgmBt1@}*b_COa2o$)5V z7^Y2)_ghA_E#>Y739-M3krj^+$P3drYCb&e6{uH%Z2tejP90^f}tld~@>g ze=-_^T3PfPZ@1iLzp}gXa_DnFs{5Mlw3XpH&%(@9|8hk!P7&^yyL#U8pFJ0B%|EQc zxR}l_drM%JcTbZjNQ%GG)O?zKtD)2=(zlniXyRhCP?`37CjQU3{=>`sXJ2VA=T>)- z_F zwR@?iT~rzy!Y(q5l*LqfHPeMJzTNZhdvn$tdX`(7Ii>N0;}vPrP&@0*yjcrjLm;AbenP z>FTCYLsy>2i5sqjGvOes?%%42AfBgDs4e&y|-f}GywB3qvy3lw{F!`7kE{`uZ;zp403i#B*z zcOebO;0?uR1bTxjaC#%{Y3Zjh)c0*n4gMk$uy?mP91NK&XCWKp6GOgqY6>KjGo?XT zNG}P7A5D+VknTn$1rW7Ye4-|oB`eIHYzLnk0Tb6)lb3KGv> zf&78Ere!Co{QB+?Vxn3{ZA=l+;0NG&1U(%t*%3a*7Z*ByG?zv9TN-px>F zxltd|)K|t1$~61z@pgFYDQ6rde}aIh35s`= zN2Dva@WvGC51YY*Odw8rga3uYXunOTp#5XEvG~XAA1r`y1Zwlonx-f~?k-&?-7cNS zmj_Dv*LBxM-s*?_i?D9ZsMCJUILWU1C_A+gH|bwk=kH4M`wVIYCR6k$JNM5Ce9KFg zw(XgzmiGESHSS~W>YPukN5+VD#@ROAed!bkXH*fCdM%tfmcH6h4kBw}3 zh36L{_6qc85m{j>OxC=Zm?_nrusR%NtiODTaP=GolVlaD^6pLNu`SP4sI9I{-9nJ7 zfCu^C)RmMr*%{O6`w1U>mz|$A1Ao3l>T(OS&3s~b5nIkPrW|CN0-MJoiK(jc1_z{t z5hQojTWe&6<<2RxOGKb%g*}c`89yW}Z6t!bU`(uJdO+X4{urM?<<5juwek|}i0Z~7C1wJg~VAwD9I{@RN1hJI0rMzlle~ctJ znpya1JAT|g(93#uQdgS!xs?rF=v0rNJ7&DFc-eBcU7@JMQZARGbr0dQCBm=dTnR{X zD44ar+pUR1V#)p6TNO4&mR4wP4$rQNi`*Ze^ zC9Sa^6&}9F6?76SGa&(z4>5Y3rf1}fsBR6Tv#j8Wde?ScrxX$E6-u@ggCpyaQ~jlk z=Qb)D4cw%Wiy&v-eshZGl7fbkzwIEAWVd}!-$d-fqNY^(E^&$q1Kw(Bz2^5kU1KNk zoZb8dPDbJU$JpA;d}k{D$qILz$Dm#SplZln5Raz6$>QNDUO-|0saH!Y(2#v{%8Mah zNGGVJi117XFXVajLiv3*R*T7&e@~MQ%Ly9^zokSjKU&eZoBHSK!SMXgM>5ABNSdf( zUIK&|fxpY3_ZXV=M<$$Edjsm(jepP|EU0%^(ir40>dzj!-cAnTqf}+fNr;)bqnqE2 zDJ?uCet&r}{cQl<#&j4V#>exxDCXvYt7|KFmf%r;8%5`nq!`R_F=&%R6j%{|>1tgf z^u1<dKPk zwl90>{wZzWBYsLofHW&|aj9fB-k4LSh9d*#+t>4=`_X7)S_>|_WPJ9*q5pFE6@DrL z5|hBY)2ykuQe#Y4?3Yjgw}+R|w|M$5+LHFgtn~cBH+pkUVn{sSj zW?vUNDX-9)SikB;;bT3)lgUDgV=0aKsSGO?A-g9-`r+wcb>!rn9iR@N8hK z=x-);)W}>h6x74$@v)zf)Fe~Ef9cF;Vx6>^vCe^8FuVj7j^Zo*w;s?(Se{Qm+SSny zvwoJRB+=64Vl~R{b^cOgf!``@_O6OT@E^IPP33ZyB=vqt2$6CqT4|+5Phcak5+X`H zu#`gEby03HsGZsv3bu@W>WWx8xt~uGeOyR4a`n|Z*65i5RSR*14TFwgWAtEYs71BW zDb_=rd)j)ci(5T4U4Dy)gAeE z8|HvgFT_~iiN4%asjK83CU5>0$YqTJt{XXBg-`Ls#&Fe{wIc|F*tyE&5=z-qE zi^K#_likclJQ`f|$ zr$W5FzI>(jZz?T7KP`7bMR)th{Q+_&k^+gR-9c4C>+Aud51<2UB+g&R>6)6Mgz%8i z?gv@N=$FfTm5%CEmap!$+j?0cH12~2H=y?1_!>AB0E>rg=vQ2@6^M#|pDg`+azBsi zG;~E%0nTGYC{O(KnTE+PYU|P>waaFgYPX5AdvS`?IyGu5CmSpw2p(*)3hQS;E+{cn z3gJOG9Pr`R(lpHB<`LF|`z0J)dp0M?0<+qN55H&c7fgMdKZ>CBDU!?SFabKBg3C zgG-_*L~Qo<1LKK%&iehNt~mHvh}jSO>=Ckuc?%ARA`TwYviwM48VetRhO~M6@4SHc z{2>>F`#1pm4Sga5X^>!w)ps;@nqA_*8sZdAa%}112~q8M$ZA93vl~&BTHvsnw?*mn za$WDaID|a91EQZQL?+Z9jJ{j6Y4Cdv5RzS@U+54+$mHv2C-$R_+oB)F6!?L}*eSbC$l&TnSM<zOD$RC-Coe+($!;j-_46nQ>AihhT3N(dYl&To{5MsK|zt1mb zRW1+nvyQrMf$1HC(V(nl{bj&@m%sfSatNLl0_W7S)oCCWp3Gcc+TZ`N*WOBV*D)dD z9p%Fjz*lD_P#F;_J(XtoDYK9^+Ed+QMuo(=L}gG42YrX%4+&4u1aD*ZkF02=>hE(1 zbMNv30tOjp=`!W*Pfvte6ED#jiCEdOW(gi1YOjK(=Z}OGb+Hmna1u?n+#(^?ps^vEOMQx8yDxdbchra`Hiy>rB61HLMr`c`_@1NekbABxxyHaG>&u1|m z+nmV!CIC&wvza4@fDJ|B_dWfp+Bn=U7AfIDH4!t7wRGx7^7l1PxPit;F%~TF6f??r z(jZ=07zVjEij~kob^T8+-<}o}8G6mXq)nEhyd2Fu2_?c6-z-}CmFgR_8~~jCCHO&s zVvTPRx0rN}S0#jV_wm_0mA9~$`7iXj?*fK{zHXr4X|k|uRKo;%7$_G?9pwt-E4?_THc&U++5@{GnX^`4G{P_hoHP0MwhFBE;XL(>T8(>q6{r8o#w1?i6xfRq! z11aIp301595Po_IIIrvC0-YNL@aX#C3(aj?}|-Pq7O@-16j zs~e~gx`0_!k7S1;>?tczCYvTT9UgqdQNIvby%>}DtPsa?%n`_U zPF#R4IK2pILi)$#tCG4gtDM&?J0ya;jCyGvCCNNYn57OBrV`*+`lki)D~}Ho!D;3- z2*KQGr%P^LL)Un@iX}HhW6I?W$~dbZi{AiBl;4ylTO`zhwcU%@#Tj>dcxuJ5C;A8A zPo-i1In9?SI~pw5xA z54|W9RJ-=+5U8eRhv}u~zua_Hh76gydl?0bLDB~Y+f*Ym{}6;I`I;rl6Y0kq@WRlh z;Qgd2XzTBb?+YK61$_(;1`e-3@;1Dn^gWObmIw5Er zJwrvD`TWaid2_BKP(lBS%D`#jJq4x2OOJ^RI)JkD_eRCb*rj76IP-CH9xg z>XhwhbP47-y!<+8$cYzy2Q2PzA1sb8i!@!&L~+i`?2eqkyEU@UY1q}Db7_*oB1EU^ zJFH{mpRsu9JF8ih55uFaivnl?Se7wDGz9rAzf)SO-@6igrCh7s=-O&wEFVqzBXBc2`vd0rO_vod+S|@RL4xfb&%Y)-vqLo$@nopsopW~O zTuBL!uPS?Jew@oWK1Yg`Zg}m}C>Hlg{pd**d}fqzzb0@Q|D5SH2rY~{RzcM%EVsBN zOX>0YNE^L6ZkJHQE6fJd3V50KsolZ5*0Tb+cVd1_jk~v@p)pv3uLp$0!%i6IaoY{vB3+Q!M+}s{(irb6Os^Td2>0@)`Ns5I zRwFG@c_WW}H?SWUen_x>gj@68kZr~7H-&QnuEC3^b7XeUnqjwB?GfdcW<`RCfp9;) zjaRM_Ca1C8e-Bdq-xR1ktek!QZL*R9iD9FehBmsQXOZo)s~1dViw;d4rC|edO9S0? z#ZlVrF_}?bS1n>>YpgqtZa6iM*)0N*oFUMvnK2<6t~XNq+I3-U9r;WnNN)FOoMhYK zxE2pL^P-IY1^(hyyYZI_b+?24Vn78;-Izb)vHUG2v?^ab`L|}OOsDm#sW^>mHcXWq zYypQgNaao3_2&qyRu)D(_Z@Pbe>`A?yJvt|yA`P}n1+O6nN4`nOsv_Q_$f>n;|?Kx zZ{W5AU@OX&(atTZI=kQ%0@O$tm*%W{aJ1+qd5*>`^$0Wdh=Fe`EnYnaay{PWpl!48 zH>bK}jz}&@cOgH_i^ID9AniNf4$B#Fh(KZ)9xiI$$A>GC%+B>U;vOZ~YD&!_NuH3Y zAv+on=USg#Y)lTSexqWr2o=&F%PB~=!WmPv9(z!oMd$Aq+O?Kt4t@5hi>S z=4R(tP$fgY1107oGfJuWjwRkqd+s8dBTNS4A|_YXs?5*Pg{T$SR2Q#SpI1~I9G%xP zrQYkzR=%8q~z;(qlk8>{tZ$kB={xEO=E5z z-wcScPCb|n63(27g11xg?GU|XE)x@G88jX+A`1Er8hK{jzsD~49yKJ4k2dU04f5RB z)H|c8CX-BPnBy!d5F>Qi5Euj(Zt73G=6jJ9jwhORdAoi!(@#M5n8}7mtWe(e1kT3r zoi7MZ1x5GimrUdt>WvPyu}j3oXOU*KI=U9EC~)622PP{9m$!;9XA?H1Z99%lotHT! zCrptDWrqbMqCDtq89q0%7qd4?5&YFRw)(W1Lz&Blca8@lg<}t13Y>O+qXFke0cw?T z{07~B5N>JG?9Yzj-n$GDGhS>B95$*9+qWa=_R!KOij82?cK>b1VY);7Szej=;-fwT zSMfuf5?Mt=)>6%IclqI}5m$4ya?qtUXQvDHCvJ#^zkD@ls+UTkDnI4ZvdZya4Tlrm zfdh*JYeNf-6m}|g>qVw=B}$r2zey}C+sJyQzE~FBzWcY4lEHh*lVohME?uRBXJNX= zNs*t|AWIvoGc1$+=cP>Ri)6|`O99-cuX92ELrx& z;qz&>4`=NoiZsSeyNTM?!u2nb+GbOZ?W~D7CC9(>7W54sA|` ztwy5NZ#E`HYb2}=8y5J#eg7G5LH-MkMxHF};hOiB{=8NHzV)wR8d)C9CO%e*-_vpx z-u7=~A{92{M=Z=mnt$*+f67t|^RYYx+$vTNJd zn-_Sfjg5%^t-aBD9nbHzqhJ~|KBI`t#JpfVk?4(sKRbKzWgCibvbw^y5V;1~T&rQI zRXX#6gGc(^#m#A93S>s@BBc|`@?5sRCsMD6nzTQcaG`gdRIY$iyjY`{!}PS19?oQ( zsg2`a;7e-1(8Ke`81e#P3E;70h~Q@7?Q<(xW9I#iV~F0NMq-V#b>#it47SJEeMR#s zfDiayZrb|7fUxGg_!G}OFa25#80e|!?XN3Y_Jopw3pV(Wjawxj{4-ubmguMozRb0;^Lm=Kt%moSB&V;>?dyH>lwA8FM9LP6!V?w+vOtcxN_;hgL+yKRtJ=)&n#N1 zXhKUe{D(BWF_DXdOiHMxi(4Fp^x!hK=O`-OCSme_(nY;}>7hNj=7z1485CbBa|VqKYm`%s(hQ*A3|?Oz>8 zZmYZcm5U9x(e{!DY>45Q%s~mLHSl_N|6pp;ii}n*clmYz;GyC7-vAu|6Pwl=Wr;j;3=5GYFJ``9SV=>=V^~#bXMNd+^|xJ$}EviK>Ax%uD2&G&2P3F8y)6-cW}J>;e6%%{OdFWce)3qg_F`(T^c&82}~85>Bapr zoJ7a`YJX$3u@WJ>x!XBWICQKNEY?t_3M?JWaa#89Q3!die@p)mgdCiZ7xR_f-e9yg zPKrgVmtDz{=|iF}(vY$4NWv#fq8DABy4mDYB)HIB!_sRT0^%a*m)Cs|9L%umy%kkR z4*`y~8Fk|RA;fBRKtGD|dhOw-$viD&8uxZc2uSK=w_L5Is2qEwNC}LHx!<$tEe^=O zLH)mK{17zpmw*Aj@V)d{e@m1&c3*^c-mA0eU3Ur*?9lIw0US?k&J(m&X*Wk9c_wr zVU=d&szsUqUA!femf>zwH6;H0q{l zUhr;T1mx;UG~KE-4$zOR-H5}V=C+CpLrn9l!}98uXorg<+c85AvOf*BRsc-cNQBoO z@~*Q*_j?Dl?6+C$tMete@g=@a3Y=bYgp}Tue;81d189xAj@1hIe`S9;(~vlYOIoQW zbyfGiEXs9>3W#I>Q+jL1XpQriBBW$OXoD(u$_y)?nTpb-Q zhP+%d1=RnkvXEh@|g>sY}VqP0-7(kPHQTKc)-qxW2p=NhNn{MaOGlI}3ij_Q5 z|Mc59;>xg4K>kpu>_RZHq)Rr|qFV#NSe+CFYsZeaWkvz}Q56rT^kKS4|p%f4-dYvldhlVt0 znLZ7;lRh5svP`L%4qnZ|PcVDNquM38ED5Q1AvQJaeyRipDrS6dJW1u1xn}n|l?@EU z+7_177RUR-GE0|;5=aUu6&;8d8vm=_-{!^cb*x=X&Q#_CWj{?irMhank8+5t;04cY z!^UEv1)1_GQ3(2fuKV2@LcLyM{Hgv7M&UR*L@uxs-hg?~KW04~_JU-M6eLWQy5N>F{=U4MmIb5f9_M}OJk$NwWvxk%tBjzHk(150RBmu%|f6(X( zAtrhUwV67ax^mR~|6g^VkC$2}NKTCIX?*VqHzu1lIX6>h;RW0rDBzjZ=Eb)?U%Oa3 zErE|2rDKby7gj0yAJx6oI$KNX{JGXe1dG{Sa)-154?OJk&@cjn!t!ffQv0Si+lD)S z`=^#yjm}%i-@c~y4--~EZaoF5n6~Uh-=D@P1W7$*5kR9F#MyR7kb-Nvw8tMbL>E5V zRuwu$TS6pE!14j-6cyvoZaC9X#?oseiKdCfZhBWaXa0JVoBrkw9=9x`w|*+!1crI5 z6z|tDNX0vBe&syer5;FGw#Y+)6FHd89q_Br$17NUeadKM{TmGaPXa<1dpiqeNJ3T& zd_h}_R&DZ3g^%oq4^x@iP@NtlRVunQFLGbvP}3Je>V-uoMH=s8+b3h=vC$G4;DxHX z=dBNK8UJ)_45pr--__!$N%Q@JYLglVUXKIfr>@NJ+g9pRmWgw?mY0_Z*c7-lkJ)TJ zk0p`;4|JJRH*3#5i7a~_DCd!i8&iJP|0Q(@KM3DG@ckS<@lSpcCwoPZPAma_9r~)Y zio>FxuOuQCaf##m$~7sP?A;-t$LRSxW|sQYbu)xI`*uUT|Ind2yYFtkP@L|Y_M%&x zPA;dM#3xSly4eM(qmNKR$M#3th!_}>4XD7HZ-`&^>U>Fq3zCLw0K_R z>#mnsF8!_D{Vh4D*rDfw4Ep(vi*8hSlNK7`Pjwr#QatJ6SCIJm8@P10f`T<3w0K1c zNUYcxZ2jw_pOBxiuR3<1@H}!J0*Z0^5_+PM-0MU7(GNQrNwGk!(p_m<8W0 zw9$jzw#`b1^GzBOm0=DR9r?X4DD1FnBcFGx)hFAYa6p=56E=QA5#ybZd<$!>erxYS zGX4yw0AuusVqxpGPIQ}yOG3k${4R2TNY-{gXjE=1G#t`@=C@L9vi-)PP=nzf@c0WF zKjREZ2+DtO*U9u79a~&dcEb~^(*M<0z9;_9EzF#OfPU(^;-wUm&y^tCZBZBwH{{iA zqRB!wV(r}GpztQM{zS)9deM}Uid?8u&LIc4P$yPL5ZYnK72biGg#7P^5D zN)=WPbTTT?z=m|rsmWdZ&>!5?unjQia0-#4Tk5Qn+J9_5`IB8$kOK%>HSh4Y;lB@w zXy?M?mC>rrW;2+c()**%GCr#e)zFl8G`+a;Cw-ti#hSn%yZ%I$qZ&z55;2j{5ey!q ze}GiEILUcj_Ts6;g2NRiUs4+-ccCF4f=}Pw7pg7>^80F-lqb7(iLSy{=x1;9;pOfg(_)#IdC~8olUY_i?kRk&38;%ap^v?VC4Lxa$|NltYFgtSJs?!9!?fp`DeJ_kkRxYO_Zh zK2A0#0+%~Z(i9B%z*L^ac`radYc0HO;bS5jB7k4om9N{k0#qC^i2lpHd{_da@14Y5 z0>JD;sL1|Q>_Hj&-vThUb*6zHNS$#%@6`)Z7UWJb0fB+btF=~Pfn3zpM4X}h_Mg^R zv$2@v3#mS9MwA^PLsfkQ9fJcs zs@k7L=P|D~Wz$Ab)(|gtDMfz>;BRFQj;E;m{ZZV4huB2GZLsCq^YAbr5$x%6CFD3S z#21m9C$0Wy7+xe7O`S>oRFq=7+>mju6QzZVZcj_MZEnNh#pXnLk6fz{lcv^=R~Or1gt~4Hy4wq#PUhX1NOen{zsN z5#`Squ;?O&SfG8!Qu>taJLNNV;I+<>eX~-&(gt7Qv#Sj$^#h2Mvu_92#d|XRcMuM0 z)5GHQn{&&{bi8W=-g?$yV1yU^&=Bj!_sh^2LX0+|hbRu)i8-G=n~i`T!Y{MQ=qVio z?p=?a=ieK72vT>v`5WsKglMP3_*fIzuvib=jr~=Zqrk=88m47o%?5va>@`}&HAV5I-owt^1k*1 zEOQD><$>(CFKA@)_D5#+|4_cZf77w!72|i9XNmuRc)F(WNSbiHn~iPTwr$(S#;-~aVCK9Mv4Ro&?e zofhgm*T6Ty+dd|Mg_*msr_~EFo@vU+(g%~pD1|n|1;~qTKX>FaH>3Li=fN44Ze-%f zUi{v#VyUu#;|;WdYHqbKM;e2NlHh&I2br16XjC{Qylch(n!=WhxnH zgKyc!KUuXid0a$MjK%ff=vc>``#i*ydbfp!x6+PzQ#+^>DN9&xDpT1b2%OAmm__RW z0eps5WN#x?6?p)2Gk=^5PhuK;AQaY@cQm*i150NKQ$27q0*{1m;N#fCIpdpI{1vqCo)xz_iCDB&^vSk{faaL|5Br@#3&WK2K8@M?|ja}W`^UZ_HB0l z*7Hmpq~-A-hT;R}#-Rv?V;fj{_<-+du6T2#tpjKhVC^2c8T%C(-6OC%1Hr+sh|k9_dCF8Z{f`_BZV}nK5DDgHHb5h&dRTl=*owOZc0W`}gzqghX7(Ei$IqpGe5x zGK%G$geQuOF@=D1ep8EYd8px7ea9zBKFLHo9?dWyXQyYQ90(#i zfgx5{0&PGx-XZf!YBzo>M>N#RmWnb@P`^EFP4N}Oyb&>tkkHMV@(M=#o+uV1RFDi# z2z|^B?D@smnn8c;5Hu@;MrIRAEF?nBdF|A_SO8@#K3m`~ltwTOHtb>$fqihw3co3E zr@=b?lL~!)lrD5tJkRoVzu_Hrwd|hRb_N42*BRC0GBT@`LDieGSv{!mdtT>+C2~A5= zai-+d?n@>6L!tU@reqS=D_-=1_|BFgHT{t{4|&XkA7-CJW{2MSIkF5?R^#tH0+bO^ zKv-KG6JOQ9Ki1(oCh21nOY~*tbRAjTi*OzZqF1H$QKC2M!(-guT_A-P2OQCLz#_~1 za9Q@%UMp{u@UreVd@f^_dl~zk#)KV7tcH^M^v?&qHmS%VWbbbX923u zWX(SJ1_FS>a$I}96EZS#lOKKya*o(K!(NQ$;^*C-u#iC|09yYv4A8139M5P{6vXWD z*YM+salkBmx@dwA7yVB75flXSn{N1K1-Io{FzXWE(E)W;zbn z-#Zc9QnH%*Cp(}!QX5&pbOb+I*U%VKraI4#aAN0NPuefbOjU+PD%y*GH^pXG**ii= zPy5I`w>y;S?q%Dxe${YS%*i_uF(#HqPlWVZh>9+-t59W(UND#gQXIFvk@jb|p8*1C zGj0q0(pKCSdVtIH{@HG#9f>op#@%$jxPT2dI#8VUH4`Hm@{&g&)v8?sR~`J36i_1O z!!!{Nygsybm&)7&^RF@WJEUdXaEup9l`ACrw+HSyM&ud$pOAEU0Vv{G$9Cj71rbt& z^;>11+TMu+?@+(S`ygTL2O%7e)i`74pgSJG!i_`K$hn{S;Q? znc|kro~d%7at~d;VNA4(no*>U*vIK2Zl`DEMd@YUWOK;8!(5iAGWBcdkIYBO6+A2V z>kZJQA{pkC(H1I8V%yW(q7c$cE!fNa7UrYA(!1&BcWNph8`a3QpeY`Me(N*&%NOra z+aZyR;{ALUyG?uY#vj2D<7)KaE-$B>x0&Q;b6HzpAuCoI(8-LU?zIJEt^ z71v2qUs!8aTWwZ<(&t)9%SAGCuumixJ96i{oDB_0;{ zq%28TU4rlBv}{!kS7t;}c3BGjlDfRu*uy@va$6kM#3u4$wVIL{y2W)Dj<-?YuO63A z3s$eWFjJ?9KMBXvaMFQu3BxiO3V`X?@_|~lmvc&>_FAz|3im+}xoN(BS$LJi3G-(+ zu0SqnAmEUtIm_5rCGp&Hd|d>NgA&r9a^km%uTfahTQs*4ymWg{_zgbD;o8fe4YD$L z4Wy4^`a!Ppl%veN@V(qnle@y^a zb-vH4jXe$h?P_<8W7B&UzV+|&)GqEu|ItFiKj8S zF2qCm?8K3~RyY(H<~zU;Q65j(H<#wbjXL*?8TmfT2pJtl6O!}JQtU1-#D&MRI@{X9ev4Q4^|SZpl%@9D;h ze3pF?2AcW4gCAi=&s ztg5zZvsc@QWebcBkpb6h1uY6-(F6GM;=+Ogw(tfm60xHV{GKV`>~wJ==_6;)Jpi2# z6;dY%sloAh%pG*LaFgi49)5Cu4B5sa9me;Z&b{ zePXf`ktc8JlcACjIKDf+_UqRZ6zYCZ!0+Gp#~MpU0TI=FzxAk$pEbyi%6Z@DULN%h z15FL>oWtpk%n_yn8g!KN&*6t5Jq%_X;gl_dDUoCzcU0O8DecekY`L`Y&D_q;{${jW zg5f(Ctv&v~x9q1GeCX;s8lZ$o7KK$)l@KbISj8++GgV=ph9C&K#?|&`r`0O-ChXi6 zk3b|U{VI-M=YxEfg2=biaX$dTL>zn~m|)3KG>2ugbrR_D6Vh~s>YbE@gsOF6`vcsC zEq{-J3(@Rt>Z%UsE-~%b6!PJr=4mFjL5!!v&IZ3GoN(TxBSr@uqKSUW!`+FBqr