diff --git a/.github/workflows/update-libarchive.yml b/.github/workflows/update-libarchive.yml deleted file mode 100644 index 760911224b29..000000000000 --- a/.github/workflows/update-libarchive.yml +++ /dev/null @@ -1,106 +0,0 @@ -name: Update libarchive - -permissions: - contents: read - -on: - schedule: - - cron: "0 3 * * 0" - workflow_dispatch: - -jobs: - check-update: - if: github.repository == 'oven-sh/bun' - runs-on: ubuntu-latest - permissions: - contents: write - pull-requests: write - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Check libarchive version - id: check-version - run: | - set -euo pipefail - - DEP_FILE=scripts/build/deps/libarchive.ts - CURRENT_VERSION=$(sed -nE 's/^const LIBARCHIVE_COMMIT = "([0-9a-f]{40})";$/\1/p' "$DEP_FILE") - - if [ -z "$CURRENT_VERSION" ]; then - echo "Error: Could not find LIBARCHIVE_COMMIT in $DEP_FILE" - exit 1 - fi - - # Validate that it looks like a git hash - if ! [[ $CURRENT_VERSION =~ ^[0-9a-f]{40}$ ]]; then - echo "Error: Invalid git hash format in $DEP_FILE" - echo "Found: $CURRENT_VERSION" - echo "Expected: 40 character hexadecimal string" - exit 1 - fi - - echo "current=$CURRENT_VERSION" >> $GITHUB_OUTPUT - - LATEST_RELEASE=$(curl -sL https://api.github.com/repos/libarchive/libarchive/releases/latest) - if [ -z "$LATEST_RELEASE" ]; then - echo "Error: Failed to fetch latest release from GitHub API" - exit 1 - fi - - LATEST_TAG=$(echo "$LATEST_RELEASE" | jq -r '.tag_name') - if [ -z "$LATEST_TAG" ] || [ "$LATEST_TAG" = "null" ]; then - echo "Error: Could not extract tag name from GitHub API response" - exit 1 - fi - - LATEST_TAG_SHA=$(curl -sL "https://api.github.com/repos/libarchive/libarchive/git/refs/tags/$LATEST_TAG" | jq -r '.object.sha') - if [ -z "$LATEST_TAG_SHA" ] || [ "$LATEST_TAG_SHA" = "null" ]; then - echo "Error: Could not fetch SHA for tag $LATEST_TAG" - exit 1 - fi - LATEST_SHA=$(curl -sL "https://api.github.com/repos/libarchive/libarchive/git/tags/$LATEST_TAG_SHA" | jq -r '.object.sha') - if [ -z "$LATEST_SHA" ] || [ "$LATEST_SHA" = "null" ]; then - echo "Error: Could not fetch SHA for tag $LATEST_TAG @ $LATEST_TAG_SHA" - exit 1 - fi - - if ! [[ $LATEST_SHA =~ ^[0-9a-f]{40}$ ]]; then - echo "Error: Invalid SHA format received from GitHub" - echo "Found: $LATEST_SHA" - echo "Expected: 40 character hexadecimal string" - exit 1 - fi - - echo "latest=$LATEST_SHA" >> $GITHUB_OUTPUT - echo "tag=$LATEST_TAG" >> $GITHUB_OUTPUT - - - name: Update version if needed - if: success() && steps.check-version.outputs.current != steps.check-version.outputs.latest - env: - LATEST: ${{ steps.check-version.outputs.latest }} - run: | - set -euo pipefail - sed -i -E 's/^(const LIBARCHIVE_COMMIT = ")[0-9a-f]{40}(";)$/\1'"$LATEST"'\2/' scripts/build/deps/libarchive.ts - - - name: Create Pull Request - if: success() && steps.check-version.outputs.current != steps.check-version.outputs.latest - uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - add-paths: | - scripts/build/deps/libarchive.ts - commit-message: "deps: update libarchive to ${{ steps.check-version.outputs.tag }} (${{ steps.check-version.outputs.latest }})" - title: "deps: update libarchive to ${{ steps.check-version.outputs.tag }}" - delete-branch: true - branch: deps/update-libarchive - body: | - ## What does this PR do? - - Updates libarchive to version ${{ steps.check-version.outputs.tag }} - - Compare: https://github.com/libarchive/libarchive/compare/${{ steps.check-version.outputs.current }}...${{ steps.check-version.outputs.latest }} - - Auto-updated by [this workflow](https://github.com/oven-sh/bun/actions/workflows/update-libarchive.yml) diff --git a/.gitignore b/.gitignore index 6a7a508c79ed..4b09e2f7fd5a 100644 --- a/.gitignore +++ b/.gitignore @@ -153,7 +153,6 @@ codegen-for-zig-team.tar.gz /src/deps/boringssl /src/deps/brotli /src/deps/c*ares -/src/deps/libarchive /src/deps/libdeflate /src/deps/libuv /src/deps/lol*html diff --git a/CLAUDE.md b/CLAUDE.md index 887894b69491..8feccff501bc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -147,7 +147,7 @@ The Rust side is a Cargo workspace of ~200 crates rooted at `Cargo.toml`. The ke #### Vendored Dependencies (`vendor/`) -Third-party native libraries are vendored locally and can be read from disk (not git submodules): boringssl (TLS/crypto), brotli, cares (async DNS), highway (SIMD), libarchive (tar/zip), libdeflate, libuv (Windows event loop), lolhtml (HTML rewriter), lshpack (HTTP/2 HPACK), lsqpack + lsquic (HTTP/3), mimalloc (allocator), nodejs (headers), picohttpparser, rust-argon2 (argon2 for `Bun.password`, Rust), tinycc (FFI JIT, fork: oven-sh/tinycc), WebKit (JavaScriptCore), zlib (zlib-ng), zstd. Build configuration for these is in `scripts/build/deps/*.ts`. +Third-party native libraries are vendored locally and can be read from disk (not git submodules): boringssl (TLS/crypto), brotli, cares (async DNS), highway (SIMD), libdeflate, libuv (Windows event loop), lolhtml (HTML rewriter), lshpack (HTTP/2 HPACK), lsqpack + lsquic (HTTP/3), mimalloc (allocator), nodejs (headers), picohttpparser, rust-argon2 (argon2 for `Bun.password`, Rust), tinycc (FFI JIT, fork: oven-sh/tinycc), WebKit (JavaScriptCore), zlib (zlib-ng), zstd. Build configuration for these is in `scripts/build/deps/*.ts`. ### JavaScript Class Implementation (C++) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a696e16c6f20..e475dd5e1b32 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -341,14 +341,6 @@ $ sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 $ sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100 ``` -### libarchive - -If you see an error on macOS when compiling `libarchive`, run: - -```bash -$ brew install pkg-config -``` - ### macOS `library not found for -lSystem` If you see this error when compiling, run: diff --git a/Cargo.lock b/Cargo.lock index fc0bd1c3794f..d0a1b614ca6c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -154,6 +154,22 @@ dependencies = [ "libc", ] +[[package]] +name = "bun_archive" +version = "0.0.0" +dependencies = [ + "bun_alloc", + "bun_collections", + "bun_core", + "bun_errno", + "bun_paths", + "bun_sys", + "bun_tar", + "bun_wyhash", + "scopeguard", + "thiserror", +] + [[package]] name = "bun_ast" version = "0.0.0" @@ -724,6 +740,7 @@ dependencies = [ "bstr", "bun_alloc", "bun_analytics", + "bun_archive", "bun_ast", "bun_base64", "bun_boringssl_sys", @@ -743,7 +760,6 @@ dependencies = [ "bun_io", "bun_js_parser", "bun_js_printer", - "bun_libarchive", "bun_libdeflate_sys", "bun_options_types", "bun_parsers", @@ -961,23 +977,6 @@ dependencies = [ "syn", ] -[[package]] -name = "bun_libarchive" -version = "0.0.0" -dependencies = [ - "bstr", - "bun_alloc", - "bun_collections", - "bun_core", - "bun_errno", - "bun_opaque", - "bun_paths", - "bun_sys", - "bun_wyhash", - "scopeguard", - "thiserror", -] - [[package]] name = "bun_libdeflate_sys" version = "0.0.0" @@ -1471,6 +1470,7 @@ dependencies = [ "bstr", "bun_alloc", "bun_analytics", + "bun_archive", "bun_ast", "bun_ast_jsc", "bun_base64", @@ -1508,7 +1508,6 @@ dependencies = [ "bun_js_parser_jsc", "bun_js_printer", "bun_jsc", - "bun_libarchive", "bun_libdeflate_sys", "bun_libuv_sys", "bun_lint_driver", @@ -1856,6 +1855,7 @@ dependencies = [ "bitflags", "bstr", "bun_alloc", + "bun_archive", "bun_ast", "bun_bundler", "bun_collections", @@ -1864,7 +1864,6 @@ dependencies = [ "bun_errno", "bun_exe_format", "bun_http", - "bun_libarchive", "bun_opaque", "bun_options_types", "bun_parsers", @@ -1916,6 +1915,17 @@ dependencies = [ "bun_sys", ] +[[package]] +name = "bun_tar" +version = "0.0.0" +dependencies = [ + "bun_alloc", + "bun_core", + "bun_zlib", + "itoa", + "thiserror", +] + [[package]] name = "bun_tcc_sys" version = "0.0.0" diff --git a/Cargo.toml b/Cargo.toml index cee148a2b450..eada0eb0235a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ members = [ "src/install/windows-shim", "src/parsers", "src/io", - "src/libarchive", + "src/archive", "src/md", "src/paths", "src/resolver", @@ -44,6 +44,7 @@ members = [ "src/sourcemap", "src/sql", "src/sys", + "src/tar", "src/threading", "src/valkey", "src/watcher", @@ -383,7 +384,7 @@ bun_http = { path = "src/http" } bun_install = { path = "src/install" } bun_parsers = { path = "src/parsers" } bun_io = { path = "src/io" } -bun_libarchive = { path = "src/libarchive" } +bun_archive = { path = "src/archive" } bun_md = { path = "src/md" } bun_paths = { path = "src/paths" } bun_resolver = { path = "src/resolver" } @@ -412,6 +413,7 @@ bun_semver = { path = "src/semver" } bun_sourcemap = { path = "src/sourcemap" } bun_sql = { path = "src/sql" } bun_sys = { path = "src/sys" } +bun_tar = { path = "src/tar" } bun_threading = { path = "src/threading" } bun_valkey = { path = "src/valkey" } bun_watcher = { path = "src/watcher" } diff --git a/LICENSE.md b/LICENSE.md index 3e1b37f1c940..aef802e5cf21 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -22,7 +22,7 @@ Bun statically links these libraries: |---------|---------| | [`boringssl`](https://boringssl.googlesource.com/boringssl/) | [several licenses](https://boringssl.googlesource.com/boringssl/+/refs/heads/master/LICENSE) | | [`brotli`](https://github.com/google/brotli) | MIT | -| [`libarchive`](https://github.com/libarchive/libarchive) | [several licenses](https://github.com/libarchive/libarchive/blob/master/COPYING) | +| [`libarchive`](https://github.com/libarchive/libarchive) (not linked: `src/tar` is derived from parts of it) | [several licenses](https://github.com/libarchive/libarchive/blob/master/COPYING) | | [`lol-html`](https://github.com/cloudflare/lol-html/tree/master/c-api) | BSD 3-Clause | | [`ls-hpack`](https://github.com/litespeedtech/ls-hpack) | MIT | | [`ls-qpack`](https://github.com/litespeedtech/ls-qpack) | MIT | diff --git a/docs/project/contributing.mdx b/docs/project/contributing.mdx index 6f76099f7996..c341ed3d01fa 100644 --- a/docs/project/contributing.mdx +++ b/docs/project/contributing.mdx @@ -345,14 +345,6 @@ sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100 ``` -### libarchive - -If you see an error on macOS when compiling `libarchive`, run: - -```bash -brew install pkg-config -``` - ### macOS `library not found for -lSystem` If you see this error when compiling, run: diff --git a/docs/project/license.mdx b/docs/project/license.mdx index e0d5e61370c3..41c1ad0cf35d 100644 --- a/docs/project/license.mdx +++ b/docs/project/license.mdx @@ -27,7 +27,7 @@ Bun statically links these libraries: | ------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------- | | [`boringssl`](https://boringssl.googlesource.com/boringssl/) | [several licenses](https://boringssl.googlesource.com/boringssl/+/refs/heads/master/LICENSE) | | [`brotli`](https://github.com/google/brotli) | MIT | -| [`libarchive`](https://github.com/libarchive/libarchive) | [several licenses](https://github.com/libarchive/libarchive/blob/master/COPYING) | +| [`libarchive`](https://github.com/libarchive/libarchive) (not linked: `src/tar` is derived from parts of it) | [several licenses](https://github.com/libarchive/libarchive/blob/master/COPYING) | | [`lol-html`](https://github.com/cloudflare/lol-html/tree/master/c-api) | BSD 3-Clause | | [`ls-hpack`](https://github.com/litespeedtech/ls-hpack) | MIT (bundled xxhash is BSD 2-Clause) | | [`ls-qpack`](https://github.com/litespeedtech/ls-qpack) | MIT | diff --git a/patches/libarchive/archive_string-codepage-cache.patch b/patches/libarchive/archive_string-codepage-cache.patch deleted file mode 100644 index 0f12156bb63c..000000000000 --- a/patches/libarchive/archive_string-codepage-cache.patch +++ /dev/null @@ -1,81 +0,0 @@ ---- a/libarchive/archive_string.c -+++ b/libarchive/archive_string.c -@@ -1561,10 +1561,10 @@ make_codepage_from_charset(const char *charset) - } - - /* -- * Return ANSI Code Page of current locale set by setlocale(). -+ * Return ANSI Code Page of current locale set by setlocale() (uncached). - */ - static unsigned --get_current_codepage(void) -+get_current_codepage_uncached(void) - { - char *locale, *p; - unsigned cp; -@@ -1585,6 +1585,28 @@ get_current_codepage(void) - return (cp); - } - -+/* -+ * Bun: cache the codepage after the first call. The process never changes -+ * its locale at runtime, but `bun install` creates a charset converter per -+ * extracted archive on many worker threads at once, and hammering the UCRT's -+ * setlocale(LC_CTYPE, NULL) query from all of them concurrently is both -+ * wasted work and a needless trip through the CRT locale machinery (and its -+ * heap) on every archive. Compute once; later callers read the cached value. -+ */ -+static unsigned -+get_current_codepage(void) -+{ -+ static volatile LONG cached_cp = -1; -+ LONG cp = cached_cp; -+ -+ if (cp == -1) { -+ cp = (LONG)get_current_codepage_uncached(); -+ InterlockedCompareExchange(&cached_cp, cp, -1); -+ cp = cached_cp; -+ } -+ return ((unsigned)cp); -+} -+ - /* - * Translation table between Locale Name and ACP/OEMCP. - */ -@@ -1638,10 +1660,10 @@ static struct { - }; - - /* -- * Return OEM Code Page of current locale set by setlocale(). -+ * Return OEM Code Page of current locale set by setlocale() (uncached). - */ - static unsigned --get_current_oemcp(void) -+get_current_oemcp_uncached(void) - { - int i; - char *locale, *p; -@@ -1663,6 +1685,23 @@ get_current_oemcp(void) - } - return (GetOEMCP()); - } -+ -+/* -+ * Bun: cached for the same reason as get_current_codepage() above. -+ */ -+static unsigned -+get_current_oemcp(void) -+{ -+ static volatile LONG cached_ocp = -1; -+ LONG ocp = cached_ocp; -+ -+ if (ocp == -1) { -+ ocp = (LONG)get_current_oemcp_uncached(); -+ InterlockedCompareExchange(&cached_ocp, ocp, -1); -+ ocp = cached_ocp; -+ } -+ return ((unsigned)ocp); -+} - #else - - /* diff --git a/patches/libarchive/archive_write_add_filter_gzip.c.patch b/patches/libarchive/archive_write_add_filter_gzip.c.patch deleted file mode 100644 index 28b236d0e517..000000000000 --- a/patches/libarchive/archive_write_add_filter_gzip.c.patch +++ /dev/null @@ -1,57 +0,0 @@ ---- a/libarchive/archive_write_add_filter_gzip.c -+++ b/libarchive/archive_write_add_filter_gzip.c -@@ -60,6 +60,7 @@ struct gzip { - int compression_level; - int timestamp; - char *original_filename; -+ unsigned char os; - #ifdef HAVE_ZLIB_H - z_stream stream; - uint64_t total_in; -@@ -109,6 +110,7 @@ archive_write_add_filter_gzip(struct archive *a) - if (gzip == NULL) - goto memerr; - gzip->original_filename = NULL; -+ gzip->os = 3; /* default Unix */ - #ifdef HAVE_ZLIB_H - gzip->compression_level = Z_DEFAULT_COMPRESSION; - -@@ -184,6 +186,29 @@ archive_compressor_gzip_options(struct archive_write_filter *f, const char *key, - } - return (ARCHIVE_OK); - } -+ if (strcmp(key, "os") == 0) { -+ if (value == NULL) -+ return (ARCHIVE_WARN); -+ -+ if (strcmp(value, "FAT") == 0) gzip->os = 0; -+ else if (strcmp(value, "Amiga") == 0) gzip->os = 1; -+ else if (strcmp(value, "VMS") == 0 || strcmp(value, "OpenVMS") == 0) gzip->os = 2; -+ else if (strcmp(value, "Unix") == 0) gzip->os = 3; -+ else if (strcmp(value, "VM") == 0 || strcmp(value, "VM/CMS") == 0) gzip->os = 4; -+ else if (strcmp(value, "Atari TOS") == 0) gzip->os = 5; -+ else if (strcmp(value, "HPFS") == 0) gzip->os = 6; -+ else if (strcmp(value, "Macintosh") == 0) gzip->os = 7; -+ else if (strcmp(value, "Z-System") == 0) gzip->os = 8; -+ else if (strcmp(value, "CP/M") == 0) gzip->os = 9; -+ else if (strcmp(value, "TOPS-20") == 0) gzip->os = 10; -+ else if (strcmp(value, "NTFS") == 0) gzip->os = 11; -+ else if (strcmp(value, "QDOS") == 0) gzip->os = 12; -+ else if (strcmp(value, "Acorn RISCOS") == 0) gzip->os = 13; -+ else if (strcmp(value, "Unknown") == 0) gzip->os = 255; -+ else return (ARCHIVE_WARN); -+ -+ return (ARCHIVE_OK); -+ } - - /* Note: The "warn" return is just to inform the options - * supervisor that we didn't handle it. It will generate -@@ -244,7 +269,7 @@ archive_compressor_gzip_open(struct archive_write_filter *f) - } else { - gzip->compressed[8] = 0; - } -- gzip->compressed[9] = 3; /* OS=Unix */ -+ gzip->compressed[9] = gzip->os; - gzip->stream.next_out += 10; - gzip->stream.avail_out -= 10; - diff --git a/patches/libarchive/gzip-verify-trailer.patch b/patches/libarchive/gzip-verify-trailer.patch deleted file mode 100644 index c4e6bdb95df7..000000000000 --- a/patches/libarchive/gzip-verify-trailer.patch +++ /dev/null @@ -1,63 +0,0 @@ ---- a/libarchive/archive_read_support_filter_gzip.c -+++ b/libarchive/archive_read_support_filter_gzip.c -@@ -67,6 +67,11 @@ - * a gzip header). - */ - char trailer_pending; -+ /* -+ * BUN PATCH: uncompressed byte count of the current member, -+ * modulo 2^32, checked against the ISIZE trailer field. -+ */ -+ uint32_t member_isize; - }; - - /* Gzip Filter. */ -@@ -374,6 +379,7 @@ - - /* Initialize CRC accumulator. */ - gzip->crc = crc32(0L, NULL, 0); -+ gzip->member_isize = 0; - - /* - * BUN PATCH: Initialize compression library. next_in / avail_in -@@ -455,7 +461,18 @@ - if (p == NULL) - return (ARCHIVE_FATAL); - -- /* XXX TODO: Verify the length and CRC. */ -+ /* -+ * BUN PATCH: compare the trailer with what was inflated. Upstream -+ * has a TODO here, so a corrupted member decompressed without any -+ * error. -+ */ -+ if (archive_le32dec(p) != (uint32_t)gzip->crc || -+ archive_le32dec(p + 4) != gzip->member_isize) { -+ archive_set_error(&f->archive->archive, -+ ARCHIVE_ERRNO_MISC, -+ "gzip trailer mismatch: bad CRC32 or uncompressed size"); -+ return (ARCHIVE_FATAL); -+ } - - /* We've verified the trailer, so consume it now. */ - __archive_read_filter_consume(f->upstream, 8); -@@ -470,6 +487,8 @@ - struct gzip *gzip = f->data; - size_t decompressed; - ssize_t avail_in, max_in; -+ unsigned char *out_before; -+ size_t produced; - int ret; - - /* Empty our output buffer. */ -@@ -532,7 +551,11 @@ - gzip->stream.avail_in = (uInt)avail_in; - - /* Decompress and consume some of that data. */ -+ out_before = gzip->stream.next_out; - ret = inflate(&(gzip->stream), 0); -+ produced = gzip->stream.next_out - out_before; -+ gzip->crc = crc32(gzip->crc, out_before, (uInt)produced); -+ gzip->member_isize += (uint32_t)produced; - switch (ret) { - case Z_OK: /* Decompressor made some progress. */ - __archive_read_filter_consume(f->upstream, diff --git a/patches/libarchive/nonblocking-read.patch b/patches/libarchive/nonblocking-read.patch deleted file mode 100644 index 7fd118debbf9..000000000000 --- a/patches/libarchive/nonblocking-read.patch +++ /dev/null @@ -1,1008 +0,0 @@ ---- a/libarchive/archive_read.c -+++ b/libarchive/archive_read.c -@@ -643,33 +643,67 @@ _archive_read_next_header2(struct archive *_a, struct archive_entry *entry) - ARCHIVE_STATE_DATA_RECOVERY, - "archive_read_next_header"); - -- archive_entry_clear(entry); -- archive_clear_error(&a->archive); -- - /* -- * If client didn't consume entire data, skip any remainder -- * (This is especially important for GNU incremental directories.) -- * A header that failed to parse (DATA_RECOVERY) still needs the -- * same treatment: whatever of its body the format reader left -- * unconsumed must be skipped before we can read the next header. -+ * BUN PATCH: when resuming after an ARCHIVE_RETRY from a -+ * non-blocking source, the format reader has already -+ * populated `entry` from consumed extension headers (pax, -+ * GNU long name/link). Clearing it here would discard that -+ * work, and since no data has been read for the pending -+ * entry there is nothing to skip. Skip straight to the -+ * format reader which will pick up where it left off. - */ -- if (a->archive.state == ARCHIVE_STATE_DATA || -- a->archive.state == ARCHIVE_STATE_DATA_RECOVERY) { -- r1 = archive_read_data_skip(&a->archive); -- if (r1 == ARCHIVE_EOF) -- archive_set_error(&a->archive, EIO, -- "Premature end-of-file"); -- if (r1 == ARCHIVE_EOF || r1 == ARCHIVE_FATAL) { -- a->archive.state = ARCHIVE_STATE_FATAL; -- return (ARCHIVE_FATAL); -+ if (!a->read_header_in_progress) { -+ archive_entry_clear(entry); -+ archive_clear_error(&a->archive); -+ -+ /* -+ * If client didn't consume entire data, skip any remainder -+ * (This is especially important for GNU incremental directories.) -+ * A header that failed to parse (DATA_RECOVERY) still needs the -+ * same treatment: whatever of its body the format reader left -+ * unconsumed must be skipped before we can read the next header. -+ */ -+ if (a->archive.state == ARCHIVE_STATE_DATA || -+ a->archive.state == ARCHIVE_STATE_DATA_RECOVERY) { -+ r1 = archive_read_data_skip(&a->archive); -+ /* -+ * BUN PATCH: the skip may run out of buffered -+ * bytes when the source is non-blocking. The -+ * format's skip handler has already recorded how -+ * much remains, and archive_read_data_skip() -+ * leaves the state untouched on ARCHIVE_RETRY, so -+ * the next next_header() call will re-enter here -+ * and continue the skip. -+ */ -+ if (r1 == ARCHIVE_RETRY) -+ return (ARCHIVE_RETRY); -+ if (r1 == ARCHIVE_EOF) -+ archive_set_error(&a->archive, EIO, -+ "Premature end-of-file"); -+ if (r1 == ARCHIVE_EOF || r1 == ARCHIVE_FATAL) { -+ a->archive.state = ARCHIVE_STATE_FATAL; -+ return (ARCHIVE_FATAL); -+ } - } -- } - -- /* Record start-of-header offset in uncompressed stream. */ -- a->header_position = a->filter->position; -+ /* Record start-of-header offset in uncompressed stream. */ -+ a->header_position = a->filter->position; - -- ++_a->file_count; -+ ++_a->file_count; -+ } - r2 = (a->format->read_header)(a, entry); -+ /* -+ * BUN PATCH: the format reader sets read_header_in_progress -+ * itself when a non-blocking source yields mid-header (see -+ * `bun_retry` in the tar reader). Clear it on any terminal -+ * result so the next call runs the full reset above. An -+ * ARCHIVE_RETRY that did not set the flag — upstream tar's -+ * pre-existing damaged-block skip — therefore behaves exactly -+ * as upstream: the next call re-runs archive_entry_clear / -+ * archive_clear_error. -+ */ -+ if (r2 != ARCHIVE_RETRY) -+ a->read_header_in_progress = 0; - - /* - * EOF and FATAL are persistent at this layer. By -@@ -986,6 +1020,14 @@ archive_read_data_skip(struct archive *_a) - if (r == ARCHIVE_EOF) - r = ARCHIVE_OK; - -+ /* -+ * BUN PATCH: a non-blocking source ran dry mid-skip. Leave the -+ * state as-is so the next archive_read_next_header() re-enters -+ * the skip with whatever the format recorded as still owed. -+ */ -+ if (r == ARCHIVE_RETRY) -+ return (r); -+ - if (r == ARCHIVE_FATAL) - a->archive.state = ARCHIVE_STATE_FATAL; - else -@@ -1447,6 +1489,33 @@ __archive_read_filter_ahead(struct archive_read_filter *f, - } - bytes_read = (f->vtable->read)(f, - &f->client_buff); -+ /* -+ * BUN PATCH: non-blocking read support. -+ * -+ * A client reader that is being fed incrementally -+ * (e.g. tarball bytes arriving from the network -+ * during `bun install`) returns ARCHIVE_RETRY when -+ * no data is currently available. Unlike a real -+ * error this must not poison the filter: whatever -+ * has already been copied into `f->buffer` -+ * stays put so the next call with the same `min` -+ * resumes exactly where we left off, and the caller -+ * can yield its thread and try again once more -+ * input has arrived. -+ * -+ * Callers that do not opt in (pass avail == NULL or -+ * do not check for ARCHIVE_RETRY) will see NULL and -+ * treat it as truncated input, which matches the -+ * pre-patch behaviour for non-streaming sources. -+ */ -+ if (bytes_read == ARCHIVE_RETRY) { -+ f->client_total = f->client_avail = 0; -+ f->client_next = -+ f->client_buff = NULL; -+ if (avail != NULL) -+ *avail = ARCHIVE_RETRY; -+ return (NULL); -+ } - if (bytes_read < 0) { /* Read error. */ - f->client_total = f->client_avail = 0; - f->client_next = -@@ -1641,7 +1710,14 @@ advance_file_pointer(struct archive_read_filter *f, int64_t request) - bytes_read = (f->vtable->read)(f, &f->client_buff); - if (bytes_read < 0) { - f->client_buff = NULL; -- f->fatal = 1; -+ /* -+ * BUN PATCH: see the matching comment in -+ * __archive_read_filter_ahead. A non-blocking -+ * reader returning ARCHIVE_RETRY must not mark -+ * the filter fatal. -+ */ -+ if (bytes_read != ARCHIVE_RETRY) -+ f->fatal = 1; - return (bytes_read); - } - ---- a/libarchive/archive_read_private.h -+++ b/libarchive/archive_read_private.h -@@ -189,6 +189,18 @@ struct archive_read { - */ - int64_t entry_bytes_declared; - -+ /* -+ * BUN PATCH: set by the format's read_header callback when it -+ * returns ARCHIVE_RETRY from a non-blocking source and wants -+ * the next call to skip archive_entry_clear() so that -+ * attributes already populated by consumed extension headers -+ * (pax, GNU long name/link, etc.) survive the resume. Format -+ * readers that return ARCHIVE_RETRY without setting this — -+ * e.g. upstream tar's damaged-block skip — get the original -+ * reset-everything semantics on re-entry. -+ */ -+ int read_header_in_progress; -+ - /* Nodes and offsets of compressed data block */ - unsigned int data_start_node; - unsigned int data_end_node; ---- a/libarchive/archive_read_support_filter_gzip.c -+++ b/libarchive/archive_read_support_filter_gzip.c -@@ -60,6 +60,13 @@ struct gzip { - uint32_t mtime; - char *name; - char eof; /* True = found end of compressed data. */ -+ /* -+ * BUN PATCH: set after Z_STREAM_END so that a retry which -+ * interrupts consume_trailer() does not re-enter -+ * consume_header() (which would try to parse deflate bytes as -+ * a gzip header). -+ */ -+ char trailer_pending; - }; - - /* Gzip Filter. */ -@@ -140,8 +147,14 @@ peek_at_header(struct archive_read_filter *f, int *pbits, - * is all fixed layout. */ - len = 10; - p = __archive_read_filter_ahead(f, len, &avail); -+ /* -+ * BUN PATCH: propagate non-blocking retry. Nothing has been -+ * consumed yet, so calling peek_at_header again once more -+ * input is available will re-read the same header bytes from -+ * f->buffer. -+ */ - if (p == NULL) -- return (0); -+ return (avail == ARCHIVE_RETRY ? ARCHIVE_RETRY : 0); - /* We only support deflation- third byte must be 0x08. */ - if (memcmp(p, "\x1F\x8B\x08", 3) != 0) - return (0); -@@ -164,7 +177,7 @@ peek_at_header(struct archive_read_filter *f, int *pbits, - if (header_flags & 4) { - p = __archive_read_filter_ahead(f, len + 2, &avail); - if (p == NULL) -- return (0); -+ return (avail == ARCHIVE_RETRY ? ARCHIVE_RETRY : 0); - len += archive_le16dec(p + len); - len += 2; - } -@@ -184,7 +197,7 @@ peek_at_header(struct archive_read_filter *f, int *pbits, - len, &avail); - } - if (p == NULL) -- return (0); -+ return (avail == ARCHIVE_RETRY ? ARCHIVE_RETRY : 0); - } while (p[len - 1] != 0); - - #ifdef HAVE_ZLIB_H -@@ -208,7 +221,7 @@ peek_at_header(struct archive_read_filter *f, int *pbits, - len, &avail); - } - if (p == NULL) -- return (0); -+ return (avail == ARCHIVE_RETRY ? ARCHIVE_RETRY : 0); - } while (p[len - 1] != 0); - } - -@@ -216,7 +229,7 @@ peek_at_header(struct archive_read_filter *f, int *pbits, - if ((header_flags & 2)) { - p = __archive_read_filter_ahead(f, len + 2, &avail); - if (p == NULL) -- return (0); -+ return (avail == ARCHIVE_RETRY ? ARCHIVE_RETRY : 0); - #if 0 - int hcrc = ((int)p[len + 1] << 8) | (int)p[len]; - int crc = /* XXX TODO: Compute header CRC. */; -@@ -243,7 +256,16 @@ gzip_bidder_bid(struct archive_read_filter_bidder *b, - - (void)b; /* UNUSED */ - -- if (peek_at_header(f, &bits_checked, NULL)) -+ /* -+ * BUN PATCH: peek_at_header() may now return ARCHIVE_RETRY -+ * from a non-blocking source. Bidding has no retry channel, -+ * so only accept a positive header length as a match. -+ * Streaming callers (bun install) always deliver at least -+ * one HTTP chunk — well over the ~10-byte gzip header — -+ * before archive_read_open() is invoked, so this branch is -+ * defensive rather than expected. -+ */ -+ if (peek_at_header(f, &bits_checked, NULL) > 0) - return (bits_checked); - return (0); - } -@@ -333,12 +355,19 @@ static int - consume_header(struct archive_read_filter *f) - { - struct gzip *gzip = f->data; -- ssize_t avail, max_in; -- size_t len; -+ ssize_t len; - int ret; - - /* If this is a real header, consume it. */ - len = peek_at_header(f->upstream, NULL, gzip); -+ /* -+ * BUN PATCH: peek_at_header consumes nothing, so on -+ * ARCHIVE_RETRY the caller can simply try again later and the -+ * partial header bytes already copied into the upstream -+ * filter's buffer will still be there. -+ */ -+ if (len == ARCHIVE_RETRY) -+ return (ARCHIVE_RETRY); - if (len == 0) - return (ARCHIVE_EOF); - __archive_read_filter_consume(f->upstream, len); -@@ -346,22 +375,16 @@ consume_header(struct archive_read_filter *f) - /* Initialize CRC accumulator. */ - gzip->crc = crc32(0L, NULL, 0); - -- /* Initialize compression library. */ -- gzip->stream.next_in = (unsigned char *)(uintptr_t) -- __archive_read_filter_ahead(f->upstream, 1, &avail); -- if (avail < 0) { -- archive_set_error(&f->archive->archive, -- ARCHIVE_ERRNO_MISC, -- "Failed to read gzip input"); -- return (ARCHIVE_FATAL); -- } -- if (UINT_MAX >= SSIZE_MAX) -- max_in = SSIZE_MAX; -- else -- max_in = UINT_MAX; -- if (avail > max_in) -- avail = max_in; -- gzip->stream.avail_in = (uInt)avail; -+ /* -+ * BUN PATCH: Initialize compression library. next_in / avail_in -+ * are unused by inflateInit2 with a negative windowBits (raw -+ * deflate); the main read loop primes them before each -+ * inflate() call, so we do not need an extra read-ahead here -+ * that could itself return ARCHIVE_RETRY after the header has -+ * already been consumed. -+ */ -+ gzip->stream.next_in = NULL; -+ gzip->stream.avail_in = 0; - ret = inflateInit2(&(gzip->stream), - -15 /* Don't check for zlib header */); - -@@ -402,20 +425,33 @@ consume_trailer(struct archive_read_filter *f) - { - struct gzip *gzip = f->data; - const unsigned char *p; -- -- gzip->in_stream = 0; -- switch (inflateEnd(&(gzip->stream))) { -- case Z_OK: -- break; -- default: -- archive_set_error(&f->archive->archive, -- ARCHIVE_ERRNO_MISC, -- "Failed to clean up gzip decompressor"); -- return (ARCHIVE_FATAL); -+ ssize_t avail; -+ -+ /* -+ * BUN PATCH: inflateEnd()/in_stream=0 are only safe to run -+ * once. If a previous consume_trailer() attempt was -+ * interrupted by ARCHIVE_RETRY while reading the 8-byte -+ * footer, `trailer_pending` stays set and we skip straight to -+ * the read-ahead below. -+ */ -+ if (!gzip->trailer_pending) { -+ gzip->in_stream = 0; -+ switch (inflateEnd(&(gzip->stream))) { -+ case Z_OK: -+ break; -+ default: -+ archive_set_error(&f->archive->archive, -+ ARCHIVE_ERRNO_MISC, -+ "Failed to clean up gzip decompressor"); -+ return (ARCHIVE_FATAL); -+ } -+ gzip->trailer_pending = 1; - } - - /* GZip trailer is a fixed 8 byte structure. */ -- p = __archive_read_filter_ahead(f->upstream, 8, NULL); -+ p = __archive_read_filter_ahead(f->upstream, 8, &avail); -+ if (p == NULL && avail == ARCHIVE_RETRY) -+ return (ARCHIVE_RETRY); - if (p == NULL) - return (ARCHIVE_FATAL); - -@@ -423,6 +459,7 @@ consume_trailer(struct archive_read_filter *f) - - /* We've verified the trailer, so consume it now. */ - __archive_read_filter_consume(f->upstream, 8); -+ gzip->trailer_pending = 0; - - return (ARCHIVE_OK); - } -@@ -441,6 +478,18 @@ gzip_filter_read(struct archive_read_filter *f, const void **p) - - /* Try to fill the output buffer. */ - while (gzip->stream.avail_out > 0 && !gzip->eof) { -+ /* -+ * BUN PATCH: finish any pending trailer first. This can -+ * only be set when a previous gzip_filter_read() call hit -+ * ARCHIVE_RETRY inside consume_trailer(). -+ */ -+ if (gzip->trailer_pending) { -+ ret = consume_trailer(f); -+ if (ret == ARCHIVE_RETRY) -+ goto bun_retry; -+ if (ret < ARCHIVE_OK) -+ return (ret); -+ } - /* If we're not in a stream, read a header - * and initialize the decompression library. */ - if (!gzip->in_stream) { -@@ -449,6 +498,8 @@ gzip_filter_read(struct archive_read_filter *f, const void **p) - gzip->eof = 1; - break; - } -+ if (ret == ARCHIVE_RETRY) -+ goto bun_retry; - if (ret < ARCHIVE_OK) - return (ret); - } -@@ -459,6 +510,14 @@ gzip_filter_read(struct archive_read_filter *f, const void **p) - gzip->stream.next_in = (unsigned char *)(uintptr_t) - __archive_read_filter_ahead(f->upstream, 1, &avail_in); - if (gzip->stream.next_in == NULL) { -+ /* -+ * BUN PATCH: upstream has no bytes right now but -+ * isn't done. zlib's inflate state is untouched, -+ * so propagate the retry (or return whatever we -+ * have already decompressed this call). -+ */ -+ if (avail_in == ARCHIVE_RETRY) -+ goto bun_retry; - archive_set_error(&f->archive->archive, - ARCHIVE_ERRNO_MISC, - "truncated gzip input"); -@@ -485,6 +544,8 @@ gzip_filter_read(struct archive_read_filter *f, const void **p) - /* Consume the stream trailer; release the - * decompression library. */ - ret = consume_trailer(f); -+ if (ret == ARCHIVE_RETRY) -+ goto bun_retry; - if (ret < ARCHIVE_OK) - return (ret); - break; -@@ -504,6 +565,24 @@ gzip_filter_read(struct archive_read_filter *f, const void **p) - else - *p = gzip->out_block; - return (decompressed); -+ -+bun_retry: -+ /* -+ * BUN PATCH: upstream ran dry mid-call. If we managed to -+ * decompress anything before that happened, hand it back -+ * now; the tar layer will consume it and the next call will -+ * start with an empty out_block and immediately retry. -+ * Otherwise propagate ARCHIVE_RETRY so the tar layer (and -+ * ultimately the extract loop in TarballStream.rs) can yield -+ * this worker and reschedule when more bytes arrive. -+ */ -+ decompressed = gzip->stream.next_out - gzip->out_block; -+ if (decompressed > 0) { -+ *p = gzip->out_block; -+ return (decompressed); -+ } -+ *p = NULL; -+ return (ARCHIVE_RETRY); - } - - /* ---- a/libarchive/archive_read_support_format_tar.c -+++ b/libarchive/archive_read_support_format_tar.c -@@ -156,6 +156,23 @@ struct tar { - int read_concatenated_archives; - int default_inode; - int default_dev; -+ -+ /* -+ * BUN PATCH: resume state for non-blocking sources. -+ * -+ * `tar_read_header` walks a variable-length sequence of -+ * extension headers (pax 'x'/'g', GNU 'L'/'K', etc.) before -+ * the real ustar header. When the underlying reader returns -+ * ARCHIVE_RETRY part-way through that walk, we must remember -+ * which extensions have already been parsed so the next call -+ * picks up at the right header instead of re-running -+ * `tar_reset_header_state` and re-bidding bytes that were -+ * already consumed. -+ */ -+ int header_in_progress; -+ int32_t header_seen; -+ int header_eof_fatal; -+ int header_err; - }; - - /* Track which size fields were present in the headers */ -@@ -232,6 +249,8 @@ static int64_t tar_atol(const char *, size_t); - static int64_t tar_atol10(const char *, size_t); - static int64_t tar_atol256(const char *, size_t); - static int64_t tar_atol8(const char *, size_t); -+static int tar_consume_retryable(struct archive_read *, struct tar *, -+ int64_t); - static int tar_read_header(struct archive_read *, struct tar *, - struct archive_entry *, int64_t *); - static int tohex(int c); -@@ -474,6 +493,53 @@ archive_read_format_tar_options(struct archive_read *a, - return (ARCHIVE_WARN); - } - -+/* -+ * BUN PATCH: consume up to `request` bytes in a way that is safe to -+ * resume from a non-blocking source. Each iteration uses -+ * __archive_read_ahead(1) to discover how many bytes are currently -+ * buffered and then __archive_read_consume()s exactly that much, so -+ * the consume never has to invoke the client reader itself. On -+ * ARCHIVE_RETRY the remaining amount is written back into -+ * `tar->entry_bytes_remaining` / `tar->entry_padding` so the next -+ * call (via archive_read_next_header → read_data_skip, or another -+ * read_data_block) continues from the right offset. -+ */ -+static int -+tar_consume_retryable(struct archive_read *a, struct tar *tar, -+ int64_t request) -+{ -+ while (request > 0) { -+ ssize_t avail = 0; -+ const void *p = __archive_read_ahead(a, 1, &avail); -+ if (p == NULL) { -+ if (avail == ARCHIVE_RETRY) { -+ /* Persist what is still owed so the -+ * next resume asks for the remainder -+ * rather than the original total. */ -+ if (request >= tar->entry_padding) { -+ tar->entry_bytes_remaining = -+ request - tar->entry_padding; -+ } else { -+ tar->entry_bytes_remaining = 0; -+ tar->entry_padding = request; -+ } -+ return (ARCHIVE_RETRY); -+ } -+ archive_set_error(&a->archive, -+ ARCHIVE_ERRNO_MISC, -+ "Truncated tar archive" -+ " detected while skipping data"); -+ return (ARCHIVE_FATAL); -+ } -+ if (avail > request) -+ avail = (ssize_t)request; -+ if (__archive_read_consume(a, avail) != avail) -+ return (ARCHIVE_FATAL); -+ request -= avail; -+ } -+ return (ARCHIVE_OK); -+} -+ - /* utility function- this exists to centralize the logic of tracking - * how much unconsumed data we have floating around, and to consume - * anything outstanding since we're going to do read_aheads -@@ -530,32 +596,52 @@ archive_read_format_tar_read_header(struct archive_read *a, - size_t l; - int64_t unconsumed = 0; - -- /* Assign default device/inode values. */ -- archive_entry_set_dev(entry, 1 + tar->default_dev); /* Don't use zero. */ -- archive_entry_set_ino(entry, ++tar->default_inode); /* Don't use zero. */ -- /* Limit generated st_ino number to 16 bits. */ -- if (tar->default_inode >= 0xffff) { -- ++tar->default_dev; -- tar->default_inode = 0; -- } -- -- tar->entry_offset = 0; -- gnu_clear_sparse_list(tar); -- tar->size_fields = 0; /* We don't have any size info yet */ -- -- /* Setup default string conversion. */ -- tar->sconv = tar->opt_sconv; -- if (tar->sconv == NULL) { -- if (!tar->init_default_conversion) { -- tar->sconv_default = -- archive_string_default_conversion_for_read(&(a->archive)); -- tar->init_default_conversion = 1; -+ /* -+ * BUN PATCH: when resuming after an ARCHIVE_RETRY from a -+ * non-blocking source, skip the fresh-entry initialisation -+ * below. Extension headers parsed before the retry have -+ * already written into `entry` and `tar->entry_*`; repeating -+ * these resets would discard that work. -+ */ -+ if (!tar->header_in_progress) { -+ /* Assign default device/inode values. */ -+ archive_entry_set_dev(entry, 1 + tar->default_dev); /* Don't use zero. */ -+ archive_entry_set_ino(entry, ++tar->default_inode); /* Don't use zero. */ -+ /* Limit generated st_ino number to 16 bits. */ -+ if (tar->default_inode >= 0xffff) { -+ ++tar->default_dev; -+ tar->default_inode = 0; -+ } -+ -+ tar->entry_offset = 0; -+ gnu_clear_sparse_list(tar); -+ tar->size_fields = 0; /* We don't have any size info yet */ -+ -+ /* Setup default string conversion. */ -+ tar->sconv = tar->opt_sconv; -+ if (tar->sconv == NULL) { -+ if (!tar->init_default_conversion) { -+ tar->sconv_default = -+ archive_string_default_conversion_for_read(&(a->archive)); -+ tar->init_default_conversion = 1; -+ } -+ tar->sconv = tar->sconv_default; - } -- tar->sconv = tar->sconv_default; - } - - r = tar_read_header(a, tar, entry, &unconsumed); - -+ /* -+ * BUN PATCH: a non-blocking yield leaves `header_in_progress` -+ * set; in that case `*unconsumed == 0` (nothing pending that -+ * isn't already flushed) so the caller can yield and re-enter -+ * later. Upstream's damaged-block ARCHIVE_RETRY clears the -+ * flag and falls through to the original post-read handling -+ * below (which is what upstream did). -+ */ -+ if (r == ARCHIVE_RETRY && tar->header_in_progress) -+ return (ARCHIVE_RETRY); -+ - tar_flush_unconsumed(a, &unconsumed); - - /* -@@ -629,9 +715,20 @@ archive_read_format_tar_read_data(struct archive_read *a, - int64_t request = tar->entry_bytes_remaining + - tar->entry_padding; - -- if (__archive_read_consume(a, request) != request) -+ /* -+ * BUN PATCH: make the end-of-entry padding -+ * consume resumable. `tar_consume_retryable` -+ * decrements `entry_bytes_remaining` / -+ * `entry_padding` as bytes become available so a -+ * subsequent call picks up with the remainder. -+ */ -+ int cr = tar_consume_retryable(a, tar, request); -+ if (cr == ARCHIVE_RETRY) -+ return (ARCHIVE_RETRY); -+ if (cr != ARCHIVE_OK) - return (ARCHIVE_FATAL); - tar->entry_padding = 0; -+ tar->entry_bytes_remaining = 0; - *buff = NULL; - *size = 0; - *offset = tar->disk_size; -@@ -640,6 +737,15 @@ archive_read_format_tar_read_data(struct archive_read *a, - - *buff = __archive_read_ahead(a, 1, &bytes_read); - if (*buff == NULL) { -+ /* -+ * BUN PATCH: propagate non-blocking retry. Every -+ * counter we would have touched -+ * (`entry_bytes_remaining`, `sparse_list`, etc.) -+ * is still at its pre-call value, so the caller -+ * can yield and re-enter read_data_block later. -+ */ -+ if (bytes_read == ARCHIVE_RETRY) -+ return (ARCHIVE_RETRY); - archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, - "Truncated tar archive" - " detected while reading data"); -@@ -669,15 +775,26 @@ archive_read_format_tar_skip(struct archive_read *a) - { - struct tar *tar = a->format->data; - int64_t request; -+ int cr; - - request = tar->entry_bytes_remaining + tar->entry_padding + - tar->entry_bytes_unconsumed; - -- if (__archive_read_consume(a, request) != request) -+ /* -+ * BUN PATCH: use the retryable consume so a non-blocking -+ * source can yield mid-skip. Progress is recorded in -+ * `entry_bytes_remaining` / `entry_padding` / -+ * `entry_bytes_unconsumed`, so archive_read_next_header's -+ * implicit skip on re-entry resumes with the remainder. -+ */ -+ tar->entry_bytes_unconsumed = 0; -+ cr = tar_consume_retryable(a, tar, request); -+ if (cr == ARCHIVE_RETRY) -+ return (ARCHIVE_RETRY); -+ if (cr != ARCHIVE_OK) - return (ARCHIVE_FATAL); - - tar->entry_bytes_remaining = 0; -- tar->entry_bytes_unconsumed = 0; - tar->entry_padding = 0; - - /* Free the sparse list. */ -@@ -711,14 +828,14 @@ tar_read_header(struct archive_read *a, struct tar *tar, - struct archive_entry *entry, int64_t *unconsumed) - { - ssize_t bytes; -- int err = ARCHIVE_OK, err2; -- int eof_fatal = 0; /* EOF is okay at some points... */ -+ int err, err2; -+ int eof_fatal; /* EOF is okay at some points... */ - const char *h; - const struct archive_entry_header_ustar *header; - const struct archive_entry_header_gnutar *gnuheader; - - /* Bitmask of what header types we've seen. */ -- int32_t seen_headers = 0; -+ int32_t seen_headers; - static const int32_t seen_A_header = 1; - static const int32_t seen_g_header = 2; - static const int32_t seen_K_header = 4; -@@ -727,7 +844,24 @@ tar_read_header(struct archive_read *a, struct tar *tar, - static const int32_t seen_x_header = 32; /* Also X */ - static const int32_t seen_mac_metadata = 512; - -- tar_reset_header_state(tar); -+ /* -+ * BUN PATCH: `header_in_progress` persists the header loop's -+ * locals across an ARCHIVE_RETRY so a non-blocking source can -+ * resume mid-sequence. On a fresh entry we initialise them -+ * exactly as before; on resume we restore and skip the reset -+ * that would otherwise wipe pax/GNU long-name data already -+ * parsed into `tar->entry_*`. -+ */ -+ if (!tar->header_in_progress) { -+ tar_reset_header_state(tar); -+ tar->header_in_progress = 1; -+ tar->header_seen = 0; -+ tar->header_eof_fatal = 0; -+ tar->header_err = ARCHIVE_OK; -+ } -+ seen_headers = tar->header_seen; -+ eof_fatal = tar->header_eof_fatal; -+ err = tar->header_err; - - /* Ensure format is set. */ - if (a->archive.archive_format_name == NULL) { -@@ -735,6 +869,17 @@ tar_read_header(struct archive_read *a, struct tar *tar, - a->archive.archive_format_name = "tar"; - } - -+/* -+ * BUN PATCH: every terminal exit from this function (anything other -+ * than the non-blocking retry path) must clear `header_in_progress` -+ * so the next entry starts with fresh `seen_headers` / header state. -+ * The macro keeps the existing `return (X)` sites readable. -+ */ -+#define TAR_HEADER_RETURN(rc) do { \ -+ tar->header_in_progress = 0; \ -+ return (rc); \ -+ } while (0) -+ - /* - * TODO: Write global/default pax options into - * 'entry' struct here before overwriting with -@@ -747,11 +892,22 @@ tar_read_header(struct archive_read *a, struct tar *tar, - /* Find the next valid header record. */ - while (1) { - if (tar_flush_unconsumed(a, unconsumed) != ARCHIVE_OK) { -- return (ARCHIVE_FATAL); -+ TAR_HEADER_RETURN(ARCHIVE_FATAL); - } - - /* Read 512-byte header record */ - h = __archive_read_ahead(a, 512, &bytes); -+ if (h == NULL && bytes == ARCHIVE_RETRY) { -+ /* -+ * BUN PATCH: non-blocking source ran -+ * out mid-header. Nothing has been -+ * added to `*unconsumed` yet this -+ * iteration, so save loop state and -+ * let the caller try again once more -+ * input arrives. -+ */ -+ goto bun_retry; -+ } - if (bytes == 0) { /* EOF at a block boundary. */ - if (eof_fatal) { - /* We've read a special header already; -@@ -759,9 +915,9 @@ tar_read_header(struct archive_read *a, struct tar *tar, - * a premature EOF. */ - archive_set_error(&a->archive, EINVAL, - "Damaged tar archive (end-of-archive within a sequence of headers)"); -- return (ARCHIVE_FATAL); -+ TAR_HEADER_RETURN(ARCHIVE_FATAL); - } else { -- return (ARCHIVE_EOF); -+ TAR_HEADER_RETURN(ARCHIVE_EOF); - } - } - if (h == NULL) { /* Short block at EOF; this is bad. */ -@@ -769,7 +925,7 @@ tar_read_header(struct archive_read *a, struct tar *tar, - ARCHIVE_ERRNO_FILE_FORMAT, - "Truncated tar archive" - " detected while reading next header"); -- return (ARCHIVE_FATAL); -+ TAR_HEADER_RETURN(ARCHIVE_FATAL); - } - *unconsumed += 512; - -@@ -789,22 +945,35 @@ tar_read_header(struct archive_read *a, struct tar *tar, - __archive_read_consume(a, 512); - - archive_clear_error(&a->archive); -- return (ARCHIVE_EOF); -+ TAR_HEADER_RETURN(ARCHIVE_EOF); - } - - /* This is NOT a null block, so it must be a valid header. */ - if (!checksum(a, h)) { - if (tar_flush_unconsumed(a, unconsumed) != ARCHIVE_OK) { -- return (ARCHIVE_FATAL); -+ TAR_HEADER_RETURN(ARCHIVE_FATAL); - } - archive_set_error(&a->archive, EINVAL, - "Damaged tar archive (bad header checksum)"); - /* If we've read some critical information (pax headers, etc) - * and _then_ see a bad header, we can't really recover. */ - if (eof_fatal) { -- return (ARCHIVE_FATAL); -+ TAR_HEADER_RETURN(ARCHIVE_FATAL); - } else { -- return (ARCHIVE_RETRY); -+ /* -+ * BUN PATCH: upstream's -+ * "damaged block, skip and retry -+ * the next one" — NOT a -+ * non-blocking yield. The next -+ * call must re-run the full -+ * state reset (archive_entry_clear, -+ * tar_reset_header_state) exactly -+ * as upstream, so clear both -+ * in-progress flags instead of -+ * routing through bun_retry. -+ */ -+ a->read_header_in_progress = 0; -+ TAR_HEADER_RETURN(ARCHIVE_RETRY); - } - } - break; -@@ -812,12 +981,73 @@ tar_read_header(struct archive_read *a, struct tar *tar, - - /* Determine the format variant. */ - header = (const struct archive_entry_header_ustar *)h; -+ -+ /* -+ * BUN PATCH: extension headers ('A','g','K','L','V', -+ * 'x','X') are followed by a payload whose length is -+ * encoded in this block's `size` field. The handlers -+ * below flush `*unconsumed` (this 512-byte block) and -+ * then __archive_read_ahead() the payload; if that -+ * ahead were to return ARCHIVE_RETRY after the flush, -+ * the block would already be consumed and the resume -+ * would read payload bytes as a header. -+ * -+ * To keep the handlers unchanged, pre-buffer the -+ * header+payload here *before* the flush. On retry we -+ * roll back `*unconsumed` so nothing is consumed and -+ * the next attempt re-reads this exact block from the -+ * filter's buffer. -+ */ -+ switch (header->typeflag[0]) { -+ case 'A': case 'g': case 'K': case 'L': -+ case 'V': case 'X': case 'x': { -+ int64_t ext_size = tar_atol(header->size, -+ sizeof(header->size)); -+ /* Bound the prebuffer before rounding -+ * (tar_atol saturates to INT64_MAX on -+ * overflow, so adding 511 first would be UB). -+ * A corrupted stream can produce garbage that -+ * decodes as an extension header with a -+ * multi-GB `size`; retrying `ahead(512+huge)` -+ * every chunk would never make progress. Real -+ * pax/GNU extension payloads are path-length -+ * bounded (a few KB), so anything beyond this -+ * is handed to the handler below, which will -+ * fail cleanly on its own -+ * `__archive_read_ahead`. */ -+ if (ext_size < 0 || ext_size > (int64_t)(1u << 20)) -+ break; -+ int64_t ext_padded = (ext_size + 511) & ~511; -+ if (__archive_read_ahead(a, -+ (size_t)(512 + ext_padded), &bytes) == NULL && -+ bytes == ARCHIVE_RETRY) { -+ *unconsumed -= 512; -+ goto bun_retry; -+ } -+ /* Re-establish `h`: the filter may have -+ * memmoved its buffer to satisfy the larger -+ * request. A fatal error from the larger -+ * read (alloc failure, downstream I/O error) -+ * leaves `filter->fatal` set, so this read -+ * can return NULL even though the original -+ * 512 bytes are still buffered; bail cleanly -+ * rather than dereferencing `header` below. */ -+ h = __archive_read_ahead(a, 512, NULL); -+ if (h == NULL) -+ TAR_HEADER_RETURN(ARCHIVE_FATAL); -+ header = (const struct archive_entry_header_ustar *)h; -+ break; -+ } -+ default: -+ break; -+ } -+ - switch(header->typeflag[0]) { - case 'A': /* Solaris tar ACL */ - if (seen_headers & seen_A_header) { - archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, - "Redundant 'A' header"); -- return (ARCHIVE_FATAL); -+ TAR_HEADER_RETURN(ARCHIVE_FATAL); - } - seen_headers |= seen_A_header; - a->archive.archive_format = ARCHIVE_FORMAT_TAR_PAX_INTERCHANGE; -@@ -828,7 +1058,7 @@ tar_read_header(struct archive_read *a, struct tar *tar, - if (seen_headers & seen_g_header) { - archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, - "Redundant 'g' header"); -- return (ARCHIVE_FATAL); -+ TAR_HEADER_RETURN(ARCHIVE_FATAL); - } - seen_headers |= seen_g_header; - a->archive.archive_format = ARCHIVE_FORMAT_TAR_PAX_INTERCHANGE; -@@ -872,7 +1102,7 @@ tar_read_header(struct archive_read *a, struct tar *tar, - if (seen_headers & seen_x_header) { - archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, - "Redundant 'X'/'x' header"); -- return (ARCHIVE_FATAL); -+ TAR_HEADER_RETURN(ARCHIVE_FATAL); - } - seen_headers |= seen_x_header; - a->archive.archive_format = ARCHIVE_FORMAT_TAR_PAX_INTERCHANGE; -@@ -884,7 +1114,7 @@ tar_read_header(struct archive_read *a, struct tar *tar, - if (seen_headers & seen_x_header) { - archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, - "Redundant 'x' header"); -- return (ARCHIVE_FATAL); -+ TAR_HEADER_RETURN(ARCHIVE_FATAL); - } - seen_headers |= seen_x_header; - a->archive.archive_format = ARCHIVE_FORMAT_TAR_PAX_INTERCHANGE; -@@ -911,7 +1141,7 @@ tar_read_header(struct archive_read *a, struct tar *tar, - err = err_combine(err, err2); - /* We return warnings or success as-is. Anything else is fatal. */ - if (err < ARCHIVE_WARN) { -- return (ARCHIVE_FATAL); -+ TAR_HEADER_RETURN(ARCHIVE_FATAL); - } - /* Filename of the form `._filename` is an AppleDouble - * extension entry. The body is the macOS metadata blob; -@@ -930,7 +1160,7 @@ tar_read_header(struct archive_read *a, struct tar *tar, - && is_mac_metadata_entry(entry)) { - err2 = read_mac_metadata_blob(a, entry, unconsumed); - if (err2 < ARCHIVE_WARN) { -- return (ARCHIVE_FATAL); -+ TAR_HEADER_RETURN(ARCHIVE_FATAL); - } - err = err_combine(err, err2); - /* Note: Other headers can appear again. */ -@@ -945,7 +1175,7 @@ tar_read_header(struct archive_read *a, struct tar *tar, - if (tar->filetype != 'S' && tar->filetype != '0') { - archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, - "Non-regular file cannot be sparse"); -- return (ARCHIVE_WARN); -+ TAR_HEADER_RETURN(ARCHIVE_WARN); - } else if (tar->sparse_gnu_major == 0 && - tar->sparse_gnu_minor == 0) { - /* Sparse map already parsed from 'x' header */ -@@ -958,22 +1188,22 @@ tar_read_header(struct archive_read *a, struct tar *tar, - ssize_t bytes_read; - bytes_read = gnu_sparse_10_read(a, tar, unconsumed); - if (bytes_read < 0) -- return ((int)bytes_read); -+ TAR_HEADER_RETURN((int)bytes_read); - tar->entry_bytes_remaining -= bytes_read; - } else { - archive_set_error(&a->archive, - ARCHIVE_ERRNO_MISC, - "Unrecognized GNU sparse file format"); -- return (ARCHIVE_WARN); -+ TAR_HEADER_RETURN(ARCHIVE_WARN); - } - } -- return (err); -+ TAR_HEADER_RETURN(err); - } - - /* We're between headers ... */ - err = err_combine(err, err2); - if (err == ARCHIVE_FATAL) -- return (err); -+ TAR_HEADER_RETURN(err); - - /* The GNU volume header and the pax `g` global header - * are both allowed to be the only header in an -@@ -983,6 +1213,26 @@ tar_read_header(struct archive_read *a, struct tar *tar, - eof_fatal = 1; - } - } -+ -+bun_retry: -+ /* -+ * BUN PATCH: persist loop state so the next tar_read_header() -+ * call resumes exactly here. At this point `*unconsumed == 0` -+ * (either nothing was added this iteration, or we rolled the -+ * 512-byte header back above), so -+ * `archive_read_format_tar_read_header` can return -+ * ARCHIVE_RETRY without flushing. Setting -+ * `read_header_in_progress` here (rather than having -+ * archive_read_next_header2 set it on every RETRY) is what -+ * lets upstream's damaged-block RETRY keep its original -+ * reset-everything semantics. -+ */ -+ tar->header_seen = seen_headers; -+ tar->header_eof_fatal = eof_fatal; -+ tar->header_err = err; -+ a->read_header_in_progress = 1; -+ return (ARCHIVE_RETRY); -+#undef TAR_HEADER_RETURN - } - - /* diff --git a/patches/libarchive/select-registered-only.patch b/patches/libarchive/select-registered-only.patch deleted file mode 100644 index c0c0a7eed810..000000000000 --- a/patches/libarchive/select-registered-only.patch +++ /dev/null @@ -1,116 +0,0 @@ ---- a/libarchive/archive_read_append_filter.c -+++ b/libarchive/archive_read_append_filter.c -@@ -42,7 +42,12 @@ archive_read_append_filter(struct archive *_a, int code) - struct archive_read_filter *f; - struct archive_read *a = (struct archive_read *)_a; - -- r2 = (ARCHIVE_OK); -+ /* -+ * BUN PATCH: append only a filter the caller already registered. -+ * Upstream registers it here via archive_read_support_filter_(), -+ * which keeps every filter linked; bun only ever appends gzip. -+ */ -+ r1 = r2 = (ARCHIVE_OK); - switch (code) - { - case ARCHIVE_FILTER_NONE: -@@ -50,19 +55,15 @@ archive_read_append_filter(struct archive *_a, int code) - * NOTE: An initial "NONE" type filter is always set at the end of the - * filter chain. - */ -- r1 = (ARCHIVE_OK); - break; - case ARCHIVE_FILTER_GZIP: - str = "gzip"; -- r1 = archive_read_support_filter_gzip(_a); - break; - case ARCHIVE_FILTER_BZIP2: - str = "bzip2"; -- r1 = archive_read_support_filter_bzip2(_a); - break; - case ARCHIVE_FILTER_COMPRESS: - str = "compress (.Z)"; -- r1 = archive_read_support_filter_compress(_a); - break; - case ARCHIVE_FILTER_PROGRAM: - archive_set_error(&a->archive, ARCHIVE_ERRNO_PROGRAMMER, -@@ -70,43 +71,33 @@ archive_read_append_filter(struct archive *_a, int code) - return (ARCHIVE_FATAL); - case ARCHIVE_FILTER_LZMA: - str = "lzma"; -- r1 = archive_read_support_filter_lzma(_a); - break; - case ARCHIVE_FILTER_XZ: - str = "xz"; -- r1 = archive_read_support_filter_xz(_a); - break; - case ARCHIVE_FILTER_UU: - str = "uu"; -- r1 = archive_read_support_filter_uu(_a); - break; - case ARCHIVE_FILTER_RPM: - str = "rpm"; -- r1 = archive_read_support_filter_rpm(_a); - break; - case ARCHIVE_FILTER_LZ4: - str = "lz4"; -- r1 = archive_read_support_filter_lz4(_a); - break; - case ARCHIVE_FILTER_ZSTD: - str = "zstd"; -- r1 = archive_read_support_filter_zstd(_a); - break; - case ARCHIVE_FILTER_LZIP: - str = "lzip"; -- r1 = archive_read_support_filter_lzip(_a); - break; - case ARCHIVE_FILTER_LZOP: - str = "lzop"; -- r1 = archive_read_support_filter_lzop(_a); - break; - case ARCHIVE_FILTER_LRZIP: - str = "lrzip"; -- r1 = archive_read_support_filter_lrzip(_a); - break; - case ARCHIVE_FILTER_GRZIP: - str = "grzip"; -- r1 = archive_read_support_filter_grzip(_a); - break; - default: - archive_set_error(&a->archive, ARCHIVE_ERRNO_PROGRAMMER, -@@ -127,7 +118,7 @@ archive_read_append_filter(struct archive *_a, int code) - if (!b->name || strcmp(b->name, str)) - { - archive_set_error(&a->archive, ARCHIVE_ERRNO_PROGRAMMER, -- "Internal error: Unable to append filter"); -+ "Filter is not registered"); - return (ARCHIVE_FATAL); - } - ---- a/libarchive/archive_read_set_format.c -+++ b/libarchive/archive_read_set_format.c -@@ -40,9 +40,12 @@ archive_read_set_format(struct archive *_a, int code) - const char *str; - struct archive_read *a = (struct archive_read *)_a; - -- if ((r1 = archive_read_support_format_by_code(_a, code)) < (ARCHIVE_OK)) -- return r1; -- -+ /* -+ * BUN PATCH: select only among formats the caller already registered. -+ * Upstream first calls archive_read_support_format_by_code(), whose -+ * switch references every reader, so the whole reader set (7zip, rar, -+ * zip, ...) stays linked even though bun only ever registers tar. -+ */ - r1 = r2 = (ARCHIVE_OK); - if (a->format) - r2 = (ARCHIVE_WARN); -@@ -108,7 +111,7 @@ archive_read_set_format(struct archive *_a, int code) - if (!a->format->name || strcmp(a->format->name, str)) - { - archive_set_error(&a->archive, ARCHIVE_ERRNO_PROGRAMMER, -- "Internal error: Unable to set format"); -+ "Format is not registered"); - r1 = (ARCHIVE_FATAL); - } - diff --git a/patches/libarchive/tar-npm-compat.patch b/patches/libarchive/tar-npm-compat.patch deleted file mode 100644 index 962548b1b1f4..000000000000 --- a/patches/libarchive/tar-npm-compat.patch +++ /dev/null @@ -1,175 +0,0 @@ ---- a/libarchive/archive_read_support_format_tar.c -+++ b/libarchive/archive_read_support_format_tar.c -@@ -173,6 +173,16 @@ - int32_t header_seen; - int header_eof_fatal; - int header_err; -+ -+ /* -+ * BUN PATCH: `npm` is the reader option of that name: read a -+ * package the way npm's own tar reader does. With it `null_blocks` -+ * counts the zero blocks in a row where a header is due; two of them -+ * end the archive, which `past_end` remembers. -+ */ -+ int npm; -+ int null_blocks; -+ int past_end; - }; - - /* Track which size fields were present in the headers */ -@@ -485,6 +495,10 @@ - } else if (strcmp(key, "read_concatenated_archives") == 0) { - tar->read_concatenated_archives = (val != NULL && val[0] != 0); - return (ARCHIVE_OK); -+ } else if (strcmp(key, "npm") == 0) { -+ /* BUN PATCH */ -+ tar->npm = (val != NULL && val[0] != 0); -+ return (ARCHIVE_OK); - } - - /* Note: The "warn" return is just to inform the options -@@ -662,7 +676,13 @@ - } - } - -- if (r == ARCHIVE_OK && archive_entry_filetype(entry) == AE_IFREG) { -+ /* -+ * BUN PATCH: npm does this for types '0' and NUL alone, also behind -+ * a header that got a warning. -+ */ -+ if ((tar->npm ? (r == ARCHIVE_OK || r == ARCHIVE_WARN) && -+ (tar->filetype == '0' || tar->filetype == '\0') : -+ r == ARCHIVE_OK) && archive_entry_filetype(entry) == AE_IFREG) { - /* - * "Regular" entry with trailing '/' is really - * directory: This is needed for certain old tar -@@ -684,6 +704,19 @@ - } - } - } -+ -+ /* -+ * BUN PATCH: npm takes types '0', NUL and '7' for files and skips -+ * an entry whose pax path has a NUL in it. Anything else that got -+ * the file type of a regular file loses it. -+ */ -+ if (tar->npm && archive_entry_filetype(entry) == AE_IFREG && -+ ((tar->filetype != '0' && tar->filetype != '\0' && -+ tar->filetype != '7') || -+ (archive_strlen(&tar->entry_pathname) > 0 && -+ memchr(tar->entry_pathname.s, '\0', -+ archive_strlen(&tar->entry_pathname)) != NULL))) -+ archive_entry_set_filetype(entry, 0); - return (r); - } - -@@ -895,8 +928,13 @@ - TAR_HEADER_RETURN(ARCHIVE_FATAL); - } - -- /* Read 512-byte header record */ -- h = __archive_read_ahead(a, 512, &bytes); -+ /* -+ * Read 512-byte header record -+ * BUN PATCH: or whatever there is behind the end -+ * of the archive. -+ */ -+ h = __archive_read_ahead(a, tar->past_end ? 1 : 512, -+ &bytes); - if (h == NULL && bytes == ARCHIVE_RETRY) { - /* - * BUN PATCH: non-blocking source ran -@@ -909,7 +947,8 @@ - goto bun_retry; - } - if (bytes == 0) { /* EOF at a block boundary. */ -- if (eof_fatal) { -+ /* BUN PATCH: npm drops such a sequence. */ -+ if (eof_fatal && !tar->npm) { - /* We've read a special header already; - * if there's no regular header, then this is - * a premature EOF. */ -@@ -927,6 +966,10 @@ - " detected while reading next header"); - TAR_HEADER_RETURN(ARCHIVE_FATAL); - } -+ if (tar->past_end) { -+ *unconsumed += bytes; -+ continue; -+ } - *unconsumed += 512; - - if (h[0] == 0 && archive_block_is_null(h)) { -@@ -937,6 +980,18 @@ - continue; - } - -+ /* -+ * BUN PATCH: for npm two in a row end the -+ * archive, and nothing behind them is an entry. -+ * Read on to the end of the input, so that the -+ * filters get to check all of it. -+ */ -+ if (tar->npm) { -+ if (++tar->null_blocks == 2) -+ tar->past_end = 1; -+ continue; -+ } -+ - /* Try to consume a second all-null record, as well. */ - /* If we can't, that's okay. */ - tar_flush_unconsumed(a, unconsumed); -@@ -947,6 +1002,7 @@ - archive_clear_error(&a->archive); - TAR_HEADER_RETURN(ARCHIVE_EOF); - } -+ tar->null_blocks = 0; - - /* This is NOT a null block, so it must be a valid header. */ - if (!checksum(a, h)) { -@@ -1127,7 +1183,10 @@ - a->archive.archive_format = ARCHIVE_FORMAT_TAR_GNUTAR; - a->archive.archive_format_name = "GNU tar format"; - err2 = header_gnutar(a, tar, entry, h, unconsumed); -- } else if (memcmp(header->magic, "ustar", 5) == 0) { -+ } else if (memcmp(header->magic, "ustar", 5) == 0 && -+ /* BUN PATCH: npm wants the version too. */ -+ (!tar->npm || memcmp(gnuheader->magic, -+ "ustar\00000", 8) == 0)) { - if (a->archive.archive_format != ARCHIVE_FORMAT_TAR_PAX_INTERCHANGE) { - a->archive.archive_format = ARCHIVE_FORMAT_TAR_USTAR; - a->archive.archive_format_name = "POSIX ustar format"; -@@ -2619,6 +2678,15 @@ - const char *p; - int err = ARCHIVE_OK; - -+ /* -+ * BUN PATCH: npm reads none of the vendor keys that matter here. -+ * They all start with a capital letter. -+ */ -+ if (tar->npm && key[0] >= 'A' && key[0] <= 'Z') { -+ __archive_read_consume(a, value_length); -+ return (ARCHIVE_OK); -+ } -+ - switch (key[0]) { - case 'G': - /* GNU.* extensions */ -@@ -3353,14 +3421,15 @@ - archive_entry_set_ctime(entry, t, 0); - } - -- if (header->realsize[0] != 0) { -+ /* BUN PATCH: npm reads neither this field nor the next. */ -+ if (!tar->npm && header->realsize[0] != 0) { - /* Treat as a synonym for the pax GNU.sparse.realsize attr */ - tar->GNU_sparse_realsize - = tar_atol(header->realsize, sizeof(header->realsize)); - tar->size_fields |= TAR_SIZE_GNU_SPARSE_REALSIZE; - } - -- if (header->sparse[0].offset[0] != 0) { -+ if (!tar->npm && header->sparse[0].offset[0] != 0) { - if (gnu_sparse_old_read(a, tar, header, unconsumed) - != ARCHIVE_OK) - return (ARCHIVE_FATAL); diff --git a/scripts/build/CLAUDE.md b/scripts/build/CLAUDE.md index 4e7606462366..55f29955c068 100644 --- a/scripts/build/CLAUDE.md +++ b/scripts/build/CLAUDE.md @@ -22,7 +22,7 @@ This directory generates `build.ninja`. The scripts **describe** the build; ninj **Deps in our graph by default; native build systems when needed.** `BuildSpec` variants: -- `direct` — list the dep's sources explicitly; each becomes a first-class `cc`/`cxx` edge in our graph and the `.o`s go straight into bun's link. The default for the C/C++ deps (zlib, zstd, boringssl, libarchive, mimalloc, …). Skips a sub-process configure entirely and lets LTO see across the dep boundary. +- `direct` — list the dep's sources explicitly; each becomes a first-class `cc`/`cxx` edge in our graph and the `.o`s go straight into bun's link. The default for the C/C++ deps (zlib, zstd, boringssl, mimalloc, …). Skips a sub-process configure entirely and lets LTO see across the dep boundary. - `nested-cmake` — invoke the dep's own cmake configure + build as ninja edges. For deps whose build is too entangled to list by hand. Flags forwarded via `-DCMAKE_C_FLAGS`; cmake's own dependency tracking handles incrementality inside. - `cargo` — (rule kept for out-of-workspace cargo deps; none today — lolhtml and rust-argon2 are path dependencies compiled as units of bun's own Rust graph). - `prebuilt` — skip build entirely, download compiled `.a`/`.lib` (WebKit, nodejs-headers). diff --git a/scripts/build/binary-expectations.ts b/scripts/build/binary-expectations.ts index d256873a9324..1f7150a244f0 100644 --- a/scripts/build/binary-expectations.ts +++ b/scripts/build/binary-expectations.ts @@ -531,11 +531,7 @@ export function binaryExpectations(cfg: Config): BinaryExpectations { "ole32.dll", ], exact: true, - // bcrypt.dll: libarchive's archive_random.c / archive_digest.c call - // BCryptOpenAlgorithmProvider and BCryptGenRandom. A release link - // drops them as unreferenced (/OPT:REF); a debug link keeps every - // section, and the import with them. Never in a shipped binary. - allowed: [...sanitizerLibs, ...(cfg.debug ? ["bcrypt.dll"] : [])], + allowed: sanitizerLibs, }, minOSVersion: "6.0", forbiddenImports: forbiddenImports(cfg), diff --git a/scripts/build/deps/README.md b/scripts/build/deps/README.md index 7e2a97f18b53..b60e6ea93be4 100644 --- a/scripts/build/deps/README.md +++ b/scripts/build/deps/README.md @@ -99,7 +99,7 @@ export const mydep: Dependency = { patches: ["patches/mydep/fix-something.patch"], // Optional: deps whose SOURCE must be ready before this one builds - // (for -I cross-dep headers). See libarchive for an example. + // (for -I cross-dep headers). See libspng for an example. fetchDeps: ["zlib"], // How to build. `direct` lists sources explicitly; emitDirect compiles @@ -148,7 +148,7 @@ export const mydep: Dependency = { - **mimalloc.ts** — direct build, single unity TU compiled as C++ - **tinycc.ts** — direct build with a build-time codegen tool - **zlib.ts** — direct build with per-source SIMD `-m` flags + `.h.in` substitution -- **libarchive.ts** / **cares.ts** — direct build with hand-written per-target config.h +- **cares.ts** — direct build with hand-written per-target config.h - **boringssl.ts** — direct build with NASM assembly (win-x64) and a large gen/ manifest; `forbidUndefined` (with libuv.ts) keeps a dep that bun points at mimalloc from calling libc's allocator behind its back - **sqlite.ts** — direct build, in-tree source (lives in `src/`, not `vendor/`) - **libuv.ts** — `enabled: cfg => cfg.windows` for a platform-only dep diff --git a/scripts/build/deps/cares.ts b/scripts/build/deps/cares.ts index 801590bbcf50..3ab129ea660c 100644 --- a/scripts/build/deps/cares.ts +++ b/scripts/build/deps/cares.ts @@ -136,8 +136,8 @@ function buildH(cfg: Config): string { } // ─────────────────────────────────────────────────────────────────────────── -// ares_config.h — private, included by every .c via ares_setup.h. Same shape -// as libarchive's: HAVE_* booleans + the recv/send prototype types c-ares +// ares_config.h — private, included by every .c via ares_setup.h. +// HAVE_* booleans + the recv/send prototype types c-ares // uses to cast its socket calls. // ─────────────────────────────────────────────────────────────────────────── diff --git a/scripts/build/deps/index.ts b/scripts/build/deps/index.ts index e86f94a29fe2..334931b8683f 100644 --- a/scripts/build/deps/index.ts +++ b/scripts/build/deps/index.ts @@ -4,7 +4,7 @@ * Order matters for two reasons: * 1. `fetchDeps` relationships — a dep with fetchDeps must come AFTER the * deps it references, so the referenced .ref stamp node exists in ninja - * when we add order-only edges to it. (zlib before libarchive.) + * when we add order-only edges to it. (zlib before libspng.) * 2. Link order — when these libs hit the final link line, static linking * resolves left-to-right. Deps that PROVIDE symbols should come after * deps that USE them. This list becomes the link order. @@ -15,7 +15,6 @@ import { boringssl } from "./boringssl.ts"; import { brotli } from "./brotli.ts"; import { cares } from "./cares.ts"; import { highway } from "./highway.ts"; -import { libarchive } from "./libarchive.ts"; import { libdeflate } from "./libdeflate.ts"; import { libjpegTurbo } from "./libjpeg-turbo.ts"; import { libspng } from "./libspng.ts"; @@ -38,7 +37,7 @@ import { zstd } from "./zstd.ts"; /** * All deps in dependency-resolution + link order. * - * zlib FIRST — libarchive's fetchDeps references it. + * zlib FIRST — libspng's and lsquic's fetchDeps reference it. * brotli libs in internal dep order (common last on link line). * boringssl near the end — many things depend on crypto/ssl symbols. */ @@ -51,7 +50,6 @@ export const allDeps: readonly Dependency[] = [ zstd, brotli, libdeflate, - libarchive, // Image codecs — libspng has fetchDeps:["zlib"], so after zlib. libjpegTurbo, libspng, @@ -82,7 +80,6 @@ export { brotli, cares, highway, - libarchive, libdeflate, libjpegTurbo, libspng, diff --git a/scripts/build/deps/libarchive.ts b/scripts/build/deps/libarchive.ts deleted file mode 100644 index 2d8e0feccd22..000000000000 --- a/scripts/build/deps/libarchive.ts +++ /dev/null @@ -1,330 +0,0 @@ -/** - * libarchive — multi-format archive reader/writer. Bun uses it for tarball - * extraction during `bun install` (npm packages ship as tarballs) and - * `bun pm pack`. - * - * Minimal config: tar + gzip only. Every other codec backend (bz2/lzma/lz4/ - * zstd/openssl/iconv/...) is left out of config.h, so the corresponding - * `archive_*_support_*` calls compile to "format not supported" stubs. - * - * DirectBuild: cmake's configure step here was ~22s of try_compile probes - * to fill in config.h. We replace it with a static per-target header and - * compile the source list straight into our ninja graph. Side benefit: - * the cmake build was non-hermetic — it would detect host libacl/libmd - * and emit references bun never links. The hand-written config.h omits - * those entirely. - */ - -import type { Config } from "../config.ts"; -import type { Dependency } from "../source.ts"; -import { depBuildDir } from "../source.ts"; - -const LIBARCHIVE_COMMIT = "27cbc7827172698143e440801fc0ba39ccb4f1f5"; - -// The unconditional list from libarchive/CMakeLists.txt, minus the read -// formats/filters bun never registers (it only registers tar/gnutar/gzip, -// and select-registered-only.patch stops set_format/append_filter from -// pulling the rest in). Kept even though unused: format_empty/format_raw -// (archive_match.c), ppmd7 (write_set_format_7zip), filter_program -// (append_filter_program_signature) — dead code that other kept files -// still reference by name. -// prettier-ignore -const SOURCES = [ - "archive_acl", "archive_check_magic", "archive_cmdline", "archive_cryptor", - "archive_digest", "archive_entry", "archive_entry_copy_stat", - "archive_entry_link_resolver", "archive_entry_sparse", "archive_entry_stat", - "archive_entry_strmode", "archive_entry_xattr", "archive_hmac", "archive_match", - "archive_options", "archive_pack_dev", "archive_parse_date", "archive_pathmatch", - "archive_ppmd7", "archive_random", "archive_rb", "archive_read", - "archive_read_add_passphrase", "archive_read_append_filter", - "archive_read_data_into_fd", "archive_read_disk_entry_from_file", - "archive_read_disk_posix", "archive_read_disk_set_standard_lookup", - "archive_read_extract", "archive_read_extract2", "archive_read_open_fd", - "archive_read_open_file", "archive_read_open_filename", "archive_read_open_memory", - "archive_read_set_format", "archive_read_set_options", - "archive_read_support_filter_gzip", "archive_read_support_filter_program", - "archive_read_support_format_empty", "archive_read_support_format_raw", - "archive_read_support_format_tar", "archive_string", "archive_string_sprintf", - "archive_time", "archive_util", "archive_version_details", "archive_virtual", - "archive_write", "archive_write_disk_posix", "archive_write_disk_set_standard_lookup", - "archive_write_open_fd", "archive_write_open_file", "archive_write_open_filename", - "archive_write_open_memory", "archive_write_add_filter", - "archive_write_add_filter_b64encode", "archive_write_add_filter_by_name", - "archive_write_add_filter_bzip2", "archive_write_add_filter_compress", - "archive_write_add_filter_grzip", "archive_write_add_filter_gzip", - "archive_write_add_filter_lrzip", "archive_write_add_filter_lz4", - "archive_write_add_filter_lzop", "archive_write_add_filter_none", - "archive_write_add_filter_program", "archive_write_add_filter_uuencode", - "archive_write_add_filter_xz", "archive_write_add_filter_zstd", - "archive_write_set_format", "archive_write_set_format_7zip", - "archive_write_set_format_ar", "archive_write_set_format_by_name", - "archive_write_set_format_cpio", "archive_write_set_format_cpio_binary", - "archive_write_set_format_cpio_newc", "archive_write_set_format_cpio_odc", - "archive_write_set_format_filter_by_ext", "archive_write_set_format_gnutar", - "archive_write_set_format_iso9660", "archive_write_set_format_mtree", - "archive_write_set_format_pax", "archive_write_set_format_raw", - "archive_write_set_format_shar", "archive_write_set_format_ustar", - "archive_write_set_format_v7tar", "archive_write_set_format_warc", - "archive_write_set_format_xar", "archive_write_set_format_zip", - "archive_write_set_options", "archive_write_set_passphrase", "filter_fork_posix", - "xxhash", -]; - -const SOURCES_WIN = [ - "archive_entry_copy_bhfi", - "archive_read_disk_windows", - "archive_windows", - "archive_write_disk_windows", - "filter_fork_windows", -]; - -export const libarchive: Dependency = { - name: "libarchive", - versionMacro: "LIBARCHIVE", - - source: () => ({ - kind: "github-archive", - repo: "libarchive/libarchive", - commit: LIBARCHIVE_COMMIT, - }), - - patches: [ - "patches/libarchive/archive_write_add_filter_gzip.c.patch", - // archive_read_set_format/append_filter register by code through a - // switch over every reader/filter, which keeps them all linked. Bun - // registers tar/gnutar/gzip itself, so make those calls only select - // an already-registered one. - "patches/libarchive/select-registered-only.patch", - // Propagate ARCHIVE_RETRY from the client read callback up through - // the gzip filter and tar reader so the worker-thread extract loop - // in `bun install` can yield and resume as HTTP chunks arrive. See - // src/install/TarballStream.rs. - "patches/libarchive/nonblocking-read.patch", - // Windows: cache get_current_codepage()/get_current_oemcp() after the - // first call instead of querying setlocale(LC_CTYPE, NULL) once per - // created charset converter — `bun install` creates one per extracted - // archive across many worker threads at once, and the process never - // changes its locale, so the repeated trips through the UCRT locale - // machinery (and its heap) are pure overhead and needless cross-thread - // CRT-heap traffic. - "patches/libarchive/archive_string-codepage-cache.patch", - // Upstream's gzip read filter never checks the CRC32/ISIZE trailer. - // Verify it, so a corrupted tarball fails the extract instead of - // installing the corrupted bytes. - "patches/libarchive/gzip-verify-trailer.patch", - // Adds the tar reader option `npm`, which `bun install` sets: where tar - // readers differ in what they take for a file, for its name or for the end - // of the archive, do what npm's does. Without the option nothing changes. - "patches/libarchive/tar-npm-compat.patch", - ], - - // zlib-ng generates zlib.h during its own build; libarchive's gzip filter - // includes it. We don't link zlib here (bun's final link does), just need - // the header on -I. - fetchDeps: ["zlib"], - - build: cfg => ({ - kind: "direct", - sources: [...SOURCES, ...(cfg.windows ? SOURCES_WIN : [])].map(s => `libarchive/${s}.c`), - // zlib's build dir holds the generated zlib.h (subst'd from .in) that - // the gzip filter includes. Absolute path → emitDirect quotes it. - // android: archive.h does `#include ` under __ANDROID__; - // that header lives under contrib/android/include. - includes: ["libarchive", depBuildDir(cfg, "zlib"), ...(cfg.abi === "android" ? ["contrib/android/include"] : [])], - pic: true, - defines: { - HAVE_CONFIG_H: 1, - LIBARCHIVE_STATIC: 1, - // ELF-only; clang-cl doesn't support __attribute__((visibility)). - ...(!cfg.windows && { __LIBARCHIVE_ENABLE_VISIBILITY: true }), - // Upstream's CMakeLists sets this on MSVC; the DirectBuild conversion - // missed it (97× -Wdeprecated-declarations on localtime/strncpy/etc). - ...(cfg.windows && { _CRT_SECURE_NO_DEPRECATE: true }), - }, - cflags: [ - ...(cfg.windows ? [] : ["-fvisibility=hidden"]), - // Vendored C; const-discard casts in archive_options/_pack_dev/etc. - "-Wno-incompatible-pointer-types-discards-qualifiers", - ], - headers: { "config.h": configH(cfg) }, - }), - - provides: cfg => ({ - libs: [], - includes: cfg.abi === "android" ? ["libarchive", "contrib/android/include"] : ["libarchive"], - }), -}; - -// ─────────────────────────────────────────────────────────────────────────── -// config.h — replaces cmake's feature-detection pass -// ─────────────────────────────────────────────────────────────────────────── -// -// Mirrors the relevant subset of build/cmake/config.h.in for our four -// targets (linux-gnu, linux-musl, darwin, windows). Anything not defined -// here makes libarchive take its portable fallback. -// -// Deliberately omitted regardless of host: -// - HAVE_{BZLIB,LZMA,LZ4,ZSTD,OPENSSL,NETTLE,MBEDTLS,ICONV,LIBXML2,EXPAT, -// LIBB2}_H — codecs we don't ship. -// - HAVE_LIB{ACL,ATTR,MD} / ARCHIVE_ACL_* / ARCHIVE_CRYPTO_* — the cmake -// build would pick these up if dev headers happen to be installed, -// producing a non-hermetic binary. Bun never calls the ACL/digest entry -// points, so the extra code was dead-stripped anyway. -// -// If a libarchive bump adds a new HAVE_* check, the worst case is a missed -// optimization. The two cases that fail loudly are SIZEOF_LONG/WCHAR_T and -// the struct-stat-nsec field name — both pinned per-target below. - -const def1 = (names: string[]) => names.map(n => `#define ${n} 1`).join("\n"); - -// prettier-ignore -const ALWAYS = def1([ - "HAVE_INT16_T", "HAVE_INT32_T", "HAVE_INT64_T", "HAVE_INTMAX_T", - "HAVE_UINT8_T", "HAVE_UINT16_T", "HAVE_UINT32_T", "HAVE_UINT64_T", "HAVE_UINTMAX_T", - "HAVE_DECL_INT32_MAX", "HAVE_DECL_INT32_MIN", "HAVE_DECL_INT64_MAX", "HAVE_DECL_INT64_MIN", - "HAVE_DECL_INTMAX_MAX", "HAVE_DECL_INTMAX_MIN", "HAVE_DECL_SIZE_MAX", - "HAVE_DECL_UINT32_MAX", "HAVE_DECL_UINT64_MAX", "HAVE_DECL_UINTMAX_MAX", - "HAVE_CTYPE_H", "HAVE_ERRNO_H", "HAVE_FCNTL_H", "HAVE_LIMITS_H", "HAVE_LOCALE_H", - "HAVE_SIGNAL_H", "HAVE_STDARG_H", "HAVE_STDINT_H", "HAVE_STDIO_H", "HAVE_STDLIB_H", - "HAVE_STRING_H", "HAVE_TIME_H", "HAVE_WCHAR_H", "HAVE_WCTYPE_H", "HAVE_SYS_STAT_H", - "HAVE_SYS_TYPES_H", "HAVE_INTTYPES_H", - "HAVE_EILSEQ", "HAVE_WCHAR_T", "HAVE_FSTAT", "HAVE_GETPID", "HAVE_MEMMOVE", - "HAVE_MEMORY_H", "HAVE_MKDIR", "HAVE_SETLOCALE", "HAVE_STRCHR", "HAVE_STRDUP", - "HAVE_STRERROR", "HAVE_STRFTIME", "HAVE_STRNLEN", "HAVE_STRRCHR", "HAVE_TZSET", - "HAVE_VPRINTF", "HAVE_WCRTOMB", "HAVE_WCSCMP", "HAVE_WCSCPY", "HAVE_WCSLEN", - "HAVE_WCTOMB", "HAVE_WMEMCMP", "HAVE_WMEMCPY", "HAVE_WMEMMOVE", "HAVE_MBRTOWC", - "HAVE_ZLIB_H", -]); - -// POSIX: every non-Windows target (linux glibc, linux musl, darwin). -// prettier-ignore -const POSIX = def1([ - "HAVE_DECL_SSIZE_MAX", "HAVE_DECL_STRERROR_R", - "HAVE_DIRENT_H", "HAVE_DLFCN_H", "HAVE_FNMATCH_H", "HAVE_GRP_H", "HAVE_LANGINFO_H", - "HAVE_PATHS_H", "HAVE_POLL_H", "HAVE_PTHREAD_H", "HAVE_PWD_H", "HAVE_REGEX_H", - "HAVE_SPAWN_H", "HAVE_STRINGS_H", "HAVE_SYS_CDEFS_H", "HAVE_SYS_IOCTL_H", - "HAVE_SYS_MOUNT_H", "HAVE_SYS_PARAM_H", "HAVE_SYS_POLL_H", "HAVE_SYS_SELECT_H", - "HAVE_SYS_STATVFS_H", "HAVE_SYS_TIME_H", "HAVE_SYS_UTSNAME_H", "HAVE_SYS_WAIT_H", - "HAVE_UNISTD_H", "HAVE_UTIME_H", - "HAVE_CHOWN", "HAVE_CHROOT", "HAVE_CTIME_R", "HAVE_DIRFD", - "HAVE_FCHDIR", "HAVE_FCHMOD", "HAVE_FCHOWN", "HAVE_FCNTL", "HAVE_FDOPENDIR", - "HAVE_FNMATCH", "HAVE_FORK", "HAVE_FSEEKO", "HAVE_FSTATAT", "HAVE_FSTATVFS", - "HAVE_FTRUNCATE", "HAVE_FUTIMENS", "HAVE_FUTIMES", "HAVE_GETEGID", "HAVE_GETEUID", - "HAVE_GETGRGID_R", "HAVE_GETGRNAM_R", "HAVE_GETLINE", "HAVE_GETPWNAM_R", - "HAVE_GETPWUID_R", "HAVE_GMTIME_R", "HAVE_LCHOWN", "HAVE_LINK", "HAVE_LINKAT", - "HAVE_LOCALTIME_R", "HAVE_LSTAT", "HAVE_LUTIMES", "HAVE_MKFIFO", "HAVE_MKNOD", - "HAVE_MKSTEMP", "HAVE_NL_LANGINFO", "HAVE_OPENAT", "HAVE_PIPE", "HAVE_POLL", - "HAVE_POSIX_SPAWNP", "HAVE_READLINK", "HAVE_READLINKAT", "HAVE_SELECT", - "HAVE_SETENV", "HAVE_SIGACTION", "HAVE_STATVFS", "HAVE_STRERROR_R", - "HAVE_SYMLINK", "HAVE_SYSCONF", "HAVE_TCGETATTR", "HAVE_TCSETATTR", "HAVE_TIMEGM", - "HAVE_UNLINKAT", "HAVE_UNSETENV", "HAVE_UTIME", "HAVE_UTIMENSAT", "HAVE_UTIMES", - "HAVE_VFORK", - "HAVE_STRUCT_STAT_ST_BLKSIZE", "HAVE_STRUCT_TM_TM_GMTOFF", -]); - -// Linux-only. xattr via directly — modern glibc/musl ship it, -// no -lattr needed. -// prettier-ignore -const LINUX = def1([ - "HAVE_LINUX_FIEMAP_H", "HAVE_LINUX_FS_H", "HAVE_LINUX_MAGIC_H", "HAVE_LINUX_TYPES_H", - "HAVE_SYS_STATFS_H", "HAVE_SYS_SYSMACROS_H", "HAVE_SYS_VFS_H", "HAVE_SYS_XATTR_H", - "HAVE_STATFS", "HAVE_FSTATFS", "HAVE_FUTIMESAT", - "HAVE_FGETXATTR", "HAVE_FLISTXATTR", "HAVE_FSETXATTR", "HAVE_GETXATTR", - "HAVE_LGETXATTR", "HAVE_LISTXATTR", "HAVE_LLISTXATTR", "HAVE_LSETXATTR", - "HAVE_STRUCT_STAT_ST_MTIM_TV_NSEC", - "HAVE_WORKING_FS_IOC_GETFLAGS", - "MAJOR_IN_SYSMACROS", "ARCHIVE_XATTR_LINUX", -]); - -// FreeBSD: BSD-style stat (st_mtim, st_flags, st_birthtim), extattr_* xattr -// API, chflags family. No — extattr lives in . -// prettier-ignore -const FREEBSD = def1([ - "HAVE_ARC4RANDOM_BUF", - "HAVE_SYS_EXTATTR_H", "HAVE_SYS_MOUNT_H", - "HAVE_STATFS", "HAVE_FSTATFS", - "HAVE_LCHMOD", "HAVE_LCHFLAGS", "HAVE_CHFLAGS", "HAVE_FCHFLAGS", - "HAVE_EXTATTR_GET_FILE", "HAVE_EXTATTR_LIST_FILE", "HAVE_EXTATTR_SET_FD", - "HAVE_EXTATTR_SET_FILE", "HAVE_DECL_EXTATTR_NAMESPACE_USER", - "HAVE_STRUCT_STAT_ST_MTIM_TV_NSEC", "HAVE_STRUCT_STAT_ST_BIRTHTIM", - "HAVE_STRUCT_STAT_ST_FLAGS", - "HAVE_READPASSPHRASE", "HAVE_READPASSPHRASE_H", - "ARCHIVE_XATTR_FREEBSD", -]); - -// prettier-ignore -const DARWIN = def1([ - // arc4random_buf: BSD libc + glibc≥2.36 only. Bun's Linux CI targets - // older glibc, so Linux falls back to /dev/urandom. - "HAVE_ARC4RANDOM_BUF", - "HAVE_SYS_XATTR_H", "HAVE_COPYFILE_H", - "HAVE_FSTATFS", "HAVE_STATFS", "HAVE_LCHMOD", "HAVE_LCHFLAGS", "HAVE_CHFLAGS", - "HAVE_FCHFLAGS", - "HAVE_FGETXATTR", "HAVE_FLISTXATTR", "HAVE_FSETXATTR", "HAVE_GETXATTR", - "HAVE_LISTXATTR", "HAVE_SETXATTR", - "HAVE_STRUCT_STAT_ST_MTIMESPEC_TV_NSEC", "HAVE_STRUCT_STAT_ST_BIRTHTIME", - "HAVE_STRUCT_STAT_ST_BIRTHTIMESPEC_TV_NSEC", "HAVE_STRUCT_STAT_ST_FLAGS", - "HAVE_STRUCT_STATFS_F_IOSIZE", - "ARCHIVE_XATTR_DARWIN", -]); - -// Windows: clang-cl + UCRT. archive_windows.h supplies most POSIX shims; -// this declares what the UCRT actually has. No ARCHIVE_CRYPTO_*_WIN — we -// don't need digests for tar/gzip. (archive_random.c/archive_util.c call -// BCryptGenRandom regardless; bun's link line already carries bcrypt.lib.) -// prettier-ignore -const WINDOWS = def1([ - "HAVE_IO_H", "HAVE_DIRECT_H", "HAVE_PROCESS_H", "HAVE_SYS_UTIME_H", "HAVE_WINDOWS_H", - "HAVE__CTIME64_S", "HAVE__FSEEKI64", "HAVE__GET_TIMEZONE", "HAVE__GMTIME64_S", - "HAVE__LOCALTIME64_S", "HAVE__MKGMTIME64", - "HAVE_STRNCPY_S", "HAVE_WCSCPY_S", "HAVE_WCSNCPY_S", -]) + ` -/* POSIX type fallbacks — UCRT's doesn't define these. - Values match cmake's WIN32 branch (CMakeLists.txt CHECK_TYPE_SIZE block). */ -#define gid_t short -#define uid_t short -#define id_t short -#define mode_t unsigned short -#define pid_t int -#define ssize_t int64_t -`; - -function configH(cfg: Config): string { - const longSize = cfg.windows ? 4 : 8; - const wcharSize = cfg.windows ? 2 : 4; - const platform = cfg.windows ? WINDOWS : `${POSIX}\n${cfg.darwin ? DARWIN : cfg.freebsd ? FREEBSD : LINUX}`; - - // Feature-test macros must come before any system header. archive_platform.h - // includes config.h first, so defining them here is early enough. - const featureTest = cfg.windows - ? "#define NTDDI_VERSION 0x0A000000\n#define _WIN32_WINNT 0x0A00\n#define WINVER 0x0A00" - : "#define _GNU_SOURCE 1\n#define _DARWIN_C_SOURCE 1\n#define __EXTENSIONS__ 1"; - - return `/* Generated by scripts/build/deps/libarchive.ts for ${cfg.os}-${cfg.arch} */ -#define __LIBARCHIVE_CONFIG_H_INCLUDED 1 -${featureTest} - -#define SIZEOF_SHORT 2 -#define SIZEOF_INT 4 -#define SIZEOF_LONG ${longSize} -#define SIZEOF_LONG_LONG 8 -#define SIZEOF_UNSIGNED_SHORT 2 -#define SIZEOF_UNSIGNED 4 -#define SIZEOF_UNSIGNED_LONG ${longSize} -#define SIZEOF_UNSIGNED_LONG_LONG 8 -#define SIZEOF_WCHAR_T ${wcharSize} -#define ICONV_CONST - -#define LIBARCHIVE_VERSION_NUMBER "3008009" -#define LIBARCHIVE_VERSION_STRING "3.8.9" -#define BSDTAR_VERSION_STRING "3.8.9" -#define BSDCPIO_VERSION_STRING "3.8.9" -#define BSDCAT_VERSION_STRING "3.8.9" -#define BSDUNZIP_VERSION_STRING "3.8.9" -#define VERSION "3.8.9" - -${ALWAYS} - -${platform} -`; -} diff --git a/scripts/build/deps/zlib.ts b/scripts/build/deps/zlib.ts index 2ee3470f9502..03179d05a335 100644 --- a/scripts/build/deps/zlib.ts +++ b/scripts/build/deps/zlib.ts @@ -236,7 +236,7 @@ export const zlib: Dependency = { }, // The substituted zlib.h / zconf.h land in the build dir, not the source - // tree, so consumers (libarchive, bun's own bindings) include from there. + // tree, so consumers (libspng, bun's own bindings) include from there. provides: cfg => ({ libs: [], includes: [depBuildDir(cfg, "zlib")], diff --git a/scripts/build/deps/zstd.ts b/scripts/build/deps/zstd.ts index 803636dea2de..351bb62c9fc9 100644 --- a/scripts/build/deps/zstd.ts +++ b/scripts/build/deps/zstd.ts @@ -83,7 +83,7 @@ export const zstd: Dependency = { pic: true, // XXH_NAMESPACE must be a bare token prefix (xxhash pastes it onto // symbol names), not a string literal — DirectBuild.defines would - // wrap it in quotes. Avoids clashes with lshpack/libarchive's copies. + // wrap it in quotes. Avoids clashes with lshpack's copy. cflags: ["-DXXH_NAMESPACE=ZSTD_"], }; return spec; diff --git a/scripts/build/fetch-cli.ts b/scripts/build/fetch-cli.ts index 1763f555f7b1..1325d2c69270 100644 --- a/scripts/build/fetch-cli.ts +++ b/scripts/build/fetch-cli.ts @@ -70,7 +70,7 @@ async function main(): Promise { text = text.split(pairs[i]!).join(pairs[i + 1]!); } // dep_subst is restat=1 — preserve mtime when the rendered text is - // unchanged so depfile-tracked .o files (libarchive ← zlib.h) don't + // unchanged so depfile-tracked .o files (libspng ← zlib.h) don't // recompile after a no-op re-fetch. writeIfChanged(outPath, text); return; diff --git a/scripts/build/source.ts b/scripts/build/source.ts index 633cf107a87e..f9db40929ff7 100644 --- a/scripts/build/source.ts +++ b/scripts/build/source.ts @@ -343,7 +343,7 @@ export interface NestedCmakeBuild { * * Most deps don't need this (we link statically into a non-PIE executable), * but some build intermediate tools or have internal shared libs that - * require PIC. cares/highway/libarchive set it. + * require PIC. cares/highway set it. */ pic?: boolean; /** @@ -447,7 +447,6 @@ export type DepName = | "brotli" | "cares" | "highway" - | "libarchive" | "libdeflate" | "libjpeg-turbo" | "libspng" @@ -491,9 +490,9 @@ export interface Dependency { /** * Other deps that must be BUILT before this dep's configure runs. - * Used for header-level dependencies — e.g. libarchive needs zlib's + * Used for header-level dependencies — e.g. a dep that needs zlib's * headers at configure time (`check_include_file("zlib.h")`). zlib-ng - * generates `zlib.h` during its own cmake configure, so libarchive must + * generates `zlib.h` during its own cmake configure, so that dep must * wait for zlib's full build, not just its source fetch. * * Resolves to the named dep's build outputs (lib files for nested-cmake, @@ -900,7 +899,7 @@ export function resolveDep( // ─── Resolve fetchDeps → extra inputs on configure + build ─── // These are deps that must be BUILT before we configure (not link). - // E.g. libarchive's configure runs check_include_file("zlib.h"), and + // E.g. a dep's configure runs check_include_file("zlib.h"), and // zlib-ng generates zlib.h during its own cmake configure — so we depend // on zlib's lib output (which implies its configure ran). // @@ -1189,7 +1188,7 @@ function emitNestedCmake( // // NOT setting TRY_COMPILE_TARGET_TYPE=STATIC_LIBRARY: it stops // try_compile from linking, which makes check_function_exists and - // check_library_exists always succeed → libarchive "finds" fork, + // check_library_exists always succeed → a dep "finds" fork, // posix_spawnp, libmd on Windows. If llvm-mt is missing, better to // fail fast at "compiler works" than mis-detect every feature. args.push(`-DCMAKE_LINKER=${slash(cfg.ld)}`); @@ -1300,7 +1299,7 @@ function emitNestedCmake( inputs: [], // Configure re-runs if: source changed, cmake binary changed, preBuild // outputs changed. fetchDeps stamps are order-only — can't configure - // until cross-dep headers are on disk (libarchive's check_include_file + // until cross-dep headers are on disk (a check_include_file // for zlib.h runs at configure time). implicitInputs: [sourceStamp, cfg.cmake, ...preBuildOutputs], orderOnlyInputs: fetchDepStamps, diff --git a/src/libarchive/Cargo.toml b/src/archive/Cargo.toml similarity index 83% rename from src/libarchive/Cargo.toml rename to src/archive/Cargo.toml index ab298977e303..0ac120a1b61f 100644 --- a/src/libarchive/Cargo.toml +++ b/src/archive/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "bun_libarchive" +name = "bun_archive" version.workspace = true edition.workspace = true @@ -11,8 +11,7 @@ workspace = true [dependencies] thiserror.workspace = true -bun_opaque.workspace = true -bstr.workspace = true +bun_tar.workspace = true scopeguard.workspace = true bun_core.workspace = true bun_alloc.workspace = true diff --git a/src/libarchive/error.rs b/src/archive/error.rs similarity index 64% rename from src/libarchive/error.rs rename to src/archive/error.rs index f91a5c69c153..882cbe97dda1 100644 --- a/src/libarchive/error.rs +++ b/src/archive/error.rs @@ -12,6 +12,21 @@ pub enum Error { Paths(#[from] bun_paths::Error), } +/// Why the contents of an entry did not reach the file. +#[derive(Debug, thiserror::Error)] +pub enum ReadDataError { + #[error(transparent)] + Archive(#[from] bun_tar::Error), + #[error("{0}")] + Sys(bun_sys::Error), +} + +impl From for ReadDataError { + fn from(e: bun_sys::Error) -> Self { + Self::Sys(e) + } +} + impl Error { pub(crate) fn name(&self) -> &'static str { match self { @@ -30,6 +45,15 @@ impl bun_core::output::ErrName for Error { } } +impl From for Error { + fn from(e: bun_tar::Error) -> Self { + match e { + bun_tar::Error::Alloc(e) => Self::Alloc(e), + _ => Self::Fail, + } + } +} + impl From for Error { fn from(e: bun_sys::Error) -> Self { Self::Sys(e.into()) diff --git a/src/archive/lib.rs b/src/archive/lib.rs new file mode 100644 index 000000000000..f8d73ee761a2 --- /dev/null +++ b/src/archive/lib.rs @@ -0,0 +1,1015 @@ +//! Extracts tar archives to disk. The format itself is `bun_tar`'s. +#![warn(unused_must_use)] + +use bun_alloc::AllocError; +use bun_collections::StringArrayHashMap; +#[cfg(unix)] +use bun_core::ZStr; +use bun_core::{MutableString, strings}; +use bun_core::{Output, slice_as_bytes}; +#[cfg(unix)] +use bun_paths::PathBuffer; +use bun_paths::{OSPathChar, SEP, SEP_STR}; +use bun_sys::{self, Fd, FdExt}; +use bun_wyhash::hash; + +pub mod error; +pub use error::{Error, ReadDataError, Result}; + +pub use bun_tar as tar; +use bun_tar::{Dialect, Entry, Event, MemoryDecoder, Options}; + +/// The entry's path the way the file system takes it. `buf` is scratch space for Windows. +#[cfg(not(windows))] +pub fn os_path<'a>( + entry: &'a Entry, + _buf: &'a mut Vec, +) -> Result<&'a [OSPathChar], AllocError> { + Ok(entry.path()) +} +#[cfg(windows)] +pub fn os_path<'a>( + entry: &'a Entry, + buf: &'a mut Vec, +) -> Result<&'a [OSPathChar], AllocError> { + buf.clear(); + bun_tar::compose::to_utf16(entry.path(), entry.path_is_utf8(), buf).map_err(|_| AllocError)?; + Ok(buf) +} + +/// The entry's path as a program sees it: the name [`os_path`] gives the file, in UTF-8 on Windows. +#[cfg(not(windows))] +pub fn display_path(entry: &Entry) -> Result, AllocError> { + Ok(entry.path().into()) +} +#[cfg(windows)] +pub fn display_path(entry: &Entry) -> Result, AllocError> { + let mut wide = Vec::new(); + Ok(strings::to_utf8_list_with_type(Vec::new(), os_path(entry, &mut wide)?)?.into()) +} + +/// A reader for an archive that is all in memory, gzipped or not. Fails for what is not one. +pub fn open_memory(bytes: &[u8], dialect: Dialect) -> bun_tar::Result> { + MemoryDecoder::new(bytes, Options::for_host(dialect)) +} + +/// Writes the contents of the current entry to `fd`. +pub fn read_data_into_fd( + reader: &mut MemoryDecoder<'_>, + fd: Fd, + strategy: &mut WriteStrategy, +) -> Result<(), ReadDataError> { + let mut writer = EntryWriter::new(fd); + loop { + match reader.next()? { + Event::Data { offset, bytes } => writer.write(strategy, offset, bytes)?, + Event::EndOfEntry { size } => return Ok(writer.finish(size)?), + Event::NeedInput | Event::Entry(_) | Event::EndOfArchive => return Ok(()), + } + } +} + +fn log_archive_error(path: &[OSPathChar], err: impl core::fmt::Display) { + Output::err( + "archive error", + "extracting {}: {}", + (bun_core::fmt::fmt_os_path(path, Default::default()), err), + ); +} + +/// One per extraction. Once `pwrite` fails, later entries don't try it again. +pub struct WriteStrategy { + pub pwrite: bool, +} + +impl Default for WriteStrategy { + fn default() -> Self { + Self { pwrite: cfg!(unix) } + } +} + +/// Never sizes a write or an allocation from the entry header or the sparse map, because +/// nothing verifies them. +pub struct EntryWriter { + fd: Fd, + /// One past the last byte written. + end: i64, + /// The file position, which only `write` and `lseek` move. + cursor: i64, + #[cfg(windows)] + marked_sparse: bool, +} + +impl EntryWriter { + pub fn new(fd: Fd) -> Self { + Self { + fd, + end: 0, + cursor: 0, + #[cfg(windows)] + marked_sparse: false, + } + } + + #[inline] + pub fn fd(&self) -> Fd { + self.fd + } + + /// NTFS gives a range that nothing writes real clusters unless the file is marked sparse. + #[inline] + fn before_hole(&mut self) { + #[cfg(windows)] + if !self.marked_sparse { + self.marked_sparse = true; + // A volume without sparse file support refuses the mark. The hole then takes up + // disk space. + let _ = bun_sys::set_sparse(self.fd); + } + } + + pub fn write( + &mut self, + strategy: &mut WriteStrategy, + offset: i64, + data: &[u8], + ) -> bun_sys::Maybe<()> { + if data.is_empty() { + return Ok(()); + } + let fd = self.fd; + let file = bun_sys::File::borrow(&fd); + + #[cfg(unix)] + if strategy.pwrite { + match file.pwrite_all(data, offset) { + Ok(()) => { + self.end = self.end.max(offset + data.len() as i64); + return Ok(()); + } + Err(_) => { + strategy.pwrite = false; + bun_core::debug_warn!( + "archive: falling back to write() after pwrite() failure", + ); + } + } + } + #[cfg(not(unix))] + let _ = strategy; + + if offset != self.cursor { + if offset > self.end { + self.before_hole(); + } + bun_sys::set_file_offset(self.fd, offset as u64)?; + self.cursor = offset; + } + + file.write_all(data)?; + self.cursor += data.len() as i64; + self.end = self.end.max(self.cursor); + Ok(()) + } + + /// `end` is the length of the file, which the last write need not have reached. + pub fn finish(&mut self, end: i64) -> bun_sys::Maybe<()> { + if end > self.end { + self.before_hole(); + bun_sys::ftruncate(self.fd, end)?; + self.end = end; + } + Ok(()) + } +} + +/// `mkdirat` mode for a directory entry. `perm` is whatever the header +/// encoded (a GNU base-256 field need not fit in `0o7777`), so mask it, then +/// make readable directories listable like node-tar does: +/// https://github.com/npm/node-tar/blob/main/lib/mode-fix.js +#[cfg(not(windows))] +pub fn directory_mode(perm: bun_sys::Mode) -> bun_sys::Mode { + let mut mode = perm & 0o7777; + if (mode & 0o400) != 0 { + mode |= 0o100; + } + if (mode & 0o40) != 0 { + mode |= 0o10; + } + if (mode & 0o4) != 0 { + mode |= 0o1; + } + mode +} + +/// Returns the path where node-tar would extract the npm tarball entry `path`, whose leading +/// `package/` is already removed. The result is empty for an entry that node-tar skips. +pub fn normalize_npm_entry_path<'a>( + path: &[OSPathChar], + buf: &'a mut [OSPathChar], +) -> &'a mut [OSPathChar] { + if path + .split(|c| bun_paths::is_sep_any_t(*c)) + .any(|part| part == [b'.' as OSPathChar; 2]) + { + return &mut buf[..0]; + } + #[cfg(windows)] + return bun_paths::resolve_path::normalize_buf_t::(path, buf); + #[cfg(not(windows))] + bun_paths::resolve_path::normalize_string_generic_t::(path, buf, b'/', |c| { + c == b'/' + }) +} + +/// Returns the first component of an entry's path, which is the archive's root folder. +pub fn first_path_component(path: &[OSPathChar]) -> &[OSPathChar] { + let sep = b'/' as OSPathChar; + let start = path.iter().take_while(|c| **c == sep).count(); + let rest = &path[start..]; + &rest[..strings::index_of_scalar(rest, sep).unwrap_or(rest.len())] +} + +/// Validates that a symlink target doesn't escape the extraction directory. +/// Returns true if the symlink is safe (target stays within extraction dir), +/// false if it would escape (e.g., via ../ traversal or absolute path). +/// +/// The check works by normalizing `symlink_dir/link_target` as a relative +/// path with leading `..` preserved; the target is unsafe if the result +/// climbs above the extraction root. +#[cfg(unix)] +pub fn is_symlink_target_safe( + symlink_path: &[u8], + link_target_bytes: &[u8], + symlink_join_buf: &mut Option, +) -> bool { + // Absolute symlink targets are never safe - they could point anywhere + if link_target_bytes.is_empty() || link_target_bytes[0] == b'/' { + return false; + } + + let mut seen_named_component = false; + for component in strings::split(link_target_bytes, b"/") { + match component { + b"" | b"." => {} + b".." => { + if seen_named_component { + return false; + } + } + _ => seen_named_component = true, + } + } + + // Get the directory containing the symlink + let symlink_dir = bun_paths::dirname_simple(symlink_path); + + let join_buf: &mut PathBuffer = + symlink_join_buf.get_or_insert_with(bun_paths::path_buffer_pool::get); + + // Normalize symlink_dir/link_target as a relative path. An absolute fake + // root cannot be used here: POSIX normalization clamps excess `..` at `/`, + // so a target like `../../../packages/x` would normalize back under any + // fake root that happens to match a real ancestor directory name. + if symlink_dir.len() + 1 + link_target_bytes.len() >= join_buf.len() { + return false; + } + let mut written = 0usize; + if !symlink_dir.is_empty() { + join_buf[..symlink_dir.len()].copy_from_slice(symlink_dir); + written = symlink_dir.len(); + join_buf[written] = b'/'; + written += 1; + } + join_buf[written..written + link_target_bytes.len()].copy_from_slice(link_target_bytes); + written += link_target_bytes.len(); + + let mut norm_buf = bun_paths::path_buffer_pool::get(); + let resolved = bun_paths::resolve_path::normalize_string_generic_t::( + &join_buf[..written], + &mut norm_buf[..], + b'/', + |c| c == b'/', + ); + + !(strings::eql(resolved, b"..") || strings::has_prefix_comptime(resolved, b"../")) +} + +#[cfg(unix)] +pub struct DeferredSymlink { + path: bun_core::ZBox, + target: bun_core::ZBox, +} + +#[cfg(unix)] +impl DeferredSymlink { + pub fn new(path: &[u8], target: &[u8]) -> Self { + Self { + path: bun_core::ZBox::from_bytes(path), + target: bun_core::ZBox::from_bytes(target), + } + } +} + +#[cfg(unix)] +fn open_dir_with_stat(dir_fd: Fd, sub_path: &[u8]) -> Option<(Fd, bun_sys::Stat)> { + let fd = bun_sys::open_dir_at(dir_fd, sub_path).ok()?; + match bun_sys::fstat(fd) { + Ok(st) => Some((fd, st)), + Err(_) => { + fd.close(); + None + } + } +} + +#[cfg(unix)] +pub fn create_deferred_symlinks(dir_fd: Fd, symlinks: &[DeferredSymlink], log: bool) -> u32 { + let mut parents: Vec> = Vec::with_capacity(symlinks.len()); + for symlink in symlinks { + let dirname = bun_paths::dirname_simple(symlink.path.as_bytes()); + if dirname.is_empty() { + parents.push(None); + continue; + } + let _ = dir_fd.make_path_u8(dirname); + parents.push(open_dir_with_stat(dir_fd, dirname).map(|(fd, st)| { + fd.close(); + st + })); + } + + let mut created: u32 = 0; + for (symlink, expected) in symlinks.iter().zip(parents) { + let dirname = bun_paths::dirname_simple(symlink.path.as_bytes()); + let target = symlink.target.as_zstr(); + let result = if dirname.is_empty() { + bun_sys::symlinkat(target, dir_fd, symlink.path.as_zstr()) + } else { + let name = + ZStr::from_slice_with_nul(&symlink.path.as_bytes_with_nul()[dirname.len() + 1..]); + match (open_dir_with_stat(dir_fd, dirname), expected) { + (Some((parent, st)), Some(expected)) + if st.st_dev == expected.st_dev && st.st_ino == expected.st_ino => + { + let result = bun_sys::symlinkat(target, parent, name); + parent.close(); + result + } + (parent, _) => { + if let Some((parent, _)) = parent { + parent.close(); + } + if log { + bun_core::warn!( + "Skipping symlink whose parent directory changed during extraction: {}\n", + bun_core::fmt::escape_control_chars(symlink.path.as_bytes()), + ); + } + continue; + } + } + }; + match result { + Ok(()) => created += 1, + Err(_) => { + if log { + bun_core::warn!( + "Skipping symlink that could not be created: {} -> {}\n", + bun_core::fmt::escape_control_chars(symlink.path.as_bytes()), + bun_core::fmt::escape_control_chars(symlink.target.as_bytes()), + ); + } + } + } + } + created +} + +/// Recursive mkdir over a WTF-16 path: component-iterates the +/// wide path and `NtCreateFile`s each prefix with `FILE_OPEN_IF`, walking back +/// on `FileNotFound` and forward again on success. This stays in WTF-16 +/// throughout (no UTF-8 round-trip — `bun_sys::make_path_w` is the *different* +/// `bun.makePathW` helper which transcodes via `from_w_path` and would lose +/// lone surrogates / skip `\??\` long-path prefixing). +#[cfg(windows)] +fn make_path_u16(dir_fd: Fd, sub_path: &[u16]) -> crate::Result<()> { + use bun_sys::{E, WindowsOpenDirOp, WindowsOpenDirOptions, open_dir_at_windows}; + // Access mask (`STANDARD_RIGHTS_READ | FILE_READ_ATTRIBUTES | + // FILE_READ_EA | SYNCHRONIZE | FILE_TRAVERSE`) is selected by setting `read_only`, + // and `FILE_OPEN_IF` via `OpenOrCreate`. + let opts = WindowsOpenDirOptions { + op: WindowsOpenDirOp::OpenOrCreate, + ..Default::default() + }; + // tar entry paths are dir-relative (no drive/UNC/`\??\`) so `init` never + // returns BadPathName here. + let it = bun_paths::ComponentIterator::init(sub_path, bun_paths::PathFormat::Windows)?; + bun_paths::make_path_with(it, |prefix| { + match open_dir_at_windows(dir_fd, prefix, opts) { + Ok(fd) => { + fd.close(); + Ok(bun_paths::MakePathStep::Created) + } + Err(e) if e.get_errno() == E::ENOENT => Ok(bun_paths::MakePathStep::NotFound(e.into())), + Err(e) => Err(e.into()), + } + }) +} + +struct OutputFile { + fd: Fd, + close: bool, +} + +impl Drop for OutputFile { + fn drop(&mut self) { + if self.close { + // Closing is slow under Windows antivirus, but closing asynchronously gains nothing and + // can make NtSetInformationFile fail. + self.fd.close(); + } + } +} + +pub struct Archiver; + +pub mod archiver { + use super::*; + + pub struct Context { + pub pluckers: Vec, + pub overwrite_list: StringArrayHashMap<()>, + } + + pub struct Plucker { + pub contents: MutableString, + pub(crate) filename_hash: u64, + pub found: bool, + pub fd: Fd, + } + + impl Plucker { + pub fn init(filepath: &[OSPathChar], estimated_size: usize) -> crate::Result { + Ok(Plucker { + contents: MutableString::init(estimated_size)?, + filename_hash: hash(slice_as_bytes(filepath)), + fd: Fd::INVALID, + found: false, + }) + } + } + + #[derive(Clone, Copy)] + pub struct ExtractOptions { + pub depth_to_skip: usize, + pub close_handles: bool, + pub log: bool, + pub npm: bool, + pub skip_symlinks: bool, + } + + impl Default for ExtractOptions { + fn default() -> Self { + Self { + depth_to_skip: 0, + close_handles: true, + log: false, + npm: false, + skip_symlinks: false, + } + } + } +} + +pub use archiver::{Context, ExtractOptions, Plucker}; + +pub trait ArchiveAppender { + /// Mirrors `@hasDecl(Child, "onFirstDirectoryName")`. + const HAS_ON_FIRST_DIRECTORY_NAME: bool = false; + + fn needs_first_dirname(&self) -> bool { + false + } + fn on_first_directory_name(&mut self, _name: &[u8]) {} + + fn append(&mut self, path: &[u8]) -> crate::Result<&[u8]> { + let _ = path; + unreachable!() + } +} + +impl ArchiveAppender for () {} + +impl Archiver { + pub fn get_overwriting_file_list( + file_buffer: &[u8], + root: &[u8], + ctx: &mut Context, + appender: &mut A, + // Called with the path of every entry below `root`. If it returns true, the entry is listed + // whatever its size. + leaves_root: &mut dyn FnMut(&[u8]) -> bun_sys::Result, + ) -> crate::Result<()> { + // Uses the bun_sys directory-fd helpers (open_dir_absolute / open_dir_at). + let dir: Fd = 'brk: { + let cwd = Fd::cwd(); + + // if the destination doesn't exist, we skip the whole thing since nothing can overwrite it. + if bun_paths::is_absolute(root) { + let Ok(d) = bun_sys::open_dir_absolute(root) else { + return Ok(()); + }; + break 'brk d; + } else { + let Ok(d) = bun_sys::open_dir_at(cwd, root) else { + return Ok(()); + }; + break 'brk d; + } + }; + let _close_dir = bun_sys::CloseOnDrop::new(dir); + + let mut reader = open_memory(file_buffer, Dialect::General)?; + let mut normalized_buf = bun_paths::path_buffer_pool::get(); + + 'loop_: while let Some(entry) = reader.next_entry()? { + let pathname = display_path(entry)?; + let pathname_bytes: &[u8] = &pathname; + + // Tokenizer semantics: `next()` skips leading + // separators and returns None only when no token remains; + // the rest-of-input slice also skips leading separators. + // An entry's name has `/` on every platform, as in `extract_to_dir`. + let mut remaining = pathname_bytes; + let mut depth_i = 0usize; + while depth_i < DEPTH_TO_SKIP { + // skip leading separators + while let [first, rest @ ..] = remaining { + if *first == b'/' { + remaining = rest; + } else { + break; + } + } + if remaining.is_empty() { + continue 'loop_; + } + match strings::index_of_char_usize(remaining, b'/') { + Some(i) => remaining = &remaining[i..], + None => remaining = &remaining[remaining.len()..], + } + depth_i += 1; + } + // skip leading separators (tokenizer.rest() does this) + while let [first, rest @ ..] = remaining { + if *first == b'/' { + remaining = rest; + } else { + break; + } + } + + let pathname = remaining; + if pathname.is_empty() || pathname.len() >= normalized_buf.len() { + continue 'loop_; + } + let normalized = bun_paths::resolve_path::normalize_buf_t::< + u8, + bun_paths::platform::Auto, + >(pathname, &mut normalized_buf[..]); + let normalized_len = normalized.len(); + let pathname: &[u8] = &normalized_buf[..normalized_len]; + if pathname.is_empty() || pathname == b"." { + continue 'loop_; + } + #[cfg(windows)] + if bun_paths::is_absolute_windows(pathname) { + continue 'loop_; + } + let dirname = strings::trim(bun_paths::dirname_simple(pathname), SEP_STR.as_bytes()); + + let size: usize = usize::try_from(entry.size().max(0)).unwrap(); + let in_the_way = leaves_root(pathname)? + || (size > 0 + && match bun_sys::openat_a(dir, pathname, bun_sys::O::WRONLY, 0) { + Ok(opened) => { + let _close = bun_sys::CloseOnDrop::new(opened); + bun_sys::get_file_size(opened)? > 0 + } + Err(_) => false, + }); + + if in_the_way { + let is_already_top_level = dirname.is_empty(); + let path_to_use_: &[u8] = 'brk: { + let __pathname: &[u8] = pathname; + + if is_already_top_level { + break 'brk __pathname; + } + + let index = strings::index_of_char_usize(__pathname, SEP).unwrap(); + break 'brk &__pathname[..index]; + }; + let mut temp_buf = [0u8; 1024]; + temp_buf[..path_to_use_.len()].copy_from_slice(path_to_use_); + let path_to_use: &[u8] = if !is_already_top_level { + temp_buf[path_to_use_.len()] = SEP; + &temp_buf[..path_to_use_.len() + 1] + } else { + &temp_buf[..path_to_use_.len()] + }; + + let overwrite_entry = ctx.overwrite_list.get_or_put(path_to_use)?; + if !overwrite_entry.found_existing { + *overwrite_entry.key_ptr = Box::from(appender.append(path_to_use)?); + } + } + } + + Ok(()) + } + + pub fn extract_to_dir( + file_buffer: &[u8], + dir: Fd, + ctx: Option<&mut Context>, + appender: &mut A, + options: ExtractOptions, + ) -> crate::Result { + let dialect = if options.npm { + Dialect::Npm + } else { + Dialect::General + }; + let mut reader = open_memory(file_buffer, dialect)?; + let mut os_path_buf: Vec = Vec::new(); + let mut count: u32 = 0; + let dir_fd = dir; + + // reshaped for borrowck — ctx is Option<&mut>, rebound as needed + let mut ctx = ctx; + + #[cfg(unix)] + let mut symlink_join_buf: Option = None; + + #[cfg(unix)] + let mut deferred_symlinks: Vec = Vec::new(); + + let mut normalized_buf = bun_paths::os_path_buffer_pool::get(); + let mut write_strategy = WriteStrategy::default(); + + 'loop_: while let Some(entry) = reader.next_entry()? { + let pathname_z = os_path(entry, &mut os_path_buf)?; + + if A::HAS_ON_FIRST_DIRECTORY_NAME { + if appender.needs_first_dirname() { + let root = first_path_component(pathname_z); + #[cfg(windows)] + { + let result = strings::to_utf8_list_with_type(Vec::new(), root)?; + // onFirstDirectoryName copies the contents of pathname to another buffer, safe to free + appender.on_first_directory_name(&result); + } + #[cfg(not(windows))] + { + appender.on_first_directory_name(root); + } + } + } + + let kind = entry.kind(); + + if options.npm { + // - ignore entries other than files (`true` can only be returned if type is file) + // https://github.com/npm/cli/blob/93883bb6459208a916584cad8c6c72a315cf32af/node_modules/pacote/lib/fetcher.js#L419-L441 + if kind != bun_sys::FileKind::File { + continue; + } + + // TODO: .npmignore, or .gitignore if it doesn't exist + // https://github.com/npm/cli/blob/93883bb6459208a916584cad8c6c72a315cf32af/node_modules/pacote/lib/fetcher.js#L434 + } + + if options.skip_symlinks && kind == bun_sys::FileKind::SymLink { + continue; + } + + // strip and normalize the path + let pathname_slice: &[OSPathChar] = pathname_z; + let mut remaining: &[OSPathChar] = pathname_slice; + { + let sep: OSPathChar = b'/' as OSPathChar; + let mut i = 0usize; + while i < options.depth_to_skip { + // npm counts the empty name before a leading `/` as a component. + while let [first, rest @ ..] = remaining { + if *first == sep && !options.npm { + remaining = rest; + } else { + break; + } + } + if remaining.is_empty() { + continue 'loop_; + } + match strings::index_of_scalar(remaining, sep) { + Some(j) => remaining = &remaining[j + 1..], + None => remaining = &remaining[remaining.len()..], + } + i += 1; + } + while let [first, rest @ ..] = remaining { + if *first == sep { + remaining = rest; + } else { + break; + } + } + } + let pathname: &[OSPathChar] = remaining; + + if pathname.len() >= normalized_buf.len() { + if options.log { + bun_core::warn!( + "Skipping entry with a path longer than the maximum path length: {}\n", + bun_core::fmt::EscapeControlChars(bun_core::fmt::fmt_os_path( + pathname, + Default::default() + )), + ); + } + continue; + } + + let normalized = if options.npm { + normalize_npm_entry_path(pathname, &mut normalized_buf[..]) + } else { + bun_paths::resolve_path::normalize_buf_t::( + pathname, + &mut normalized_buf[..], + ) + }; + let normalized_len = normalized.len(); + normalized_buf[normalized_len] = 0; + // SAFETY: we just wrote a NUL at normalized_buf[normalized_len] + let path: &mut [OSPathChar] = &mut normalized_buf[..normalized_len]; + if path.is_empty() || (path.len() == 1 && path[0] == b'.' as OSPathChar) { + continue; + } + + // Skip entries whose normalized path is absolute on Windows. + // `openatWindows` ignores `dir_fd` for absolute inputs (drive + // letter or UNC), so without this guard a tar entry could + // resolve outside the extraction directory. On POSIX the + // tokenize-on-'/' step already strips any leading separators, + // so `normalizeBufT` cannot produce an absolute output. + #[cfg(windows)] + { + if bun_paths::is_absolute_windows_t::(path) { + continue 'loop_; + } + } + + #[cfg(windows)] + if options.npm { + // When writing files on Windows, translate the characters to their + // 0xf000 higher-encoded versions. + // https://github.com/isaacs/node-tar/blob/0510c9ea6d000c40446d56674a7efeec8e72f052/lib/winchars.js + let mut remain: &mut [OSPathChar] = path; + if strings::starts_with_windows_drive_letter_t::(remain) { + // don't encode `:` from the drive letter + // https://github.com/npm/cli/blob/93883bb6459208a916584cad8c6c72a315cf32af/node_modules/tar/lib/unpack.js#L327 + remain = &mut remain[2..]; + } + + for ch in remain.iter_mut() { + match *ch { + c if c == b'|' as OSPathChar + || c == b'<' as OSPathChar + || c == b'>' as OSPathChar + || c == b'?' as OSPathChar + || c == b':' as OSPathChar => + { + *ch += 0xf000; + } + _ => {} + } + } + } + + let path_slice: &[OSPathChar] = &path[..]; + + if options.log { + bun_core::prettyln!( + " {}", + bun_core::fmt::EscapeControlChars(bun_core::fmt::fmt_os_path( + path_slice, + Default::default() + )) + ); + } + + count += 1; + + match kind { + bun_sys::FileKind::Directory => { + #[cfg(windows)] + { + make_path_u16(dir, path_slice)?; + } + #[cfg(not(windows))] + { + let mode = directory_mode(entry.perm()); + // SAFETY: normalized_buf[path_slice.len()] == 0 (written above), + // so path_slice is a NUL-terminated [:0]u8. + let path_z: &ZStr = + unsafe { ZStr::from_raw(path_slice.as_ptr(), path_slice.len()) }; + match bun_sys::mkdirat_z(dir_fd, path_z, mode) { + Ok(()) => {} + Err(err) => { + // It's possible for some tarballs to return a directory twice, with and + // without `./` in the beginning. So if it already exists, continue to the + // next entry. + match err.get_errno() { + bun_sys::E::EEXIST | bun_sys::E::ENOTDIR => continue, + _ => {} + } + let dirname = bun_paths::dirname_simple(path_slice); + if dirname.is_empty() { + return Err(err.into()); + } + let _ = dir.make_path_u8(dirname); + let _ = bun_sys::mkdirat_z(dir_fd, path_z, 0o777); + } + } + } + } + bun_sys::FileKind::SymLink => { + let link_target = entry.symlink().unwrap_or_default(); + #[cfg(unix)] + { + // Validate that the symlink target doesn't escape the extraction directory. + // This prevents path traversal attacks where a malicious tarball creates a symlink + // pointing outside (e.g., to /tmp), then writes files through that symlink. + if !is_symlink_target_safe(path_slice, link_target, &mut symlink_join_buf) { + // Skip symlinks that would escape the extraction directory + if options.log { + bun_core::warn!( + "Skipping symlink with unsafe target: {} -> {}\n", + bun_core::fmt::EscapeControlChars(bun_core::fmt::fmt_os_path( + path_slice, + Default::default(), + )), + bun_core::fmt::escape_control_chars(link_target), + ); + } + continue; + } + deferred_symlinks.push(DeferredSymlink::new(path_slice, link_target)); + } + #[cfg(not(unix))] + { + let _ = link_target; + } + } + bun_sys::FileKind::File => { + // first https://github.com/npm/cli/blob/feb54f7e9a39bd52519221bae4fafc8bc70f235e/node_modules/pacote/lib/fetcher.js#L65-L66 + // this.fmode = opts.fmode || 0o666 + // + // then https://github.com/npm/cli/blob/feb54f7e9a39bd52519221bae4fafc8bc70f235e/node_modules/pacote/lib/fetcher.js#L402-L411 + // + // we simplify and turn it into `entry.mode || 0o666` because we aren't accepting a umask or fmask option. + #[cfg(not(windows))] + let mode: bun_sys::Mode = (entry.perm() & 0o777) | 0o666; + + let flags = bun_sys::O::WRONLY | bun_sys::O::CREAT | bun_sys::O::TRUNC; + + #[cfg(windows)] + let file_handle_native: Fd = + match bun_sys::openat_windows(dir_fd, path_slice, flags, 0) { + Ok(fd) => fd, + Err(e) => match e.get_errno() { + bun_sys::E::EPERM | bun_sys::E::ENOENT => { + // `Dirname::dirname` strips + // trailing separators. + let Some(dirname) = bun_paths::Dirname::dirname(path_slice) + else { + return Err(e.into()); + }; + let _ = make_path_u16(dir, dirname); + bun_sys::openat_windows(dir_fd, path_slice, flags, 0)? + } + _ => return Err(e.into()), + }, + }; + + #[cfg(not(windows))] + let file_handle_native: Fd = { + // dir.createFileZ(.{truncate, mode}) → bun_sys::openat + // SAFETY: normalized_buf[path_slice.len()] == 0 (written above). + let path_z: &ZStr = + unsafe { ZStr::from_raw(path_slice.as_ptr(), path_slice.len()) }; + match bun_sys::openat(dir_fd, path_z, flags, mode) { + Ok(fd) => fd, + Err(err) => match err.get_errno() { + bun_sys::E::EACCES | bun_sys::E::EPERM | bun_sys::E::ENOENT => { + let dirname = bun_paths::dirname_simple(path_slice); + if dirname.is_empty() { + return Err(err.into()); + } + let _ = dir.make_path_u8(dirname); + bun_sys::openat(dir_fd, path_z, flags, mode)? + } + _ => return Err(err.into()), + }, + } + }; + + let mut output = match file_handle_native.make_lib_uv_owned() { + Ok(fd) => OutputFile { + fd, + close: options.close_handles, + }, + Err(err) => { + file_handle_native.close(); + return Err(err.into()); + } + }; + + let size: usize = usize::try_from(entry.size().max(0)).unwrap(); + + if size > 0 { + if let Some(ctx_) = ctx.as_deref_mut() { + let h: u64 = if !ctx_.pluckers.is_empty() { + hash(slice_as_bytes(path_slice)) + } else { + 0u64 + }; + + for plucker_ in ctx_.pluckers.iter_mut() { + if plucker_.filename_hash == h { + plucker_.contents.list.clear(); + if let Err(err) = + reader.read_data(size, &mut plucker_.contents.list) + { + if options.log { + log_archive_error(path_slice, err); + } + return Err(err.into()); + } + plucker_.found = !plucker_.contents.list.is_empty(); + plucker_.fd = output.fd; + output.close = false; + continue 'loop_; + } + } + } + if let Err(err) = + read_data_into_fd(&mut reader, output.fd, &mut write_strategy) + { + if options.log { + log_archive_error(path_slice, err); + } + return Err(crate::Error::Fail); + } + } + } + _ => {} + } + } + + #[cfg(unix)] + create_deferred_symlinks(dir_fd, &deferred_symlinks, options.log); + + Ok(count) + } + + pub fn extract_to_disk( + file_buffer: &[u8], + root: &[u8], + ctx: Option<&mut Context>, + appender: &mut A, + options: ExtractOptions, + ) -> crate::Result { + let dir: Fd = 'brk: { + let cwd = Fd::cwd(); + let _ = cwd.make_path_u8(root); + + if bun_paths::is_absolute(root) { + break 'brk bun_sys::open_dir_absolute(root)?; + } else { + break 'brk bun_sys::open_dir_at(cwd, root)?; + } + }; + + let _close_guard = scopeguard::guard(dir, |d| { + if options.close_handles { + d.close(); + } + }); + + Self::extract_to_dir(file_buffer, dir, ctx, appender, options) + } +} diff --git a/src/bun_core/env_var.rs b/src/bun_core/env_var.rs index 78e86cefe1c9..0227e811986c 100644 --- a/src/bun_core/env_var.rs +++ b/src/bun_core/env_var.rs @@ -118,7 +118,7 @@ new!(pub BUN_INSTALL: string, "BUN_INSTALL", {}); new!(pub BUN_INSTALL_BIN: string, "BUN_INSTALL_BIN", {}); new!(pub BUN_INSTALL_GLOBAL_DIR: string, "BUN_INSTALL_GLOBAL_DIR", {}); // Minimum response `Content-Length` (in bytes) for `bun install` to -// stream a tarball directly into libarchive instead of buffering the +// stream a tarball directly into the extractor instead of buffering the // whole body first. Smaller tarballs stay on the buffered path where // the fixed overhead of the resumable state machine isn't worth it. new!(pub BUN_INSTALL_STREAMING_MIN_SIZE: unsigned, "BUN_INSTALL_STREAMING_MIN_SIZE", { default: 2 * 1024 * 1024 }); diff --git a/src/bun_core/lib.rs b/src/bun_core/lib.rs index b3e478c39c93..69eeeeb62598 100644 --- a/src/bun_core/lib.rs +++ b/src/bun_core/lib.rs @@ -701,7 +701,7 @@ pub const unsafe fn container_of(field: *const F, offset: usize) -> *mut P /// Recover a typed `&mut T` from a C-callback's opaque user-data pointer. /// /// This is the canonical spelling for the ubiquitous trampoline pattern where -/// a C library (libarchive, c-ares, uWS, libuv, lol-html, BoringSSL, …) round- +/// a C library (c-ares, uWS, libuv, lol-html, BoringSSL, …) round- /// trips a Rust object through a `void *user_data` slot and hands it back to /// an `extern "C" fn` thunk. Earlier ports open-coded this as /// `unsafe { &mut *ctx.cast::() }` at every site; centralising it here diff --git a/src/bun_core/util.rs b/src/bun_core/util.rs index 2ceeb1857e42..7d554a243440 100644 --- a/src/bun_core/util.rs +++ b/src/bun_core/util.rs @@ -195,7 +195,7 @@ impl ZStr { } /// Borrow a NUL-terminated FFI C string as `&ZStr`, or `EMPTY` if `p` is /// null. Single audited funnel for the `strlen`-then-`from_raw` shape that - /// previously appeared as ad-hoc local helpers in libarchive / uSockets / + /// previously appeared as ad-hoc local helpers in uSockets / /// HTTPCertError. Returns `&'a ZStr` so the *caller* picks the lifetime. /// /// # Safety @@ -467,22 +467,6 @@ impl WStr { // release) and `buf[..buf.len()-1]` is in-bounds by slice invariant. unsafe { Self::from_raw(buf.as_ptr(), buf.len() - 1) } } - /// Borrow a NUL-terminated FFI wide string as `&WStr`, or [`EMPTY`] if - /// `p` is null. UTF-16 mirror of [`ZStr::from_c_ptr`]; single audited - /// funnel for the `wcslen`-then-`from_raw` shape at libarchive `_w` - /// accessors and Windows path-API ingestion points. - /// - /// # Safety - /// If non-null, `p` must point to a NUL-terminated u16 sequence readable - /// for `'a`. Null is explicitly allowed (→ `WStr::EMPTY`). - #[inline] - pub unsafe fn from_ptr<'a>(p: *const u16) -> &'a WStr { - if p.is_null() { - return Self::EMPTY; - } - // SAFETY: non-null and NUL-terminated per caller contract. - unsafe { Self::from_raw(p, crate::ffi::wcslen(p)) } - } #[inline] pub const fn as_slice(&self) -> &[u16] { &self.0 @@ -1528,7 +1512,7 @@ pub mod fd { } // ─── FileKind / Mode / kind_from_mode (from bun_sys) ────────────────────── -// Pure S_IFMT arithmetic, no syscalls (libarchive_sys req). +// Pure S_IFMT arithmetic, no syscalls. pub type Mode = u32; // POSIX `mode_t` /// `stat` mode-flag constants and predicates. diff --git a/src/install/Cargo.toml b/src/install/Cargo.toml index a8a45c65c7e7..feaca157d586 100644 --- a/src/install/Cargo.toml +++ b/src/install/Cargo.toml @@ -46,7 +46,7 @@ bun_parsers.workspace = true bun_io.workspace = true bun_js_parser.workspace = true bun_js_printer.workspace = true -bun_libarchive.workspace = true +bun_archive.workspace = true bun_libdeflate_sys.workspace = true bun_ast.workspace = true bun_options_types.workspace = true diff --git a/src/install/NetworkTask.rs b/src/install/NetworkTask.rs index b7d2e66c39f6..6019ab60adb4 100644 --- a/src/install/NetworkTask.rs +++ b/src/install/NetworkTask.rs @@ -72,7 +72,7 @@ pub struct NetworkTask { pub(crate) next: bun_threading::Link, /// Producer/consumer buffer that feeds tarball bytes from the HTTP thread - /// to a worker running libarchive. `None` when streaming extraction is + /// to a worker that extracts them. `None` when streaming extraction is /// disabled or this task is not a tarball download. pub(crate) tarball_stream: Option>, /// Extract `Task` pre-created on the main thread so the HTTP thread can @@ -949,10 +949,10 @@ impl NetworkTask { // Tell the HTTP client to invoke `notify` for every body chunk // instead of buffering the whole response. `notify` pushes each // chunk into `tarball_stream`, which schedules a drain task on - // `thread_pool`; the drain task calls into libarchive until it - // reports ARCHIVE_RETRY (out of input), then returns so the + // `thread_pool`; the drain task runs the decoder until it is + // out of input, then returns so the // worker can be reused for other install work. The next chunk - // reschedules it and libarchive — whose state lives on the heap + // reschedules it and the decoder — whose state lives on the heap // — resumes exactly where it stopped. // // The stream itself is created by the caller (see diff --git a/src/install/TarballStream.rs b/src/install/TarballStream.rs index eb913f976b33..5468e32083c1 100644 --- a/src/install/TarballStream.rs +++ b/src/install/TarballStream.rs @@ -2,31 +2,26 @@ //! //! The HTTP thread hands each body chunk to `on_chunk`, which appends to a //! small pending buffer and (if not already running) schedules -//! `drain_task` on `PackageManager.thread_pool`. The drain task calls into -//! libarchive to gunzip and untar whatever is available, writing files as -//! their data arrives, until libarchive asks for more compressed bytes -//! than are currently buffered. At that point the read callback returns -//! `ARCHIVE_RETRY`, libarchive propagates it (see the BUN PATCHes in -//! `vendor/libarchive`), and the drain task returns — the worker is -//! released. The next HTTP chunk reschedules the drain task, which calls -//! back into libarchive and resumes exactly where it left off because the -//! `struct archive *`, the gzip inflate state, the partially-read tar -//! header and the open output `bun.FD` all live on the heap in this -//! struct. +//! `drain_task` on `PackageManager.thread_pool`. The drain task feeds +//! whatever is available to a `bun_tar::Decoder`, writing files as their +//! data arrives, until the decoder has consumed everything that is +//! buffered. At that point the drain task returns and the worker is +//! released. The next HTTP chunk reschedules the drain task, which resumes +//! exactly where it left off because the decoder and the open output +//! `bun.FD` live on the heap in this struct. //! //! This lets `bun install` overlap download and extraction on the normal //! resolve thread pool without ever parking a worker on a condvar, and //! without holding the full compressed or decompressed tarball in memory. -use core::ffi::{c_int, c_void}; use core::mem::ManuallyDrop; use core::sync::atomic::{AtomicBool, Ordering}; +use bun_archive::tar::{self, Event}; use bun_collections::VecExt; #[cfg(windows)] use bun_core::strings; use bun_core::{self, Output, ZBox, env_var, fmt as bun_fmt}; -use bun_libarchive::lib; use bun_paths::resolve_path::{self, platform}; use bun_paths::{self, OSPathChar, OSPathSliceZ}; #[cfg(not(windows))] @@ -48,17 +43,6 @@ type Task = crate::package_manager_task::Task<'static>; type OSPathZ<'a> = &'a OSPathSliceZ; type OSPathZMut<'a> = &'a mut OSPathSliceZ; -#[derive(Clone, Copy, PartialEq, Eq)] -enum Phase { - /// Call `archive_read_next_header` next. - WantHeader, - /// Currently writing the body of `out`; call - /// `archive_read_data_block` next. - WantData, - /// `archive_read_next_header` returned EOF; we are done. - Done, -} - // `extract_task` / `package_manager` are raw pointers, not // `&'a mut` / `&'a`. This struct is heap-allocated (`heap::alloc`), // crosses threads via `drain_task`, and self-destroys in `finish()`, so a @@ -72,15 +56,15 @@ pub struct TarballStream { mutex: Mutex, /// Compressed .tgz bytes that have arrived from the HTTP thread but have - /// not yet been consumed by libarchive. + /// not yet been moved to `reading`. pending: Vec, /// True once the HTTP thread has delivered the final chunk (or an error). closed: bool, - /// Set if the HTTP request failed mid-stream. Unless libarchive still + /// Set if the HTTP request failed mid-stream. Unless the decoder still /// reached end-of-archive and the digest verifies, this — not - /// libarchive's truncation error — is the failure, and `finish()` hands + /// the decoder's truncation error — is the failure, and `finish()` hands /// the NetworkTask back to be retried as a failed download. http_err: Option, @@ -97,27 +81,42 @@ pub struct TarballStream { // --------------------------------------------------------------------- // Drain-side state (touched only by one drain task at a time) // --------------------------------------------------------------------- - /// Bytes currently being consumed by libarchive. Populated by swapping - /// with `pending` under the mutex so the HTTP thread can keep appending - /// while libarchive decompresses without the lock held. libarchive's - /// read callback hands out `reading[read_pos..]` and advances - /// `read_pos`; the slice must remain valid until the next callback, so - /// we only recycle this buffer on the *following* swap. + /// Bytes currently being decoded. Populated by swapping with `pending` + /// under the mutex so the HTTP thread can keep appending while the + /// decoder works without the lock held. reading: Vec, read_pos: usize, - archive_holds_reading: bool, - archive: Option, + decoder: Option, + extraction: Extraction, + + /// Incremental SHA over the *compressed* bytes, matching + /// `Integrity.verify` / `Integrity.forBytes` in the buffered path. + hasher: integrity::Streaming, - /// Where we are in the per-entry state machine between drain - /// invocations. libarchive preserves everything else (filter buffers, - /// zlib stream, tar header progress) on its own heap. - phase: Phase, + bytes_received: usize, + /// `Content-Length` of the response, when the server sent one. + pub(crate) content_length: Option, + fail: Option, + + /// Thread-pool task that runs `drain`. Re-enqueued whenever new data + /// arrives and no drain is currently in flight. + drain_task: thread_pool::Task, + + network_task: *mut NetworkTask, + package_manager: *mut PackageManager, +} + +/// What the decoder's events are applied to. Nothing in here is touched by the HTTP thread, so the +/// drain task can hold `&mut` to it for as long as it likes. +struct Extraction { + /// The decoder reached the end of the archive. + done: bool, /// Output file for the entry currently being written. `None` while /// between entries or when the current entry is being skipped. - out: Option, - write_strategy: lib::WriteStrategy, + out: Option, + write_strategy: bun_archive::WriteStrategy, /// Temp directory files are written into before being renamed into the /// cache. Lazily opened on the first drain so the HTTP thread never @@ -127,44 +126,30 @@ pub struct TarballStream { // `ZBox` is the owned NUL-terminated counterpart of `&ZStr`. tmpname: ZBox, - /// Incremental SHA over the *compressed* bytes, matching - /// `Integrity.verify` / `Integrity.forBytes` in the buffered path. - hasher: integrity::Streaming, - /// Resolved first-directory name for GitHub tarballs (written to /// `.bun-tag` and used for the cache folder name). resolved_github_dirname: &'static [u8], want_first_dirname: bool, npm_mode: bool, - bytes_received: usize, - /// Compressed bytes handed to libarchive by the read callback so far. + /// Compressed bytes the decoder has consumed so far. bytes_consumed: usize, - /// `Content-Length` of the response, when the server sent one. - pub(crate) content_length: Option, entry_count: u32, - fail: Option, - /// libarchive's error string for `fail == Some(Fail)`. + /// The decoder's message for `fail == Some(Fail)`. fail_detail: Vec, invalid_name: bool, - /// Thread-pool task that runs `drain`. Re-enqueued whenever new data - /// arrives and no drain is currently in flight. - drain_task: thread_pool::Task, - /// Completion task that carries the final result back to the main /// thread. Populated by `finish()` and pushed onto `resolve_tasks` there. /// BACKREF — `*mut Task` constructed via `ParentRef::from_raw_mut` so the /// read-only `request_extract()` accessor in `open_destination` goes /// through safe `Deref`; `finish()` recovers the raw via `as_mut_ptr()`. extract_task: bun_ptr::ParentRef, - network_task: *mut NetworkTask, - package_manager: *mut PackageManager, } impl TarballStream { /// Minimum Content-Length for which the streaming path is used. Below - /// this the whole body is buffered as before; the resumable libarchive + /// this the whole body is buffered as before; the resumable /// state machine is only worth its per-chunk overhead for tarballs that /// would otherwise consume a noticeable amount of memory. pub(crate) fn min_size() -> usize { @@ -250,36 +235,37 @@ impl TarballStream { draining: AtomicBool::new(false), reading: Vec::new(), read_pos: 0, - archive_holds_reading: false, - archive: None, - phase: Phase::WantHeader, - out: None, - write_strategy: lib::WriteStrategy::default(), - dest: None, - tmpname: ZBox::from_bytes(b""), + decoder: None, + extraction: Extraction { + done: false, + out: None, + write_strategy: bun_archive::WriteStrategy::default(), + dest: None, + tmpname: ZBox::from_bytes(b""), + resolved_github_dirname, + want_first_dirname, + npm_mode, + bytes_consumed: 0, + entry_count: 0, + fail_detail: Vec::new(), + invalid_name: false, + extract_task, + }, hasher, - resolved_github_dirname, - want_first_dirname, - npm_mode, bytes_received: 0, - bytes_consumed: 0, content_length: None, - entry_count: 0, fail: None, - fail_detail: Vec::new(), - invalid_name: false, drain_task: thread_pool::Task { node: thread_pool::Node::default(), callback: drain_callback, }, - extract_task, network_task, package_manager: manager, })) } /// Called from the HTTP thread for each response-body chunk. Returns - /// without touching the filesystem or libarchive; actual processing is + /// without touching the filesystem or the decoder; actual processing is /// deferred to `drain` on a worker so the HTTP event loop stays /// responsive. /// @@ -347,9 +333,9 @@ impl TarballStream { } } - /// Pull whatever compressed bytes are available into libarchive, writing - /// entries to disk, until libarchive reports `ARCHIVE_RETRY` (out of - /// input — yield) or a terminal state (EOF / error — finish). + /// Feed whatever compressed bytes are available to the decoder, writing + /// entries to disk, until it is out of input (yield) or in a terminal + /// state (EOF / error — finish). /// /// # Safety /// `this` must be the live pointer returned by `init()`. Runs on a @@ -366,8 +352,8 @@ impl TarballStream { // free `*this`; each `return` after it touches nothing. unsafe { loop { - if (*this).fail.is_none() && (*this).phase != Phase::Done { - // Only pull bytes into `reading` while libarchive is still + if (*this).fail.is_none() && !(*this).extraction.done { + // Only pull bytes into `reading` while the decoder is still // going to consume them. After EOF/failure `step()` is // never called again, so appending here would let // `reading` grow by one HTTP chunk per wakeup for the @@ -376,7 +362,7 @@ impl TarballStream { if let Err(err) = Self::step(this) { // If the body was cut short by a transport error, - // that is the failure; libarchive's complaint about + // that is the failure; the decoder's complaint about // the truncated input is just its symptom. (*this).mutex.lock(); let http_err = (*this).http_err; @@ -385,14 +371,14 @@ impl TarballStream { (crate::Error::Fail, Some(http_err)) => http_err, (err, _) => err, }); - (*this).close_output_file(); + (*this).extraction.close_output_file(); } - if (*this).fail.is_none() && (*this).phase != Phase::Done { + if (*this).fail.is_none() && !(*this).extraction.done { if more { continue; } - // libarchive consumed everything we had. Yield the + // The decoder consumed everything we had. Yield the // worker until the HTTP thread delivers the next // chunk. (*this).mutex.lock(); @@ -408,23 +394,22 @@ impl TarballStream { } } - // Terminal: archive finished or extraction failed. libarchive + // Terminal: archive finished or extraction failed. The decoder // will not be called again, so `reading` is dead — drop it // now rather than carrying its capacity until `finish()`. - // `reading` is drain-local (only the read callback touches - // it, and that runs inside `step()`), so this needs no lock. + // `reading` is drain-local, so this needs no lock. (*this).reading = Vec::new(); (*this).read_pos = 0; (*this).mutex.lock(); - // Hash any bytes that arrived after libarchive hit + // Hash any bytes that arrived after the decoder hit // end-of-archive so the integrity digest covers the full // response (tar zero-padding, gzip footer). Skip this once // an error is recorded — the digest won't be checked anyway. if (*this).fail.is_none() && !(*this).pending.is_empty() { (*this).hasher.update(&(*this).pending); } - // After EOF/failure we stop feeding libarchive but must keep + // After EOF/failure we stop feeding the decoder but must keep // consuming (and discarding) chunks until the HTTP thread // closes the stream; freeing ourselves earlier would let the // next `notify` dereference a dead pointer. @@ -454,14 +439,12 @@ impl TarballStream { } // unsafe } - /// Move any bytes still sitting in `pending` into `reading` so the read - /// callback can hand them to libarchive. Returns true if new bytes were - /// added or the stream is now closed. + /// Move any bytes still sitting in `pending` into `reading` for `step()`. + /// Returns true if new bytes were added or the stream is now closed. /// /// # Safety - /// `this` must be live. Called both from `drain()` and re-entrantly - /// from inside libarchive's read callback (while `step()` is on the - /// stack), so this must NOT materialise `&mut TarballStream` — all + /// `this` must be live. The HTTP thread may be inside `on_chunk()`, so + /// this must NOT materialise `&mut TarballStream` — all /// access is via raw-ptr field projection. Producer fields (`pending`/`closed`) /// are synchronised by `mutex`; drain-side fields (`reading`/ /// `read_pos`/`hasher`) are owned by the single active drain task. @@ -476,7 +459,7 @@ impl TarballStream { return closed; } - // Hash before libarchive sees the bytes so integrity covers exactly + // Hash before the decoder sees the bytes so integrity covers exactly // what came off the socket. (*this).hasher.update(&(*this).pending); @@ -487,13 +470,7 @@ impl TarballStream { core::mem::swap(&mut (*this).reading, &mut (*this).pending); (*this).read_pos = 0; } else { - // libarchive still holds a slice into `reading` (the read - // callback contract keeps the last-returned buffer valid until - // the next call). Appending would realloc and invalidate that - // slice, so instead shift the unconsumed tail down and append - // in place — the callback is not running concurrently with us - // (single drain at a time) and will be re-primed with the new - // base on its next invocation. + // Keep the unconsumed tail in front of the new bytes. let read_pos = (*this).read_pos; (*this).reading.drain_front(read_pos); (*this).read_pos = 0; @@ -505,186 +482,94 @@ impl TarballStream { } // unsafe } - /// Run libarchive until it needs more input (`Retry`) or hits a - /// terminal state. All libarchive state persists on the heap, so - /// returning from here and re-entering later is safe. + /// Run the decoder until it needs more input or hits a terminal state. /// /// # Safety - /// `this` must be live. Takes `*mut Self` (not `&mut self`) because - /// `open_archive()` hands `this` to libarchive as client_data and the - /// read callback dereferences it across MULTIPLE `step()` invocations. - /// A `&mut self` receiver would mint a fresh Unique tag on each call, - /// popping the SharedRW tag the stored client_data pointer carries and - /// leaving the callback with dead provenance (Stacked Borrows UB). - /// Threading the Box-rooted `*mut Self` from `drain()` keeps one - /// provenance alive for the lifetime of the archive. + /// `this` must be live, and the caller the one active drain task. The + /// HTTP thread may be inside `on_chunk()`, so the producer fields are + /// only read under `mutex`. unsafe fn step(this: *mut Self) -> crate::Result<()> { - // SAFETY: see fn-level # Safety — raw-ptr field projection only; no - // `&mut TarballStream` is held across any libarchive call (which may - // re-enter `archive_read_callback` and access `*this` via the same - // provenance). Transient `&mut *this` for `open_destination` / - // `begin_entry` / `write_block` / `finish_output_file` is sound: - // those do not call into libarchive. + // SAFETY: see fn-level # Safety. Everything but `mutex`, `pending` + // and `closed` is drain-local. unsafe { - if (*this).archive.is_none() { + (*this).mutex.lock(); + // Nothing follows `reading` once the body has ended and all of it was taken. + let eof = (*this).closed && (*this).pending.is_empty(); + (*this).mutex.unlock(); + + if (*this).decoder.is_none() { let head: &[u8] = &(*this).reading; - let gzip = head.starts_with(&[0x1f, 0x8b]); - if head.len() < 2 { - // Telling gzip from tar takes two bytes. Wait for more unless the body ended. - (*this).mutex.lock(); - let closed = (*this).closed; - (*this).mutex.unlock(); - if !closed { - return Ok(()); - } + if head.len() < 2 && !eof { + // Telling gzip from tar takes two bytes. + return Ok(()); } - Self::open_archive(this, gzip, (*this).npm_mode)?; + // Plain tar is what a .tgz sent with `Content-Encoding: gzip` arrives as, + // because the HTTP client has already decompressed it. + let gzip = head.starts_with(&[0x1f, 0x8b]); + let dialect = if (*this).extraction.npm_mode { + tar::Dialect::Npm + } else { + tar::Dialect::General + }; + (*this).decoder = Some( + tar::Decoder::new(tar::Options::for_host(dialect), gzip) + .map_err(|_| crate::Error::Alloc(bun_alloc::AllocError))?, + ); } - if (*this).dest.is_none() { - (*this).open_destination()?; + // References to single drain-local fields, never to the whole struct. + let extraction = &mut (*this).extraction; + if extraction.dest.is_none() { + extraction.open_destination()?; } + let Some(decoder) = (*this).decoder.as_mut() else { + return Ok(()); + }; + extraction.run(decoder, &(*this).reading, &mut (*this).read_pos, eof) + } + } +} - // `archive` points to a libarchive heap allocation disjoint from - // `*this`; holding `&lib::Archive` across the loop does not - // alias any access to `*this`. - let archive: &lib::Archive = (*this).archive.as_deref().unwrap(); - - loop { - match (*this).phase { - Phase::Done => return Ok(()), - Phase::WantHeader => { - let mut entry: *mut lib::Entry = core::ptr::null_mut(); - match archive.read_next_header(&mut entry) { - lib::Result::Retry if (*this).archive_holds_reading => continue, - lib::Result::Retry => return Ok(()), - lib::Result::Eof => { - // A body that is not an archive also ends here, because the tar - // reader skips every block that is not a valid header. - if (*this).entry_count == 0 { - (*this).fail_detail = b"archive is empty".to_vec(); - return Err(crate::Error::Fail); - } - (*this).phase = Phase::Done; - return Ok(()); - } - lib::Result::Ok | lib::Result::Warn => { - // libarchive returned OK/WARN with a valid entry - // pointer owned by `archive`; it stays valid until - // the next `read_next_header`. No other Rust - // reference to it exists. - (*this).begin_entry(&mut *entry)?; - } - lib::Result::Failed | lib::Result::Fatal => { - (*this).fail_detail = archive.error_string().to_vec(); - return Err(crate::Error::Fail); - } - } +impl Extraction { + fn run( + &mut self, + decoder: &mut tar::Decoder, + reading: &[u8], + read_pos: &mut usize, + eof: bool, + ) -> crate::Result<()> { + loop { + let (used, event) = match decoder.next(&reading[*read_pos..], eof) { + Ok(next) => next, + Err(tar::Error::Alloc(err)) => return Err(err.into()), + Err(err) => { + self.fail_detail = err.to_string().into_bytes(); + return Err(crate::Error::Fail); + } + }; + *read_pos += used; + self.bytes_consumed += used; + match event { + Event::NeedInput => return Ok(()), + Event::Entry(entry) => { + self.begin_entry(entry)?; + if self.out.is_none() { + decoder.skip_entry(); } - Phase::WantData => { - let mut offset: i64 = 0; - let Some(block) = archive.next(&mut offset) else { - // This is the end of the entry's data, and `offset` is its length. - (*this).finish_output_file(offset)?; - (*this).phase = Phase::WantHeader; - continue; - }; - match block.result { - lib::Result::Retry if !(*this).archive_holds_reading => return Ok(()), - lib::Result::Ok | lib::Result::Warn => { - (*this).write_block(&block)?; - } - _ => { - (*this).fail_detail = archive.error_string().to_vec(); - return Err(crate::Error::Fail); - } - } + } + Event::Data { offset, bytes } => self.write_block(offset, bytes)?, + Event::EndOfEntry { size } => self.finish_output_file(size)?, + Event::EndOfArchive => { + // A body that is not an archive also ends here, because the tar + // reader skips every block that is not a valid header. + if self.entry_count == 0 { + self.fail_detail = b"archive is empty".to_vec(); + return Err(crate::Error::Fail); } + self.done = true; + return Ok(()); } } - } // unsafe - } - - /// # Safety - /// `this` must be live and rooted at the Box allocation (i.e. the - /// pointer threaded from `drain_callback` → `drain` → `step`, NOT a - /// `&mut self as *mut Self` reborrow). libarchive stores `this` as - /// client_data and `archive_read_callback` dereferences it across the - /// lifetime of the archive — see `step()` # Safety for the provenance - /// requirement. - unsafe fn open_archive(this: *mut Self, gzip: bool, npm: bool) -> crate::Result<()> { - let archive = lib::ReadArchive::new(); - // Bypass bidding entirely. `gzip` is false for a .tgz sent with `Content-Encoding: gzip`, - // because the HTTP client has already decompressed it to a plain tar. - // With patches/libarchive/select-registered-only.patch, - // archive_read_append_filter / archive_read_set_format only select a - // filter/format that was already registered, so register gzip and tar - // first. Tar must also be registered before read_set_options (so the - // option has a slot) and set_format must come after it: libarchive's - // archive_set_format_option() clobbers `a->format` while dispatching, - // and losing the selected format means archive_read_open1() falls - // back to bidding, which fails when the first HTTP chunk is short. - // ARCHIVE_FILTER_GZIP = 1, ARCHIVE_FORMAT_TAR = 0x30000. - if gzip { - let _ = archive.read_support_filter_gzip(); - // SAFETY: archive is a valid non-null handle from read_new(); FFI call has no other preconditions. - if unsafe { lib::archive_read_append_filter(archive.as_mut_ptr(), 1) } != 0 { - return Err(crate::Error::Fail); - } - } - let _ = archive.read_support_format_tar(); - // patches/libarchive/tar-npm-compat.patch adds the `npm` option. - let _ = archive.read_set_options(if npm { - c"npm" - } else { - c"read_concatenated_archives" - }); - // SAFETY: archive is a valid non-null handle from read_new(); FFI call has no other preconditions. - if unsafe { lib::archive_read_set_format(archive.as_mut_ptr(), 0x30000) } != 0 { - return Err(crate::Error::Fail); - } - - // SAFETY: archive is a valid handle; `this` outlives the archive - // (dropped below on failure, otherwise as a field of `*this`). See - // fn-level # Safety for why client_data must be the Box-rooted - // `this` and not a `&mut self`-derived pointer. - let rc_raw: c_int = unsafe { - lib::archive_read_open( - archive.as_mut_ptr(), - this.cast::(), - None, - Some(archive_read_callback), - None, - ) - }; - // PORTING.md §Forbidden: `transmute::` is UB for any value not - // declared as a discriminant. Map known ARCHIVE_* codes explicitly and treat - // anything else as Fatal. - let rc: lib::Result = match rc_raw { - x if x == lib::Result::Ok as c_int => lib::Result::Ok, - x if x == lib::Result::Eof as c_int => lib::Result::Eof, - x if x == lib::Result::Retry as c_int => lib::Result::Retry, - x if x == lib::Result::Warn as c_int => lib::Result::Warn, - x if x == lib::Result::Failed as c_int => lib::Result::Failed, - _ => lib::Result::Fatal, - }; - match rc { - lib::Result::Ok | lib::Result::Warn => {} - lib::Result::Retry => { - // open() runs the filter bidder which we bypassed, but the - // client open path may still probe; treat as transient. - // SAFETY: see fn-level # Safety — raw-ptr field write. - unsafe { (*this).archive = Some(archive) }; - return Ok(()); - } - _ => { - // SAFETY: see fn-level # Safety — raw-ptr field write. - unsafe { (*this).fail_detail = archive.error_string().to_vec() }; - return Err(crate::Error::Fail); - } } - // SAFETY: see fn-level # Safety — raw-ptr field write. - unsafe { (*this).archive = Some(archive) }; - Ok(()) } fn open_destination(&mut self) -> crate::Result<()> { @@ -738,27 +623,25 @@ impl TarballStream { } } - fn write_block(&mut self, block: &lib::Block<'_>) -> crate::Result<()> { + fn write_block(&mut self, offset: i64, bytes: &[u8]) -> crate::Result<()> { let Some(out) = self.out.as_mut() else { return Ok(()); }; - out.write(&mut self.write_strategy, block.offset, block.bytes) + out.write(&mut self.write_strategy, offset, bytes) .map_err(|e| e.to_zig_err().into()) } - /// Process one entry header returned by `read_next_header`. Opens the - /// output file (or creates the directory) and transitions to - /// `WantData` so the next `step()` iteration starts pulling its body. - fn begin_entry(&mut self, entry: &mut lib::Entry) -> crate::Result<()> { - #[cfg(windows)] - let pathname: OSPathZ = entry.pathname_w(); - #[cfg(not(windows))] - let pathname: OSPathZ = entry.pathname(); + /// Process one entry header. Opens the output file (or creates the + /// directory). With `out` left empty, the entry's data is skipped. + fn begin_entry(&mut self, entry: &tar::Entry) -> crate::Result<()> { + self.out = None; + let mut os_path_buf: Vec = Vec::new(); + let pathname: &[OSPathChar] = bun_archive::os_path(entry, &mut os_path_buf)?; if self.want_first_dirname { self.want_first_dirname = false; // GitHub's first entry is the folder `repo-sha/`. A mirror's may be `repo-sha/file`. - let root = bun_libarchive::first_path_component(&pathname[..]); + let root = bun_archive::first_path_component(pathname); #[cfg(windows)] { let result = strings::to_utf8_list_with_type(Vec::new(), root)?; @@ -777,13 +660,11 @@ impl TarballStream { } } - let kind = bun_sys::kind_from_mode(entry.filetype() as Mode); + let kind = entry.kind(); if self.npm_mode && kind != FileKind::File { - // npm tarballs only contain files; matching the libarchive path - // in Archiver.extractToDir we skip everything else. - self.phase = Phase::WantData; - self.out = None; + // npm tarballs only contain files; like Archiver.extractToDir + // we skip everything else. return Ok(()); } @@ -794,8 +675,6 @@ impl TarballStream { .split(|c| *c == ('/' as OSPathChar)) .filter(|s| !s.is_empty()); if tokenizer.next().is_none() { - self.phase = Phase::WantData; - self.out = None; return Ok(()); } // tokenizeScalar.rest() — need byte offset of remainder, not just @@ -804,7 +683,7 @@ impl TarballStream { // straight to `normalize_buf_t` (which takes `&[OSPathChar]`, not a // NUL-terminated slice) so there is no need to reconstruct an // `OSPathSliceZ` suffix view here. - let rest: &[OSPathChar] = tokenize_rest_after_first(&pathname[..], self.npm_mode); + let rest: &[OSPathChar] = tokenize_rest_after_first(pathname, self.npm_mode); let mut norm_buf = bun_paths::os_path_buffer_pool::get(); if rest.len() >= norm_buf.len() { @@ -812,12 +691,10 @@ impl TarballStream { "Skipping entry with a path longer than the maximum path length: {}\n", bun_fmt::EscapeControlChars(bun_fmt::fmt_os_path(rest, Default::default())), ); - self.phase = Phase::WantData; - self.out = None; return Ok(()); } let normalized = if self.npm_mode { - bun_libarchive::normalize_npm_entry_path(rest, &mut norm_buf[..]) + bun_archive::normalize_npm_entry_path(rest, &mut norm_buf[..]) } else { resolve_path::normalize_buf_t::(rest, &mut norm_buf[..]) }; @@ -827,8 +704,6 @@ impl TarballStream { let path: OSPathZMut = unsafe { OSPathSliceZ::from_raw_mut(norm_buf.as_mut_ptr(), norm_len) }; if path.is_empty() || (path.len() == 1 && path[0] == ('.' as OSPathChar)) { - self.phase = Phase::WantData; - self.out = None; return Ok(()); } // `normalize_buf_t` collapses interior `..` but leaves a leading `..` @@ -841,15 +716,11 @@ impl TarballStream { && path[1] == ('.' as OSPathChar) && (path.len() == 2 || path[2] == bun_paths::SEP as OSPathChar) { - self.phase = Phase::WantData; - self.out = None; return Ok(()); } #[cfg(windows)] { if bun_paths::is_absolute_windows_wtf16(&path[..]) { - self.phase = Phase::WantData; - self.out = None; return Ok(()); } if self.npm_mode { @@ -866,8 +737,6 @@ impl TarballStream { match kind { FileKind::Directory => { make_directory(entry, dest, path, path_slice); - self.phase = Phase::WantData; - self.out = None; } FileKind::File => { #[cfg(windows)] @@ -878,17 +747,15 @@ impl TarballStream { let mode: Mode = Mode::try_from((entry.perm() & 0o777) | 0o666).expect("int cast"); let fd = open_output_file(dest, path, path_slice, mode)?; self.entry_count += 1; - self.out = Some(lib::EntryWriter::new(fd)); - self.phase = Phase::WantData; - } - _ => { - self.phase = Phase::WantData; - self.out = None; + self.out = Some(bun_archive::EntryWriter::new(fd)); } + _ => {} } Ok(()) } +} +impl TarballStream { /// # Safety /// `this` must be the live pointer returned by `init()`. Frees `*this` /// — caller must not touch it after return. Takes a raw pointer (not @@ -906,11 +773,11 @@ impl TarballStream { // Fields are already raw pointers (see the struct-level note), so copying // them out before `heap::take(this)` is just a pointer copy — no // reborrow of `&mut Task` is ever materialised from a stored `&mut`. - let task: *mut Task = (*this).extract_task.as_mut_ptr(); + let task: *mut Task = (*this).extraction.extract_task.as_mut_ptr(); let network: *mut NetworkTask = (*this).network_task; let manager: *mut PackageManager = (*this).package_manager; - (*this).close_output_file(); + (*this).extraction.close_output_file(); // The HTTP thread has delivered the final `has_more=false` chunk // (that's the only way `closed` gets set) and `notify()` does not @@ -927,21 +794,21 @@ impl TarballStream { let http_err = (*this).http_err; // SAFETY: `task` is live until pushed onto `resolve_tasks` below. - // `(*this).extract_task` is a raw `*mut Task` (not `&mut`), so this + // `(*this).extraction.extract_task` is a raw `*mut Task` (not `&mut`), so this // is the only writer — no aliasing with a stored reference. - // `populate_result` does not touch `(*this).extract_task`. + // `populate_result` does not touch `(*this).extraction.extract_task`. (*this).populate_result(task, http_err); // Temp-dir cleanup must happen before we release the stream or - // publish the task: both `(*this).tmpname` and + // publish the task: both `(*this).extraction.tmpname` and // `task.request.extract.tarball.temp_dir` become invalid once // `Drop` runs / the main thread recycles the Task. // SAFETY: task is live (see above). - if (*task).status != TaskStatus::Success && !(*this).tmpname.is_empty() { + if (*task).status != TaskStatus::Success && !(*this).extraction.tmpname.is_empty() { // `populate_result` closes `dest` on the success path before the // rename; the early-return failure paths leave it open, so close // it here first — Windows can't remove an open directory. - if let Some(d) = (*this).dest.take() { + if let Some(d) = (*this).extraction.dest.take() { d.close(); } // SAFETY: task is live (see above). `request` is an untagged @@ -949,7 +816,7 @@ impl TarballStream { // implicit autoref through the raw-ptr deref) for the // `ManuallyDrop` → `ExtractRequest` deref. let _ = Dir::borrow(&(&(*task).request.extract).tarball.temp_dir) - .delete_tree((*this).tmpname.as_bytes()); + .delete_tree((*this).extraction.tmpname.as_bytes()); } if let Some(crate::Error::Http(err)) = (*task).err @@ -1024,7 +891,7 @@ impl TarballStream { if let Some(err) = self.fail { if matches!(err, crate::Error::Http(_)) { // Reported (or retried) by `run_tasks`; see `finish()`. - } else if self.invalid_name { + } else if self.extraction.invalid_name { (*task).log.add_error_fmt( None, bun_ast::Loc::EMPTY, @@ -1041,13 +908,15 @@ impl TarballStream { "{} extracting tarball for \"{}\"{}{} (at byte {} of {})", err.name(), bun_fmt::for_terminal(bstr::BStr::new(tarball.name.slice())), - if self.fail_detail.is_empty() { + if self.extraction.fail_detail.is_empty() { "" } else { ": " }, - bun_fmt::EscapeControlChars(bstr::BStr::new(&self.fail_detail)), - self.bytes_consumed, + bun_fmt::EscapeControlChars(bstr::BStr::new( + &self.extraction.fail_detail + )), + self.extraction.bytes_consumed, self.content_length .as_ref() .map_or(&"unknown" as &dyn core::fmt::Display, |n| n), @@ -1062,7 +931,7 @@ impl TarballStream { if !tarball.skip_verify && tarball.integrity.tag.is_supported() { if !self.hasher.verify() { if let Some(http_err) = http_err { - // libarchive found the end-of-archive marker but the + // The decoder found the end-of-archive marker but the // body still ended early (gzip trailer, tar padding): // the same failed download as above. (*task).err = Some(http_err); @@ -1085,11 +954,11 @@ impl TarballStream { if tarball.resolution.tag == ResolutionTag::Github { 'insert_tag: { - if self.resolved_github_dirname.is_empty() { + if self.extraction.resolved_github_dirname.is_empty() { break 'insert_tag; } if bun_sys::File::openat( - self.dest.unwrap(), + self.extraction.dest.unwrap(), bun_core::zstr!(".bun-tag"), O::WRONLY | O::CREAT @@ -1097,16 +966,19 @@ impl TarballStream { | if cfg!(windows) { 0 } else { O::NOFOLLOW }, 0o664, ) - .and_then(|f| f.write_all(self.resolved_github_dirname)) + .and_then(|f| f.write_all(self.extraction.resolved_github_dirname)) .is_err() { - let _ = bun_sys::unlinkat(self.dest.unwrap(), bun_core::zstr!(".bun-tag")); + let _ = bun_sys::unlinkat( + self.extraction.dest.unwrap(), + bun_core::zstr!(".bun-tag"), + ); } } } // Close the temp dir handle before renaming so Windows can move it. - if let Some(d) = self.dest.take() { + if let Some(d) = self.extraction.dest.take() { d.close(); } @@ -1115,10 +987,10 @@ impl TarballStream { let integrity = tarball.lockfile_integrity(|| self.hasher.final_()); let mut result = match tarball.move_to_cache_directory( &mut (*task).log, - StagedEntry::new(tarball.temp_dir, self.tmpname.as_zstr()), + StagedEntry::new(tarball.temp_dir, self.extraction.tmpname.as_zstr()), name, basename, - self.resolved_github_dirname, + self.extraction.resolved_github_dirname, &integrity, ) { Ok(r) => r, @@ -1135,7 +1007,7 @@ impl TarballStream { "[{}] Streamed {} tarball → {} entries", bun_fmt::for_terminal(bstr::BStr::new(name)), bun_fmt::size(self.bytes_received, Default::default()), - self.entry_count, + self.extraction.entry_count, ); Output::flush(); } @@ -1163,10 +1035,10 @@ impl TarballStream { impl Drop for TarballStream { fn drop(&mut self) { - if let Some(out) = self.out.take() { + if let Some(out) = self.extraction.out.take() { out.fd().close(); } - if let Some(d) = self.dest { + if let Some(d) = self.extraction.dest { d.close(); } } @@ -1189,98 +1061,6 @@ fn drain_callback(task: *mut thread_pool::Task) { unsafe { TarballStream::drain(this) }; } -/// libarchive client read callback. Returns whatever compressed bytes -/// are currently buffered in `reading`; if none, returns `ARCHIVE_RETRY` -/// (when more data is still expected) so libarchive unwinds with a -/// resumable status, or `0` (EOF) once the HTTP response is complete. -/// -/// Safe-fn: only ever invoked by libarchive itself (never from Rust), with -/// `ctx`/`out_buffer` it threaded through from `open_archive()`. Every -/// pointer dereference inside carries its own SAFETY justification grounded -/// in that setup, so there is no caller-side precondition to encode in the -/// signature. The fn-pointer still coerces to the binding's expected type. -extern "C" fn archive_read_callback( - _a: *mut lib::Archive, - ctx: *mut c_void, - out_buffer: *mut *const c_void, -) -> lib::la_ssize_t { - // SAFETY: `ctx` is the Box-rooted `*mut TarballStream` threaded from - // `drain()` → `step()` → `open_archive()`; libarchive passes it back - // unchanged. `step()`/`open_archive()` take `*mut Self` (not - // `&mut self`) precisely so this pointer's provenance survives every - // re-entry — see `step()` # Safety. We keep `this` as a raw pointer and - // access fields through it directly; no `&mut TarballStream` is live anywhere on the - // call stack while libarchive runs. All fields touched here (`reading`, - // `read_pos`, `mutex`, `pending`, `closed`, `hasher`) are drain-side / - // mutex-guarded and are not accessed by `step()` across the FFI call - // boundary. - let this: *mut TarballStream = ctx.cast::(); - - // SAFETY: `this` is valid (see above); `reading`/`read_pos` are owned by - // the single active drain task. - unsafe { - // Explicit `&` on the `Vec` field (no implicit autoref through the - // raw-ptr deref) for `Index::index`. - let remaining = &(&(*this).reading)[(*this).read_pos..]; - if !remaining.is_empty() { - *out_buffer = remaining.as_ptr().cast(); - (*this).read_pos = (*this).reading.len(); - (*this).bytes_consumed += remaining.len(); - (*this).archive_holds_reading = true; - return lib::la_ssize_t::try_from(remaining.len()).expect("int cast"); - } - (*this).archive_holds_reading = false; - } - - // No data left in `reading`. Check for more under the lock — - // libarchive may have called us more than once for a single - // `step()` (e.g. gzip header + first deflate block), and `on_chunk` - // might have landed a fresh chunk in the meantime. - // SAFETY: `mutex`/`pending`/`closed` accessed via raw ptr; producer side - // is synchronised by the mutex itself. - let (has_pending, closed) = unsafe { - (*this).mutex.lock(); - let r = (!(*this).pending.is_empty(), (*this).closed); - (*this).mutex.unlock(); - r - }; - - if has_pending { - // Pull the new bytes into `reading` and retry the read. We are - // the only consumer of `reading`/`read_pos`, and `take_pending` - // only touches producer state under the same mutex. - // SAFETY: `take_pending` takes `*mut Self` and accesses fields via - // raw-ptr projection, never forming `&mut TarballStream`; `step()` - // holds no `&mut TarballStream` across the libarchive call that - // re-entered us. - unsafe { - let _ = TarballStream::take_pending(this); - // Explicit `&` on the `Vec` field (no implicit autoref through - // the raw-ptr deref) for `Index::index`. - let again = &(&(*this).reading)[(*this).read_pos..]; - if !again.is_empty() { - *out_buffer = again.as_ptr().cast(); - (*this).read_pos = (*this).reading.len(); - (*this).bytes_consumed += again.len(); - (*this).archive_holds_reading = true; - return lib::la_ssize_t::try_from(again.len()).expect("int cast"); - } - } - } - - if closed { - // SAFETY: out_buffer is a valid out-param; ptr is unused when len==0. - unsafe { *out_buffer = this.cast() }; - return 0; - } - - // Tell libarchive to unwind with a resumable status. The BUN PATCHes - // in vendor/libarchive make every layer (filter_ahead → gzip → tar) - // preserve its state and propagate ARCHIVE_RETRY to our `step()` - // loop, which then returns so this worker can be reused. - lib::Result::Retry as lib::la_ssize_t -} - fn open_output_file( dest_fd: Fd, path: OSPathZ, @@ -1325,7 +1105,7 @@ fn open_output_file( } } -fn make_directory(entry: &mut lib::Entry, dest_fd: Fd, path: OSPathZ, path_slice: &[OSPathChar]) { +fn make_directory(entry: &tar::Entry, dest_fd: Fd, path: OSPathZ, path_slice: &[OSPathChar]) { #[cfg(windows)] { let _ = bun_sys::make_path::make_path::(Dir::borrow(&dest_fd), &path[..]); @@ -1333,7 +1113,11 @@ fn make_directory(entry: &mut lib::Entry, dest_fd: Fd, path: OSPathZ, path_slice } #[cfg(not(windows))] { - match bun_sys::mkdirat_z(dest_fd, path, bun_libarchive::directory_mode(entry.perm())) { + match bun_sys::mkdirat_z( + dest_fd, + path, + bun_archive::directory_mode(entry.perm() as Mode), + ) { Ok(()) => {} Err(e) => match e.get_errno() { bun_sys::E::EEXIST | bun_sys::E::ENOTDIR => {} diff --git a/src/install/cache_pack.rs b/src/install/cache_pack.rs index 97db80fb8c99..2b60ecf3da5d 100644 --- a/src/install/cache_pack.rs +++ b/src/install/cache_pack.rs @@ -498,7 +498,7 @@ fn pack_entry( let len = sys::readlinkat(entry.dir, entry.basename, &mut target_buf[..])?; let target = ZStr::from_buf(&target_buf[..], len); if len > MAX_PACK_PATH_LEN - || !bun_libarchive::is_symlink_target_safe(rel, target, &mut join_buf) + || !bun_archive::is_symlink_target_safe(rel, target, &mut join_buf) { return Err(refuse( "cache entry has symlink that may point outside it", @@ -749,7 +749,7 @@ fn unpack_package( let mut target: Vec = Vec::new(); // Symlinks are created last, so that no path can go through one while files are being written. #[cfg(unix)] - let mut links: Vec = Vec::new(); + let mut links: Vec = Vec::new(); #[cfg(unix)] let mut join_buf = None; @@ -804,7 +804,7 @@ fn unpack_package( { let target = ZStr::from_slice_with_nul(&target); if strings::contains_char(target.as_bytes(), 0) - || !bun_libarchive::is_symlink_target_safe( + || !bun_archive::is_symlink_target_safe( rel.as_bytes(), target, &mut join_buf, @@ -812,7 +812,7 @@ fn unpack_package( { return Err(r.refuse("unsafe symlink target", rel.as_bytes())); } - links.push(bun_libarchive::DeferredSymlink::new( + links.push(bun_archive::DeferredSymlink::new( rel.as_bytes(), target.as_bytes(), )); @@ -829,7 +829,7 @@ fn unpack_package( } #[cfg(unix)] - if bun_libarchive::create_deferred_symlinks(dir.fd(), &links, false) as usize != links.len() { + if bun_archive::create_deferred_symlinks(dir.fd(), &links, false) as usize != links.len() { return Err(r.refuse("symlink collides with another entry", name.as_bytes())); } // Windows requires the directory to be closed before it's renamed. diff --git a/src/install/error.rs b/src/install/error.rs index 6501a1d207fb..7c77823fd7d0 100644 --- a/src/install/error.rs +++ b/src/install/error.rs @@ -396,14 +396,14 @@ impl From for Error { } } -impl From for Error { - fn from(e: bun_libarchive::Error) -> Self { +impl From for Error { + fn from(e: bun_archive::Error) -> Self { match e { - bun_libarchive::Error::Fail => Self::Fail, - bun_libarchive::Error::Sys(s) => Self::Sys(s), - bun_libarchive::Error::Alloc(a) => Self::Alloc(a), - bun_libarchive::Error::MakeLibUvOwned(_) => Self::SystemFdQuotaExceeded, - bun_libarchive::Error::Paths(p) => Self::Paths(p), + bun_archive::Error::Fail => Self::Fail, + bun_archive::Error::Sys(s) => Self::Sys(s), + bun_archive::Error::Alloc(a) => Self::Alloc(a), + bun_archive::Error::MakeLibUvOwned(_) => Self::SystemFdQuotaExceeded, + bun_archive::Error::Paths(p) => Self::Paths(p), } } } diff --git a/src/install/extract_tarball.rs b/src/install/extract_tarball.rs index 8d377c76dd3f..e095baf20cfd 100644 --- a/src/install/extract_tarball.rs +++ b/src/install/extract_tarball.rs @@ -9,13 +9,13 @@ use bun_paths::{self as path, PathBuffer}; use bun_semver::Version; use bun_sys::{self as sys, Dir, Fd}; +use bun_archive::{ArchiveAppender, ExtractOptions}; use bun_install::install::{self as Install, DependencyID, ExtractData}; use bun_install::integrity::Integrity; use bun_install::npm::{self as Npm}; use bun_install::package_manager_real::PackageManager; use bun_install::package_manager_real::directories; use bun_install::resolution::{Resolution, Tag as ResolutionTag}; -use bun_libarchive::{ArchiveAppender, ExtractOptions}; use bun_resolver::fs::FileSystem; use crate::cache_publish::{self, StagedEntry}; @@ -291,7 +291,7 @@ impl ExtractTarball { } }; - use bun_libarchive::Archiver; + use bun_archive::Archiver; let mut zlib_pool = Npm::Registry::BodyPool::get(); zlib_pool.reset(); // `defer Npm.Registry.BodyPool.release(zlib_pool)` → PoolGuard's Drop releases. @@ -302,7 +302,7 @@ impl ExtractTarball { 0 }; - // libarchive gunzips on the fly (`BufferReadStream::open_read`), + // `extract_to_dir` gunzips on the fly, // so hand it the compressed bytes and never buffer the full tar. // Small tarballs still try libdeflate first for speed; the gzip // ISIZE trailer (size mod 2^32) is only trusted when small. @@ -329,7 +329,7 @@ impl ExtractTarball { libdeflate::Encoding::Gzip, ); // libdeflate only reads the first gzip member, so a tarball with more - // than one member is left to libarchive. + // than one member is left to `extract_to_dir`. if result.status == libdeflate::Status::Success && result.read == tgz_bytes.len() { diff --git a/src/js/internal-for-testing.ts b/src/js/internal-for-testing.ts index 9e61a10b3bb1..33e1ba347f30 100644 --- a/src/js/internal-for-testing.ts +++ b/src/js/internal-for-testing.ts @@ -185,6 +185,9 @@ export const npm_manifest_test_helpers = $bindgen("npm_jsc.bind.ts", "npmManifes export const readTarball = $bindgen("pack_command.bind.ts", "readTarball"); +export const readTar = $bindgen("ArchiveInternals.bind.ts", "readTar"); +export const writeTar = $bindgen("ArchiveInternals.bind.ts", "writeTar"); + export const isArchitectureMatch: (architecture: string[]) => boolean = $bindgen( "npm_jsc.bind.ts", "architectureIsMatch", diff --git a/src/jsc/bindings/BunProcess.cpp b/src/jsc/bindings/BunProcess.cpp index 00e342dac979..ab0193ba344f 100644 --- a/src/jsc/bindings/BunProcess.cpp +++ b/src/jsc/bindings/BunProcess.cpp @@ -276,7 +276,6 @@ static JSValue constructVersions(VM& vm, JSObject* processObject) { "openssl", "1.1.0" }, // keep in sync with src/jsc/bindings/node/http/llhttp/README.md { "llhttp", "9.4.2" }, - { "libarchive", BUN_VERSION_LIBARCHIVE }, { "mimalloc", BUN_VERSION_MIMALLOC }, { "picohttpparser", BUN_VERSION_PICOHTTPPARSER }, { "uwebsockets", BUN_VERSION_UWS }, diff --git a/src/jsc/bindings/workaround-missing-symbols.cpp b/src/jsc/bindings/workaround-missing-symbols.cpp index 0ee12bafda18..0c75d8269a4f 100644 --- a/src/jsc/bindings/workaround-missing-symbols.cpp +++ b/src/jsc/bindings/workaround-missing-symbols.cpp @@ -356,7 +356,7 @@ extern "C" __attribute__((used)) char __libc_single_threaded = 0; // ─────────────────────────────────────────────────────────────────────────── // glibc 2.27–2.35 symbols pulled in by Rust std, host crt1.o, libgcc_eh, -// libarchive, and the ASAN runtime when building on a glibc ≥ 2.27 host. +// and the ASAN runtime when building on a glibc ≥ 2.27 host. // All have ABI-compatible older versions (or syscall fallbacks); pinning // each __wrap_X to the floor version keeps verneed ≤ 2.17 regardless of // build host. On a glibc 2.17 host these wraps are inert (the linker never diff --git a/src/libarchive/lib.rs b/src/libarchive/lib.rs deleted file mode 100644 index 556fbcc36cc9..000000000000 --- a/src/libarchive/lib.rs +++ /dev/null @@ -1,1996 +0,0 @@ -// @link "../deps/libarchive.a" -#![warn(unused_must_use)] -// ────────────────────────────────────────────────────────────────────────── -// Thin `extern "C"` wrappers over the libarchive C library live in `mod lib` -// below; higher-level extraction logic (`Archiver`, `BufferReadStream`) sits -// on top and uses `bun_sys` for I/O. -// ────────────────────────────────────────────────────────────────────────── -use core::ptr; - -use bun_collections::StringArrayHashMap; -use bun_core::{MutableString, slice_to_nul, strings}; -use bun_core::{Output, ZStr, slice_as_bytes}; -#[cfg(unix)] -use bun_paths::PathBuffer; -use bun_paths::{OSPathChar, SEP, SEP_STR}; -use bun_sys::{self, Fd, FdExt}; -use bun_wyhash::hash; - -pub mod error; -pub use error::{Error, Result}; - -// ────────────────────────────────────────────────────────────────────────── -// Local libarchive C-API surface. Thin safe(ish) wrappers over the raw -// `extern "C"` libarchive symbols. The opaque `Archive` / `Entry` types -// here are layout-compatible with libarchive's `struct archive` / -// `struct archive_entry` (zero-sized, `#[repr(C)]`, !Unpin). -// ────────────────────────────────────────────────────────────────────────── -#[allow(non_camel_case_types)] -pub mod lib { - use super::*; - use core::ffi::{c_char, c_int, c_long, c_uint, c_void}; - - pub type la_ssize_t = isize; - pub(crate) type la_int64_t = i64; - type time_t = isize; - - bun_opaque::opaque_ffi! { - /// Opaque libarchive `struct archive`. Always used behind `*mut Archive`. - /// Contains `UnsafeCell` so that `&Archive` does not assert immutability - /// (libarchive mutates through every call), making `&self -> *mut Self` - /// sound under Stacked Borrows. - pub struct Archive; - /// Opaque libarchive `struct archive_entry`. Always used behind `*mut Entry`. - /// Contains `UnsafeCell` for the same reason as `Archive` — the C side - /// mutates through getter/setter calls that take `&self` here. - pub struct Entry; - } - - #[repr(i32)] - #[derive(Copy, Clone, Eq, PartialEq, Debug)] - pub enum Result { - Eof = 1, - Ok = 0, - Retry = -10, - Warn = -20, - Failed = -25, - Fatal = -30, - } - - impl Result { - /// `Ok` or `Warn`: the call completed. libarchive returns `Warn` for - /// recoverable per-call issues while still producing a result, e.g. a - /// `read_next_header` that fell back to the raw pathname bytes. - #[inline] - pub fn succeeded(self) -> bool { - matches!(self, Result::Ok | Result::Warn) - } - } - - // ── raw libarchive C FFI ─────────────────────────────────────────────── - // Signatures match `vendor/libarchive/archive.h` exactly. `Result` is `#[repr(i32)]` - // so it is ABI-compatible with the C `int` return values. - unsafe extern "C" { - // read side - fn archive_read_new() -> *mut Archive; - fn archive_read_close(a: *mut Archive) -> Result; - fn archive_read_free(a: *mut Archive) -> Result; - fn archive_read_support_format_tar(a: *mut Archive) -> Result; - fn archive_read_support_format_gnutar(a: *mut Archive) -> Result; - fn archive_read_support_filter_gzip(a: *mut Archive) -> Result; - fn archive_read_set_options(a: *mut Archive, opts: *const c_char) -> Result; - fn archive_read_open_memory(a: *mut Archive, buf: *const c_void, size: usize) -> Result; - fn archive_read_next_header(a: *mut Archive, entry: *mut *mut Entry) -> Result; - fn archive_read_data(a: *mut Archive, buf: *mut c_void, size: usize) -> la_ssize_t; - fn archive_read_data_block( - a: *mut Archive, - buff: *mut *const c_void, - size: *mut usize, - offset: *mut la_int64_t, - ) -> Result; - fn archive_error_string(a: *mut Archive) -> *const c_char; - // streaming-read setup (used by TarballStream's resumable extractor) - pub fn archive_read_set_format(a: *mut Archive, code: c_int) -> c_int; - pub fn archive_read_append_filter(a: *mut Archive, code: c_int) -> c_int; - pub fn archive_read_open( - a: *mut Archive, - client_data: *mut c_void, - open: Option, - read: Option, - close: Option, - ) -> c_int; - - // write side - fn archive_write_new() -> *mut Archive; - fn archive_write_free(a: *mut Archive) -> Result; - fn archive_write_close(a: *mut Archive) -> Result; - fn archive_write_set_bytes_in_last_block(a: *mut Archive, bytes: c_int) -> Result; - fn archive_set_error(a: *mut Archive, err: c_int, fmt: *const c_char, ...); - fn archive_write_set_format_pax_restricted(a: *mut Archive) -> Result; - fn archive_write_add_filter_gzip(a: *mut Archive) -> Result; - fn archive_write_set_filter_option( - a: *mut Archive, - module: *const c_char, - option: *const c_char, - value: *const c_char, - ) -> Result; - fn archive_write_set_options(a: *mut Archive, opts: *const c_char) -> Result; - fn archive_write_header(a: *mut Archive, entry: *mut Entry) -> Result; - fn archive_write_data(a: *mut Archive, data: *const c_void, size: usize) -> la_ssize_t; - fn archive_write_finish_entry(a: *mut Archive) -> Result; - #[link_name = "archive_write_open2"] - fn archive_write_open2_raw( - a: *mut Archive, - client_data: *mut c_void, - open: Option, - write: Option, - close: Option, - free: Option, - ) -> c_int; - - // entry - fn archive_entry_new() -> *mut Entry; - fn archive_entry_new2(a: *mut Archive) -> *mut Entry; - fn archive_entry_free(e: *mut Entry); - fn archive_entry_clear(e: *mut Entry) -> *mut Entry; - fn archive_entry_pathname(e: *mut Entry) -> *const c_char; - #[cfg(windows)] - fn archive_entry_pathname_w(e: *mut Entry) -> *const u16; - fn archive_entry_symlink(e: *mut Entry) -> *const c_char; - fn archive_entry_perm(e: *mut Entry) -> bun_sys::Mode; - fn archive_entry_size(e: *mut Entry) -> la_int64_t; - fn archive_entry_filetype(e: *mut Entry) -> bun_sys::Mode; - fn archive_entry_mtime(e: *mut Entry) -> time_t; - fn archive_entry_set_pathname(e: *mut Entry, name: *const c_char); - fn archive_entry_set_pathname_utf8(e: *mut Entry, name: *const c_char); - fn archive_entry_set_size(e: *mut Entry, s: la_int64_t); - fn archive_entry_set_filetype(e: *mut Entry, t: c_uint); - fn archive_entry_set_perm(e: *mut Entry, p: bun_sys::Mode); - fn archive_entry_set_mtime(e: *mut Entry, secs: time_t, nsecs: c_long); - } - - /// One block from `archive_read_data_block`. `bytes` borrows libarchive's - /// internal buffer (valid until the next read call on the owning archive). - pub struct Block<'a> { - pub bytes: &'a [u8], - pub offset: i64, - pub result: Result, - } - - /// One per extraction. Once `pwrite` fails, later entries don't try it again. - pub struct WriteStrategy { - pub pwrite: bool, - } - - impl Default for WriteStrategy { - fn default() -> Self { - Self { pwrite: cfg!(unix) } - } - } - - /// Never sizes a write or an allocation from the entry header or the sparse map, because - /// nothing verifies them. - pub struct EntryWriter { - fd: Fd, - /// One past the last byte written. - end: i64, - /// The file position, which only `write` and `lseek` move. - cursor: i64, - #[cfg(windows)] - marked_sparse: bool, - } - - impl EntryWriter { - pub fn new(fd: Fd) -> Self { - Self { - fd, - end: 0, - cursor: 0, - #[cfg(windows)] - marked_sparse: false, - } - } - - #[inline] - pub fn fd(&self) -> Fd { - self.fd - } - - /// NTFS gives a range that nothing writes real clusters unless the file is marked sparse. - #[inline] - fn before_hole(&mut self) { - #[cfg(windows)] - if !self.marked_sparse { - self.marked_sparse = true; - // A volume without sparse file support refuses the mark. The hole then takes up - // disk space. - let _ = bun_sys::set_sparse(self.fd); - } - } - - pub fn write( - &mut self, - strategy: &mut WriteStrategy, - offset: i64, - data: &[u8], - ) -> bun_sys::Maybe<()> { - if data.is_empty() { - return Ok(()); - } - let fd = self.fd; - let file = bun_sys::File::borrow(&fd); - - #[cfg(unix)] - if strategy.pwrite { - match file.pwrite_all(data, offset) { - Ok(()) => { - self.end = self.end.max(offset + data.len() as i64); - return Ok(()); - } - Err(_) => { - strategy.pwrite = false; - bun_core::debug_warn!( - "libarchive: falling back to write() after pwrite() failure", - ); - } - } - } - #[cfg(not(unix))] - let _ = strategy; - - if offset != self.cursor { - if offset > self.end { - self.before_hole(); - } - bun_sys::set_file_offset(self.fd, offset as u64)?; - self.cursor = offset; - } - - file.write_all(data)?; - self.cursor += data.len() as i64; - self.end = self.end.max(self.cursor); - Ok(()) - } - - /// `end` is the offset that libarchive returned with `ARCHIVE_EOF`, which is the length of - /// the file. - pub fn finish(&mut self, end: i64) -> bun_sys::Maybe<()> { - if end > self.end { - self.before_hole(); - bun_sys::ftruncate(self.fd, end)?; - self.end = end; - } - Ok(()) - } - } - - impl Archive { - pub fn read_new() -> *mut Archive { - // SAFETY: FFI call with no preconditions. - let p = unsafe { archive_read_new() }; - // libarchive's `archive_read_new()` returns NULL on calloc failure. - // Every caller immediately dereferences the result (forming - // `&Archive`), so fail loudly here instead of invoking UB at the - // first accessor call. - assert!(!p.is_null(), "archive_read_new returned NULL (OOM)"); - p - } - pub fn read_close(&self) -> Result { - // SAFETY: self came from archive_read_new(). - unsafe { archive_read_close(self.as_mut_ptr()) } - } - pub fn read_free(&self) -> Result { - // SAFETY: self came from archive_read_new(); not used after this. - unsafe { archive_read_free(self.as_mut_ptr()) } - } - pub fn read_support_format_tar(&self) -> Result { - // SAFETY: self valid. - unsafe { archive_read_support_format_tar(self.as_mut_ptr()) } - } - pub fn read_support_format_gnutar(&self) -> Result { - // SAFETY: self valid. - unsafe { archive_read_support_format_gnutar(self.as_mut_ptr()) } - } - pub fn read_support_filter_gzip(&self) -> Result { - // SAFETY: self valid. - unsafe { archive_read_support_filter_gzip(self.as_mut_ptr()) } - } - pub fn read_set_options(&self, opts: &core::ffi::CStr) -> Result { - // SAFETY: self valid; opts is NUL-terminated. - unsafe { archive_read_set_options(self.as_mut_ptr(), opts.as_ptr()) } - } - pub fn read_open_memory(&self, buf: &[u8]) -> Result { - // SAFETY: self valid; buf outlives the archive (caller contract, - // see `BufferReadStream::buf` field comment). - unsafe { archive_read_open_memory(self.as_mut_ptr(), buf.as_ptr().cast(), buf.len()) } - } - pub fn read_next_header(&self, entry: &mut *mut Entry) -> Result { - // SAFETY: self valid; entry is a valid out-ptr. - unsafe { - archive_read_next_header(self.as_mut_ptr(), std::ptr::from_mut::<*mut Entry>(entry)) - } - } - pub fn read_data(&self, buf: &mut [u8]) -> isize { - // SAFETY: self valid; buf writable for buf.len(). - unsafe { archive_read_data(self.as_mut_ptr(), buf.as_mut_ptr().cast(), buf.len()) } - } - - /// Reads in steps, so that an untrusted `size` doesn't drive the allocation. Returns - /// `Ok(false)` on a read error. - pub fn read_data_to_vec( - &self, - size: usize, - out: &mut Vec, - ) -> core::result::Result { - let start = out.len(); - while out.len() - start < size { - let to_read = (size - (out.len() - start)).min(64 * 1024); - out.try_reserve(to_read).map_err(|_| bun_core::AllocError)?; - // SAFETY: `archive_read_data` only writes into the slice; the written prefix is committed below. - let dest = unsafe { &mut bun_core::vec::spare_bytes_mut(out)[..to_read] }; - let read = self.read_data(dest); - if read < 0 { - return Ok(false); - } - if read == 0 { - break; - } - // SAFETY: `archive_read_data` returns exactly the byte count it wrote (`<= to_read`). - unsafe { - bun_core::vec::commit_spare(out, usize::try_from(read).expect("int cast")) - }; - } - Ok(true) - } - - /// Calls `archive_read_data_block`. Returns `None` on EOF, and `*offset` is then the - /// entry's length. - pub fn next(&self, offset: &mut i64) -> Option> { - let mut buff: *const c_void = core::ptr::null(); - let mut size: usize = 0; - // SAFETY: self valid; out-ptrs are valid stack locations. - let r = unsafe { - archive_read_data_block(self.as_mut_ptr(), &raw mut buff, &raw mut size, offset) - }; - if r == Result::Eof { - return None; - } - if r != Result::Ok { - return Some(Block { - bytes: &[], - offset: *offset, - result: r, - }); - } - // SAFETY: on ARCHIVE_OK, libarchive guarantees buff[0..size] is - // readable until the next read call on this archive. - let bytes = unsafe { core::slice::from_raw_parts(buff.cast::(), size) }; - Some(Block { - bytes, - offset: *offset, - result: r, - }) - } - - pub fn read_data_into_fd(&self, fd: Fd, strategy: &mut WriteStrategy) -> Result { - let mut writer = EntryWriter::new(fd); - let mut offset: i64 = 0; - while let Some(block) = self.next(&mut offset) { - if block.result != Result::Ok { - return block.result; - } - if writer.write(strategy, block.offset, block.bytes).is_err() { - return Result::Failed; - } - } - match writer.finish(offset) { - Ok(()) => Result::Ok, - Err(_) => Result::Failed, - } - } - - // `self` must be a live archive handle from `archive_{read,write}_new()`. - // `Archive` is `opaque_ffi!`-backed (UnsafeCell), so `&self → *mut Self` - // is sound; libarchive never returns null from `*_new()`. - pub fn error_string(&self) -> &'static [u8] { - // SAFETY: `self` is a live archive handle. - let p = unsafe { archive_error_string(self.as_mut_ptr()) }; - if p.is_null() { - return b""; - } - // SAFETY: libarchive owns the error string for the lifetime of the - // archive; callers treat it as borrowed-until-next-call. The - // `'static` here is a lifetime erasure — the caller must not let - // the slice outlive the archive. - unsafe { ZStr::from_c_ptr(p) }.as_bytes() - } - - // ── write side ───────────────────────────────────────────────────── - pub fn write_new() -> *mut Archive { - // SAFETY: FFI call with no preconditions. - unsafe { archive_write_new() } - } - pub fn write_close(&self) -> Result { - // SAFETY: self valid. - unsafe { archive_write_close(self.as_mut_ptr()) } - } - /// Sets the padding unit for the last output block. The default pads it to 10 KiB, and 1 - /// disables padding. - pub fn write_set_bytes_in_last_block(&self, bytes: c_int) -> Result { - // SAFETY: self valid. - unsafe { archive_write_set_bytes_in_last_block(self.as_mut_ptr(), bytes) } - } - pub fn write_set_format_pax_restricted(&self) -> Result { - // SAFETY: self valid. - unsafe { archive_write_set_format_pax_restricted(self.as_mut_ptr()) } - } - pub fn write_add_filter_gzip(&self) -> Result { - // SAFETY: self valid. - unsafe { archive_write_add_filter_gzip(self.as_mut_ptr()) } - } - pub fn write_set_filter_option( - &self, - module: Option<&ZStr>, - option: &ZStr, - value: &ZStr, - ) -> Result { - // SAFETY: self valid; ZStr guarantees NUL-termination. - unsafe { - archive_write_set_filter_option( - self.as_mut_ptr(), - module.map_or(core::ptr::null(), |m| m.as_ptr().cast()), - option.as_ptr().cast(), - value.as_ptr().cast(), - ) - } - } - pub fn write_set_options(&self, opts: &ZStr) -> Result { - // SAFETY: self valid; ZStr guarantees NUL-termination. - unsafe { archive_write_set_options(self.as_mut_ptr(), opts.as_ptr().cast()) } - } - pub fn write_header(&self, entry: &Entry) -> Result { - // SAFETY: self valid; entry came from Entry::new()/read_next_header(). - // `Entry` has interior mutability so `&Entry -> *mut Entry` is sound. - unsafe { archive_write_header(self.as_mut_ptr(), entry.as_mut_ptr()) } - } - pub fn write_data(&self, data: &[u8]) -> isize { - // SAFETY: self valid; data readable for data.len(). - unsafe { archive_write_data(self.as_mut_ptr(), data.as_ptr().cast(), data.len()) } - } - pub fn write_finish_entry(&self) -> Result { - // SAFETY: self valid. - unsafe { archive_write_finish_entry(self.as_mut_ptr()) } - } - } - - impl Entry { - pub fn pathname(&self) -> &ZStr { - // SAFETY: self valid; returned string owned by libarchive for the - // lifetime of this entry. - unsafe { ZStr::from_c_ptr(archive_entry_pathname(self.as_mut_ptr())) } - } - #[cfg(windows)] - pub fn pathname_w(&self) -> &bun_core::WStr { - // SAFETY: self valid. - unsafe { bun_core::WStr::from_ptr(archive_entry_pathname_w(self.as_mut_ptr())) } - } - pub fn symlink(&self) -> &ZStr { - // SAFETY: self valid. - unsafe { ZStr::from_c_ptr(archive_entry_symlink(self.as_mut_ptr())) } - } - pub fn perm(&self) -> u32 { - // SAFETY: self valid. - unsafe { archive_entry_perm(self.as_mut_ptr()) as u32 } - } - pub fn size(&self) -> i64 { - // SAFETY: self valid. - unsafe { archive_entry_size(self.as_mut_ptr()) } - } - pub fn filetype(&self) -> u32 { - // SAFETY: self valid. - unsafe { archive_entry_filetype(self.as_mut_ptr()) as u32 } - } - pub fn mtime(&self) -> i64 { - // SAFETY: self valid. - unsafe { archive_entry_mtime(self.as_mut_ptr()) as i64 } - } - - // ── write side ───────────────────────────────────────────────────── - pub(crate) fn new() -> *mut Entry { - // SAFETY: FFI call with no preconditions. - unsafe { archive_entry_new() } - } - /// `archive_entry_new2(archive)` — ties the entry to the archive's - /// charset-conversion context (preferred over `new()` when an archive - /// is available). `archive` is a live handle from `read_new()`/`write_new()`. - pub fn new2(archive: &Archive) -> *mut Entry { - // SAFETY: `archive` is a live handle (opaque_ffi! `&self → *mut Self`). - unsafe { archive_entry_new2(archive.as_mut_ptr()) } - } - pub fn free(&self) { - // SAFETY: self came from Entry::new(); not used after this. - unsafe { archive_entry_free(self.as_mut_ptr()) } - } - pub fn clear(&self) -> *mut Entry { - // SAFETY: self valid. - unsafe { archive_entry_clear(self.as_mut_ptr()) } - } - /// Raw `archive_entry_set_pathname` — bytes are stored verbatim (no - /// charset conversion). - pub fn set_pathname(&self, name: &ZStr) { - // SAFETY: self valid; name is NUL-terminated. - unsafe { archive_entry_set_pathname(self.as_mut_ptr(), name.as_ptr()) } - } - pub fn set_pathname_utf8(&self, name: &ZStr) { - // SAFETY: self valid; name is NUL-terminated. - unsafe { archive_entry_set_pathname_utf8(self.as_mut_ptr(), name.as_ptr()) } - } - pub fn set_size(&self, s: i64) { - // SAFETY: self valid. - unsafe { archive_entry_set_size(self.as_mut_ptr(), s) } - } - pub fn set_filetype(&self, t: u32) { - // SAFETY: self valid. - unsafe { archive_entry_set_filetype(self.as_mut_ptr(), t as c_uint) } - } - pub fn set_perm(&self, p: u32) { - // SAFETY: self valid. - unsafe { archive_entry_set_perm(self.as_mut_ptr(), p as bun_sys::Mode) } - } - pub fn set_mtime(&self, secs: isize, nsecs: core::ffi::c_long) { - // SAFETY: self valid. - unsafe { archive_entry_set_mtime(self.as_mut_ptr(), secs as time_t, nsecs) } - } - } - - // ── RAII owners ──────────────────────────────────────────────────────── - // - // The raw `*mut Archive` / `*mut Entry` constructors above mirror the C - // API. These thin owners pair them with the matching `*_free` on `Drop` - // so callers stop hand-rolling `defer { (*archive).read_free() }`. - - /// Owns a `*mut Archive` opened with [`Archive::read_new`]; calls - /// `archive_read_free` on drop. Derefs to `&Archive`. - pub struct ReadArchive(core::ptr::NonNull); - impl ReadArchive { - #[inline] - pub fn new() -> Self { - Self( - core::ptr::NonNull::new(Archive::read_new()) - .expect("archive_read_new returned null"), - ) - } - } - impl core::ops::Deref for ReadArchive { - type Target = Archive; - #[inline] - fn deref(&self) -> &Archive { - // SAFETY: handle is live until Drop; libarchive owns the storage. - unsafe { self.0.as_ref() } - } - } - impl Drop for ReadArchive { - #[inline] - fn drop(&mut self) { - // SAFETY: handle came from archive_read_new() and is freed exactly once. - let _ = unsafe { archive_read_free(self.0.as_ptr()) }; - } - } - - /// Owns a `*mut Archive` opened with [`Archive::write_new`]; calls - /// `archive_write_free` on drop. Derefs to `&Archive`. - pub struct WriteArchive(core::ptr::NonNull); - impl WriteArchive { - #[inline] - pub fn new() -> Self { - Self( - core::ptr::NonNull::new(Archive::write_new()) - .expect("archive_write_new returned null"), - ) - } - } - impl core::ops::Deref for WriteArchive { - type Target = Archive; - #[inline] - fn deref(&self) -> &Archive { - // SAFETY: handle is live until Drop; libarchive owns the storage. - unsafe { self.0.as_ref() } - } - } - impl Drop for WriteArchive { - #[inline] - fn drop(&mut self) { - // SAFETY: handle came from archive_write_new() and is freed exactly once. - let _ = unsafe { archive_write_free(self.0.as_ptr()) }; - } - } - - /// Owns a `*mut Entry` created with [`Entry::new`] / [`Entry::new2`]; - /// calls `archive_entry_free` on drop. Derefs to `&Entry`. - pub struct OwnedEntry(core::ptr::NonNull); - impl OwnedEntry { - #[inline] - pub fn new() -> Self { - Self(core::ptr::NonNull::new(Entry::new()).expect("archive_entry_new returned null")) - } - } - impl core::ops::Deref for OwnedEntry { - type Target = Entry; - #[inline] - fn deref(&self) -> &Entry { - // SAFETY: handle is live until Drop; libarchive owns the storage. - unsafe { self.0.as_ref() } - } - } - impl Drop for OwnedEntry { - #[inline] - fn drop(&mut self) { - // SAFETY: handle came from archive_entry_new()/new2() and is freed exactly once. - unsafe { archive_entry_free(self.0.as_ptr()) }; - } - } - - // ── Archive::Iterator ────────────────────────────────────────────────── - // - // Thin streaming reader over a tar.gz blob: `init` opens the archive in - // memory, `next` yields one header at a time, `read_entry_data` slurps the - // current entry's payload, `close` tears down. Errors are surfaced as the - // libarchive `*mut Archive` plus a static message so callers can append - // `Archive::error_string`. - - /// Generic result type used by [`ArchiveIterator`]. - pub enum IteratorResult { - Err { - archive: *mut Archive, - message: &'static [u8], - }, - Result(T), - } - - impl IteratorResult { - #[inline] - pub(crate) fn init_err(arch: *mut Archive, msg: &'static [u8]) -> Self { - Self::Err { - message: msg, - archive: arch, - } - } - #[inline] - pub(crate) fn init_res(value: T) -> Self { - Self::Result(value) - } - } - - /// Iterates over the entries of an open archive, skipping entries whose - /// file kind has its bit set in `filter`. - pub struct ArchiveIterator { - pub archive: *mut Archive, - // A u16 bitmask over - // `bun_sys::FileKind` variants. - pub(crate) filter: u16, - } - - /// One entry returned from [`ArchiveIterator::next`]. - pub struct NextEntry { - pub entry: *mut Entry, - pub kind: bun_sys::FileKind, - } - - impl NextEntry { - /// The header `next()` just read; valid until the iterator advances. - pub fn entry(&self) -> &Entry { - // SAFETY: `entry` is the non-null header libarchive handed to `next()` and stays live until the next read. - unsafe { &*self.entry } - } - } - - impl ArchiveIterator { - /// Borrow the underlying libarchive handle. - /// - /// SAFETY (invariant): `self.archive` is set to a fresh non-null - /// handle by `Archive::read_new()` in [`init`] and remains valid - /// until `read_free()` in [`close`]. All `Archive` methods take - /// `&self` (FFI interior mutability), so a shared borrow suffices. - #[inline] - pub fn archive(&self) -> &Archive { - // SAFETY: see doc comment — non-null for the lifetime of `self`. - unsafe { &*self.archive } - } - - /// Reads the body of the entry `next()` just returned. - pub fn read_entry_data( - &mut self, - next: &NextEntry, - ) -> core::result::Result>, bun_core::OOM> { - next.read_entry_data(self.archive()) - } - - pub fn init(tarball_bytes: &[u8]) -> IteratorResult { - let archive = Archive::read_new(); - // SAFETY: archive_read_new() returns a non-null handle owned by libarchive. - let a = unsafe { &*archive }; - - match a.read_support_format_tar() { - Result::Failed | Result::Fatal | Result::Warn => { - return IteratorResult::init_err( - archive, - b"failed to enable tar format support", - ); - } - _ => {} - } - match a.read_support_format_gnutar() { - Result::Failed | Result::Fatal | Result::Warn => { - return IteratorResult::init_err( - archive, - b"failed to enable gnutar format support", - ); - } - _ => {} - } - match a.read_support_filter_gzip() { - Result::Failed | Result::Fatal | Result::Warn => { - return IteratorResult::init_err( - archive, - b"failed to enable support for gzip compression", - ); - } - _ => {} - } - match a.read_set_options(c"read_concatenated_archives") { - Result::Failed | Result::Fatal | Result::Warn => { - return IteratorResult::init_err( - archive, - b"failed to set option `read_concatenated_archives`", - ); - } - _ => {} - } - match a.read_open_memory(tarball_bytes) { - Result::Failed | Result::Fatal | Result::Warn => { - return IteratorResult::init_err(archive, b"failed to read tarball"); - } - _ => {} - } - - IteratorResult::init_res(Self { archive, filter: 0 }) - } - - pub fn next(&mut self) -> IteratorResult> { - let a = self.archive(); - let mut entry: *mut Entry = core::ptr::null_mut(); - loop { - return match a.read_next_header(&mut entry) { - Result::Retry => continue, - Result::Eof => IteratorResult::init_res(None), - // `Warn` still yields a fully populated entry; see `Result::succeeded`. - Result::Ok | Result::Warn => { - let kind = bun_sys::kind_from_mode( - Entry::opaque_ref(entry).filetype() as bun_sys::Mode - ); - if (self.filter & (1u16 << (kind as u8))) != 0 { - continue; - } - IteratorResult::init_res(Some(NextEntry { entry, kind })) - } - _ => IteratorResult::init_err(self.archive, b"failed to read archive header"), - }; - } - } - - /// Returns a `Result` the caller inspects, so this - /// cannot be `Drop`. Explicit-close per PORTING.md §Idiom map. - pub fn close(self) -> IteratorResult<()> { - let a = self.archive(); - match a.read_close() { - Result::Failed | Result::Fatal | Result::Warn => { - return IteratorResult::init_err(self.archive, b"failed to close archive read"); - } - _ => {} - } - match a.read_free() { - Result::Failed | Result::Fatal | Result::Warn => { - return IteratorResult::init_err(self.archive, b"failed to free archive read"); - } - _ => {} - } - IteratorResult::init_res(()) - } - } - - impl NextEntry { - /// Reads this entry's full data into a heap buffer. `archive` is the - /// live handle this `NextEntry` was yielded from. - pub fn read_entry_data( - &self, - archive: &Archive, - ) -> core::result::Result>, bun_core::OOM> { - // SAFETY: self.entry is the libarchive-owned entry from read_next_header. - let size = unsafe { (*self.entry).size() }; - let Ok(size) = usize::try_from(size) else { - return Ok(IteratorResult::init_err( - archive.as_mut_ptr(), - b"invalid archive entry size", - )); - }; - let mut buf: Vec = Vec::new(); - if !archive.read_data_to_vec(size, &mut buf)? { - return Ok(IteratorResult::init_err( - archive.as_mut_ptr(), - b"failed to read archive data", - )); - } - Ok(IteratorResult::init_res(buf.into_boxed_slice())) - } - } - - // ── write-open callback surface (libarchive `archive_write_open2`) ───── - type archive_open_callback = unsafe extern "C" fn(*mut Archive, *mut c_void) -> c_int; - pub type archive_read_callback = - unsafe extern "C" fn(*mut Archive, *mut c_void, *mut *const c_void) -> la_ssize_t; - type archive_write_callback = - unsafe extern "C" fn(*mut Archive, *mut c_void, *const c_void, usize) -> la_ssize_t; - type archive_close_callback = unsafe extern "C" fn(*mut Archive, *mut c_void) -> c_int; - type archive_free_callback = unsafe extern "C" fn(*mut Archive, *mut c_void) -> c_int; - - /// `a` is a live `archive_write_new()` handle. `client_data` is forwarded - /// opaquely to the callbacks (never dereferenced here); its lifetime must - /// outlast the registered callbacks. - #[allow(clippy::not_unsafe_ptr_arg_deref)] - pub fn archive_write_open2( - a: &Archive, - client_data: *mut c_void, - open: Option, - write: Option, - close: Option, - free: Option, - ) -> c_int { - // SAFETY: `a` is a live handle (`opaque_ffi!` `&self → *mut Self`); - // `client_data` is opaque to libarchive until a callback dereferences it. - unsafe { archive_write_open2_raw(a.as_mut_ptr(), client_data, open, write, close, free) } - } - - /// Growing memory buffer for archive writes with libarchive callbacks. - pub struct GrowingBuffer<'a> { - pub(crate) list: Vec, - pub(crate) had_error: bool, - /// When set, every write goes to this callback instead of `list`, so `list` stays empty. - pub(crate) forward_to: Option<&'a mut dyn FnMut(&[u8])>, - } - - impl<'a> GrowingBuffer<'a> { - pub fn init() -> Self { - GrowingBuffer { - list: Vec::new(), - had_error: false, - forward_to: None, - } - } - - pub fn forward_to(write: &'a mut dyn FnMut(&[u8])) -> Self { - GrowingBuffer { - forward_to: Some(write), - ..Self::init() - } - } - - pub fn to_owned_slice(&mut self) -> core::result::Result, bun_core::OOM> { - if self.had_error { - return Err(bun_core::AllocError); - } - Ok(core::mem::take(&mut self.list)) - } - - pub unsafe extern "C" fn open_callback( - _a: *mut Archive, - client_data: *mut c_void, - ) -> c_int { - // SAFETY: client_data is a *mut GrowingBuffer registered via archive_write_open2. - let this = unsafe { bun_core::callback_ctx::(client_data) }; - this.list.clear(); - this.had_error = false; - 0 - } - - pub unsafe extern "C" fn write_callback( - a: *mut Archive, - client_data: *mut c_void, - buff: *const c_void, - length: usize, - ) -> la_ssize_t { - // SAFETY: client_data is a *mut GrowingBuffer registered via archive_write_open2. - let this = unsafe { bun_core::callback_ctx::(client_data) }; - if buff.is_null() || length == 0 { - return 0; - } - // SAFETY: buff[0..length] is valid for reads per libarchive contract. - let data = unsafe { core::slice::from_raw_parts(buff.cast::(), length) }; - if let Some(write) = this.forward_to.as_deref_mut() { - write(data); - return la_ssize_t::try_from(length).expect("int cast"); - } - if this.list.try_reserve(length).is_err() { - this.had_error = true; - // libarchive sets no error of its own for a failed client write. - // SAFETY: `a` is the archive this callback was invoked for; the - // format string has no conversions, so no varargs are read. - unsafe { archive_set_error(a, bun_sys::E::ENOMEM as c_int, c"No memory".as_ptr()) }; - return -1; - } - this.list.extend_from_slice(data); - la_ssize_t::try_from(length).expect("int cast") - } - - pub unsafe extern "C" fn close_callback( - _a: *mut Archive, - _client_data: *mut c_void, - ) -> c_int { - 0 - } - } - - // ── Archive::Iterator ────────────────────────────────────────────────── - // Thin - // wrapper that opens a tarball from memory and yields one - // `IteratorEntry` per `next()`, used by `bun publish `. -} - -use lib::Archive; - -pub struct BufferReadStream { - buf: *const [u8], - - archive: *mut Archive, -} - -impl BufferReadStream { - /// Construct a stream over `buf`. - /// - /// # Safety - /// `buf` is type-erased to a raw `*const [u8]` (no lifetime parameter on - /// `BufferReadStream` — see field comment). The caller - /// **must** guarantee that the slice `buf` points to remains valid and - /// unmoved for the entire lifetime of the returned `BufferReadStream` - /// (including its `Drop`). Violating this makes [`buf()`], [`buf_left()`], - /// and [`open_read()`] dereference a dangling pointer (UB). - pub(crate) unsafe fn init(buf: &[u8]) -> Self { - // was an out-param constructor (`this.* = ...`) - Self { - buf: std::ptr::from_ref::<[u8]>(buf), - archive: Archive::read_new(), - } - } - - /// Borrow the underlying libarchive handle. - /// - /// SAFETY (invariant): `self.archive` is set to a fresh non-null handle by - /// `Archive::read_new()` in `init()` (asserted there) and remains valid - /// until `read_free()` in `Drop`. All `Archive` methods take `&self` - /// (FFI interior mutability), so a shared borrow is sufficient. - #[inline] - fn archive(&self) -> &Archive { - // SAFETY: see doc comment — non-null for the lifetime of `self`. - unsafe { &*self.archive } - } - - /// Borrow the input buffer. - /// - /// SAFETY (invariant): `self.buf` is a fat pointer captured from the - /// `&[u8]` passed to `init()`; the caller guarantees it outlives `self` - /// (see field comment). Never null, never mutated. - #[inline] - fn buf(&self) -> &[u8] { - // SAFETY: see doc comment — borrowed for `self`'s lifetime. - unsafe { &*self.buf } - } - - /// `npm` makes libarchive read the tarball the way npm's own tar reader does. See - /// patches/libarchive/tar-npm-compat.patch. - pub(crate) fn open_read(&mut self, npm: bool) -> lib::Result { - let archive = self.archive(); - - let _ = archive.read_support_format_tar(); - let _ = archive.read_support_format_gnutar(); - let _ = archive.read_support_filter_gzip(); - - // Ignore zeroed blocks in the archive, which occurs when multiple tar archives - // have been concatenated together. - // Without this option, only the contents of - // the first concatenated archive would be read. - let _ = archive.read_set_options(if npm { - c"npm" - } else { - c"read_concatenated_archives" - }); - - archive.read_open_memory(self.buf()) - } -} - -impl Drop for BufferReadStream { - fn drop(&mut self) { - let _ = self.archive().read_close(); - let _ = self.archive().read_free(); - } -} - -/// `mkdirat` mode for a directory entry. `perm` is whatever the header -/// encoded (a GNU base-256 field need not fit in `0o7777`), so mask it, then -/// make readable directories listable like node-tar does: -/// https://github.com/npm/node-tar/blob/main/lib/mode-fix.js -#[cfg(not(windows))] -pub fn directory_mode(perm: bun_sys::Mode) -> bun_sys::Mode { - let mut mode = perm & 0o7777; - if (mode & 0o400) != 0 { - mode |= 0o100; - } - if (mode & 0o40) != 0 { - mode |= 0o10; - } - if (mode & 0o4) != 0 { - mode |= 0o1; - } - mode -} - -/// Returns the path where node-tar would extract the npm tarball entry `path`, whose leading -/// `package/` is already removed. The result is empty for an entry that node-tar skips. -pub fn normalize_npm_entry_path<'a>( - path: &[OSPathChar], - buf: &'a mut [OSPathChar], -) -> &'a mut [OSPathChar] { - if path - .split(|c| bun_paths::is_sep_any_t(*c)) - .any(|part| part == [b'.' as OSPathChar; 2]) - { - return &mut buf[..0]; - } - #[cfg(windows)] - return bun_paths::resolve_path::normalize_buf_t::(path, buf); - #[cfg(not(windows))] - bun_paths::resolve_path::normalize_string_generic_t::(path, buf, b'/', |c| { - c == b'/' - }) -} - -/// Returns the first component of an entry's path, which is the archive's root folder. -pub fn first_path_component(path: &[OSPathChar]) -> &[OSPathChar] { - let sep = b'/' as OSPathChar; - let start = path.iter().take_while(|c| **c == sep).count(); - let rest = &path[start..]; - &rest[..strings::index_of_scalar(rest, sep).unwrap_or(rest.len())] -} - -/// Validates that a symlink target doesn't escape the extraction directory. -/// Returns true if the symlink is safe (target stays within extraction dir), -/// false if it would escape (e.g., via ../ traversal or absolute path). -/// -/// The check works by normalizing `symlink_dir/link_target` as a relative -/// path with leading `..` preserved; the target is unsafe if the result -/// climbs above the extraction root. -#[cfg(unix)] -pub fn is_symlink_target_safe( - symlink_path: &[u8], - link_target: &ZStr, - symlink_join_buf: &mut Option, -) -> bool { - // Absolute symlink targets are never safe - they could point anywhere - let link_target_bytes = link_target.as_bytes(); - if link_target_bytes.is_empty() || link_target_bytes[0] == b'/' { - return false; - } - - let mut seen_named_component = false; - for component in strings::split(link_target_bytes, b"/") { - match component { - b"" | b"." => {} - b".." => { - if seen_named_component { - return false; - } - } - _ => seen_named_component = true, - } - } - - // Get the directory containing the symlink - let symlink_dir = bun_paths::dirname_simple(symlink_path); - - let join_buf: &mut PathBuffer = - symlink_join_buf.get_or_insert_with(bun_paths::path_buffer_pool::get); - - // Normalize symlink_dir/link_target as a relative path. An absolute fake - // root cannot be used here: POSIX normalization clamps excess `..` at `/`, - // so a target like `../../../packages/x` would normalize back under any - // fake root that happens to match a real ancestor directory name. - if symlink_dir.len() + 1 + link_target_bytes.len() >= join_buf.len() { - return false; - } - let mut written = 0usize; - if !symlink_dir.is_empty() { - join_buf[..symlink_dir.len()].copy_from_slice(symlink_dir); - written = symlink_dir.len(); - join_buf[written] = b'/'; - written += 1; - } - join_buf[written..written + link_target_bytes.len()].copy_from_slice(link_target_bytes); - written += link_target_bytes.len(); - - let mut norm_buf = bun_paths::path_buffer_pool::get(); - let resolved = bun_paths::resolve_path::normalize_string_generic_t::( - &join_buf[..written], - &mut norm_buf[..], - b'/', - |c| c == b'/', - ); - - !(strings::eql(resolved, b"..") || strings::has_prefix_comptime(resolved, b"../")) -} - -#[cfg(unix)] -pub struct DeferredSymlink { - path: bun_core::ZBox, - target: bun_core::ZBox, -} - -#[cfg(unix)] -impl DeferredSymlink { - pub fn new(path: &[u8], target: &[u8]) -> Self { - Self { - path: bun_core::ZBox::from_bytes(path), - target: bun_core::ZBox::from_bytes(target), - } - } -} - -#[cfg(unix)] -fn open_dir_with_stat(dir_fd: Fd, sub_path: &[u8]) -> Option<(Fd, bun_sys::Stat)> { - let fd = bun_sys::open_dir_at(dir_fd, sub_path).ok()?; - match bun_sys::fstat(fd) { - Ok(st) => Some((fd, st)), - Err(_) => { - fd.close(); - None - } - } -} - -#[cfg(unix)] -pub fn create_deferred_symlinks(dir_fd: Fd, symlinks: &[DeferredSymlink], log: bool) -> u32 { - let mut parents: Vec> = Vec::with_capacity(symlinks.len()); - for symlink in symlinks { - let dirname = bun_paths::dirname_simple(symlink.path.as_bytes()); - if dirname.is_empty() { - parents.push(None); - continue; - } - let _ = dir_fd.make_path_u8(dirname); - parents.push(open_dir_with_stat(dir_fd, dirname).map(|(fd, st)| { - fd.close(); - st - })); - } - - let mut created: u32 = 0; - for (symlink, expected) in symlinks.iter().zip(parents) { - let dirname = bun_paths::dirname_simple(symlink.path.as_bytes()); - let target = symlink.target.as_zstr(); - let result = if dirname.is_empty() { - bun_sys::symlinkat(target, dir_fd, symlink.path.as_zstr()) - } else { - let name = - ZStr::from_slice_with_nul(&symlink.path.as_bytes_with_nul()[dirname.len() + 1..]); - match (open_dir_with_stat(dir_fd, dirname), expected) { - (Some((parent, st)), Some(expected)) - if st.st_dev == expected.st_dev && st.st_ino == expected.st_ino => - { - let result = bun_sys::symlinkat(target, parent, name); - parent.close(); - result - } - (parent, _) => { - if let Some((parent, _)) = parent { - parent.close(); - } - if log { - bun_core::warn!( - "Skipping symlink whose parent directory changed during extraction: {}\n", - bun_core::fmt::escape_control_chars(symlink.path.as_bytes()), - ); - } - continue; - } - } - }; - match result { - Ok(()) => created += 1, - Err(_) => { - if log { - bun_core::warn!( - "Skipping symlink that could not be created: {} -> {}\n", - bun_core::fmt::escape_control_chars(symlink.path.as_bytes()), - bun_core::fmt::escape_control_chars(symlink.target.as_bytes()), - ); - } - } - } - } - created -} - -/// Recursive mkdir over a WTF-16 path: component-iterates the -/// wide path and `NtCreateFile`s each prefix with `FILE_OPEN_IF`, walking back -/// on `FileNotFound` and forward again on success. This stays in WTF-16 -/// throughout (no UTF-8 round-trip — `bun_sys::make_path_w` is the *different* -/// `bun.makePathW` helper which transcodes via `from_w_path` and would lose -/// lone surrogates / skip `\??\` long-path prefixing). -#[cfg(windows)] -fn make_path_u16(dir_fd: Fd, sub_path: &[u16]) -> crate::Result<()> { - use bun_sys::{E, WindowsOpenDirOp, WindowsOpenDirOptions, open_dir_at_windows}; - // Access mask (`STANDARD_RIGHTS_READ | FILE_READ_ATTRIBUTES | - // FILE_READ_EA | SYNCHRONIZE | FILE_TRAVERSE`) is selected by setting `read_only`, - // and `FILE_OPEN_IF` via `OpenOrCreate`. - let opts = WindowsOpenDirOptions { - op: WindowsOpenDirOp::OpenOrCreate, - ..Default::default() - }; - // tar entry paths are dir-relative (no drive/UNC/`\??\`) so `init` never - // returns BadPathName here. - let it = bun_paths::ComponentIterator::init(sub_path, bun_paths::PathFormat::Windows)?; - bun_paths::make_path_with(it, |prefix| { - match open_dir_at_windows(dir_fd, prefix, opts) { - Ok(fd) => { - fd.close(); - Ok(bun_paths::MakePathStep::Created) - } - Err(e) if e.get_errno() == E::ENOENT => Ok(bun_paths::MakePathStep::NotFound(e.into())), - Err(e) => Err(e.into()), - } - }) -} - -struct OutputFile { - fd: Fd, - close: bool, -} - -impl Drop for OutputFile { - fn drop(&mut self) { - if self.close { - // Closing is slow under Windows antivirus, but closing asynchronously gains nothing and - // can make NtSetInformationFile fail. - self.fd.close(); - } - } -} - -pub struct Archiver; - -pub mod archiver { - use super::*; - - pub struct Context { - pub pluckers: Vec, - pub overwrite_list: StringArrayHashMap<()>, - } - - pub struct Plucker { - pub contents: MutableString, - pub(crate) filename_hash: u64, - pub found: bool, - pub fd: Fd, - } - - impl Plucker { - pub fn init(filepath: &[OSPathChar], estimated_size: usize) -> crate::Result { - Ok(Plucker { - contents: MutableString::init(estimated_size)?, - filename_hash: hash(slice_as_bytes(filepath)), - fd: Fd::INVALID, - found: false, - }) - } - } - - #[derive(Clone, Copy)] - pub struct ExtractOptions { - pub depth_to_skip: usize, - pub close_handles: bool, - pub log: bool, - pub npm: bool, - pub skip_symlinks: bool, - } - - impl Default for ExtractOptions { - fn default() -> Self { - Self { - depth_to_skip: 0, - close_handles: true, - log: false, - npm: false, - skip_symlinks: false, - } - } - } -} - -pub use archiver::{Context, ExtractOptions, Plucker}; - -pub trait ArchiveAppender { - /// Mirrors `@hasDecl(Child, "onFirstDirectoryName")`. - const HAS_ON_FIRST_DIRECTORY_NAME: bool = false; - - fn needs_first_dirname(&self) -> bool { - false - } - fn on_first_directory_name(&mut self, _name: &[u8]) {} - - fn append(&mut self, path: &[u8]) -> crate::Result<&[u8]> { - let _ = path; - unreachable!() - } -} - -impl ArchiveAppender for () {} - -impl Archiver { - pub fn get_overwriting_file_list( - file_buffer: &[u8], - root: &[u8], - ctx: &mut Context, - appender: &mut A, - // Called with the path of every entry below `root`. If it returns true, the entry is listed - // whatever its size. - leaves_root: &mut dyn FnMut(&[u8]) -> bun_sys::Result, - ) -> crate::Result<()> { - let mut entry: *mut lib::Entry = ptr::null_mut(); - - // SAFETY: `file_buffer` outlives `stream` (stack-local, dropped at fn exit). - let mut stream = unsafe { BufferReadStream::init(file_buffer) }; - let _ = stream.open_read(false); - let archive = stream.archive; - - // Uses the bun_sys directory-fd helpers (open_dir_absolute / open_dir_at). - let dir: Fd = 'brk: { - let cwd = Fd::cwd(); - - // if the destination doesn't exist, we skip the whole thing since nothing can overwrite it. - if bun_paths::is_absolute(root) { - let Ok(d) = bun_sys::open_dir_absolute(root) else { - return Ok(()); - }; - break 'brk d; - } else { - let Ok(d) = bun_sys::open_dir_at(cwd, root) else { - return Ok(()); - }; - break 'brk d; - } - }; - let _close_dir = bun_sys::CloseOnDrop::new(dir); - - let mut normalized_buf = bun_paths::path_buffer_pool::get(); - - 'loop_: loop { - // SAFETY: archive valid for stream lifetime - let r = unsafe { (*archive).read_next_header(&mut entry) }; - - match r { - lib::Result::Eof => break 'loop_, - lib::Result::Retry => continue 'loop_, - lib::Result::Failed | lib::Result::Fatal => { - return Err(crate::Error::Fail); - } - _ => { - // do not use the utf8 name there - // it will require us to pull in libiconv - // though we should probably validate the utf8 here nonetheless - // SAFETY: entry was just populated by read_next_header - #[cfg(not(windows))] - let pathname_bytes = lib::Entry::opaque_ref(entry).pathname().as_bytes(); - // The name `extract_to_dir` writes to. - #[cfg(windows)] - let pathname_utf8 = strings::to_utf8_list_with_type( - Vec::new(), - lib::Entry::opaque_ref(entry).pathname_w().as_slice(), - )?; - #[cfg(windows)] - let pathname_bytes: &[u8] = &pathname_utf8; - - // Tokenizer semantics: `next()` skips leading - // separators and returns None only when no token remains; - // the rest-of-input slice also skips leading separators. - // An entry's name has `/` on every platform, as in `extract_to_dir`. - let mut remaining = pathname_bytes; - let mut depth_i = 0usize; - while depth_i < DEPTH_TO_SKIP { - // skip leading separators - while let [first, rest @ ..] = remaining { - if *first == b'/' { - remaining = rest; - } else { - break; - } - } - if remaining.is_empty() { - continue 'loop_; - } - match strings::index_of_char_usize(remaining, b'/') { - Some(i) => remaining = &remaining[i..], - None => remaining = &remaining[remaining.len()..], - } - depth_i += 1; - } - // skip leading separators (tokenizer.rest() does this) - while let [first, rest @ ..] = remaining { - if *first == b'/' { - remaining = rest; - } else { - break; - } - } - - // pathname = sliceTo(remaining[..len :0], 0) - let pathname = slice_to_nul(remaining); - if pathname.is_empty() || pathname.len() >= normalized_buf.len() { - continue 'loop_; - } - let normalized = bun_paths::resolve_path::normalize_buf_t::< - u8, - bun_paths::platform::Auto, - >(pathname, &mut normalized_buf[..]); - let normalized_len = normalized.len(); - let pathname: &[u8] = &normalized_buf[..normalized_len]; - if pathname.is_empty() || pathname == b"." { - continue 'loop_; - } - #[cfg(windows)] - if bun_paths::is_absolute_windows(pathname) { - continue 'loop_; - } - let dirname = - strings::trim(bun_paths::dirname_simple(pathname), SEP_STR.as_bytes()); - - // SAFETY: entry valid - let size: usize = - usize::try_from(lib::Entry::opaque_ref(entry).size().max(0)).unwrap(); - let in_the_way = leaves_root(pathname)? - || (size > 0 - && match bun_sys::openat_a(dir, pathname, bun_sys::O::WRONLY, 0) { - Ok(opened) => { - let _close = bun_sys::CloseOnDrop::new(opened); - bun_sys::get_file_size(opened)? > 0 - } - Err(_) => false, - }); - - if in_the_way { - let is_already_top_level = dirname.is_empty(); - let path_to_use_: &[u8] = 'brk: { - let __pathname: &[u8] = pathname; - - if is_already_top_level { - break 'brk __pathname; - } - - let index = strings::index_of_char_usize(__pathname, SEP).unwrap(); - break 'brk &__pathname[..index]; - }; - let mut temp_buf = [0u8; 1024]; - temp_buf[..path_to_use_.len()].copy_from_slice(path_to_use_); - let path_to_use: &[u8] = if !is_already_top_level { - temp_buf[path_to_use_.len()] = SEP; - &temp_buf[..path_to_use_.len() + 1] - } else { - &temp_buf[..path_to_use_.len()] - }; - - let overwrite_entry = ctx.overwrite_list.get_or_put(path_to_use)?; - if !overwrite_entry.found_existing { - *overwrite_entry.key_ptr = Box::from(appender.append(path_to_use)?); - } - } - } - } - } - - Ok(()) - } - - pub fn extract_to_dir( - file_buffer: &[u8], - dir: Fd, - ctx: Option<&mut Context>, - appender: &mut A, - options: ExtractOptions, - ) -> crate::Result { - let mut entry: *mut lib::Entry = ptr::null_mut(); - - // SAFETY: `file_buffer` outlives `stream` (stack-local, dropped at fn exit). - let mut stream = unsafe { BufferReadStream::init(file_buffer) }; - let _ = stream.open_read(options.npm); - let archive = stream.archive; - let mut count: u32 = 0; - let dir_fd = dir; - - // reshaped for borrowck — ctx is Option<&mut>, rebound as needed - let mut ctx = ctx; - - #[cfg(unix)] - let mut symlink_join_buf: Option = None; - - #[cfg(unix)] - let mut deferred_symlinks: Vec = Vec::new(); - - let mut normalized_buf = bun_paths::os_path_buffer_pool::get(); - let mut write_strategy = lib::WriteStrategy::default(); - - 'loop_: loop { - // SAFETY: archive valid for stream lifetime - let r = unsafe { (*archive).read_next_header(&mut entry) }; - - match r { - lib::Result::Eof => break 'loop_, - lib::Result::Retry => continue 'loop_, - lib::Result::Failed | lib::Result::Fatal => { - return Err(crate::Error::Fail); - } - _ => { - // TODO: - // Due to path separator replacement and other copies that happen internally, libarchive changes the - // storage type of paths on windows to wide character strings. Using `archive_entry_pathname` or `archive_entry_pathname_utf8` - // on an wide character string will return null if there are non-ascii characters. - // (this can be seen by installing @fastify/send, which has a path "@fastify\send\test\fixtures\snow ☃") - // - // Ideally, we find a way to tell libarchive to not convert the strings to wide characters and also to not - // replace path separators. We can do both of these with our own normalization and utf8/utf16 string conversion code. - // SAFETY: entry was just populated by read_next_header - #[cfg(windows)] - let pathname_z = lib::Entry::opaque_ref(entry).pathname_w(); - // SAFETY: entry was just populated by read_next_header - #[cfg(not(windows))] - let pathname_z = lib::Entry::opaque_ref(entry).pathname(); - - if A::HAS_ON_FIRST_DIRECTORY_NAME { - if appender.needs_first_dirname() { - let root = first_path_component(&pathname_z[..]); - #[cfg(windows)] - { - let result = strings::to_utf8_list_with_type(Vec::new(), root)?; - // onFirstDirectoryName copies the contents of pathname to another buffer, safe to free - appender.on_first_directory_name(&result); - } - #[cfg(not(windows))] - { - appender.on_first_directory_name(root); - } - } - } - - // SAFETY: entry valid - let kind = bun_sys::kind_from_mode(lib::Entry::opaque_ref(entry).filetype()); - - if options.npm { - // - ignore entries other than files (`true` can only be returned if type is file) - // https://github.com/npm/cli/blob/93883bb6459208a916584cad8c6c72a315cf32af/node_modules/pacote/lib/fetcher.js#L419-L441 - if kind != bun_sys::FileKind::File { - continue; - } - - // TODO: .npmignore, or .gitignore if it doesn't exist - // https://github.com/npm/cli/blob/93883bb6459208a916584cad8c6c72a315cf32af/node_modules/pacote/lib/fetcher.js#L434 - } - - if options.skip_symlinks && kind == bun_sys::FileKind::SymLink { - continue; - } - - // strip and normalize the path - // `pathname_z` is `&ZStr` on POSIX (`as_bytes() → &[u8]`) - // and `&WStr` on Windows (`as_slice() → &[u16]`); both - // deref to `&[OSPathChar]`. - let pathname_slice: &[OSPathChar] = &pathname_z[..]; - let mut remaining: &[OSPathChar] = pathname_slice; - { - let sep: OSPathChar = b'/' as OSPathChar; - let mut i = 0usize; - while i < options.depth_to_skip { - // npm counts the empty name before a leading `/` as a component. - while let [first, rest @ ..] = remaining { - if *first == sep && !options.npm { - remaining = rest; - } else { - break; - } - } - if remaining.is_empty() { - continue 'loop_; - } - match strings::index_of_scalar(remaining, sep) { - Some(j) => remaining = &remaining[j + 1..], - None => remaining = &remaining[remaining.len()..], - } - i += 1; - } - while let [first, rest @ ..] = remaining { - if *first == sep { - remaining = rest; - } else { - break; - } - } - } - // pathname = rest.ptr[0..rest.len :0] (NUL is at original buffer end) - // SAFETY: `remaining` is a tail slice of `pathname_z`, which is NUL-terminated - // at its original `.len()`; therefore `remaining[remaining.len()] == 0`. - let pathname: &[OSPathChar] = remaining; - - if pathname.len() >= normalized_buf.len() { - if options.log { - bun_core::warn!( - "Skipping entry with a path longer than the maximum path length: {}\n", - bun_core::fmt::EscapeControlChars(bun_core::fmt::fmt_os_path( - pathname, - Default::default() - )), - ); - } - continue; - } - - let normalized = if options.npm { - normalize_npm_entry_path(pathname, &mut normalized_buf[..]) - } else { - bun_paths::resolve_path::normalize_buf_t::< - OSPathChar, - bun_paths::platform::Auto, - >(pathname, &mut normalized_buf[..]) - }; - let normalized_len = normalized.len(); - normalized_buf[normalized_len] = 0; - // SAFETY: we just wrote a NUL at normalized_buf[normalized_len] - let path: &mut [OSPathChar] = &mut normalized_buf[..normalized_len]; - if path.is_empty() || (path.len() == 1 && path[0] == b'.' as OSPathChar) { - continue; - } - - // Skip entries whose normalized path is absolute on Windows. - // `openatWindows` ignores `dir_fd` for absolute inputs (drive - // letter or UNC), so without this guard a tar entry could - // resolve outside the extraction directory. On POSIX the - // tokenize-on-'/' step already strips any leading separators, - // so `normalizeBufT` cannot produce an absolute output. - #[cfg(windows)] - { - if bun_paths::is_absolute_windows_t::(path) { - continue 'loop_; - } - } - - #[cfg(windows)] - if options.npm { - // When writing files on Windows, translate the characters to their - // 0xf000 higher-encoded versions. - // https://github.com/isaacs/node-tar/blob/0510c9ea6d000c40446d56674a7efeec8e72f052/lib/winchars.js - let mut remain: &mut [OSPathChar] = path; - if strings::starts_with_windows_drive_letter_t::(remain) { - // don't encode `:` from the drive letter - // https://github.com/npm/cli/blob/93883bb6459208a916584cad8c6c72a315cf32af/node_modules/tar/lib/unpack.js#L327 - remain = &mut remain[2..]; - } - - for ch in remain.iter_mut() { - match *ch { - c if c == b'|' as OSPathChar - || c == b'<' as OSPathChar - || c == b'>' as OSPathChar - || c == b'?' as OSPathChar - || c == b':' as OSPathChar => - { - *ch += 0xf000; - } - _ => {} - } - } - } - - let path_slice: &[OSPathChar] = &path[..]; - - if options.log { - bun_core::prettyln!( - " {}", - bun_core::fmt::EscapeControlChars(bun_core::fmt::fmt_os_path( - path_slice, - Default::default() - )) - ); - } - - count += 1; - - match kind { - bun_sys::FileKind::Directory => { - #[cfg(windows)] - { - make_path_u16(dir, path_slice)?; - } - #[cfg(not(windows))] - { - // SAFETY: entry valid - let mode = directory_mode(lib::Entry::opaque_ref(entry).perm()); - // SAFETY: normalized_buf[path_slice.len()] == 0 (written above), - // so path_slice is a NUL-terminated [:0]u8. - let path_z: &ZStr = unsafe { - ZStr::from_raw(path_slice.as_ptr(), path_slice.len()) - }; - match bun_sys::mkdirat_z(dir_fd, path_z, mode) { - Ok(()) => {} - Err(err) => { - // It's possible for some tarballs to return a directory twice, with and - // without `./` in the beginning. So if it already exists, continue to the - // next entry. - match err.get_errno() { - bun_sys::E::EEXIST | bun_sys::E::ENOTDIR => continue, - _ => {} - } - let dirname = bun_paths::dirname_simple(path_slice); - if dirname.is_empty() { - return Err(err.into()); - } - let _ = dir.make_path_u8(dirname); - let _ = bun_sys::mkdirat_z(dir_fd, path_z, 0o777); - } - } - } - } - bun_sys::FileKind::SymLink => { - // SAFETY: entry valid - let link_target = lib::Entry::opaque_ref(entry).symlink(); - #[cfg(unix)] - { - // Validate that the symlink target doesn't escape the extraction directory. - // This prevents path traversal attacks where a malicious tarball creates a symlink - // pointing outside (e.g., to /tmp), then writes files through that symlink. - if !is_symlink_target_safe( - path_slice, - link_target, - &mut symlink_join_buf, - ) { - // Skip symlinks that would escape the extraction directory - if options.log { - bun_core::warn!( - "Skipping symlink with unsafe target: {} -> {}\n", - bun_core::fmt::EscapeControlChars( - bun_core::fmt::fmt_os_path( - path_slice, - Default::default(), - ) - ), - bun_core::fmt::escape_control_chars( - link_target.as_bytes() - ), - ); - } - continue; - } - deferred_symlinks - .push(DeferredSymlink::new(path_slice, link_target.as_bytes())); - } - #[cfg(not(unix))] - { - let _ = link_target; - } - } - bun_sys::FileKind::File => { - // first https://github.com/npm/cli/blob/feb54f7e9a39bd52519221bae4fafc8bc70f235e/node_modules/pacote/lib/fetcher.js#L65-L66 - // this.fmode = opts.fmode || 0o666 - // - // then https://github.com/npm/cli/blob/feb54f7e9a39bd52519221bae4fafc8bc70f235e/node_modules/pacote/lib/fetcher.js#L402-L411 - // - // we simplify and turn it into `entry.mode || 0o666` because we aren't accepting a umask or fmask option. - #[cfg(not(windows))] - let mode: bun_sys::Mode = bun_sys::Mode::try_from( - // SAFETY: entry valid - (lib::Entry::opaque_ref(entry).perm() & 0o777) | 0o666, - ) - .unwrap(); - - let flags = bun_sys::O::WRONLY | bun_sys::O::CREAT | bun_sys::O::TRUNC; - - #[cfg(windows)] - let file_handle_native: Fd = - match bun_sys::openat_windows(dir_fd, path_slice, flags, 0) { - Ok(fd) => fd, - Err(e) => match e.get_errno() { - bun_sys::E::EPERM | bun_sys::E::ENOENT => { - // `Dirname::dirname` strips - // trailing separators. - let Some(dirname) = - bun_paths::Dirname::dirname(path_slice) - else { - return Err(e.into()); - }; - let _ = make_path_u16(dir, dirname); - bun_sys::openat_windows(dir_fd, path_slice, flags, 0)? - } - _ => return Err(e.into()), - }, - }; - - #[cfg(not(windows))] - let file_handle_native: Fd = { - // dir.createFileZ(.{truncate, mode}) → bun_sys::openat - // SAFETY: normalized_buf[path_slice.len()] == 0 (written above). - let path_z: &ZStr = unsafe { - ZStr::from_raw(path_slice.as_ptr(), path_slice.len()) - }; - match bun_sys::openat(dir_fd, path_z, flags, mode) { - Ok(fd) => fd, - Err(err) => match err.get_errno() { - bun_sys::E::EACCES - | bun_sys::E::EPERM - | bun_sys::E::ENOENT => { - let dirname = bun_paths::dirname_simple(path_slice); - if dirname.is_empty() { - return Err(err.into()); - } - let _ = dir.make_path_u8(dirname); - bun_sys::openat(dir_fd, path_z, flags, mode)? - } - _ => return Err(err.into()), - }, - } - }; - - let mut output = match file_handle_native.make_lib_uv_owned() { - Ok(fd) => OutputFile { - fd, - close: options.close_handles, - }, - Err(err) => { - file_handle_native.close(); - return Err(err.into()); - } - }; - - // SAFETY: entry valid - let size: usize = - usize::try_from(lib::Entry::opaque_ref(entry).size().max(0)) - .unwrap(); - - if size > 0 { - if let Some(ctx_) = ctx.as_deref_mut() { - let h: u64 = if !ctx_.pluckers.is_empty() { - hash(slice_as_bytes(path_slice)) - } else { - 0u64 - }; - - for plucker_ in ctx_.pluckers.iter_mut() { - if plucker_.filename_hash == h { - plucker_.contents.list.clear(); - // SAFETY: archive valid - let read_ok = unsafe { &*archive }.read_data_to_vec( - size, - &mut plucker_.contents.list, - )?; - if !read_ok { - if options.log { - // SAFETY: `archive` is the live - // `read_new()` handle this - // extraction loop is iterating. - let archive_error = slice_to_nul( - unsafe { &*archive }.error_string(), - ); - Output::err( - "libarchive error", - "extracting {}: {}", - ( - bun_core::fmt::fmt_os_path( - path_slice, - Default::default(), - ), - bstr::BStr::new(archive_error), - ), - ); - } - return Err(crate::Error::Fail); - } - plucker_.found = !plucker_.contents.list.is_empty(); - plucker_.fd = output.fd; - output.close = false; - continue 'loop_; - } - } - } - let mut retries_remaining: u8 = 5; - - 'possibly_retry: while retries_remaining != 0 { - // SAFETY: archive valid - match unsafe { - (*archive).read_data_into_fd(output.fd, &mut write_strategy) - } { - lib::Result::Eof => break 'loop_, - lib::Result::Ok => break 'possibly_retry, - lib::Result::Retry => { - if options.log { - Output::err( - "libarchive error", - "extracting {}, retry {} / {}", - ( - bun_core::fmt::fmt_os_path( - path_slice, - Default::default(), - ), - retries_remaining, - 5, - ), - ); - } - } - _ => { - if options.log { - // SAFETY: `archive` is the live - // `read_new()` handle this - // extraction loop is iterating. - let archive_error = slice_to_nul( - unsafe { &*archive }.error_string(), - ); - Output::err( - "libarchive error", - "extracting {}: {}", - ( - bun_core::fmt::fmt_os_path( - path_slice, - Default::default(), - ), - bstr::BStr::new(archive_error), - ), - ); - } - return Err(crate::Error::Fail); - } - } - retries_remaining -= 1; - } - } - } - _ => {} - } - } - } - } - - #[cfg(unix)] - create_deferred_symlinks(dir_fd, &deferred_symlinks, options.log); - - Ok(count) - } - - pub fn extract_to_disk( - file_buffer: &[u8], - root: &[u8], - ctx: Option<&mut Context>, - appender: &mut A, - options: ExtractOptions, - ) -> crate::Result { - let dir: Fd = 'brk: { - let cwd = Fd::cwd(); - let _ = cwd.make_path_u8(root); - - if bun_paths::is_absolute(root) { - break 'brk bun_sys::open_dir_absolute(root)?; - } else { - break 'brk bun_sys::open_dir_at(cwd, root)?; - } - }; - - let _close_guard = scopeguard::guard(dir, |d| { - if options.close_handles { - d.close(); - } - }); - - Self::extract_to_dir(file_buffer, dir, ctx, appender, options) - } -} diff --git a/src/opaque/lib.rs b/src/opaque/lib.rs index 246284c5fb2a..e63b38750d22 100644 --- a/src/opaque/lib.rs +++ b/src/opaque/lib.rs @@ -339,7 +339,7 @@ pub mod ffi { /// excluding) the first NUL. Single audited funnel for the hand-rolled /// `while *p.add(n) != 0 { n += 1 }` loop that appeared at every /// `LPCWSTR` / `char16_t*` ingestion point (Windows path APIs, N-API - /// `napi_create_string_utf16`, libarchive `_w` accessors, env-block + /// `napi_create_string_utf16`, env-block /// scan). Adds a `debug_assert!(!p.is_null())` — same precondition as /// `CStr::from_ptr`. /// diff --git a/src/paths/component_iterator.rs b/src/paths/component_iterator.rs index 48b7632b2ff7..a76d4085be71 100644 --- a/src/paths/component_iterator.rs +++ b/src/paths/component_iterator.rs @@ -6,7 +6,7 @@ //! This is the single source of truth for "split a path into prefix slices and //! never yield the Windows root (`C:`, `C:\`, `\\server\share\`) as a component". //! Replaces the four hand-rolled copies in `bun_sys` (posix+windows), -//! `bun_libarchive` (u16) and `bun` (`make_path`, which already called the +//! `bun_archive` (u16) and `bun` (`make_path`, which already called the //! `component_iterator` free fn below before it existed). use crate::PathChar; diff --git a/src/runtime/Cargo.toml b/src/runtime/Cargo.toml index 4127e3bb18e1..b14bfc41c456 100644 --- a/src/runtime/Cargo.toml +++ b/src/runtime/Cargo.toml @@ -71,7 +71,7 @@ bun_format_conformance.workspace = true bun_sema_baselines.workspace = true bun_js_printer.workspace = true bun_jsc.workspace = true -bun_libarchive.workspace = true +bun_archive.workspace = true bun_libdeflate_sys.workspace = true bun_ast.workspace = true bun_ast_jsc.workspace = true diff --git a/src/runtime/api/Archive.rs b/src/runtime/api/Archive.rs index cf70e4fff35b..b507d8f05ccb 100644 --- a/src/runtime/api/Archive.rs +++ b/src/runtime/api/Archive.rs @@ -1,23 +1,17 @@ -//! `Bun.Archive` — tar/tgz pack + extract over libarchive. - -use std::ffi::CString; +//! `Bun.Archive` — tar/tgz pack + extract. use crate::webcore::Blob; use crate::webcore::BlobExt as _; use crate::webcore::blob::Store; -use bun_core::{self, EncodedSlice, Output, Utf8Bytes, ZBox, strings}; +use bun_archive::tar; +use bun_core::{self, Output, Utf8Bytes, ZBox, strings}; use bun_glob as glob; use bun_jsc::bindgen::{JsLocal, JsScope}; use bun_jsc::{self as jsc, JSGlobalObject, JSMap, JSPromise, JSPromiseStrong, JSValue, JsResult}; -use bun_jsc::{EncodedSliceJsc as _, StringJsc as _, SysErrorJsc as _}; -use bun_libarchive as libarchive; +use bun_jsc::{StringJsc as _, SysErrorJsc as _}; use bun_ptr::RefPtr; use bun_sys::{self, Fd, FdDirExt as _, FdExt as _, Mode}; -/// libarchive `AE_IFREG` (== `S_IFREG`). The Rust `bun_libarchive::lib` port -/// does not yet expose `FileType`, so mirror the constant locally. -const FILETYPE_REGULAR: u32 = 0o100000; - /// Compression options for the archive #[derive(Clone, Copy, Default)] pub(crate) enum Compression { @@ -94,40 +88,17 @@ impl Archive { } } -/// Configure archive for reading tar/tar.gz -fn configure_archive_reader(archive: &libarchive::lib::Archive) { - let _ = archive.read_support_format_tar(); - let _ = archive.read_support_format_gnutar(); - let _ = archive.read_support_filter_gzip(); - let _ = archive.read_set_options(c"read_concatenated_archives"); -} - -/// Entry pathname as owned UTF-8 bytes. libarchive on Windows keeps a -/// charset-converted name (every pax `path=`) only in the wide-string slot; -/// `archive_entry_pathname` lossily narrows that through the "C" locale. -#[cfg(windows)] -fn entry_pathname_utf8(entry: &libarchive::lib::Entry) -> Result, bun_alloc::AllocError> { - bun_core::strings::to_utf8_list_with_type(Vec::new(), entry.pathname_w().as_slice()) -} - /// Count the number of files in an archive fn count_files_in_archive(data: &[u8]) -> u32 { - use libarchive::lib; - let archive = lib::ReadArchive::new(); - configure_archive_reader(&archive); - - if archive.read_open_memory(data) != lib::Result::Ok { + let Ok(mut reader) = bun_archive::open_memory(data, tar::Dialect::General) else { return 0; - } - + }; let mut count: u32 = 0; - let mut entry: *mut lib::Entry = core::ptr::null_mut(); - while archive.read_next_header(&mut entry).succeeded() { - if lib::Entry::opaque_ref(entry).filetype() == FILETYPE_REGULAR { + while let Ok(Some(entry)) = reader.next_entry() { + if entry.kind() == bun_sys::FileKind::File { count += 1; } } - count } @@ -250,45 +221,17 @@ fn blob_from_js(value: JSValue) -> Option<&'static Blob> { /// Shared helper that builds tarball bytes from a JS object fn build_tarball_from_object(global: &JSGlobalObject, obj: JSValue) -> JsResult> { - use libarchive::lib; - let Some(js_obj) = obj.get_object() else { return Err(global.throw_invalid_arguments(format_args!("Expected an object"))); }; - // Set up archive first - let mut growing_buffer = lib::GrowingBuffer::init(); - // errdefer growing_buffer.deinit() — handled by Drop on Vec - - let archive = lib::WriteArchive::new(); - let archive_ref: &lib::Archive = &archive; - - if archive_ref.write_set_format_pax_restricted() != lib::Result::Ok { - return Err(global.throw_invalid_arguments(format_args!( - "Failed to create tarball: ArchiveFormatError" - ))); - } - - // `archive` is a live `archive_write_new()` handle (see `Archive::write_new` - // above); `growing_buffer` is stack-local and outlives all callback invocations - // (the archive is closed before this fn returns). - let open_rc = lib::archive_write_open2( - &archive, - (&raw mut growing_buffer).cast(), - Some(lib::GrowingBuffer::open_callback), - Some(lib::GrowingBuffer::write_callback), - Some(lib::GrowingBuffer::close_callback), - None, - ); - if open_rc != 0 { - return Err(global - .throw_invalid_arguments(format_args!("Failed to create tarball: ArchiveOpenError"))); - } - - let entry = lib::OwnedEntry::new(); - let entry_ref: &lib::Entry = &entry; + let mut writer = tar::writer::Writer::new(Vec::::new(), tar::writer::Names::for_host()); + // Writing to a `Vec` fails for want of memory only. + let fail = |_: tar::Error| { + global.throw_invalid_arguments(format_args!("Failed to create tarball: OutOfMemory")) + }; - let now_secs: isize = isize::try_from(bun_core::time::milli_timestamp() / 1000).unwrap_or(0); + let now_secs: i64 = bun_core::time::milli_timestamp() / 1000; // Iterate over object properties and write directly to archive let iter = jsc::JSPropertyIterator::init( @@ -305,61 +248,25 @@ fn build_tarball_from_object(global: &JSGlobalObject, obj: JSValue) -> JsResult< continue; } - // Get the key as a null-terminated string let key_slice = key.to_utf8(); - let key_str = ZBox::from_vec_with_nul(key_slice.slice().to_vec()); // Get data - use view for Blob/ArrayBuffer, convert for strings let mut array_buffer = None; let data_slice = get_entry_data(global, value, &mut array_buffer)?; - - // Write entry to archive let data = data_slice.slice(); - let _ = entry_ref.clear(); - // Same platform split as `pack_command::add_archive_entry`: the process - // locale is always "C", so libarchive's locale-keyed pax writer is only - // lossless with raw bytes on POSIX and with the UTF-8 form on Windows. - #[cfg(windows)] - entry_ref.set_pathname_utf8(key_str.as_zstr()); - #[cfg(not(windows))] - entry_ref.set_pathname(key_str.as_zstr()); - entry_ref.set_size(i64::try_from(data.len()).expect("int cast")); - entry_ref.set_filetype(FILETYPE_REGULAR); - entry_ref.set_perm(0o644); - entry_ref.set_mtime(now_secs, 0); - - // `Warn` means the header was still written (libarchive fell back to a - // per-entry binary hdrcharset for a name its locale machinery could not - // convert); only `Failed`/`Fatal` mean no header was produced. - if !archive_ref.write_header(entry_ref).succeeded() { - return Err(global.throw_invalid_arguments(format_args!( - "Failed to create tarball: ArchiveHeaderError" - ))); - } - if archive_ref.write_data(data) < 0 { - return Err(global.throw_invalid_arguments(format_args!( - "Failed to create tarball: ArchiveWriteError" - ))); - } - if archive_ref.write_finish_entry() != lib::Result::Ok { - return Err(global.throw_invalid_arguments(format_args!( - "Failed to create tarball: ArchiveFinishEntryError" - ))); - } - } - if archive_ref.write_close() != lib::Result::Ok { - return Err(global - .throw_invalid_arguments(format_args!("Failed to create tarball: ArchiveCloseError"))); + writer + .header(&tar::writer::File { + path: key_slice.slice(), + size: data.len() as u64, + perm: 0o644, + mtime: now_secs, + }) + .map_err(fail)?; + writer.data(data).map_err(fail)?; } - match growing_buffer.to_owned_slice() { - Ok(v) => Ok(v), - Err(_) => { - Err(global - .throw_invalid_arguments(format_args!("Failed to create tarball: OutOfMemory"))) - } - } + writer.finish(true).map_err(fail) } fn get_entry_data<'a>( @@ -717,12 +624,12 @@ impl ExtractContext { } // Otherwise use the fast path without filtering - let count = match libarchive::Archiver::extract_to_disk( + let count = match bun_archive::Archiver::extract_to_disk( self.store.shared_view(), &self.path, None, &mut (), - libarchive::ExtractOptions { + bun_archive::ExtractOptions { depth_to_skip: 0, close_handles: true, log: false, @@ -969,7 +876,7 @@ enum FilesError { enum FilesResult { Success(FileEntryList), - LibarchiveErr(CString), + ArchiveErr(tar::Error), Err(FilesError), } @@ -982,48 +889,31 @@ pub(crate) struct FilesContext { } impl FilesContext { - fn read_error(archive: &libarchive::lib::Archive) -> FilesResult { - let err_str = archive.error_string(); - match CString::new(err_str) { - Ok(err) if !err_str.is_empty() => FilesResult::LibarchiveErr(err), - _ => FilesResult::Err(FilesError::ReadError), - } - } - fn do_run(&mut self) -> Result { - use libarchive::lib; - let archive = lib::ReadArchive::new(); - configure_archive_reader(&archive); - - if archive.read_open_memory(self.store.shared_view()) != lib::Result::Ok { - return Ok(Self::read_error(&archive)); - } + let read_error = |err: tar::Error| match err { + tar::Error::Alloc(err) => Err(err), + err => Ok(FilesResult::ArchiveErr(err)), + }; + let mut reader = + match bun_archive::open_memory(self.store.shared_view(), tar::Dialect::General) { + Ok(reader) => reader, + Err(err) => return read_error(err), + }; let mut entries: FileEntryList = Vec::new(); - // errdefer freeEntries(&entries) — handled by Drop on `entries` - let mut entry: *mut lib::Entry = core::ptr::null_mut(); loop { - match archive.read_next_header(&mut entry) { - lib::Result::Eof => break, - lib::Result::Retry => continue, - lib::Result::Failed | lib::Result::Fatal => return Ok(Self::read_error(&archive)), - lib::Result::Ok | lib::Result::Warn => {} - } - let entry_ref = lib::Entry::opaque_ref(entry); - if entry_ref.filetype() != FILETYPE_REGULAR { + let entry_ref = match reader.next_entry() { + Ok(None) => break, + Ok(Some(entry)) => entry, + Err(err) => return read_error(err), + }; + if entry_ref.kind() != bun_sys::FileKind::File { continue; } - // POSIX: the raw header/pax bytes; the locale-converting - // `archive_entry_pathname_utf8` returns NULL for every non-ASCII - // name in the "C" locale, which would key the Map by "". - #[cfg(not(windows))] - let pathname = entry_ref.pathname().as_bytes(); - #[cfg(windows)] - let pathname_owned = entry_pathname_utf8(entry_ref)?; - #[cfg(windows)] - let pathname: &[u8] = &pathname_owned; + let pathname = bun_archive::display_path(entry_ref)?; + let pathname: &[u8] = &pathname; // Apply glob pattern filtering (supports both positive and negative patterns) if let Some(patterns) = &self.glob_patterns { if !match_glob_patterns(patterns, pathname) { @@ -1034,14 +924,13 @@ impl FilesContext { let size: usize = usize::try_from(entry_ref.size().max(0)).expect("int cast"); let mtime: i64 = entry_ref.mtime(); + let path_copy: Box<[u8]> = Box::from(pathname); + let mut data: Vec = Vec::new(); - if !archive.read_data_to_vec(size, &mut data)? { - return Ok(Self::read_error(&archive)); + if let Err(err) = reader.read_data(size, &mut data) { + return read_error(err); } - let path_copy: Box<[u8]> = Box::from(pathname); - // errdefer free(path_copy) — handled by Drop - entries.push(FileEntry { path: path_copy, data, @@ -1089,8 +978,8 @@ impl TaskContext for FilesContext { Ok(PromiseResult::Resolve(map)) } - FilesResult::LibarchiveErr(err_msg) => Ok(PromiseResult::Reject( - EncodedSlice::utf8(err_msg.to_bytes()).to_error_instance(global), + FilesResult::ArchiveErr(err) => Ok(PromiseResult::Reject( + global.create_error_instance(format_args!("{err}")), )), FilesResult::Err(e) => Ok(PromiseResult::Reject( global.create_error_instance(format_args!("{}", <&'static str>::from(&*e))), @@ -1247,13 +1136,11 @@ fn extract_to_disk_filtered( root: &[u8], glob_patterns: Option<&[GlobPattern]>, ) -> crate::Result { - use libarchive::lib; - let archive = lib::ReadArchive::new(); - configure_archive_reader(&archive); - - if archive.read_open_memory(file_buffer) != lib::Result::Ok { - return Err(crate::Error::ReadError); - } + let mut reader = + bun_archive::open_memory(file_buffer, tar::Dialect::General).map_err(|err| match err { + tar::Error::Alloc(err) => crate::Error::Alloc(err), + _ => crate::Error::ReadError, + })?; // Open/create target directory using bun.sys let cwd = Fd::cwd(); @@ -1279,28 +1166,16 @@ fn extract_to_disk_filtered( let _dir_close = bun_sys::CloseOnDrop::new(dir_fd); let mut count: u32 = 0; - let mut entry: *mut lib::Entry = core::ptr::null_mut(); - let mut write_strategy = lib::WriteStrategy::default(); + let mut write_strategy = bun_archive::WriteStrategy::default(); loop { - match archive.read_next_header(&mut entry) { - lib::Result::Eof => break, - lib::Result::Retry => continue, - lib::Result::Failed | lib::Result::Fatal => return Err(crate::Error::ReadError), - lib::Result::Ok | lib::Result::Warn => {} - } - let entry_ref = lib::Entry::opaque_ref(entry); - // Same platform split as `FilesContext::do_run`; see `entry_pathname_utf8`. - #[cfg(not(windows))] - let raw_pathname_z = entry_ref.pathname(); - #[cfg(windows)] - let raw_pathname_zbox = ZBox::from_vec_with_nul( - entry_pathname_utf8(entry_ref) - .map_err(|_| crate::Error::Alloc(bun_alloc::AllocError))?, - ); - #[cfg(windows)] - let raw_pathname_z = raw_pathname_zbox.as_zstr(); - let raw_pathname = raw_pathname_z.as_bytes(); + let entry_ref = match reader.next_entry() { + Ok(None) => break, + Ok(Some(entry)) => entry, + Err(_) => return Err(crate::Error::ReadError), + }; + let raw_pathname = bun_archive::display_path(entry_ref)?; + let raw_pathname: &[u8] = &raw_pathname; let mut normalized_buf = bun_paths::path_buffer_pool::get(); if raw_pathname.len() >= normalized_buf.len() { @@ -1325,10 +1200,7 @@ fn extract_to_disk_filtered( } } - let filetype = entry_ref.filetype(); - let kind = bun_sys::kind_from_mode(filetype); - - match kind { + match entry_ref.kind() { bun_sys::FileKind::Directory => { match dir_fd.make_path(pathname) { // Directory already exists - don't count as extracted @@ -1373,7 +1245,8 @@ fn extract_to_disk_filtered( }; let write_success = size == 0 - || archive.read_data_into_fd(file_fd, &mut write_strategy) == lib::Result::Ok; + || bun_archive::read_data_into_fd(&mut reader, file_fd, &mut write_strategy) + .is_ok(); let _ = file_fd.close(); if !write_success { @@ -1384,7 +1257,8 @@ fn extract_to_disk_filtered( count += 1; } bun_sys::FileKind::SymLink => { - let link_target_z = entry_ref.symlink(); + let link_target = ZBox::from_bytes(entry_ref.symlink().unwrap_or_default()); + let link_target_z = link_target.as_zstr(); // Validate symlink target is also safe if !is_safe_path(link_target_z.as_bytes()) { continue; @@ -1417,3 +1291,248 @@ fn extract_to_disk_filtered( Ok(count) } + +/// `bun:internal-for-testing` +pub(crate) mod testing { + use crate::generated_bindgen::archive_internals::{ + ReadTarOptions, ReadTarSkip as Skip, WriteTarFile, WriteTarOptions, + }; + use bun_archive::tar::output::Output; + use bun_archive::tar::writer::{File, Names, Writer}; + use bun_archive::tar::{self, Event}; + use bun_core::{S, strings}; + use bun_jsc::bindgen::{self, BufferSource, JsHeap, JsLocal}; + use core::fmt::Write as _; + + enum Source<'a> { + Whole(tar::MemoryDecoder<'a>), + Pieces(tar::Decoder), + } + + fn type_name(filetype: u32) -> Option<&'static str> { + Some(match filetype { + S::IFREG => "file", + S::IFDIR => "directory", + S::IFLNK => "symlink", + S::IFCHR => "character-device", + S::IFBLK => "block-device", + S::IFIFO => "fifo", + _ => return None, + }) + } + + /// A long name is cut short, with what tells it from another. + fn write_name(log: &mut String, name: &[u8]) { + const LONGEST: usize = 300; + const SHOWN: usize = 32; + if name.len() <= LONGEST { + _ = write!(log, "\"{}\"", name.escape_ascii()); + } else { + _ = write!( + log, + "\"{}\"... {} bytes crc32={:08x}", + name[..SHOWN].escape_ascii(), + name.len(), + bun_zlib::crc32_bytes(0, name), + ); + } + } + + /// Contents that follow each other, however many pieces they came in. + #[derive(Default)] + struct Run { + offset: i64, + len: i64, + } + + impl Run { + fn flush(&mut self, log: &mut String) { + if self.len != 0 { + _ = writeln!(log, " data {}+{}", self.offset, self.len); + } + *self = Self::default(); + } + } + + pub(crate) fn read_tar(data: &[u8], options: &ReadTarOptions) -> String { + let reader_options = tar::Options { + dialect: if options.npm { + tar::Dialect::Npm + } else { + tar::Dialect::General + }, + charset: if options.windows { + tar::Charset::Utf8 + } else { + tar::Charset::Ascii + }, + mac_metadata: options.macos, + }; + let opened = if options.stream { + tar::Decoder::new(reader_options, data.starts_with(&[0x1f, 0x8b])).map(Source::Pieces) + } else { + tar::MemoryDecoder::new(data, reader_options).map(Source::Whole) + }; + let mut source = match opened { + Ok(source) => source, + Err(err) => return format!("refused: {err}\n"), + }; + + let mut log = String::new(); + // `data[pos..end]` has arrived and was not taken yet. + let (mut pos, mut end) = (0usize, 0usize); + let mut index = 0usize; + let mut crc = 0u32; + let mut run = Run::default(); + loop { + let mut skip = false; + let ended = end == data.len() && (!options.late_end || pos == end); + let next = match &mut source { + Source::Whole(reader) => reader.next().map(|event| (0, event)), + Source::Pieces(decoder) => decoder.next(&data[pos..end], ended), + }; + let (used, event) = match next { + Ok(next) => next, + Err(err) => { + run.flush(&mut log); + _ = writeln!(log, "error: {err}"); + return log; + } + }; + match event { + Event::NeedInput if ended => { + log.push_str("error: the reader wants more input than there is\n"); + return log; + } + Event::NeedInput => { + end = match options.piece_size { + 0 => data.len(), + size => end.saturating_add(size).min(data.len()), + }; + } + Event::Entry(entry) => { + match type_name(entry.filetype()) { + Some(name) => log.push_str(name), + None => _ = write!(log, "type-{:o}", entry.filetype()), + } + _ = write!( + log, + " perm={:o} size={} mtime={} path=", + entry.perm(), + entry.size(), + entry.mtime(), + ); + let mut wide = Vec::new(); + if options.windows + && tar::compose::to_utf16(entry.path(), entry.path_is_utf8(), &mut wide) + .is_ok() + { + // A name that is not UTF-8 has one unit for each byte. + let name = if entry.path_is_utf8() { + String::from_utf16_lossy(&wide).into_bytes() + } else { + wide.into_iter().map(|unit| unit as u8).collect() + }; + write_name(&mut log, &name); + } else { + write_name(&mut log, entry.path()); + } + // Nothing reads a link's target on Windows. + if let Some(target) = entry.symlink().filter(|_| !options.windows) { + log.push_str(" -> "); + write_name(&mut log, target); + } + log.push('\n'); + skip = match options.skip { + Skip::None => false, + Skip::All => true, + Skip::Odd => index % 2 == 1, + Skip::OtherThanFiles => entry.filetype() != S::IFREG, + }; + index += 1; + crc = 0; + run = Run::default(); + } + Event::Data { offset, bytes } => { + crc = bun_zlib::crc32_bytes(crc, bytes); + if offset != run.offset + run.len { + run.flush(&mut log); + run.offset = offset; + } + run.len += bytes.len() as i64; + } + Event::EndOfEntry { size } => { + run.flush(&mut log); + _ = writeln!(log, " end {size} crc32={crc:08x}"); + } + Event::EndOfArchive => { + log.push_str("end\n"); + return log; + } + } + pos += used; + if skip { + log.push_str(" skipped\n"); + match &mut source { + Source::Whole(reader) => reader.skip_entry(), + Source::Pieces(decoder) => decoder.skip_entry(), + } + } + } + } + + fn write_files( + out: O, + files: Vec, + names: &[u8], + mut contents: &[u8], + options: &WriteTarOptions, + ) -> tar::Result> { + let mut writer = Writer::new( + out, + if options.windows { + Names::Utf8 + } else { + Names::Binary + }, + ); + for (file, path) in files.into_iter().zip(strings::split(names, b"\0")) { + writer.header(&File { + path, + size: file.size as u64, + perm: file.perm, + mtime: file.mtime as i64, + })?; + let (given, rest) = contents.split_at(file.given.min(contents.len())); + contents = rest; + for piece in given.chunks(options.write_size.max(1)) { + writer.data(piece)?; + } + } + Ok(writer) + } + + pub(crate) fn write_tar<'a>( + heap: &JsHeap<'a>, + files: Vec, + names: &[u8], + contents: &[u8], + options: &WriteTarOptions, + ) -> bindgen::Result<'a, JsLocal<'a, BufferSource>> { + let written = match options.gzip_level { + _ if !options.finish => { + let mut written = Vec::new(); + let sink = |bytes: &[u8]| written.extend_from_slice(bytes); + let result = write_files(sink, files, names, contents, options).map(drop); + result.map(|()| written) + } + None => write_files(Vec::new(), files, names, contents, options) + .and_then(|writer| writer.finish(true)), + Some(level) => tar::gzip::Encoder::new(Vec::new(), level) + .and_then(|encoder| write_files(encoder, files, names, contents, options)) + .and_then(|writer| writer.finish(false)) + .and_then(|encoder| encoder.finish()), + }; + heap.create_buffer(&written.map_err(|err| heap.error(format_args!("{err}")))?) + } +} diff --git a/src/runtime/api/ArchiveInternals.bind.ts b/src/runtime/api/ArchiveInternals.bind.ts new file mode 100644 index 000000000000..571d1c4affdf --- /dev/null +++ b/src/runtime/api/ArchiveInternals.bind.ts @@ -0,0 +1,60 @@ +import * as b from "../../codegen/bindgen-lib.ts"; + +export default b.module({ rustPath: "crate::api::archive::testing" }); + +export const ReadTarSkip = b.enumeration(["none", "all", "odd", "otherThanFiles"]); + +export const ReadTarOptions = b.dictionary({ + /** The rules of npm's own tar reader, as for a package. */ + npm: b.bool.default(false), + /** What differs on these systems is an option of the reader, so every system can test it. */ + windows: b.bool.default(false), + macos: b.bool.default(false), + /** Read like `bun install` reads a download: nothing is detected but gzip, by its first two bytes. */ + stream: b.bool.default(false), + /** With `stream`, how much of the input arrives at a time. 0 is all of it. */ + pieceSize: b.usize.default(0), + /** With `stream`, the reader learns that the input has ended only after it has taken the last piece. */ + lateEnd: b.bool.default(false), + /** The entries whose contents are not read. */ + skip: ReadTarSkip.default("none"), +}); + +/** Everything the tar reader reports about `data`, one line each. */ +export const readTar = b.fn({ + args: { data: b.bytes({ from: ["buffer"] }), options: ReadTarOptions.default({}) }, + returns: b.string.utf8, +}); + +export const WriteTarFile = b.dictionary({ + /** What the header announces. */ + size: b.f64, + /** How much of `contents` is written for it, which may be less or more. */ + given: b.usize, + perm: b.u32, + mtime: b.f64, +}); + +export const WriteTarOptions = b.dictionary({ + /** Names are recorded the way Windows does. */ + windows: b.bool.default(false), + /** As `bun pm pack` writes: compressed, and not padded to a full record. */ + gzipLevel: b.u8.optional, + /** How much of a file's contents is written at a time. */ + writeSize: b.usize.default(8192), + /** Without it the result ends where the last write did, so that a size can be announced and not written. */ + finish: b.bool.default(true), +}); + +/** `names` has a NUL behind each name, and `contents` is what the files hold, one after the other. */ +export const writeTar = b.fn({ + args: { + files: b.array(WriteTarFile), + names: b.bytes({ from: ["buffer"] }), + contents: b.bytes({ from: ["buffer"] }), + options: WriteTarOptions.default({}), + }, + returns: b.bufferSource, + capability: "heap", + throws: true, +}); diff --git a/src/runtime/cli/create_command.rs b/src/runtime/cli/create_command.rs index 9d53acf0954b..ecd97d3afce5 100644 --- a/src/runtime/cli/create_command.rs +++ b/src/runtime/cli/create_command.rs @@ -4,6 +4,7 @@ use std::io::Write as _; use crate::api::bun_process::Status; use crate::api::bun_process::sync as spawn_sync; +use bun_archive::{Archiver, archiver}; use bun_clap as clap; use bun_core::Progress::{Node as ProgressNode, Progress}; use bun_core::{Global, Output, pretty_error, pretty_errorln}; @@ -11,7 +12,6 @@ use bun_core::{MutableString, strings}; use bun_dotenv as DotEnv; use bun_http as HTTP; use bun_js_printer as JSPrinter; -use bun_libarchive::{Archiver, archiver}; use bun_parsers::json as JSON; use bun_paths::{OSPathSlice, PathBuffer}; use bun_resolver::fs; @@ -549,8 +549,8 @@ impl CreateCommand { struct OverwriteListAppender { buf: Vec, } - impl bun_libarchive::ArchiveAppender for OverwriteListAppender { - fn append(&mut self, path: &[u8]) -> Result<&[u8], bun_libarchive::Error> { + impl bun_archive::ArchiveAppender for OverwriteListAppender { + fn append(&mut self, path: &[u8]) -> Result<&[u8], bun_archive::Error> { self.buf.clear(); self.buf.extend_from_slice(path); Ok(&self.buf) diff --git a/src/runtime/cli/pack_command.rs b/src/runtime/cli/pack_command.rs index c82d34628c2a..2046c25a98b8 100644 --- a/src/runtime/cli/pack_command.rs +++ b/src/runtime/cli/pack_command.rs @@ -5,6 +5,7 @@ use std::io::Write as _; use crate::cli::Command; use crate::cli::publish_command as Publish; use bun_alloc::AllocError; +use bun_archive::tar; use bun_ast::{E, Expr, ExprData}; use bun_collections::StringHashMap; use bun_core::{Global, Output, Progress, fmt as bun_fmt}; @@ -14,10 +15,6 @@ use bun_install::package_manager::LogLevel; use bun_install::package_manager::workspace_manifests::WorkspaceManifests; use bun_install::package_manager::workspace_package_json_cache as WorkspacePackageJSONCache; use bun_js_printer as js_printer; -use bun_libarchive::lib::{ - Archive, Entry as ArchiveEntry, GrowingBuffer, Result as ArchiveResult, WriteArchive, - archive_write_open2, -}; use bun_parsers::json as JSON; use bun_paths::{self as path, SEP_STR}; use bun_ptr::cow_slice::CowSlice; @@ -1644,18 +1641,7 @@ fn fail_file(err: impl bun_core::output::ErrName, action: &str, path: &[u8]) -> Global::crash(); } -fn expect_archive_ok(archive: &Archive, status: ArchiveResult, action: &str) { - if matches!( - status, - ArchiveResult::Failed | ArchiveResult::Fatal | ArchiveResult::Warn - ) { - Output::err_generic( - "failed to {}: {}", - (action, bstr::BStr::new(archive.error_string())), - ); - Global::crash(); - } -} +type TarballWriter<'a> = tar::writer::Writer>; /// `Some` exactly when `for_publish`. #[inline(never)] @@ -1876,53 +1862,18 @@ pub(crate) fn pack( })); } - let archive = WriteArchive::new(); - - expect_archive_ok( - &archive, - archive.write_set_format_pax_restricted(), - "set archive format", - ); - expect_archive_ok( - &archive, - archive.write_add_filter_gzip(), - "set archive compression to gzip", - ); - // default is 9 // https://github.com/npm/cli/blob/ec105f400281a5bfd17885de1ea3d54d0c231b27/node_modules/pacote/lib/util/tar-create-options.js#L12 - let compression_level: &[u8] = ctx.manager.options.pack_gzip_level.unwrap_or(b"9"); - match archive.write_set_filter_option( - None, - bun_core::zstr!("compression-level"), - &ZBox::from_bytes(compression_level), - ) { - ArchiveResult::Failed | ArchiveResult::Fatal | ArchiveResult::Warn => { + let compression_level: u8 = match ctx.manager.options.pack_gzip_level.unwrap_or(b"9") { + &[digit @ b'0'..=b'9'] => digit - b'0', + level => { Output::err_generic( "compression level must be between 0 and 9, received {}", - format_args!("{}", bstr::BStr::new(compression_level)), + format_args!("{}", bstr::BStr::new(level)), ); Global::crash(); } - _ => {} - } - - expect_archive_ok( - &archive, - archive.write_set_filter_option(None, bun_core::zstr!("os"), bun_core::zstr!("Unknown")), - "set os to `Unknown`", - ); - expect_archive_ok( - &archive, - archive.write_set_options(bun_core::zstr!("gzip:!timestamp")), - "unset gzip timestamp option", - ); - // Same as `archive_write_open_filename` set for a file, so the output stays byte-identical. - expect_archive_ok( - &archive, - archive.write_set_bytes_in_last_block(1), - "set archive block padding", - ); + }; // `bun publish` uploads the bytes from memory. Only `bun pm pack` writes a file. let mut dest_buf = bun_paths::path_buffer_pool::get(); @@ -1960,34 +1911,20 @@ pub(crate) fn pack( fail_write_tarball(err, dest); } }; - let mut tarball_buffer = GrowingBuffer::forward_to(&mut write_tarball); - // Rebound here so that it is dropped before `tarball_buffer`. Closing the archive, which its - // `Drop` also does, writes into that buffer. - let archive = archive; - - if archive_write_open2( - &archive, - (&raw mut tarball_buffer).cast(), - Some(GrowingBuffer::open_callback), - Some(GrowingBuffer::write_callback), - Some(GrowingBuffer::close_callback), - None, - ) != 0 - { - Output::err_generic( - "failed to open archive for writing: {}", - format_args!("{}", bstr::BStr::new(archive.error_string())), - ); - Global::crash(); - } + let mut writer: TarballWriter<'_> = tar::writer::Writer::new( + tar::gzip::Encoder::new( + &mut write_tarball as &mut dyn FnMut(&[u8]), + compression_level, + ) + .unwrap_or_else(|err| archive_write_failed(err)), + tar::writer::Names::for_host(), + ); // append removed items from `pack_queue` with their file size let mut pack_list: PackList = Vec::new(); let mut read_buf = vec![0u8; 512 * 1024]; - let mut entry = ArchiveEntry::new2(&archive); - { let mut progress = Progress::Progress::default(); let mut node: Option<&mut Progress::Node> = None; @@ -1998,7 +1935,7 @@ pub(crate) fn pack( node = Some(started); } - entry = archive_package_json(&mut ctx, &archive, entry, &root_dir, &edited_package_json); + archive_package_json(&mut ctx, &mut writer, &root_dir, &edited_package_json); if let Some(node) = node.as_mut() { node.complete_one(); } @@ -2028,14 +1965,13 @@ pub(crate) fn pack( size: usize::try_from(stat.st_size).expect("int cast"), }); - entry = add_archive_entry( + add_archive_entry( &mut ctx, fd, &stat, &item.path, &mut read_buf, - &archive, - entry, + &mut writer, &bins, None, ); @@ -2053,14 +1989,13 @@ pub(crate) fn pack( .stat() .unwrap_or_else(|err| fail_file(err, "stat", path)); - entry = add_archive_entry( + add_archive_entry( &mut ctx, file.handle, &stat, &item.path, &mut read_buf, - &archive, - entry, + &mut writer, &bins, item.edited.as_deref(), ); @@ -2075,17 +2010,11 @@ pub(crate) fn pack( } } - ArchiveEntry::opaque_ref(entry).free(); - - match archive.write_close() { - ArchiveResult::Failed | ArchiveResult::Fatal | ArchiveResult::Warn => { - archive_write_failed(&archive); - } - _ => {} + // The archive ends right behind its two end-of-archive blocks, with no padding to a full record. + if let Err(err) = writer.finish(false).and_then(|encoder| encoder.finish()) { + archive_write_failed(err); } - drop(archive); - let mut shasum: [u8; sha::SHA1::DIGEST] = [0; sha::SHA1::DIGEST]; sha1.r#final(&mut shasum); @@ -2423,20 +2352,16 @@ fn write_tarball_in_place(abs_tarball_dest: &ZStr, tarball_bytes: &[u8], is_type } #[cold] -fn archive_write_failed(archive: &Archive) -> ! { - Output::err_generic( - "failed to write tarball: {}", - format_args!("{}", bstr::BStr::new(archive.error_string())), - ); +fn archive_write_failed(err: tar::Error) -> ! { + Output::err_generic("failed to write tarball: {}", format_args!("{err}")); Global::crash(); } -/// `archive_write_data` returns the byte count, or a negative status when the write fails. -fn write_archive_data(archive: &Archive, data: &[u8]) -> usize { - match usize::try_from(archive.write_data(data)) { - Ok(written) => written, - Err(_) => archive_write_failed(archive), - } +/// Returns how much was taken, which stops at the size in the entry's header. +fn write_archive_data(writer: &mut TarballWriter<'_>, data: &[u8]) -> usize { + writer + .data(data) + .unwrap_or_else(|err| archive_write_failed(err)) } fn stat_package_json(root_dir: &Dir) -> bun_sys::Stat { @@ -2450,39 +2375,24 @@ fn stat_package_json(root_dir: &Dir) -> bun_sys::Stat { fn archive_package_json( ctx: &mut Context<'_>, - archive: &Archive, - entry: *mut ArchiveEntry, + writer: &mut TarballWriter<'_>, root_dir: &Dir, edited_package_json: &[u8], -) -> *mut ArchiveEntry { - // `entry` is the same pointer after `.clear()`. - let entry = ArchiveEntry::opaque_ref(entry); +) { let stat = stat_package_json(root_dir); - entry.set_pathname(bun_core::zstr!("package/package.json")); - entry.set_size(i64::try_from(edited_package_json.len()).expect("int cast")); - // https://github.com/libarchive/libarchive/blob/898dc8319355b7e985f68a9819f182aaed61b53a/libarchive/archive_entry.h#L185 - entry.set_filetype(0o100000); - entry.set_perm(stat.st_mode as bun_sys::Mode); - // '1985-10-26T08:15:00.000Z' - // https://github.com/npm/cli/blob/ec105f400281a5bfd17885de1ea3d54d0c231b27/node_modules/pacote/lib/util/tar-create-options.js#L28 - entry.set_mtime(499162500, 0); - - match archive.write_header(entry) { - ArchiveResult::Fatal => archive_write_failed(archive), - ArchiveResult::Failed | ArchiveResult::Warn => { - Output::err_generic( - "failed to write tarball header: {}", - format_args!("{}", bstr::BStr::new(archive.error_string())), - ); - Global::crash(); - } - _ => {} - } - - ctx.stats.unpacked_size += write_archive_data(archive, edited_package_json); + writer + .header(&tar::writer::File { + path: b"package/package.json", + size: edited_package_json.len() as u64, + perm: stat.st_mode as u32, + // '1985-10-26T08:15:00.000Z' + // https://github.com/npm/cli/blob/ec105f400281a5bfd17885de1ea3d54d0c231b27/node_modules/pacote/lib/util/tar-create-options.js#L28 + mtime: 499162500, + }) + .unwrap_or_else(|err| archive_write_failed(err)); - entry.clear() + ctx.stats.unpacked_size += write_archive_data(writer, edited_package_json); } fn add_archive_entry( @@ -2491,52 +2401,33 @@ fn add_archive_entry( stat: &bun_sys::Stat, filename: &ZStr, read_buf: &mut [u8], - archive: &Archive, - entry: *mut ArchiveEntry, + writer: &mut TarballWriter<'_>, bins: &[BinInfo], // If set, these bytes are archived instead of the contents of `file`. edited: Option<&[u8]>, -) -> *mut ArchiveEntry { - // `entry` is the same pointer after `.clear()`. - let entry = ArchiveEntry::opaque_ref(entry); - let pathname = [b"package/".as_slice(), filename.as_bytes_with_nul()].concat(); - let pathname = ZBox::from_vec_with_nul(pathname); - #[cfg(windows)] - entry.set_pathname_utf8(&pathname); - #[cfg(not(windows))] - entry.set_pathname(&pathname); - - entry.set_size(edited.map_or(stat.st_size as i64, |edited| edited.len() as i64)); - - // https://github.com/libarchive/libarchive/blob/898dc8319355b7e985f68a9819f182aaed61b53a/libarchive/archive_entry.h#L185 - entry.set_filetype(0o100000); +) { + let pathname = [b"package/".as_slice(), filename.as_bytes()].concat(); - let mut perm: bun_sys::Mode = stat.st_mode as bun_sys::Mode; + let mut perm = stat.st_mode as u32; // https://github.com/npm/cli/blob/ec105f400281a5bfd17885de1ea3d54d0c231b27/node_modules/pacote/lib/util/tar-create-options.js#L20 if is_package_bin(bins, filename.as_bytes()) { perm |= 0o111; } - entry.set_perm(perm); - - // '1985-10-26T08:15:00.000Z' - // https://github.com/npm/cli/blob/ec105f400281a5bfd17885de1ea3d54d0c231b27/node_modules/pacote/lib/util/tar-create-options.js#L28 - entry.set_mtime(499162500, 0); - match archive.write_header(entry) { - ArchiveResult::Fatal => archive_write_failed(archive), - ArchiveResult::Failed => { - Output::err_generic( - "failed to write tarball header: {}", - format_args!("{}", bstr::BStr::new(archive.error_string())), - ); - Global::crash(); - } - _ => {} - } + writer + .header(&tar::writer::File { + path: &pathname, + size: edited.map_or(stat.st_size as u64, |edited| edited.len() as u64), + perm, + // '1985-10-26T08:15:00.000Z' + // https://github.com/npm/cli/blob/ec105f400281a5bfd17885de1ea3d54d0c231b27/node_modules/pacote/lib/util/tar-create-options.js#L28 + mtime: 499162500, + }) + .unwrap_or_else(|err| archive_write_failed(err)); if let Some(edited) = edited { - ctx.stats.unpacked_size += write_archive_data(archive, edited); - return entry.clear(); + ctx.stats.unpacked_size += write_archive_data(writer, edited); + return; } let mut written: usize = 0; @@ -2548,12 +2439,12 @@ fn add_archive_entry( }; // The size of each write reaches the compressor, so it is part of what keeps the bytes stable. for chunk in read_buf[..read].chunks(8192) { - written += write_archive_data(archive, chunk); + written += write_archive_data(writer, chunk); } } ctx.stats.unpacked_size += written; - // libarchive drops bytes past the size in the header, and pads an entry that is too short with + // The writer drops bytes past the size in the header, and pads an entry that is too short with // zeros, so a file that shrank has to be caught here. if (written as i64) < stat.st_size as i64 { Output::err_generic( @@ -2562,8 +2453,6 @@ fn add_archive_entry( ); Global::crash(); } - - entry.clear() } /// What replaces a `workspace:` or `catalog:` specifier in the tarball's package.json. @@ -3195,100 +3084,45 @@ pub(crate) mod bindings { let mut entries: Vec = Vec::new(); - let archive = Archive::opaque_ref(Archive::read_new()); - let check = |status: ArchiveResult, action: &str| -> bindgen::Result<'a, ()> { - match status { - ArchiveResult::Failed | ArchiveResult::Fatal | ArchiveResult::Warn => Err(heap - .error(format_args!( - "failed to {}: {}", - action, - bstr::BStr::new(archive.error_string()) - ))), - _ => Ok(()), - } - }; - - check(archive.read_support_format_tar(), "support tar")?; - check(archive.read_support_format_gnutar(), "support gnutar")?; - check( - archive.read_support_filter_gzip(), - "support gzip compression", - )?; - check( - archive.read_set_options(c"read_concatenated_archives"), - "set read_concatenated_archives option", - )?; - check(archive.read_open_memory(&tarball), "open archive in memory")?; - - let mut archive_entry: *mut ArchiveEntry = core::ptr::null_mut(); - let mut header_status = archive.read_next_header(&mut archive_entry); - + let mut reader = bun_archive::open_memory(&tarball, tar::Dialect::General) + .map_err(|err| heap.error(format_args!("failed to read tarball: {err}")))?; let mut read_buf: Vec = Vec::new(); - while header_status != ArchiveResult::Eof { - match header_status { - ArchiveResult::Eof => unreachable!(), - ArchiveResult::Retry => { - header_status = archive.read_next_header(&mut archive_entry); - continue; + loop { + let archive_entry = match reader.next_entry() { + Ok(None) => break, + Ok(Some(entry)) => entry, + Err(err) => { + return Err(heap.error(format_args!("failed to read archive header: {err}"))); } - ArchiveResult::Failed | ArchiveResult::Fatal => { + }; + let pathname = BunString::clone_utf8(&bun_core::handle_oom(bun_archive::display_path( + archive_entry, + ))); + + let kind = archive_entry.kind(); + let size: usize = usize::try_from(archive_entry.size()).expect("int cast"); + let mut entry = TarballEntry { + pathname, + kind: tarball_entry_kind(kind), + perm: f64::from(archive_entry.perm()), + contents: None, + }; + + if kind == bun_sys::FileKind::File { + if let Err(err) = reader.read_data(size, &mut read_buf) { + let pathname_utf8 = entry.pathname.to_utf8(); return Err(heap.error(format_args!( - "failed to read archive header: {}", - bstr::BStr::new(archive.error_string()), + "failed to read archive entry \"{}\": {err}", + bstr::BStr::new(pathname_utf8.slice()), ))); } - _ => { - let archive_entry_ref = ArchiveEntry::opaque_mut(archive_entry); - #[cfg(windows)] - let pathname_string = { - let pathname_w = archive_entry_ref.pathname_w(); - let result = bun_core::handle_oom(strings::to_utf8_list_with_type( - Vec::new(), - pathname_w, - )); - BunString::clone_utf8(&result) - }; - #[cfg(not(windows))] - let pathname_string = BunString::clone_utf8(archive_entry_ref.pathname()); - - let kind = bun_sys::kind_from_mode(archive_entry_ref.filetype()); - let perm = archive_entry_ref.perm(); - - let mut entry = TarballEntry { - pathname: pathname_string, - kind: tarball_entry_kind(kind), - perm: f64::from(perm), - contents: None, - }; - - if kind == bun_sys::FileKind::File { - let size: usize = - usize::try_from(archive_entry_ref.size()).expect("int cast"); - read_buf.resize(size, 0); - - let read = archive.read_data(&mut read_buf); - if read < 0 { - let pathname_utf8 = entry.pathname.to_utf8(); - return Err(heap.error(format_args!( - "failed to read archive entry \"{}\": {}", - bstr::BStr::new(pathname_utf8.slice()), - bstr::BStr::new(archive.error_string()), - ))); - } - read_buf.truncate(usize::try_from(read).expect("int cast")); - entry.contents = Some(BunString::clone_utf8(&read_buf)); - read_buf.clear(); - } - - entries.push(entry); - } + entry.contents = Some(BunString::clone_utf8(&read_buf)); + read_buf.clear(); } - header_status = archive.read_next_header(&mut archive_entry); - } - check(archive.read_close(), "close read archive")?; - check(archive.read_free(), "close read archive")?; + entries.push(entry); + } Ok(Tarball { entries, diff --git a/src/runtime/cli/pm_diff_command.rs b/src/runtime/cli/pm_diff_command.rs index 087d77f5a1cc..459470603431 100644 --- a/src/runtime/cli/pm_diff_command.rs +++ b/src/runtime/cli/pm_diff_command.rs @@ -5,6 +5,7 @@ use std::io::Write as _; use bstr::BStr; use bun_alloc::Arena as Bump; +use bun_archive::tar; use bun_collections::VecExt as _; use bun_core::MutableString; use bun_core::fmt as bun_fmt; @@ -19,7 +20,6 @@ use bun_install::npm::{self, PackageManifest}; use bun_install::package_manager::options::Do; use bun_install::package_manager::workspace_manifests::WorkspaceManifests; use bun_install::{PackageManager, resolution}; -use bun_libarchive::lib::{ArchiveIterator, IteratorResult as ArchiveIterResult}; use bun_sys::{Fd, FdExt as _, dir_iterator as DirIterator}; use bun_url::URL; @@ -531,36 +531,27 @@ fn materialize( } fn read_tarball_into(bytes: &[u8], tree: &mut Tree) -> Result<(), crate::Error> { - let mut iter = match ArchiveIterator::init(bytes) { - ArchiveIterResult::Result(it) => it, - ArchiveIterResult::Err { message, .. } => { - Status::clear(); - Output::err_generic("{}: {}", (BStr::new(&tree.label), BStr::new(message))); - Global::exit(1); + let fail = |err: tar::Error, message: &str| -> crate::Error { + if let tar::Error::Alloc(err) = err { + return err.into(); } + Status::clear(); + Output::err_generic("{}: {}", (BStr::new(&tree.label), message)); + Global::exit(1); }; - loop { - let next = match iter.next() { - ArchiveIterResult::Result(Some(n)) => n, - ArchiveIterResult::Result(None) => break, - ArchiveIterResult::Err { message, .. } => { - Status::clear(); - Output::err_generic("{}: {}", (BStr::new(&tree.label), BStr::new(message))); - Global::exit(1); - } - }; - if next.kind != bun_sys::FileKind::File { + let mut reader = bun_archive::open_memory(bytes, tar::Dialect::General) + .map_err(|err| fail(err, "failed to read tarball"))?; + while let Some(entry) = reader + .next_entry() + .map_err(|err| fail(err, "failed to read archive header"))? + { + if entry.kind() != bun_sys::FileKind::File { continue; } - let perm = next.entry().perm() & 0o777; - // libarchive returns NULL from the narrow accessor for non-ASCII names on Windows; go through UTF-16 there. - #[cfg(windows)] - let wide_path = - strings::to_utf8_list_with_type(Vec::new(), next.entry().pathname_w().as_slice())?; - #[cfg(windows)] - let path = wide_path.as_slice(); - #[cfg(not(windows))] - let path = next.entry().pathname().as_bytes(); + let perm = (entry.perm() & 0o777) as bun_sys::Mode; + let size = usize::try_from(entry.size().max(0)).expect("int cast"); + let path = bun_archive::display_path(entry)?; + let path: &[u8] = &path; // npm tarballs root everything under one folder (usually `package/`). let rel = match strings::index_of_char(path, b'/') { Some(i) => &path[i as usize + 1..], @@ -569,21 +560,26 @@ fn read_tarball_into(bytes: &[u8], tree: &mut Tree) -> Result<(), crate::Error> if rel.is_empty() { continue; } - let data = match iter.read_entry_data(&next)? { - ArchiveIterResult::Result(d) => d, - ArchiveIterResult::Err { message, .. } => { - Status::clear(); - Output::err_generic( - "{}: {}: {}", - (BStr::new(&tree.label), BStr::new(rel), BStr::new(message)), - ); - Global::exit(1); + let rel = rel.to_vec(); + let mut data = Vec::new(); + if let Err(err) = reader.read_data(size, &mut data) { + if let tar::Error::Alloc(err) = err { + return Err(err.into()); } - }; - tree.modes.insert(rel.to_vec(), perm); - tree.files.insert(rel.to_vec(), data.into_vec()); + Status::clear(); + Output::err_generic( + "{}: {}: {}", + ( + BStr::new(&tree.label), + BStr::new(&rel), + "failed to read archive data", + ), + ); + Global::exit(1); + } + tree.modes.insert(rel.clone(), perm); + tree.files.insert(rel, data); } - let _ = iter.close(); Ok(()) } diff --git a/src/runtime/cli/publish_command.rs b/src/runtime/cli/publish_command.rs index fedb790237ec..1e1c99ea9424 100644 --- a/src/runtime/cli/publish_command.rs +++ b/src/runtime/cli/publish_command.rs @@ -3,6 +3,7 @@ use std::io::Write as _; use crate::Error; use bun_alloc::AllocError; +use bun_archive::tar; use bun_ast::{E, Expr, G}; use bun_core::MutableString; use bun_core::fmt as bun_fmt; @@ -12,7 +13,6 @@ use bun_http as http; use bun_install::package_manager::options::PublishTarget; use bun_install::package_manager::workspace_manifests; use bun_install::{self as install, Npm, PackageManager, Subcommand}; -use bun_libarchive::lib::{Archive, ArchiveIterator, IteratorResult as ArchiveIterResult}; use bun_parsers::json as json_mod; use bun_paths as path; use bun_paths::resolve_path::{join_abs_string_buf_checked, normalize_buf_z_spill}; @@ -71,20 +71,14 @@ fn is_readme_filename>(name: &[T]) -> bool { && (name.len() == README.len() || name[README.len()].into() == u32::from(b'.')) } -fn archive_ok(result: ArchiveIterResult) -> T { - match result { - ArchiveIterResult::Result(value) => value, - ArchiveIterResult::Err { archive, message } => { - Output::err_generic( - "{}: {}", - ( - bstr::BStr::new(message), - bstr::BStr::new(Archive::opaque_ref(archive).error_string()), - ), - ); - Global::crash(); +fn archive_ok(result: Result, message: &str) -> Result { + result.map_err(|err| { + if matches!(err, tar::Error::Alloc(_)) { + return PackError::OutOfMemory; } - } + Output::err_generic("{}: {}", (message, err)); + Global::crash(); + }) } /// `E::String` erases the lifetime of its bytes, so these have to outlive the printing of the AST. @@ -172,25 +166,26 @@ impl Context { let mut maybe_readme: Option = None; let mut files: Vec> = Vec::new(); - let mut iter = archive_ok(ArchiveIterator::init(&tarball_bytes)); + let mut reader = archive_ok( + bun_archive::open_memory(&tarball_bytes, tar::Dialect::General), + "failed to read tarball", + )?; + let mut os_path_buf: Vec = Vec::new(); let mut unpacked_size: usize = 0; let mut total_files: usize = 0; Output::print(format_args!("\n")); - while let Some(next) = archive_ok(iter.next()) { - // SAFETY: `next.entry` is valid until the next `iter.next()` call. - let entry = unsafe { &*next.entry }; - #[cfg(windows)] - let pathname: &[OSPathChar] = entry.pathname_w().as_slice(); - #[cfg(not(windows))] - let pathname: &[OSPathChar] = entry.pathname().as_bytes(); + while let Some(entry) = archive_ok(reader.next_entry(), "failed to read archive header")? { + let pathname: &[OSPathChar] = bun_archive::os_path(entry, &mut os_path_buf) + .map_err(|_| PackError::OutOfMemory)?; + let kind = entry.kind(); let size = usize::try_from(entry.size().max(0)).expect("int cast"); unpacked_size += size; - total_files += usize::from(next.kind == FileKind::File); + total_files += usize::from(kind == FileKind::File); // this is option `strip: 1` (npm expects a `package/` prefix for all paths) let stripped = (pathname.iter().position(|&c| bun_paths::is_sep_any_t(c))) @@ -214,7 +209,7 @@ impl Context { ); let Some(filename) = stripped else { continue }; - if next.kind != FileKind::File { + if kind != FileKind::File { continue; } #[cfg(not(windows))] @@ -228,19 +223,26 @@ impl Context { if maybe_package_json_contents.is_none() && strings::eql_case_insensitive_t(filename, b"package.json") { - maybe_package_json_contents = Some(archive_ok(iter.read_entry_data(&next)?)); + let mut contents = Vec::new(); + archive_ok( + reader.read_data(size, &mut contents), + "failed to read archive data", + )?; + maybe_package_json_contents = Some(contents.into_boxed_slice()); } else if maybe_readme.is_none() && is_readme_filename(filename) { - let contents = archive_ok(iter.read_entry_data(&next)?).into_vec(); #[cfg(not(windows))] let filename: Vec = filename.to_vec(); #[cfg(windows)] let filename: Vec = strings::to_utf8_alloc(filename); + let mut contents = Vec::new(); + archive_ok( + reader.read_data(size, &mut contents), + "failed to read archive data", + )?; maybe_readme = Some(ReadmeInfo { filename, contents }); } } - archive_ok(iter.close()); - let Some(package_json_contents) = maybe_package_json_contents else { Output::err_generic( "failed to find package.json in tarball '{}'", diff --git a/src/runtime/error.rs b/src/runtime/error.rs index 254c6d9d673a..64924d2bb73b 100644 --- a/src/runtime/error.rs +++ b/src/runtime/error.rs @@ -274,11 +274,11 @@ impl From for Error { } } -impl From for Error { - fn from(e: bun_libarchive::Error) -> Self { +impl From for Error { + fn from(e: bun_archive::Error) -> Self { match e { - bun_libarchive::Error::Sys(s) => Self::Sys(s), - bun_libarchive::Error::Alloc(a) => Self::Alloc(a), + bun_archive::Error::Sys(s) => Self::Sys(s), + bun_archive::Error::Alloc(a) => Self::Alloc(a), _ => Self::Unexpected, } } diff --git a/src/standalone_graph/Cargo.toml b/src/standalone_graph/Cargo.toml index 7ade11b6adfd..bbf9eb90b516 100644 --- a/src/standalone_graph/Cargo.toml +++ b/src/standalone_graph/Cargo.toml @@ -27,7 +27,7 @@ bun_dotenv.workspace = true bun_exe_format.workspace = true bun_http.workspace = true bun_parsers.workspace = true -bun_libarchive.workspace = true +bun_archive.workspace = true bun_ast.workspace = true bun_options_types.workspace = true bun_paths.workspace = true diff --git a/src/standalone_graph/StandaloneModuleGraph.rs b/src/standalone_graph/StandaloneModuleGraph.rs index d5da84e8451a..8dbc4d8c9b73 100644 --- a/src/standalone_graph/StandaloneModuleGraph.rs +++ b/src/standalone_graph/StandaloneModuleGraph.rs @@ -2741,12 +2741,12 @@ pub(crate) fn download_to_path( scopeguard::defer! { let _ = bun_sys::Dir::cwd().delete_tree(tempdir_name.as_bytes()); } - let extract_res = bun_libarchive::Archiver::extract_to_dir( + let extract_res = bun_archive::Archiver::extract_to_dir( tarball_bytes.as_slice(), tmpdir.fd(), None, &mut (), - bun_libarchive::ExtractOptions { + bun_archive::ExtractOptions { // "package/bin" depth_to_skip: 2, ..Default::default() diff --git a/src/tar/Cargo.toml b/src/tar/Cargo.toml new file mode 100644 index 000000000000..5a6059594687 --- /dev/null +++ b/src/tar/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "bun_tar" +version.workspace = true +edition.workspace = true + +[lib] +path = "lib.rs" + +[lints] +workspace = true + +[dependencies] +itoa.workspace = true +thiserror.workspace = true +bun_alloc.workspace = true +bun_core.workspace = true +bun_zlib.workspace = true diff --git a/src/tar/compose.rs b/src/tar/compose.rs new file mode 100644 index 000000000000..8028a877058b --- /dev/null +++ b/src/tar/compose.rs @@ -0,0 +1,273 @@ +//! What Windows does to a UTF-8 name. Not NFC: nothing is decomposed, and the data is Unicode 6.0. + +mod tables; + +use crate::error::Result; +use tables::{COMBINING_CLASSES, COMPOSING_BLOCKS, COMPOSITIONS}; + +/// More combining marks than this on one character make the name count as unconvertible. +// https://github.com/libarchive/libarchive/blob/27cbc7827172698143e440801fc0ba39ccb4f1f5/libarchive/archive_string.c#L2903 +const MAX_MARKS: usize = 10; +/// The combining class that is exempt from the rule that classes rise. +// https://github.com/libarchive/libarchive/blob/27cbc7827172698143e440801fc0ba39ccb4f1f5/libarchive/archive_string.c#L3018 +const ABOVE_LEFT: u8 = 228; + +mod hangul { + pub(super) const S_BASE: u32 = 0xAC00; + pub(super) const L_BASE: u32 = 0x1100; + pub(super) const V_BASE: u32 = 0x1161; + pub(super) const T_BASE: u32 = 0x11A7; + pub(super) const L_COUNT: u32 = 19; + pub(super) const V_COUNT: u32 = 21; + pub(super) const T_COUNT: u32 = 28; + pub(super) const S_COUNT: u32 = L_COUNT * V_COUNT * T_COUNT; +} + +fn compose_pair(first: u32, second: u32) -> Option { + let index = COMPOSITIONS + .binary_search_by_key(&(first, second), |&(a, b, _)| (a, b)) + .ok()?; + COMPOSITIONS.get(index).map(|&(_, _, composed)| composed) +} + +fn in_ranges(ranges: &[T], value: u32, bounds: impl Fn(&T) -> (u32, u32)) -> Option<&T> { + let index = ranges.partition_point(|range| bounds(range).1 < value); + ranges.get(index).filter(|range| bounds(range).0 <= value) +} + +fn combining_class(c: u32) -> u8 { + in_ranges(&COMBINING_CLASSES, c, |&(first, last, _)| (first, last)) + .map_or(0, |&(_, _, class)| class) +} + +fn may_compose(c: u32) -> bool { + in_ranges(&COMPOSING_BLOCKS, c >> 8, |&range| range).is_some() +} + +/// UTF-8, or CESU-8 for a surrogate pair. `None` for anything else. +fn decode(bytes: &[u8]) -> Result>> { + let mut out = Vec::new(); + out.try_reserve(bytes.len())?; + let mut rest = bytes; + while let Some(chunk) = rest.utf8_chunks().next() { + out.extend(chunk.valid().chars().map(u32::from)); + rest = match rest.get(chunk.valid().len()..) { + None | Some([]) => break, + Some( + &[ + 0xed, + a @ 0xa0..=0xaf, + b @ 0x80..=0xbf, + 0xed, + c @ 0xb0..=0xbf, + d @ 0x80..=0xbf, + ref tail @ .., + ], + ) => { + let [a, b, c, d] = [a & 0x0f, b & 0x3f, c & 0x0f, d & 0x3f].map(u32::from); + out.push(0x10000 + ((a << 16) | (b << 10) | (c << 6) | d)); + tail + } + Some(_) => return Ok(None), + }; + } + Ok(Some(out)) +} + +/// A run of combining marks behind one character, each of a higher class than the one before. +struct Marks { + marks: [u32; MAX_MARKS], + count: usize, + last_class: u8, + /// The class of the character that ended the run, unless the input did. + stopper: Option, +} + +impl Marks { + /// Takes marks from `chars[*read..]` while their classes keep rising. `None` past [`MAX_MARKS`]. + fn collect(&mut self, chars: &[u32], read: &mut usize) -> Option<()> { + loop { + let slot = self.marks.get_mut(self.count)?; + let Some(&c) = chars.get(*read) else { + self.stopper = None; + return Some(()); + }; + let class = combining_class(c); + if self.last_class >= class && self.last_class != ABOVE_LEFT && class != ABOVE_LEFT { + self.stopper = Some(class); + return Some(()); + } + *slot = c; + *read = read.saturating_add(1); + self.last_class = class; + self.count = self.count.saturating_add(1); + } + } + + fn remove(&mut self, index: usize) { + self.marks + .copy_within(index.saturating_add(1)..self.count, index); + self.count = self.count.saturating_sub(1); + } + + fn as_slice(&self) -> &[u32] { + self.marks.get(..self.count).unwrap_or_default() + } +} + +/// `None` past [`MAX_MARKS`]. `out` has room for all of `chars`. +fn compose_chars(chars: &[u32], out: &mut Vec) -> Option<()> { + let mut read = 0usize; + 'starter: while let Some(&first) = chars.get(read) { + let mut current = first; + read = read.saturating_add(1); + loop { + let Some(&next) = chars.get(read) else { + out.push(current); + break 'starter; + }; + read = read.saturating_add(1); + + if !may_compose(next) { + out.push(current); + current = next; + continue; + } + let l_index = current.wrapping_sub(hangul::L_BASE); + let s_index = current.wrapping_sub(hangul::S_BASE); + if l_index < hangul::L_COUNT { + let v_index = next.wrapping_sub(hangul::V_BASE); + if v_index < hangul::V_COUNT { + current = + hangul::S_BASE + (l_index * hangul::V_COUNT + v_index) * hangul::T_COUNT; + } else { + out.push(current); + current = next; + } + continue; + } + if s_index < hangul::S_COUNT && s_index % hangul::T_COUNT == 0 { + let t_index = next.wrapping_sub(hangul::T_BASE); + if 0 < t_index && t_index < hangul::T_COUNT { + current += t_index; + } else { + out.push(current); + current = next; + } + continue; + } + if let Some(composed) = compose_pair(current, next) { + current = composed; + continue; + } + let class = combining_class(next); + if class == 0 { + out.push(current); + current = next; + continue; + } + + // A mark behind `next` may still combine, unless one in between has its class or a higher one. + let mut marks = Marks { + marks: [next; MAX_MARKS], + count: 1, + last_class: class, + stopper: None, + }; + marks.collect(chars, &mut read)?; + let mut i = 1usize; + while let Some(&mark) = marks.as_slice().get(i) { + let Some(composed) = compose_pair(current, mark) else { + i = i.saturating_add(1); + continue; + }; + current = composed; + marks.remove(i); + // The mark that is gone may have been what kept the next one out. + if marks.count > 0 && i == marks.count && marks.stopper == Some(marks.last_class) { + marks.last_class = marks + .as_slice() + .last() + .map_or(0, |&mark| combining_class(mark)); + marks.collect(chars, &mut read)?; + } + i = 0; + } + + out.push(current); + out.extend_from_slice(marks.as_slice()); + if marks.stopper == Some(marks.last_class) { + let mut last_class = marks.last_class; + while let Some(&c) = chars.get(read) { + let class = combining_class(c); + if last_class > class { + break; + } + read = read.saturating_add(1); + last_class = class; + out.push(c); + } + } + continue 'starter; + } + } + Some(()) +} + +/// `None` if `name` does not convert: Windows then takes each byte for one UTF-16 unit. +pub fn compose(name: &[u8]) -> Result>> { + let Some(chars) = decode(name)? else { + return Ok(None); + }; + let mut out = Vec::new(); + out.try_reserve(chars.len())?; + Ok(compose_chars(&chars, &mut out).map(|()| out)) +} + +/// Whether [`compose`] would return something. +pub(crate) fn converts(name: &[u8]) -> Result { + Ok(name.is_ascii() || compose(name)?.is_some()) +} + +fn push_utf8(out: &mut Vec, c: u32) { + let c = char::from_u32(c).unwrap_or(char::REPLACEMENT_CHARACTER); + out.extend_from_slice(c.encode_utf8(&mut [0; 4]).as_bytes()); +} + +/// The name an entry gets when it is written on Windows, or `None` if `name` does not convert. +pub(crate) fn windows_name(name: &[u8]) -> Result>> { + let mut out = Vec::new(); + out.try_reserve(name.len())?; + if name.is_ascii() { + out.extend_from_slice(name); + } else { + let Some(chars) = compose(name)? else { + return Ok(None); + }; + for c in chars { + push_utf8(&mut out, c); + } + } + for b in &mut out { + if *b == b'\\' { + *b = b'/'; + } + } + Ok(Some(out)) +} + +/// Appends `name` as Windows sees it. `is_utf8` is [`crate::Entry::path_is_utf8`]. +pub fn to_utf16(name: &[u8], is_utf8: bool, out: &mut Vec) -> Result<()> { + out.try_reserve(name.len())?; + if is_utf8 && !name.is_ascii() { + if let Some(chars) = compose(name)? { + for c in chars { + let c = char::from_u32(c).unwrap_or(char::REPLACEMENT_CHARACTER); + out.extend_from_slice(c.encode_utf16(&mut [0; 2])); + } + return Ok(()); + } + } + out.extend(name.iter().map(|&b| u16::from(b))); + Ok(()) +} diff --git a/src/tar/compose/tables.rs b/src/tar/compose/tables.rs new file mode 100644 index 000000000000..325a78e5eb62 --- /dev/null +++ b/src/tar/compose/tables.rs @@ -0,0 +1,1228 @@ +//! Unicode 6.0 data for [`super`]. Generated: each table is what its comment says, taken from the +//! Unicode Character Database. + +/// `(first, second, composed)` for every character whose canonical decomposition is a pair and +/// that is not a composition exclusion, sorted. Hangul syllables are computed instead. +pub(super) static COMPOSITIONS: [(u32, u32, u32); 931] = [ + (0x003C, 0x0338, 0x226E), + (0x003D, 0x0338, 0x2260), + (0x003E, 0x0338, 0x226F), + (0x0041, 0x0300, 0x00C0), + (0x0041, 0x0301, 0x00C1), + (0x0041, 0x0302, 0x00C2), + (0x0041, 0x0303, 0x00C3), + (0x0041, 0x0304, 0x0100), + (0x0041, 0x0306, 0x0102), + (0x0041, 0x0307, 0x0226), + (0x0041, 0x0308, 0x00C4), + (0x0041, 0x0309, 0x1EA2), + (0x0041, 0x030A, 0x00C5), + (0x0041, 0x030C, 0x01CD), + (0x0041, 0x030F, 0x0200), + (0x0041, 0x0311, 0x0202), + (0x0041, 0x0323, 0x1EA0), + (0x0041, 0x0325, 0x1E00), + (0x0041, 0x0328, 0x0104), + (0x0042, 0x0307, 0x1E02), + (0x0042, 0x0323, 0x1E04), + (0x0042, 0x0331, 0x1E06), + (0x0043, 0x0301, 0x0106), + (0x0043, 0x0302, 0x0108), + (0x0043, 0x0307, 0x010A), + (0x0043, 0x030C, 0x010C), + (0x0043, 0x0327, 0x00C7), + (0x0044, 0x0307, 0x1E0A), + (0x0044, 0x030C, 0x010E), + (0x0044, 0x0323, 0x1E0C), + (0x0044, 0x0327, 0x1E10), + (0x0044, 0x032D, 0x1E12), + (0x0044, 0x0331, 0x1E0E), + (0x0045, 0x0300, 0x00C8), + (0x0045, 0x0301, 0x00C9), + (0x0045, 0x0302, 0x00CA), + (0x0045, 0x0303, 0x1EBC), + (0x0045, 0x0304, 0x0112), + (0x0045, 0x0306, 0x0114), + (0x0045, 0x0307, 0x0116), + (0x0045, 0x0308, 0x00CB), + (0x0045, 0x0309, 0x1EBA), + (0x0045, 0x030C, 0x011A), + (0x0045, 0x030F, 0x0204), + (0x0045, 0x0311, 0x0206), + (0x0045, 0x0323, 0x1EB8), + (0x0045, 0x0327, 0x0228), + (0x0045, 0x0328, 0x0118), + (0x0045, 0x032D, 0x1E18), + (0x0045, 0x0330, 0x1E1A), + (0x0046, 0x0307, 0x1E1E), + (0x0047, 0x0301, 0x01F4), + (0x0047, 0x0302, 0x011C), + (0x0047, 0x0304, 0x1E20), + (0x0047, 0x0306, 0x011E), + (0x0047, 0x0307, 0x0120), + (0x0047, 0x030C, 0x01E6), + (0x0047, 0x0327, 0x0122), + (0x0048, 0x0302, 0x0124), + (0x0048, 0x0307, 0x1E22), + (0x0048, 0x0308, 0x1E26), + (0x0048, 0x030C, 0x021E), + (0x0048, 0x0323, 0x1E24), + (0x0048, 0x0327, 0x1E28), + (0x0048, 0x032E, 0x1E2A), + (0x0049, 0x0300, 0x00CC), + (0x0049, 0x0301, 0x00CD), + (0x0049, 0x0302, 0x00CE), + (0x0049, 0x0303, 0x0128), + (0x0049, 0x0304, 0x012A), + (0x0049, 0x0306, 0x012C), + (0x0049, 0x0307, 0x0130), + (0x0049, 0x0308, 0x00CF), + (0x0049, 0x0309, 0x1EC8), + (0x0049, 0x030C, 0x01CF), + (0x0049, 0x030F, 0x0208), + (0x0049, 0x0311, 0x020A), + (0x0049, 0x0323, 0x1ECA), + (0x0049, 0x0328, 0x012E), + (0x0049, 0x0330, 0x1E2C), + (0x004A, 0x0302, 0x0134), + (0x004B, 0x0301, 0x1E30), + (0x004B, 0x030C, 0x01E8), + (0x004B, 0x0323, 0x1E32), + (0x004B, 0x0327, 0x0136), + (0x004B, 0x0331, 0x1E34), + (0x004C, 0x0301, 0x0139), + (0x004C, 0x030C, 0x013D), + (0x004C, 0x0323, 0x1E36), + (0x004C, 0x0327, 0x013B), + (0x004C, 0x032D, 0x1E3C), + (0x004C, 0x0331, 0x1E3A), + (0x004D, 0x0301, 0x1E3E), + (0x004D, 0x0307, 0x1E40), + (0x004D, 0x0323, 0x1E42), + (0x004E, 0x0300, 0x01F8), + (0x004E, 0x0301, 0x0143), + (0x004E, 0x0303, 0x00D1), + (0x004E, 0x0307, 0x1E44), + (0x004E, 0x030C, 0x0147), + (0x004E, 0x0323, 0x1E46), + (0x004E, 0x0327, 0x0145), + (0x004E, 0x032D, 0x1E4A), + (0x004E, 0x0331, 0x1E48), + (0x004F, 0x0300, 0x00D2), + (0x004F, 0x0301, 0x00D3), + (0x004F, 0x0302, 0x00D4), + (0x004F, 0x0303, 0x00D5), + (0x004F, 0x0304, 0x014C), + (0x004F, 0x0306, 0x014E), + (0x004F, 0x0307, 0x022E), + (0x004F, 0x0308, 0x00D6), + (0x004F, 0x0309, 0x1ECE), + (0x004F, 0x030B, 0x0150), + (0x004F, 0x030C, 0x01D1), + (0x004F, 0x030F, 0x020C), + (0x004F, 0x0311, 0x020E), + (0x004F, 0x031B, 0x01A0), + (0x004F, 0x0323, 0x1ECC), + (0x004F, 0x0328, 0x01EA), + (0x0050, 0x0301, 0x1E54), + (0x0050, 0x0307, 0x1E56), + (0x0052, 0x0301, 0x0154), + (0x0052, 0x0307, 0x1E58), + (0x0052, 0x030C, 0x0158), + (0x0052, 0x030F, 0x0210), + (0x0052, 0x0311, 0x0212), + (0x0052, 0x0323, 0x1E5A), + (0x0052, 0x0327, 0x0156), + (0x0052, 0x0331, 0x1E5E), + (0x0053, 0x0301, 0x015A), + (0x0053, 0x0302, 0x015C), + (0x0053, 0x0307, 0x1E60), + (0x0053, 0x030C, 0x0160), + (0x0053, 0x0323, 0x1E62), + (0x0053, 0x0326, 0x0218), + (0x0053, 0x0327, 0x015E), + (0x0054, 0x0307, 0x1E6A), + (0x0054, 0x030C, 0x0164), + (0x0054, 0x0323, 0x1E6C), + (0x0054, 0x0326, 0x021A), + (0x0054, 0x0327, 0x0162), + (0x0054, 0x032D, 0x1E70), + (0x0054, 0x0331, 0x1E6E), + (0x0055, 0x0300, 0x00D9), + (0x0055, 0x0301, 0x00DA), + (0x0055, 0x0302, 0x00DB), + (0x0055, 0x0303, 0x0168), + (0x0055, 0x0304, 0x016A), + (0x0055, 0x0306, 0x016C), + (0x0055, 0x0308, 0x00DC), + (0x0055, 0x0309, 0x1EE6), + (0x0055, 0x030A, 0x016E), + (0x0055, 0x030B, 0x0170), + (0x0055, 0x030C, 0x01D3), + (0x0055, 0x030F, 0x0214), + (0x0055, 0x0311, 0x0216), + (0x0055, 0x031B, 0x01AF), + (0x0055, 0x0323, 0x1EE4), + (0x0055, 0x0324, 0x1E72), + (0x0055, 0x0328, 0x0172), + (0x0055, 0x032D, 0x1E76), + (0x0055, 0x0330, 0x1E74), + (0x0056, 0x0303, 0x1E7C), + (0x0056, 0x0323, 0x1E7E), + (0x0057, 0x0300, 0x1E80), + (0x0057, 0x0301, 0x1E82), + (0x0057, 0x0302, 0x0174), + (0x0057, 0x0307, 0x1E86), + (0x0057, 0x0308, 0x1E84), + (0x0057, 0x0323, 0x1E88), + (0x0058, 0x0307, 0x1E8A), + (0x0058, 0x0308, 0x1E8C), + (0x0059, 0x0300, 0x1EF2), + (0x0059, 0x0301, 0x00DD), + (0x0059, 0x0302, 0x0176), + (0x0059, 0x0303, 0x1EF8), + (0x0059, 0x0304, 0x0232), + (0x0059, 0x0307, 0x1E8E), + (0x0059, 0x0308, 0x0178), + (0x0059, 0x0309, 0x1EF6), + (0x0059, 0x0323, 0x1EF4), + (0x005A, 0x0301, 0x0179), + (0x005A, 0x0302, 0x1E90), + (0x005A, 0x0307, 0x017B), + (0x005A, 0x030C, 0x017D), + (0x005A, 0x0323, 0x1E92), + (0x005A, 0x0331, 0x1E94), + (0x0061, 0x0300, 0x00E0), + (0x0061, 0x0301, 0x00E1), + (0x0061, 0x0302, 0x00E2), + (0x0061, 0x0303, 0x00E3), + (0x0061, 0x0304, 0x0101), + (0x0061, 0x0306, 0x0103), + (0x0061, 0x0307, 0x0227), + (0x0061, 0x0308, 0x00E4), + (0x0061, 0x0309, 0x1EA3), + (0x0061, 0x030A, 0x00E5), + (0x0061, 0x030C, 0x01CE), + (0x0061, 0x030F, 0x0201), + (0x0061, 0x0311, 0x0203), + (0x0061, 0x0323, 0x1EA1), + (0x0061, 0x0325, 0x1E01), + (0x0061, 0x0328, 0x0105), + (0x0062, 0x0307, 0x1E03), + (0x0062, 0x0323, 0x1E05), + (0x0062, 0x0331, 0x1E07), + (0x0063, 0x0301, 0x0107), + (0x0063, 0x0302, 0x0109), + (0x0063, 0x0307, 0x010B), + (0x0063, 0x030C, 0x010D), + (0x0063, 0x0327, 0x00E7), + (0x0064, 0x0307, 0x1E0B), + (0x0064, 0x030C, 0x010F), + (0x0064, 0x0323, 0x1E0D), + (0x0064, 0x0327, 0x1E11), + (0x0064, 0x032D, 0x1E13), + (0x0064, 0x0331, 0x1E0F), + (0x0065, 0x0300, 0x00E8), + (0x0065, 0x0301, 0x00E9), + (0x0065, 0x0302, 0x00EA), + (0x0065, 0x0303, 0x1EBD), + (0x0065, 0x0304, 0x0113), + (0x0065, 0x0306, 0x0115), + (0x0065, 0x0307, 0x0117), + (0x0065, 0x0308, 0x00EB), + (0x0065, 0x0309, 0x1EBB), + (0x0065, 0x030C, 0x011B), + (0x0065, 0x030F, 0x0205), + (0x0065, 0x0311, 0x0207), + (0x0065, 0x0323, 0x1EB9), + (0x0065, 0x0327, 0x0229), + (0x0065, 0x0328, 0x0119), + (0x0065, 0x032D, 0x1E19), + (0x0065, 0x0330, 0x1E1B), + (0x0066, 0x0307, 0x1E1F), + (0x0067, 0x0301, 0x01F5), + (0x0067, 0x0302, 0x011D), + (0x0067, 0x0304, 0x1E21), + (0x0067, 0x0306, 0x011F), + (0x0067, 0x0307, 0x0121), + (0x0067, 0x030C, 0x01E7), + (0x0067, 0x0327, 0x0123), + (0x0068, 0x0302, 0x0125), + (0x0068, 0x0307, 0x1E23), + (0x0068, 0x0308, 0x1E27), + (0x0068, 0x030C, 0x021F), + (0x0068, 0x0323, 0x1E25), + (0x0068, 0x0327, 0x1E29), + (0x0068, 0x032E, 0x1E2B), + (0x0068, 0x0331, 0x1E96), + (0x0069, 0x0300, 0x00EC), + (0x0069, 0x0301, 0x00ED), + (0x0069, 0x0302, 0x00EE), + (0x0069, 0x0303, 0x0129), + (0x0069, 0x0304, 0x012B), + (0x0069, 0x0306, 0x012D), + (0x0069, 0x0308, 0x00EF), + (0x0069, 0x0309, 0x1EC9), + (0x0069, 0x030C, 0x01D0), + (0x0069, 0x030F, 0x0209), + (0x0069, 0x0311, 0x020B), + (0x0069, 0x0323, 0x1ECB), + (0x0069, 0x0328, 0x012F), + (0x0069, 0x0330, 0x1E2D), + (0x006A, 0x0302, 0x0135), + (0x006A, 0x030C, 0x01F0), + (0x006B, 0x0301, 0x1E31), + (0x006B, 0x030C, 0x01E9), + (0x006B, 0x0323, 0x1E33), + (0x006B, 0x0327, 0x0137), + (0x006B, 0x0331, 0x1E35), + (0x006C, 0x0301, 0x013A), + (0x006C, 0x030C, 0x013E), + (0x006C, 0x0323, 0x1E37), + (0x006C, 0x0327, 0x013C), + (0x006C, 0x032D, 0x1E3D), + (0x006C, 0x0331, 0x1E3B), + (0x006D, 0x0301, 0x1E3F), + (0x006D, 0x0307, 0x1E41), + (0x006D, 0x0323, 0x1E43), + (0x006E, 0x0300, 0x01F9), + (0x006E, 0x0301, 0x0144), + (0x006E, 0x0303, 0x00F1), + (0x006E, 0x0307, 0x1E45), + (0x006E, 0x030C, 0x0148), + (0x006E, 0x0323, 0x1E47), + (0x006E, 0x0327, 0x0146), + (0x006E, 0x032D, 0x1E4B), + (0x006E, 0x0331, 0x1E49), + (0x006F, 0x0300, 0x00F2), + (0x006F, 0x0301, 0x00F3), + (0x006F, 0x0302, 0x00F4), + (0x006F, 0x0303, 0x00F5), + (0x006F, 0x0304, 0x014D), + (0x006F, 0x0306, 0x014F), + (0x006F, 0x0307, 0x022F), + (0x006F, 0x0308, 0x00F6), + (0x006F, 0x0309, 0x1ECF), + (0x006F, 0x030B, 0x0151), + (0x006F, 0x030C, 0x01D2), + (0x006F, 0x030F, 0x020D), + (0x006F, 0x0311, 0x020F), + (0x006F, 0x031B, 0x01A1), + (0x006F, 0x0323, 0x1ECD), + (0x006F, 0x0328, 0x01EB), + (0x0070, 0x0301, 0x1E55), + (0x0070, 0x0307, 0x1E57), + (0x0072, 0x0301, 0x0155), + (0x0072, 0x0307, 0x1E59), + (0x0072, 0x030C, 0x0159), + (0x0072, 0x030F, 0x0211), + (0x0072, 0x0311, 0x0213), + (0x0072, 0x0323, 0x1E5B), + (0x0072, 0x0327, 0x0157), + (0x0072, 0x0331, 0x1E5F), + (0x0073, 0x0301, 0x015B), + (0x0073, 0x0302, 0x015D), + (0x0073, 0x0307, 0x1E61), + (0x0073, 0x030C, 0x0161), + (0x0073, 0x0323, 0x1E63), + (0x0073, 0x0326, 0x0219), + (0x0073, 0x0327, 0x015F), + (0x0074, 0x0307, 0x1E6B), + (0x0074, 0x0308, 0x1E97), + (0x0074, 0x030C, 0x0165), + (0x0074, 0x0323, 0x1E6D), + (0x0074, 0x0326, 0x021B), + (0x0074, 0x0327, 0x0163), + (0x0074, 0x032D, 0x1E71), + (0x0074, 0x0331, 0x1E6F), + (0x0075, 0x0300, 0x00F9), + (0x0075, 0x0301, 0x00FA), + (0x0075, 0x0302, 0x00FB), + (0x0075, 0x0303, 0x0169), + (0x0075, 0x0304, 0x016B), + (0x0075, 0x0306, 0x016D), + (0x0075, 0x0308, 0x00FC), + (0x0075, 0x0309, 0x1EE7), + (0x0075, 0x030A, 0x016F), + (0x0075, 0x030B, 0x0171), + (0x0075, 0x030C, 0x01D4), + (0x0075, 0x030F, 0x0215), + (0x0075, 0x0311, 0x0217), + (0x0075, 0x031B, 0x01B0), + (0x0075, 0x0323, 0x1EE5), + (0x0075, 0x0324, 0x1E73), + (0x0075, 0x0328, 0x0173), + (0x0075, 0x032D, 0x1E77), + (0x0075, 0x0330, 0x1E75), + (0x0076, 0x0303, 0x1E7D), + (0x0076, 0x0323, 0x1E7F), + (0x0077, 0x0300, 0x1E81), + (0x0077, 0x0301, 0x1E83), + (0x0077, 0x0302, 0x0175), + (0x0077, 0x0307, 0x1E87), + (0x0077, 0x0308, 0x1E85), + (0x0077, 0x030A, 0x1E98), + (0x0077, 0x0323, 0x1E89), + (0x0078, 0x0307, 0x1E8B), + (0x0078, 0x0308, 0x1E8D), + (0x0079, 0x0300, 0x1EF3), + (0x0079, 0x0301, 0x00FD), + (0x0079, 0x0302, 0x0177), + (0x0079, 0x0303, 0x1EF9), + (0x0079, 0x0304, 0x0233), + (0x0079, 0x0307, 0x1E8F), + (0x0079, 0x0308, 0x00FF), + (0x0079, 0x0309, 0x1EF7), + (0x0079, 0x030A, 0x1E99), + (0x0079, 0x0323, 0x1EF5), + (0x007A, 0x0301, 0x017A), + (0x007A, 0x0302, 0x1E91), + (0x007A, 0x0307, 0x017C), + (0x007A, 0x030C, 0x017E), + (0x007A, 0x0323, 0x1E93), + (0x007A, 0x0331, 0x1E95), + (0x00A8, 0x0300, 0x1FED), + (0x00A8, 0x0301, 0x0385), + (0x00A8, 0x0342, 0x1FC1), + (0x00C2, 0x0300, 0x1EA6), + (0x00C2, 0x0301, 0x1EA4), + (0x00C2, 0x0303, 0x1EAA), + (0x00C2, 0x0309, 0x1EA8), + (0x00C4, 0x0304, 0x01DE), + (0x00C5, 0x0301, 0x01FA), + (0x00C6, 0x0301, 0x01FC), + (0x00C6, 0x0304, 0x01E2), + (0x00C7, 0x0301, 0x1E08), + (0x00CA, 0x0300, 0x1EC0), + (0x00CA, 0x0301, 0x1EBE), + (0x00CA, 0x0303, 0x1EC4), + (0x00CA, 0x0309, 0x1EC2), + (0x00CF, 0x0301, 0x1E2E), + (0x00D4, 0x0300, 0x1ED2), + (0x00D4, 0x0301, 0x1ED0), + (0x00D4, 0x0303, 0x1ED6), + (0x00D4, 0x0309, 0x1ED4), + (0x00D5, 0x0301, 0x1E4C), + (0x00D5, 0x0304, 0x022C), + (0x00D5, 0x0308, 0x1E4E), + (0x00D6, 0x0304, 0x022A), + (0x00D8, 0x0301, 0x01FE), + (0x00DC, 0x0300, 0x01DB), + (0x00DC, 0x0301, 0x01D7), + (0x00DC, 0x0304, 0x01D5), + (0x00DC, 0x030C, 0x01D9), + (0x00E2, 0x0300, 0x1EA7), + (0x00E2, 0x0301, 0x1EA5), + (0x00E2, 0x0303, 0x1EAB), + (0x00E2, 0x0309, 0x1EA9), + (0x00E4, 0x0304, 0x01DF), + (0x00E5, 0x0301, 0x01FB), + (0x00E6, 0x0301, 0x01FD), + (0x00E6, 0x0304, 0x01E3), + (0x00E7, 0x0301, 0x1E09), + (0x00EA, 0x0300, 0x1EC1), + (0x00EA, 0x0301, 0x1EBF), + (0x00EA, 0x0303, 0x1EC5), + (0x00EA, 0x0309, 0x1EC3), + (0x00EF, 0x0301, 0x1E2F), + (0x00F4, 0x0300, 0x1ED3), + (0x00F4, 0x0301, 0x1ED1), + (0x00F4, 0x0303, 0x1ED7), + (0x00F4, 0x0309, 0x1ED5), + (0x00F5, 0x0301, 0x1E4D), + (0x00F5, 0x0304, 0x022D), + (0x00F5, 0x0308, 0x1E4F), + (0x00F6, 0x0304, 0x022B), + (0x00F8, 0x0301, 0x01FF), + (0x00FC, 0x0300, 0x01DC), + (0x00FC, 0x0301, 0x01D8), + (0x00FC, 0x0304, 0x01D6), + (0x00FC, 0x030C, 0x01DA), + (0x0102, 0x0300, 0x1EB0), + (0x0102, 0x0301, 0x1EAE), + (0x0102, 0x0303, 0x1EB4), + (0x0102, 0x0309, 0x1EB2), + (0x0103, 0x0300, 0x1EB1), + (0x0103, 0x0301, 0x1EAF), + (0x0103, 0x0303, 0x1EB5), + (0x0103, 0x0309, 0x1EB3), + (0x0112, 0x0300, 0x1E14), + (0x0112, 0x0301, 0x1E16), + (0x0113, 0x0300, 0x1E15), + (0x0113, 0x0301, 0x1E17), + (0x014C, 0x0300, 0x1E50), + (0x014C, 0x0301, 0x1E52), + (0x014D, 0x0300, 0x1E51), + (0x014D, 0x0301, 0x1E53), + (0x015A, 0x0307, 0x1E64), + (0x015B, 0x0307, 0x1E65), + (0x0160, 0x0307, 0x1E66), + (0x0161, 0x0307, 0x1E67), + (0x0168, 0x0301, 0x1E78), + (0x0169, 0x0301, 0x1E79), + (0x016A, 0x0308, 0x1E7A), + (0x016B, 0x0308, 0x1E7B), + (0x017F, 0x0307, 0x1E9B), + (0x01A0, 0x0300, 0x1EDC), + (0x01A0, 0x0301, 0x1EDA), + (0x01A0, 0x0303, 0x1EE0), + (0x01A0, 0x0309, 0x1EDE), + (0x01A0, 0x0323, 0x1EE2), + (0x01A1, 0x0300, 0x1EDD), + (0x01A1, 0x0301, 0x1EDB), + (0x01A1, 0x0303, 0x1EE1), + (0x01A1, 0x0309, 0x1EDF), + (0x01A1, 0x0323, 0x1EE3), + (0x01AF, 0x0300, 0x1EEA), + (0x01AF, 0x0301, 0x1EE8), + (0x01AF, 0x0303, 0x1EEE), + (0x01AF, 0x0309, 0x1EEC), + (0x01AF, 0x0323, 0x1EF0), + (0x01B0, 0x0300, 0x1EEB), + (0x01B0, 0x0301, 0x1EE9), + (0x01B0, 0x0303, 0x1EEF), + (0x01B0, 0x0309, 0x1EED), + (0x01B0, 0x0323, 0x1EF1), + (0x01B7, 0x030C, 0x01EE), + (0x01EA, 0x0304, 0x01EC), + (0x01EB, 0x0304, 0x01ED), + (0x0226, 0x0304, 0x01E0), + (0x0227, 0x0304, 0x01E1), + (0x0228, 0x0306, 0x1E1C), + (0x0229, 0x0306, 0x1E1D), + (0x022E, 0x0304, 0x0230), + (0x022F, 0x0304, 0x0231), + (0x0292, 0x030C, 0x01EF), + (0x0391, 0x0300, 0x1FBA), + (0x0391, 0x0301, 0x0386), + (0x0391, 0x0304, 0x1FB9), + (0x0391, 0x0306, 0x1FB8), + (0x0391, 0x0313, 0x1F08), + (0x0391, 0x0314, 0x1F09), + (0x0391, 0x0345, 0x1FBC), + (0x0395, 0x0300, 0x1FC8), + (0x0395, 0x0301, 0x0388), + (0x0395, 0x0313, 0x1F18), + (0x0395, 0x0314, 0x1F19), + (0x0397, 0x0300, 0x1FCA), + (0x0397, 0x0301, 0x0389), + (0x0397, 0x0313, 0x1F28), + (0x0397, 0x0314, 0x1F29), + (0x0397, 0x0345, 0x1FCC), + (0x0399, 0x0300, 0x1FDA), + (0x0399, 0x0301, 0x038A), + (0x0399, 0x0304, 0x1FD9), + (0x0399, 0x0306, 0x1FD8), + (0x0399, 0x0308, 0x03AA), + (0x0399, 0x0313, 0x1F38), + (0x0399, 0x0314, 0x1F39), + (0x039F, 0x0300, 0x1FF8), + (0x039F, 0x0301, 0x038C), + (0x039F, 0x0313, 0x1F48), + (0x039F, 0x0314, 0x1F49), + (0x03A1, 0x0314, 0x1FEC), + (0x03A5, 0x0300, 0x1FEA), + (0x03A5, 0x0301, 0x038E), + (0x03A5, 0x0304, 0x1FE9), + (0x03A5, 0x0306, 0x1FE8), + (0x03A5, 0x0308, 0x03AB), + (0x03A5, 0x0314, 0x1F59), + (0x03A9, 0x0300, 0x1FFA), + (0x03A9, 0x0301, 0x038F), + (0x03A9, 0x0313, 0x1F68), + (0x03A9, 0x0314, 0x1F69), + (0x03A9, 0x0345, 0x1FFC), + (0x03AC, 0x0345, 0x1FB4), + (0x03AE, 0x0345, 0x1FC4), + (0x03B1, 0x0300, 0x1F70), + (0x03B1, 0x0301, 0x03AC), + (0x03B1, 0x0304, 0x1FB1), + (0x03B1, 0x0306, 0x1FB0), + (0x03B1, 0x0313, 0x1F00), + (0x03B1, 0x0314, 0x1F01), + (0x03B1, 0x0342, 0x1FB6), + (0x03B1, 0x0345, 0x1FB3), + (0x03B5, 0x0300, 0x1F72), + (0x03B5, 0x0301, 0x03AD), + (0x03B5, 0x0313, 0x1F10), + (0x03B5, 0x0314, 0x1F11), + (0x03B7, 0x0300, 0x1F74), + (0x03B7, 0x0301, 0x03AE), + (0x03B7, 0x0313, 0x1F20), + (0x03B7, 0x0314, 0x1F21), + (0x03B7, 0x0342, 0x1FC6), + (0x03B7, 0x0345, 0x1FC3), + (0x03B9, 0x0300, 0x1F76), + (0x03B9, 0x0301, 0x03AF), + (0x03B9, 0x0304, 0x1FD1), + (0x03B9, 0x0306, 0x1FD0), + (0x03B9, 0x0308, 0x03CA), + (0x03B9, 0x0313, 0x1F30), + (0x03B9, 0x0314, 0x1F31), + (0x03B9, 0x0342, 0x1FD6), + (0x03BF, 0x0300, 0x1F78), + (0x03BF, 0x0301, 0x03CC), + (0x03BF, 0x0313, 0x1F40), + (0x03BF, 0x0314, 0x1F41), + (0x03C1, 0x0313, 0x1FE4), + (0x03C1, 0x0314, 0x1FE5), + (0x03C5, 0x0300, 0x1F7A), + (0x03C5, 0x0301, 0x03CD), + (0x03C5, 0x0304, 0x1FE1), + (0x03C5, 0x0306, 0x1FE0), + (0x03C5, 0x0308, 0x03CB), + (0x03C5, 0x0313, 0x1F50), + (0x03C5, 0x0314, 0x1F51), + (0x03C5, 0x0342, 0x1FE6), + (0x03C9, 0x0300, 0x1F7C), + (0x03C9, 0x0301, 0x03CE), + (0x03C9, 0x0313, 0x1F60), + (0x03C9, 0x0314, 0x1F61), + (0x03C9, 0x0342, 0x1FF6), + (0x03C9, 0x0345, 0x1FF3), + (0x03CA, 0x0300, 0x1FD2), + (0x03CA, 0x0301, 0x0390), + (0x03CA, 0x0342, 0x1FD7), + (0x03CB, 0x0300, 0x1FE2), + (0x03CB, 0x0301, 0x03B0), + (0x03CB, 0x0342, 0x1FE7), + (0x03CE, 0x0345, 0x1FF4), + (0x03D2, 0x0301, 0x03D3), + (0x03D2, 0x0308, 0x03D4), + (0x0406, 0x0308, 0x0407), + (0x0410, 0x0306, 0x04D0), + (0x0410, 0x0308, 0x04D2), + (0x0413, 0x0301, 0x0403), + (0x0415, 0x0300, 0x0400), + (0x0415, 0x0306, 0x04D6), + (0x0415, 0x0308, 0x0401), + (0x0416, 0x0306, 0x04C1), + (0x0416, 0x0308, 0x04DC), + (0x0417, 0x0308, 0x04DE), + (0x0418, 0x0300, 0x040D), + (0x0418, 0x0304, 0x04E2), + (0x0418, 0x0306, 0x0419), + (0x0418, 0x0308, 0x04E4), + (0x041A, 0x0301, 0x040C), + (0x041E, 0x0308, 0x04E6), + (0x0423, 0x0304, 0x04EE), + (0x0423, 0x0306, 0x040E), + (0x0423, 0x0308, 0x04F0), + (0x0423, 0x030B, 0x04F2), + (0x0427, 0x0308, 0x04F4), + (0x042B, 0x0308, 0x04F8), + (0x042D, 0x0308, 0x04EC), + (0x0430, 0x0306, 0x04D1), + (0x0430, 0x0308, 0x04D3), + (0x0433, 0x0301, 0x0453), + (0x0435, 0x0300, 0x0450), + (0x0435, 0x0306, 0x04D7), + (0x0435, 0x0308, 0x0451), + (0x0436, 0x0306, 0x04C2), + (0x0436, 0x0308, 0x04DD), + (0x0437, 0x0308, 0x04DF), + (0x0438, 0x0300, 0x045D), + (0x0438, 0x0304, 0x04E3), + (0x0438, 0x0306, 0x0439), + (0x0438, 0x0308, 0x04E5), + (0x043A, 0x0301, 0x045C), + (0x043E, 0x0308, 0x04E7), + (0x0443, 0x0304, 0x04EF), + (0x0443, 0x0306, 0x045E), + (0x0443, 0x0308, 0x04F1), + (0x0443, 0x030B, 0x04F3), + (0x0447, 0x0308, 0x04F5), + (0x044B, 0x0308, 0x04F9), + (0x044D, 0x0308, 0x04ED), + (0x0456, 0x0308, 0x0457), + (0x0474, 0x030F, 0x0476), + (0x0475, 0x030F, 0x0477), + (0x04D8, 0x0308, 0x04DA), + (0x04D9, 0x0308, 0x04DB), + (0x04E8, 0x0308, 0x04EA), + (0x04E9, 0x0308, 0x04EB), + (0x0627, 0x0653, 0x0622), + (0x0627, 0x0654, 0x0623), + (0x0627, 0x0655, 0x0625), + (0x0648, 0x0654, 0x0624), + (0x064A, 0x0654, 0x0626), + (0x06C1, 0x0654, 0x06C2), + (0x06D2, 0x0654, 0x06D3), + (0x06D5, 0x0654, 0x06C0), + (0x0928, 0x093C, 0x0929), + (0x0930, 0x093C, 0x0931), + (0x0933, 0x093C, 0x0934), + (0x09C7, 0x09BE, 0x09CB), + (0x09C7, 0x09D7, 0x09CC), + (0x0B47, 0x0B3E, 0x0B4B), + (0x0B47, 0x0B56, 0x0B48), + (0x0B47, 0x0B57, 0x0B4C), + (0x0B92, 0x0BD7, 0x0B94), + (0x0BC6, 0x0BBE, 0x0BCA), + (0x0BC6, 0x0BD7, 0x0BCC), + (0x0BC7, 0x0BBE, 0x0BCB), + (0x0C46, 0x0C56, 0x0C48), + (0x0CBF, 0x0CD5, 0x0CC0), + (0x0CC6, 0x0CC2, 0x0CCA), + (0x0CC6, 0x0CD5, 0x0CC7), + (0x0CC6, 0x0CD6, 0x0CC8), + (0x0CCA, 0x0CD5, 0x0CCB), + (0x0D46, 0x0D3E, 0x0D4A), + (0x0D46, 0x0D57, 0x0D4C), + (0x0D47, 0x0D3E, 0x0D4B), + (0x0DD9, 0x0DCA, 0x0DDA), + (0x0DD9, 0x0DCF, 0x0DDC), + (0x0DD9, 0x0DDF, 0x0DDE), + (0x0DDC, 0x0DCA, 0x0DDD), + (0x1025, 0x102E, 0x1026), + (0x1B05, 0x1B35, 0x1B06), + (0x1B07, 0x1B35, 0x1B08), + (0x1B09, 0x1B35, 0x1B0A), + (0x1B0B, 0x1B35, 0x1B0C), + (0x1B0D, 0x1B35, 0x1B0E), + (0x1B11, 0x1B35, 0x1B12), + (0x1B3A, 0x1B35, 0x1B3B), + (0x1B3C, 0x1B35, 0x1B3D), + (0x1B3E, 0x1B35, 0x1B40), + (0x1B3F, 0x1B35, 0x1B41), + (0x1B42, 0x1B35, 0x1B43), + (0x1E36, 0x0304, 0x1E38), + (0x1E37, 0x0304, 0x1E39), + (0x1E5A, 0x0304, 0x1E5C), + (0x1E5B, 0x0304, 0x1E5D), + (0x1E62, 0x0307, 0x1E68), + (0x1E63, 0x0307, 0x1E69), + (0x1EA0, 0x0302, 0x1EAC), + (0x1EA0, 0x0306, 0x1EB6), + (0x1EA1, 0x0302, 0x1EAD), + (0x1EA1, 0x0306, 0x1EB7), + (0x1EB8, 0x0302, 0x1EC6), + (0x1EB9, 0x0302, 0x1EC7), + (0x1ECC, 0x0302, 0x1ED8), + (0x1ECD, 0x0302, 0x1ED9), + (0x1F00, 0x0300, 0x1F02), + (0x1F00, 0x0301, 0x1F04), + (0x1F00, 0x0342, 0x1F06), + (0x1F00, 0x0345, 0x1F80), + (0x1F01, 0x0300, 0x1F03), + (0x1F01, 0x0301, 0x1F05), + (0x1F01, 0x0342, 0x1F07), + (0x1F01, 0x0345, 0x1F81), + (0x1F02, 0x0345, 0x1F82), + (0x1F03, 0x0345, 0x1F83), + (0x1F04, 0x0345, 0x1F84), + (0x1F05, 0x0345, 0x1F85), + (0x1F06, 0x0345, 0x1F86), + (0x1F07, 0x0345, 0x1F87), + (0x1F08, 0x0300, 0x1F0A), + (0x1F08, 0x0301, 0x1F0C), + (0x1F08, 0x0342, 0x1F0E), + (0x1F08, 0x0345, 0x1F88), + (0x1F09, 0x0300, 0x1F0B), + (0x1F09, 0x0301, 0x1F0D), + (0x1F09, 0x0342, 0x1F0F), + (0x1F09, 0x0345, 0x1F89), + (0x1F0A, 0x0345, 0x1F8A), + (0x1F0B, 0x0345, 0x1F8B), + (0x1F0C, 0x0345, 0x1F8C), + (0x1F0D, 0x0345, 0x1F8D), + (0x1F0E, 0x0345, 0x1F8E), + (0x1F0F, 0x0345, 0x1F8F), + (0x1F10, 0x0300, 0x1F12), + (0x1F10, 0x0301, 0x1F14), + (0x1F11, 0x0300, 0x1F13), + (0x1F11, 0x0301, 0x1F15), + (0x1F18, 0x0300, 0x1F1A), + (0x1F18, 0x0301, 0x1F1C), + (0x1F19, 0x0300, 0x1F1B), + (0x1F19, 0x0301, 0x1F1D), + (0x1F20, 0x0300, 0x1F22), + (0x1F20, 0x0301, 0x1F24), + (0x1F20, 0x0342, 0x1F26), + (0x1F20, 0x0345, 0x1F90), + (0x1F21, 0x0300, 0x1F23), + (0x1F21, 0x0301, 0x1F25), + (0x1F21, 0x0342, 0x1F27), + (0x1F21, 0x0345, 0x1F91), + (0x1F22, 0x0345, 0x1F92), + (0x1F23, 0x0345, 0x1F93), + (0x1F24, 0x0345, 0x1F94), + (0x1F25, 0x0345, 0x1F95), + (0x1F26, 0x0345, 0x1F96), + (0x1F27, 0x0345, 0x1F97), + (0x1F28, 0x0300, 0x1F2A), + (0x1F28, 0x0301, 0x1F2C), + (0x1F28, 0x0342, 0x1F2E), + (0x1F28, 0x0345, 0x1F98), + (0x1F29, 0x0300, 0x1F2B), + (0x1F29, 0x0301, 0x1F2D), + (0x1F29, 0x0342, 0x1F2F), + (0x1F29, 0x0345, 0x1F99), + (0x1F2A, 0x0345, 0x1F9A), + (0x1F2B, 0x0345, 0x1F9B), + (0x1F2C, 0x0345, 0x1F9C), + (0x1F2D, 0x0345, 0x1F9D), + (0x1F2E, 0x0345, 0x1F9E), + (0x1F2F, 0x0345, 0x1F9F), + (0x1F30, 0x0300, 0x1F32), + (0x1F30, 0x0301, 0x1F34), + (0x1F30, 0x0342, 0x1F36), + (0x1F31, 0x0300, 0x1F33), + (0x1F31, 0x0301, 0x1F35), + (0x1F31, 0x0342, 0x1F37), + (0x1F38, 0x0300, 0x1F3A), + (0x1F38, 0x0301, 0x1F3C), + (0x1F38, 0x0342, 0x1F3E), + (0x1F39, 0x0300, 0x1F3B), + (0x1F39, 0x0301, 0x1F3D), + (0x1F39, 0x0342, 0x1F3F), + (0x1F40, 0x0300, 0x1F42), + (0x1F40, 0x0301, 0x1F44), + (0x1F41, 0x0300, 0x1F43), + (0x1F41, 0x0301, 0x1F45), + (0x1F48, 0x0300, 0x1F4A), + (0x1F48, 0x0301, 0x1F4C), + (0x1F49, 0x0300, 0x1F4B), + (0x1F49, 0x0301, 0x1F4D), + (0x1F50, 0x0300, 0x1F52), + (0x1F50, 0x0301, 0x1F54), + (0x1F50, 0x0342, 0x1F56), + (0x1F51, 0x0300, 0x1F53), + (0x1F51, 0x0301, 0x1F55), + (0x1F51, 0x0342, 0x1F57), + (0x1F59, 0x0300, 0x1F5B), + (0x1F59, 0x0301, 0x1F5D), + (0x1F59, 0x0342, 0x1F5F), + (0x1F60, 0x0300, 0x1F62), + (0x1F60, 0x0301, 0x1F64), + (0x1F60, 0x0342, 0x1F66), + (0x1F60, 0x0345, 0x1FA0), + (0x1F61, 0x0300, 0x1F63), + (0x1F61, 0x0301, 0x1F65), + (0x1F61, 0x0342, 0x1F67), + (0x1F61, 0x0345, 0x1FA1), + (0x1F62, 0x0345, 0x1FA2), + (0x1F63, 0x0345, 0x1FA3), + (0x1F64, 0x0345, 0x1FA4), + (0x1F65, 0x0345, 0x1FA5), + (0x1F66, 0x0345, 0x1FA6), + (0x1F67, 0x0345, 0x1FA7), + (0x1F68, 0x0300, 0x1F6A), + (0x1F68, 0x0301, 0x1F6C), + (0x1F68, 0x0342, 0x1F6E), + (0x1F68, 0x0345, 0x1FA8), + (0x1F69, 0x0300, 0x1F6B), + (0x1F69, 0x0301, 0x1F6D), + (0x1F69, 0x0342, 0x1F6F), + (0x1F69, 0x0345, 0x1FA9), + (0x1F6A, 0x0345, 0x1FAA), + (0x1F6B, 0x0345, 0x1FAB), + (0x1F6C, 0x0345, 0x1FAC), + (0x1F6D, 0x0345, 0x1FAD), + (0x1F6E, 0x0345, 0x1FAE), + (0x1F6F, 0x0345, 0x1FAF), + (0x1F70, 0x0345, 0x1FB2), + (0x1F74, 0x0345, 0x1FC2), + (0x1F7C, 0x0345, 0x1FF2), + (0x1FB6, 0x0345, 0x1FB7), + (0x1FBF, 0x0300, 0x1FCD), + (0x1FBF, 0x0301, 0x1FCE), + (0x1FBF, 0x0342, 0x1FCF), + (0x1FC6, 0x0345, 0x1FC7), + (0x1FF6, 0x0345, 0x1FF7), + (0x1FFE, 0x0300, 0x1FDD), + (0x1FFE, 0x0301, 0x1FDE), + (0x1FFE, 0x0342, 0x1FDF), + (0x2190, 0x0338, 0x219A), + (0x2192, 0x0338, 0x219B), + (0x2194, 0x0338, 0x21AE), + (0x21D0, 0x0338, 0x21CD), + (0x21D2, 0x0338, 0x21CF), + (0x21D4, 0x0338, 0x21CE), + (0x2203, 0x0338, 0x2204), + (0x2208, 0x0338, 0x2209), + (0x220B, 0x0338, 0x220C), + (0x2223, 0x0338, 0x2224), + (0x2225, 0x0338, 0x2226), + (0x223C, 0x0338, 0x2241), + (0x2243, 0x0338, 0x2244), + (0x2245, 0x0338, 0x2247), + (0x2248, 0x0338, 0x2249), + (0x224D, 0x0338, 0x226D), + (0x2261, 0x0338, 0x2262), + (0x2264, 0x0338, 0x2270), + (0x2265, 0x0338, 0x2271), + (0x2272, 0x0338, 0x2274), + (0x2273, 0x0338, 0x2275), + (0x2276, 0x0338, 0x2278), + (0x2277, 0x0338, 0x2279), + (0x227A, 0x0338, 0x2280), + (0x227B, 0x0338, 0x2281), + (0x227C, 0x0338, 0x22E0), + (0x227D, 0x0338, 0x22E1), + (0x2282, 0x0338, 0x2284), + (0x2283, 0x0338, 0x2285), + (0x2286, 0x0338, 0x2288), + (0x2287, 0x0338, 0x2289), + (0x2291, 0x0338, 0x22E2), + (0x2292, 0x0338, 0x22E3), + (0x22A2, 0x0338, 0x22AC), + (0x22A8, 0x0338, 0x22AD), + (0x22A9, 0x0338, 0x22AE), + (0x22AB, 0x0338, 0x22AF), + (0x22B2, 0x0338, 0x22EA), + (0x22B3, 0x0338, 0x22EB), + (0x22B4, 0x0338, 0x22EC), + (0x22B5, 0x0338, 0x22ED), + (0x3046, 0x3099, 0x3094), + (0x304B, 0x3099, 0x304C), + (0x304D, 0x3099, 0x304E), + (0x304F, 0x3099, 0x3050), + (0x3051, 0x3099, 0x3052), + (0x3053, 0x3099, 0x3054), + (0x3055, 0x3099, 0x3056), + (0x3057, 0x3099, 0x3058), + (0x3059, 0x3099, 0x305A), + (0x305B, 0x3099, 0x305C), + (0x305D, 0x3099, 0x305E), + (0x305F, 0x3099, 0x3060), + (0x3061, 0x3099, 0x3062), + (0x3064, 0x3099, 0x3065), + (0x3066, 0x3099, 0x3067), + (0x3068, 0x3099, 0x3069), + (0x306F, 0x3099, 0x3070), + (0x306F, 0x309A, 0x3071), + (0x3072, 0x3099, 0x3073), + (0x3072, 0x309A, 0x3074), + (0x3075, 0x3099, 0x3076), + (0x3075, 0x309A, 0x3077), + (0x3078, 0x3099, 0x3079), + (0x3078, 0x309A, 0x307A), + (0x307B, 0x3099, 0x307C), + (0x307B, 0x309A, 0x307D), + (0x309D, 0x3099, 0x309E), + (0x30A6, 0x3099, 0x30F4), + (0x30AB, 0x3099, 0x30AC), + (0x30AD, 0x3099, 0x30AE), + (0x30AF, 0x3099, 0x30B0), + (0x30B1, 0x3099, 0x30B2), + (0x30B3, 0x3099, 0x30B4), + (0x30B5, 0x3099, 0x30B6), + (0x30B7, 0x3099, 0x30B8), + (0x30B9, 0x3099, 0x30BA), + (0x30BB, 0x3099, 0x30BC), + (0x30BD, 0x3099, 0x30BE), + (0x30BF, 0x3099, 0x30C0), + (0x30C1, 0x3099, 0x30C2), + (0x30C4, 0x3099, 0x30C5), + (0x30C6, 0x3099, 0x30C7), + (0x30C8, 0x3099, 0x30C9), + (0x30CF, 0x3099, 0x30D0), + (0x30CF, 0x309A, 0x30D1), + (0x30D2, 0x3099, 0x30D3), + (0x30D2, 0x309A, 0x30D4), + (0x30D5, 0x3099, 0x30D6), + (0x30D5, 0x309A, 0x30D7), + (0x30D8, 0x3099, 0x30D9), + (0x30D8, 0x309A, 0x30DA), + (0x30DB, 0x3099, 0x30DC), + (0x30DB, 0x309A, 0x30DD), + (0x30EF, 0x3099, 0x30F7), + (0x30F0, 0x3099, 0x30F8), + (0x30F1, 0x3099, 0x30F9), + (0x30F2, 0x3099, 0x30FA), + (0x30FD, 0x3099, 0x30FE), + (0x11099, 0x110BA, 0x1109A), + (0x1109B, 0x110BA, 0x1109C), + (0x110A5, 0x110BA, 0x110AB), +]; + +/// `(first, last, class)` for every run of characters that share a non-zero canonical combining +/// class, sorted. +pub(super) static COMBINING_CLASSES: [(u32, u32, u8); 266] = [ + (0x0300, 0x0314, 230), + (0x0315, 0x0315, 232), + (0x0316, 0x0319, 220), + (0x031A, 0x031A, 232), + (0x031B, 0x031B, 216), + (0x031C, 0x0320, 220), + (0x0321, 0x0322, 202), + (0x0323, 0x0326, 220), + (0x0327, 0x0328, 202), + (0x0329, 0x0333, 220), + (0x0334, 0x0338, 1), + (0x0339, 0x033C, 220), + (0x033D, 0x0344, 230), + (0x0345, 0x0345, 240), + (0x0346, 0x0346, 230), + (0x0347, 0x0349, 220), + (0x034A, 0x034C, 230), + (0x034D, 0x034E, 220), + (0x0350, 0x0352, 230), + (0x0353, 0x0356, 220), + (0x0357, 0x0357, 230), + (0x0358, 0x0358, 232), + (0x0359, 0x035A, 220), + (0x035B, 0x035B, 230), + (0x035C, 0x035C, 233), + (0x035D, 0x035E, 234), + (0x035F, 0x035F, 233), + (0x0360, 0x0361, 234), + (0x0362, 0x0362, 233), + (0x0363, 0x036F, 230), + (0x0483, 0x0487, 230), + (0x0591, 0x0591, 220), + (0x0592, 0x0595, 230), + (0x0596, 0x0596, 220), + (0x0597, 0x0599, 230), + (0x059A, 0x059A, 222), + (0x059B, 0x059B, 220), + (0x059C, 0x05A1, 230), + (0x05A2, 0x05A7, 220), + (0x05A8, 0x05A9, 230), + (0x05AA, 0x05AA, 220), + (0x05AB, 0x05AC, 230), + (0x05AD, 0x05AD, 222), + (0x05AE, 0x05AE, 228), + (0x05AF, 0x05AF, 230), + (0x05B0, 0x05B0, 10), + (0x05B1, 0x05B1, 11), + (0x05B2, 0x05B2, 12), + (0x05B3, 0x05B3, 13), + (0x05B4, 0x05B4, 14), + (0x05B5, 0x05B5, 15), + (0x05B6, 0x05B6, 16), + (0x05B7, 0x05B7, 17), + (0x05B8, 0x05B8, 18), + (0x05B9, 0x05BA, 19), + (0x05BB, 0x05BB, 20), + (0x05BC, 0x05BC, 21), + (0x05BD, 0x05BD, 22), + (0x05BF, 0x05BF, 23), + (0x05C1, 0x05C1, 24), + (0x05C2, 0x05C2, 25), + (0x05C4, 0x05C4, 230), + (0x05C5, 0x05C5, 220), + (0x05C7, 0x05C7, 18), + (0x0610, 0x0617, 230), + (0x0618, 0x0618, 30), + (0x0619, 0x0619, 31), + (0x061A, 0x061A, 32), + (0x064B, 0x064B, 27), + (0x064C, 0x064C, 28), + (0x064D, 0x064D, 29), + (0x064E, 0x064E, 30), + (0x064F, 0x064F, 31), + (0x0650, 0x0650, 32), + (0x0651, 0x0651, 33), + (0x0652, 0x0652, 34), + (0x0653, 0x0654, 230), + (0x0655, 0x0656, 220), + (0x0657, 0x065B, 230), + (0x065C, 0x065C, 220), + (0x065D, 0x065E, 230), + (0x065F, 0x065F, 220), + (0x0670, 0x0670, 35), + (0x06D6, 0x06DC, 230), + (0x06DF, 0x06E2, 230), + (0x06E3, 0x06E3, 220), + (0x06E4, 0x06E4, 230), + (0x06E7, 0x06E8, 230), + (0x06EA, 0x06EA, 220), + (0x06EB, 0x06EC, 230), + (0x06ED, 0x06ED, 220), + (0x0711, 0x0711, 36), + (0x0730, 0x0730, 230), + (0x0731, 0x0731, 220), + (0x0732, 0x0733, 230), + (0x0734, 0x0734, 220), + (0x0735, 0x0736, 230), + (0x0737, 0x0739, 220), + (0x073A, 0x073A, 230), + (0x073B, 0x073C, 220), + (0x073D, 0x073D, 230), + (0x073E, 0x073E, 220), + (0x073F, 0x0741, 230), + (0x0742, 0x0742, 220), + (0x0743, 0x0743, 230), + (0x0744, 0x0744, 220), + (0x0745, 0x0745, 230), + (0x0746, 0x0746, 220), + (0x0747, 0x0747, 230), + (0x0748, 0x0748, 220), + (0x0749, 0x074A, 230), + (0x07EB, 0x07F1, 230), + (0x07F2, 0x07F2, 220), + (0x07F3, 0x07F3, 230), + (0x0816, 0x0819, 230), + (0x081B, 0x0823, 230), + (0x0825, 0x0827, 230), + (0x0829, 0x082D, 230), + (0x0859, 0x085B, 220), + (0x093C, 0x093C, 7), + (0x094D, 0x094D, 9), + (0x0951, 0x0951, 230), + (0x0952, 0x0952, 220), + (0x0953, 0x0954, 230), + (0x09BC, 0x09BC, 7), + (0x09CD, 0x09CD, 9), + (0x0A3C, 0x0A3C, 7), + (0x0A4D, 0x0A4D, 9), + (0x0ABC, 0x0ABC, 7), + (0x0ACD, 0x0ACD, 9), + (0x0B3C, 0x0B3C, 7), + (0x0B4D, 0x0B4D, 9), + (0x0BCD, 0x0BCD, 9), + (0x0C4D, 0x0C4D, 9), + (0x0C55, 0x0C55, 84), + (0x0C56, 0x0C56, 91), + (0x0CBC, 0x0CBC, 7), + (0x0CCD, 0x0CCD, 9), + (0x0D4D, 0x0D4D, 9), + (0x0DCA, 0x0DCA, 9), + (0x0E38, 0x0E39, 103), + (0x0E3A, 0x0E3A, 9), + (0x0E48, 0x0E4B, 107), + (0x0EB8, 0x0EB9, 118), + (0x0EC8, 0x0ECB, 122), + (0x0F18, 0x0F19, 220), + (0x0F35, 0x0F35, 220), + (0x0F37, 0x0F37, 220), + (0x0F39, 0x0F39, 216), + (0x0F71, 0x0F71, 129), + (0x0F72, 0x0F72, 130), + (0x0F74, 0x0F74, 132), + (0x0F7A, 0x0F7D, 130), + (0x0F80, 0x0F80, 130), + (0x0F82, 0x0F83, 230), + (0x0F84, 0x0F84, 9), + (0x0F86, 0x0F87, 230), + (0x0FC6, 0x0FC6, 220), + (0x1037, 0x1037, 7), + (0x1039, 0x103A, 9), + (0x108D, 0x108D, 220), + (0x135D, 0x135F, 230), + (0x1714, 0x1714, 9), + (0x1734, 0x1734, 9), + (0x17D2, 0x17D2, 9), + (0x17DD, 0x17DD, 230), + (0x18A9, 0x18A9, 228), + (0x1939, 0x1939, 222), + (0x193A, 0x193A, 230), + (0x193B, 0x193B, 220), + (0x1A17, 0x1A17, 230), + (0x1A18, 0x1A18, 220), + (0x1A60, 0x1A60, 9), + (0x1A75, 0x1A7C, 230), + (0x1A7F, 0x1A7F, 220), + (0x1B34, 0x1B34, 7), + (0x1B44, 0x1B44, 9), + (0x1B6B, 0x1B6B, 230), + (0x1B6C, 0x1B6C, 220), + (0x1B6D, 0x1B73, 230), + (0x1BAA, 0x1BAA, 9), + (0x1BE6, 0x1BE6, 7), + (0x1BF2, 0x1BF3, 9), + (0x1C37, 0x1C37, 7), + (0x1CD0, 0x1CD2, 230), + (0x1CD4, 0x1CD4, 1), + (0x1CD5, 0x1CD9, 220), + (0x1CDA, 0x1CDB, 230), + (0x1CDC, 0x1CDF, 220), + (0x1CE0, 0x1CE0, 230), + (0x1CE2, 0x1CE8, 1), + (0x1CED, 0x1CED, 220), + (0x1DC0, 0x1DC1, 230), + (0x1DC2, 0x1DC2, 220), + (0x1DC3, 0x1DC9, 230), + (0x1DCA, 0x1DCA, 220), + (0x1DCB, 0x1DCC, 230), + (0x1DCD, 0x1DCD, 234), + (0x1DCE, 0x1DCE, 214), + (0x1DCF, 0x1DCF, 220), + (0x1DD0, 0x1DD0, 202), + (0x1DD1, 0x1DE6, 230), + (0x1DFC, 0x1DFC, 233), + (0x1DFD, 0x1DFD, 220), + (0x1DFE, 0x1DFE, 230), + (0x1DFF, 0x1DFF, 220), + (0x20D0, 0x20D1, 230), + (0x20D2, 0x20D3, 1), + (0x20D4, 0x20D7, 230), + (0x20D8, 0x20DA, 1), + (0x20DB, 0x20DC, 230), + (0x20E1, 0x20E1, 230), + (0x20E5, 0x20E6, 1), + (0x20E7, 0x20E7, 230), + (0x20E8, 0x20E8, 220), + (0x20E9, 0x20E9, 230), + (0x20EA, 0x20EB, 1), + (0x20EC, 0x20EF, 220), + (0x20F0, 0x20F0, 230), + (0x2CEF, 0x2CF1, 230), + (0x2D7F, 0x2D7F, 9), + (0x2DE0, 0x2DFF, 230), + (0x302A, 0x302A, 218), + (0x302B, 0x302B, 228), + (0x302C, 0x302C, 232), + (0x302D, 0x302D, 222), + (0x302E, 0x302F, 224), + (0x3099, 0x309A, 8), + (0xA66F, 0xA66F, 230), + (0xA67C, 0xA67D, 230), + (0xA6F0, 0xA6F1, 230), + (0xA806, 0xA806, 9), + (0xA8C4, 0xA8C4, 9), + (0xA8E0, 0xA8F1, 230), + (0xA92B, 0xA92D, 220), + (0xA953, 0xA953, 9), + (0xA9B3, 0xA9B3, 7), + (0xA9C0, 0xA9C0, 9), + (0xAAB0, 0xAAB0, 230), + (0xAAB2, 0xAAB3, 230), + (0xAAB4, 0xAAB4, 220), + (0xAAB7, 0xAAB8, 230), + (0xAABE, 0xAABF, 230), + (0xAAC1, 0xAAC1, 230), + (0xABED, 0xABED, 9), + (0xFB1E, 0xFB1E, 26), + (0xFE20, 0xFE26, 230), + (0x101FD, 0x101FD, 220), + (0x10A0D, 0x10A0D, 220), + (0x10A0F, 0x10A0F, 230), + (0x10A38, 0x10A38, 230), + (0x10A39, 0x10A39, 1), + (0x10A3A, 0x10A3A, 220), + (0x10A3F, 0x10A3F, 9), + (0x11046, 0x11046, 9), + (0x110B9, 0x110B9, 9), + (0x110BA, 0x110BA, 7), + (0x1D165, 0x1D166, 216), + (0x1D167, 0x1D169, 1), + (0x1D16D, 0x1D16D, 226), + (0x1D16E, 0x1D172, 216), + (0x1D17B, 0x1D182, 220), + (0x1D185, 0x1D189, 230), + (0x1D18A, 0x1D18B, 220), + (0x1D1AA, 0x1D1AD, 230), + (0x1D242, 0x1D244, 230), +]; + +/// `(first, last)` for every run of 256-character blocks that hold a character from +/// [`COMBINING_CLASSES`] or a Hangul jamo or syllable, sorted. A block is a code point `>> 8`. +pub(super) static COMPOSING_BLOCKS: [(u32, u32); 14] = [ + (0x03, 0x11), + (0x13, 0x13), + (0x17, 0x1D), + (0x20, 0x20), + (0x2C, 0x2D), + (0x30, 0x30), + (0xA6, 0xA6), + (0xA8, 0xD7), + (0xFB, 0xFB), + (0xFE, 0xFE), + (0x101, 0x101), + (0x10A, 0x10A), + (0x110, 0x110), + (0x1D1, 0x1D2), +]; diff --git a/src/tar/decoder.rs b/src/tar/decoder.rs new file mode 100644 index 000000000000..29f195d35b4a --- /dev/null +++ b/src/tar/decoder.rs @@ -0,0 +1,344 @@ +//! A tar reader with the gzip layers in front of it. + +use crate::error::{Error, Result}; +use crate::gzip::{self, Progress}; +use crate::output::Output as _; +use crate::reader::{Entry, Event, Options, Poll, Reader}; + +// https://github.com/libarchive/libarchive/blob/27cbc7827172698143e440801fc0ba39ccb4f1f5/libarchive/archive_read_support_filter_gzip.c#L306 +const BLOCK: usize = 64 * 1024; +// https://github.com/libarchive/libarchive/blob/27cbc7827172698143e440801fc0ba39ccb4f1f5/libarchive/archive_read.c#L578 +const MAX_LAYERS: usize = 24; + +/// Where a layer's next block goes. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Place { + InsteadOfTheLast, + BehindTheLast, +} + +struct Layer { + gzip: gzip::Decoder, + out: Vec, + pos: usize, + done: bool, +} + +impl Layer { + fn new() -> Result { + let mut out = Vec::new(); + out.try_reserve_exact(BLOCK)?; + Ok(Self { + gzip: gzip::Decoder::new()?, + out, + pos: 0, + done: false, + }) + } + + fn unread(&self) -> &[u8] { + self.out.get(self.pos..).unwrap_or_default() + } +} + +/// Takes compressed or plain bytes as they come and gives tar events back. +pub struct Decoder { + /// The first one reads the caller's input, and each next one the output of the one before. + layers: Vec, + tar: Reader, +} + +impl Decoder { + /// Nothing is detected, so this works however little of the input is there yet. + pub fn new(options: Options, gzip: bool) -> Result { + let mut layers = Vec::new(); + if gzip { + layers.push(Layer::new()?); + } + Ok(Self { + layers, + tar: Reader::new(options), + }) + } + + pub fn entry(&self) -> &Entry { + self.tar.entry() + } + + /// See [`Reader::skip_entry`]. + pub fn skip_entry(&mut self) { + self.tar.skip_entry(); + } + + /// Gives layer `index` a full block, dropped whole if it fails. `false`: no output for want of input. + fn refill( + &mut self, + index: usize, + input: &[u8], + eof: bool, + place: Place, + ) -> Result<(usize, bool)> { + let mut used_total = 0usize; + let Some(layer) = self.layers.get_mut(index) else { + return Ok((0, true)); + }; + if place == Place::InsteadOfTheLast { + layer.out.clear(); + layer.pos = 0; + } + let start = layer.out.len(); + loop { + let (below, rest) = self.layers.split_at_mut(index); + let Some(layer) = rest.first_mut() else { + return Ok((used_total, true)); + }; + let (source, source_eof) = match below.last() { + Some(lower) => (lower.unread(), lower.done), + None => (input.get(used_total..).unwrap_or_default(), eof), + }; + let decoded = layer.gzip.decode( + source, + source_eof, + &mut layer.out, + start.saturating_add(BLOCK), + ); + let (used, progress) = match decoded { + Ok(decoded) => decoded, + Err(error) => { + layer.out.truncate(start); + return Err(error); + } + }; + match below.last_mut() { + Some(lower) => lower.pos = lower.pos.saturating_add(used), + None => used_total = used_total.saturating_add(used), + } + match progress { + Progress::Done => { + layer.done = true; + return Ok((used_total, true)); + } + Progress::OutputFull => return Ok((used_total, true)), + Progress::NeedInput => {} + } + let has_output = layer.out.len() > start; + let Some(lower) = index.checked_sub(1) else { + return Ok((used_total, has_output)); + }; + let rest = input.get(used_total..).unwrap_or_default(); + used_total = used_total.saturating_add(self.refill_source( + lower, + rest, + eof, + Place::InsteadOfTheLast, + )?); + } + } + + /// To its reader a layer that fails has ended: an error inside a member, nothing between two. + fn refill_source( + &mut self, + index: usize, + input: &[u8], + eof: bool, + place: Place, + ) -> Result { + match self.refill(index, input, eof, place) { + Ok((used, _)) => Ok(used), + Err(error @ Error::Alloc(_)) => Err(error), + Err(_) => { + if let Some(layer) = self.layers.get_mut(index) { + layer.done = true; + } + Ok(0) + } + } + } + + /// If layer `index`, which is new, does not start with a header, none of its source counts as read. + fn read_first_header(&mut self, index: usize, input: &[u8]) -> Result<(usize, bool)> { + let mut used_total = 0usize; + let mut header_len = 0usize; + loop { + let (below, rest) = self.layers.split_at_mut(index); + let Some(layer) = rest.first_mut() else { + return Ok((used_total, false)); + }; + let (source, source_eof) = match below.last() { + Some(lower) => (lower.unread(), lower.done), + None => (input, true), + }; + let source = source.get(header_len..).unwrap_or_default(); + let (used, progress) = layer.gzip.decode(source, source_eof, &mut layer.out, 0)?; + header_len = header_len.saturating_add(used); + match progress { + Progress::Done => return Ok((used_total, false)), + Progress::OutputFull => { + match below.last_mut() { + Some(lower) => lower.pos = lower.pos.saturating_add(header_len), + None => used_total = header_len, + } + return Ok((used_total, true)); + } + Progress::NeedInput => {} + } + let Some(lower) = index.checked_sub(1) else { + return Ok((used_total, false)); + }; + let rest = input.get(used_total..).unwrap_or_default(); + used_total = used_total.saturating_add(self.refill_source( + lower, + rest, + true, + Place::BehindTheLast, + )?); + } + } + + /// Like [`Reader::next`], with `input` being what the outermost layer reads. + pub fn next<'a>(&'a mut self, input: &'a [u8], eof: bool) -> Result<(usize, Event<'a>)> { + let Some(top) = self.layers.len().checked_sub(1) else { + return self.tar.next(input, eof); + }; + let mut used_total = 0usize; + let poll = loop { + let Some(layer) = self.layers.get_mut(top) else { + break Poll::EndOfArchive; + }; + let base = layer.pos; + let (used, poll) = self.tar.poll(layer.unread(), layer.done)?; + layer.pos = layer.pos.saturating_add(used); + match poll { + Poll::NeedInput => {} + Poll::Data { offset, start, len } => { + break Poll::Data { + offset, + start: base.saturating_add(start), + len, + }; + } + other => break other, + } + let (used, ready) = self.refill( + top, + input.get(used_total..).unwrap_or_default(), + eof, + Place::InsteadOfTheLast, + )?; + used_total = used_total.saturating_add(used); + if !ready { + break Poll::NeedInput; + } + }; + let decoded = self.layers.get(top).map_or(&[][..], |layer| &layer.out); + Ok((used_total, poll.event(self.tar.entry(), decoded))) + } +} + +/// Reads an archive that is all in memory, and works out by itself how it is wrapped. +pub struct MemoryDecoder<'a> { + data: &'a [u8], + pos: usize, + decoder: Decoder, +} + +impl<'a> MemoryDecoder<'a> { + /// Fails for what is not an archive, and if the first block of a gzip layer does not decompress. + pub fn new(data: &'a [u8], options: Options) -> Result { + let mut this = Self { + data, + pos: 0, + decoder: Decoder::new(options, false)?, + }; + loop { + let rest = data.get(this.pos..).unwrap_or_default(); + let layers = &mut this.decoder.layers; + if !gzip::starts_like_header(layers.last().map_or(rest, Layer::unread)) { + break; + } + layers.push(Layer::new()?); + let top = layers.len().saturating_sub(1); + let (used, is_gzip) = this.decoder.read_first_header(top, rest)?; + this.pos = this.pos.saturating_add(used); + if !is_gzip { + this.decoder.layers.pop(); + break; + } + if top >= MAX_LAYERS { + return Err(Error::TooManyLayers); + } + let rest = data.get(this.pos..).unwrap_or_default(); + let (used, _) = this + .decoder + .refill(top, rest, true, Place::InsteadOfTheLast)?; + this.pos = this.pos.saturating_add(used); + } + match this.decoder.layers.last_mut() { + None => this.pos = this.decoder.tar.detect(data)?, + Some(layer) => layer.pos = this.decoder.tar.detect(layer.unread())?, + } + Ok(this) + } + + pub fn entry(&self) -> &Entry { + self.decoder.entry() + } + + pub fn skip_entry(&mut self) { + self.decoder.skip_entry(); + } + + /// Skips what is left of the current entry. `None` at the end of the archive. + pub fn next_entry(&mut self) -> Result> { + self.skip_entry(); + loop { + match self.next()? { + Event::Entry(_) => break, + Event::EndOfArchive => return Ok(None), + Event::NeedInput | Event::Data { .. } | Event::EndOfEntry { .. } => {} + } + } + Ok(Some(self.entry())) + } + + /// Appends at most `size` bytes of the current entry's contents. Holes read as zeros. + pub fn read_data(&mut self, size: usize, out: &mut Vec) -> Result<()> { + let start = out.len(); + let zeros_up_to = |out: &mut Vec, offset: i64| -> Result<()> { + let have = out.len().saturating_sub(start); + let offset = usize::try_from(offset).map_err(|_| Error::OutOfOrderSparse)?; + if offset < have { + return Err(Error::OutOfOrderSparse); + } + let zeros = offset.min(size).saturating_sub(have); + out.try_reserve(zeros)?; + out.resize(out.len().saturating_add(zeros), 0); + Ok(()) + }; + while out.len().saturating_sub(start) < size { + match self.next()? { + Event::Data { offset, bytes } => { + zeros_up_to(out, offset)?; + let room = size.saturating_sub(out.len().saturating_sub(start)); + let bytes = bytes.get(..room.min(bytes.len())).unwrap_or_default(); + out.write(bytes)?; + } + Event::EndOfEntry { size: end } => { + zeros_up_to(out, end)?; + break; + } + Event::NeedInput | Event::Entry(_) | Event::EndOfArchive => break, + } + } + Ok(()) + } + + /// Never returns [`Event::NeedInput`]. + pub fn next(&mut self) -> Result> { + let (used, event) = self + .decoder + .next(self.data.get(self.pos..).unwrap_or_default(), true)?; + self.pos = self.pos.saturating_add(used); + Ok(event) + } +} diff --git a/src/tar/error.rs b/src/tar/error.rs new file mode 100644 index 000000000000..184b4e7d84fb --- /dev/null +++ b/src/tar/error.rs @@ -0,0 +1,75 @@ +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +pub enum Error { + #[error("Unrecognized archive format")] + UnrecognizedFormat, + #[error("Damaged tar archive (bad header checksum)")] + BadChecksum, + #[error("Damaged tar archive (end-of-archive within a sequence of headers)")] + EndWithinHeaders, + #[error("Truncated tar archive detected while reading next header")] + TruncatedHeader, + #[error("Truncated tar archive detected while reading data")] + TruncatedData, + #[error("Truncated archive detected while reading metadata")] + TruncatedMetadata, + #[error("Truncated tar archive detected while reading pax attribute name")] + TruncatedPaxName, + #[error("Truncated archive detected while reading macOS metadata")] + TruncatedMacMetadata, + #[error("Truncated tar archive detected while skipping data")] + TruncatedSkip, + #[error("Truncated input file (needed {needed} bytes, only {available} available)")] + TruncatedInput { needed: i64, available: i64 }, + #[error("Truncated tar archive detected while reading sparse file data")] + TruncatedSparse, + #[error("Redundant '{0}' header")] + RedundantHeader(&'static str), + #[error("Special header has invalid size: {0}")] + InvalidSpecialHeaderSize(i64), + #[error("pax extension header has invalid size: {0}")] + InvalidPaxHeaderSize(i64), + #[error("Invalid empty pax extended header")] + EmptyPaxHeader, + #[error("Invalid empty pax global extended header")] + EmptyPaxGlobalHeader, + #[error("Malformed pax attribute")] + MalformedPaxAttribute, + #[error("Tar size attribute overflow")] + PaxSizeOverflow, + #[error("Tar entry has negative file size")] + NegativeFileSize, + #[error("Tar entry has negative size")] + NegativeSize, + #[error("Tar entry size overflow")] + SizeOverflow, + #[error("Line too long")] + LineTooLong, + #[error("Malformed sparse map data")] + MalformedSparseMap, + #[error("Encountered out-of-order sparse blocks")] + OutOfOrderSparse, + #[error("Unreasonably large sparse map")] + SparseMapTooLarge, + #[error("Oversized AppleDouble extension")] + OversizedMacMetadata, + #[error("Input requires too many filters for decoding")] + TooManyLayers, + #[error("truncated gzip input")] + TruncatedGzip, + #[error("gzip decompression failed")] + GzipDecompressionFailed, + #[error("GZip compression failed")] + GzipCompressionFailed, + #[error("gzip trailer mismatch: bad CRC32 or uncompressed size")] + GzipTrailerMismatch, + #[error(transparent)] + Alloc(#[from] bun_alloc::AllocError), +} + +impl From for Error { + fn from(_: std::collections::TryReserveError) -> Self { + Self::Alloc(bun_alloc::AllocError) + } +} + +pub type Result = core::result::Result; diff --git a/src/tar/gzip.rs b/src/tar/gzip.rs new file mode 100644 index 000000000000..a879b76840a0 --- /dev/null +++ b/src/tar/gzip.rs @@ -0,0 +1,404 @@ +//! The gzip container (RFC 1952). DEFLATE itself is zlib's. + +use crate::error::{Error, Result}; +use crate::output::Output; +use bun_alloc::AllocError; +use bun_core::strings; +use bun_zlib::{InflateDecoder, ReturnCode, crc32_bytes}; + +mod flag { + pub(super) const HEADER_CRC: u8 = 2; + pub(super) const EXTRA: u8 = 4; + pub(super) const NAME: u8 = 8; + pub(super) const COMMENT: u8 = 16; + pub(super) const RESERVED: u8 = 0xe0; +} + +const RAW_DEFLATE: core::ffi::c_int = -15; +/// The two identification bytes, and the compression method, which is always DEFLATE. +const MAGIC: [u8; 3] = [0x1f, 0x8b, 8]; +const FLAGS: usize = MAGIC.len(); +const FIXED_HEADER_SIZE: usize = 10; +/// The length of the extra field, and the CRC-16 of the header. +const SHORT_SIZE: usize = 2; +/// The CRC-32 and the size. +const TRAILER_SIZE: usize = 8; + +fn fixed_header_flags(fixed: &[u8; FIXED_HEADER_SIZE]) -> Option { + let flags = fixed[FLAGS]; + (fixed.starts_with(&MAGIC) && flags & flag::RESERVED == 0).then_some(flags) +} + +/// Whether `data` starts like a gzip header. The optional fields are not looked at. +pub(crate) fn starts_like_header(data: &[u8]) -> bool { + data.first_chunk::() + .is_some_and(|fixed| fixed_header_flags(fixed).is_some()) +} + +// https://github.com/libarchive/libarchive/blob/27cbc7827172698143e440801fc0ba39ccb4f1f5/libarchive/archive_read_support_filter_gzip.c#L120 +/// Only for a header that goes on past the piece of input it starts in. +const SPILLED_HEADER_LIMIT: usize = 1024 * 1024; + +/// In the order they appear in. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +enum Field { + Extra, + Name, + Comment, + HeaderCrc, + Body, +} + +#[derive(Debug, Clone, Copy)] +enum State { + Fixed, + ExtraLength, + Extra(u16), + Name, + Comment, + HeaderCrc, + Body, + Trailer, + Done, + Failed(Error), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Progress { + /// All of the input was consumed and more is wanted. + NeedInput, + /// `out` has reached `limit`. + OutputFull, + /// The last member has ended. What follows it, if anything, is not gzip and is left alone. + Done, +} + +/// Decompresses a sequence of gzip members, checking the CRC-32 and the size of each. +pub struct Decoder { + state: State, + inflate: InflateDecoder, + flags: u8, + small: [u8; FIXED_HEADER_SIZE], + small_len: usize, + /// How much of the current header is behind us. + header_len: usize, + /// The current header started in an earlier piece of input. + header_spilled: bool, + crc: u32, + /// Modulo 2^32, like the field it is compared with. + size: u32, +} + +impl Decoder { + pub fn new() -> Result { + Ok(Self { + state: State::Fixed, + inflate: InflateDecoder::new(RAW_DEFLATE).map_err(|_| Error::Alloc(AllocError))?, + flags: 0, + small: [0; FIXED_HEADER_SIZE], + small_len: 0, + header_len: 0, + header_spilled: false, + crc: 0, + size: 0, + }) + } + + /// Gathers `want` bytes in `small`. `false` when `input` ran out first. + fn gather(&mut self, want: usize, input: &[u8], used: &mut usize) -> bool { + let take = want.saturating_sub(self.small_len).min(input.len()); + if let (Some(dest), Some(src)) = ( + self.small + .get_mut(self.small_len..self.small_len.saturating_add(take)), + input.get(..take), + ) { + dest.copy_from_slice(src); + } + self.small_len = self.small_len.saturating_add(take); + *used = take; + if self.small_len < want { + return false; + } + self.small_len = 0; + true + } + + /// Moves to the first optional field at or behind `from` that the flags announce. + fn next_header_field(&mut self, from: Field) { + let has = |flag: u8| self.flags & flag != 0; + self.state = if from <= Field::Extra && has(flag::EXTRA) { + State::ExtraLength + } else if from <= Field::Name && has(flag::NAME) { + State::Name + } else if from <= Field::Comment && has(flag::COMMENT) { + State::Comment + } else if from <= Field::HeaderCrc && has(flag::HEADER_CRC) { + State::HeaderCrc + } else { + self.crc = 0; + self.size = 0; + State::Body + }; + } + + /// Fills `out` up to `limit` bytes. A call that fails appends nothing, and every later one fails too. + pub fn decode( + &mut self, + input: &[u8], + eof: bool, + out: &mut Vec, + limit: usize, + ) -> Result<(usize, Progress)> { + let before = out.len(); + let mut pos = 0usize; + loop { + let rest = input.get(pos..).unwrap_or_default(); + let mut used = 0usize; + let in_header = !matches!(self.state, State::Body | State::Trailer); + let step = self.step(rest, eof, out, limit, &mut used); + pos = pos.saturating_add(used); + if in_header { + self.header_len = self.header_len.saturating_add(used); + if matches!(step, Ok(Some(Progress::NeedInput))) { + self.header_spilled = self.header_len > 0; + } + } + match step { + Ok(None) => {} + Ok(Some(progress)) => return Ok((pos, progress)), + Err(error) => { + self.state = State::Failed(error); + out.truncate(before); + return Err(error); + } + } + } + } + + fn step( + &mut self, + input: &[u8], + eof: bool, + out: &mut Vec, + limit: usize, + used: &mut usize, + ) -> Result> { + // A header that the input ends in the middle of is not one: the stream ended before it. + let starved = |this: &mut Self| { + if eof { + this.state = State::Done; + Some(Progress::Done) + } else { + Some(Progress::NeedInput) + } + }; + match self.state { + State::Failed(error) => Err(error), + State::Done => Ok(Some(Progress::Done)), + + State::Fixed => { + if !self.gather(FIXED_HEADER_SIZE, input, used) { + return Ok(starved(self)); + } + let Some(flags) = fixed_header_flags(&self.small) else { + self.state = State::Done; + return Ok(Some(Progress::Done)); + }; + self.flags = flags; + self.next_header_field(Field::Extra); + Ok(None) + } + State::ExtraLength => { + if !self.gather(SHORT_SIZE, input, used) { + return Ok(starved(self)); + } + self.state = State::Extra(u16::from_le_bytes([self.small[0], self.small[1]])); + Ok(None) + } + State::Extra(left) => { + let take = usize::from(left).min(input.len()); + *used = take; + let left = left.saturating_sub(u16::try_from(take).unwrap_or(u16::MAX)); + if left > 0 { + self.state = State::Extra(left); + if eof && self.flags & (flag::NAME | flag::COMMENT | flag::HEADER_CRC) == 0 { + return Err(Error::TruncatedGzip); + } + return Ok(starved(self)); + } + self.next_header_field(Field::Name); + Ok(None) + } + State::Name | State::Comment => { + let not_a_header = |this: &mut Self| { + this.state = State::Done; + Ok(Some(Progress::Done)) + }; + let looked_at = if self.header_spilled { + let room = (SPILLED_HEADER_LIMIT + 1).saturating_sub(self.header_len); + input.get(..room.min(input.len())).unwrap_or_default() + } else { + input + }; + let Some(end) = strings::index_of_char_usize(looked_at, 0) else { + *used = looked_at.len(); + if looked_at.len() < input.len() + || self.header_len.saturating_add(input.len()) > SPILLED_HEADER_LIMIT + { + return not_a_header(self); + } + return Ok(starved(self)); + }; + *used = end.saturating_add(1); + self.next_header_field(if matches!(self.state, State::Name) { + Field::Comment + } else { + Field::HeaderCrc + }); + Ok(None) + } + State::HeaderCrc => { + if !self.gather(SHORT_SIZE, input, used) { + return Ok(starved(self)); + } + self.next_header_field(Field::Body); + Ok(None) + } + + State::Body => { + let room = limit.saturating_sub(out.len()); + if room == 0 { + return Ok(Some(Progress::OutputFull)); + } + if input.is_empty() { + if eof { + return Err(Error::TruncatedGzip); + } + return Ok(Some(Progress::NeedInput)); + } + let start = out.len(); + let (consumed, code) = self.inflate.step_limited(input, out, room); + *used = consumed; + let produced = out.get(start..).unwrap_or_default(); + self.crc = crc32_bytes(self.crc, produced); + self.size = self.size.wrapping_add(produced.len() as u32); + match code { + ReturnCode::Ok => Ok(None), + ReturnCode::StreamEnd => { + if self.inflate.reset() != ReturnCode::Ok { + return Err(Error::GzipDecompressionFailed); + } + self.state = State::Trailer; + Ok(None) + } + ReturnCode::MemError => Err(Error::Alloc(AllocError)), + _ => Err(Error::GzipDecompressionFailed), + } + } + State::Trailer => { + if !self.gather(TRAILER_SIZE, input, used) { + if eof { + return Err(Error::TruncatedGzip); + } + return Ok(Some(Progress::NeedInput)); + } + let [c0, c1, c2, c3, s0, s1, s2, s3, ..] = self.small; + if u32::from_le_bytes([c0, c1, c2, c3]) != self.crc + || u32::from_le_bytes([s0, s1, s2, s3]) != self.size + { + return Err(Error::GzipTrailerMismatch); + } + self.state = State::Fixed; + self.header_len = 0; + self.header_spilled = false; + // What follows a member is not looked at for a block that is already full. + Ok((out.len() >= limit).then_some(Progress::OutputFull)) + } + } + } +} + +/// One gzip member with no name and no time, so the same input always gives the same bytes. +pub struct Encoder { + out: O, + deflate: bun_zlib::DeflateEncoder, + buffer: Vec, + crc: u32, + size: u32, +} + +// https://github.com/libarchive/libarchive/blob/27cbc7827172698143e440801fc0ba39ccb4f1f5/libarchive/archive_write_add_filter_gzip.c#L206 +/// At level 0 the room `deflate()` is shown decides where a stored block ends. +const ENCODER_BUFFER: usize = 61440; +const OS_UNKNOWN: u8 = 255; + +impl Encoder { + /// `level` is 0 to 9. + pub fn new(out: O, level: u8) -> Result { + let mut buffer = Vec::new(); + buffer.try_reserve_exact(ENCODER_BUFFER)?; + let extra_flags = match level { + 9 => 2, + 1 => 4, + _ => 0, + }; + buffer.extend_from_slice(&MAGIC); + buffer.extend_from_slice(&[0, 0, 0, 0, 0, extra_flags, OS_UNKNOWN]); + Ok(Self { + out, + deflate: bun_zlib::DeflateEncoder::new(i32::from(level), RAW_DEFLATE, 8, 0) + .map_err(|_| Error::Alloc(AllocError))?, + buffer, + crc: 0, + size: 0, + }) + } + + fn drive(&mut self, mut input: &[u8], finishing: bool) -> Result<()> { + loop { + if self.buffer.len() >= ENCODER_BUFFER { + self.out.write(&self.buffer)?; + self.buffer.clear(); + } + if !finishing && input.is_empty() { + return Ok(()); + } + let room = ENCODER_BUFFER.saturating_sub(self.buffer.len()); + let (used, code) = self.deflate.step_limited( + input, + &mut self.buffer, + room, + if finishing { + bun_zlib::FlushValue::Finish + } else { + bun_zlib::FlushValue::NoFlush + }, + ); + input = input.get(used..).unwrap_or_default(); + match code { + ReturnCode::Ok => {} + ReturnCode::StreamEnd => return Ok(()), + ReturnCode::MemError => return Err(Error::Alloc(AllocError)), + _ => return Err(Error::GzipCompressionFailed), + } + } + } + + pub fn finish(mut self) -> Result { + self.drive(&[], true)?; + self.out.write(&self.buffer)?; + let mut trailer = [0u8; TRAILER_SIZE]; + trailer[..4].copy_from_slice(&self.crc.to_le_bytes()); + trailer[4..].copy_from_slice(&self.size.to_le_bytes()); + self.out.write(&trailer)?; + Ok(self.out) + } +} + +impl Output for Encoder { + fn write(&mut self, bytes: &[u8]) -> Result<()> { + self.crc = crc32_bytes(self.crc, bytes); + self.size = self.size.wrapping_add(bytes.len() as u32); + self.drive(bytes, false) + } +} diff --git a/src/tar/header.rs b/src/tar/header.rs new file mode 100644 index 000000000000..c1b59cd4f1de --- /dev/null +++ b/src/tar/header.rs @@ -0,0 +1,146 @@ +//! The 512-byte header block. + +use crate::num::parse_field; +use bun_core::strings; +use core::ops::Range; + +pub(crate) const BLOCK_SIZE: usize = 512; +pub(crate) type Block = [u8; BLOCK_SIZE]; + +/// How many bytes fill the last block up behind `size` bytes. +pub(crate) fn padding_for(size: u64) -> u64 { + size.wrapping_neg() % BLOCK_SIZE as u64 +} + +pub(crate) const NAME: Range = 0..100; +pub(crate) const MODE: Range = 100..108; +pub(crate) const UID: Range = 108..116; +pub(crate) const GID: Range = 116..124; +pub(crate) const SIZE: Range = 124..136; +pub(crate) const MTIME: Range = 136..148; +pub(crate) const CHECKSUM: Range = 148..156; +pub(crate) const TYPEFLAG: usize = 156; +pub(crate) const LINKNAME: Range = 157..257; +/// `magic` and `version` together. +pub(crate) const MAGIC: Range = 257..265; +pub(crate) const DEVMAJOR: Range = 329..337; +pub(crate) const DEVMINOR: Range = 337..345; +pub(crate) const PREFIX: Range = 345..500; + +pub(crate) const USTAR_MAGIC: &[u8; 8] = b"ustar\x0000"; + +pub(crate) fn name(h: &Block) -> &[u8] { + &h[NAME] +} +pub(crate) fn mode(h: &Block) -> i64 { + parse_field(&h[MODE]) +} +pub(crate) fn size(h: &Block) -> i64 { + parse_field(&h[SIZE]) +} +pub(crate) fn mtime(h: &Block) -> i64 { + parse_field(&h[MTIME]) +} +pub(crate) fn typeflag(h: &Block) -> u8 { + h[TYPEFLAG] +} +pub(crate) fn linkname(h: &Block) -> &[u8] { + &h[LINKNAME] +} +pub(crate) fn magic(h: &Block) -> &[u8] { + &h[MAGIC] +} +pub(crate) fn prefix(h: &Block) -> &[u8] { + &h[PREFIX] +} + +/// The GNU header reuses the `prefix` area. +pub(crate) mod gnu { + use super::Block; + use core::ops::Range; + + pub(crate) const MAGIC: &[u8; 8] = b"ustar \0"; + + /// An offset and a length. + pub(crate) const SPARSE_ENTRY_SIZE: usize = 2 * SPARSE_NUMBER_SIZE; + pub(crate) const SPARSE_NUMBER_SIZE: usize = 12; + + const SPARSE: Range = 386..482; + const IS_EXTENDED: usize = 482; + const REAL_SIZE: Range = 483..495; + /// A block of 21 more sparse entries. + const EXTENSION_SPARSE: Range = 0..504; + const EXTENSION_IS_EXTENDED: usize = 504; + + pub(crate) fn sparse(h: &Block) -> &[u8] { + &h[SPARSE] + } + pub(crate) fn is_extended(h: &Block) -> bool { + h[IS_EXTENDED] != 0 + } + pub(crate) fn real_size(h: &Block) -> &[u8] { + &h[REAL_SIZE] + } + pub(crate) fn extension_sparse(h: &Block) -> &[u8] { + &h[EXTENSION_SPARSE] + } + pub(crate) fn extension_is_extended(h: &Block) -> bool { + h[EXTENSION_IS_EXTENDED] != 0 + } +} + +pub(crate) fn is_null(h: &Block) -> bool { + strings::index_of_not_char(h, 0).is_none() +} + +/// Old BSD, Solaris and HP-UX tars summed signed bytes, so either sum is accepted. +pub(crate) fn checksum_ok(h: &Block) -> bool { + let field = &h[CHECKSUM]; + if !field.iter().all(|c| matches!(c, b' ' | 0 | b'0'..=b'7')) { + return false; + } + let expected = parse_field(field); + let sum = |bytes: &[u8]| bytes.iter().map(|&b| u32::from(b)).sum::(); + // The field itself counts as spaces. + let unsigned = i64::from(sum(h) - sum(field) + sum(&[b' '; CHECKSUM.end - CHECKSUM.start])); + if expected == unsigned { + return true; + } + let negative = h.iter().filter(|&&b| b.cast_signed() < 0).count() as i64; + expected == unsigned - negative * (1 << u8::BITS) +} + +fn number_field_looks_valid(field: &[u8]) -> bool { + // Base-256, or empty. + if matches!(field.first(), Some(0x80 | 0xff | 0)) { + return true; + } + let rest = strings::trim_leading_char(field, b' '); + let digits = rest + .iter() + .position(|c| !matches!(c, b'0'..=b'7')) + .unwrap_or(rest.len()); + rest.iter().skip(digits).all(|c| matches!(c, b' ' | 0)) +} + +pub(crate) fn has_magic(h: &Block) -> bool { + magic(h) == USTAR_MAGIC || magic(h) == gnu::MAGIC +} + +// https://github.com/libarchive/libarchive/blob/27cbc7827172698143e440801fc0ba39ccb4f1f5/libarchive/archive_read_support_format_tar.c#L393 +/// Whether every field that can be checked holds what a header would. +pub(crate) fn is_plausible(h: &Block) -> bool { + checksum_ok(h) + && matches!(typeflag(h), 0 | b'0'..=b'9' | b'A'..=b'Z' | b'a'..=b'z') + && [ + &h[MODE], + &h[UID], + &h[GID], + &h[MTIME], + &h[SIZE], + &h[DEVMAJOR], + &h[DEVMINOR], + ] + .into_iter() + .all(number_field_looks_valid) +} diff --git a/src/tar/lib.rs b/src/tar/lib.rs new file mode 100644 index 000000000000..7dadfee1cb8c --- /dev/null +++ b/src/tar/lib.rs @@ -0,0 +1,25 @@ +//! tar and gzip framing without I/O, behaving as libarchive did. Parts derive from it (LICENSE.md). +#![forbid(unsafe_code)] +// Archives come from anywhere and a panic aborts the process. +#![deny( + clippy::indexing_slicing, + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::unreachable +)] +#![warn(unused_must_use)] + +pub mod compose; +pub mod decoder; +pub mod error; +pub mod gzip; +mod header; +mod num; +pub mod output; +pub mod reader; +pub mod writer; + +pub use decoder::{Decoder, MemoryDecoder}; +pub use error::{Error, Result}; +pub use reader::{Charset, Dialect, Entry, Event, Options, Reader}; diff --git a/src/tar/num.rs b/src/tar/num.rs new file mode 100644 index 000000000000..34048a0e7258 --- /dev/null +++ b/src/tar/num.rs @@ -0,0 +1,97 @@ +//! Number fields. Overflow saturates, and a field ends at the first byte that is not a digit. + +fn parse_base_n(field: &[u8], base: u8) -> i64 { + let mut bytes = field.iter().copied().peekable(); + while bytes.next_if(|c| matches!(c, b' ' | b'\t')).is_some() {} + let negative = bytes.next_if_eq(&b'-').is_some(); + let mut value: i64 = 0; + for c in bytes { + let digit = c.wrapping_sub(b'0'); + if digit >= base { + break; + } + let digit = if negative { + -i64::from(digit) + } else { + i64::from(digit) + }; + let Some(next) = value + .checked_mul(i64::from(base)) + .and_then(|v| v.checked_add(digit)) + else { + return if negative { i64::MIN } else { i64::MAX }; + }; + value = next; + } + value +} + +pub(crate) fn parse_decimal(field: &[u8]) -> i64 { + parse_base_n(field, 10) +} + +/// Big-endian two's complement whose top bit is the base-256 marker. +fn parse_base_256(field: &[u8]) -> i64 { + let Some((&first, mut rest)) = field.split_first() else { + return 0; + }; + let negative = first & 0x40 != 0; + let fill: u8 = if negative { 0xff } else { 0 }; + let saturated = if negative { i64::MIN } else { i64::MAX }; + let mut c = if negative { first | 0x80 } else { first & 0x7f }; + while rest.len() >= size_of::() { + if c != fill { + return saturated; + } + let Some((&next, tail)) = rest.split_first() else { + break; + }; + c = next; + rest = tail; + } + if (c ^ fill) & 0x80 != 0 { + return saturated; + } + let mut value: u64 = if negative { u64::MAX } else { 0 }; + value = (value << 8) | u64::from(c); + for &b in rest { + value = (value << 8) | u64::from(b); + } + value.cast_signed() +} + +/// A header field: octal, or base-256 when the top bit of the first byte is set. +pub(crate) fn parse_field(field: &[u8]) -> i64 { + match field.first() { + Some(first) if first & 0x80 != 0 => parse_base_256(field), + _ => parse_base_n(field, 8), + } +} + +/// `seconds[.fraction]`. `None` for anything else, or for seconds outside `i64`. +pub(crate) fn parse_pax_time(value: &[u8]) -> Option { + if value.is_empty() { + return Some(0); + } + let (negative, digits) = match value.split_first() { + Some((b'-', rest)) => (true, rest), + _ => (false, value), + }; + let int_len = digits + .iter() + .position(|c| !c.is_ascii_digit()) + .unwrap_or(digits.len()); + let (int, fraction) = digits.split_at(int_len); + let mut seconds: i64 = 0; + for c in int { + seconds = seconds + .checked_mul(10)? + .checked_add(i64::from(c.wrapping_sub(b'0')))?; + } + match fraction.split_first() { + None => {} + Some((b'.', rest)) if rest.iter().all(u8::is_ascii_digit) => {} + Some(_) => return None, + } + Some(if negative { -seconds } else { seconds }) +} diff --git a/src/tar/output.rs b/src/tar/output.rs new file mode 100644 index 000000000000..aff9568984da --- /dev/null +++ b/src/tar/output.rs @@ -0,0 +1,23 @@ +//! Where bytes go. + +use crate::error::Result; + +/// The size of each write matters: behind a compressor it decides what the compressed bytes are. +pub trait Output { + fn write(&mut self, bytes: &[u8]) -> Result<()>; +} + +impl Output for Vec { + fn write(&mut self, bytes: &[u8]) -> Result<()> { + self.try_reserve(bytes.len())?; + self.extend_from_slice(bytes); + Ok(()) + } +} + +impl Output for F { + fn write(&mut self, bytes: &[u8]) -> Result<()> { + self(bytes); + Ok(()) + } +} diff --git a/src/tar/reader.rs b/src/tar/reader.rs new file mode 100644 index 000000000000..0fdef5efbdfb --- /dev/null +++ b/src/tar/reader.rs @@ -0,0 +1,1649 @@ +use crate::error::{Error, Result}; +use crate::header::{self, BLOCK_SIZE, Block, gnu}; +use crate::num::{parse_decimal, parse_field, parse_pax_time}; +use crate::output::Output as _; +use bun_core::{FileKind, S, kind_from_mode, slice_to_nul, strings}; +use std::borrow::Cow; + +/// Whose answer to give where tar readers disagree. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Dialect { + /// Zero blocks never end the archive, so archives that were concatenated read as one. + General, + /// What npm's own tar reader does with a package. + Npm, +} + +/// Which pax strings convert. An entry with one that does not has its headers trusted less. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Charset { + /// POSIX in the "C" locale: only ASCII converts. + Ascii, + /// Windows, where names become UTF-16: valid UTF-8 converts. + Utf8, +} + +#[derive(Debug, Clone, Copy)] +pub struct Options { + pub dialect: Dialect, + pub charset: Charset, + /// macOS: an entry named `._name` is the AppleDouble metadata of the entry behind it, and dropped. + pub mac_metadata: bool, +} + +impl Options { + /// What the platform this was compiled for has always done. + pub fn for_host(dialect: Dialect) -> Self { + Self { + dialect, + charset: if cfg!(windows) { + Charset::Utf8 + } else { + Charset::Ascii + }, + mac_metadata: cfg!(target_os = "macos"), + } + } +} + +#[derive(Debug, Default)] +pub struct Entry { + path: Vec, + path_is_utf8: bool, + link: Vec, + is_symlink: bool, + mode: u32, + size: i64, + mtime: Option, +} + +impl Entry { + /// Never holds a NUL. + pub fn path(&self) -> &[u8] { + &self.path + } + /// The path is a pax record that [`Charset`] could convert. Otherwise its encoding is unknown. + pub fn path_is_utf8(&self) -> bool { + self.path_is_utf8 + } + pub fn symlink(&self) -> Option<&[u8]> { + self.is_symlink.then_some(&self.link[..]) + } + /// One of `S::IF*`, or what the mode field held for the few types that say nothing about it. + pub fn filetype(&self) -> u32 { + self.mode & S::IFMT + } + pub fn kind(&self) -> FileKind { + kind_from_mode(self.filetype()) + } + /// Not masked to the permission bits: a base-256 mode field can hold more. + pub fn perm(&self) -> u32 { + self.mode & !S::IFMT + } + /// The length of the file on disk, which for a sparse entry is more than the archive stores. + pub fn size(&self) -> i64 { + self.size + } + pub fn mtime(&self) -> i64 { + self.mtime.unwrap_or(0) + } +} + +#[derive(Debug)] +pub enum Event<'a> { + /// Everything given was consumed. Call again with more, or with `eof` set. + NeedInput, + Entry(&'a Entry), + /// `offset` only jumps for a sparse entry. + Data { + offset: i64, + bytes: &'a [u8], + }, + /// `size` is the length of the file, which may lie past the last `Data`. + EndOfEntry { + size: i64, + }, + EndOfArchive, +} + +/// Ordered so that the worse of two is the smaller one. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +enum Status { + /// Becomes an error once the entry's own header has been read. + Failed, + Warn, + Ok, +} + +impl Status { + fn warn_unless(ok: bool) -> Self { + if ok { Self::Ok } else { Self::Warn } + } +} + +/// Sticks across entries. It decides whether the size of a hardlink entry is believed. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Format { + Old, + Ustar, + Pax, + Gnu, +} + +#[derive(Debug, Clone, Copy)] +struct SparseBlock { + offset: i64, + remaining: i64, + hole: bool, +} + +mod seen { + pub(super) const SOLARIS_ACL: u32 = 1; + pub(super) const PAX_GLOBAL: u32 = 2; + pub(super) const LONG_LINK: u32 = 4; + pub(super) const LONG_NAME: u32 = 8; + pub(super) const VOLUME: u32 = 16; + pub(super) const PAX: u32 = 32; + pub(super) const MAC_METADATA: u32 = 512; +} + +mod size_field { + pub(super) const GNU_SPARSE_REALSIZE: u8 = 1; + pub(super) const GNU_SPARSE_SIZE: u8 = 2; + pub(super) const SCHILY_REALSIZE: u8 = 4; + pub(super) const PAX_SIZE: u8 = 8; +} + +// https://github.com/libarchive/libarchive/blob/27cbc7827172698143e440801fc0ba39ccb4f1f5/libarchive/archive_read_support_format_tar.c#L247 +const PATHNAME_LIMIT: i64 = 1024 * 1024; +const USER_NAME_LIMIT: i64 = 65536; +const SPARSE_MAP_LIMIT: i64 = 8 * 1024 * 1024; +const XATTR_LIMIT: i64 = 16 * 1024 * 1024; +const FFLAGS_LIMIT: i64 = 512; +const ENTRY_LIMIT: i64 = 0x0fff_ffff_ffff_ffff; +const PAX_BODY_LIMIT: i64 = 1024 * 1024 * 1024; +const PAX_LINE_LIMIT: i64 = 99_999_999; +const NUMBER_LIMIT: i64 = 64; +const TIME_LIMIT: i64 = 128; +const SYMLINK_TYPE_LIMIT: i64 = 16; +const CHARSET_LIMIT: i64 = 64; +const XATTR_NAME_LIMIT: usize = 128; +const SPARSE_VERSION_LIMIT: i64 = 10; +const SPARSE_MAP_LINE_LIMIT: usize = 100; +const LIBARCHIVE_XATTR: &[u8] = b"LIBARCHIVE.xattr."; +const SCHILY_XATTR: &[u8] = b"SCHILY.xattr."; +/// Longer than every key that is matched in full. +const KEY_CAPACITY: usize = 32; + +fn padding_for(size: i64) -> i64 { + header::padding_for(size.cast_unsigned()).cast_signed() +} + +fn assign(dest: &mut Vec, bytes: &[u8]) -> Result<()> { + dest.clear(); + dest.write(bytes) +} + +#[derive(Debug, Clone, Copy)] +enum AfterSkip { + /// The body of a header that says something about the entry behind it. + BetweenHeaders(Status), + PaxNewline(Status), + MacMetadata, + SparseMap, + EndOfEntry, + NextEntry, +} + +#[derive(Debug, Clone, Copy)] +enum Number { + SparseOffset, + SparseNumBytes, + SparseSize, + SparseMajor, + SparseMinor, + SparseRealSize, + SchilyRealSize, + Size, + /// Only whether it parses matters. + Unused, +} + +#[derive(Debug, Clone, Copy)] +enum Text { + Path, + PathOverride, + LinkPath, + UserName, + GroupName, +} + +#[derive(Debug, Clone, Copy)] +enum Collect { + LongName, + LongLink, + Number(Number), + /// `true` for `mtime`. Of the others only whether they parse matters. + Time(bool), + Text(Text), + HeaderCharset, + SymlinkType, + SparseMap, + HolesData, +} + +#[derive(Debug, Clone, Copy)] +enum SparseMapField { + Count, + Offset, + Size, +} + +#[derive(Debug, Clone, Copy)] +enum State { + NextEntry, + Header, + Skip(i64, AfterSkip), + Collect(usize, Collect), + PaxLength, + PaxKey, + PaxNewline(Status), + OldSparseExtension, + SparseMapLine(SparseMapField), + Emit, + Data, + /// Behind the end of an npm archive, which is still read to its end so that all of it is checked. + Drain, + Done, + Failed(Error), +} + +enum Step { + Continue, + NeedInput, + Entry, + Data { offset: i64, len: usize }, + EndOfEntry, + EndOfArchive, +} + +/// [`Event`] without the borrows. `Data` is `input[start..start + len]`. +#[derive(Debug, Clone, Copy)] +pub(crate) enum Poll { + NeedInput, + Entry, + Data { + offset: i64, + start: usize, + len: usize, + }, + EndOfEntry { + size: i64, + }, + EndOfArchive, +} + +impl Poll { + pub(crate) fn event<'a>(self, entry: &'a Entry, input: &'a [u8]) -> Event<'a> { + match self { + Poll::NeedInput => Event::NeedInput, + Poll::Entry => Event::Entry(entry), + Poll::Data { offset, start, len } => Event::Data { + offset, + bytes: input + .get(start..start.saturating_add(len)) + .unwrap_or_default(), + }, + Poll::EndOfEntry { size } => Event::EndOfEntry { size }, + Poll::EndOfArchive => Event::EndOfArchive, + } + } +} + +pub struct Reader { + options: Options, + state: State, + + block: Block, + block_len: usize, + /// The entry's own header, kept while the sparse extension blocks behind it are read. + main_header: Block, + collected: Vec, + + // One sequence of headers, which ends with the entry's own. + status: Status, + eof_is_fatal: bool, + seen: u32, + pax_is_utf8: bool, + sparse_attributes_seen: bool, + pax_path: Vec, + pax_path_override: Vec, + pax_link: Vec, + user_name_converts: bool, + group_name_converts: bool, + + // One pax header. + pax_status: Status, + pax_left: i64, + pax_padding: i64, + pax_line_length: i64, + pax_line_pos: i64, + pax_key: [u8; KEY_CAPACITY], + /// Up to the first NUL, and possibly more than `pax_key` holds. + pax_key_len: usize, + pax_key_ended: bool, + pax_key_started: bool, + + // One entry. + entry: Entry, + size_fields: u8, + sparse: Vec, + sparse_head: usize, + sparse_map_left: i64, + sparse_map_entries: i64, + sparse_map_offset: i64, + line: [u8; SPARSE_MAP_LINE_LIMIT], + line_len: usize, + + // Never reset, so a header that leaves one out inherits it from an earlier entry. + format: Format, + typeflag: u8, + sparse_major: i64, + sparse_minor: i64, + sparse_offset: i64, + sparse_num_bytes: i64, + gnu_sparse_size: i64, + gnu_sparse_real_size: i64, + schily_real_size: i64, + pax_size: i64, + disk_size: i64, + bytes_remaining: i64, + /// How much of the current `State::Skip` is behind us. + skipped: i64, + padding: i64, + null_blocks: u32, +} + +impl Reader { + pub fn new(options: Options) -> Self { + Self { + options, + state: State::NextEntry, + block: [0; BLOCK_SIZE], + block_len: 0, + main_header: [0; BLOCK_SIZE], + collected: Vec::new(), + status: Status::Ok, + eof_is_fatal: false, + seen: 0, + pax_is_utf8: true, + sparse_attributes_seen: false, + pax_path: Vec::new(), + pax_path_override: Vec::new(), + pax_link: Vec::new(), + user_name_converts: true, + group_name_converts: true, + pax_status: Status::Ok, + pax_left: 0, + pax_padding: 0, + pax_line_length: 0, + pax_line_pos: 0, + pax_key: [0; KEY_CAPACITY], + pax_key_len: 0, + pax_key_ended: false, + pax_key_started: false, + entry: Entry::default(), + size_fields: 0, + sparse: Vec::new(), + sparse_head: 0, + sparse_map_left: 0, + sparse_map_entries: 0, + sparse_map_offset: 0, + line: [0; SPARSE_MAP_LINE_LIMIT], + line_len: 0, + format: Format::Old, + typeflag: 0, + sparse_major: 0, + sparse_minor: 0, + sparse_offset: 0, + sparse_num_bytes: 0, + gnu_sparse_size: 0, + gnu_sparse_real_size: 0, + schily_real_size: 0, + pax_size: 0, + disk_size: 0, + bytes_remaining: 0, + skipped: 0, + padding: 0, + null_blocks: 0, + } + } + + fn is_npm(&self) -> bool { + self.options.dialect == Dialect::Npm + } + + /// Returns how much of `input` was consumed. `eof`: nothing follows it. An error is final. + pub fn next<'a>(&'a mut self, input: &'a [u8], eof: bool) -> Result<(usize, Event<'a>)> { + let (used, poll) = self.poll(input, eof)?; + Ok((used, poll.event(&self.entry, input))) + } + + /// The entry of the last [`Event::Entry`]. + pub fn entry(&self) -> &Entry { + &self.entry + } + + /// Refuses input that does not start like tar, given its first block if there is that much. + pub(crate) fn detect(&mut self, start: &[u8]) -> Result { + let mut used = 0usize; + if self.fill_block(start, &mut used).is_none() + || !(header::is_null(&self.block) || header::is_plausible(&self.block)) + { + self.state = State::Failed(Error::UnrecognizedFormat); + return Err(Error::UnrecognizedFormat); + } + // `block` stays full, so `Header` takes it from here. + Ok(used) + } + + pub(crate) fn poll(&mut self, input: &[u8], eof: bool) -> Result<(usize, Poll)> { + let mut pos = 0usize; + loop { + let rest = input.get(pos..).unwrap_or_default(); + let mut used = 0usize; + let step = match self.step(rest, eof, &mut used) { + Ok(step) => step, + Err(error) => { + self.state = State::Failed(error); + return Err(error); + } + }; + let start = pos; + pos = pos.saturating_add(used); + let poll = match step { + Step::Continue => continue, + Step::NeedInput => Poll::NeedInput, + Step::Entry => Poll::Entry, + Step::Data { offset, len } => Poll::Data { offset, start, len }, + Step::EndOfEntry => Poll::EndOfEntry { + size: self.disk_size, + }, + Step::EndOfArchive => Poll::EndOfArchive, + }; + return Ok((pos, poll)); + } + } + + /// Drops what is left of the current entry's data. No `EndOfEntry` follows. + pub fn skip_entry(&mut self) { + if matches!(self.state, State::Data) { + self.state = State::NextEntry; + } + } + + /// `None` when `input` ran out first. + fn fill_block(&mut self, input: &[u8], used: &mut usize) -> Option<()> { + let want = self.block.len().saturating_sub(self.block_len); + let take = want.min(input.len()); + if let (Some(dest), Some(src)) = ( + self.block + .get_mut(self.block_len..self.block_len.saturating_add(take)), + input.get(..take), + ) { + dest.copy_from_slice(src); + } + self.block_len = self.block_len.saturating_add(take); + *used = take; + (self.block_len == self.block.len()).then_some(()) + } + + /// Like `fill_block`, but a block that `input` holds all of is not copied. + fn take_block<'a>(&mut self, input: &'a [u8], used: &mut usize) -> Option> { + if self.block_len == 0 + && let Some(block) = input.first_chunk() + { + *used = block.len(); + return Some(Cow::Borrowed(block)); + } + self.fill_block(input, used)?; + self.block_len = 0; + Some(Cow::Owned(self.block)) + } + + fn step(&mut self, input: &[u8], eof: bool, used: &mut usize) -> Result { + match self.state { + State::Failed(error) => Err(error), + State::Done => Ok(Step::EndOfArchive), + + State::NextEntry => { + let left = self.bytes_remaining.saturating_add(self.padding); + self.bytes_remaining = 0; + self.padding = 0; + self.state = State::Skip(left, AfterSkip::NextEntry); + Ok(Step::Continue) + } + + State::Header => { + let Some(block) = self.take_block(input, used) else { + if !eof { + return Ok(Step::NeedInput); + } + if self.block_len != 0 { + return Err(Error::TruncatedHeader); + } + if self.eof_is_fatal && !self.is_npm() { + return Err(Error::EndWithinHeaders); + } + self.state = State::Done; + return Ok(Step::EndOfArchive); + }; + self.on_header(&block)?; + Ok(Step::Continue) + } + + State::Skip(left, after) => { + let take = usize::try_from(left).unwrap_or(usize::MAX).min(input.len()); + *used = take; + let taken = i64::try_from(take).unwrap_or(i64::MAX); + let left = left.saturating_sub(taken); + self.skipped = self.skipped.saturating_add(taken); + if left > 0 { + self.state = State::Skip(left, after); + if !eof { + return Ok(Step::NeedInput); + } + return Err(match after { + AfterSkip::EndOfEntry | AfterSkip::NextEntry => Error::TruncatedSkip, + AfterSkip::MacMetadata => Error::TruncatedMacMetadata, + _ => Error::TruncatedInput { + needed: left.saturating_add(self.skipped), + available: self.skipped, + }, + }); + } + self.skipped = 0; + self.after_skip(after) + } + + State::Collect(left, what) => { + let take = left.min(input.len()); + self.collected + .write(input.get(..take).unwrap_or_default())?; + *used = take; + let left = left.saturating_sub(take); + if left > 0 { + self.state = State::Collect(left, what); + if !eof { + return Ok(Step::NeedInput); + } + return Err(Error::TruncatedMetadata); + } + self.on_collected(what)?; + Ok(Step::Continue) + } + + State::PaxLength => { + for &c in input { + if self.pax_line_pos >= self.pax_left { + break; + } + *used = used.saturating_add(1); + self.pax_line_pos = self.pax_line_pos.saturating_add(1); + match c { + b' ' => { + if self.pax_line_length > self.pax_left { + self.ignore_rest_of_pax(); + } else { + self.pax_key_len = 0; + self.pax_key_ended = false; + self.state = State::PaxKey; + } + return Ok(Step::Continue); + } + b'0'..=b'9' => { + self.pax_line_length = self + .pax_line_length + .saturating_mul(10) + .saturating_add(i64::from(c.wrapping_sub(b'0'))); + if self.pax_line_length > PAX_LINE_LIMIT { + self.ignore_rest_of_pax(); + return Ok(Step::Continue); + } + } + _ => { + self.ignore_rest_of_pax(); + return Ok(Step::Continue); + } + } + } + self.pax_prologue_ran_out(eof) + } + + State::PaxKey => { + for &c in input { + if self.pax_line_pos >= self.pax_left + || self.pax_line_pos >= self.pax_line_length + { + break; + } + *used = used.saturating_add(1); + self.pax_line_pos = self.pax_line_pos.saturating_add(1); + if c != b'=' { + if c == 0 { + self.pax_key_ended = true; + } + if !self.pax_key_ended { + if let Some(slot) = self.pax_key.get_mut(self.pax_key_len) { + *slot = c; + } + self.pax_key_len = self.pax_key_len.saturating_add(1); + } + // A key that starts with a NUL is still a key. + self.pax_key_started = true; + continue; + } + if !self.pax_key_started { + self.ignore_rest_of_pax(); + return Ok(Step::Continue); + } + self.on_pax_key()?; + return Ok(Step::Continue); + } + self.pax_prologue_ran_out(eof) + } + + State::PaxNewline(status) => { + self.pax_line_pos = 0; + if status == Status::Failed { + self.skip_rest_of_pax(Status::Failed); + return Ok(Step::Continue); + } + let Some(&c) = input.first() else { + if !eof { + return Ok(Step::NeedInput); + } + return Err(Error::TruncatedMetadata); + }; + self.pax_status = self.pax_status.min(status); + if c != b'\n' { + self.ignore_rest_of_pax(); + return Ok(Step::Continue); + } + *used = 1; + self.pax_left = self.pax_left.saturating_sub(1); + if self.pax_left > 0 { + self.begin_pax_line(); + } else { + self.finish_pax()?; + } + Ok(Step::Continue) + } + + State::OldSparseExtension => { + let Some(block) = self.take_block(input, used) else { + if !eof { + return Ok(Step::NeedInput); + } + return Err(Error::TruncatedSparse); + }; + self.add_old_sparse_entries(gnu::extension_sparse(&block))?; + if !gnu::extension_is_extended(&block) { + let h = self.main_header; + self.header_common(&h)?; + } + Ok(Step::Continue) + } + + State::SparseMapLine(field) => { + let limit = self.sparse_map_left.min(SPARSE_MAP_LINE_LIMIT as i64); + let limit = usize::try_from(limit).unwrap_or(0); + for &c in input { + *used = used.saturating_add(1); + if self.line_len >= limit { + return Err(Error::LineTooLong); + } + if let Some(slot) = self.line.get_mut(self.line_len) { + *slot = c; + } + self.line_len = self.line_len.saturating_add(1); + if c == b'\n' { + self.on_sparse_map_line(field)?; + return Ok(Step::Continue); + } + } + if !eof { + return Ok(Step::NeedInput); + } + Err(Error::MalformedSparseMap) + } + + State::Emit => { + self.state = State::Data; + Ok(Step::Entry) + } + + State::Data => { + while self + .sparse + .get(self.sparse_head) + .is_some_and(|block| block.remaining == 0) + { + self.sparse_head = self.sparse_head.saturating_add(1); + } + let Some(block) = self + .sparse + .get_mut(self.sparse_head) + .filter(|_| self.bytes_remaining != 0) + else { + let left = self.bytes_remaining.saturating_add(self.padding); + self.bytes_remaining = 0; + self.padding = 0; + self.state = State::Skip(left, AfterSkip::EndOfEntry); + return Ok(Step::Continue); + }; + if input.is_empty() { + if !eof { + return Ok(Step::NeedInput); + } + return Err(Error::TruncatedData); + } + let len = i64::try_from(input.len()) + .unwrap_or(i64::MAX) + .min(self.bytes_remaining) + .min(block.remaining); + let offset = block.offset; + block.remaining = block.remaining.saturating_sub(len); + block.offset = block.offset.saturating_add(len); + self.bytes_remaining = self.bytes_remaining.saturating_sub(len); + let len = usize::try_from(len).unwrap_or(0); + *used = len; + if block.hole { + return Ok(Step::Continue); + } + Ok(Step::Data { offset, len }) + } + + State::Drain => { + *used = input.len(); + if !eof { + return Ok(Step::NeedInput); + } + self.state = State::Done; + Ok(Step::EndOfArchive) + } + } + } + + fn after_skip(&mut self, after: AfterSkip) -> Result { + match after { + AfterSkip::BetweenHeaders(status) => self.between_headers(status), + AfterSkip::PaxNewline(status) => self.state = State::PaxNewline(status), + AfterSkip::MacMetadata => { + self.entry.clear(); + self.seen = seen::MAC_METADATA; + self.reset_header_state(); + self.between_headers(Status::Ok); + } + AfterSkip::SparseMap => self.finish_header(self.status)?, + AfterSkip::EndOfEntry => { + self.state = State::NextEntry; + return Ok(Step::EndOfEntry); + } + AfterSkip::NextEntry => { + self.entry.clear(); + self.sparse.clear(); + self.sparse_head = 0; + self.size_fields = 0; + self.status = Status::Ok; + self.eof_is_fatal = false; + self.seen = 0; + self.reset_header_state(); + self.state = State::Header; + } + } + Ok(Step::Continue) + } + + fn reset_header_state(&mut self) { + self.pax_is_utf8 = true; + self.sparse_attributes_seen = false; + self.pax_path.clear(); + self.pax_path_override.clear(); + self.pax_link.clear(); + self.user_name_converts = true; + self.group_name_converts = true; + } + + fn between_headers(&mut self, status: Status) { + self.status = self.status.min(status); + if self.seen & !(seen::VOLUME | seen::PAX_GLOBAL) != 0 { + self.eof_is_fatal = true; + } + self.state = State::Header; + } + + fn mark_seen(&mut self, bit: u32) -> bool { + let before = self.seen & bit != 0; + self.seen |= bit; + before + } + + fn on_header(&mut self, block: &Block) -> Result<()> { + if header::is_null(block) { + if self.is_npm() { + self.null_blocks = self.null_blocks.saturating_add(1); + if self.null_blocks == 2 { + self.state = State::Drain; + } + } + return Ok(()); + } + self.null_blocks = 0; + + if !header::checksum_ok(block) { + if self.eof_is_fatal { + return Err(Error::BadChecksum); + } + // Look for a header one block further on, with nothing carried over. + self.state = State::NextEntry; + return Ok(()); + } + + let size = header::size(block); + match header::typeflag(block) { + b'A' => { + if self.mark_seen(seen::SOLARIS_ACL) { + return Err(Error::RedundantHeader("A")); + } + self.format = Format::Pax; + // A Solaris ACL. As in GNU tar and node-tar, permissions come from the header's mode alone. + self.begin_body(size, None) + } + b'g' => { + if self.mark_seen(seen::PAX_GLOBAL) { + return Err(Error::RedundantHeader("g")); + } + self.format = Format::Pax; + if !(0..=ENTRY_LIMIT).contains(&size) { + return Err(Error::InvalidSpecialHeaderSize(size)); + } + if size == 0 { + return Err(Error::EmptyPaxGlobalHeader); + } + self.skip_body(size, Status::Ok); + Ok(()) + } + b'K' => { + if self.mark_seen(seen::LONG_LINK) { + self.status = self.status.min(Status::Warn); + } + self.format = Format::Gnu; + self.begin_body(size, Some(Collect::LongLink)) + } + b'L' => { + if self.mark_seen(seen::LONG_NAME) { + self.status = self.status.min(Status::Warn); + } + self.format = Format::Gnu; + self.begin_body(size, Some(Collect::LongName)) + } + b'V' => { + if self.mark_seen(seen::VOLUME) { + self.status = self.status.min(Status::Warn); + } + self.format = Format::Gnu; + if !(0..=PATHNAME_LIMIT).contains(&size) { + return Err(Error::InvalidSpecialHeaderSize(size)); + } + self.skip_body(size, Status::Ok); + Ok(()) + } + flag @ (b'x' | b'X') => { + if self.mark_seen(seen::PAX) { + return Err(Error::RedundantHeader(if flag == b'x' { + "x" + } else { + "X'/'x" + })); + } + self.format = Format::Pax; + self.begin_pax(size) + } + _ => self.on_main_header(block), + } + } + + fn skip_body(&mut self, size: i64, status: Status) { + self.state = State::Skip( + size.saturating_add(padding_for(size)), + AfterSkip::BetweenHeaders(status), + ); + } + + /// The body of a header whose size is not trusted past [`PATHNAME_LIMIT`]. + fn begin_body(&mut self, size: i64, collect: Option) -> Result<()> { + if !(0..=ENTRY_LIMIT).contains(&size) { + return Err(Error::InvalidSpecialHeaderSize(size)); + } + if size > PATHNAME_LIMIT { + self.skip_body(size, Status::Warn); + return Ok(()); + } + let Some(collect) = collect else { + self.skip_body(size, Status::Ok); + return Ok(()); + }; + self.collected.clear(); + self.state = State::Collect(usize::try_from(size).unwrap_or(0), collect); + Ok(()) + } + + fn begin_pax(&mut self, size: i64) -> Result<()> { + if !(0..=ENTRY_LIMIT).contains(&size) { + return Err(Error::InvalidPaxHeaderSize(size)); + } + if size == 0 { + return Err(Error::EmptyPaxHeader); + } + if size > PAX_BODY_LIMIT { + self.skip_body(size, Status::Warn); + return Ok(()); + } + self.pax_status = Status::Ok; + self.pax_left = size; + self.pax_padding = padding_for(size); + self.begin_pax_line(); + Ok(()) + } + + fn begin_pax_line(&mut self) { + self.pax_line_length = 0; + self.pax_line_pos = 0; + self.pax_key_started = false; + self.state = State::PaxLength; + } + + /// One malformed record makes the rest of the header unreadable. What came before it counts. + fn ignore_rest_of_pax(&mut self) { + self.skip_rest_of_pax(Status::Warn); + } + + fn skip_rest_of_pax(&mut self, status: Status) { + let left = self + .pax_left + .saturating_sub(self.pax_line_pos) + .saturating_add(self.pax_padding); + self.state = State::Skip(left, AfterSkip::BetweenHeaders(status)); + } + + fn pax_prologue_ran_out(&mut self, eof: bool) -> Result { + let at_end = self.pax_line_pos >= self.pax_left + || (matches!(self.state, State::PaxKey) && self.pax_line_pos >= self.pax_line_length); + if at_end { + self.ignore_rest_of_pax(); + return Ok(Step::Continue); + } + if !eof { + return Ok(Step::NeedInput); + } + Err(Error::TruncatedPaxName) + } + + fn on_pax_key(&mut self) -> Result<()> { + self.pax_left = self.pax_left.saturating_sub(self.pax_line_pos); + let with_newline = self.pax_line_length.saturating_sub(self.pax_line_pos); + self.pax_line_pos = 0; + if with_newline == 0 { + self.ignore_rest_of_pax(); + return Ok(()); + } + let length = with_newline.saturating_sub(1); + self.pax_left = self.pax_left.saturating_sub(length); + + enum Value { + Skip(Status), + Collect(Collect), + } + use Value::{Collect as Take, Skip}; + + let number = |which: Number| -> Result { + if length > NUMBER_LIMIT { + return Err(Error::MalformedPaxAttribute); + } + Ok(Take(Collect::Number(which))) + }; + let time = |is_mtime: bool| -> Result { + if length > TIME_LIMIT { + return Err(Error::MalformedPaxAttribute); + } + Ok(Take(Collect::Time(is_mtime))) + }; + let warn_unless = |ok: bool| Skip(Status::warn_unless(ok)); + + let stored = self.pax_key_len.min(KEY_CAPACITY); + let key_buf = self.pax_key; + let key = key_buf.get(..stored).unwrap_or_default(); + let whole = self.pax_key_len <= KEY_CAPACITY; + let is = |name: &[u8]| whole && key == name; + + let value = if self.is_npm() && key.first().is_some_and(u8::is_ascii_uppercase) { + Skip(Status::Ok) + } else if let Some(sparse) = key.strip_prefix(b"GNU.sparse") { + if sparse.is_empty() || (sparse.len() > 1 && sparse.first() == Some(&b'.')) { + self.sparse_attributes_seen = true; + } + match sparse { + _ if !whole => Skip(Status::Ok), + b".numblocks" => { + self.sparse_offset = -1; + self.sparse_num_bytes = -1; + self.sparse_major = 0; + self.sparse_minor = 0; + Skip(Status::Ok) + } + b".offset" => number(Number::SparseOffset)?, + b".numbytes" => number(Number::SparseNumBytes)?, + b".size" => number(Number::SparseSize)?, + b".map" => { + self.sparse_major = 0; + self.sparse_minor = 1; + if length > SPARSE_MAP_LIMIT { + Skip(Status::Failed) + } else { + Take(Collect::SparseMap) + } + } + b".major" => number(Number::SparseMajor)?, + b".minor" => number(Number::SparseMinor)?, + b".name" if length > PATHNAME_LIMIT => Skip(Status::Warn), + b".name" => Take(Collect::Text(Text::PathOverride)), + b".realsize" => number(Number::SparseRealSize)?, + _ => Skip(Status::Ok), + } + } else if is(b"LIBARCHIVE.creationtime") || is(b"atime") || is(b"ctime") { + time(false)? + } else if is(b"LIBARCHIVE.symlinktype") { + if length < SYMLINK_TYPE_LIMIT { + Take(Collect::SymlinkType) + } else { + Skip(Status::Warn) + } + } else if (key.starts_with(LIBARCHIVE_XATTR) && self.pax_key_len > LIBARCHIVE_XATTR.len()) + || is(b"RHT.security.selinux") + { + warn_unless(length <= XATTR_LIMIT) + } else if is(b"SCHILY.devmajor") + || is(b"SCHILY.devminor") + || is(b"SCHILY.dev") + || is(b"SCHILY.ino") + || is(b"SCHILY.nlink") + || is(b"gid") + || is(b"uid") + { + number(Number::Unused)? + } else if is(b"SCHILY.fflags") { + warn_unless(length < FFLAGS_LIMIT) + } else if is(b"SCHILY.realsize") { + number(Number::SchilyRealSize)? + } else if key.starts_with(SCHILY_XATTR) && self.pax_key_len > SCHILY_XATTR.len() { + let name_len = self.pax_key_len.saturating_sub(SCHILY_XATTR.len()); + warn_unless(length < XATTR_LIMIT && name_len <= XATTR_NAME_LIMIT) + } else if is(b"SUN.holesdata") { + if length < SPARSE_MAP_LIMIT { + Take(Collect::HolesData) + } else { + Skip(Status::Failed) + } + } else if is(b"gname") || is(b"uname") { + if length > USER_NAME_LIMIT { + Skip(Status::Warn) + } else if is(b"gname") { + Take(Collect::Text(Text::GroupName)) + } else { + Take(Collect::Text(Text::UserName)) + } + } else if is(b"hdrcharset") { + if length < CHARSET_LIMIT { + Take(Collect::HeaderCharset) + } else { + Skip(Status::Warn) + } + } else if is(b"linkpath") || is(b"path") { + if length > PATHNAME_LIMIT { + Skip(Status::Warn) + } else if is(b"path") { + Take(Collect::Text(Text::Path)) + } else { + Take(Collect::Text(Text::LinkPath)) + } + } else if is(b"mtime") { + time(true)? + } else if is(b"size") { + number(Number::Size)? + } else { + Skip(Status::Ok) + }; + + match value { + Skip(status) => self.state = State::Skip(length, AfterSkip::PaxNewline(status)), + Take(what) => { + self.collected.clear(); + self.state = State::Collect(usize::try_from(length).unwrap_or(0), what); + } + } + Ok(()) + } + + fn converts(&self, text: &[u8]) -> Result { + let text = slice_to_nul(text); + match self.options.charset { + Charset::Ascii => Ok(text.is_ascii()), + Charset::Utf8 => crate::compose::converts(text), + } + } + + fn finish_pax(&mut self) -> Result<()> { + let mut status = self.pax_status; + let utf8 = self.pax_is_utf8; + + let path = if self.pax_path_override.is_empty() { + &self.pax_path + } else { + &self.pax_path_override + }; + if !path.is_empty() { + let converts = !utf8 || self.converts(path)?; + assign(&mut self.entry.path, slice_to_nul(path))?; + self.entry.path_is_utf8 = utf8 && converts; + if !converts { + status = Status::Warn; + } + } + if utf8 && !(self.user_name_converts && self.group_name_converts) { + status = Status::Warn; + } + if !self.pax_link.is_empty() { + let converts = !utf8 || self.converts(&self.pax_link)?; + assign(&mut self.entry.link, slice_to_nul(&self.pax_link))?; + if !converts { + status = Status::Warn; + } + } + self.state = State::Skip( + self.pax_left.saturating_add(self.pax_padding), + AfterSkip::BetweenHeaders(status), + ); + Ok(()) + } + + fn on_collected(&mut self, what: Collect) -> Result<()> { + let collected = core::mem::take(&mut self.collected); + let result = self.use_collected(what, &collected); + self.collected = collected; + result + } + + fn use_collected(&mut self, what: Collect, value: &[u8]) -> Result<()> { + let status = match what { + Collect::LongName | Collect::LongLink => { + if matches!(what, Collect::LongName) { + assign(&mut self.entry.path, slice_to_nul(value))?; + self.entry.path_is_utf8 = false; + } else { + assign(&mut self.entry.link, slice_to_nul(value))?; + } + let padding = padding_for(i64::try_from(value.len()).unwrap_or(0)); + self.state = State::Skip(padding, AfterSkip::BetweenHeaders(Status::Ok)); + return Ok(()); + } + Collect::Number(which) => { + let parsed = parse_decimal(value); + let valid = parsed >= 0 && parsed != i64::MAX; + if valid { + self.on_pax_number(which, parsed)?; + } else if matches!(which, Number::Size) { + return Err(Error::PaxSizeOverflow); + } + Status::warn_unless(valid) + } + Collect::Time(is_mtime) => { + let seconds = parse_pax_time(value); + if is_mtime && seconds.is_some() { + self.entry.mtime = seconds; + } + Status::warn_unless(seconds.is_some()) + } + Collect::Text(which) => { + match which { + Text::Path => assign(&mut self.pax_path, value)?, + Text::PathOverride => assign(&mut self.pax_path_override, value)?, + Text::LinkPath => assign(&mut self.pax_link, value)?, + Text::UserName => self.user_name_converts = self.converts(value)?, + Text::GroupName => self.group_name_converts = self.converts(value)?, + } + Status::Ok + } + Collect::HeaderCharset => { + let utf8 = match value { + b"BINARY" => Some(false), + b"ISO-IR 10646 2000 UTF-8" => Some(true), + _ => None, + }; + self.pax_is_utf8 = utf8.unwrap_or(self.pax_is_utf8); + Status::warn_unless(utf8.is_some()) + } + Collect::SymlinkType => Status::warn_unless(matches!(value, b"file" | b"dir")), + Collect::SparseMap => Status::warn_unless(self.parse_sparse_map_0_1(value)?), + Collect::HolesData => Status::warn_unless(self.parse_holes_data(value)?), + }; + self.state = State::PaxNewline(status); + Ok(()) + } + + fn on_pax_number(&mut self, which: Number, value: i64) -> Result<()> { + match which { + Number::SparseOffset => { + self.sparse_offset = value; + if self.sparse_num_bytes != -1 { + self.add_pending_sparse_block()?; + } + } + Number::SparseNumBytes => { + self.sparse_num_bytes = value; + if self.sparse_offset != -1 { + self.add_pending_sparse_block()?; + } + } + Number::SparseSize => { + self.gnu_sparse_size = value; + self.size_fields |= size_field::GNU_SPARSE_SIZE; + } + Number::SparseMajor if value <= SPARSE_VERSION_LIMIT => self.sparse_major = value, + Number::SparseMinor if value <= SPARSE_VERSION_LIMIT => self.sparse_minor = value, + Number::SparseMajor | Number::SparseMinor | Number::Unused => {} + Number::SparseRealSize => { + self.gnu_sparse_real_size = value; + self.size_fields |= size_field::GNU_SPARSE_REALSIZE; + } + Number::SchilyRealSize => { + self.schily_real_size = value; + self.size_fields |= size_field::SCHILY_REALSIZE; + } + Number::Size => { + self.pax_size = value; + self.size_fields |= size_field::PAX_SIZE; + } + } + Ok(()) + } + + fn add_pending_sparse_block(&mut self) -> Result<()> { + if !self.add_sparse_block(self.sparse_offset, self.sparse_num_bytes, false)? { + return Err(Error::MalformedSparseMap); + } + self.sparse_offset = -1; + self.sparse_num_bytes = -1; + Ok(()) + } + + /// `false` for a block that cannot be. An empty one takes its place, so the list is not empty. + fn add_sparse_block(&mut self, offset: i64, remaining: i64, hole: bool) -> Result { + self.sparse.try_reserve(1)?; + let valid = remaining >= 0 && offset >= 0 && offset.checked_add(remaining).is_some(); + self.sparse.push(if valid { + SparseBlock { + offset, + remaining, + hole, + } + } else { + SparseBlock { + offset: 0, + remaining: 0, + hole: false, + } + }); + Ok(valid) + } + + /// `offset,size,offset,size,...` + fn parse_sparse_map_0_1(&mut self, map: &[u8]) -> Result { + let mut offset: Option = None; + for field in strings::split(map, b",") { + if !field.iter().all(u8::is_ascii_digit) { + return Ok(false); + } + let value = parse_decimal(field); + match offset.take() { + None => offset = Some(value), + Some(offset) => { + if !self.add_sparse_block(offset, value, false)? { + return Ok(false); + } + } + } + } + Ok(true) + } + + /// ` end end end ...`, where the ranges between the ends alternate between hole and data. + fn parse_holes_data(&mut self, value: &[u8]) -> Result { + let Some(mut rest) = value.strip_prefix(b" ") else { + return Ok(false); + }; + let mut end: i64 = 0; + let mut hole = true; + loop { + let len = strings::index_of_char_usize(rest, b' ').unwrap_or(rest.len()); + let (field, tail) = rest.split_at(len); + if !field.iter().all(u8::is_ascii_digit) { + return Ok(false); + } + let start = end; + end = parse_decimal(field); + if start < end && !self.add_sparse_block(start, end.saturating_sub(start), hole)? { + return Err(Error::MalformedSparseMap); + } + let Some(tail) = tail.strip_prefix(b" ") else { + return Ok(true); + }; + rest = tail; + hole = !hole; + } + } + + fn add_old_sparse_entries(&mut self, entries: &[u8]) -> Result<()> { + for entry in entries.as_chunks::<{ gnu::SPARSE_ENTRY_SIZE }>().0 { + let (offset, num_bytes) = entry.split_at(gnu::SPARSE_NUMBER_SIZE); + if offset.first() == Some(&0) { + break; + } + if !self.add_sparse_block(parse_field(offset), parse_field(num_bytes), false)? { + return Err(Error::MalformedSparseMap); + } + } + Ok(()) + } + + fn on_main_header(&mut self, h: &Block) -> Result<()> { + let magic = header::magic(h); + let npm = self.is_npm(); + + if magic == gnu::MAGIC { + self.format = Format::Gnu; + if self.entry.path.is_empty() { + self.entry.path_is_utf8 = false; + assign(&mut self.entry.path, slice_to_nul(header::name(h)))?; + } + if !npm { + if gnu::real_size(h).first() != Some(&0) { + self.gnu_sparse_real_size = parse_field(gnu::real_size(h)); + self.size_fields |= size_field::GNU_SPARSE_REALSIZE; + } + if gnu::sparse(h).first() != Some(&0) { + self.add_old_sparse_entries(gnu::sparse(h))?; + if gnu::is_extended(h) { + self.main_header = *h; + self.state = State::OldSparseExtension; + return Ok(()); + } + } + } + } else if magic.starts_with(b"ustar") && (!npm || magic == header::USTAR_MAGIC) { + if self.format != Format::Pax { + self.format = Format::Ustar; + } + if self.entry.path.is_empty() { + self.entry.path_is_utf8 = false; + let prefix = slice_to_nul(header::prefix(h)); + assign(&mut self.entry.path, prefix)?; + if !prefix.is_empty() && prefix.last() != Some(&b'/') { + self.entry.path.write(b"/")?; + } + self.entry.path.write(slice_to_nul(header::name(h)))?; + } + } else { + self.format = Format::Old; + if self.entry.path.is_empty() { + self.entry.path_is_utf8 = false; + assign(&mut self.entry.path, slice_to_nul(header::name(h)))?; + } + } + self.header_common(h) + } + + fn header_common(&mut self, h: &Block) -> Result<()> { + // Truncated the way a 32-bit `mode_t` is. + self.entry.mode = header::mode(h) as u32; + if self.entry.mtime.is_none() { + self.entry.mtime = Some(header::mtime(h)); + } + + let header_size = header::size(h); + let has = |field: u8| self.size_fields & field != 0; + + self.disk_size = if has(size_field::GNU_SPARSE_REALSIZE) { + self.gnu_sparse_real_size + } else if has(size_field::GNU_SPARSE_SIZE) && self.sparse_major == 0 { + self.gnu_sparse_size + } else if has(size_field::SCHILY_REALSIZE) { + self.schily_real_size + } else if has(size_field::PAX_SIZE) { + self.pax_size + } else { + header_size + }; + if self.disk_size < 0 { + return Err(Error::NegativeFileSize); + } + if self.disk_size > ENTRY_LIMIT { + return Err(Error::SizeOverflow); + } + self.entry.size = self.disk_size; + + self.bytes_remaining = if has(size_field::GNU_SPARSE_SIZE) && self.sparse_major == 1 { + self.gnu_sparse_size + } else if has(size_field::PAX_SIZE) { + self.pax_size + } else { + header_size + }; + if self.bytes_remaining < 0 { + self.bytes_remaining = 0; + return Err(Error::NegativeSize); + } + if self.bytes_remaining > ENTRY_LIMIT { + self.bytes_remaining = 0; + return Err(Error::SizeOverflow); + } + + self.typeflag = header::typeflag(h); + let mut no_body = |entry: &mut Entry, filetype: u32| { + entry.set_filetype(filetype); + entry.size = 0; + self.bytes_remaining = 0; + }; + match self.typeflag { + b'1' => { + if self.entry.size > 0 { + self.entry.set_filetype(S::IFREG); + // Only pax lets a hardlink carry a body, and a ustar header does not say whether it is pax. + let ignore_size = match self.format { + Format::Pax => false, + Format::Old | Format::Gnu => true, + Format::Ustar => header::has_magic(h) && header::is_plausible(h), + }; + if ignore_size { + self.entry.size = 0; + self.bytes_remaining = 0; + } + } + } + b'2' => { + self.entry.is_symlink = true; + if self.entry.link.is_empty() { + assign(&mut self.entry.link, slice_to_nul(header::linkname(h)))?; + } + no_body(&mut self.entry, S::IFLNK); + } + b'3' => no_body(&mut self.entry, S::IFCHR), + b'4' => no_body(&mut self.entry, S::IFBLK), + b'5' => no_body(&mut self.entry, S::IFDIR), + b'6' => no_body(&mut self.entry, S::IFIFO), + b'D' => self.entry.set_filetype(S::IFDIR), + b'M' => {} + _ => self.entry.set_filetype(S::IFREG), + } + self.padding = padding_for(self.bytes_remaining); + self.after_main_header() + } + + fn after_main_header(&mut self) -> Result<()> { + if self.status == Status::Failed { + return Err(Error::SparseMapTooLarge); + } + + if self.options.mac_metadata + && self.seen & seen::MAC_METADATA == 0 + && is_mac_metadata_name(&self.entry.path) + { + let size = self.entry.size; + if size > XATTR_LIMIT { + return Err(Error::OversizedMacMetadata); + } + self.state = State::Skip( + size.saturating_add(padding_for(size)), + AfterSkip::MacMetadata, + ); + return Ok(()); + } + + if self.sparse_attributes_seen { + if !matches!(self.typeflag, b'S' | b'0') { + return self.finish_header(Status::Warn); + } + match (self.sparse_major, self.sparse_minor) { + (0, 0 | 1) => {} + (1, 0) => { + self.sparse.clear(); + self.sparse_head = 0; + self.sparse_map_left = self.bytes_remaining; + self.line_len = 0; + self.state = State::SparseMapLine(SparseMapField::Count); + return Ok(()); + } + _ => return self.finish_header(Status::Warn), + } + } + self.finish_header(self.status) + } + + /// Format 1.0: a count, then pairs of offset and size, each on a line, in front of the data. + fn on_sparse_map_line(&mut self, field: SparseMapField) -> Result<()> { + let line_buf = self.line; + let line = line_buf.get(..self.line_len).unwrap_or_default(); + self.sparse_map_left = self + .sparse_map_left + .saturating_sub(i64::try_from(line.len()).unwrap_or(0)); + self.line_len = 0; + if line.first() == Some(&b'#') { + return Ok(()); + } + let digits = line.strip_suffix(b"\n").unwrap_or(line); + if !digits.iter().all(u8::is_ascii_digit) { + return Err(Error::MalformedSparseMap); + } + let value = parse_decimal(digits); + + match field { + SparseMapField::Count => self.sparse_map_entries = value, + SparseMapField::Offset => { + self.sparse_map_offset = value; + self.state = State::SparseMapLine(SparseMapField::Size); + return Ok(()); + } + SparseMapField::Size => { + if !self.add_sparse_block(self.sparse_map_offset, value, false)? { + return Err(Error::MalformedSparseMap); + } + } + } + if self.sparse_map_entries > 0 { + self.sparse_map_entries = self.sparse_map_entries.saturating_sub(1); + self.state = State::SparseMapLine(SparseMapField::Offset); + return Ok(()); + } + + let read = self.bytes_remaining.saturating_sub(self.sparse_map_left); + let to_skip = padding_for(read); + if to_skip > self.sparse_map_left { + return Err(Error::MalformedSparseMap); + } + self.bytes_remaining = self + .bytes_remaining + .saturating_sub(read.saturating_add(to_skip)); + self.state = State::Skip(to_skip, AfterSkip::SparseMap); + Ok(()) + } + + fn finish_header(&mut self, status: Status) -> Result<()> { + if self.sparse.is_empty() { + self.add_sparse_block(0, self.bytes_remaining, false)?; + } + + let plain_file = matches!(self.typeflag, b'0' | 0); + let may_be_directory = if self.is_npm() { + plain_file + } else { + status == Status::Ok + }; + if may_be_directory + && self.entry.filetype() == S::IFREG + && self.entry.path.last() == Some(&b'/') + { + self.entry.set_filetype(S::IFDIR); + self.bytes_remaining = 0; + self.padding = 0; + } + + if self.is_npm() + && self.entry.filetype() == S::IFREG + && (!(plain_file || self.typeflag == b'7') || strings::contains_char(&self.pax_path, 0)) + { + self.entry.set_filetype(0); + } + + self.state = State::Emit; + Ok(()) + } +} + +impl Entry { + fn clear(&mut self) { + self.path.clear(); + self.link.clear(); + *self = Entry { + path: core::mem::take(&mut self.path), + link: core::mem::take(&mut self.link), + ..Entry::default() + }; + } + + fn set_filetype(&mut self, filetype: u32) { + self.mode = (self.mode & !S::IFMT) | filetype; + } +} + +/// The last component is `._` and at least one more character. +fn is_mac_metadata_name(path: &[u8]) -> bool { + let mut name = path; + let mut rest = path; + while let Some(slash) = strings::index_of_char_usize(rest, b'/') { + rest = rest.get(slash.saturating_add(1)..).unwrap_or_default(); + if !rest.is_empty() { + name = rest; + } + } + name.len() > 2 && name.starts_with(b"._") +} diff --git a/src/tar/writer.rs b/src/tar/writer.rs new file mode 100644 index 000000000000..b48596ecd26e --- /dev/null +++ b/src/tar/writer.rs @@ -0,0 +1,434 @@ +//! Writes "restricted pax": ustar, plus a pax header where that falls short. Regular files only. + +use crate::error::Result; +use crate::header::{ + BLOCK_SIZE, Block, CHECKSUM, DEVMAJOR, DEVMINOR, GID, MAGIC, MODE, MTIME, NAME, PREFIX, SIZE, + TYPEFLAG, UID, USTAR_MAGIC, padding_for, +}; +use crate::output::Output; +use bun_core::{slice_to_nul, strings}; +use core::ops::Range; + +/// How a name that is not ASCII is recorded. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Names { + /// POSIX: the bytes as they are, marked `hdrcharset=BINARY`. + Binary, + /// Windows: UTF-8 with its characters composed, and `\` turned into `/`. + Utf8, +} + +impl Names { + pub fn for_host() -> Self { + if cfg!(windows) { + Self::Utf8 + } else { + Self::Binary + } + } +} + +#[derive(Debug, Clone, Copy)] +pub struct File<'a> { + /// Ends at its first NUL. + pub path: &'a [u8], + pub size: u64, + pub perm: u32, + pub mtime: i64, +} + +const RECORD_SIZE: u64 = 10240; +/// The last time that fits in eleven octal digits, which still gets a pax record. +const USTAR_MTIME_MAX: i64 = (1 << 33) - 1; +const USTAR_SIZE_END: u64 = 1 << 33; +const MODE_DIGITS: usize = 6; +const SIZE_DIGITS: usize = 11; +const MTIME_DIGITS: usize = 11; +/// As many as always fit in an `i64`. +const MAX_OCTAL_DIGITS: usize = (i64::BITS / 3) as usize; +/// A NUL and a space follow them. +const CHECKSUM_DIGITS: Range = CHECKSUM.start..CHECKSUM.start + 6; +static ZEROS: [u8; 1024] = [0; 1024]; + +pub struct Writer { + out: O, + names: Names, + written: u64, + /// Of the current entry: the data still owed, and the zeros behind it. + remaining: u64, + padding: u64, +} + +impl Writer { + pub fn new(out: O, names: Names) -> Self { + Self { + out, + names, + written: 0, + remaining: 0, + padding: 0, + } + } + + fn put(&mut self, bytes: &[u8]) -> Result<()> { + self.written = self.written.saturating_add(bytes.len() as u64); + self.out.write(bytes) + } + + fn put_zeros(&mut self, mut count: u64) -> Result<()> { + while count > 0 { + let take = count.min(ZEROS.len() as u64); + self.put(ZEROS.get(..take as usize).unwrap_or_default())?; + count = count.saturating_sub(take); + } + Ok(()) + } + + /// Starts an entry, after finishing the one before it. + pub fn header(&mut self, file: &File<'_>) -> Result<()> { + self.finish_entry()?; + + let given = slice_to_nul(file.path); + let converted; + let (path, binary) = match self.names { + Names::Binary => (given, !given.is_ascii()), + Names::Utf8 => match crate::compose::windows_name(given)? { + Some(name) => { + converted = name; + (&converted[..], false) + } + None => (given, true), + }, + }; + + let mut pax: Vec = Vec::new(); + if binary { + add_record(&mut pax, b"hdrcharset", b"BINARY")?; + } + let shortened; + let mut name = path; + if !path.is_ascii() || !fits_in_ustar(path) { + add_record(&mut pax, b"path", path)?; + shortened = ustar_entry_name(path, None); + name = &shortened; + } + let mtime_fits = (0..USTAR_MTIME_MAX).contains(&file.mtime); + if !mtime_fits { + add_record( + &mut pax, + b"mtime", + itoa::Buffer::new().format(file.mtime).as_bytes(), + )?; + } + if file.size >= USTAR_SIZE_END { + let size = i64::try_from(file.size).unwrap_or(i64::MAX); + add_record( + &mut pax, + b"size", + itoa::Buffer::new().format(size).as_bytes(), + )?; + } + + if !pax.is_empty() { + let block = format_header(&Fields { + name: &pax_attribute_name(name), + perm: file.perm & 0o777, + size: pax.len() as u64, + mtime: file.mtime.clamp(0, USTAR_MTIME_MAX), + typeflag: b'x', + strict: true, + }); + self.put(&block)?; + self.put(&pax)?; + self.put_zeros(padding_for(pax.len() as u64))?; + } + + let block = format_header(&Fields { + name, + perm: file.perm, + size: file.size, + mtime: file.mtime, + typeflag: b'0', + strict: false, + }); + self.put(&block)?; + self.remaining = file.size; + self.padding = padding_for(file.size); + Ok(()) + } + + /// Returns how much was taken. What goes past the size in the header is dropped. + pub fn data(&mut self, bytes: &[u8]) -> Result { + let take = usize::try_from(self.remaining) + .unwrap_or(usize::MAX) + .min(bytes.len()); + if take > 0 { + self.put(bytes.get(..take).unwrap_or_default())?; + self.remaining = self.remaining.saturating_sub(take as u64); + } + Ok(take) + } + + /// Data that was announced and never written is made up with zeros. + fn finish_entry(&mut self) -> Result<()> { + let zeros = self.remaining.saturating_add(self.padding); + self.remaining = 0; + self.padding = 0; + self.put_zeros(zeros) + } + + /// `pad_to_record` fills the last 10 KiB record up, the way tar writes to a tape. + pub fn finish(mut self, pad_to_record: bool) -> Result { + self.finish_entry()?; + self.put_zeros(2 * BLOCK_SIZE as u64)?; + if pad_to_record { + let over = self.written % RECORD_SIZE; + if over != 0 { + self.put_zeros(RECORD_SIZE.saturating_sub(over))?; + } + } + Ok(self.out) + } +} + +/// ` =\n`, where the length counts its own digits too. +fn add_record(pax: &mut Vec, key: &[u8], value: &[u8]) -> Result<()> { + let without_length = key.len().saturating_add(value.len()).saturating_add(3); + let mut digits = 0usize; + let mut next_ten = 1usize; + let mut left = without_length; + while left > 0 { + left /= 10; + digits = digits.saturating_add(1); + next_ten = next_ten.saturating_mul(10); + } + if without_length.saturating_add(digits) >= next_ten { + digits = digits.saturating_add(1); + } + let length = without_length.saturating_add(digits); + pax.try_reserve(length)?; + pax.extend_from_slice(itoa::Buffer::new().format(length).as_bytes()); + pax.push(b' '); + pax.extend_from_slice(key); + pax.push(b'='); + pax.extend_from_slice(value); + pax.push(b'\n'); + Ok(()) +} + +fn find_slash(path: &[u8], from: usize) -> Option { + let tail = path.get(from..)?; + Some(from.saturating_add(strings::index_of_char_usize(tail, b'/')?)) +} + +/// The first `/` that leaves a name of at most 100 bytes behind a prefix that is not empty. +fn split_point(path: &[u8]) -> Option { + let mut at = find_slash(path, path.len().saturating_sub(NAME.len() + 1))?; + if at == 0 { + at = find_slash(path, 1)?; + } + (at.saturating_add(1) < path.len() && at <= PREFIX.len()).then_some(at) +} + +fn fits_in_ustar(path: &[u8]) -> bool { + path.len() <= NAME.len() || split_point(path).is_some() +} + +fn byte_at(path: &[u8], index: usize) -> u8 { + path.get(index).copied().unwrap_or(0) +} + +/// A stand-in for readers that do not know pax. `insert` becomes the last directory. +fn ustar_entry_name(src: &[u8], insert: Option<&[u8]>) -> Vec { + let insert_length = insert.map_or(0, |name| name.len().saturating_add(2)); + if src.len() < NAME.len() && insert.is_none() { + return src.to_vec(); + } + + let mut need_slash = false; + let mut filename_end = src.len(); + loop { + if filename_end > 0 && byte_at(src, filename_end.wrapping_sub(1)) == b'/' { + filename_end = filename_end.saturating_sub(1); + } else if filename_end > 1 + && byte_at(src, filename_end.wrapping_sub(1)) == b'.' + && byte_at(src, filename_end.wrapping_sub(2)) == b'/' + { + filename_end = filename_end.saturating_sub(2); + } else { + break; + } + need_slash = true; + } + let mut dest = Vec::new(); + if filename_end == 0 { + dest.extend_from_slice(insert.unwrap_or_default()); + dest.push(b'/'); + return dest; + } + + let mut suffix_length = NAME.len() - 2 - usize::from(need_slash); + let mut filename = filename_end.saturating_sub(1); + while filename > 0 && byte_at(src, filename) != b'/' { + filename = filename.saturating_sub(1); + } + if byte_at(src, filename) == b'/' && filename < filename_end.saturating_sub(1) { + filename = filename.saturating_add(1); + } + suffix_length = suffix_length.saturating_sub(insert_length); + filename_end = filename_end.min(filename.saturating_add(suffix_length)); + suffix_length = suffix_length.saturating_sub(filename_end.saturating_sub(filename)); + + // Backs up to a `/`, and then takes it along. + let directory_end = |start: usize, limit: usize| { + let mut end = limit.min(filename).max(start); + while end > start && byte_at(src, end) != b'/' { + end = end.saturating_sub(1); + } + if end < filename && byte_at(src, end) == b'/' { + end = end.saturating_add(1); + } + end + }; + let prefix_end = directory_end(0, PREFIX.len() - 1); + let suffix_end = directory_end(prefix_end, prefix_end.saturating_add(suffix_length)); + + dest.extend_from_slice(src.get(..suffix_end).unwrap_or_default()); + if let Some(insert) = insert { + dest.extend_from_slice(insert); + dest.push(b'/'); + } + dest.extend_from_slice(src.get(filename..filename_end).unwrap_or_default()); + if need_slash { + dest.push(b'/'); + } + dest +} + +/// The name of the pax header's own entry: `dir/PaxHeader/file` for `dir/file`. +fn pax_attribute_name(src: &[u8]) -> Vec { + if src.is_empty() { + return b"PaxHeader/blank".to_vec(); + } + let mut end = src.len(); + loop { + let slash = end > 0 && byte_at(src, end.wrapping_sub(1)) == b'/'; + let slash_dot = end > 1 + && byte_at(src, end.wrapping_sub(1)) == b'.' + && byte_at(src, end.wrapping_sub(2)) == b'/'; + if !slash && !slash_dot { + break; + } + end = end.saturating_sub(1); + } + if end == 0 { + return b"/PaxHeader/rootdir".to_vec(); + } + if end == 1 && src.first() == Some(&b'.') { + return b"PaxHeader/currentdir".to_vec(); + } + ustar_entry_name(src.get(..end).unwrap_or_default(), Some(b"PaxHeader")) +} + +struct Fields<'a> { + name: &'a [u8], + perm: u32, + size: u64, + mtime: i64, + typeflag: u8, + /// Keep every number in octal with its terminator, whatever it takes. + strict: bool, +} + +fn copy_into(block: &mut Block, at: usize, bytes: &[u8]) { + if let Some(dest) = block.get_mut(at..at.saturating_add(bytes.len())) { + dest.copy_from_slice(bytes); + } +} + +/// A value too large for the field fills it with sevens. +fn format_octal(mut value: i64, field: &mut [u8]) { + if value < 0 { + field.fill(b'0'); + return; + } + for slot in field.iter_mut().rev() { + *slot = b'0'.wrapping_add((value & 7) as u8); + value >>= 3; + } + if value != 0 { + field.fill(b'7'); + } +} + +/// A number too long for `digits` overwrites the terminator. Base-256 takes what still won't fit. +fn format_number(value: i64, field: &mut [u8], digits: usize, strict: bool) { + if strict { + return format_octal(value, field.get_mut(..digits).unwrap_or_default()); + } + if value >= 0 { + for width in digits..=field.len() { + if width >= MAX_OCTAL_DIGITS || value < 1i64 << width.saturating_mul(3) { + return format_octal(value, field.get_mut(..width).unwrap_or_default()); + } + } + } + let mut left = value; + for slot in field.iter_mut().rev() { + *slot = (left & 0xff) as u8; + left >>= 8; + } + if let Some(first) = field.first_mut() { + *first |= 0x80; + } +} + +fn format_header(fields: &Fields<'_>) -> Block { + let mut h: Block = [0; BLOCK_SIZE]; + for field in [MODE, UID, GID, DEVMAJOR, DEVMINOR] { + copy_into(&mut h, field.start, b"000000 "); + } + copy_into(&mut h, SIZE.start, b"00000000000 "); + copy_into(&mut h, MTIME.start, b"00000000000 "); + h[CHECKSUM].fill(b' '); + h[MAGIC].copy_from_slice(USTAR_MAGIC); + + if fields.name.len() <= NAME.len() { + copy_into(&mut h, NAME.start, fields.name); + } else if let Some(at) = split_point(fields.name) { + copy_into( + &mut h, + PREFIX.start, + fields.name.get(..at).unwrap_or_default(), + ); + copy_into( + &mut h, + NAME.start, + fields.name.get(at.saturating_add(1)..).unwrap_or_default(), + ); + } + format_number( + i64::from(fields.perm & 0o7777), + &mut h[MODE], + MODE_DIGITS, + fields.strict, + ); + format_number( + i64::try_from(fields.size).unwrap_or(i64::MAX), + &mut h[SIZE], + SIZE_DIGITS, + fields.strict, + ); + format_number( + fields.mtime, + &mut h[MTIME.start..MTIME.start + MTIME_DIGITS], + MTIME_DIGITS, + fields.strict, + ); + h[TYPEFLAG] = fields.typeflag; + + let checksum: u32 = h.iter().map(|&b| u32::from(b)).sum(); + h[CHECKSUM_DIGITS.end] = 0; + format_octal(i64::from(checksum), &mut h[CHECKSUM_DIGITS]); + h +} diff --git a/src/zlib/lib.rs b/src/zlib/lib.rs index b9aee44c945b..cf296b28f015 100644 --- a/src/zlib/lib.rs +++ b/src/zlib/lib.rs @@ -793,6 +793,18 @@ impl DeflateEncoder { deflate, ) } + + /// Like [`step`](Self::step), but `deflate()` sees exactly `limit` bytes of output space, + /// whatever `out`'s capacity is. At level 0 that space decides where a stored block ends. + pub fn step_limited( + &mut self, + input: &[u8], + out: &mut Vec, + limit: usize, + flush: FlushValue, + ) -> (usize, ReturnCode) { + step(&mut self.strm, input, out, limit, limit, flush, deflate) + } } impl Drop for DeflateEncoder { @@ -872,6 +884,24 @@ impl InflateDecoder { ) } + /// Like [`step`](Self::step), but appends at most `limit` bytes to `out`. + pub fn step_limited( + &mut self, + input: &[u8], + out: &mut Vec, + limit: usize, + ) -> (usize, ReturnCode) { + step( + &mut self.strm, + input, + out, + limit, + limit, + FlushValue::NoFlush, + inflate, + ) + } + /// Append decompressed output to `out` (growing by 4096-byte steps, capped at /// `max_output_size`) until `input` is consumed or `out.len()` reaches `max_output`. /// Returns the input bytes consumed. Returns `ShortRead` when more input is required and diff --git a/test/cli/install/GHSA-pfwx-36v6-832x.test.ts b/test/cli/install/GHSA-pfwx-36v6-832x.test.ts index c9ca2a047e33..27fe140b6e6c 100644 --- a/test/cli/install/GHSA-pfwx-36v6-832x.test.ts +++ b/test/cli/install/GHSA-pfwx-36v6-832x.test.ts @@ -73,7 +73,7 @@ async function lockedEntry(dir: string) { } // Tarballs smaller than BUN_INSTALL_STREAMING_MIN_SIZE (2 MiB by default) are extracted -// once the whole body has been buffered; larger ones are streamed into libarchive while +// once the whole body has been buffered; larger ones are streamed into the extractor while // they download. The two extractors hash and verify the tarball independently, so every // test runs against both. `--verbose` output names the extractor that ran. const variants = [ diff --git a/test/cli/install/bun-install-streaming-extract.test.ts b/test/cli/install/bun-install-streaming-extract.test.ts index d254ef2f066f..47af5ad8872d 100644 --- a/test/cli/install/bun-install-streaming-extract.test.ts +++ b/test/cli/install/bun-install-streaming-extract.test.ts @@ -1,9 +1,9 @@ // Verifies that `bun install` can extract a tarball while it is still // downloading. A local registry drip-feeds the .tgz body in small // chunks so the HTTP thread delivers multiple progress callbacks; the -// streaming extractor (TarballStream.zig + the ARCHIVE_RETRY patches in -// vendor/libarchive) must reassemble them into the same on-disk layout -// the buffered extractor would produce. +// streaming extractor (TarballStream.rs on top of bun_tar's push decoder) +// must reassemble them into the same on-disk layout the buffered +// extractor would produce. import { describe, expect, setDefaultTimeout, test } from "bun:test"; import { bunEnv, bunExe, isMacOS, isWindows, readdirSorted, runCommandMaxRSS, tempDir } from "harness"; @@ -68,8 +68,8 @@ function pad512(len: number): Buffer { function tarFile(name: string, body: Buffer): Buffer[] { // ustar stores at most 100 bytes of name; longer paths need a pax - // 'x' record. npm's `tar` uses pax, so this exercises the resumable - // `tar_read_header` path in the libarchive patch. + // 'x' record. npm's `tar` uses pax, so this exercises a header that + // arrives in pieces. if (name.length > 100) { // Build the pax record so that the declared length includes the // length field itself. Iterate because adding digits to the length @@ -133,7 +133,7 @@ function makeEntries(): Entry[] { { path: "index.js", body: Buffer.from("module.exports = 'ok';\n") }, { // > 100 chars → pax extended header → exercises the resumable - // header path in the libarchive patch. + // header path. path: "very/deeply/nested/directory/structure/that/exceeds/the/one/hundred/byte/ustar/limit/long-name-file.txt", body: Buffer.from("long path ok\n"), }, @@ -540,7 +540,7 @@ describe("streaming tarball extraction", () => { test("streaming rejects a tarball whose integrity does not match", async () => { // Serve the valid tarball but advertise the integrity of a // *different* blob. Extraction will stream to completion (so we - // exercise the full ARCHIVE_RETRY path through libarchive), the + // exercise running out of input all the way through), the // incremental hasher produces the real SHA-512, and `finish()` // must notice the mismatch before the temp tree is promoted into // the cache. @@ -569,7 +569,7 @@ describe("streaming tarball extraction", () => { // A connection that dies part-way through the body is a failed download, // whichever extractor is consuming it: report the transport error, retry // like any other failed download, and never hand the truncated body to - // libarchive (streaming) or the integrity check (buffered). + // the decoder (streaming) or the integrity check (buffered). describe.each([ ["streaming", {}], ["buffered", { BUN_FEATURE_FLAG_DISABLE_STREAMING_INSTALL: "1" }], @@ -638,9 +638,9 @@ describe("streaming tarball extraction", () => { }); // A body that arrives in full but is not a valid archive is an extraction - // failure: not retried, and the message carries libarchive's reason and + // failure: not retried, and the message carries the decoder's reason and // the offset it gave up at. - test.concurrent("streaming reports libarchive's error for a corrupt tarball", async () => { + test.concurrent("streaming reports the decoder's error for a corrupt tarball", async () => { const corrupt = Buffer.from(tgz); corrupt.fill(0x55, corrupt.length >> 1, (corrupt.length >> 1) + 4096); const corruptIntegrity = "sha512-" + createHash("sha512").update(corrupt).digest("base64"); @@ -837,25 +837,17 @@ describe("streaming tarball extraction", () => { }); // ------------------------------------------------------------------- -// Regression: the nonblocking-read patch routed upstream libarchive's -// pre-existing damaged-block ARCHIVE_RETRY through the same `bun_retry` -// path as a non-blocking yield, so `seen_headers` / entry state leaked -// across the retry. A second pax 'g' global header after the damaged -// block would then trip "Redundant 'g' header" → ARCHIVE_FATAL even -// though upstream libarchive (and a `tar` CLI) accepts this layout. +// A block with a bad checksum is skipped, and what was seen before it is +// forgotten. If it were not, a second pax 'g' global header after the +// damaged block would trip "Redundant 'g' header", though a `tar` CLI +// accepts this layout. // // This test goes through the buffered extractor only: local `file:` -// tarballs are read fully into memory by PackageManagerTask.readAndExtract -// and handed to Archiver.extractToDir, which loops on readNextHeader with -// `.retry => continue`. The streaming reader is never involved, so any -// behaviour change here is the libarchive patch leaking into the shared -// buffered codepath. +// tarballs are read fully into memory. // ------------------------------------------------------------------- test("buffered extract: damaged-block retry resets header state (upstream semantics)", async () => { - // One pax 'g' extended-header payload. libarchive's header_pax_global - // just skips it, but parsing it sets `seen_headers |= seen_g_header`; - // seeing a second one without an intervening state reset is what - // triggers the "Redundant 'g' header" FATAL. + // One pax 'g' extended-header payload. It is skipped, but remembered: + // a second one before the entry they belong to is refused. const pax = Buffer.from("16 comment=test\n", "utf8"); expect(pax.length).toBe(16); const paxEntry = () => [tarHeader("pax_global_header", pax.length, "g"), pax, pad512(pax.length)]; @@ -889,10 +881,8 @@ test("buffered extract: damaged-block retry resets header state (upstream semant const { stderr, exitCode } = await runInstall(String(dir)); - // With the broken patch the second 'g' header trips - // "Redundant 'g' header" → ARCHIVE_FATAL inside libarchive; the Zig - // extract loop surfaces that as `error.Fail` → "Fail extracting - // tarball". With upstream semantics restored the damaged block is + // If the first 'g' header were remembered across the damaged block, the + // second one would surface as "Fail extracting tarball". The damaged block is // skipped, state is fully reset, and the file following the second // 'g' header is extracted normally. expect(stderr).not.toContain("Fail extracting tarball"); @@ -905,13 +895,13 @@ test("buffered extract: damaged-block retry resets header state (upstream semant // ------------------------------------------------------------------- // Buffered extract: the decompressed tar is never materialised in -// memory. libarchive gunzips on the fly, so a highly compressible .tgz +// memory. It is gunzipped on the fly, so a highly compressible .tgz // installs without an RSS spike of roughly its decompressed size. // Covers `file:` dependencies, which always take the buffered path. // ------------------------------------------------------------------- test("buffered extract does not hold the decompressed local tarball in memory", async () => { // 256 MiB of zeros: above the 64 MB gzip ISIZE cutoff that gates the - // libdeflate fast path, so libarchive (not libdeflate) decompresses, + // libdeflate fast path, so zlib (not libdeflate) decompresses, // and a multiple of 512 so the tar entry needs no trailing pad block. // The old path inflated this into a ~256 MB Vec before extraction, // which shows up directly in the child's maxRSS. @@ -975,7 +965,7 @@ test("buffered extract does not hold the decompressed local tarball in memory", // decompressed tar in memory. With the old pre-decompress path the // install's maxRSS was well over 3x PAYLOAD_SIZE (Vec growth // reallocations): ~780 MB release, ~1 GB debug+ASAN. Streaming - // through libarchive it stays at baseline (~40 MB release, ~240 MB + // through zlib it stays at baseline (~40 MB release, ~240 MB // debug+ASAN, reported as the spawner's ~320 MB floor), so the // midpoint gives wide margin both ways without needing to branch on // build type. @@ -1038,7 +1028,7 @@ test("streaming extract skips a damaged header block and extracts the entries af }); // Unlike registry tarballs, a `github:` tarball has its directory entries -// created, with the mode libarchive reports for them. A GNU base-256 mode +// created, with the mode the tar reader reports for them. A GNU base-256 mode // field can set bits far above the twelve mode bits; both extractors must // drop those instead of aborting on them. test.concurrent.each([ @@ -1552,11 +1542,11 @@ function readFiles(root: string): Record { } // Installs `tgz` as `stream-pkg`, `installs` times with one cache, and returns what each install left. -// "libdeflate" and "libarchive": a `file:` tarball, which the buffered extractor decompresses that way. +// "libdeflate" and "zlib": a `file:` tarball, which the buffered extractor decompresses that way. // "buffered" and "streaming": downloaded, from the registry or as a URL dependency, and extracted that way. // `files`: also what `readFiles` finds in the installed package. // `gzipEncoded`: the server compresses `tgz` and says so with `Content-Encoding: gzip`. -type ExtractPath = "libdeflate" | "libarchive" | "buffered" | "streaming"; +type ExtractPath = "libdeflate" | "zlib" | "buffered" | "streaming"; async function installStreamPkg( path: ExtractPath, tgz: Buffer, @@ -1605,12 +1595,12 @@ async function installStreamPkg( BUN_TMPDIR: tmp, BUN_INSTALL_STREAMING_MIN_SIZE: "1", BUN_INSTALL_STREAMING_DRAIN_THRESHOLD: "1", - ...(path === "libarchive" ? { BUN_FEATURE_FLAG_NO_LIBDEFLATE: "1" } : {}), + ...(path === "zlib" ? { BUN_FEATURE_FLAG_NO_LIBDEFLATE: "1" } : {}), ...(path === "buffered" ? { BUN_FEATURE_FLAG_DISABLE_STREAMING_INSTALL: "1" } : {}), }); results.push({ // What another thread writes to the verbose log can come anywhere in this line. - error: stderr.replace(/\[stream-pkg\] Extract .*\n(libarchive error: .*\n)?/g, "").match(/^error: .*$/m)?.[0], + error: stderr.replace(/\[stream-pkg\] Extract .*\n(archive error: .*\n)?/g, "").match(/^error: .*$/m)?.[0], streamed: stderr.includes("Streamed "), installed: existsSync(join(String(dir), "node_modules", "stream-pkg")), cached: readdirSync(join(String(dir), ".cache")).filter(name => /^(stream-pkg@|@T@)/.test(name)).length, @@ -1630,7 +1620,7 @@ async function installStreamPkg( // install, a registry without `dist.integrity`, `--no-verify`) that // trailer is the only thing that catches a damaged tarball. All three // ways to decompress one must check it: libdeflate in memory and -// libarchive's gzip filter in the buffered extractor, and that filter +// bun_tar's gzip decoder in the buffered extractor, and that decoder // again in the streaming extractor. // ------------------------------------------------------------------- describe.concurrent("gzip trailer", () => { @@ -1670,7 +1660,7 @@ describe.concurrent("gzip trailer", () => { // Too long for the attempt in memory. ["a wrong ISIZE", changed(good, good.length - 4, (b, i) => b.writeUInt32LE(100 * 1024 * 1024, i))], ]; - const paths = ["libdeflate", "libarchive", "streaming"] as const; + const paths = ["libdeflate", "zlib", "streaming"] as const; const install = async (path: (typeof paths)[number], tgz: Buffer) => (await installStreamPkg(path, tgz))[0]; test.each(paths)("a valid tarball installs (%s)", async path => { @@ -1690,10 +1680,7 @@ describe.concurrent("gzip trailer", () => { expect(await install(path, tgz)).toEqual({ error: path === "streaming" - ? // libarchive's tar reader overwrites the message of the filter that failed. - expect.stringMatching( - /^error: Fail extracting tarball for "stream-pkg": (gzip trailer mismatch|Truncated tar archive)/, - ) + ? expect.stringMatching(/^error: Fail extracting tarball for "stream-pkg": gzip trailer mismatch/) : "error: Fail extracting tarball from stream-pkg", streamed: false, installed: false, @@ -1795,6 +1782,56 @@ describe.concurrent("gzip trailer", () => { }); }); +// ------------------------------------------------------------------- +// libarchive could wait for more of a download in some of its reads only. +// A piece that ended inside one of the others failed the install. +// ------------------------------------------------------------------- +describe.concurrent("a piece of the download may end anywhere", () => { + const index = `module.exports = "GOOD";\n`; + const json = JSON.stringify({ name: "stream-pkg", version: "1.0.0" }); + const comment = Buffer.from(`1126417 comment=${Buffer.alloc(1100 * 1024, "c")}\n`); + expect(comment.length).toBe(1126417); + const tar = (...before: Buffer[]) => + Buffer.concat([ + ...tarFile("package/package.json", Buffer.from(json)), + ...before, + ...tarFile("package/index.js", Buffer.from(index)), + Buffer.alloc(1024), + ]); + const gz = gzipSync(tar(), { level: 0 }); + + test.each([ + // The first piece is 512 bytes. + [ + "in an optional field of the gzip header", + Buffer.concat([ + gz.subarray(0, 3), + Buffer.from([4]), + gz.subarray(4, 10), + Buffer.from([0xe8, 3]), + Buffer.alloc(1000), + gz.subarray(10), + ]), + ], + [ + "in a pax header of more than 1 MiB", + gzipSync(tar(tarHeader("PaxHeader", comment.length, "x"), comment, pad512(comment.length)), { level: 0 }), + ], + ])("%s", async (_, tgz) => { + expect(await installStreamPkg("streaming", tgz, { files: true })).toEqual([ + { + error: undefined, + streamed: true, + installed: true, + cached: 1, + tmp: [".keep"], + exitCode: 0, + files: { "package.json": json, "index.js": index }, + }, + ]); + }); +}); + // ------------------------------------------------------------------- // bun reads the package.json of a tarball dependency, and of a registry // package that may run scripts, as soon as it is extracted. One that diff --git a/test/cli/install/bun-pack.test.ts b/test/cli/install/bun-pack.test.ts index 81778503351c..ff9781f7543d 100644 --- a/test/cli/install/bun-pack.test.ts +++ b/test/cli/install/bun-pack.test.ts @@ -109,9 +109,8 @@ test.concurrent("basic", async () => { expect(tarballEntries(join(dir, "pack-basic-1.2.3.tgz"))).toEqual(["package/package.json", "package/index.js"]); }); -// The archive ends right after the two end-of-archive blocks. libarchive's default for a custom -// output sink would pad it to a full 10 KiB record instead, which would change the size and shasum -// of every tarball bun produces. +// The archive ends right after the two end-of-archive blocks. Padding it to a full 10 KiB record, +// as Bun.Archive does, would change the size and shasum of every tarball bun produces. test.concurrent("the archive is not padded to a full tar record", async () => { using dir = tempDir("pack-unpadded", { "package.json": JSON.stringify({ name: "pack-unpadded", version: "1.0.0" }), @@ -125,6 +124,52 @@ test.concurrent("the archive is not padded to a full tar record", async () => { expect(tar.byteLength).toBe(6 * 512); }); +// A package packed twice must give the same tarball, whichever version of bun packs it. What the +// compressor makes of the archive depends on the CPU, so that part is compared decompressed. +// The hashes are from Bun 1.4.4. Not on Windows, where files have no permission bits to record. +test.concurrent.skipIf(isWindows).each([ + [[], "1f8b08000000000002ff"], + [["--gzip-level", "0"], "1f8b08000000000000ff"], + [["--gzip-level", "1"], "1f8b08000000000004ff"], + [["--gzip-level", "6"], "1f8b08000000000000ff"], +])("the bytes of a tarball do not change %j", async (args, gzipHeader) => { + const n = (count: number) => Buffer.alloc(count, "n").toString(); + const files = { + "package.json": JSON.stringify({ name: "@scope/pack-bytes", version: "1.2.3", bin: { x: "bin/x.js" } }), + "index.js": "module.exports = 1;", + "empty.txt": "", + "bin/x.js": "#!/usr/bin/env node\n", + "run.sh": "#!/bin/sh\n", + "café.txt": "not ASCII", + [n(92) + ".js"]: "the name fits once package/ is in front of it", + [n(93) + ".js"]: "the name does not fit", + [n(200) + ".js"]: "the name does not fit and cannot be split", + [`${n(50)}/${n(50)}/${n(50)}/split.js`]: "the name fits once it is split", + [`${n(150)}/${n(100)}/no-split.js`]: "the name cannot be split", + "blocks/511": n(511), + "blocks/512": n(512), + "blocks/513": n(513), + "blocks/100000": n(100000), + }; + using dir = tempDir("pack-bytes", files); + // The archive records them, and what they start as depends on the umask. + for (const path of Object.keys(files)) await chmod(join(String(dir), path), path === "run.sh" ? 0o755 : 0o644); + + await pack(dir, bunEnv, ...args); + + const tgz = await file(join(dir, "scope-pack-bytes-1.2.3.tgz")).bytes(); + const tar = gunzipSync(tgz); + expect({ + gzipHeader: Buffer.from(tgz.subarray(0, 10)).toString("hex"), + length: tar.byteLength, + sha256: new Bun.CryptoHasher("sha256").update(tar).digest("hex"), + }).toEqual({ + gzipHeader, + length: 119296, + sha256: "f77c6cf6bbb5acff151d905cc601904310c7eebdfa00bc346b8a8c7465f19294", + }); +}); + // pack only reads package.json, so a file that is readable but not writable (a read-only // checkout, a file owned by another user) must not stop it. test.concurrent.skipIf(isWindows)("read-only package.json", async () => { @@ -657,7 +702,7 @@ describe.concurrent("flags", () => { }); // Every write(2) to /dev/full fails with ENOSPC, the same as a tarball destination on a full disk. - // "an entry": enough incompressible data that libarchive flushes a block while it writes index.js. + // "an entry": enough incompressible data that a block is flushed while it writes index.js. // "the end of the archive": small enough that the only write(2) happens when the archive is closed. test.skipIf(!isLinux).each([ ["an entry", () => `// ${randomBytes(128 * 1024).toString("base64")}`], @@ -5389,7 +5434,7 @@ test.concurrent("unicode", async () => { // names into JS strings, which maps the bytes this test is about onto U+FFFD. // Names are returned latin1-decoded (one char per stored byte) and collected // from every place the archive spells a name: the ustar header `name` field -// and, when libarchive emits one, the `path` record of the pax header. +// and, when there is one, the `path` record of the pax header. function rawTarballMembers(tgz: Uint8Array) { const tar = Buffer.from(Bun.gunzipSync(tgz)); const members: { names: Set; contents: string }[] = []; diff --git a/test/cli/install/tarball-golden/bun-install-tarball-golden.test.ts b/test/cli/install/tarball-golden/bun-install-tarball-golden.test.ts new file mode 100644 index 000000000000..cb07a81cdd5c --- /dev/null +++ b/test/cli/install/tarball-golden/bun-install-tarball-golden.test.ts @@ -0,0 +1,49 @@ +// Tarballs of every shape the tar reader knows about, well formed and damaged, with what +// `bun install` made of each one when it was recorded. See record.ts. +import { describe, expect, test } from "bun:test"; +import { bunExe, isLinux, isMacOS, isWindows } from "harness"; +import { join } from "node:path"; +import { loadCases } from "../../../js/bun/archive-golden/observe"; +import { install, type Case } from "./install"; + +const all = await loadCases(join(import.meta.dir, "cases.json.gz")); +const cases = all + .map((c, id) => ({ ...c, id })) + // Recorded on Linux. Names outside ASCII are decoded differently on Windows, and neither its file + // systems nor those of macOS take every name. + .filter(c => isLinux || ((isMacOS ? c.mac : c.win) && !/[^\x20-\x7e\n]|[:*?<>|\\]|^link /m.test(c.tree ?? ""))); +// Windows has no permission bits. +const comparable = (tree: string | null | undefined) => (isWindows ? tree?.replace(/^(\w+ ".*") \d+/gm, "$1") : tree); + +// One `bun install` per group, small enough for a debug build to finish in time. +const groups = (of: T[]) => + Array.from({ length: Math.ceil(of.length / 60) }, (_, i) => [of.slice(i * 60, i * 60 + 60)]); +const accepted = cases.filter(c => c.tree !== null); +const refused = cases.filter(c => c.tree === null); + +test("there are cases for this platform", () => { + expect(accepted.length).toBeGreaterThan(50); + expect(refused.length).toBeGreaterThan(20); +}); + +describe.each([ + ["while they download", true], + ["from memory", false], +])("tarballs extracted %s", (_, streaming) => { + test.each(groups(accepted))("are installed the way it was recorded", async group => { + const { trees, exitCode } = await install(bunExe(), group, streaming); + const different = group + .filter(c => comparable(trees.get(c.id)) !== comparable(c.tree)) + .map(c => `case ${c.id}\n--- expected\n${c.tree}--- actual\n${trees.get(c.id)}`); + expect(different.slice(0, 5).join("\n")).toBe(""); + expect(exitCode).toBe(0); + }); + + test.each(groups(refused))("are refused if they were when it was recorded", async group => { + const { trees, exitCode, stderr } = await install(bunExe(), group, streaming); + expect(group.filter(c => trees.get(c.id) !== null).map(c => c.id)).toEqual([]); + // Only the streaming extractor says how far into the download it got. + expect(stderr.includes("(at byte ")).toBe(streaming); + expect(exitCode).toBe(1); + }); +}); diff --git a/test/cli/install/tarball-golden/cases.json.gz b/test/cli/install/tarball-golden/cases.json.gz new file mode 100644 index 000000000000..b9cc0f4ce97d Binary files /dev/null and b/test/cli/install/tarball-golden/cases.json.gz differ diff --git a/test/cli/install/tarball-golden/install.ts b/test/cli/install/tarball-golden/install.ts new file mode 100644 index 000000000000..4be507aeb173 --- /dev/null +++ b/test/cli/install/tarball-golden/install.ts @@ -0,0 +1,106 @@ +// Installs many tarballs at once with `bun install` and says what became of each. +import { bunEnv, tempDir } from "harness"; +import { createServer, type ServerResponse } from "node:http"; +import { once } from "node:events"; +import { existsSync, readdirSync } from "node:fs"; +import { join } from "node:path"; +import { tarHeader, tarPadded, tree } from "../../../js/bun/archive-golden/observe"; + +export type Case = { + /** What follows `package/package.json` in the tarball, base64. */ + data: string; + /** Whether macOS and Windows read it the way Linux does. */ + mac: boolean; + win: boolean; + /** The installed package, or null if the tarball is refused. */ + tree: string | null; +}; + +function tarball(name: string, rest: Buffer) { + const json = Buffer.from(JSON.stringify({ name, version: "1.0.0" })); + return Buffer.from( + Bun.gzipSync(Buffer.concat([tarHeader("package/package.json", json.length), tarPadded(json), rest])), + ); +} + +/** + * `streaming` is whether tarballs are extracted while they download. They are sent in pieces, since + * one that arrives whole is extracted from memory either way. + */ +export async function install(exe: string, cases: { id: number; data: string }[], streaming: boolean) { + const tarballs = new Map(cases.map(c => [`/c${c.id}.tgz`, tarball(`c${c.id}`, Buffer.from(c.data, "base64"))])); + // A tarball is streamed only if its body arrives in more than one piece. The extractor makes its + // directory when it gets the first one, so the rest waits until that exists or bun has hung up. + let tmp = ""; + const waiting = new Map void>(); + async function extractorStarted(name: string, res: ServerResponse) { + const { promise, resolve } = Promise.withResolvers(); + res.once("close", resolve); + waiting.set(name, resolve); + if (waiting.size === 1) { + (async () => { + for (; waiting.size > 0; await Bun.sleep(1)) { + for (const dir of readdirSync(tmp)) waiting.get(dir.slice(dir.lastIndexOf(".") + 1))?.(); + } + })(); + } + await promise; + waiting.delete(name); + } + const server = createServer((req, res) => { + const body = tarballs.get(req.url!)!; + res.writeHead(200, { "content-length": body.length, "content-type": "application/octet-stream" }); + const step = Math.ceil(body.length / 4); + let at = 0; + (function send() { + if (at >= body.length) return void res.end(); + const first = at === 0; + res.write(body.subarray(at, (at += step)), async () => { + if (first && streaming) await extractorStarted(req.url!.slice(1, -".tgz".length), res); + send(); + }); + })(); + }).listen(0, "127.0.0.1"); + await once(server, "listening"); + using _ = { [Symbol.dispose]: () => void server.close() }; + const { port } = server.address() as import("node:net").AddressInfo; + + using dir = tempDir("tarball-golden", { + "package.json": JSON.stringify({ + name: "root", + dependencies: Object.fromEntries(cases.map(c => [`c${c.id}`, `http://127.0.0.1:${port}/c${c.id}.tgz`])), + }), + "tmp/.keep": "", + }); + tmp = join(String(dir), "tmp"); + await using proc = Bun.spawn({ + cmd: [exe, "install", "--ignore-scripts"], + cwd: String(dir), + env: { + ...bunEnv, + BUN_INSTALL_CACHE_DIR: join(String(dir), ".cache"), + BUN_TMPDIR: tmp, + ...(streaming + ? { BUN_INSTALL_STREAMING_MIN_SIZE: "0", BUN_INSTALL_STREAMING_DRAIN_THRESHOLD: "1" } + : { BUN_FEATURE_FLAG_DISABLE_STREAMING_INSTALL: "1" }), + }, + stdout: "ignore", + stderr: "pipe", + }); + const [stderr, exitCode] = await Promise.all([proc.stderr.text(), proc.exited]); + + const refused = new Set( + Array.from(stderr.matchAll(/extracting tarball (?:for|from) "?c(\d+)\b/g), m => Number(m[1])), + ); + const trees = new Map(); + for (const { id } of cases) { + const path = join(String(dir), "node_modules", `c${id}`); + const lines: string[] = []; + if (existsSync(path)) tree(Buffer.from(path), Buffer.alloc(0), lines); + // `.bun-tag` holds a hash of the compressed tarball, which depends on the compressor. How many + // links a file has depends on how the platform copies it out of the cache. + const shown = lines.filter(line => !line.startsWith('file "/.bun-tag"')).join("\n"); + trees.set(id, refused.has(id) ? null : shown.replace(/ nlink=\d+/g, "") + "\n"); + } + return { trees, exitCode, stderr }; +} diff --git a/test/cli/install/tarball-golden/record.ts b/test/cli/install/tarball-golden/record.ts new file mode 100644 index 000000000000..df8e59d7dc24 --- /dev/null +++ b/test/cli/install/tarball-golden/record.ts @@ -0,0 +1,28 @@ +// Fills in the expected results of cases.json.gz from the Bun that runs this script: +// +// bun record.ts [--accept] [new-cases.json] +// +// The results in the repository come from the last Bun that read tarballs with libarchive (1.4.4). +import { join } from "node:path"; +import { loadCases, recordArguments } from "../../../js/bun/archive-golden/observe"; +import { install, type Case } from "./install"; + +const fixture = join(import.meta.dir, "cases.json.gz"); +const [newCases] = recordArguments(); +const cases: Case[] = newCases ? await Bun.file(newCases).json() : await loadCases(fixture); +const numbered = cases.map((c, id) => ({ ...c, id })); + +// Nothing is installed when one tarball is refused, so those are found first. +const first = await install(process.execPath, numbered, false); +const accepted = numbered.filter(c => first.trees.get(c.id) !== null); +const second = await install(process.execPath, accepted, false); +if (second.exitCode !== 0) throw new Error(second.stderr); + +const recorded: Case[] = numbered.map(({ id, data, mac, win }) => ({ + data, + mac, + win, + tree: second.trees.get(id) ?? null, +})); +await Bun.write(fixture, Bun.gzipSync(Buffer.from(JSON.stringify(recorded)), { level: 9 })); +console.log(`recorded ${recorded.length} cases, ${accepted.length} of them accepted, with Bun ${Bun.version}`); diff --git a/test/integration/bun-types/bun-types.test.ts b/test/integration/bun-types/bun-types.test.ts index ab79e2b9218f..0bba8099b085 100644 --- a/test/integration/bun-types/bun-types.test.ts +++ b/test/integration/bun-types/bun-types.test.ts @@ -310,6 +310,7 @@ function checkForEmptyInterfaces(program: ts.Program) { */ const PUBLISHED_AS: Record = { "Archive.bind.ts": "typeof Bun", + "ArchiveInternals.bind.ts": null, "Blob.bind.ts": "typeof Bun & { Blob: Class }", "BlockList.bind.ts": null, "BunObject.bind.ts": diff --git a/test/js/bun/archive-golden/__snapshots__/archive-golden.test.ts.snap b/test/js/bun/archive-golden/__snapshots__/archive-golden.test.ts.snap new file mode 100644 index 000000000000..56dfddb076f6 --- /dev/null +++ b/test/js/bun/archive-golden/__snapshots__/archive-golden.test.ts.snap @@ -0,0 +1,270 @@ +// Bun Snapshot v1, https://bun.sh/docs/test/snapshots + +exports[`Bun.Archive writes archives the way it was recorded to nothing 1`] = ` +"10240 bytes +crc=271dde9a +" +`; + +exports[`Bun.Archive writes archives the way it was recorded to one file 1`] = ` +"10240 bytes +type=0 name=612e747874 prefix= mode=000644 size=1 checksum=ok mtime=now +crc=b662bc81 +" +`; + +exports[`Bun.Archive writes archives the way it was recorded to empty file 1`] = ` +"10240 bytes +type=0 name=61 prefix= mode=000644 size=0 checksum=ok mtime=now +crc=94772c35 +" +`; + +exports[`Bun.Archive writes archives the way it was recorded to block sizes 1`] = ` +"30720 bytes +type=0 name=353131 prefix= mode=000644 size=511 checksum=ok mtime=now +type=0 name=353132 prefix= mode=000644 size=512 checksum=ok mtime=now +type=0 name=353133 prefix= mode=000644 size=513 checksum=ok mtime=now +type=0 name=39393939 prefix= mode=000644 size=9999 checksum=ok mtime=now +type=0 name=3130323430 prefix= mode=000644 size=10240 checksum=ok mtime=now +crc=cf34ffcf +" +`; + +exports[`Bun.Archive writes archives the way it was recorded to directories in names 1`] = ` +"10240 bytes +type=0 name=612f622f632e747874 prefix= mode=000644 size=1 checksum=ok mtime=now +type=0 name=612f prefix= mode=000644 size=0 checksum=ok mtime=now +type=0 name=2e2f64 prefix= mode=000644 size=1 checksum=ok mtime=now +type=0 name=2f616273 prefix= mode=000644 size=1 checksum=ok mtime=now +type=0 name=2e2e2f7570 prefix= mode=000644 size=1 checksum=ok mtime=now +type=0 name=612f2f62 prefix= mode=000644 size=1 checksum=ok mtime=now +crc=a73012f2 +" +`; + +exports[`Bun.Archive writes archives the way it was recorded to name of 99 1`] = ` +"10240 bytes +type=0 name=6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e prefix= mode=000644 size=1 checksum=ok mtime=now +crc=5d60b2af +" +`; + +exports[`Bun.Archive writes archives the way it was recorded to name of 100 1`] = ` +"10240 bytes +type=0 name=6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e prefix= mode=000644 size=1 checksum=ok mtime=now +crc=d3ddafe6 +" +`; + +exports[`Bun.Archive writes archives the way it was recorded to name of 101 1`] = ` +"10240 bytes +type=x name=5061784865616465722f6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e prefix= mode=000644 size=111 checksum=ok mtime=now + pax=31313120706174683d6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e0a +type=0 name=6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e prefix= mode=000644 size=1 checksum=ok mtime=now +crc=d0141907 +" +`; + +exports[`Bun.Archive writes archives the way it was recorded to name of 300 1`] = ` +"10240 bytes +type=x name=5061784865616465722f6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e prefix= mode=000644 size=310 checksum=ok mtime=now + pax=33313020706174683d6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e0a +type=0 name=6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e prefix= mode=000644 size=1 checksum=ok mtime=now +crc=46e84fe4 +" +`; + +exports[`Bun.Archive writes archives the way it was recorded to prefix of 1 1`] = ` +"10240 bytes +type=0 name=6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e prefix=64 mode=000644 size=1 checksum=ok mtime=now +crc=414067fc +" +`; + +exports[`Bun.Archive writes archives the way it was recorded to prefix of 154 1`] = ` +"10240 bytes +type=0 name=6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e prefix=6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e mode=000644 size=1 checksum=ok mtime=now +crc=7fd5468b +" +`; + +exports[`Bun.Archive writes archives the way it was recorded to prefix of 155 1`] = ` +"10240 bytes +type=0 name=6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e prefix=6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e mode=000644 size=1 checksum=ok mtime=now +crc=b2ef0992 +" +`; + +exports[`Bun.Archive writes archives the way it was recorded to prefix of 156 1`] = ` +"10240 bytes +type=x name=5061784865616465722f6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e prefix= mode=000644 size=267 checksum=ok mtime=now + pax=32363720706174683d6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e2f6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e0a +type=0 name=6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e prefix= mode=000644 size=1 checksum=ok mtime=now +crc=9d8e1670 +" +`; + +exports[`Bun.Archive writes archives the way it was recorded to prefix then name of 101 1`] = ` +"10240 bytes +type=x name=642f5061784865616465722f6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e prefix= mode=000644 size=113 checksum=ok mtime=now + pax=31313320706174683d642f6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e0a +type=0 name=642f6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e prefix= mode=000644 size=1 checksum=ok mtime=now +crc=f142a1f9 +" +`; + +exports[`Bun.Archive writes archives the way it was recorded to many parts 1`] = ` +"10240 bytes +type=0 name=6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f66 prefix=6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f6469722f646972 mode=000644 size=1 checksum=ok mtime=now +crc=c9c98458 +" +`; + +exports[`Bun.Archive writes archives the way it was recorded to many long parts 1`] = ` +"10240 bytes +type=x name=5061784865616465722f6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e prefix=6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e mode=000644 size=737 checksum=ok mtime=now + pax=37333720706174683d6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e2f6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e2f6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e2f6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e2f6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e2f6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e2f6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e2f6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e0a +type=0 name=6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e prefix=6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e mode=000644 size=1 checksum=ok mtime=now +crc=e4ae53b1 +" +`; + +exports[`Bun.Archive writes archives the way it was recorded to long with a trailing slash 1`] = ` +"10240 bytes +type=x name=5061784865616465722f6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e prefix= mode=000644 size=131 checksum=ok mtime=now + pax=31333120706174683d6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e2f0a +type=0 name=6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e2f prefix= mode=000644 size=0 checksum=ok mtime=now +crc=da38570e +" +`; + +exports[`Bun.Archive writes archives the way it was recorded to long name ending in a dot 1`] = ` +"10240 bytes +type=0 name=2e prefix=642f6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e mode=000644 size=1 checksum=ok mtime=now +crc=db6cf95c +" +`; + +exports[`Bun.Archive writes archives the way it was recorded to control characters 1`] = ` +"10240 bytes +type=0 name=610a62 prefix= mode=000644 size=1 checksum=ok mtime=now +type=0 name=610962 prefix= mode=000644 size=1 checksum=ok mtime=now +type=0 name=613d62 prefix= mode=000644 size=1 checksum=ok mtime=now +type=0 name=20 prefix= mode=000644 size=1 checksum=ok mtime=now +crc=e24de551 +" +`; + +exports[`Bun.Archive writes archives the way it was recorded to binary data 1`] = ` +"10240 bytes +type=0 name=62696e prefix= mode=000644 size=1000 checksum=ok mtime=now +crc=98aa872e +" +`; + +exports[`Bun.Archive writes archives the way it was recorded to a hundred files 1`] = ` +"153600 bytes +type=0 name=646972302f66696c6530 prefix= mode=000644 size=0 checksum=ok mtime=now +type=0 name=646972312f66696c6531 prefix= mode=000644 size=13 checksum=ok mtime=now +type=0 name=646972322f66696c6532 prefix= mode=000644 size=26 checksum=ok mtime=now +type=0 name=646972332f66696c6533 prefix= mode=000644 size=39 checksum=ok mtime=now +type=0 name=646972342f66696c6534 prefix= mode=000644 size=52 checksum=ok mtime=now +type=0 name=646972352f66696c6535 prefix= mode=000644 size=65 checksum=ok mtime=now +type=0 name=646972362f66696c6536 prefix= mode=000644 size=78 checksum=ok mtime=now +type=0 name=646972302f66696c6537 prefix= mode=000644 size=91 checksum=ok mtime=now +type=0 name=646972312f66696c6538 prefix= mode=000644 size=104 checksum=ok mtime=now +type=0 name=646972322f66696c6539 prefix= mode=000644 size=117 checksum=ok mtime=now +type=0 name=646972332f66696c653130 prefix= mode=000644 size=130 checksum=ok mtime=now +type=0 name=646972342f66696c653131 prefix= mode=000644 size=143 checksum=ok mtime=now +type=0 name=646972352f66696c653132 prefix= mode=000644 size=156 checksum=ok mtime=now +type=0 name=646972362f66696c653133 prefix= mode=000644 size=169 checksum=ok mtime=now +type=0 name=646972302f66696c653134 prefix= mode=000644 size=182 checksum=ok mtime=now +type=0 name=646972312f66696c653135 prefix= mode=000644 size=195 checksum=ok mtime=now +type=0 name=646972322f66696c653136 prefix= mode=000644 size=208 checksum=ok mtime=now +type=0 name=646972332f66696c653137 prefix= mode=000644 size=221 checksum=ok mtime=now +type=0 name=646972342f66696c653138 prefix= mode=000644 size=234 checksum=ok mtime=now +type=0 name=646972352f66696c653139 prefix= mode=000644 size=247 checksum=ok mtime=now +type=0 name=646972362f66696c653230 prefix= mode=000644 size=260 checksum=ok mtime=now +type=0 name=646972302f66696c653231 prefix= mode=000644 size=273 checksum=ok mtime=now +type=0 name=646972312f66696c653232 prefix= mode=000644 size=286 checksum=ok mtime=now +type=0 name=646972322f66696c653233 prefix= mode=000644 size=299 checksum=ok mtime=now +type=0 name=646972332f66696c653234 prefix= mode=000644 size=312 checksum=ok mtime=now +type=0 name=646972342f66696c653235 prefix= mode=000644 size=325 checksum=ok mtime=now +type=0 name=646972352f66696c653236 prefix= mode=000644 size=338 checksum=ok mtime=now +type=0 name=646972362f66696c653237 prefix= mode=000644 size=351 checksum=ok mtime=now +type=0 name=646972302f66696c653238 prefix= mode=000644 size=364 checksum=ok mtime=now +type=0 name=646972312f66696c653239 prefix= mode=000644 size=377 checksum=ok mtime=now +type=0 name=646972322f66696c653330 prefix= mode=000644 size=390 checksum=ok mtime=now +type=0 name=646972332f66696c653331 prefix= mode=000644 size=403 checksum=ok mtime=now +type=0 name=646972342f66696c653332 prefix= mode=000644 size=416 checksum=ok mtime=now +type=0 name=646972352f66696c653333 prefix= mode=000644 size=429 checksum=ok mtime=now +type=0 name=646972362f66696c653334 prefix= mode=000644 size=442 checksum=ok mtime=now +type=0 name=646972302f66696c653335 prefix= mode=000644 size=455 checksum=ok mtime=now +type=0 name=646972312f66696c653336 prefix= mode=000644 size=468 checksum=ok mtime=now +type=0 name=646972322f66696c653337 prefix= mode=000644 size=481 checksum=ok mtime=now +type=0 name=646972332f66696c653338 prefix= mode=000644 size=494 checksum=ok mtime=now +type=0 name=646972342f66696c653339 prefix= mode=000644 size=507 checksum=ok mtime=now +type=0 name=646972352f66696c653430 prefix= mode=000644 size=520 checksum=ok mtime=now +type=0 name=646972362f66696c653431 prefix= mode=000644 size=533 checksum=ok mtime=now +type=0 name=646972302f66696c653432 prefix= mode=000644 size=546 checksum=ok mtime=now +type=0 name=646972312f66696c653433 prefix= mode=000644 size=559 checksum=ok mtime=now +type=0 name=646972322f66696c653434 prefix= mode=000644 size=572 checksum=ok mtime=now +type=0 name=646972332f66696c653435 prefix= mode=000644 size=585 checksum=ok mtime=now +type=0 name=646972342f66696c653436 prefix= mode=000644 size=598 checksum=ok mtime=now +type=0 name=646972352f66696c653437 prefix= mode=000644 size=611 checksum=ok mtime=now +type=0 name=646972362f66696c653438 prefix= mode=000644 size=624 checksum=ok mtime=now +type=0 name=646972302f66696c653439 prefix= mode=000644 size=637 checksum=ok mtime=now +type=0 name=646972312f66696c653530 prefix= mode=000644 size=650 checksum=ok mtime=now +type=0 name=646972322f66696c653531 prefix= mode=000644 size=663 checksum=ok mtime=now +type=0 name=646972332f66696c653532 prefix= mode=000644 size=676 checksum=ok mtime=now +type=0 name=646972342f66696c653533 prefix= mode=000644 size=689 checksum=ok mtime=now +type=0 name=646972352f66696c653534 prefix= mode=000644 size=702 checksum=ok mtime=now +type=0 name=646972362f66696c653535 prefix= mode=000644 size=715 checksum=ok mtime=now +type=0 name=646972302f66696c653536 prefix= mode=000644 size=728 checksum=ok mtime=now +type=0 name=646972312f66696c653537 prefix= mode=000644 size=741 checksum=ok mtime=now +type=0 name=646972322f66696c653538 prefix= mode=000644 size=754 checksum=ok mtime=now +type=0 name=646972332f66696c653539 prefix= mode=000644 size=767 checksum=ok mtime=now +type=0 name=646972342f66696c653630 prefix= mode=000644 size=780 checksum=ok mtime=now +type=0 name=646972352f66696c653631 prefix= mode=000644 size=793 checksum=ok mtime=now +type=0 name=646972362f66696c653632 prefix= mode=000644 size=806 checksum=ok mtime=now +type=0 name=646972302f66696c653633 prefix= mode=000644 size=819 checksum=ok mtime=now +type=0 name=646972312f66696c653634 prefix= mode=000644 size=832 checksum=ok mtime=now +type=0 name=646972322f66696c653635 prefix= mode=000644 size=845 checksum=ok mtime=now +type=0 name=646972332f66696c653636 prefix= mode=000644 size=858 checksum=ok mtime=now +type=0 name=646972342f66696c653637 prefix= mode=000644 size=871 checksum=ok mtime=now +type=0 name=646972352f66696c653638 prefix= mode=000644 size=884 checksum=ok mtime=now +type=0 name=646972362f66696c653639 prefix= mode=000644 size=897 checksum=ok mtime=now +type=0 name=646972302f66696c653730 prefix= mode=000644 size=910 checksum=ok mtime=now +type=0 name=646972312f66696c653731 prefix= mode=000644 size=923 checksum=ok mtime=now +type=0 name=646972322f66696c653732 prefix= mode=000644 size=936 checksum=ok mtime=now +type=0 name=646972332f66696c653733 prefix= mode=000644 size=949 checksum=ok mtime=now +type=0 name=646972342f66696c653734 prefix= mode=000644 size=962 checksum=ok mtime=now +type=0 name=646972352f66696c653735 prefix= mode=000644 size=975 checksum=ok mtime=now +type=0 name=646972362f66696c653736 prefix= mode=000644 size=988 checksum=ok mtime=now +type=0 name=646972302f66696c653737 prefix= mode=000644 size=1001 checksum=ok mtime=now +type=0 name=646972312f66696c653738 prefix= mode=000644 size=1014 checksum=ok mtime=now +type=0 name=646972322f66696c653739 prefix= mode=000644 size=1027 checksum=ok mtime=now +type=0 name=646972332f66696c653830 prefix= mode=000644 size=1040 checksum=ok mtime=now +type=0 name=646972342f66696c653831 prefix= mode=000644 size=1053 checksum=ok mtime=now +type=0 name=646972352f66696c653832 prefix= mode=000644 size=1066 checksum=ok mtime=now +type=0 name=646972362f66696c653833 prefix= mode=000644 size=1079 checksum=ok mtime=now +type=0 name=646972302f66696c653834 prefix= mode=000644 size=1092 checksum=ok mtime=now +type=0 name=646972312f66696c653835 prefix= mode=000644 size=1105 checksum=ok mtime=now +type=0 name=646972322f66696c653836 prefix= mode=000644 size=1118 checksum=ok mtime=now +type=0 name=646972332f66696c653837 prefix= mode=000644 size=1131 checksum=ok mtime=now +type=0 name=646972342f66696c653838 prefix= mode=000644 size=1144 checksum=ok mtime=now +type=0 name=646972352f66696c653839 prefix= mode=000644 size=1157 checksum=ok mtime=now +type=0 name=646972362f66696c653930 prefix= mode=000644 size=1170 checksum=ok mtime=now +type=0 name=646972302f66696c653931 prefix= mode=000644 size=1183 checksum=ok mtime=now +type=0 name=646972312f66696c653932 prefix= mode=000644 size=1196 checksum=ok mtime=now +type=0 name=646972322f66696c653933 prefix= mode=000644 size=1209 checksum=ok mtime=now +type=0 name=646972332f66696c653934 prefix= mode=000644 size=1222 checksum=ok mtime=now +type=0 name=646972342f66696c653935 prefix= mode=000644 size=1235 checksum=ok mtime=now +type=0 name=646972352f66696c653936 prefix= mode=000644 size=1248 checksum=ok mtime=now +type=0 name=646972362f66696c653937 prefix= mode=000644 size=1261 checksum=ok mtime=now +type=0 name=646972302f66696c653938 prefix= mode=000644 size=1274 checksum=ok mtime=now +type=0 name=646972312f66696c653939 prefix= mode=000644 size=1287 checksum=ok mtime=now +crc=fac35742 +" +`; diff --git a/test/js/bun/archive-golden/archive-golden.test.ts b/test/js/bun/archive-golden/archive-golden.test.ts new file mode 100644 index 000000000000..f187a0b3cb77 --- /dev/null +++ b/test/js/bun/archive-golden/archive-golden.test.ts @@ -0,0 +1,267 @@ +// Archives of every shape the tar reader knows about, well formed and damaged, with what Bun.Archive +// made of each one when it was recorded. See record.ts. +import { describe, expect, test } from "bun:test"; +import { isLinux, isMacOS, isPosix, isWindows, tempDir } from "harness"; +import { readdirSync } from "node:fs"; +import { join } from "node:path"; +import { + differences, + loadCases, + observeExtract, + observeFiles, + observeTar, + paxHeader, + paxRecord, + tarHeader, + tarPadded, + type Case, +} from "./observe"; + +const all = await loadCases(join(import.meta.dir, "cases.json.gz")); +const cases = all + .map((c, index) => ({ ...c, index, archive: Buffer.from(c.data, "base64") })) + // Recorded on Linux. Names outside ASCII are decoded differently on Windows, and macOS file + // systems do not take all of them. + .filter(c => isLinux || ((isMacOS ? c.mac : c.win) && !/[^\x20-\x7e\n]/.test(c.files + c.extract))); + +// What a refused archive leaves behind depends on which error a system call fails with, and that +// differs between systems: opening "dir/" for writing is EISDIR on Linux and ENOENT on macOS. +const comparable = (extracted: string) => (!isLinux && extracted.startsWith("rejected\n") ? "rejected\n" : extracted); + +describe("Bun.Archive reads archives the way it was recorded to", () => { + test("files()", async () => { + expect(cases.length).toBeGreaterThan(500); + const results = await Promise.all( + cases.map(async c => ({ index: c.index, expected: c.files, actual: await observeFiles(c.archive) })), + ); + expect(differences(results)).toBe(""); + }); + + // The recorded trees have permission bits, links and names that Windows does not. + test.skipIf(!isPosix)("extract()", async () => { + using dir = tempDir("archive-golden", {}); + const umask = process.umask(0o022); + try { + const results = await Promise.all( + cases.map(async c => ({ + index: c.index, + expected: comparable(c.extract), + actual: comparable(await observeExtract(c.archive, join(String(dir), String(c.index)))), + })), + ); + expect(differences(results)).toBe(""); + } finally { + process.umask(umask); + } + }); +}); + +const file = (name: Buffer | string, body = "x") => + Buffer.concat([tarHeader(name, body.length), tarPadded(Buffer.from(body))]); +const pax = (path: Buffer, before: Buffer[] = []) => + Buffer.concat([paxHeader(...before, paxRecord("path", path)), file("short")]); +function gnuLongName(path: Buffer) { + const body = Buffer.concat([path, Buffer.from([0])]); + return Buffer.concat([tarHeader("././@LongLink", body.length, "L"), tarPadded(body), file("short")]); +} +const end = Buffer.alloc(1024); +const units = (s: string) => + Array.from({ length: s.length }, (_, i) => s.charCodeAt(i).toString(16).padStart(4, "0")).join(" "); +const names = async (...parts: Buffer[]) => [...(await new Bun.Archive(Buffer.concat(parts)).files()).keys()]; + +// Measured with Bun 1.4.4 on Linux and macOS and Bun 1.4.3 on Windows. +describe("entry names", () => { + test.each([ + // name, POSIX, Windows unless it is a pax path, Windows pax path + ["63c3a9", "0063 00e9", "0063 00c3 00a9", "0063 00e9"], + ["6365cc81", "0063 0065 0301", "0063 0065 00cc 0081", "0063 00e9"], + ["63e284ab", "0063 212b", "0063 00e2 0084 00ab", "0063 212b"], + ["63e18492e185a1", "0063 1112 1161", "0063 00e1 0084 0092 00e1 0085 00a1", "0063 d558"], + ["63efac81", "0063 fb01", "0063 00ef 00ac 0081", "0063 fb01"], + ["63f09f9880", "0063 d83d de00", "0063 00f0 009f 0098 0080", "0063 d83d de00"], + ["63fffe", "0063 fffd fffd", "0063 00ff 00fe", "0063 00ff 00fe"], + ["63c3a9ff", "0063 00e9 fffd", "0063 00c3 00a9 00ff", "0063 00c3 00a9 00ff"], + ["63eda080", "0063 fffd fffd fffd", "0063 00ed 00a0 0080", "0063 00ed 00a0 0080"], + ["645c65", "0064 005c 0065", "0064 005c 0065", "0064 005c 0065"], + ["643a65", "0064 003a 0065", "0064 003a 0065", "0064 003a 0065"], + ])("%s", async (hex, posix, windowsBytes, windowsPax) => { + const name = Buffer.from(hex, "hex"); + const read = async (tar: Buffer) => (await names(tar, end)).map(units); + expect({ + ustar: await read(file(name)), + pax: await read(pax(name)), + binary: await read(pax(name, [paxRecord("hdrcharset", "BINARY")])), + gnu: await read(gnuLongName(name)), + }).toEqual({ + ustar: [isWindows ? windowsBytes : posix], + pax: [isWindows ? windowsPax : posix], + binary: [isWindows ? windowsBytes : posix], + gnu: [isWindows ? windowsBytes : posix], + }); + }); + + test("a ._ file belongs to the entry after it on macOS only", async () => { + expect(await names(file("._a"), file("a"), end)).toEqual(isMacOS ? ["a"] : ["._a", "a"]); + const nothingAfter = /end-of-archive within a sequence of headers/; + for (const tar of [[file("._a"), end], [file("._a")], [file("a"), file("._z"), end]]) { + if (isMacOS) await expect(names(...tar)).rejects.toThrow(nothingAfter); + else expect((await names(...tar)).at(-1)).toStartWith("._"); + } + }); +}); + +describe("header checksum", () => { + // The linkname field holds the bytes over 127, which a regular file makes no use of. + const withChecksum = (sum: (bytes: Buffer) => number) => { + const header = tarHeader("a", 0); + header.fill(0xe9, 157, 200); + header.fill(" ", 148, 156); + header.write(sum(header).toString(8).padStart(6, "0") + "\0 ", 148); + return header; + }; + const unsigned = (bytes: Buffer) => bytes.reduce((a, b) => a + b, 0); + const signed = (bytes: Buffer) => new Int8Array(bytes).reduce((a, b) => a + b, 0); + + test.each([ + ["of unsigned bytes", unsigned, 0, ["first", "a", "last"]], + ["of signed bytes, as some old tars wrote it", signed, 0, ["first", "a", "last"]], + ["that is off by one", unsigned, 1, ["first", "last"]], + ["of signed bytes that is off by one", signed, -1, ["first", "last"]], + ] as const)("%s", async (_, sum, off, expected) => { + const header = withChecksum(bytes => sum(bytes) + off); + expect(await names(file("first"), header, file("last"), end)).toEqual([...expected]); + }); +}); + +describe("Bun.Archive writes archives the way it was recorded to", () => { + const write = async (files: Record, options?: Bun.ArchiveOptions) => { + const started = Math.floor(Date.now() / 1000); + const bytes = await new Bun.Archive(files, options).bytes(); + return observeTar(options?.compress ? Bun.gunzipSync(bytes) : bytes, started); + }; + const n = (count: number) => Buffer.alloc(count, "n").toString(); + + // The snapshots come from the same Bun as cases.json.gz. + test.each([ + ["nothing", {}], + ["one file", { "a.txt": "x" }], + ["empty file", { "a": "" }], + ["block sizes", { "511": n(511), "512": n(512), "513": n(513), "9999": n(9999), "10240": n(10240) }], + ["directories in names", { "a/b/c.txt": "x", "a/": "", "./d": "x", "/abs": "x", "../up": "x", "a//b": "x" }], + ["name of 99", { [n(99)]: "x" }], + ["name of 100", { [n(100)]: "x" }], + ["name of 101", { [n(101)]: "x" }], + ["name of 300", { [n(300)]: "x" }], + ["prefix of 1", { ["d/" + n(100)]: "x" }], + ["prefix of 154", { [n(154) + "/" + n(100)]: "x" }], + ["prefix of 155", { [n(155) + "/" + n(100)]: "x" }], + ["prefix of 156", { [n(156) + "/" + n(100)]: "x" }], + ["prefix then name of 101", { ["d/" + n(101)]: "x" }], + ["many parts", { [Array(60).fill("dir").join("/") + "/f"]: "x" }], + ["many long parts", { [Array(8).fill(n(90)).join("/")]: "x" }], + ["long with a trailing slash", { [n(120) + "/"]: "" }], + ["long name ending in a dot", { ["d/" + n(120) + "/."]: "x" }], + ["control characters", { "a\nb": "x", "a\tb": "x", "a=b": "x", " ": "x" }], + ["binary data", { "bin": new Uint8Array(1000).map((_, i) => i * 7) }], + ["a hundred files", Object.fromEntries(Array.from({ length: 100 }, (_, i) => [`dir${i % 7}/file${i}`, n(i * 13)]))], + ] as [string, Record][])("%s", async (_, files) => { + const plain = await write(files); + expect(plain).toMatchSnapshot(); + expect(await write(files, { compress: "gzip" })).toBe(plain); + }); + + test.each([ + [ + "cé", + "323120686472636861727365743d42494e4152590a313220706174683d63c3a90a", + "63c3a9", + "313220706174683d63c3a90a", + "63c3a9", + ], + [ + "cé", + "323120686472636861727365743d42494e4152590a313320706174683d6365cc810a", + "6365cc81", + "313220706174683d63c3a90a", + "63c3a9", + ], + [ + "a\ud800b", + "323120686472636861727365743d42494e4152590a313420706174683d61efbfbd620a", + "61efbfbd62", + "313420706174683d61efbfbd620a", + "61efbfbd62", + ], + ])("name outside ASCII %j", async (name, posixPax, posixName, windowsPax, windowsName) => { + const [paxRecords, stored] = isWindows ? [windowsPax, windowsName] : [posixPax, posixName]; + const lines = (await write({ [name]: "x" })).split("\n"); + expect(lines.slice(0, 4)).toEqual([ + "10240 bytes", + `type=x name=${Buffer.from("PaxHeader/").toString("hex")}${stored} prefix= mode=000644 size=${paxRecords.length / 2} checksum=ok mtime=now`, + ` pax=${paxRecords}`, + `type=0 name=${stored} prefix= mode=000644 size=1 checksum=ok mtime=now`, + ]); + }); + + test("a backslash is a separator on Windows only", async () => { + expect((await write({ "d\\e": "x" })).split("\n")[1]).toBe( + `type=0 name=${isWindows ? "642f65" : "645c65"} prefix= mode=000644 size=1 checksum=ok mtime=now`, + ); + }); +}); + +// Measured with Bun 1.4.4. +describe("gzip inside gzip", () => { + function gzip(data: Uint8Array, { name = 0, badCrc = false } = {}) { + const trailer = Buffer.alloc(8); + trailer.writeUInt32LE((Bun.hash.crc32(data) ^ Number(badCrc)) >>> 0, 0); + trailer.writeUInt32LE(data.length >>> 0, 4); + return Buffer.concat([ + Buffer.from([0x1f, 0x8b, 8, name ? 8 : 0, 0, 0, 0, 0, 0, 255]), + name ? Buffer.alloc(name + 1, "n").fill(0, name) : Buffer.alloc(0), + Bun.deflateSync(data), + trailer, + ]); + } + const tar = Buffer.concat([file("a"), end]); + const incompressible = (length: number) => { + let state = 1; + return Buffer.from(Array.from({ length }, () => (state = (state * 48271) % 2147483647) & 0xff)); + }; + + test("is unwrapped 24 times and no more", async () => { + let wrapped = tar; + for (let i = 0; i < 24; i++) wrapped = gzip(wrapped); + expect(await names(wrapped)).toEqual(["a"]); + await expect(names(gzip(wrapped))).rejects.toThrow("Input requires too many filters for decoding"); + }); + + // Each leaves less than a tar header, or less than the inner gzip header, in the outer layer's first 64 KiB. + test.each([65225, 65485, 66000, 140000])("is recognized behind an inner header of %d bytes", async name => { + expect(await names(gzip(gzip(tar, { name })))).toEqual(["a"]); + }); + + test("an inner name has to end within 1 MiB", async () => { + expect(await names(gzip(gzip(tar, { name: 1048576 - 10 })))).toEqual(["a"]); + await expect(names(gzip(gzip(tar, { name: 1048576 - 9 })))).rejects.toThrow("Unrecognized archive format"); + expect(await names(gzip(tar, { name: 1048576 + 100 }))).toEqual(["a"]); + }); + + test("the outer layer is read no further than the inner one needs", async () => { + const inner = Buffer.concat([gzip(tar), incompressible(200_000)]); + expect(await names(gzip(inner, { badCrc: true }))).toEqual(["a"]); + expect(await names(gzip(inner).subarray(0, 150_000))).toEqual(["a"]); + await expect(names(gzip(gzip(tar), { badCrc: true }))).rejects.toThrow("gzip trailer mismatch"); + }); + + test("what follows a member that ends with a 64 KiB block does not spoil that block", async () => { + const block = Buffer.concat([file("a", Buffer.alloc(64000, "x").toString()), end]); + expect(block.length).toBe(65536); + // A header that announces nine bytes of extra field and ends after one. + const cutHeader = Buffer.from([0x1f, 0x8b, 8, 4, 0, 0, 0, 0, 0, 255, 9, 0, 0]); + using dir = tempDir("archive-golden-member", {}); + const archive = new Bun.Archive(gzip(Buffer.concat([gzip(block), cutHeader]))); + await expect(archive.extract(String(dir))).rejects.toThrow("ReadError"); + expect(readdirSync(String(dir))).toEqual(["a"]); + }); +}); diff --git a/test/js/bun/archive-golden/bun-tar.test.ts b/test/js/bun/archive-golden/bun-tar.test.ts new file mode 100644 index 000000000000..48c9ad99108a --- /dev/null +++ b/test/js/bun/archive-golden/bun-tar.test.ts @@ -0,0 +1,281 @@ +// Bun.Archive shows the regular files of an archive, read by the rules of the system it runs on, and +// writes them with one mode and the current time. These tests reach the tar reader and writer +// themselves, with every set of rules on any system. +import { readTar, writeTar } from "bun:internal-for-testing"; +import { describe, expect, test } from "bun:test"; +import { join } from "node:path"; +import { + contents, + differences, + gzipStored, + loadCases, + namesToCompose, + paxHeader, + paxRecord, + readerArchives, + tarHeader, + tarPadded, + type ReaderFixture, + type WriterCase, +} from "./observe"; + +// See record-reader.ts. +describe("the tar reader reports what libarchive did", async () => { + const [fixture] = await loadCases(join(import.meta.dir, "reader.json.gz")); + const archives = await readerArchives(); + const modes = fixture.modes.map((mode, column) => [JSON.stringify(mode), mode, column] as const); + const recorded = (index: number, column: number) => fixture.logs[fixture.cases[index][column]]; + + test.each(modes)("%s", (_, mode, column) => { + const results = archives.map(({ name, archive }, index) => ({ + index: name, + expected: recorded(index, column), + actual: readTar(archive, mode), + })); + expect(differences(results)).toBe(""); + }); + + const cut = modes.filter(([, mode]) => mode.stream && (mode.skip === "none" || mode.skip === "odd")); + test.each(cut)("however the input is cut into pieces: %s", (_, mode, column) => { + const results = archives.map(({ name, archive }, index) => { + // How much of a damaged gzip stream is seen before it fails depends on where the pieces end. + const comparable = (log: string) => (archive[0] === 0x1f && !/^end$/m.test(log) ? "fails\n" : log); + const pieceSize = archive.length > 20_000 ? [4093, 65536][index % 2] : [1, 7, 511, 512, 513][index % 5]; + return { + index: name, + expected: comparable(recorded(index, column)), + actual: comparable(readTar(archive, { ...mode, pieceSize, lateEnd: index % 2 === 1 })), + }; + }); + expect(differences(results)).toBe(""); + }); +}); + +// The rules in short. libarchive gave the same answers. +describe("the tar reader", () => { + const end = Buffer.alloc(1024); + const file = (name: string, body = "x") => + Buffer.concat([tarHeader(name, body.length), tarPadded(Buffer.from(body))]); + const pax = (records: Record, then: Buffer) => + Buffer.concat([paxHeader(...Object.entries(records).map(([key, value]) => paxRecord(key, value))), then]); + const x = " data 0+1\n end 1 crc32=8cdc1683\n"; + + test("reports entries of every type", () => { + const archive = Buffer.concat([ + tarHeader("d/", 0, "5"), + tarHeader("l", 0, "2", "../target"), + tarHeader("h", 0, "1", "f"), + tarHeader("c", 0, "3"), + tarHeader("b", 0, "4"), + tarHeader("p", 0, "6"), + file("f", "hello"), + end, + ]); + expect(readTar(archive).replaceAll(" end 0 crc32=00000000\n", "")).toBe( + [ + 'directory perm=644 size=0 mtime=0 path="d/"', + 'symlink perm=644 size=0 mtime=0 path="l" -> "../target"', + 'type-0 perm=644 size=0 mtime=0 path="h"', + 'character-device perm=644 size=0 mtime=0 path="c"', + 'block-device perm=644 size=0 mtime=0 path="b"', + 'fifo perm=644 size=0 mtime=0 path="p"', + 'file perm=644 size=5 mtime=0 path="f"', + " data 0+5", + " end 5 crc32=3610a686", + "end\n", + ].join("\n"), + ); + }); + + test("stops at two empty blocks by npm's rules, and reads on otherwise", () => { + const archive = Buffer.concat([file("a"), end, file("b"), end]); + const [a, b] = ["a", "b"].map(name => `file perm=644 size=1 mtime=0 path="${name}"\n${x}`); + expect(readTar(archive)).toBe(a + b + "end\n"); + expect(readTar(archive, { npm: true })).toBe(a + "end\n"); + }); + + test("takes a ._ file for metadata of the entry after it on macOS", () => { + const archive = Buffer.concat([file("._a", "metadata"), file("a"), end]); + const a = `file perm=644 size=1 mtime=0 path="a"\n${x}end\n`; + expect(readTar(archive, { macos: true })).toBe(a); + expect(readTar(archive)).toBe('file perm=644 size=8 mtime=0 path="._a"\n data 0+8\n end 8 crc32=4f143414\n' + a); + }); + + test("composes the characters of a pax path on Windows", () => { + const archive = Buffer.concat([pax({ path: "cafe\u0301" }, file("short")), end]); + expect(readTar(archive)).toBe(`file perm=644 size=1 mtime=0 path="cafe\\xcc\\x81"\n${x}end\n`); + expect(readTar(archive, { windows: true })).toBe(`file perm=644 size=1 mtime=0 path="caf\\xc3\\xa9"\n${x}end\n`); + }); + + test("puts the data of a sparse file where it belongs, which npm's rules know nothing of", () => { + const records = { + "GNU.sparse.major": "0", + "GNU.sparse.minor": "0", + "GNU.sparse.size": "5000", + "GNU.sparse.map": "1000,3,4000,2", + }; + const archive = Buffer.concat([pax(records, file("s", "abcde")), end]); + expect(readTar(archive)).toBe( + 'file perm=644 size=5000 mtime=0 path="s"\n data 1000+3\n data 4000+2\n end 5000 crc32=8587d865\nend\n', + ); + expect(readTar(archive, { npm: true })).toBe( + 'file perm=644 size=5 mtime=0 path="s"\n data 0+5\n end 5 crc32=8587d865\nend\n', + ); + }); + + test.each([{}, { stream: true, pieceSize: 100 }])("says where an archive is cut short %j", mode => { + const archive = Buffer.concat([file("a"), tarHeader("b", 1000), Buffer.alloc(100, "y")]); + expect(readTar(archive, mode)).toBe( + `file perm=644 size=1 mtime=0 path="a"\n${x}file perm=644 size=1000 mtime=0 path="b"\n data 0+100\n` + + "error: Truncated tar archive detected while reading data\n", + ); + }); + + // libarchive let them change the permissions. + test("passes over ACLs", () => { + const script = 'file perm=755 size=0 mtime=0 path="run.sh"\n end 0 crc32=00000000\nend\n'; + const entry = tarHeader("run.sh", 0, "0", "", header => void header.write("0000755\0", 100)); + const solaris = Buffer.concat([tarHeader("run.sh", 20, "A"), tarPadded(Buffer.from("1000003\0user::rw-,x\0"))]); + expect(readTar(Buffer.concat([solaris, entry, end]))).toBe(script); + expect(readTar(Buffer.concat([pax({ "SCHILY.acl.access": "user::rw-,group::r--,other::r--" }, entry), end]))).toBe( + script, + ); + expect(readTar(Buffer.concat([pax({ "SCHILY.acl.default": "not an ACL" }, entry), end]))).toBe(script); + expect(readTar(Buffer.concat([solaris, solaris, entry, end]))).toStartWith("error: Redundant 'A' header"); + }); + + // libarchive looked for the "=" in what was left of its 64 KiB block of unpacked bytes, or in 512 bytes if that + // was more, and passed over the rest of the header when it was not there. Only a header over 1 MiB got that far. + test.each([1, 300, 604, 605])("reads a long pax key that starts %i bytes before the end of a block of gzip", left => { + const filler = paxRecord("comment", Buffer.alloc(17 * 65536 - 512 - left - 17, "c")); + const records = [filler, paxRecord(Buffer.alloc(600, "k").toString(), "1"), paxRecord("path", "renamed")]; + expect(512 + filler.length).toBe(17 * 65536 - left); + const archive = Buffer.concat([paxHeader(...records), file("short"), end]); + const renamed = `file perm=644 size=1 mtime=0 path="renamed"\n${x}end\n`; + expect([archive, gzipStored(archive)].flatMap(bytes => [readTar(bytes), readTar(bytes, { stream: true })])).toEqual( + Array(4).fill(renamed), + ); + }); + + test("refuses what is not an archive, unless it is told that it is one", () => { + expect(readTar(Buffer.alloc(2000, "q"))).toBe("refused: Unrecognized archive format\n"); + expect(readTar(Buffer.alloc(2048, "q"), { stream: true })).toBe("end\n"); + }); +}); + +function write({ files, windows, gzipLevel, writeSize, finish }: Omit) { + return writeTar( + files.map(file => ({ size: Number(file.size), given: file.length, perm: file.perm, mtime: Number(file.mtime) })), + Buffer.concat(files.map(file => Buffer.from(file.path + "00", "hex"))), + Buffer.concat(files.map(file => contents(file.length, file.kind, file.seed))), + { windows, gzipLevel: gzipLevel ?? undefined, writeSize, finish }, + ); +} + +describe("the tar writer", () => { + const written = (name: string, windows: boolean) => { + const tar = Buffer.from( + writeTar([{ size: 1, given: 1, perm: 0o644, mtime: 0 }], Buffer.from(name + "\0"), Buffer.from("x"), { windows }), + ); + return [/path="(.*)"/.exec(readTar(tar, { windows }))![1], tar.includes("hdrcharset=BINARY") ? "binary" : "utf8"]; + }; + + test("records a name as it is on POSIX", () => { + expect(written("cafe\u0301", false)).toEqual(["cafe\\xcc\\x81", "binary"]); + expect(written("d\\e", false)).toEqual(["d\\\\e", "utf8"]); + }); + + test("composes a name and turns its backslashes on Windows", () => { + expect(written("cafe\u0301", true)).toEqual(["caf\\xc3\\xa9", "utf8"]); + expect(written("d\\e", true)).toEqual(["d/e", "utf8"]); + }); + + // libarchive failed such an entry. + test("records a name with more marks than can be composed as it is on Windows", () => { + const marks = String.fromCodePoint(0x334, 0x93c, 0x3099, 0x94d, 0x5b0, 0x5b1, 0x5b2, 0x5b3, 0x5b4, 0x5b5); + const escaped = Array.from(Buffer.from(marks), byte => "\\x" + byte.toString(16)).join(""); + expect(written("x" + marks, true)).toEqual(["x" + escaped, "binary"]); + expect(written("x" + marks.slice(0, -1), true)).toEqual(["x" + escaped.slice(0, -8), "utf8"]); + }); +}); + +// libarchive read the same names. +describe("the reader and the writer compose on Windows", () => { + const composed: Record = { + "European marks": "379841 bytes crc32=243fa655", + "marks of other scripts": "43463 bytes crc32=72de80e0", + "class 7": "97181 bytes crc32=132ab45d", + "class 8": "97010 bytes crc32=f78f5f27", + "class 202": "67955 bytes crc32=32a1ef7e", + "class 216": "67949 bytes crc32=aa8cbffa", + "class 220": "68083 bytes crc32=0010de8b", + "class 230 in the first plane": "441650 bytes crc32=8d01ca63", + "class 230 in the second plane": "524259 bytes crc32=1f95bc1c", + "class 240": "77199 bytes crc32=70cdc340", + "classes against each other, 1 of 4": "167406 bytes crc32=1d332031", + "classes against each other, 2 of 4": "164850 bytes crc32=45a68ed3", + "classes against each other, 3 of 4": "175786 bytes crc32=f0a7c7e3", + "classes against each other, 4 of 4": "133132 bytes crc32=66d2fdd4", + "characters without a class": "230435 bytes crc32=1c014ca0", + "Hangul consonants and vowels": "2184 bytes crc32=7a92fcc9", + "Hangul syllables and final consonants": "48881 bytes crc32=af722c7b", + "Hangul syllables": "77021 bytes crc32=cedcd6ca", + }; + const withoutStart = (log: string) => log.replace(/".*"\.\.\. /, ""); + + test.each(Object.entries(namesToCompose()))("%s", (names, make) => { + const name = make(); + const written = writeTar( + [{ size: 0, given: 0, perm: 0o644, mtime: 0 }], + Buffer.concat([name, Buffer.alloc(1)]), + Buffer.alloc(0), + { windows: true }, + ); + const archive = Buffer.concat([paxHeader(paxRecord("path", name)), tarHeader("short", 0), Buffer.alloc(1024)]); + const log = `file perm=644 size=0 mtime=0 path=${composed[names]}\n end 0 crc32=00000000\nend\n`; + expect({ + read: withoutStart(readTar(archive, { windows: true })), + written: withoutStart(readTar(Buffer.from(written))), + }).toEqual({ read: log, written: log }); + }); +}); + +// Each output was compared with libarchive's, byte for byte, when it was recorded. A compressed or a +// long one is kept as its hash. +describe("the tar writer writes what libarchive did", async () => { + const cases = (await loadCases(join(import.meta.dir, "writer.json.gz"))).map((c, index) => ({ + ...c, + index, + })); + const sha256 = (bytes: Uint8Array) => + `${bytes.length} bytes, sha256 ${new Bun.CryptoHasher("sha256").update(bytes).digest("hex")}\n`; + // One header or one run of other bytes to a line. + const blocks = (bytes: Buffer) => + `${bytes.length} bytes\n` + + Array.from({ length: Math.ceil(bytes.length / 512) }, (_, i) => + bytes.subarray(i * 512, i * 512 + 512).toString("latin1"), + ) + .map(block => JSON.stringify(block.replace(/\0+$/, ""))) + .join("\n") + + "\n"; + + test.each([ + ["with the names of POSIX", (c: WriterCase) => !c.windows && c.gzipLevel === null && c.finish], + ["with the names of Windows", (c: WriterCase) => c.windows && c.gzipLevel === null && c.finish], + ["compressed", (c: WriterCase) => c.gzipLevel !== null], + ["for sizes that are not written out", (c: WriterCase) => !c.finish], + ])("%s", (_, belongs) => { + const group = cases.filter(belongs); + expect(group.length).toBeGreaterThan(20); + const results = group.map(c => { + const actual = Buffer.from(write(c)); + if (!("expected" in c)) { + return { index: c.index, expected: `${c.length} bytes, sha256 ${c.sha256}\n`, actual: sha256(actual) }; + } + const expected = Buffer.from(c.expected, "base64"); + if (actual.equals(expected)) return { index: c.index, expected: "", actual: "" }; + return { index: c.index, expected: blocks(expected), actual: blocks(actual) }; + }); + expect(differences(results)).toBe(""); + }); +}); diff --git a/test/js/bun/archive-golden/cases.json.gz b/test/js/bun/archive-golden/cases.json.gz new file mode 100644 index 000000000000..f8c432215d81 Binary files /dev/null and b/test/js/bun/archive-golden/cases.json.gz differ diff --git a/test/js/bun/archive-golden/observe.ts b/test/js/bun/archive-golden/observe.ts new file mode 100644 index 000000000000..c93a38ab74d7 --- /dev/null +++ b/test/js/bun/archive-golden/observe.ts @@ -0,0 +1,1114 @@ +// What a program can see of Bun.Archive, as text. `record.ts` saves it and the test compares it. +import { chmodSync, lstatSync, mkdirSync, readFileSync, readdirSync, readlinkSync } from "node:fs"; +import { join } from "node:path"; + +export type Case = { + /** The archive, base64. */ + data: string; + /** Whether macOS and Windows read it the way Linux does. */ + mac: boolean; + win: boolean; + files: string; + extract: string; +}; + +export async function loadCases(path: string): Promise { + return JSON.parse(Buffer.from(Bun.gunzipSync(await Bun.file(path).bytes())).toString()); +} + +/** A ustar header with mode 644 and everything else zero. */ +/** One failure message for all the cases that differ, instead of one test per case. */ +export function differences(results: { index: number | string; expected: string; actual: string }[]) { + const different = results.filter(r => r.actual !== r.expected); + if (different.length === 0) return ""; + return [ + `${different.length} of ${results.length} differ`, + ...different + .slice(0, 5) + .map( + r => + `${typeof r.index === "number" ? "case " : ""}${r.index}\n--- expected\n${r.expected}--- actual\n${r.actual}`, + ), + ].join("\n"); +} + +/** The options of `readTar` in `bun:internal-for-testing`. */ +export type ReaderMode = { + npm?: boolean; + windows?: boolean; + macos?: boolean; + stream?: boolean; + pieceSize?: number; + lateEnd?: boolean; + skip?: "none" | "all" | "odd" | "otherThanFiles"; +}; +export type ReaderFixture = { + modes: ReaderMode[]; + logs: string[]; + /** For each of `readerArchives()`, the index in `logs` of what each mode reports. */ + cases: number[][]; +}; + +const systems: ReaderMode[] = [{}, { macos: true }, { windows: true }]; +/** Both sets of rules, on all three kinds of system, from memory and as a download, reading all, some or none of the contents. */ +export const readerModes: ReaderMode[] = systems.flatMap(system => + [false, true].flatMap(npm => + [false, true].flatMap(stream => + (["none", "all", "odd", "otherThanFiles"] as const).map(skip => ({ ...system, npm, stream, skip })), + ), + ), +); + +/** gzip with stored blocks, made by hand so that the bytes are the same whatever zlib does. */ +export function gzipStored(data: Uint8Array, { flags = 0, fields = Buffer.alloc(0) } = {}) { + const parts = [Buffer.from([0x1f, 0x8b, 8, flags, 0, 0, 0, 0, 0, 255]), fields]; + for (let at = 0; at === 0 || at < data.length; at += 65535) { + const block = data.subarray(at, at + 65535); + const head = Buffer.alloc(5); + head[0] = at + 65535 >= data.length ? 1 : 0; + head.writeUInt16LE(block.length, 1); + head.writeUInt16LE(~block.length & 0xffff, 3); + parts.push(head, Buffer.from(block)); + } + const trailer = Buffer.alloc(8); + trailer.writeUInt32LE(Bun.hash.crc32(data), 0); + trailer.writeUInt32LE(data.length >>> 0, 4); + return Buffer.concat([...parts, trailer]); +} + +export function paxRecord(key: string, value: string | Uint8Array) { + const rest = Buffer.concat([Buffer.from(` ${key}=`), Buffer.from(value), Buffer.from("\n")]); + let length = rest.length + 1; + while (String(length).length + rest.length !== length) length = String(length).length + rest.length; + return Buffer.concat([Buffer.from(String(length)), rest]); +} + +/** Applies to the entry behind it. */ +export function paxHeader(...records: Uint8Array[]) { + const body = Buffer.concat(records); + return Buffer.concat([tarHeader("PaxHeader/x", body.length, "x"), tarPadded(body)]); +} + +/** Archives that are made here and not stored: gzip in every state of repair, and what is at or over a limit of the reader. */ +function madeArchives() { + const made: { name: string; archive: Buffer }[] = []; + const add = (name: string, ...parts: Uint8Array[]) => void made.push({ name, archive: Buffer.concat(parts) }); + const changed = (bytes: Buffer, at: number, xor: number) => { + const copy = Buffer.from(bytes); + copy[at < 0 ? copy.length + at : at] ^= xor; + return copy; + }; + const end = Buffer.alloc(1024); + const file = (name: Buffer | string, body: Uint8Array | string = "x", type = "0") => + Buffer.concat([tarHeader(name, body.length, type), tarPadded(Buffer.from(body))]); + const tar = Buffer.concat([file("a", "hello"), file("b", Buffer.alloc(600, "b")), end]); + const gz = gzipStored(tar); + + for (let at = 0; at < gz.length; at = at < 20 || at >= gz.length - 11 ? at + 1 : Math.min(at + 211, gz.length - 11)) + add(`gzip cut at ${at}`, gz.subarray(0, at)); + for (let at = 0; at < 15; at++) for (const xor of [1, 0x80]) add(`gzip byte ${at} ^ ${xor}`, changed(gz, at, xor)); + for (let at = 1; at <= 8; at++) add(`gzip byte -${at} ^ 1`, changed(gz, -at, 1)); + + const fieldsOf = (flags: number) => + Buffer.concat([ + Buffer.from(flags & 4 ? [3, 0, 1, 2, 3] : []), + Buffer.from(flags & 8 ? "name\0" : ""), + Buffer.from(flags & 16 ? "comment\0" : ""), + Buffer.from(flags & 2 ? [0, 0] : []), + ]); + for (let flags = 1; flags < 32; flags++) { + const fields = fieldsOf(flags); + add(`gzip flags ${flags}`, gzipStored(tar, { flags, fields })); + // The same header behind a whole member, where it is the start of another member or it is nothing. + for (let cut = 0; cut < fields.length; cut++) { + add(`gzip flags ${flags}, header cut at ${10 + cut}`, gzipStored(tar, { flags, fields }).subarray(0, 10 + cut)); + add( + `gzip, then flags ${flags}, header cut at ${10 + cut}`, + gz, + gzipStored(tar, { flags, fields }).subarray(0, 10 + cut), + ); + } + } + for (const flags of [0x20, 0x40, 0x80]) add(`gzip flags ${flags}`, gzipStored(tar, { flags })); + + for (const at of [0, 1, 512, 700, 1024, 2560, tar.length]) { + add(`gzip members split at ${at}`, gzipStored(tar.subarray(0, at)), gzipStored(tar.subarray(at))); + add( + `gzip members split at ${at}, second damaged`, + gzipStored(tar.subarray(0, at)), + changed(gzipStored(tar.subarray(at)), -6, 1), + ); + add( + `gzip members split at ${at}, first damaged`, + changed(gzipStored(tar.subarray(0, at)), -6, 1), + gzipStored(tar.subarray(at)), + ); + } + for (const after of [ + Buffer.alloc(1), + Buffer.alloc(8), + Buffer.alloc(512), + Buffer.alloc(1024), + Buffer.from("garbage"), + tar, + ]) { + add(`gzip, then ${after.length} other bytes`, gz, after); + } + for (let length = 1; length <= 20; length++) + add(`gzip, then the first ${length} bytes of another`, gz, gz.subarray(0, length)); + + const layers = (bytes: Buffer, count: number) => Array.from({ length: count }).reduce(gzipStored, bytes); + for (const count of [2, 3, 23, 24, 25, 26]) add(`gzip ${count} times`, layers(tar, count)); + for (const [what, damage] of [ + ["trailer damaged", (bytes: Buffer) => changed(bytes, -6, 1)], + ["cut short", (bytes: Buffer) => bytes.subarray(0, bytes.length - 100)], + ["followed by garbage", (bytes: Buffer) => Buffer.concat([bytes, Buffer.from("garbage")])], + ["followed by another member", (bytes: Buffer) => Buffer.concat([bytes, gzipStored(file("c"))])], + [ + "followed by a cut header", + (bytes: Buffer) => Buffer.concat([bytes, Buffer.from([0x1f, 0x8b, 8, 4, 0, 0, 0, 0, 0, 255, 9, 0, 0])]), + ], + ] as const) { + add(`gzip in gzip, inner ${what}`, gzipStored(damage(gz))); + add(`gzip in gzip, outer ${what}`, damage(gzipStored(gz))); + add(`gzip in gzip in gzip, middle ${what}`, gzipStored(damage(gzipStored(gz)))); + } + + // A reader of gzip hands on 64 KiB at a time, and what fails takes the block it is in with it. + const many = Buffer.concat(Array.from({ length: 140 }, (_, i) => file(`f${i}`, "x"))); + for (const length of [65024, 65536, 66048, 131072]) { + for (const [what, after] of [ + ["a cut header", Buffer.from([0x1f, 0x8b, 8, 4, 0, 0, 0, 0, 0, 255, 9, 0, 0])], + ["a damaged member", changed(gzipStored(file("last")), -6, 1)], + ["a member", gzipStored(Buffer.concat([file("last"), end]))], + ["nothing", Buffer.alloc(0)], + ] as const) { + add(`gzip of ${length} bytes, then ${what}`, gzipStored(many.subarray(0, length)), after); + add( + `gzip in gzip of ${length} bytes, then ${what}`, + gzipStored(Buffer.concat([gzipStored(many.subarray(0, length)), after])), + ); + } + add(`gzip of ${length} bytes, trailer damaged`, changed(gzipStored(many.subarray(0, length)), -6, 1)); + } + add("a tar whose first name looks like gzip", file(Buffer.from([0x1f, 0x8b, 8])), end); + + const pax = (records: Uint8Array[], then = file("short")) => + Buffer.concat([paxHeader(...records), then, file("after"), end]); + for (const [key, limit] of [ + ["path", 1048576], + ["linkpath", 1048576], + ["GNU.sparse.name", 1048576], + ["uname", 65536], + ["gname", 65536], + ["SCHILY.fflags", 511], + ["hdrcharset", 63], + ["LIBARCHIVE.symlinktype", 15], + ["size", 64], + ["uid", 64], + ["GNU.sparse.offset", 64], + ["mtime", 128], + ["atime", 128], + ] as const) { + for (const length of [limit, limit + 1]) + add(`pax ${key} of ${length} bytes`, pax([paxRecord(key, Buffer.alloc(length, "1"))])); + } + for (const length of [128, 129]) + add( + `pax SCHILY.xattr with a name of ${length} bytes`, + pax([paxRecord("SCHILY.xattr." + Buffer.alloc(length, "n"), "v")]), + ); + for (const key of ["GNU.sparse.map", "SUN.holesdata", "SCHILY.xattr.x", "LIBARCHIVE.xattr.x"]) { + const limit = key.includes("xattr") ? 16 << 20 : 8 << 20; + add(`pax ${key} over its limit`, pax([paxRecord(key, Buffer.alloc(limit + 1, "1"))])); + } + for (const type of ["L", "K", "V"]) { + for (const length of [1048576, 1048577]) + add( + `${type} header of ${length} bytes`, + file("././@LongLink", Buffer.alloc(length, "n"), type), + file("short"), + end, + ); + } + for (const size of [16 << 20, (16 << 20) + 1]) + add(`._ file of ${size} bytes, cut short`, tarHeader("._big", size), file("after"), end); + + add("pax record with a length over the limit", pax([Buffer.from("100000000 path=x\n")])); + add("pax record with no key", pax([Buffer.from("4 =\n"), paxRecord("path", "named")])); + const sparse = (records: Record, body: Uint8Array | string = "abcde") => + pax( + Object.entries(records).map(([key, value]) => paxRecord(`GNU.sparse.${key}`, value)), + file("s", body), + ); + add("sparse 0.0, a block past the end of all numbers", sparse({ offset: "9223372036854775806", numbytes: "5" })); + add( + "sparse 1.0", + sparse({ major: "1", minor: "0", realsize: "5000" }, tarPadded(Buffer.from("2\n1000\n3\n4000\n2\n")) + "abcde"), + ); + add( + "sparse 1.0, sized by GNU.sparse.size", + sparse({ major: "1", minor: "0", size: "517" }, tarPadded(Buffer.from("1\n0\n5\n")) + "abcde"), + ); + add( + "sparse 1.0, a block past the end of all numbers", + sparse({ major: "1", minor: "0" }, tarPadded(Buffer.from("1\n9223372036854775806\n5\n"))), + ); + add("sparse 1.0, the map is cut short", sparse({ major: "1", minor: "0" }, "2\n1000\n")); + add("sparse 2.0", sparse({ major: "2", minor: "0", realsize: "5000" })); + add( + "more stored than a size can be", + pax([paxRecord("GNU.sparse.realsize", "5"), paxRecord("size", "1152921504606846976")]), + ); + const oldSparse = (extensions: number) => { + const pair = (at: number, offset: number, length: number) => (h: Buffer) => { + h.write(offset.toString(8).padStart(11, "0") + "\0", at); + h.write(length.toString(8).padStart(11, "0") + "\0", at + 12); + }; + const header = tarHeader("s", 4 + extensions, "S", "", h => { + h.write("ustar \0", 257); + for (let i = 0; i < 4; i++) pair(386 + i * 24, i * 1000, 1)(h); + h[482] = extensions ? 1 : 0; + h.write((9000).toString(8).padStart(11, "0") + "\0", 483); + }); + const blocks = Array.from({ length: extensions }, (_, i) => { + const block = Buffer.alloc(512); + pair(0, 4000 + i * 1000, 1)(block); + block[504] = i + 1 < extensions ? 1 : 0; + return block; + }); + return [header, ...blocks, tarPadded(Buffer.alloc(4 + extensions, "d")), file("after"), end]; + }; + for (const extensions of [0, 1, 3]) + add(`old GNU sparse with ${extensions} more blocks of map`, ...oldSparse(extensions)); + add("old GNU sparse, cut in its map", ...oldSparse(3).slice(0, 2), Buffer.alloc(100)); + + const withMap = sparse( + { major: "1", minor: "0", name: "s", realsize: "20" }, + tarPadded(Buffer.from("1\n10\n5\n")) + "hello", + ); + for (const length of [2, 3]) + add(`sparse 1.0, cut ${length} bytes into its map`, withMap.subarray(0, withMap.indexOf("1\n10\n5\n") + length)); + for (const [offset, numbytes] of [ + ["9223372036854775807", "1"], + ["9223372036854775806", "2"], + ["-1", "1"], + ]) + add(`sparse 0.0, ${numbytes} bytes at ${offset}`, sparse({ numblocks: "1", offset, numbytes, size: "5" })); + add( + "sparse 0.0, the length before the offset", + sparse({ numblocks: "1", numbytes: "4611686018427387904", offset: "4611686018427387904" }), + ); + + const acl = file("a", "1000007\0user::rw-,group::r--,other::r--\0", "A"); + add("Solaris ACL", acl, file("a"), end); + add("Solaris ACL twice", acl, acl, file("a"), end); + add("Solaris ACL of no size", file("a", "", "A"), file("a"), end); + + // Long enough for the inner layer to have handed on a block when the outer one fails. + const twice = layers(Buffer.concat([file("first"), file("big", Buffer.alloc(100_000, "b")), file("last"), end]), 2); + add("long gzip in gzip, outer cut at 60%", twice.subarray(0, Math.floor(twice.length * 0.6))); + add("long gzip in gzip, outer cut near its end", twice.subarray(0, twice.length - 700)); + add("long gzip in gzip, damaged", changed(changed(twice, twice.length >> 1, 0xff), 70_000, 0xff)); + + for (const [what, at, bytes] of [ + ["time of 2^63", 136, [0x80, 0, 0, 0, 0x80]], + ["time below -2^63", 136, [0xff, 0xff, 0xff, 0xff, 0x7f]], + ["size of 2^63", 124, [0x80, 0, 0, 0, 0x80]], + ] as const) { + const number = Buffer.concat([Buffer.from(bytes), Buffer.alloc(7)]); + add( + `base-256 ${what}`, + tarHeader("m", 1, "0", "", h => void number.copy(h, at)), + tarPadded(Buffer.from("x")), + end, + ); + } + for (const size of [2 ** 30, 2 ** 30 + 1]) + add(`pax header that announces ${size} bytes`, tarHeader("PaxHeader/x", size, "x"), file("a"), end); + for (const bytes of [ + [0x61, 0xf0, 0x41, 0x80, 0x80], + [0x61, 0xf0, 0x9f, 0x98, 0x41], + ]) + add(`pax path ${Buffer.from(bytes).toString("hex")}`, pax([paxRecord("path", Buffer.from(bytes))])); + + // Names that Windows composes, and ones it cannot. + let state = 1; + const random = (below: number) => ((state = (Math.imul(state, 1103515245) + 12345) >>> 0) >>> 8) % below; + const pick = (from: T[]) => from[random(from.length)]; + const bases = [ + ..."aAeEoOuUiIcCnNsSyY<=>\u00a8\u00c2\u00ca\u01a1\u0391\u03b1\u0415\u0627\u0928\u09c7\u1025\u1100\u1112\uac00\uac01", + ].concat([..."\u1e36\u1f00\u2190\u3046\u304b\u30ab\u212b\ufb01\u{11099}\u{1d157}"]); + const marks = [ + 0x300, 0x301, 0x302, 0x303, 0x304, 0x306, 0x307, 0x308, 0x30a, 0x30b, 0x30c, 0x323, 0x327, 0x328, 0x331, 0x338, + ] + .concat([ + 0x342, 0x345, 0x3099, 0x309a, 0x5b0, 0x5bc, 0x93c, 0x653, 0x654, 0x655, 0x1161, 0x1162, 0x1175, 0x11a8, 0x11c2, + ]) + .concat([0x9be, 0x9d7, 0x102e, 0x110ba, 0x1d165, 0x1d16e]); + const notUtf8 = ["ff", "eda080", "c3", "f4908080", "c0af", "e282"].map(hex => Buffer.from(hex, "hex")); + for (let archive = 0; archive < 5; archive++) { + const entries = Array.from({ length: 80 }, () => { + const clusters = Array.from({ length: 1 + random(4) }, () => { + const cluster = + pick(bases) + + String.fromCodePoint(...Array.from({ length: pick([0, 0, 1, 1, 2, 3, 4, 9, 10, 11]) }, () => pick(marks))); + return random(12) === 0 ? Buffer.concat([Buffer.from(cluster), pick(notUtf8)]) : Buffer.from(cluster); + }); + return Buffer.concat([paxHeader(paxRecord("path", Buffer.concat(clusters))), tarHeader("short", 0)]); + }); + add(`names to compose ${archive}`, ...entries, end); + } + return made; +} + +/** + * Archives written for lines and branches of libarchive's own tar reader that no other archive here made it run, + * found by measuring its coverage. + */ +function archivesForLibarchiveBranches() { + const end = Buffer.alloc(1024); + const long = (n: number, c = "v") => Buffer.alloc(n, c); + const file = (name: string, body = "x") => + Buffer.concat([tarHeader(name, body.length), tarPadded(Buffer.from(body))]); + const rest = Buffer.concat([file("first", "hello"), file("second"), end]); + const pax = (...records: [string, string | Uint8Array][]) => + Buffer.concat([paxHeader(...records.map(([k, v]) => paxRecord(k, v))), rest]); + const gnu = (h: Buffer) => void h.write("ustar \0", 257); + const octal = (h: Buffer, at: number, n: number) => void h.write(n.toString(8).padStart(11, "0") + "\0", at); + const special = (type: string, body: Buffer | string, announced = Buffer.from(body).length) => + Buffer.concat([ + tarHeader(type === "A" ? "first" : "././@LongLink", announced, type, "", type === "A" ? undefined : gnu), + tarPadded(Buffer.from(body)), + ]); + /** Where the value of `key` starts in an archive that begins with one pax header. */ + const valueAt = (archive: Buffer, key: string) => archive.indexOf(` ${key}=`) + key.length + 2; + /** The parser looks at 512 bytes from the start of a record before it reads the value, so only a cut behind those is the value's own. */ + const cutInValue = (key: string, value: string | Uint8Array) => { + const a = pax([key, value], ["path", "renamed"]); + return a.subarray(0, valueAt(a, key) + 600); + }; + const oldSparse = (map: [number, number][], extended: boolean, more: Buffer[], body: string) => + Buffer.concat([ + tarHeader("sparse", body.length, "S", "", h => { + gnu(h); + map.forEach(([o, n], i) => { + octal(h, 386 + 24 * i, o); + octal(h, 398 + 24 * i, n); + }); + h[482] = extended ? 1 : 0; + octal(h, 483, 40); + }), + ...more, + tarPadded(Buffer.from(body)), + rest, + ]); + const sparse10 = (map: string) => + Buffer.concat([ + paxHeader( + paxRecord("GNU.sparse.major", "1"), + paxRecord("GNU.sparse.minor", "0"), + paxRecord("GNU.sparse.name", "sparse"), + paxRecord("GNU.sparse.realsize", "5"), + ), + tarHeader("GNUSparseFile.0/sparse", 512 + 5), + tarPadded(Buffer.from(map)), + tarPadded(Buffer.from("hello")), + rest, + ]); + + const shapes: Record = { + // pax keys whose first letter belongs to a known key (the `break` of every case in pax_attribute) + "pax keys that only start like known ones": pax( + ...[ + "GNU.x", + "GNU.sparse.x", + "LIBARCHIVE.x", + "Rx", + "RHT.security.selinuy", + "SCHILY.x", + "SUN.x", + "ax", + "cx", + "gx", + "hx", + "lx", + "mx", + "px", + "rx", + "sx", + "ux", + ].map(k => [k, "1"] as [string, string]), + ["path", "renamed"], + ), + // pax_time: what is behind the ninth digit of the fraction + "pax mtime with ten digits of fraction": pax(["mtime", "5.1234567891"]), + "pax mtime with a letter behind nine digits of fraction": pax(["mtime", "5.123456789x"]), + // header_gnutar: atime and ctime + "GNU header with atime and ctime": Buffer.concat([ + tarHeader("first", 1, "0", "", h => { + gnu(h); + octal(h, 345, 7); + octal(h, 357, 8); + }), + tarPadded(Buffer.from("x")), + end, + ]), + // old GNU sparse: a bad pair in a block of more map + "old GNU sparse, a pair in the second block overflows": oldSparse( + [[0, 5]], + true, + [ + (() => { + const b = Buffer.alloc(512); + b.fill(0xff, 0, 12); + b[0] = 0x80; + b.fill(0x7f, 4, 5); + octal(b, 12, 5); + return b; + })(), + ], + "hello", + ), + "old GNU sparse that says there is more map, with none at all": oldSparse([], true, [Buffer.alloc(512)], ""), + // sparse 1.0: numbers that overflow, in the multiplication and in the addition + "sparse 1.0 offset of 2^63": sparse10("1\n9223372036854775808\n5\n"), + "sparse 1.0 size of 2^63": sparse10("1\n0\n9223372036854775808\n"), + "sparse 1.0 count of 2^63": sparse10("9223372036854775808\n0\n5\n"), + "sparse 1.0 offset of twenty digits": sparse10("1\n99999999999999999999\n5\n"), + // ACL records: skipped, but they have a limit of their own and can be cut + "pax ACL records": pax( + ["SCHILY.acl.access", "user::rwx,group::r-x,other::r--"], + ["SCHILY.acl.default", "user::rwx,group::r-x,other::r--"], + ["path", "renamed"], + ), + "pax NFSv4 ACL record": pax(["SCHILY.acl.ace", "owner@:rwx::allow,everyone@:r::allow"], ["path", "renamed"]), + "pax ACL record that does not parse": pax(["SCHILY.acl.access", "nonsense"], ["path", "renamed"]), + "pax ACL record of 131072 bytes": pax(["SCHILY.acl.access", long(131072)], ["path", "renamed"]), + "pax ACL record of 131073 bytes": pax(["SCHILY.acl.access", long(131073)], ["path", "renamed"]), + "pax ACL record cut in its value": cutInValue("SCHILY.acl.access", long(3000, "user::rwx,")), + "Solaris ACL header with a digit that is not octal": Buffer.concat([special("A", "1000009\0user::rwx\0"), rest]), + "Solaris ACL header, NFSv4": Buffer.concat([special("A", "3000003\0owner@:rwx::allow\0"), rest]), + "Solaris ACL header with too large a count": Buffer.concat([special("A", "1000000000\0user::rwx\0"), rest]), + "Solaris ACL header with nothing behind the count": Buffer.concat([special("A", "1000003\0"), rest]), + "Solaris ACL header that does not parse": Buffer.concat([special("A", "1000003\0nonsense\0"), rest]), + "Solaris ACL header cut in its body": Buffer.concat([special("A", "1000003\0user::rwx\0", 2000)]).subarray( + 0, + 512 + 100, + ), + // values that are cut behind the 512 bytes the parser looks at first + "pax GNU.sparse.map cut in its value": cutInValue("GNU.sparse.map", long(1603, "0,1,")), + "pax LIBARCHIVE.xattr cut in its value": cutInValue("LIBARCHIVE.xattr.user.x", long(3200, "QUJD")), + "pax RHT.security.selinux cut in its value": cutInValue("RHT.security.selinux", long(3000)), + "pax SCHILY.xattr cut in its value": cutInValue("SCHILY.xattr.user.x", long(3000)), + "pax SUN.holesdata cut in its value": cutInValue( + "SUN.holesdata", + " " + Array.from({ length: 900 }, (_, i) => i).join(" "), + ), + "pax SCHILY.fflags cut in its value": (() => { + const a = pax(["SCHILY.fflags", long(511, "nodump,")], ["path", "renamed"]); + return a.subarray(0, valueAt(a, "SCHILY.fflags") + 505); + })(), + "pax record of an unknown key cut in its value": cutInValue("comment", long(3000)), + "pax record cut before its newline": (() => { + const a = pax(["comment", long(3000)], ["path", "renamed"]); + return a.subarray(0, valueAt(a, "comment") + 3000); + })(), + // limits + "pax RHT.security.selinux of 16777216 bytes": pax(["RHT.security.selinux", long(16 << 20)], ["path", "renamed"]), + "pax RHT.security.selinux of 16777217 bytes": pax( + ["RHT.security.selinux", long((16 << 20) + 1)], + ["path", "renamed"], + ), + "L header of 1048577 bytes, cut": special("L", long(5000), (1 << 20) + 1), + // a header that failed stays failed behind a second L, K or V header + ...Object.fromEntries( + ["L", "K", "V"].map(t => [ + `sparse map over the limit, then two ${t} headers`, + Buffer.concat([ + paxHeader(paxRecord("GNU.sparse.map", long((8 << 20) + 1, "0"))), + special(t, "one\0"), + special(t, "two\0"), + rest, + ]), + ]), + ), + // %XX in the name of a LIBARCHIVE.xattr record + "LIBARCHIVE.xattr names with % in them": pax( + ...["%41", "a%4Ab", "%4a%6A", "%", "a%", "a%4", "%zz", "%4z", "%z4", "%FF%ff", "%25"].map( + n => [`LIBARCHIVE.xattr.${n}`, "QUJD"] as [string, string], + ), + ["path", "renamed"], + ), + // bidding: a type that is neither a letter nor a digit + "first header of type '-'": Buffer.concat([tarHeader("first", 1, "-"), tarPadded(Buffer.from("x")), rest]), + // gzip that goes bad inside the body of a pax header (the streaming reader asks for header and body at once) + // libarchive inflates 64 KiB at a time and drops a block that ends in an error, so the header block has to end one block and the damage has to be two further on. + "gzip damaged inside the body of a pax header": (() => { + const tar = Buffer.concat([ + tarHeader("filler", 64512), + long(64512), + paxHeader(paxRecord("comment", long(140000)), paxRecord("path", "renamed")), + rest, + ]); + const g = Buffer.from(gzipStored(tar)); + g.writeUInt16LE(1, 10 + 2 * (5 + 65535) + 1); + return g; + })(), + // a value over its limit is skipped without a look, so a cut in it shows up only afterwards + "pax path of 1048577 bytes, cut": pax(["path", long((1 << 20) + 1)]).subarray(0, 9000), + "pax uname of 65537 bytes, cut": pax(["uname", long(65537)]).subarray(0, 9000), + // ---- branches: the other answer of a test that the fixture only ever answers one way + // every known key with its last letter changed: right length, wrong text + "pax keys of the length of known ones": pax( + ...[ + "GNU.sparsf", + "GNU.sparse.numblockt", + "GNU.sparse.offseu", + "GNU.sparse.numbytet", + "GNU.sparse.sizf", + "GNU.sparse.maq", + "GNU.sparse.majos", + "GNU.sparse.minos", + "GNU.sparse.namf", + "GNU.sparse.realsizf", + "GNUx.sparse", + "LIBARCHIVEx.xattr", + "LIBARCHIVE.creationtimf", + "LIBARCHIVE.symlinktypf", + "LIBARCHIVE.xattrx.a", + "RHT.security.selinuy", + "SCHILY.acl.accest", + "SCHILY.acl.defaulu", + "SCHILY.acl.acf", + "SCHILY.devmajos", + "SCHILY.devminos", + "SCHILY.fflagt", + "SCHILY.dew", + "SCHILY.inp", + "SCHILY.nlinl", + "SCHILY.realsizf", + "SCHILY.xattrx.a", + "SCHILYx.ino", + "SUN.holesdatb", + "SUNx.holesdata", + "atimf", + "ctimf", + "charseu", + "commenu", + "gie", + "gnamf", + "hdrcharseu", + "linkpati", + "mtimf", + "pati", + "sizf", + "uie", + "unamf", + ].map(k => [k, "1"] as [string, string]), + ["path", "renamed"], + ), + "pax sparse versions 0.2, 1.1 and 2.0": Buffer.concat( + [ + ["0", "2"], + ["1", "1"], + ["2", "0"], + ] + .map(([a, b]) => + Buffer.concat([ + paxHeader(paxRecord("GNU.sparse.major", a), paxRecord("GNU.sparse.minor", b)), + file("v" + a + b, "hello"), + ]), + ) + .concat(end), + ), + "pax sparse version that is not a number": pax(["GNU.sparse.major", "x"], ["GNU.sparse.minor", "-1"]), + "pax symlinktype of the right length": Buffer.concat([ + paxHeader(paxRecord("LIBARCHIVE.symlinktype", "filf")), + file("a"), + paxHeader(paxRecord("LIBARCHIVE.symlinktype", "dis")), + rest, + ]), + "pax hdrcharset of the length of the UTF-8 one": pax( + ["hdrcharset", "ISO-IR 10646 2000 UTF-9"], + ["path", "caf\u00e9"], + ), + "pax record whose length ends inside its key": Buffer.concat([ + tarHeader("PaxHeader/x", 12, "x"), + tarPadded(Buffer.from("5 path=abc\n\n")), + rest, + ]), + "pax record whose length ends at its first letter": Buffer.concat([ + tarHeader("PaxHeader/x", 10, "x"), + tarPadded(Buffer.from("2 path=ab\n")), + rest, + ]), + "pax header that ends inside a key": Buffer.concat([ + tarHeader("PaxHeader/x", 6, "x"), + tarPadded(Buffer.from("20 pat")), + rest, + ]), + "pax header that ends behind a length": Buffer.concat([ + tarHeader("PaxHeader/x", 3, "x"), + tarPadded(Buffer.from("20 ")), + rest, + ]), + "pax key of more than 512 bytes": pax([long(600, "k").toString(), "1"], ["path", "renamed"]), + "pax mtime of 2^63": pax(["mtime", "9223372036854775808"]), + "pax mtime of -2^63 - 1": pax(["mtime", "-9223372036854775809"]), + "pax mtime with a slash in its fraction": pax(["mtime", "5.12/"]), + "pax mtime with a slash behind nine digits of fraction": pax(["mtime", "5.123456789/"]), + "pax size of 2^63 and of -2^63 - 1": Buffer.concat([ + paxHeader(paxRecord("size", "9223372036854775808")), + file("a"), + paxHeader(paxRecord("uid", "-9223372036854775809")), + rest, + ]), + "pax empty linkpath, uname and gname in front of a header that has them": Buffer.concat([ + paxHeader(paxRecord("linkpath", ""), paxRecord("uname", ""), paxRecord("gname", "")), + tarHeader("link", 0, "2", "target", h => { + h.write("user", 265); + h.write("group", 297); + }), + paxHeader(paxRecord("linkpath", ""), paxRecord("uname", ""), paxRecord("gname", "")), + tarHeader("hard", 0, "1", "target", h => { + gnu(h); + h.write("user", 265); + h.write("group", 297); + }), + rest, + ]), + "pax device numbers in front of a GNU device header": Buffer.concat([ + paxHeader(paxRecord("SCHILY.devmajor", "3"), paxRecord("SCHILY.devminor", "4")), + tarHeader("dev", 0, "3", "", h => { + gnu(h); + h.write("0000001\0", 329); + h.write("0000002\0", 337); + }), + rest, + ]), + "sparse 0.1 map with a character below 0": pax(["GNU.sparse.map", "0,1,-2,3"]), + "SUN.holesdata with a character below 0": pax(["SUN.holesdata", " 0 1 -2 3"]), + "sparse 1.0 number with a character above 9": sparse10("1\n0:\n5\n"), + "Solaris ACL count with a character below 0": Buffer.concat([special("A", "10000-3\0user::rwx\0"), rest]), + "two entries with a Solaris ACL header": Buffer.concat([ + special("A", "1000003\0user::rwx\0"), + file("a"), + special("A", "1000003\0user::r--\0"), + rest, + ]), + "LIBARCHIVE.xattr values that end early or hold other bytes": pax( + ["LIBARCHIVE.xattr.a", "QUJD_QUJD"], + ["LIBARCHIVE.xattr.b", "QUJD=QUJD"], + ["LIBARCHIVE.xattr.c", Buffer.from([0x51, 0xff, 0x55, 0x1f, 0x4a])], + ["LIBARCHIVE.xattr.d", "Q"], + ["LIBARCHIVE.xattr.e", "QU"], + ["LIBARCHIVE.xattr.f", "QUJ"], + ["LIBARCHIVE.xattr.g", "="], + ["path", "renamed"], + ), + "LIBARCHIVE.xattr names with % and what is next to the hex digits": pax( + ...["%/0", "%:0", "%@0", "%G0", "%`0", "%g0", "%0/", "%0:", "%0@", "%0G", "%0`", "%0g"].map( + n => [`LIBARCHIVE.xattr.${n}`, "QUJD"] as [string, string], + ), + ["path", "renamed"], + ), + // bidding looks at the number fields of the first header + "first header with number fields of nothing but blanks": Buffer.concat([ + tarHeader("first", 0, "0", "", h => { + for (const [at, n] of [ + [100, 8], + [108, 8], + [116, 8], + [136, 12], + [329, 8], + [337, 8], + ]) + h.fill(" ", at, at + n); + }), + rest, + ]), + "first header with number fields of nothing but digits": Buffer.concat([ + tarHeader("first", 0, "0", "", h => { + for (const [at, n] of [ + [100, 8], + [108, 8], + [116, 8], + [329, 8], + [337, 8], + ]) + h.fill("0", at, at + n); + }), + rest, + ]), + "first header with a letter in rdevmajor": Buffer.concat([ + tarHeader("first", 0, "0", "", h => void h.write("x", 329)), + rest, + ]), + "first header with a letter in rdevminor": Buffer.concat([ + tarHeader("first", 0, "0", "", h => void h.write("x", 337)), + rest, + ]), + "first header of type '{'": Buffer.concat([tarHeader("first", 1, "{"), tarPadded(Buffer.from("x")), rest]), + // octal fields at the edge of 64 bits (only a 12-byte field is long enough, and only with a sign) + "mtime of twelve sevens and of a minus and eleven sevens": Buffer.concat([ + tarHeader("a", 0, "0", "", h => void h.write("777777777777", 136)), + tarHeader("b", 0, "0", "", h => void h.write("-77777777777", 136)), + rest, + ]), + "K header with an empty name in front of a symlink and of a hardlink": Buffer.concat([ + special("K", "\0"), + tarHeader("link", 0, "2", "target"), + special("K", "\0"), + tarHeader("hard", 0, "1", "target"), + rest, + ]), + "pax uname and gname that start with NUL": Buffer.concat([ + paxHeader(paxRecord("uname", "\0x"), paxRecord("gname", "\0x")), + tarHeader("a", 0, "0", "", h => { + h.write("user", 265); + h.write("group", 297); + }), + paxHeader(paxRecord("uname", "\0x"), paxRecord("gname", "\0x")), + tarHeader("b", 0, "0", "", h => { + gnu(h); + h.write("user", 265); + h.write("group", 297); + }), + rest, + ]), + "pax keys that are only the start of a vendor's": pax( + ["G", "1"], + ["GNU.", "1"], + ["GNU.sparsex.a", "1"], + ["L", "1"], + ["LIBARCHIVE.", "1"], + ["S", "1"], + ["SCHILY.", "1"], + ["SUN.", "1"], + ["path", "renamed"], + ), + "pax sparse major below zero": pax(["GNU.sparse.major", "-1"], ["GNU.sparse.minor", "0"]), + "pax uid of -2^63 - 1": pax(["uid", "-9223372036854775809"], ["gid", "-9223372036854775808"]), + "pax header that is a length and a blank": Buffer.concat([ + tarHeader("PaxHeader/x", 2, "x"), + tarPadded(Buffer.from("2 ")), + rest, + ]), + }; + return Object.entries(shapes).map(([name, archive]) => ({ name, archive })); +} + +/** + * Long names of short clusters with a "_" after each. Between them they hold every pair that Windows + * composes, beside many more that it does not, and tell the combining classes apart. + */ +export function namesToCompose(): Record Buffer> { + const range = (first: number, last: number) => Array.from({ length: last - first + 1 }, (_, i) => first + i); + // From the space on, without surrogates and without the backslash, which the writer makes a slash. + function blocks(...numbers: number[]) { + const characters: number[] = []; + for (const block of numbers) { + for (let c = Math.max(block << 8, 0x20); c <= ((block << 8) | 0xff); c++) { + if ((c < 0xd800 || c > 0xdfff) && c !== 0x5c) characters.push(c); + } + } + return characters; + } + function name(characters: Uint32Array) { + let text = ""; + for (let at = 0; at < characters.length; at += 4096) { + text += String.fromCodePoint.apply(null, characters.subarray(at, at + 4096) as unknown as number[]); + } + return Buffer.from(text); + } + function pairs(firsts: number[], seconds: number[]) { + const characters = new Uint32Array(firsts.length * seconds.length * 3).fill(0x5f); + let at = 0; + for (const first of firsts) { + for (const second of seconds) { + characters[at] = first; + characters[at + 1] = second; + at += 3; + } + } + return name(characters); + } + // Whether the mark composes with the base depends on the class of what stands between them. + function between(base: number, mark: number, middles: number[]) { + const characters = new Uint32Array(middles.length * 4).fill(0x5f); + let at = 0; + for (const middle of middles) { + characters[at] = base; + characters[at + 1] = middle; + characters[at + 2] = mark; + at += 4; + } + return name(characters); + } + + const european = [...range(0x300, 0x30c), 0x30f, 0x311, 0x313, 0x314, 0x31b, ...range(0x323, 0x328)].concat([ + 0x32d, 0x32e, 0x330, 0x331, 0x338, 0x342, 0x345, + ]); + const others = [0x653, 0x654, 0x655, 0x93c, 0x9be, 0x9d7, 0xb3e, 0xb56, 0xb57, 0xbbe, 0xbd7, 0xc56, 0xcc2].concat([ + 0xcd5, 0xcd6, 0xd3e, 0xd57, 0xdca, 0xdcf, 0xddf, 0x102e, 0x1b35, 0x3099, 0x309a, 0x110ba, + ]); + const withMarks = () => + blocks(...range(0x03, 0x10), 0x13, ...range(0x17, 0x1d), 0x20, 0x2c, 0x2d, 0x30, 0xa6).concat( + blocks(...range(0xa8, 0xab), 0xfb, 0xfe, 0x101, 0x10a, 0x110, 0x1d1, 0x1d2), + ); + // The first and the last of each run of characters that have a combining class in Unicode 6.0. + const runsOfMarks = [ + 0x300, 0x34e, 0x350, 0x36f, 0x483, 0x487, 0x591, 0x5bd, 0x5bf, 0x5bf, 0x5c1, 0x5c2, 0x5c4, 0x5c5, 0x5c7, 0x5c7, + 0x610, 0x61a, 0x64b, 0x65f, 0x670, 0x670, 0x6d6, 0x6dc, 0x6df, 0x6e4, 0x6e7, 0x6e8, 0x6ea, 0x6ed, 0x711, 0x711, + 0x730, 0x74a, 0x7eb, 0x7f3, 0x816, 0x819, 0x81b, 0x823, 0x825, 0x827, 0x829, 0x82d, 0x859, 0x85b, 0x93c, 0x93c, + 0x94d, 0x94d, 0x951, 0x954, 0x9bc, 0x9bc, 0x9cd, 0x9cd, 0xa3c, 0xa3c, 0xa4d, 0xa4d, 0xabc, 0xabc, 0xacd, 0xacd, + 0xb3c, 0xb3c, 0xb4d, 0xb4d, 0xbcd, 0xbcd, 0xc4d, 0xc4d, 0xc55, 0xc56, 0xcbc, 0xcbc, 0xccd, 0xccd, 0xd4d, 0xd4d, + 0xdca, 0xdca, 0xe38, 0xe3a, 0xe48, 0xe4b, 0xeb8, 0xeb9, 0xec8, 0xecb, 0xf18, 0xf19, 0xf35, 0xf35, 0xf37, 0xf37, + 0xf39, 0xf39, 0xf71, 0xf72, 0xf74, 0xf74, 0xf7a, 0xf7d, 0xf80, 0xf80, 0xf82, 0xf84, 0xf86, 0xf87, 0xfc6, 0xfc6, + 0x1037, 0x1037, 0x1039, 0x103a, 0x108d, 0x108d, 0x135d, 0x135f, 0x1714, 0x1714, 0x1734, 0x1734, 0x17d2, 0x17d2, + 0x17dd, 0x17dd, 0x18a9, 0x18a9, 0x1939, 0x193b, 0x1a17, 0x1a18, 0x1a60, 0x1a60, 0x1a75, 0x1a7c, 0x1a7f, 0x1a7f, + 0x1b34, 0x1b34, 0x1b44, 0x1b44, 0x1b6b, 0x1b73, 0x1baa, 0x1baa, 0x1be6, 0x1be6, 0x1bf2, 0x1bf3, 0x1c37, 0x1c37, + 0x1cd0, 0x1cd2, 0x1cd4, 0x1ce0, 0x1ce2, 0x1ce8, 0x1ced, 0x1ced, 0x1dc0, 0x1de6, 0x1dfc, 0x1dff, 0x20d0, 0x20dc, + 0x20e1, 0x20e1, 0x20e5, 0x20f0, 0x2cef, 0x2cf1, 0x2d7f, 0x2d7f, 0x2de0, 0x2dff, 0x302a, 0x302f, 0x3099, 0x309a, + 0xa66f, 0xa66f, 0xa67c, 0xa67d, 0xa6f0, 0xa6f1, 0xa806, 0xa806, 0xa8c4, 0xa8c4, 0xa8e0, 0xa8f1, 0xa92b, 0xa92d, + 0xa953, 0xa953, 0xa9b3, 0xa9b3, 0xa9c0, 0xa9c0, 0xaab0, 0xaab0, 0xaab2, 0xaab4, 0xaab7, 0xaab8, 0xaabe, 0xaabf, + 0xaac1, 0xaac1, 0xabed, 0xabed, 0xfb1e, 0xfb1e, 0xfe20, 0xfe26, 0x101fd, 0x101fd, 0x10a0d, 0x10a0d, 0x10a0f, + 0x10a0f, 0x10a38, 0x10a3a, 0x10a3f, 0x10a3f, 0x11046, 0x11046, 0x110b9, 0x110ba, 0x1d165, 0x1d169, 0x1d16d, 0x1d172, + 0x1d17b, 0x1d182, 0x1d185, 0x1d18b, 0x1d1aa, 0x1d1ad, 0x1d242, 0x1d244, + ]; + const oneOfEachClass = [ + 0x334, 0x93c, 0x3099, 0x94d, 0x5b0, 0x5b1, 0x5b2, 0x5b3, 0x5b4, 0x5b5, 0x5b6, 0x5b7, 0x5b8, 0x5b9, 0x5bb, 0x5bc, + 0x5bd, 0x5bf, 0x5c1, 0x5c2, 0xfb1e, 0x64b, 0x64c, 0x64d, 0x618, 0x619, 0x61a, 0x651, 0x652, 0x670, 0x711, 0xc55, + 0xc56, 0xe38, 0xe48, 0xeb8, 0xec8, 0xf71, 0xf72, 0xf74, 0x321, 0x1dce, 0x31b, 0x302a, 0x316, 0x59a, 0x302e, 0x1d16d, + 0x5ae, 0x305, 0x315, 0x35c, 0x35d, + ]; + // U+0345 has the highest class. It composes with U+1FB6, as nothing else does, if the classes of the two + // marks between them rise. + function compared(references: number[]) { + const marks = runsOfMarks.flatMap((first, index) => (index % 2 ? [] : range(first, runsOfMarks[index + 1]))); + const characters = new Uint32Array(references.length * marks.length * 10).fill(0x5f); + let at = 0; + for (const reference of references) { + for (const mark of marks) { + characters.set([0x1fb6, reference, mark, 0x345, 0x5f, 0x1fb6, mark, reference, 0x345], at); + at += 10; + } + } + return name(characters); + } + // Finds a class in a character that has none. Behind U+05AE, whose class is exempt from the rule that classes + // rise, anything counts as a mark, and U+0334 joins it only behind class 0 or the exempt one. Behind U+0334 + // only a higher class counts. + function unclassed(middles: number[]) { + const characters = new Uint32Array(middles.length * 11).fill(0x5f); + let at = 0; + for (const middle of middles) { + characters.set([0x1fb6, 0x5ae, middle, 0x334, 0x345, 0x5f, 0x1fb6, 0x334, middle, 0x345], at); + at += 11; + } + return name(characters); + } + // With the character on either side of them. + const syllables = range(0xabff, 0xd7a4); + + return { + "European marks": () => pairs(blocks(0, 1, 2, 3, 4, 0x1e, 0x1f, 0x21, 0x22), european), + "marks of other scripts": () => Buffer.concat(others.map(mark => pairs(blocks(mark >> 8), [mark]))), + "class 7": () => between(0x928, 0x93c, withMarks()), + "class 8": () => between(0x304b, 0x3099, withMarks()), + "class 202": () => between(0x61, 0x328, withMarks()), + "class 216": () => between(0x6f, 0x31b, withMarks()), + "class 220": () => between(0x61, 0x323, withMarks()), + "class 230 in the first plane": () => between(0x61, 0x301, blocks(...range(0, 0xff))), + "class 230 in the second plane": () => between(0x61, 0x301, blocks(...range(0x100, 0x1ff))), + "class 240": () => between(0x3b1, 0x345, withMarks()), + ...Object.fromEntries( + [0, 1, 2, 3].map(quarter => [ + `classes against each other, ${quarter + 1} of 4`, + () => compared(oneOfEachClass.slice(quarter * 14, quarter * 14 + 14)), + ]), + ), + // The three blocks more hold a neighbour of a mark. + "characters without a class": () => unclassed(withMarks().concat(blocks(0x02, 0x1e, 0x2e))), + "Hangul consonants and vowels": () => pairs(range(0x10ff, 0x1113), range(0x1160, 0x1176)), + "Hangul syllables and final consonants": () => + pairs( + syllables.filter(syllable => (syllable - 0xac00) % 28 === 0), + range(0x11a7, 0x11c3), + ), + "Hangul syllables": () => pairs(syllables, [0x11a8]), + }; +} + +/** Every archive that `readTar` is tried on. */ +export async function readerArchives() { + const stored = async (file: string, name: string) => + (await loadCases<{ data: string }>(join(import.meta.dir, file))).map((c, index) => ({ + name: `${name} ${index}`, + archive: Buffer.from(c.data, "base64"), + })); + return [ + ...(await stored("cases.json.gz", "case")), + ...(await stored("reader-cases.json.gz", "reader case")), + ...madeArchives(), + ...archivesForLibarchiveBranches(), + ]; +} + +export type WriterCase = { + files: { + /** Hex. */ + path: string; + /** The arguments of `contents`. */ + length: number; + kind: number; + seed: number; + /** What the header announces, which need not be `length`. */ + size: string; + perm: number; + mtime: string; + }[]; + windows: boolean; + gzipLevel: number | null; + writeSize: number; + finish: boolean; +} & ({ expected: string } | { length: number; sha256: string }); + +/** What a file of a `WriterCase` holds. */ +export function contents(length: number, kind: number, seed: number) { + if (kind === 2) return Buffer.alloc(length, "const x = require('y');\n"); + const bytes = Buffer.alloc(length); + for (let i = 0; i < length; i++) { + seed = (Math.imul(seed, 1103515245) + 12345) >>> 0; + bytes[i] = kind === 1 ? i / 7 : seed >>> 16; + } + return bytes; +} + +/** The arguments of a record.ts, which has to be told that it may replace what libarchive did. */ +export function recordArguments() { + const args = process.argv.slice(2); + if (!process.versions.libarchive && !args.includes("--accept")) { + throw new Error( + `Bun ${Bun.version} does not use libarchive, so this would record the code under test. ` + + "Run it with Bun 1.4.4, or pass --accept after a change that is meant.", + ); + } + return args.filter(arg => arg !== "--accept"); +} + +export function tarHeader( + name: Buffer | string, + size: number, + type = "0", + linkname = "", + /** Changes the header before its checksum is taken. */ + change: (header: Buffer) => void = () => {}, +) { + const h = Buffer.alloc(512); + Buffer.from(name).copy(h, 0, 0, 100); + h.write("0000644\0", 100); + h.write("0000000\0", 108); + h.write("0000000\0", 116); + h.write(size.toString(8).padStart(11, "0") + "\0", 124); + h.write("00000000000\0", 136); + h.fill(" ", 148, 156); + h.write(type, 156); + h.write(linkname, 157); + h.write("ustar\0" + "00", 257); + change(h); + h.write( + h + .reduce((a, b) => a + b, 0) + .toString(8) + .padStart(6, "0") + "\0 ", + 148, + ); + return h; +} + +export const tarPadded = (body: Buffer) => Buffer.concat([body, Buffer.alloc((512 - (body.length % 512)) % 512)]); + +const crc = (bytes: Uint8Array) => Bun.hash.crc32(bytes).toString(16).padStart(8, "0"); + +export async function observeFiles(archive: Uint8Array): Promise { + let map: Map; + try { + map = await new Bun.Archive(archive).files(); + } catch { + return "rejected\n"; + } + let out = ""; + for (const [name, file] of map) { + out += `${JSON.stringify(name)} name=${JSON.stringify(file.name)} size=${file.size} mtime=${file.lastModified} crc=${crc(await file.bytes())}\n`; + } + return out; +} + +/** Everything under `root` + `relative`. Makes it all readable on the way. */ +export function tree(root: Buffer, relative: Buffer, out: string[]) { + const here = Buffer.concat([root, relative]); + for (const name of readdirSync(here, { encoding: "buffer" }).sort(Buffer.compare)) { + const child = Buffer.concat([relative, Buffer.from("/"), name]); + const path = Buffer.concat([root, child]); + const stat = lstatSync(path); + const shown = `${JSON.stringify(child.latin1Slice())} ${(stat.mode & 0o777).toString(8)}`; + if (stat.isSymbolicLink()) { + out.push( + `link ${JSON.stringify(child.latin1Slice())} -> ${JSON.stringify(readlinkSync(path, "buffer").latin1Slice())}`, + ); + } else if (stat.isDirectory()) { + out.push(`dir ${shown}`); + chmodSync(path, 0o700); + tree(root, child, out); + } else { + chmodSync(path, 0o600); + out.push(`file ${shown} nlink=${stat.nlink} size=${stat.size} crc=${crc(readFileSync(path))}`); + } + } +} + +/** `into` must not exist yet. What a rejected extraction leaves behind is part of the result. */ +export async function observeExtract(archive: Uint8Array, into: string): Promise { + mkdirSync(into); + const out: string[] = []; + try { + out.push(`extracted ${await new Bun.Archive(archive).extract(into)}`); + } catch { + out.push("rejected"); + } + tree(Buffer.from(into), Buffer.alloc(0), out); + return out.join("\n") + "\n"; +} + +const text = (field: Uint8Array) => { + const end = field.indexOf(0); + return Buffer.from(end < 0 ? field : field.subarray(0, end)); +}; + +/** + * The headers of a tar made by Bun.Archive, then a checksum of all of it. Entries are stamped with + * the current time, so that field is checked and blanked first, with the header checksum it feeds. + */ +export function observeTar(tar: Uint8Array, notBefore: number): string { + const bytes = Buffer.from(tar); + let out = `${bytes.length} bytes\n`; + for (let at = 0; at + 512 <= bytes.length;) { + const header = bytes.subarray(at, at + 512); + if (header.every(byte => byte === 0)) break; + const stored = parseInt(text(header.subarray(148, 156)).toString(), 8); + header.fill(" ", 148, 156); + const sum = header.reduce((a, b) => a + b, 0); + const mtime = parseInt(text(header.subarray(136, 148)).toString(), 8); + const now = Math.floor(Date.now() / 1000); + header.fill(0, 136, 156); + const size = parseInt(text(header.subarray(124, 136)).toString(), 8); + const type = String.fromCharCode(header[156]); + out += `type=${type} name=${text(header.subarray(0, 100)).toString("hex")} prefix=${text(header.subarray(345, 500)).toString("hex")} mode=${text(header.subarray(100, 108))} size=${size}`; + out += ` checksum=${stored === sum ? "ok" : "bad"} mtime=${mtime >= notBefore && mtime <= now ? "now" : mtime}\n`; + if (type === "x") out += ` pax=${bytes.subarray(at + 512, at + 512 + size).toString("hex")}\n`; + at += 512 + Math.ceil(size / 512) * 512; + } + return out + `crc=${crc(bytes)}\n`; +} diff --git a/test/js/bun/archive-golden/reader-cases.json.gz b/test/js/bun/archive-golden/reader-cases.json.gz new file mode 100644 index 000000000000..6e92720f0b6d Binary files /dev/null and b/test/js/bun/archive-golden/reader-cases.json.gz differ diff --git a/test/js/bun/archive-golden/reader.json.gz b/test/js/bun/archive-golden/reader.json.gz new file mode 100644 index 000000000000..028dedf13fff Binary files /dev/null and b/test/js/bun/archive-golden/reader.json.gz differ diff --git a/test/js/bun/archive-golden/record-reader.ts b/test/js/bun/archive-golden/record-reader.ts new file mode 100644 index 000000000000..118f91126219 --- /dev/null +++ b/test/js/bun/archive-golden/record-reader.ts @@ -0,0 +1,33 @@ +// Fills reader.json.gz with what the tar reader of the Bun that runs this script reports about the +// archives of `readerArchives()`, in every mode: +// +// bun record-reader.ts --accept +// +// No Bun that read archives with libarchive has `readTar`. The logs in the repository were compared +// with what libarchive itself reports, line by line, when they were recorded. Only the wording +// behind `error:` and `refused:` is Bun's own, and the permissions of an entry that has an ACL. +import { readTar } from "bun:internal-for-testing"; +import { join } from "node:path"; +import { readerArchives, readerModes, type ReaderFixture } from "./observe"; + +if (!process.argv.includes("--accept")) { + throw new Error("This records the code under test. Pass --accept after a change that is meant."); +} + +const cases = await readerArchives(); +const indexes = new Map(); +const fixture: ReaderFixture = { modes: readerModes, logs: [], cases: [] }; +for (const { archive } of cases) { + fixture.cases.push( + readerModes.map(mode => { + const log = readTar(archive, mode); + if (!indexes.has(log)) indexes.set(log, fixture.logs.push(log) - 1); + return indexes.get(log)!; + }), + ); +} +await Bun.write( + join(import.meta.dir, "reader.json.gz"), + Bun.gzipSync(Buffer.from(JSON.stringify([fixture])), { level: 9 }), +); +console.log(`recorded ${cases.length} cases in ${readerModes.length} modes, ${fixture.logs.length} different logs`); diff --git a/test/js/bun/archive-golden/record.ts b/test/js/bun/archive-golden/record.ts new file mode 100644 index 000000000000..df729450ed2a --- /dev/null +++ b/test/js/bun/archive-golden/record.ts @@ -0,0 +1,28 @@ +// Fills in the expected results of cases.json.gz from the Bun that runs this script: +// +// bun record.ts [--accept] [new-cases.json] +// +// The results in the repository come from the last Bun that read archives with libarchive (1.4.4). +import { tempDir } from "harness"; +import { join } from "node:path"; +import { loadCases, observeExtract, observeFiles, recordArguments, type Case } from "./observe"; + +const fixture = join(import.meta.dir, "cases.json.gz"); +const [newCases] = recordArguments(); +const cases: Case[] = newCases ? await Bun.file(newCases).json() : await loadCases(fixture); + +process.umask(0o022); +using dir = tempDir("archive-golden", {}); +const recorded: Case[] = []; +for (const [index, { data, mac, win }] of cases.entries()) { + const archive = Buffer.from(data, "base64"); + recorded.push({ + data, + mac, + win, + files: await observeFiles(archive), + extract: await observeExtract(archive, join(String(dir), String(index))), + }); +} +await Bun.write(fixture, Bun.gzipSync(Buffer.from(JSON.stringify(recorded)), { level: 9 })); +console.log(`recorded ${recorded.length} cases with Bun ${Bun.version}`); diff --git a/test/js/bun/archive-golden/writer.json.gz b/test/js/bun/archive-golden/writer.json.gz new file mode 100644 index 000000000000..fd90ed292e48 Binary files /dev/null and b/test/js/bun/archive-golden/writer.json.gz differ diff --git a/test/js/bun/archive.test.ts b/test/js/bun/archive.test.ts index 98da1af4e6fd..4a7d6e941d53 100644 --- a/test/js/bun/archive.test.ts +++ b/test/js/bun/archive.test.ts @@ -680,7 +680,7 @@ describe("Bun.Archive", () => { }); test("extracts entries whose mode field has bits above the permission bits", async () => { - // libarchive hands back whatever the header's mode field encoded, and a + // The tar reader hands back whatever the header's mode field encoded, and a // base-256 field can encode far more than the 12 mode bits. Only the // permission bits may reach mkdirat()/openat(); the rest are dropped. const high = 1n << 31n; @@ -757,9 +757,51 @@ describe("Bun.Archive", () => { expect(lstatSync(linkPath).isSymbolicLink()).toBe(true); expect(readlinkSync(linkPath)).toBe("index.js"); }); + + // `tar --acls` stores these. GNU tar and node-tar take the permissions from the header. + describe("ACL records", () => { + const withAcl = (acl: string, mode: string, typeflag = "0", name = "run.sh") => { + const body = ` SCHILY.acl.access=${acl}\n`; + let length = body.length + 1; + while (String(length).length + body.length !== length) length++; + const records = Buffer.from(`${length}${body}`); + return Buffer.concat([ + ustarHeader("PaxHeaders/0", records.length, "x"), + records, + Buffer.alloc(512 - records.length), + ustarHeader(name, 0, typeflag, { mode: Buffer.from(mode + "\0") }), + Buffer.alloc(1024), + ]); + }; + + test.skipIf(isWindows)("do not change the permissions of a file", async () => { + using dir = tempDir("archive-acl-file", {}); + await new Bun.Archive(withAcl("user::rw-,group::r--,other::r--", "0000755")).extract(String(dir)); + expect(statSync(join(String(dir), "run.sh")).mode & 0o100).toBe(0o100); + }); + + test.skipIf(isWindows)("do not change the permissions of a directory", async () => { + using dir = tempDir("archive-acl-dir", {}); + await new Bun.Archive(withAcl("user::---,group::---,other::---", "0000755", "5", "d/")).extract(String(dir)); + expect(statSync(join(String(dir), "d")).mode & 0o700).toBe(0o700); + }); + }); }); describe("corrupted archives", () => { + test("a damaged block between two files is skipped, and Bun.inspect counts what files() returns", async () => { + const archive = new Bun.Archive( + Buffer.concat([ + ustarEntry("a", Buffer.from("a")), + Buffer.alloc(512, "junk"), + ustarEntry("b", Buffer.from("b")), + Buffer.alloc(1024), + ]), + ); + expect([...(await archive.files()).keys()]).toEqual(["a", "b"]); + expect(Bun.inspect(archive)).toContain("files: 2"); + }); + test("throws when extracting corrupted archive data", async () => { // Create garbage data that's not a valid archive const corruptedData = new Uint8Array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); @@ -824,7 +866,7 @@ describe("Bun.Archive", () => { test("files() does not leak entries when readData fails mid-stream", async () => { // Regression test: a tarball with two valid 64KB entries followed by a third - // entry whose header advertises 64KB but whose data is truncated. libarchive + // entry whose header advertises 64KB but whose data is truncated. The reader // successfully reads the first two entries (allocating path + data for each), // then readData() fails on the third. The already-collected entries must be // freed on that error path; previously they were leaked on every call. @@ -1560,7 +1602,7 @@ describe("Bun.Archive", () => { }); test.skipIf(isWindows)("files() keys an entry whose name is not even valid UTF-8", async () => { - // "foo\xff\xfe.txt": 0xFF and 0xFE are never valid UTF-8, so libarchive + // "foo\xff\xfe.txt": 0xFF and 0xFE are never valid UTF-8, so the reader // hands the raw header bytes back verbatim. They must not collapse onto // an empty Map key. const invalidUtf8Name = Buffer.from([0x66, 0x6f, 0x6f, 0xff, 0xfe, 0x2e, 0x74, 0x78, 0x74]); @@ -1599,6 +1641,63 @@ describe("Bun.Archive", () => { expect(await dataFile!.text()).toBe(JSON.stringify({ foo: "bar" })); }); + describe.each([ + ["a tar", (tar: Buffer) => tar], + ["a tar.gz", (tar: Buffer) => Buffer.from(Bun.gzipSync(tar))], + ])("the files of %s", (_, wrap) => { + const a = Buffer.alloc(3000, "a"); + const b = randomBytes(70_000); + const archive = () => + new Bun.Archive( + wrap(Buffer.concat([ustarEntry("a", a), ustarEntry("empty", Buffer.alloc(0)), ustarEntry("b", b), end])), + ); + const end = Buffer.alloc(1024); + + test("can be read in every way a File can", async () => { + const files = await archive().files(); + expect([...files].map(([name, file]) => [name, file.name, file.size])).toEqual([ + ["a", "a", a.length], + ["empty", "empty", 0], + ["b", "b", b.length], + ]); + expect(await files.get("a")!.text()).toBe(a.toString()); + expect(await files.get("empty")!.text()).toBe(""); + expect(Buffer.from(await files.get("b")!.arrayBuffer())).toEqual(b); + expect(Buffer.from(await files.get("b")!.bytes())).toEqual(b); + expect(Buffer.from(await new Response(files.get("b")!.stream()).arrayBuffer())).toEqual(b); + expect(Buffer.from(await files.get("b")!.slice(10, 5000).arrayBuffer())).toEqual(b.subarray(10, 5000)); + expect(Buffer.from(await new Blob([files.get("a")!, files.get("b")!]).arrayBuffer())).toEqual( + Buffer.concat([a, b]), + ); + }); + + test("print like any other File", async () => { + const [whole, one] = [await archive().files(), await archive().files("a")]; + const shown = (file: Blob) => Bun.inspect(file).replace(/\s+/g, " "); + expect(shown(whole.get("a")!)).toBe('File (3.0 KB) { name: "a" }'); + expect(shown(one.get("a")!)).toBe(shown(whole.get("a")!)); + expect(shown(whole.get("a")!.slice(1, 3))).toContain("offset: 1 "); + }); + + test("outlive the archive", async () => { + const files = await archive().files(); + Bun.gc(true); + expect(Buffer.from(await files.get("b")!.arrayBuffer())).toEqual(b); + }); + + test("are the same whether one or all are asked for", async () => { + const [[name, file], ...rest] = await archive().files("a"); + expect(rest).toEqual([]); + expect([name, await file.text()]).toEqual(["a", a.toString()]); + }); + + test("can be written to disk", async () => { + using dir = tempDir("archive-files-write", {}); + await Bun.write(join(String(dir), "b"), (await archive().files()).get("b")!); + expect(readFileSync(join(String(dir), "b"))).toEqual(b); + }); + }); + test("returns empty Map for empty archive", async () => { const archive = new Bun.Archive({}); const files = await archive.files(); diff --git a/test/js/node/process/process.test.js b/test/js/node/process/process.test.js index 781c8c37606b..9633bbbb89f4 100644 --- a/test/js/node/process/process.test.js +++ b/test/js/node/process/process.test.js @@ -754,7 +754,6 @@ it("process.versions", async () => { const depsDir = resolve(import.meta.dir, "../../../../scripts/build/deps"); const deps = { boringssl: "boringssl", - libarchive: "libarchive", mimalloc: "mimalloc", picohttpparser: "picohttpparser", zlib: "zlib", diff --git a/test/leaksan.supp b/test/leaksan.supp index 1403d57c9a9f..14db5147aaa0 100644 --- a/test/leaksan.supp +++ b/test/leaksan.supp @@ -102,7 +102,6 @@ leak:fse_run_loop leak:Zig::NapiClass_ConstructorFunction leak:runtime.webcore.fetch.FetchTasklet.toResponse leak:JSC::jsonProtoFuncStringify -leak:libarchive_sys.bindings.Archive.readNew leak:Zig::SourceProvider::create leak:fromErrorInstance