diff --git a/docs/pm/cli/install.mdx b/docs/pm/cli/install.mdx index 81fa16e03bf9..8a6f330d0fe1 100644 --- a/docs/pm/cli/install.mdx +++ b/docs/pm/cli/install.mdx @@ -192,6 +192,8 @@ bun install --offline `--prefer-offline` is also what `install.prefer = "offline"` in `bunfig.toml` selects; `install.offline = true` is the config form of `--offline`. With a complete restored cache, `--offline --frozen-lockfile` makes a CI install fully deterministic and network-free; `--prefer-offline` still fetches whatever the cache is missing. +An install is only certain to cache the packages that it places in `node_modules`. To warm the cache for an `--offline` install, run an install on the same platform that places at least the same packages. An install with `--production` or `--omit` does not cache the groups it leaves out. + See [lockfile](/pm/lockfile) for more on `bun.lock`. --- diff --git a/src/install/PackageManager.rs b/src/install/PackageManager.rs index e2c97f58844d..d1075ccba9a3 100644 --- a/src/install/PackageManager.rs +++ b/src/install/PackageManager.rs @@ -2497,6 +2497,7 @@ fn init_with_runtime_once( max_concurrent_lifecycle_scripts: cli .concurrent_scripts .unwrap_or((cpu_count * 2) as usize), + runtime_auto_install: true, ..Default::default() } ); diff --git a/src/install/PackageManager/PackageManagerEnqueue.rs b/src/install/PackageManager/PackageManagerEnqueue.rs index a7c46c834667..ea9ec05aa566 100644 --- a/src/install/PackageManager/PackageManagerEnqueue.rs +++ b/src/install/PackageManager/PackageManagerEnqueue.rs @@ -2417,6 +2417,17 @@ fn get_or_put_resolved_package_with_find_result( let this: &mut PackageManager = unsafe { &mut *guard.0 }; // The scopeguard runs on ALL exits, never disarmed. + // `remote_package_features` only lacks groups that no remote package has. + let placed = behavior.is_placed(this.options.local_package_features); + // `is_filtered_dependency_or_workspace` filters this package and everything below it. + let unplaced = !this.options.runtime_auto_install + && (!placed + || package.is_disabled(this.options.cpu, this.options.os) + || this.lockfile.is_in_unplaced_subtree(dependency_id)); + if unplaced { + this.lockfile.mark_unplaced_subtree(package.dependencies); + } + // non-null if the package is in "patchedDependencies" let mut name_and_version_hash: Option = None; let mut patchfile_hash: Option = None; @@ -2434,6 +2445,12 @@ fn get_or_put_resolved_package_with_find_result( is_first_time: true, task: None, }), + // If a placed dependency needs it too, the install phase fetches and patches it. + _ if unplaced => Some(ResolvedPackageResult { + package, + is_first_time: true, + task: None, + }), // Do we need to download the tarball? install::PreinstallState::Extract => 'extract: { // Skip tarball download when prefetch_resolved_tarballs is disabled (e.g., --lockfile-only) diff --git a/src/install/PackageManager/PackageManagerOptions.rs b/src/install/PackageManager/PackageManagerOptions.rs index e89a22f8b2e0..03ad74e180cd 100644 --- a/src/install/PackageManager/PackageManagerOptions.rs +++ b/src/install/PackageManager/PackageManagerOptions.rs @@ -73,6 +73,9 @@ pub struct Options { pub(crate) lockfile_only: bool, + /// Set by `init_with_runtime`: no install phase, so only a resolve downloads a package. + pub(crate) runtime_auto_install: bool, + // `bun pm version` command options pub git_tag_version: bool, pub allow_same_version: bool, @@ -161,6 +164,7 @@ impl Default for Options { ca_file_name: b"", save_text_lockfile: None, lockfile_only: false, + runtime_auto_install: false, git_tag_version: true, allow_same_version: false, preid: b"", diff --git a/src/install/lockfile.rs b/src/install/lockfile.rs index e44ef3156924..e753842367a5 100644 --- a/src/install/lockfile.rs +++ b/src/install/lockfile.rs @@ -734,7 +734,7 @@ impl Lockfile { let dep = &self.buffers.dependencies[dep_id as usize]; - dep.behavior.is_bundled() || !dep.behavior.is_enabled(features) + !dep.behavior.is_placed(features) } pub fn resolve_catalog_dependency(&self, dep: &Dependency) -> Option { @@ -2196,6 +2196,26 @@ impl Lockfile { self.exact_pinned.set(i); } + /// See `Scratch::unplaced_subtree`. + pub(crate) fn mark_unplaced_subtree(&mut self, dependencies: DependencySlice) { + let range = bun_collections::bit_set::Range { + start: dependencies.begin() as usize, + end: dependencies.end() as usize, + }; + let unplaced_subtree = &mut self.scratch.unplaced_subtree; + if unplaced_subtree.bit_length() < range.end { + bun_core::handle_oom(unplaced_subtree.resize(range.end, false)); + } + unplaced_subtree.set_range_value(range, true); + } + + #[inline] + pub(crate) fn is_in_unplaced_subtree(&self, id: DependencyID) -> bool { + self.scratch + .unplaced_subtree + .is_set_allow_out_of_bound(id as usize, false) + } + pub(crate) fn get_package_id( &self, name_hash: u64, @@ -2539,6 +2559,8 @@ impl Lockfile { pub struct Scratch { pub(crate) duplicate_checker_map: DuplicateCheckerMap, pub(crate) dependency_list_queue: DependencyQueue, + /// `bit[dependency_id]`: this resolve reached it below a dependency that the installers filter. + pub(crate) unplaced_subtree: DynamicBitSet, } pub(crate) type DuplicateCheckerMap = @@ -2550,6 +2572,7 @@ impl Scratch { Scratch { dependency_list_queue: DependencyQueue::init(), duplicate_checker_map: DuplicateCheckerMap::default(), + unplaced_subtree: DynamicBitSet::default(), } } } diff --git a/src/install/lockfile/Tree.rs b/src/install/lockfile/Tree.rs index 1a36c544db77..3d99cadfed3f 100644 --- a/src/install/lockfile/Tree.rs +++ b/src/install/lockfile/Tree.rs @@ -590,17 +590,13 @@ pub(crate) fn is_filtered_dependency_or_workspace( return true; } - if dep.behavior.is_bundled() { - return true; - } - let dep_features = if parent_res.tag.is_local_package() { manager.options.local_package_features } else { manager.options.remote_package_features }; - if !dep.behavior.is_enabled(dep_features) { + if !dep.behavior.is_placed(dep_features) { return true; } diff --git a/src/install_types/resolver_hooks.rs b/src/install_types/resolver_hooks.rs index 79ed00b6764a..4a528a15776f 100644 --- a/src/install_types/resolver_hooks.rs +++ b/src/install_types/resolver_hooks.rs @@ -260,6 +260,12 @@ impl Behavior { || (features.workspaces && self.is_workspace()) } + /// False when the installers filter the dependency, and with it everything below. + #[inline] + pub fn is_placed(self, features: Features) -> bool { + !self.is_bundled() && self.is_enabled(features) + } + pub fn cmp(self, rhs: Self) -> core::cmp::Ordering { use core::cmp::Ordering::*; if self == rhs { diff --git a/test/cli/install/bun-install-registry.test.ts b/test/cli/install/bun-install-registry.test.ts index 08bdf5c9bd11..b378627540d1 100644 --- a/test/cli/install/bun-install-registry.test.ts +++ b/test/cli/install/bun-install-registry.test.ts @@ -1587,6 +1587,283 @@ describe("bundledDependencies", () => { }); }); +// A fresh resolve starts to download tarballs before the install phase runs. +// It must ask for the tarballs that an install from its lockfile asks for, and +// for no others. The installers place nothing below a bundled dependency, a +// package for another platform, or a group that --production or --omit turns off. +describe.concurrent("tarballs of a fresh resolve", () => { + const packages: Record = { + // The tarball of outer ships bd and tr in its node_modules. + "outer": { dependencies: { bd: "1.0.0" }, bundleDependencies: ["bd"] }, + "bd": { dependencies: { tr: "1.0.0" } }, + // No CI machine has this cpu. + "uses-native": { optionalDependencies: { "native-wasm": "1.0.0" } }, + "native-wasm": { cpu: ["wasm32"], dependencies: { tr: "1.0.0" } }, + "tool": { dependencies: { tr: "1.0.0" } }, + "uses-tr": { dependencies: { tr: "1.0.0" } }, + "tr": {}, + "leaf": {}, + }; + const packageJsonOf = (name: string) => JSON.stringify({ name, version: "1.0.0", ...packages[name] }); + const files: Record> = { + "outer": { + "index.js": `module.exports = require("bd");`, + "node_modules/bd/package.json": packageJsonOf("bd"), + "node_modules/bd/index.js": `module.exports = "bundled bd, " + require("tr");`, + "node_modules/tr/package.json": packageJsonOf("tr"), + "node_modules/tr/index.js": `module.exports = "bundled tr";`, + }, + "bd": { "index.js": `module.exports = "registry bd, " + require("tr");\n` }, + "tr": { "index.js": `module.exports = "registry tr";\n` }, + "uses-tr": { "index.js": `module.exports = require("tr");` }, + }; + const tarballs: Record = {}; + + beforeAll(async () => { + for (const name of Object.keys(packages)) { + const archive: Record = { "package/package.json": packageJsonOf(name) }; + for (const [path, contents] of Object.entries(files[name] ?? {})) archive[`package/${path}`] = contents; + tarballs[name] = await new Bun.Archive(archive, { compress: "gzip" }).bytes(); + } + }); + + type Requests = { tarballs: string[]; manifests: Set }; + + /** Serves the packages above and records what it is asked for. `holdManifest` can delay a manifest. */ + function serveRegistry(requests: Requests, holdManifest: (name: string) => Promise | void = () => {}) { + return Bun.serve({ + port: 0, + async fetch(request) { + const { origin, pathname } = new URL(request.url); + const tarballOf = pathname.match(/^\/(.+)-1\.0\.0\.tgz$/)?.[1]; + const name = tarballOf ?? pathname.slice(1); + if (!(name in packages)) return new Response("not found", { status: 404 }); + if (tarballOf) { + requests.tarballs.push(name); + return new Response(tarballs[name]); + } + requests.manifests.add(name); + await holdManifest(name); + const integrity = "sha512-" + new Bun.CryptoHasher("sha512").update(tarballs[name]).digest("base64"); + return Response.json({ + name, + "dist-tags": { latest: "1.0.0" }, + versions: { + "1.0.0": { + name, + version: "1.0.0", + ...packages[name], + dist: { tarball: `${origin}/${name}-1.0.0.tgz`, integrity }, + }, + }, + }); + }, + }); + } + + function envFor(cwd: string) { + const tmp = join(cwd, ".bun-tmp"); + return { ...env, BUN_INSTALL_CACHE_DIR: join(cwd, ".bun-cache"), BUN_TMPDIR: tmp, TMPDIR: tmp, TEMP: tmp }; + } + + type Project = { + manifest: object; + files?: Record; + args?: string[]; + /** The tarballs that the installers need, sorted. */ + tarballs: string[]; + /** What `require(name)` returns from the root after the install. */ + requires?: Record; + /** Hold the manifest of `uses-tr` until the other dependency has asked for the manifest of `tr`, so that one resolves tr@1.0.0 first. */ + usesTrResolvesLast?: boolean; + }; + + /** + * Installs the project three times. A fresh resolve with a cold cache takes + * its manifests from the registry. A fresh resolve that finds the manifests + * in the cache resolves before anything else can finish, a patch hash for + * one. The last install reads the lockfile, with a cold cache (--production + * saves no lockfile and resolves again). + */ + async function installEachWay(linker: string, project: Project) { + const trRequested = Promise.withResolvers(); + const requests: Requests = { tarballs: [], manifests: new Set() }; + await using registry = serveRegistry(requests, name => { + if (name === "tr") trRequested.resolve(); + if (name === "uses-tr" && project.usesTrResolvesLast) return trRequested.promise; + }); + using dir = tempDir("fresh-resolve-tarballs", { + ...project.files, + "package.json": JSON.stringify({ name: "foo", ...project.manifest }), + "bunfig.toml": Bun.TOML.stringify({ install: { registry: registry.url.href, linker } }), + }); + const cwd = String(dir); + const cache = join(cwd, ".bun-cache"); + + const asked: Record = {}; + for (const phase of ["fresh resolve", "fresh resolve, manifests in the cache", "from the lockfile"]) { + await rm(join(cwd, "node_modules"), { recursive: true, force: true }); + if (phase === "fresh resolve, manifests in the cache") { + // bun install does not wait for the manifest cache writes before it exits. + const deadline = Date.now() + 5_000; + let entries = await readdirSorted(cache); + while ( + entries.filter(entry => entry.endsWith(".npm")).length < requests.manifests.size && + Date.now() < deadline + ) { + await Bun.sleep(10); + entries = await readdirSorted(cache); + } + await Promise.all([ + rm(join(cwd, "bun.lock"), { force: true }), + ...entries.filter(entry => !entry.endsWith(".npm")).map(entry => rm(join(cache, entry), { recursive: true })), + ]); + } else { + await rm(cache, { recursive: true, force: true }); + } + requests.tarballs.length = 0; + await using proc = spawn({ + cmd: [bunExe(), "install", ...(project.args ?? [])], + cwd, + stdout: "pipe", + stderr: "pipe", + env: envFor(cwd), + }); + const [stderr, exitCode] = await Promise.all([proc.stderr.text(), proc.exited]); + expect(stderr).not.toContain("error:"); + expect(exitCode).toBe(0); + asked[phase] = requests.tarballs.toSorted(); + } + expect(asked).toEqual({ + "fresh resolve": project.tarballs, + "fresh resolve, manifests in the cache": project.tarballs, + "from the lockfile": project.tarballs, + }); + + const names = Object.keys(project.requires ?? {}); + if (names.length === 0) return; + await using proc = spawn({ + cmd: [bunExe(), "-p", `JSON.stringify([${names.map(name => `require(${JSON.stringify(name)})`).join(", ")}])`], + cwd, + stdout: "pipe", + stderr: "pipe", + env: envFor(cwd), + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + expect(stderr).toBe(""); + expect(JSON.parse(stdout)).toEqual(Object.values(project.requires!)); + expect(exitCode).toBe(0); + } + + /** A patch for the index.js that the registry copy of `name` has. */ + const patchOf = (name: string) => `diff --git a/index.js b/index.js +index 0000000..1111111 100644 +--- a/index.js ++++ b/index.js +@@ -1 +1 @@ +-${files[name]["index.js"]}+${files[name]["index.js"].replace("registry", "patched")}`; + + const projects: Record = { + "below a bundled dependency": { + manifest: { dependencies: { outer: "1.0.0" } }, + tarballs: ["outer"], + requires: { outer: "bundled bd, bundled tr" }, + }, + "below a package for another platform": { + manifest: { dependencies: { "uses-native": "1.0.0" } }, + tarballs: ["uses-native"], + }, + "below a devDependency with --production": { + manifest: { dependencies: { leaf: "1.0.0" }, devDependencies: { tool: "1.0.0" } }, + args: ["--production"], + tarballs: ["leaf"], + }, + "below a devDependency with --omit=dev": { + manifest: { dependencies: { leaf: "1.0.0" }, devDependencies: { tool: "1.0.0" } }, + args: ["--omit=dev"], + tarballs: ["leaf"], + }, + "below an optionalDependency with --omit=optional": { + manifest: { dependencies: { leaf: "1.0.0" }, optionalDependencies: { tool: "1.0.0" } }, + args: ["--omit=optional"], + tarballs: ["leaf"], + }, + "below a peerDependency with --omit=peer": { + manifest: { dependencies: { leaf: "1.0.0" }, peerDependencies: { tool: "1.0.0" } }, + args: ["--omit=peer"], + tarballs: ["leaf"], + }, + // The install phase downloads a package that was first resolved below a + // dependency the installers do not place, when another dependency needs it. + "first below a bundled dependency, then needed": { + manifest: { dependencies: { outer: "1.0.0", "uses-tr": "1.0.0" } }, + tarballs: ["outer", "tr", "uses-tr"], + requires: { outer: "bundled bd, bundled tr", "uses-tr": "registry tr" }, + usesTrResolvesLast: true, + }, + "first below a package for another platform, then needed": { + manifest: { dependencies: { "uses-native": "1.0.0", "uses-tr": "1.0.0" } }, + tarballs: ["tr", "uses-native", "uses-tr"], + requires: { "uses-tr": "registry tr" }, + usesTrResolvesLast: true, + }, + "first below a devDependency with --production, then needed": { + manifest: { dependencies: { "uses-tr": "1.0.0" }, devDependencies: { tool: "1.0.0" } }, + args: ["--production"], + tarballs: ["tr", "uses-tr"], + requires: { "uses-tr": "registry tr" }, + usesTrResolvesLast: true, + }, + // A package with a patch takes other arms of the resolve: it waits for + // the hash of the patch, then it downloads. + "with a patch, below a bundled dependency": { + manifest: { dependencies: { outer: "1.0.0" }, patchedDependencies: { "bd@1.0.0": "patches/bd.patch" } }, + files: { "patches/bd.patch": patchOf("bd") }, + tarballs: ["outer"], + requires: { outer: "bundled bd, bundled tr" }, + }, + "with a patch, first below a bundled dependency, then needed": { + manifest: { + dependencies: { outer: "1.0.0", "uses-tr": "1.0.0" }, + patchedDependencies: { "tr@1.0.0": "patches/tr.patch" }, + }, + files: { "patches/tr.patch": patchOf("tr") }, + tarballs: ["outer", "tr", "uses-tr"], + requires: { outer: "bundled bd, bundled tr", "uses-tr": "patched tr" }, + usesTrResolvesLast: true, + }, + }; + + for (const linker of ["hoisted", "isolated"]) { + for (const [name, project] of Object.entries(projects)) { + test(`(${linker}) ${name}`, () => installEachWay(linker, project)); + } + } + + // The runtime has no install phase. With --install=force it loads every + // package from the cache, a bundled dependency too. + test("the runtime auto-install downloads a bundled dependency", async () => { + const requests: Requests = { tarballs: [], manifests: new Set() }; + await using registry = serveRegistry(requests); + using dir = tempDir("fresh-resolve-tarballs", { + "bunfig.toml": Bun.TOML.stringify({ install: { registry: registry.url.href } }), + }); + + await using proc = spawn({ + cmd: [bunExe(), "--install=force", "-p", `require("outer@1.0.0")`], + cwd: String(dir), + stdout: "pipe", + stderr: "pipe", + env: envFor(String(dir)), + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + expect(stderr).toBe(""); + expect(stdout).toBe("registry bd, registry tr\n"); + expect(requests.tarballs.toSorted()).toEqual(["bd", "outer", "tr"]); + expect(exitCode).toBe(0); + }); +}); + describe("optionalDependencies", () => { for (const optional of [true, false]) { test(`exit code is ${optional ? 0 : 1} when ${optional ? "optional" : ""} dependency tarball is missing`, async () => {