diff --git a/docs/pm/cli/install.mdx b/docs/pm/cli/install.mdx index a596c7309898..a6b463c240cf 100644 --- a/docs/pm/cli/install.mdx +++ b/docs/pm/cli/install.mdx @@ -202,6 +202,8 @@ bun install --offline An install is only certain to cache the packages that it places in `node_modules`. To warm the cache for an `--offline` install, run an install on the same platform that places at least the same packages. An install with `--production` or `--omit` does not cache the groups it leaves out. +Both flags resolve versions from the registry metadata in the cache. `bun install`, `bun add`, `bun update`, `bun outdated` and `bun audit fix` store the metadata that they fetch, so an offline command after one of them resolves the versions that it saw. `bun install` does not resolve a version that `bun.lock` already holds. If `--offline` resolves a package to a version whose tarball is not in the cache, it reports the package as not in the cache. `bun update --offline` does this after a command that fetched metadata and did not download every new version, such as `bun outdated` or `bun update `. A project with [`minimumReleaseAge`](#minimum-release-age) needs the full metadata. For most packages, only a command that runs with `minimumReleaseAge` stores it. + `--offline` skips an optional dependency that is not cached, and anything below it that is not cached either. A package that an installed package requires is still an error. See [lockfile](/pm/lockfile) for more on `bun.lock`. @@ -532,9 +534,9 @@ bun add --force bun install --force ``` -Bun does not resolve a version that `bun.lock` already holds, and it takes an exact version such as `1.2.3` from the cached manifest, so `--force` sends no request for those. To move a package in `bun.lock` to the new version, run `bun update `, which never reads the manifest cache. +Bun does not resolve a version that `bun.lock` already holds, and it takes an exact version such as `1.2.3` from the cached manifest, so `--force` sends no request for those. To move a package in `bun.lock` to the new version, run `bun update `. Unless an [offline mode](#offline-installs) is on, it reads no cached manifest: it asks the registry for each one and stores the answer in the cache. -`bun install --no-cache` skips the manifest cache for one run. To turn the manifest cache off for a project, set `disableManifest = true` under `[install.cache]` in `bunfig.toml`. See [Global cache](/pm/global-cache). +`bun install --no-cache` skips the manifest cache for one run: it reads no cached manifest and stores none. To turn the manifest cache off for a project, set `disableManifest = true` under `[install.cache]` in `bunfig.toml`. An [offline mode](#offline-installs) resolves from the manifest cache, so it reads and stores manifests with either setting. See [Global cache](/pm/global-cache). To delete the cache: diff --git a/docs/pm/global-cache.mdx b/docs/pm/global-cache.mdx index dec39094f74f..c0489460ad2b 100644 --- a/docs/pm/global-cache.mdx +++ b/docs/pm/global-cache.mdx @@ -16,7 +16,8 @@ dir = "~/.bun/install/cache" # Bun extracts packages to node_modules/.cache instead disable = false -# when true, always resolve the latest versions from the registry +# when true, always resolve the latest versions from the registry, +# and don't store registry metadata in the cache disableManifest = false ``` diff --git a/docs/runtime/bunfig.mdx b/docs/runtime/bunfig.mdx index 74e7bbf633d3..83d272da0882 100644 --- a/docs/runtime/bunfig.mdx +++ b/docs/runtime/bunfig.mdx @@ -1069,7 +1069,8 @@ dir = "~/.bun/install/cache" # Bun extracts packages to node_modules/.cache instead disable = false -# when true, always resolve the latest versions from the registry +# when true, always resolve the latest versions from the registry, +# and don't store registry metadata in the cache disableManifest = false ``` diff --git a/src/install/PackageManager.rs b/src/install/PackageManager.rs index 9197d5ec109b..ac500b0600f0 100644 --- a/src/install/PackageManager.rs +++ b/src/install/PackageManager.rs @@ -2869,10 +2869,7 @@ pub fn init( { manager.options.offline = options::OfflineMode::PreferOffline; // the manifest cache is the data source in this mode (see Options::load) - manager - .options - .enable - .set(options::Enable::MANIFEST_CACHE, true); + manager.options.enable.set_manifest_cache(true); } if let Some(network_concurrency) = cli.network_concurrency { diff --git a/src/install/PackageManager/PackageManagerDirectories.rs b/src/install/PackageManager/PackageManagerDirectories.rs index c6d631fcb460..a6b7a986bab7 100644 --- a/src/install/PackageManager/PackageManagerDirectories.rs +++ b/src/install/PackageManager/PackageManagerDirectories.rs @@ -61,6 +61,17 @@ impl PackageManager { get_temporary_directory(self) } + /// Called on the thread that runs `run_tasks`, so that no pool thread opens the directories. + pub(crate) fn manifest_store(&mut self) -> Option { + if !self.options.enable.manifest_cache_store() { + return None; + } + // If the cache cannot be written, the manifest is still used. It is just not saved. + let tmpdir = try_get_temporary_directory(self).ok()?.handle.fd; + let cache_dir = try_get_cache_directory(self).ok()?; + Some(ManifestStore { tmpdir, cache_dir }) + } + /// Prints `why` and exits. Auto-install only logs `why`, and the import that needed the /// directory then throws it. pub(crate) fn exit_or_log(&mut self, why: &DirectoryError) { @@ -203,6 +214,13 @@ pub struct TemporaryDirectory { pub(crate) name: &'static [u8], } +/// The open directories that a manifest cache file is written through. +#[derive(Clone, Copy)] +pub struct ManifestStore { + pub(crate) tmpdir: Fd, + pub(crate) cache_dir: Fd, +} + // `TemporaryDirectory` is auto-`Send + Sync`: `Dir` wraps `Fd` (an integer), // `ZBox` wraps `Box<[u8]>`, and `&'static [u8]` is `Sync`. No `unsafe impl`. const _: fn() = || { diff --git a/src/install/PackageManager/PackageManagerEnqueue.rs b/src/install/PackageManager/PackageManagerEnqueue.rs index 97e5ca021193..4831f4ca563a 100644 --- a/src/install/PackageManager/PackageManagerEnqueue.rs +++ b/src/install/PackageManager/PackageManagerEnqueue.rs @@ -546,6 +546,7 @@ pub unsafe fn enqueue_parse_npm_package( name: StringOrTinyString, network_task: *mut NetworkTask, ) -> *mut ThreadPool::Task { + let store = this.manifest_store(); // SAFETY: `this` is a live `&mut PackageManager`; `network_task` is a // freshly-vended pool slot whose `'static` reborrow matches the // `Task<'static>` slot lifetime. @@ -561,6 +562,7 @@ pub unsafe fn enqueue_parse_npm_package( crate::package_manager_task::PackageManifestRequest { network: &mut *network_task, name, + store, }, ), }, diff --git a/src/install/PackageManager/PackageManagerOptions.rs b/src/install/PackageManager/PackageManagerOptions.rs index d407a60187a8..215038a35883 100644 --- a/src/install/PackageManager/PackageManagerOptions.rs +++ b/src/install/PackageManager/PackageManagerOptions.rs @@ -706,10 +706,8 @@ impl Options { ) -> crate::Result<()> { let bunfigs = [runtime_bunfigs[0], bun_install_ref, runtime_bunfigs[1]]; if let Some(manifest_cache) = env.get(b"BUN_MANIFEST_CACHE") { - self.enable.set( - Enable::MANIFEST_CACHE, - matches!(manifest_cache, b"1" | b"2"), - ); + self.enable + .set_manifest_cache(matches!(manifest_cache, b"1" | b"2")); self.enable .set(Enable::MANIFEST_CACHE_CONTROL, manifest_cache == b"2"); } @@ -816,11 +814,11 @@ impl Options { if config.disable_cache.unwrap_or(false) { self.enable.set(Enable::CACHE, false); - self.enable.set(Enable::MANIFEST_CACHE, false); + self.enable.set_manifest_cache(false); } if config.disable_manifest_cache.unwrap_or(false) { - self.enable.set(Enable::MANIFEST_CACHE, false); + self.enable.set_manifest_cache(false); } if config.force.unwrap_or(false) { @@ -1076,7 +1074,7 @@ impl Options { self.json_output = cli.json_output; if cli.no_cache { - self.enable.set(Enable::MANIFEST_CACHE, false); + self.enable.set_manifest_cache(false); self.enable.set(Enable::MANIFEST_CACHE_CONTROL, false); } @@ -1277,7 +1275,7 @@ impl Options { // The manifest cache is the data source for --prefer-offline/--offline; keep it on // even where it is otherwise bypassed (`bun update`, `--no-cache`, `--force`). if self.offline != OfflineMode::Online { - self.enable.set(Enable::MANIFEST_CACHE, true); + self.enable.set_manifest_cache(true); } // Prefetching resolved tarballs is a latency optimisation for downloads; under // --offline there is nothing to download and the install phase reports misses. @@ -1331,6 +1329,7 @@ impl Default for Do { bitflags::bitflags! { #[derive(Copy, Clone, PartialEq, Eq)] pub struct Enable: u16 { + /// Lookups read the manifests in the cache. const MANIFEST_CACHE = 1 << 0; const MANIFEST_CACHE_CONTROL = 1 << 1; const CACHE = 1 << 2; @@ -1353,13 +1352,18 @@ bitflags::bitflags! { const GLOBAL_VIRTUAL_STORE = 1 << 9; /// `bun add` was given a group, so an entry that's in another group moves into it. const MOVE_TO_GROUP = 1 << 10; - // _: u5 padding + /// Fetched manifests are stored. `bun update` clears `MANIFEST_CACHE` and keeps this. + const MANIFEST_CACHE_STORE = 1 << 11; + // _: u4 padding } } impl Default for Enable { fn default() -> Self { - Enable::MANIFEST_CACHE | Enable::MANIFEST_CACHE_CONTROL | Enable::CACHE + Enable::MANIFEST_CACHE + | Enable::MANIFEST_CACHE_CONTROL + | Enable::MANIFEST_CACHE_STORE + | Enable::CACHE } } @@ -1423,6 +1427,15 @@ impl Enable { self.contains(Enable::MANIFEST_CACHE_CONTROL) } #[inline] + pub(crate) fn manifest_cache_store(self) -> bool { + self.contains(Enable::MANIFEST_CACHE_STORE) + } + /// The settings that name the manifest cache turn reading and storing on or off together. + #[inline] + pub(crate) fn set_manifest_cache(&mut self, on: bool) { + self.set(Enable::MANIFEST_CACHE | Enable::MANIFEST_CACHE_STORE, on); + } + #[inline] pub(crate) fn fail_early(self) -> bool { self.contains(Enable::FAIL_EARLY) } diff --git a/src/install/PackageManager/runTasks.rs b/src/install/PackageManager/runTasks.rs index bcb7c79f4e6d..5b1aef65ab5c 100644 --- a/src/install/PackageManager/runTasks.rs +++ b/src/install/PackageManager/runTasks.rs @@ -35,7 +35,7 @@ use bun_install::package_manager_task as Task; // Import the *module* under the `Options` name so `Options::LogLevel` resolves as a path // (matches the `Task` module-alias pattern above and `CommandLineArguments.rs`). use super::package_manager_options as Options; -use super::package_manager_options::{Do, Enable}; +use super::package_manager_options::Do; use crate::isolated_install::store as Store; macro_rules! add_error_or_warning_pretty { @@ -882,18 +882,7 @@ fn run_tasks_erased( .insert(name_hash, ManifestEntry::Manifest(manifest)); manager.look_ahead.arrived.push(task.task_id); - if manager.options.enable.contains(Enable::MANIFEST_CACHE) - && let Ok(temp_dir) = directories::try_get_temporary_directory(manager) - { - // reshaped for borrowck — compute the - // `&mut`-taking directory accessors first so the - // shared `scope_for_package_name` / `manifests` - // borrows below do not overlap them. `save_async` - // only needs `&PackageManifest`, so reborrow the - // freshly-inserted entry immutably alongside the - // scope (both `&manager`, no conflict). - let tmp_fd = temp_dir.handle.fd; - let cache_fd = directories::get_cache_directory(manager); + if let Some(store) = manager.manifest_store() { npm::package_manifest::Serializer::save_async( manager .manifests @@ -902,8 +891,8 @@ fn run_tasks_erased( .unwrap() .manifest(), manager.scope_for_package_name(name), - tmp_fd, - cache_fd, + store.tmpdir, + store.cache_dir, ); } diff --git a/src/install/PackageManagerTask.rs b/src/install/PackageManagerTask.rs index c702c723924a..6a29e1ff30e5 100644 --- a/src/install/PackageManagerTask.rs +++ b/src/install/PackageManagerTask.rs @@ -256,10 +256,7 @@ impl<'a> Task<'a> { // concurrently (and concurrently with the main thread), so binding a // long-lived `&mut PackageManager` here would be aliased-`&mut` UB. // Subfields touched cross-thread (`resolve_tasks`, `wake`) are reached - // through raw-ptr/shared accessors below; the few callees whose - // signatures still take `&mut PackageManager` (`get_cache_directory`, - // `get_package_metadata`) are dereferenced inline at the call boundary - // only. + // through raw-ptr/shared accessors below. let manager_ref = this.package_manager.expect("Task.package_manager unset"); let manager: *mut PackageManager = manager_ref.as_mut_ptr(); @@ -322,25 +319,16 @@ impl<'a> Task<'a> { let is_extended_manifest = *is_extended_manifest; // Shared read of `manager.options` (never mutated by worker - // threads) via the `ParentRef` safe `Deref`. Wrap as - // `BackRef` so the `&PackageManager` autoref does not stay - // live across the `&mut *manager` below. - let scope = bun_ptr::BackRef::new( - manager_ref.scope_for_package_name(manifest.name.slice()), - ); + // threads) via the `ParentRef` safe `Deref`. + let scope = manager_ref.scope_for_package_name(manifest.name.slice()); let package_manifest = match npm::Registry::get_package_metadata( - // scope is borrowed from manager.options which is not - // touched by get_package_metadata (only the cache-dir fields are). - scope.get(), + scope, metadata.response, body.slice(), &mut this.log, manifest.name.slice(), loaded_manifest, - // SAFETY: see `manager` decl — short-lived `&mut` at call - // boundary only (callee touches `cache_directory` / - // `get_temporary_directory` lazily). - unsafe { &mut *manager }, + manifest.store, is_extended_manifest, ) { Ok(v) => v, @@ -554,6 +542,8 @@ pub union Request<'a> { } pub struct PackageManifestRequest<'a> { + /// Resolved by `enqueue_parse_npm_package` on the main thread. + pub(crate) store: Option, pub(crate) name: StringOrTinyString, // BORROW_PARAM per LIFETIMES.tsv // TODO: lifetime — see note on `Task<'a>`; likely should demote to `*mut NetworkTask`. diff --git a/src/install/npm.rs b/src/install/npm.rs index 8451545dfb1d..fc5b03fb9581 100644 --- a/src/install/npm.rs +++ b/src/install/npm.rs @@ -731,7 +731,7 @@ pub mod registry { log: &mut bun_ast::Log, package_name: &[u8], loaded_manifest: Option, - package_manager: &mut PackageManager, + store: Option, is_extended_manifest: bool, ) -> Result { match response.status_code { @@ -766,15 +766,12 @@ pub mod registry { manifest_expiry(&response.headers), is_extended_manifest, )? { - // If the cache cannot be written, the manifest is still used. It is just not saved. - if package_manager.options.enable.manifest_cache() - && let Ok(temp_dir) = directories::try_get_temporary_directory(package_manager) - { + if let Some(store) = store { package_manifest::Serializer::save_async( &package, scope, - temp_dir.handle.fd, - package_manager.get_cache_directory(), + store.tmpdir, + store.cache_dir, ); } diff --git a/test/cli/install/bun-audit.test.ts b/test/cli/install/bun-audit.test.ts index 571b0b7ec162..66e4e62c784a 100644 --- a/test/cli/install/bun-audit.test.ts +++ b/test/cli/install/bun-audit.test.ts @@ -122,7 +122,10 @@ type RegistryOptions = { // Publish times overlaid on a package's manifest: { "a-dep": { "1.0.4": iso } }. rewriteTime?: Record>; // Versions added to a package's manifest, each a copy of a published one: { "a-dep": { "2.0.0": "1.0.4" } }. + // Mutable, so a test can publish a version after installing. copyVersions?: Record>; + // The version that a package's `latest` tag is moved to: { "a-dep": "2.0.0" }. + latest?: Record; }; // Answers the bulk-advisory endpoint itself and proxies everything else to the fixture registry, pointing manifest tarball URLs back at itself. @@ -181,6 +184,7 @@ function startRegistry(advisories: Record, options: Registry manifest.versions[version] = { ...manifest.versions[from], version }; if (manifest.time) manifest.time[version] = manifest.time[from]; } + if (options.latest?.[packageName]) manifest["dist-tags"].latest = options.latest[packageName]; return Response.json(manifest, { status: up.status }); }, }); @@ -2095,6 +2099,28 @@ describe("`bun audit fix`", () => { expect(await installedVersion(dir, "a-dep")).toBe("1.0.4"); }); + // The fix comes from a manifest that is newer than the cached one. `bun update --offline` resolves from the cache, + // so the fix has to leave its manifest there. + test.concurrent("a fix to a version published after the install survives an offline update", async () => { + const published: Pick = { copyVersions: {}, latest: {} }; + await using server = startRegistry({ "a-dep": [adv("<1.0.11")] }, published); + using dir = await setup(server, { name: "foo", dependencies: { "a-dep": "^1.0.2" } }); + expect(await lock(dir)).toContain('"a-dep@1.0.10"'); + published.copyVersions!["a-dep"] = { "1.0.11": "1.0.10" }; + published.latest!["a-dep"] = "1.0.11"; + + const fixed = await auditFix(dir); + expect(fixed.stdout).toContain(" ^ a-dep 1.0.10 -> 1.0.11"); + expect(fixed.exitCode).toBe(0); + + const { stdout, stderr, exitCode } = await run(dir, ["update", "--offline"]); + expect(stderr).not.toContain("error:"); + expect(stdout).not.toContain(" -> "); + expect(await lock(dir)).toContain('"a-dep@1.0.11"'); + expect(await lock(dir)).not.toContain('"a-dep@1.0.10"'); + expect(exitCode).toBe(0); + }); + test.concurrent("refuses when package.json has changed since bun.lock was written", async () => { await using server = startRegistry({ "a-dep": [adv("<1.0.4")] }); using dir = await setupVulnerableADep(server); diff --git a/test/cli/install/bun-install-offline.test.ts b/test/cli/install/bun-install-offline.test.ts index 5f4dd7b8f5ab..a9081590c53f 100644 --- a/test/cli/install/bun-install-offline.test.ts +++ b/test/cli/install/bun-install-offline.test.ts @@ -48,9 +48,9 @@ afterEach(async () => { // bunfig cache dirs these tests populate and inspect const { BUN_INSTALL_CACHE_DIR: _ciCacheDir, ...installEnv } = env; -async function install(cwd: string, args: string[], extraEnv: Record = {}) { +async function install(cwd: string, args: string[], extraEnv: Record = {}, command = "install") { await using proc = spawn({ - cmd: [bunExe(), "install", ...args], + cmd: [bunExe(), command, ...args], cwd, env: { ...installEnv, ...extraEnv }, stdout: "pipe", @@ -142,6 +142,23 @@ int renameat(int from_dir, const char *from, int to_dir, const char *to) { }, ); +it("--offline resolves from the manifests that an online `bun update` fetched", async () => { + const urls: string[] = []; + setHandler(dummyRegistry(urls, { "0.0.3": {}, "0.0.5": {} })); + // The update is the only command that reaches the registry. It starts with an empty cache. + let r = await install(await newProject({ baz: ">=0.0.3" }), [], {}, "update"); + expect(r.err).not.toContain("error:"); + expect(r.code).toBe(0); + const before = urls.length; + expect(before).toBeGreaterThan(0); + + r = await install(await newProject({ baz: ">=0.0.3" }), ["--offline"]); + expect(r.err).not.toContain("error:"); + expect(r.out).toContain("+ baz@0.0.5"); + expect(urls.slice(before)).toEqual([]); + expect(r.code).toBe(0); +}); + describe.each(["hoisted", "isolated"] as const)("--offline (%s linker)", linker => { it("never issues a request and fails cleanly on a cache miss", async () => { const urls: string[] = []; diff --git a/test/cli/install/bun-update.test.ts b/test/cli/install/bun-update.test.ts index 732c8682eb8d..770e3eef8e45 100644 --- a/test/cli/install/bun-update.test.ts +++ b/test/cli/install/bun-update.test.ts @@ -1,4 +1,5 @@ import { file, spawn } from "bun"; +import { npm_manifest_test_helpers } from "bun:internal-for-testing"; import { afterAll, beforeAll, describe, expect, it } from "bun:test"; import { access, appendFile, exists, mkdir, readFile, rm, writeFile } from "fs/promises"; import { @@ -62,14 +63,15 @@ const PROGRESS_LINES = /^(?:Resolving dependencies|Resolved, downloaded and extr // stdout comes back normalized for inline snapshots. stderr comes back without the two progress lines every // resolving run prints, so callers assert the rest of it exactly. -async function run(ctx: TestContext, args: string[], cwd = "") { +async function run(ctx: TestContext, args: string[], cwd = "", extraEnv: Record = {}, keys?: string) { await using proc = spawn({ cmd: [bunExe(), ...args], cwd: join(ctx.package_dir, cwd), - env, + env: { ...env, ...extraEnv }, stdout: "pipe", stderr: "pipe", - stdin: "ignore", + // `keys` is what a user types into an interactive command. + stdin: keys === undefined ? "ignore" : Buffer.from(keys), }); const [out, err, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); return { out: normalizeBunSnapshot(out), err: err.replace(PROGRESS_LINES, ""), exitCode }; @@ -1372,6 +1374,313 @@ it.concurrent("bun update rejects a name that is not in the lockfile", as expect(await lockText(ctx)).toBe(lockBefore); }); +// `bun update` asks the registry for every manifest. It stores what the registry sends, so the cache is not older than +// the bun.lock that the update wrote. +describe("bun update and the manifest cache", () => { + type Published = Record; + + // The project gets a manifest cache of its own (the context's bunfig has none). `published` lists the versions of + // each package, and a test changes it to publish or remove one. `dependencies` lists what each package depends on. + // Every manifest has an ETag, so a request that was made from a cached manifest would be conditional. If a test + // sets `served.cacheControl`, the registry sends it, and answers a request whose ETag still holds with 304. + async function cachedProject( + ctx: TestContext, + published: Published, + packageJson: Json, + install: Json = {}, + dependencies: Record = {}, + ) { + const cache = join(ctx.package_dir, ".bun-cache"); + const requests: string[] = []; + const served: { cacheControl?: string } = {}; + const tarballs = new Map(); + setContextHandler(ctx, async request => { + const path = request.url.slice(ctx.registry_url.length); + requests.push(request.headers.has("if-none-match") ? `${path} (conditional)` : path); + const [, name, version] = path.match(/^(.+)-(\d+\.\d+\.\d+)\.tgz$/) ?? []; + if (name) { + const files = { "package/package.json": JSON.stringify({ name, version, dependencies: dependencies[name] }) }; + if (!tarballs.has(path)) tarballs.set(path, await new Bun.Archive(files, { compress: "gzip" }).blob()); + return new Response(tarballs.get(path)); + } + if (!published[path]) return new Response("not found", { status: 404 }); + const versions = published[path].map(version => [ + version, + { + name: path, + version, + dependencies: dependencies[path], + dist: { tarball: `${ctx.registry_url}${path}-${version}.tgz` }, + }, + ]); + const etag = `"${published[path].join()}"`; + const headers: Record = served.cacheControl ? { "cache-control": served.cacheControl } : {}; + if (served.cacheControl && request.headers.get("if-none-match") === etag) { + return new Response(null, { status: 304, headers }); + } + return Response.json( + { name: path, versions: Object.fromEntries(versions), "dist-tags": { latest: published[path].at(-1) } }, + { headers: { etag, ...headers } }, + ); + }); + const bunfig = { registry: ctx.registry_url, saveTextLockfile: true, linker: "hoisted", ...install }; + await Promise.all([ + writeFile(join(ctx.package_dir, "bunfig.toml"), Bun.TOML.stringify({ install: bunfig })), + writeFile(join(ctx.package_dir, "package.json"), JSON.stringify(packageJson)), + ]); + const manifestFiles = async () => (await readdirSorted(cache)).filter(name => name.endsWith(".npm")); + return { + requests, + served, + // CI exports BUN_INSTALL_CACHE_DIR, which wins over bunfig.toml. + bun: (args: string[], extraEnv: Record = {}, keys?: string) => + run(ctx, args, "", { BUN_INSTALL_CACHE_DIR: cache, ...extraEnv }, keys), + // The versions that each manifest in the cache lists. The version of a dist-tag is listed again. + async cached(): Promise { + const manifests = (await manifestFiles()).map(name => + npm_manifest_test_helpers.parseManifest(join(cache, name), ctx.registry_url), + ); + return Object.fromEntries( + manifests.map(({ name, versions }) => [name, [...new Set(versions)].sort(Bun.semver.order)]), + ); + }, + // Leaves the packages in the cache and removes their manifests. + forgetManifests: async () => void (await Promise.all((await manifestFiles()).map(name => rm(join(cache, name))))), + // The versions of `name` in bun.lock. + locked: async (name: string) => (await lockText(ctx)).match(new RegExp(`"${name}@[\\d.]+"`, "g")), + }; + } + + async function installed(project: Awaited>) { + expect(await project.bun(["install"])).toMatchObject({ err: "Saved lockfile\n", exitCode: 0 }); + return project; + } + + // `req` is a dependency and `leaf` an optional dependency, both on `range`. They are installed at 1.0.0, then 1.1.0 + // is published and an online `bun update` moves both. + async function updatedProject(ctx: TestContext, range: string) { + const published = { leaf: ["1.0.0"], req: ["1.0.0"] }; + const packageJson = { name: "foo", dependencies: { req: range }, optionalDependencies: { leaf: range } }; + const project = await installed(await cachedProject(ctx, published, packageJson)); + published.leaf.push("1.1.0"); + published.req.push("1.1.0"); + const { out, exitCode } = await project.bun(["update"]); + expect(out).toContain("^ leaf 1.0.0 -> 1.1.0\n^ req 1.0.0 -> 1.1.0"); + expect(exitCode).toBe(0); + project.requests.length = 0; + return project; + } + + // `req` depends on `dep ^1.0.0`, and `pinned` is an exact version. + async function transitiveProject(ctx: TestContext, published: Published) { + const packageJson = { name: "foo", dependencies: { pinned: "1.0.0", req: "^1.0.0" } }; + return installed(await cachedProject(ctx, published, packageJson, {}, { req: { dep: "^1.0.0" } })); + } + + // One row per setting that decides whether an online update stores in the cache directory. The update runs on a + // cache that has no manifest, so every manifest that is there afterwards came from it. `[install.cache] disable` + // leaves no cache directory to list: bun-install.test.ts lists `node_modules/.cache` for it. + it.concurrent.each<[string, boolean, string[], { install?: Json; env?: Record }?]>([ + ["bun update", true, []], + ["bun update req", true, ["req"]], + ["bun update --latest", true, ["--latest"]], + ["bun update --force", true, ["--force"]], + ["bun update --no-cache", false, ["--no-cache"]], + ["bun update with install.cache.disableManifest", false, [], { install: { cache: { disableManifest: true } } }], + ["bun update with BUN_MANIFEST_CACHE=0", false, [], { env: { BUN_MANIFEST_CACHE: "0" } }], + // The offline modes resolve from the cache, so they store what they have to fetch whatever else is set. + ["bun update --prefer-offline --no-cache", true, ["--prefer-offline", "--no-cache"]], + ['bun update --no-cache with install.prefer = "offline"', true, ["--no-cache"], { install: { prefer: "offline" } }], + ])("%s stores the manifest that it fetches: %p", async (_, stores, args, { install, env: extraEnv } = {}) => { + using ctx = await testContext(); + const published = { req: ["1.0.0"] }; + const packageJson = { name: "foo", dependencies: { req: ">=1.0.0" } }; + const project = await installed(await cachedProject(ctx, published, packageJson, install)); + await project.forgetManifests(); + published.req.push("1.1.0"); + + const { out, exitCode } = await project.bun(["update", ...args], extraEnv); + expect(out).toContain("^ req 1.0.0 -> 1.1.0"); + expect(await project.cached()).toEqual(stores ? { req: ["1.0.0", "1.1.0"] } : {}); + expect(exitCode).toBe(0); + }); + + // The menu lists what the registry sent. Space selects the one row and Enter confirms. + it.concurrent("bun update --interactive stores the manifest that it fetches", async () => { + using ctx = await testContext(); + const published = { req: ["1.0.0"] }; + const packageJson = { name: "foo", dependencies: { req: ">=1.0.0" } }; + const project = await installed(await cachedProject(ctx, published, packageJson)); + await project.forgetManifests(); + published.req.push("1.1.0"); + + const { err, exitCode } = await project.bun(["update", "--interactive"], {}, " \r"); + expect(err).not.toContain("error:"); + expect(await project.cached()).toEqual({ req: ["1.0.0", "1.1.0"] }); + expect(await project.locked("req")).toEqual(['"req@1.1.0"']); + expect(exitCode).toBe(0); + }); + + // A store of only the packages that moved, or of only the entries of package.json, would leave the cache of an + // earlier version of bun, or of a pulled bun.lock, as it is. + it.concurrent("asks for every manifest and stores each one: unmoved, transitive and exact", async () => { + using ctx = await testContext(); + const published = { dep: ["1.0.0"], pinned: ["1.0.0"], req: ["1.0.0"] }; + const project = await transitiveProject(ctx, published); + await project.forgetManifests(); + + project.requests.length = 0; + const unmoved = await project.bun(["update"]); + expect(project.requests.sort()).toEqual(["dep", "pinned", "req"]); + expect(unmoved.out).not.toContain(" -> "); + expect(await project.cached()).toEqual({ dep: ["1.0.0"], pinned: ["1.0.0"], req: ["1.0.0"] }); + expect(unmoved.exitCode).toBe(0); + + // The stored manifests do not answer the next update, and its requests are not conditional. + published.dep.push("1.1.0"); + project.requests.length = 0; + const moved = await project.bun(["update"]); + expect(project.requests.filter(path => !path.endsWith(".tgz")).sort()).toEqual(["dep", "pinned", "req"]); + expect(moved.out).toContain("^ dep 1.0.0 -> 1.1.0"); + expect(await project.cached()).toEqual({ dep: ["1.0.0", "1.1.0"], pinned: ["1.0.0"], req: ["1.0.0"] }); + expect(moved.exitCode).toBe(0); + }); + + // The cached manifests list what the online update installed, so an update that resolves from them changes nothing. + describe.each(["--offline", "--prefer-offline"])("bun update %s after an online update", flag => { + it.concurrent.each<[string, string, string[], string]>([ + ["a range that already allowed the new version", ">=1.0.0", [], ">=1.0.0"], + ["a range that the online update rewrote", "^1.0.0", [], "^1.1.0"], + ["--latest", ">=1.0.0", ["--latest"], "^1.1.0"], + ["named packages", ">=1.0.0", ["req", "leaf"], ">=1.0.0"], + ])("%s: moves nothing and sends no request", async (_, range, args, rangeAfter) => { + using ctx = await testContext(); + const project = await updatedProject(ctx, range); + + const { out, err, exitCode } = await project.bun(["update", flag, ...args]); + expect(err).not.toContain("error:"); + expect(out).not.toContain(" -> "); + expect(project.requests).toEqual([]); + expect(await packageJsonOf(ctx)).toEqual({ + name: "foo", + dependencies: { req: rangeAfter }, + optionalDependencies: { leaf: rangeAfter }, + }); + expect([await project.locked("leaf"), await project.locked("req")]).toEqual([['"leaf@1.1.0"'], ['"req@1.1.0"']]); + expect(exitCode).toBe(0); + }); + }); + + // A bare offline update leaves a transitive package where it is. These two resolve it again. + it.concurrent.each<[string, string]>([ + ["update dep --offline", "(no changes)"], + ["add dep", "installed dep@1.1.0"], + ])("bun %s keeps the transitive package where an online update put it", async (command, printed) => { + using ctx = await testContext(); + const published = { dep: ["1.0.0"], pinned: ["1.0.0"], req: ["1.0.0"] }; + const project = await transitiveProject(ctx, published); + published.dep.push("1.1.0"); + expect((await project.bun(["update"])).out).toContain("^ dep 1.0.0 -> 1.1.0"); + project.requests.length = 0; + + const { out, err, exitCode } = await project.bun(command.split(" ")); + expect(err).not.toContain("error:"); + expect(out).toContain(printed); + expect(project.requests).toEqual([]); + expect(await project.locked("dep")).toEqual(['"dep@1.1.0"']); + expect(exitCode).toBe(0); + }); + + // The manifests that the install cached are still fresh, so these commands send no request. The manifests must not + // be older than what the update saw. + it.concurrent("bun add after an online update resolves the version that the update installed", async () => { + using ctx = await testContext(); + const project = await updatedProject(ctx, ">=1.0.0"); + + const { out, err, exitCode } = await project.bun(["add", "req"]); + expect(err).not.toContain("error:"); + expect(out).toContain("installed req@1.1.0"); + expect(project.requests).toEqual([]); + expect((await packageJsonOf(ctx)).dependencies).toEqual({ req: "^1.1.0" }); + expect(exitCode).toBe(0); + }); + + it.concurrent("bun outdated after an online update lists nothing", async () => { + using ctx = await testContext(); + const project = await updatedProject(ctx, ">=1.0.0"); + + expect(await project.bun(["outdated"])).toEqual({ + out: "bun outdated ()", + err: "", + exitCode: 0, + }); + expect(project.requests).toEqual([]); + }); + + // The entry that the update stored holds the ETag, so the next command asks with it. The 304 has a new max-age, + // and the command after it sends no request only if the 304 was stored too. + it.concurrent("a manifest that bun update stored is checked with its ETag, and the 304 is stored", async () => { + using ctx = await testContext(); + const published = { req: ["1.0.0"] }; + const packageJson = { name: "foo", dependencies: { req: ">=1.0.0" } }; + const project = await installed(await cachedProject(ctx, published, packageJson)); + await project.forgetManifests(); + published.req.push("1.1.0"); + project.served.cacheControl = "max-age=0"; + expect((await project.bun(["update"])).out).toContain("^ req 1.0.0 -> 1.1.0"); + + project.requests.length = 0; + project.served.cacheControl = "max-age=300"; + for (const expected of [["req (conditional)"], []]) { + const { out, exitCode } = await project.bun(["add", "req"]); + expect(out).toContain("installed req@1.1.0"); + expect(project.requests.splice(0)).toEqual(expected); + expect(exitCode).toBe(0); + } + }); + + // The registry no longer lists 1.0.1, and the online update moved off it. + it.concurrent("bun update --offline does not go back to a version that the registry removed", async () => { + using ctx = await testContext(); + const published = { req: ["1.0.0", "1.0.1"] }; + const project = await installed( + await cachedProject(ctx, published, { name: "foo", dependencies: { req: "^1.0.0" } }), + ); + published.req.pop(); + expect((await project.bun(["update"])).out).toContain("^ req 1.0.1 -> 1.0.0"); + project.requests.length = 0; + + const { out, err, exitCode } = await project.bun(["update", "--offline"]); + expect(err).toBe(""); + expect(out).not.toContain(" -> "); + expect(project.requests).toEqual([]); + expect(await project.locked("req")).toEqual(['"req@1.0.0"']); + expect(exitCode).toBe(0); + }); + + // `--lockfile-only` downloads no tarball. The cache then lists a version that an offline update cannot install. + it.concurrent( + "bun update --offline after bun update --lockfile-only fails on the tarball that is not cached", + async () => { + using ctx = await testContext(); + const published = { req: ["1.0.0"] }; + const project = await installed( + await cachedProject(ctx, published, { name: "foo", dependencies: { req: ">=1.0.0" } }), + ); + published.req.push("1.1.0"); + expect((await project.bun(["update", "--lockfile-only"])).out).toContain("^ req 1.0.0 -> 1.1.0"); + project.requests.length = 0; + + expect(await project.bun(["update", "--offline"])).toMatchObject({ + err: 'error: --offline: "req" is not in the cache\n', + exitCode: 1, + }); + expect(project.requests).toEqual([]); + expect(await project.locked("req")).toEqual(['"req@1.1.0"']); + }, + ); +}); + // Registry: no-deps 1.0.0/1.0.1/1.1.0/2.0.0; a-dep 1.0.1..1.0.10; dep-with-tags latest=3.0.0, pre-2=2.0.1; @types/* 1.0.0/2.0.0. describe("bun update semantics", () => { type Json = Record;