From 0ac040eacd9a84ab6097094166bd9edc54626fae Mon Sep 17 00:00:00 2001 From: jdx <216188+jdx@users.noreply.github.com> Date: Fri, 9 Oct 2026 10:29:03 +0000 Subject: [PATCH 01/17] docs(http): say the shared extraction store is trusted as is Co-Authored-By: Claude Sonnet 5.5 Entire-Checkpoint: 01M4G3B3JQ8AD2K44CKXVFGSWA --- docs/dev-tools/backends/http.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/dev-tools/backends/http.md b/docs/dev-tools/backends/http.md index 933ddfe7d1f..e96526f84d6 100644 --- a/docs/dev-tools/backends/http.md +++ b/docs/dev-tools/backends/http.md @@ -552,6 +552,12 @@ Changes to shared files, including changes made by a `postinstall` hook, affect every install that uses them. `mise install --system`, `mise install --shared` and `mise install-into` destinations always get their own copy. +mise does not re-check files already in the store against the lockfile checksum +when it links an install to them. Treat `http-tarballs` as trusted as is and +protect it like the binaries themselves: do not let untrusted jobs, users or +refs write to a store on a shared volume or in a restored CI cache. To verify +the pinned checksum, run `mise install --locked` with a fresh data directory. + `mise prune`, `mise cache prune` and `mise cache clear` leave `http-tarballs` alone, because installs may still link to it. Installs made by older mise versions may also link into it. To give an install its own copy, remove From b49bac0965e83fb9b487573a44722b9657206ac4 Mon Sep 17 00:00:00 2001 From: jdx <216188+jdx@users.noreply.github.com> Date: Fri, 9 Oct 2026 10:32:45 +0000 Subject: [PATCH 02/17] docs(lock): explain why mise can't validate restored caches Co-Authored-By: Claude Sonnet 5.5 Entire-Checkpoint: 01M4G3HX1TGRXPZN16ACSFDQ55 --- docs/continuous-integration.md | 5 +++++ docs/dev-tools/backends/http.md | 13 ++++++++----- docs/dev-tools/mise-lock.md | 34 +++++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 5 deletions(-) diff --git a/docs/continuous-integration.md b/docs/continuous-integration.md index f9e73cc64d9..bf83f3afe6b 100644 --- a/docs/continuous-integration.md +++ b/docs/continuous-integration.md @@ -38,6 +38,11 @@ jobs: - run: mise run test ``` +::: warning +A restored tool cache is trusted as is: mise does not check installed files against +`mise.lock`. See [Restored caches and installed tools](/dev-tools/mise-lock.html#restored-caches). +::: + The example assumes a `test` task in `mise.toml`, such as `test = "npm ci && npm test"` under `[tasks]`. `mise exec -- npm test` works as well. diff --git a/docs/dev-tools/backends/http.md b/docs/dev-tools/backends/http.md index e96526f84d6..779c61768e7 100644 --- a/docs/dev-tools/backends/http.md +++ b/docs/dev-tools/backends/http.md @@ -552,11 +552,14 @@ Changes to shared files, including changes made by a `postinstall` hook, affect every install that uses them. `mise install --system`, `mise install --shared` and `mise install-into` destinations always get their own copy. -mise does not re-check files already in the store against the lockfile checksum -when it links an install to them. Treat `http-tarballs` as trusted as is and -protect it like the binaries themselves: do not let untrusted jobs, users or -refs write to a store on a shared volume or in a restored CI cache. To verify -the pinned checksum, run `mise install --locked` with a fresh data directory. +::: warning The shared store is trusted as is +mise does not re-check files already in `http-tarballs` against the lockfile +checksum when it links an install to them. Protect the store like the binaries +themselves: do not let untrusted jobs, users or refs write to one on a shared +volume or in a restored CI cache. See +[Restored caches and installed tools](/dev-tools/mise-lock.html#restored-caches) +for why mise cannot validate this for you. +::: `mise prune`, `mise cache prune` and `mise cache clear` leave `http-tarballs` alone, because installs may still link to it. Installs made by older mise diff --git a/docs/dev-tools/mise-lock.md b/docs/dev-tools/mise-lock.md index 6f32cf4f8d6..13614a4a36e 100644 --- a/docs/dev-tools/mise-lock.md +++ b/docs/dev-tools/mise-lock.md @@ -411,6 +411,40 @@ Lifecycle scripts can still produce different output on each machine. See for the layout and for editing sidecars, and the [npm](/dev-tools/backends/npm.html) and [PyPI](/dev-tools/backends/pypi.html) backend pages for installer limits. +## Restored caches and installed tools {#restored-caches} + +::: warning +`mise.lock` verifies what mise downloads. It does not verify what is already on +disk. A restored CI cache, a shared volume, or an +[`http` shared extraction store](/dev-tools/backends/http.html#shared-extraction) +is trusted as is. +::: + +When a version is already installed, `mise install` skips the download, so the +checksum in `mise.lock` is not checked again. mise can't validate the installed +files against the lockfile, for two reasons: + +- **Installed trees are mutable on purpose.** Tools can update themselves, + install packages, and generate files after install, and `postinstall` hooks + can change them. The lockfile records the checksum of the downloaded artifact, + not a digest of the installed result, so there is nothing stable to compare + the directory against. +- **The artifact doesn't determine the install.** For tools built from source, + such as Python and Ruby, the checksum covers the source tarball and not the + compiled output. Re-extracting that tarball would prove nothing about the + restored tree, and reinstalling from it means compiling again. + +Treat a cache as part of your trusted build input instead: + +- Don't restore caches written by untrusted refs or triggers, and be wary of + write access to the cache from low-trust triggers such as pull requests from + forks. +- Save a manifest of the cache when you write it and check it on restore. That + catches a modified cache entry for every backend, but it proves the cache is + unchanged since your CI saved it, not that it matches the lockfile. +- For a fully verified install, run `mise install --locked` with an empty data + directory so every artifact is downloaded and checked against `mise.lock`. + ## Provenance and verification {#provenance-and-security} For supported backends, `mise lock` records provenance such as SLSA, Cosign, From a5b5a543ad343b00a1ef4955ab606e64e7ec5e0b Mon Sep 17 00:00:00 2001 From: jdx <216188+jdx@users.noreply.github.com> Date: Fri, 9 Oct 2026 10:42:15 +0000 Subject: [PATCH 03/17] docs(lock): note that entries without a checksum are generated, not validated Co-Authored-By: Claude Sonnet 5.5 Entire-Checkpoint: 01M4G438ZQSCB421NJV9FJMC79 --- docs/dev-tools/mise-lock.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/dev-tools/mise-lock.md b/docs/dev-tools/mise-lock.md index 13614a4a36e..78a377a6202 100644 --- a/docs/dev-tools/mise-lock.md +++ b/docs/dev-tools/mise-lock.md @@ -443,7 +443,9 @@ Treat a cache as part of your trusted build input instead: catches a modified cache entry for every backend, but it proves the cache is unchanged since your CI saved it, not that it matches the lockfile. - For a fully verified install, run `mise install --locked` with an empty data - directory so every artifact is downloaded and checked against `mise.lock`. + directory so every artifact is downloaded and checked against the checksum + in `mise.lock`. An entry without a checksum has one generated instead of + validated, so commit entries that have checksums. ## Provenance and verification {#provenance-and-security} From 6eb9b9d2350e0230429c78ca6e7dc29e6debfb06 Mon Sep 17 00:00:00 2001 From: jdx <216188+jdx@users.noreply.github.com> Date: Fri, 9 Oct 2026 10:47:46 +0000 Subject: [PATCH 04/17] docs(http): say how to discard an untrusted shared extraction store Co-Authored-By: Claude Sonnet 5.5 Entire-Checkpoint: 01M4G4DCX7EZFFR1XYPE5AJ11X --- docs/dev-tools/backends/http.md | 5 ++++- docs/dev-tools/mise-lock.md | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/dev-tools/backends/http.md b/docs/dev-tools/backends/http.md index 779c61768e7..70c5b24eda7 100644 --- a/docs/dev-tools/backends/http.md +++ b/docs/dev-tools/backends/http.md @@ -556,7 +556,10 @@ and `mise install-into` destinations always get their own copy. mise does not re-check files already in `http-tarballs` against the lockfile checksum when it links an install to them. Protect the store like the binaries themselves: do not let untrusted jobs, users or refs write to one on a shared -volume or in a restored CI cache. See +volume or in a restored CI cache. To stop trusting a store, remove the +installs that link into it and the `http-tarballs` directory, then reinstall; +mise extracts a fresh copy from a download it checks against the lockfile. +`mise install --force` alone reuses the store. See [Restored caches and installed tools](/dev-tools/mise-lock.html#restored-caches) for why mise cannot validate this for you. ::: diff --git a/docs/dev-tools/mise-lock.md b/docs/dev-tools/mise-lock.md index 78a377a6202..75280135efa 100644 --- a/docs/dev-tools/mise-lock.md +++ b/docs/dev-tools/mise-lock.md @@ -442,6 +442,10 @@ Treat a cache as part of your trusted build input instead: - Save a manifest of the cache when you write it and check it on restore. That catches a modified cache entry for every backend, but it proves the cache is unchanged since your CI saved it, not that it matches the lockfile. +- For an `http` tool with `shared_extraction`, remove the installs that link + into `$MISE_DATA_DIR/http-tarballs` and the directory itself, then reinstall. + mise downloads each artifact, checks it against its checksum and extracts a + fresh copy. `mise install --force` alone reuses the store's files. - For a fully verified install, run `mise install --locked` with an empty data directory so every artifact is downloaded and checked against the checksum in `mise.lock`. An entry without a checksum has one generated instead of From 2311d94c1e81ee6a87f3036d9c8d03e305a23963 Mon Sep 17 00:00:00 2001 From: jdx <216188+jdx@users.noreply.github.com> Date: Fri, 9 Oct 2026 10:51:56 +0000 Subject: [PATCH 05/17] docs(lock): shorten the restored cache warning Co-Authored-By: Claude Sonnet 5.5 Entire-Checkpoint: 01M4G4N0MFZ8B4BA3TAF1ZMWXJ --- docs/dev-tools/backends/http.md | 10 ++----- docs/dev-tools/mise-lock.md | 47 ++++++++++----------------------- 2 files changed, 16 insertions(+), 41 deletions(-) diff --git a/docs/dev-tools/backends/http.md b/docs/dev-tools/backends/http.md index 70c5b24eda7..60031692dec 100644 --- a/docs/dev-tools/backends/http.md +++ b/docs/dev-tools/backends/http.md @@ -554,14 +554,8 @@ and `mise install-into` destinations always get their own copy. ::: warning The shared store is trusted as is mise does not re-check files already in `http-tarballs` against the lockfile -checksum when it links an install to them. Protect the store like the binaries -themselves: do not let untrusted jobs, users or refs write to one on a shared -volume or in a restored CI cache. To stop trusting a store, remove the -installs that link into it and the `http-tarballs` directory, then reinstall; -mise extracts a fresh copy from a download it checks against the lockfile. -`mise install --force` alone reuses the store. See -[Restored caches and installed tools](/dev-tools/mise-lock.html#restored-caches) -for why mise cannot validate this for you. +checksum. Don't let untrusted jobs or users write to a shared or restored store. +See [Restored caches and installed tools](/dev-tools/mise-lock.html#restored-caches). ::: `mise prune`, `mise cache prune` and `mise cache clear` leave `http-tarballs` diff --git a/docs/dev-tools/mise-lock.md b/docs/dev-tools/mise-lock.md index 75280135efa..b46e82a59d9 100644 --- a/docs/dev-tools/mise-lock.md +++ b/docs/dev-tools/mise-lock.md @@ -414,42 +414,23 @@ and [PyPI](/dev-tools/backends/pypi.html) backend pages for installer limits. ## Restored caches and installed tools {#restored-caches} ::: warning -`mise.lock` verifies what mise downloads. It does not verify what is already on -disk. A restored CI cache, a shared volume, or an -[`http` shared extraction store](/dev-tools/backends/http.html#shared-extraction) -is trusted as is. +`mise.lock` verifies what mise downloads, not what is already on disk. A +restored CI cache or a shared extraction store is trusted as is. ::: -When a version is already installed, `mise install` skips the download, so the -checksum in `mise.lock` is not checked again. mise can't validate the installed -files against the lockfile, for two reasons: - -- **Installed trees are mutable on purpose.** Tools can update themselves, - install packages, and generate files after install, and `postinstall` hooks - can change them. The lockfile records the checksum of the downloaded artifact, - not a digest of the installed result, so there is nothing stable to compare - the directory against. -- **The artifact doesn't determine the install.** For tools built from source, - such as Python and Ruby, the checksum covers the source tarball and not the - compiled output. Re-extracting that tarball would prove nothing about the - restored tree, and reinstalling from it means compiling again. - -Treat a cache as part of your trusted build input instead: - -- Don't restore caches written by untrusted refs or triggers, and be wary of - write access to the cache from low-trust triggers such as pull requests from - forks. -- Save a manifest of the cache when you write it and check it on restore. That - catches a modified cache entry for every backend, but it proves the cache is - unchanged since your CI saved it, not that it matches the lockfile. -- For an `http` tool with `shared_extraction`, remove the installs that link - into `$MISE_DATA_DIR/http-tarballs` and the directory itself, then reinstall. - mise downloads each artifact, checks it against its checksum and extracts a - fresh copy. `mise install --force` alone reuses the store's files. +mise can't validate an installed tree against the lockfile. The checksum covers +the downloaded artifact, and installs are mutable on purpose: tools update +themselves and `postinstall` hooks change files. For tools built from source, +such as Python and Ruby, it covers only the source tarball, not the compiled +output. + +So treat a cache as trusted build input: + +- Don't restore caches written by untrusted refs or triggers. +- Save a manifest with the cache and check it on restore. That proves the cache + is unchanged since CI saved it, not that it matches the lockfile. - For a fully verified install, run `mise install --locked` with an empty data - directory so every artifact is downloaded and checked against the checksum - in `mise.lock`. An entry without a checksum has one generated instead of - validated, so commit entries that have checksums. + directory. ## Provenance and verification {#provenance-and-security} From 633292619c9a055046a83d7f5c1be528648a9227 Mon Sep 17 00:00:00 2001 From: jdx <216188+jdx@users.noreply.github.com> Date: Fri, 9 Oct 2026 11:09:11 +0000 Subject: [PATCH 06/17] feat(http): reuse verified downloads from a checksum-keyed cache Co-Authored-By: Claude Sonnet 5.5 Entire-Checkpoint: 01M4G5MKCXMJ0D857G5ZRZH5J3 --- docs/dev-tools/mise-lock.md | 13 ++- e2e/backend/test_http_download_cache | 39 ++++++++ schema/mise.json | 5 ++ settings.toml | 21 ++++- src/backend/http.rs | 35 +++++++- src/download_cache.rs | 127 +++++++++++++++++++++++++++ src/lib.rs | 1 + 7 files changed, 231 insertions(+), 10 deletions(-) create mode 100644 e2e/backend/test_http_download_cache create mode 100644 src/download_cache.rs diff --git a/docs/dev-tools/mise-lock.md b/docs/dev-tools/mise-lock.md index b46e82a59d9..4b00672208e 100644 --- a/docs/dev-tools/mise-lock.md +++ b/docs/dev-tools/mise-lock.md @@ -424,11 +424,18 @@ themselves and `postinstall` hooks change files. For tools built from source, such as Python and Ruby, it covers only the source tarball, not the compiled output. -So treat a cache as trusted build input: +Downloads are different. With a pinned checksum, the `http` backend keeps the +verified artifact in `$MISE_CACHE_DIR/downloads-cas`, hashes it again on every +use, and falls back to downloading when it doesn't match. See +[`download_cache`](/configuration/settings.html#download_cache). + +So treat a cache of installed files as trusted build input: - Don't restore caches written by untrusted refs or triggers. -- Save a manifest with the cache and check it on restore. That proves the cache - is unchanged since CI saved it, not that it matches the lockfile. +- Record a manifest of the cache outside it, in trusted CI metadata, and check + the restored files against it. A manifest stored in the cache can be replaced + along with it. That proves the cache is unchanged since CI saved it, not that + it matches the lockfile. - For a fully verified install, run `mise install --locked` with an empty data directory. diff --git a/e2e/backend/test_http_download_cache b/e2e/backend/test_http_download_cache new file mode 100644 index 00000000000..7b75167a46d --- /dev/null +++ b/e2e/backend/test_http_download_cache @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +# A download pinned by a checksum is kept by that checksum and reused, but only +# while it still hashes to the pin. + +SRV="$PWD/srv" +mkdir -p "$SRV" +printf '#!/bin/sh\necho cached tool ok\n' >"$SRV/mytool" +SHA="$(sha256sum "$SRV/mytool" | cut -d' ' -f1)" + +start_http_server --dir "$SRV" + +cat >mise.toml <>"$ENTRY" +mise uninstall http:cas-tool@1.0.0 +assert_fail "mise install" +assert_fail "test -f $ENTRY" diff --git a/schema/mise.json b/schema/mise.json index 97a7f7df984..2233c2f5175 100644 --- a/schema/mise.json +++ b/schema/mise.json @@ -640,6 +640,11 @@ "description": "Keep downloaded archives and sources after installing, for debugging.", "type": "boolean" }, + "download_cache": { + "default": true, + "description": "Reuse a verified download instead of fetching it again.", + "type": "boolean" + }, "always_keep_install": { "description": "Keep a failed installation's partial install directory and downloads, for debugging.", "type": "boolean" diff --git a/settings.toml b/settings.toml index 36a09c569be..de02221d47c 100644 --- a/settings.toml +++ b/settings.toml @@ -107,14 +107,27 @@ type = "Bool" description = "Keep downloaded archives and sources after installing, for debugging." docs = """ mise keeps the files under `~/.local/share/mise/downloads` so you can inspect -them while debugging a backend or plugin. This is not a download cache: some -backends skip a download when the file already exists, but that is not -guaranteed. To avoid reinstalling tools in CI, cache -`~/.local/share/mise/installs` instead. +them while debugging a backend or plugin. This is not a download cache; see +[`download_cache`](#download_cache). """ env = "MISE_ALWAYS_KEEP_DOWNLOAD" type = "Bool" +[download_cache] +default = true +description = "Reuse a verified download instead of fetching it again." +docs = """ +When a tool pins a checksum, in its options or in `mise.lock`, mise keeps the +verified download in `$MISE_CACHE_DIR/downloads-cas`, named by that checksum. +Later installs hash the kept file again and use it instead of downloading when +it matches, so a restored or shared cache can save a download but cannot change +what gets installed. A file that no longer matches is removed. Only the `http` +backend uses it so far. Extracted files are not covered; see +[Restored caches and installed tools](/dev-tools/mise-lock.html#restored-caches). +""" +env = "MISE_DOWNLOAD_CACHE" +type = "Bool" + [always_keep_install] description = "Keep a failed installation's partial install directory and downloads, for debugging." env = "MISE_ALWAYS_KEEP_INSTALL" diff --git a/src/backend/http.rs b/src/backend/http.rs index b65a6214d14..84b291b8404 100644 --- a/src/backend/http.rs +++ b/src/backend/http.rs @@ -15,6 +15,7 @@ use crate::backend::static_helpers::{ use crate::backend::version_list; use crate::config::Config; use crate::config::Settings; +use crate::download_cache; use crate::http::{DownloadFileMetadata, HTTP}; use crate::install_context::InstallContext; use crate::lockfile::PlatformInfo; @@ -1374,8 +1375,28 @@ impl Backend for HttpBackend { .and_then(|p| p.checksum.as_ref()) .is_some(); - let download = match local_artifact_path(&url)? { - Some(src) => { + // A checksum the tool pinned names the artifact, so a copy kept from an + // earlier install can stand in for the download once it hashes to it. + let pinned = opts.checksum().or_else(|| { + tv.lock_platforms + .get(&platform_key) + .and_then(|p| p.checksum.clone()) + }); + let local_src = local_artifact_path(&url)?; + let restored = match (&pinned, &local_src) { + (Some(checksum), None) => { + ctx.pr.set_message(format!("cached {filename}")); + download_cache::restore(checksum, &file_path, Some(ctx.pr.as_ref()))? + } + _ => None, + }; + let from_cache = restored.is_some(); + + let download = match (restored, local_src) { + (Some(restored), _) => DownloadFileMetadata { + effective_filename: restored.effective_filename, + }, + (None, Some(src)) => { ctx.pr.set_message(format!("copy {filename}")); if let Some(parent) = file_path.parent() { file::create_dir_all(parent)?; @@ -1383,7 +1404,7 @@ impl Backend for HttpBackend { file::run_blocking(|| file::copy(&src, &file_path))?; DownloadFileMetadata::default() } - None => { + (None, None) => { ctx.pr.set_message(format!("download {filename}")); let headers = opts.headers(Some(tv.version.as_str()))?; if headers.is_empty() { @@ -1461,6 +1482,14 @@ impl Backend for HttpBackend { } self.verify_checksum(ctx, &mut tv, &file_path)?; + // Verified against the pin above, so it is safe to keep for next time. + if let Some(checksum) = pinned.as_ref().filter(|_| !from_cache) + && let Err(err) = + download_cache::store(checksum, &file_path, download.effective_filename.as_deref()) + { + debug!("could not keep {filename} in the download cache: {err}"); + } + Ok(tv) } diff --git a/src/download_cache.rs b/src/download_cache.rs new file mode 100644 index 00000000000..da4f1d53479 --- /dev/null +++ b/src/download_cache.rs @@ -0,0 +1,127 @@ +//! A content-addressed cache of verified download artifacts. +//! +//! Entries are keyed by the checksum a tool pinned (in its options or in +//! `mise.lock`), never by URL, and every entry is hashed again before it is +//! used. A cache that was restored, shared or edited can therefore cost a +//! download but never changes what is installed. + +use std::path::{Path, PathBuf}; + +use crate::config::Settings; +use crate::file; +use crate::hash; +use crate::result::Result; +use crate::ui::progress_report::SingleReport; + +/// Algorithms an entry may be keyed by. `md5` and `sha1` are not collision +/// resistant enough to address content by. +const ALGOS: &[&str] = &["sha256", "sha384", "sha512", "blake3"]; + +/// A cached artifact and the file name it was downloaded as. +pub struct Restored { + pub effective_filename: Option, +} + +fn root() -> PathBuf { + crate::dirs::CACHE.join("downloads-cas") +} + +/// The entry for a pinned `algo:hex` checksum, or `None` when the checksum +/// cannot safely name a file. +fn entry(checksum: &str) -> Option { + let (algo, hex) = checksum.split_once(':')?; + let hex = hex.to_lowercase(); + if !ALGOS.contains(&algo) || hex.is_empty() || !hex.bytes().all(|b| b.is_ascii_hexdigit()) { + return None; + } + Some(root().join(algo).join(hex)) +} + +fn name_path(entry: &Path) -> PathBuf { + entry.with_extension("name") +} + +/// Copy the artifact pinned by `checksum` to `dest` if the cache holds it and +/// it still hashes to `checksum`. An entry that does not is removed. +pub fn restore( + checksum: &str, + dest: &Path, + pr: Option<&dyn SingleReport>, +) -> Result> { + if !Settings::get().download_cache { + return Ok(None); + } + let Some(entry) = entry(checksum) else { + return Ok(None); + }; + if !entry.is_file() { + return Ok(None); + } + let (algo, hex) = checksum.split_once(':').expect("entry() checked the form"); + if let Err(err) = hash::ensure_checksum(&entry, hex, pr, algo) { + debug!("dropping download cache entry {}: {err}", entry.display()); + let _ = file::remove_file(&entry); + let _ = file::remove_file(name_path(&entry)); + return Ok(None); + } + if let Some(parent) = dest.parent() { + file::create_dir_all(parent)?; + } + file::copy(&entry, dest)?; + let effective_filename = std::fs::read_to_string(name_path(&entry)) + .ok() + .filter(|name| !name.is_empty()); + Ok(Some(Restored { effective_filename })) +} + +/// Keep `src`, which the caller has verified against `checksum`. +pub fn store(checksum: &str, src: &Path, effective_filename: Option<&str>) -> Result<()> { + if !Settings::get().download_cache { + return Ok(()); + } + let Some(entry) = entry(checksum) else { + return Ok(()); + }; + if entry.is_file() { + return Ok(()); + } + let dir = entry.parent().expect("entry() joins a file name"); + file::create_dir_all(dir)?; + // Written beside the entry and renamed into place, so a reader never + // sees a partial file. + let tmp = dir.join(format!( + "{}.tmp-{}", + entry.file_name().unwrap().to_string_lossy(), + std::process::id() + )); + file::copy(src, &tmp)?; + if let Some(name) = effective_filename { + file::write(name_path(&entry), name)?; + } + if let Err(err) = file::rename(&tmp, &entry) { + let _ = file::remove_file(&tmp); + return Err(err); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn entry_accepts_pinned_hex_digests_only() { + let sha = format!("sha256:{}", "ab".repeat(32)); + assert!( + entry(&sha) + .unwrap() + .ends_with(format!("sha256/{}", "ab".repeat(32))) + ); + assert!(entry(&sha.to_uppercase().replace("SHA256", "sha256")).is_some()); + assert!(entry("md5:d41d8cd98f00b204e9800998ecf8427e").is_none()); + assert!(entry("sha1:da39a3ee5e6b4b0d3255bfef95601890afd80709").is_none()); + assert!(entry("sha256:../../etc/passwd").is_none()); + assert!(entry("sha256:").is_none()); + assert!(entry("deadbeef").is_none()); + } +} diff --git a/src/lib.rs b/src/lib.rs index acfff7290d0..7d0e8e96518 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -44,6 +44,7 @@ pub mod deps; pub(crate) mod deps_graph; pub mod direnv; pub mod dirs; +pub mod download_cache; pub mod duration; pub mod env; pub mod env_diff; From 76e79648833f40002cf9dd2f60ac502eb3bf59e7 Mon Sep 17 00:00:00 2001 From: jdx <216188+jdx@users.noreply.github.com> Date: Fri, 9 Oct 2026 11:18:32 +0000 Subject: [PATCH 07/17] fix(http): give each download cache store its own temp file Co-Authored-By: Claude Sonnet 5.5 Entire-Checkpoint: 01M4G65Q02AYAQ6GDWG35N0BMA --- src/download_cache.rs | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/src/download_cache.rs b/src/download_cache.rs index da4f1d53479..a36ede38d73 100644 --- a/src/download_cache.rs +++ b/src/download_cache.rs @@ -87,21 +87,14 @@ pub fn store(checksum: &str, src: &Path, effective_filename: Option<&str>) -> Re } let dir = entry.parent().expect("entry() joins a file name"); file::create_dir_all(dir)?; - // Written beside the entry and renamed into place, so a reader never - // sees a partial file. - let tmp = dir.join(format!( - "{}.tmp-{}", - entry.file_name().unwrap().to_string_lossy(), - std::process::id() - )); - file::copy(src, &tmp)?; + // Copied to a name no other install shares and renamed into place, so a + // reader never sees a partial file and parallel stores can't truncate one. + let tmp = tempfile::NamedTempFile::new_in(dir)?; + std::fs::copy(src, tmp.path())?; if let Some(name) = effective_filename { file::write(name_path(&entry), name)?; } - if let Err(err) = file::rename(&tmp, &entry) { - let _ = file::remove_file(&tmp); - return Err(err); - } + tmp.persist(&entry).map_err(|err| err.error)?; Ok(()) } From 81e623e0f377b9b8afb69317fd2f0cf631c0e993 Mon Sep 17 00:00:00 2001 From: jdx <216188+jdx@users.noreply.github.com> Date: Fri, 9 Oct 2026 11:30:37 +0000 Subject: [PATCH 08/17] feat: reuse downloads the server confirms are unchanged Co-Authored-By: Claude Sonnet 5.5 Entire-Checkpoint: 01M4G6VVSESS1FWEPXDNBCKB11 --- crates/mise-util/src/http.rs | 103 +++++++++++++++- crates/mise-util/src/http/revalidate.rs | 157 ++++++++++++++++++++++++ crates/mise-util/src/http/tests/mod.rs | 62 ++++++++++ docs/dev-tools/mise-lock.md | 5 +- e2e/backend/test_http_download_cache | 23 ++++ schema/mise.json | 2 +- settings.toml | 23 ++-- 7 files changed, 360 insertions(+), 15 deletions(-) create mode 100644 crates/mise-util/src/http/revalidate.rs diff --git a/crates/mise-util/src/http.rs b/crates/mise-util/src/http.rs index dae0699fb81..ad5d9c77694 100644 --- a/crates/mise-util/src/http.rs +++ b/crates/mise-util/src/http.rs @@ -11,7 +11,8 @@ use regex::Regex; use reqwest::StatusCode; use reqwest::header::{ ACCEPT_ENCODING, AUTHORIZATION, CONTENT_RANGE, CONTENT_TYPE, COOKIE, DATE, ETAG, HeaderMap, - HeaderName, HeaderValue, IF_RANGE, LAST_MODIFIED, PROXY_AUTHORIZATION, RANGE, + HeaderName, HeaderValue, IF_MODIFIED_SINCE, IF_NONE_MATCH, IF_RANGE, LAST_MODIFIED, + PROXY_AUTHORIZATION, RANGE, }; use reqwest::{ClientBuilder, IntoUrl, Method, Response}; use serde::{Deserialize, Serialize}; @@ -28,6 +29,8 @@ use crate::time::format_duration; use crate::{env, file}; use mise_settings::Settings; +mod revalidate; + pub static HTTP: Lazy = Lazy::new(|| { Client::new_shared( crate::network::http_timeout(&Settings::get()), @@ -222,6 +225,17 @@ impl DownloadValidator { } } +/// What an attempt learned beyond the bytes it wrote. +#[derive(Default)] +struct AttemptOutcome { + /// The validator of the stored copy, sent so the server can confirm it. + cond: Option, + /// The server confirmed the stored copy is current; nothing was written. + not_modified: std::sync::atomic::AtomicBool, + /// The validator that came with a complete response, for keeping it. + validator: Mutex>, +} + #[derive(Debug)] struct DownloadSizeMismatch { expected: u64, @@ -1310,6 +1324,22 @@ impl Client { headers: &HeaderMap, pr: Option<&dyn SingleReport>, total_timeout: Duration, + ) -> Result { + let use_cache = Settings::get().download_cache; + self.download_file_inner(url, path, headers, pr, total_timeout, use_cache) + .await + } + + /// With `use_cache`, a finished download of the same request is offered to + /// the server for revalidation and reused on `304 Not Modified`. + async fn download_file_inner( + &self, + url: U, + path: &Path, + headers: &HeaderMap, + pr: Option<&dyn SingleReport>, + total_timeout: Duration, + use_cache: bool, ) -> Result { ensure!( !crate::network::offline(&Settings::get()), @@ -1334,6 +1364,16 @@ impl Client { .await??; let attempt = Arc::new(AtomicUsize::new(0)); let progress = Arc::new(DownloadProgress::default()); + let request_hash = download_request_hash(&url, headers); + let cached = if use_cache { + revalidate::lookup(&request_hash) + } else { + None + }; + let outcome = Arc::new(AttemptOutcome { + cond: cached.as_ref().map(|c| c.validator().clone()), + ..Default::default() + }); // Retry the whole transfer, resuming a validated partial response when // possible. send_once_with_https_fallback_allow_416 (not @@ -1343,10 +1383,11 @@ impl Client { let progress = progress.clone(); let request_url = url.clone(); let partial = partial.clone(); + let outcome = outcome.clone(); async move { attempt.fetch_add(1, Ordering::Relaxed); progress.start_attempt(); - self.download_file_attempt(request_url, headers, &partial, pr, &progress) + self.download_file_attempt(request_url, headers, &partial, pr, &progress, &outcome) .await } }); @@ -1386,7 +1427,40 @@ impl Client { // of `timeout` prevents us from returning an error while it can still // install the destination in the background. let path = path.to_path_buf(); - tokio::task::spawn_blocking(move || partial.persist(&path)).await??; + if outcome.not_modified.load(Ordering::Relaxed) + && let Some(cached) = cached + { + let effective_filename = cached.effective_filename(); + let dest = path.clone(); + let restored = tokio::task::spawn_blocking(move || cached.restore(&dest)).await??; + if restored { + return Ok(DownloadFileMetadata { effective_filename }); + } + // The stored file no longer matches what was recorded: fetch it again. + drop(_download_lock); + return Box::pin(self.download_file_inner( + url, + &path, + headers, + pr, + total_timeout, + false, + )) + .await; + } + let persist_path = path.clone(); + tokio::task::spawn_blocking(move || partial.persist(&persist_path)).await??; + let validator = outcome.validator.lock().unwrap().take(); + if use_cache && let Some(validator) = validator { + let effective_filename = metadata.effective_filename.clone(); + let stored = tokio::task::spawn_blocking(move || { + revalidate::store(&request_hash, &path, &validator, effective_filename) + }) + .await; + if let Ok(Err(err)) | Err(err) = stored.map_err(Report::from) { + debug!("could not keep the download for revalidation: {err:#}"); + } + } Ok(metadata) } @@ -1397,6 +1471,7 @@ impl Client { partial: &PartialDownload, pr: Option<&dyn SingleReport>, progress: &DownloadProgress, + outcome: &AttemptOutcome, ) -> Result { let mut restarted_without_resume = false; loop { @@ -1420,6 +1495,20 @@ impl Client { let offset = resume.as_ref().map(|(_, size)| *size).unwrap_or(0); let mut request_headers = headers.clone(); request_headers.insert(ACCEPT_ENCODING, HeaderValue::from_static("identity")); + let mut conditional = false; + if resume.is_none() + && let Some(cond) = &outcome.cond + { + conditional = true; + match cond { + DownloadValidator::Etag(value) => { + request_headers.insert(IF_NONE_MATCH, HeaderValue::from_str(value)?); + } + DownloadValidator::LastModified { value, .. } => { + request_headers.insert(IF_MODIFIED_SINCE, HeaderValue::from_str(value)?); + } + } + } if let Some((state, _)) = &resume { request_headers.insert(RANGE, HeaderValue::from_str(&format!("bytes={offset}-"))?); request_headers.insert( @@ -1449,6 +1538,14 @@ impl Client { let served_by = resp.url(); progress.served_by(served_by); + if conditional && resp.status() == StatusCode::NOT_MODIFIED { + outcome.not_modified.store(true, Ordering::Relaxed); + return Ok(DownloadFileMetadata::default()); + } + *outcome.validator.lock().unwrap() = (resp.status() != StatusCode::PARTIAL_CONTENT) + .then(|| response_validator(resp.headers())) + .flatten(); + if resp.status() == StatusCode::RANGE_NOT_SATISFIABLE { if let Some(ParsedContentRange::Unsatisfied { total }) = resp .headers() diff --git a/crates/mise-util/src/http/revalidate.rs b/crates/mise-util/src/http/revalidate.rs new file mode 100644 index 00000000000..7c845c69e1b --- /dev/null +++ b/crates/mise-util/src/http/revalidate.rs @@ -0,0 +1,157 @@ +//! Reuse of a finished download when the server says it hasn't changed. +//! +//! Each entry holds the file, the validator (`ETag` or `Last-Modified`) the +//! server sent with it, and the file's BLAKE3 hash. The next request for the +//! same URL and headers carries that validator; a `304 Not Modified` answers +//! with no body, and the stored file is used after it hashes to the recorded +//! value again. + +use std::path::{Path, PathBuf}; +use std::time::SystemTime; + +use eyre::Result; +use serde::{Deserialize, Serialize}; + +use super::DownloadValidator; +use crate::file; + +/// Entries are evicted, oldest use first, past this total size. +const MAX_CACHE_BYTES: u64 = 2 * 1024 * 1024 * 1024; + +const FILE: &str = "file"; +const META: &str = "meta.json"; + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct Meta { + validator: DownloadValidator, + blake3: String, + effective_filename: Option, +} + +pub(super) struct Cached { + dir: PathBuf, + meta: Meta, +} + +impl Cached { + pub(super) fn validator(&self) -> &DownloadValidator { + &self.meta.validator + } + + pub(super) fn effective_filename(&self) -> Option { + self.meta.effective_filename.clone() + } + + /// Copy the stored file to `dest` if it still hashes to what was recorded. + /// An entry that doesn't is removed. + pub(super) fn restore(&self, dest: &Path) -> Result { + let stored = self.dir.join(FILE); + let intact = stored.is_file() + && crate::hash::file_hash_blake3(&stored, None) + .is_ok_and(|hash| hash == self.meta.blake3); + if !intact { + let _ = file::remove_all(&self.dir); + return Ok(false); + } + if let Some(parent) = dest.parent() { + file::create_dir_all(parent)?; + } + file::copy(&stored, dest)?; + // Marks the entry as recently used for eviction. + if let Ok(f) = std::fs::File::options() + .write(true) + .open(self.dir.join(META)) + { + let _ = f.set_modified(SystemTime::now()); + } + Ok(true) + } +} + +/// Tests that don't ask for a directory never touch the user's cache. +#[cfg(test)] +pub(super) static TEST_ROOT: std::sync::Mutex> = std::sync::Mutex::new(None); + +fn root() -> Option { + #[cfg(test)] + { + TEST_ROOT.lock().unwrap().clone() + } + #[cfg(not(test))] + { + Some(crate::dirs::CACHE.join("downloads-cache")) + } +} + +pub(super) fn lookup(request_hash: &str) -> Option { + let dir = root()?.join(request_hash); + let meta: Meta = serde_json::from_slice(&std::fs::read(dir.join(META)).ok()?).ok()?; + meta.validator.is_valid().then_some(Cached { dir, meta }) +} + +/// Keep `path`, just downloaded with `validator`. +pub(super) fn store( + request_hash: &str, + path: &Path, + validator: &DownloadValidator, + effective_filename: Option, +) -> Result<()> { + let Some(root) = root() else { + return Ok(()); + }; + let blake3 = crate::hash::file_hash_blake3(path, None)?; + let dir = root.join(request_hash); + let parent = dir.parent().expect("entry_dir joins a name"); + file::create_dir_all(parent)?; + // Built beside the entry under a name nothing else shares, then swapped in. + let staging = tempfile::Builder::new() + .prefix(".staging-") + .tempdir_in(parent)?; + file::copy(path, staging.path().join(FILE))?; + let meta = Meta { + validator: validator.clone(), + blake3, + effective_filename, + }; + file::write(staging.path().join(META), serde_json::to_vec(&meta)?)?; + let _ = file::remove_all(&dir); + let staged = staging.keep(); + if let Err(err) = std::fs::rename(&staged, &dir) { + let _ = file::remove_all(&staged); + // Another process stored the same download first. + if !dir.join(META).is_file() { + return Err(err.into()); + } + } + evict(parent); + Ok(()) +} + +fn evict(root: &Path) { + let Ok(read) = std::fs::read_dir(root) else { + return; + }; + let mut entries = Vec::new(); + let mut total = 0u64; + for entry in read.flatten() { + let path = entry.path(); + if entry.file_name().to_string_lossy().starts_with('.') { + continue; + } + let size = std::fs::metadata(path.join(FILE)).map_or(0, |m| m.len()); + let used = std::fs::metadata(path.join(META)) + .and_then(|m| m.modified()) + .unwrap_or(SystemTime::UNIX_EPOCH); + total += size; + entries.push((used, size, path)); + } + entries.sort(); + for (_, size, path) in entries { + if total <= MAX_CACHE_BYTES { + break; + } + if file::remove_all(&path).is_ok() { + total -= size; + } + } +} diff --git a/crates/mise-util/src/http/tests/mod.rs b/crates/mise-util/src/http/tests/mod.rs index 960d1757856..e0eb11843fa 100644 --- a/crates/mise-util/src/http/tests/mod.rs +++ b/crates/mise-util/src/http/tests/mod.rs @@ -559,3 +559,65 @@ refresh_expires_at = "2099-01-01T00:00:00Z" fn json_empty_array_response() -> &'static str { "HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\n[]" } + +fn cacheable_download_response() -> &'static str { + concat!( + "HTTP/1.1 200 OK\r\n", + "Content-Length: 5\r\n", + "ETag: \"artifact-v1\"\r\n", + "Connection: close\r\n", + "\r\n", + "hello" + ) +} +fn not_modified_response() -> &'static str { + "HTTP/1.1 304 Not Modified\r\nETag: \"artifact-v1\"\r\nConnection: close\r\n\r\n" +} + +#[tokio::test(flavor = "current_thread")] +async fn test_download_reuses_stored_file_on_not_modified() { + let _settings = set_test_http_retries(0); + let cache = tempfile::tempdir().unwrap(); + *revalidate::TEST_ROOT.lock().unwrap() = Some(cache.path().to_path_buf()); + let (port, count, requests) = spawn_recording_server(vec![ + cacheable_download_response(), + not_modified_response(), + not_modified_response(), + cacheable_download_response(), + ]) + .await; + let url = format!("http://127.0.0.1:{port}/artifact"); + let client = Client::new(Duration::from_secs(2), ClientKind::Http).unwrap(); + let dir = tempfile::tempdir().unwrap(); + let dest = dir.path().join("artifact"); + + client.download_file(&url, &dest, None).await.unwrap(); + assert_eq!(std::fs::read(&dest).unwrap(), b"hello"); + + // The second request is conditional, and a 304 serves the stored file. + std::fs::remove_file(&dest).unwrap(); + client.download_file(&url, &dest, None).await.unwrap(); + assert_eq!(std::fs::read(&dest).unwrap(), b"hello"); + assert!( + requests.lock().unwrap()[1] + .to_lowercase() + .contains("if-none-match: \"artifact-v1\"") + ); + + // A stored file that no longer hashes to what was recorded is not used, + // even though the server says it is current. + let stored = std::fs::read_dir(cache.path()) + .unwrap() + .next() + .unwrap() + .unwrap() + .path() + .join("file"); + std::fs::write(&stored, b"tampered").unwrap(); + std::fs::remove_file(&dest).unwrap(); + client.download_file(&url, &dest, None).await.unwrap(); + assert_eq!(std::fs::read(&dest).unwrap(), b"hello"); + assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 4); + + *revalidate::TEST_ROOT.lock().unwrap() = None; +} diff --git a/docs/dev-tools/mise-lock.md b/docs/dev-tools/mise-lock.md index 4b00672208e..5df2d3f1a53 100644 --- a/docs/dev-tools/mise-lock.md +++ b/docs/dev-tools/mise-lock.md @@ -424,9 +424,8 @@ themselves and `postinstall` hooks change files. For tools built from source, such as Python and Ruby, it covers only the source tarball, not the compiled output. -Downloads are different. With a pinned checksum, the `http` backend keeps the -verified artifact in `$MISE_CACHE_DIR/downloads-cas`, hashes it again on every -use, and falls back to downloading when it doesn't match. See +Downloads are different. mise keeps them in a cache and reuses one only when +the server confirms it is unchanged, or when it hashes to a pinned checksum. See [`download_cache`](/configuration/settings.html#download_cache). So treat a cache of installed files as trusted build input: diff --git a/e2e/backend/test_http_download_cache b/e2e/backend/test_http_download_cache index 7b75167a46d..4743ace8f26 100644 --- a/e2e/backend/test_http_download_cache +++ b/e2e/backend/test_http_download_cache @@ -24,6 +24,29 @@ assert_contains "mise x -- mytool" "cached tool ok" ENTRY="$MISE_CACHE_DIR/downloads-cas/sha256/$SHA" assert "test -f $ENTRY && echo kept" "kept" +# Without a pin, the server is asked whether the kept download is still current. +printf '#!/bin/sh\necho unpinned tool ok\n' >"$SRV/mytool2" +touch -d '2020-01-01' "$SRV/mytool2" +cat >mise.toml <mise.toml < Date: Fri, 9 Oct 2026 11:32:27 +0000 Subject: [PATCH 09/17] fix(http): copy into the download cache through a closed temp path Co-Authored-By: Claude Sonnet 5.5 Entire-Checkpoint: 01M4G6Z6VHA7G28DADC6A6JT5K --- src/download_cache.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/download_cache.rs b/src/download_cache.rs index a36ede38d73..7332e18c563 100644 --- a/src/download_cache.rs +++ b/src/download_cache.rs @@ -89,8 +89,9 @@ pub fn store(checksum: &str, src: &Path, effective_filename: Option<&str>) -> Re file::create_dir_all(dir)?; // Copied to a name no other install shares and renamed into place, so a // reader never sees a partial file and parallel stores can't truncate one. - let tmp = tempfile::NamedTempFile::new_in(dir)?; - std::fs::copy(src, tmp.path())?; + // A `TempPath` holds no open handle, which Windows needs to copy over it. + let tmp = tempfile::NamedTempFile::new_in(dir)?.into_temp_path(); + std::fs::copy(src, &tmp)?; if let Some(name) = effective_filename { file::write(name_path(&entry), name)?; } From 00a77aa44d939e53e81582eee8fb4212a3c12de9 Mon Sep 17 00:00:00 2001 From: jdx <216188+jdx@users.noreply.github.com> Date: Fri, 9 Oct 2026 11:40:42 +0000 Subject: [PATCH 10/17] fix(http): restore cached downloads atomically and state what the cache trusts Co-Authored-By: Claude Sonnet 5.5 Entire-Checkpoint: 01M4G7EAMNE8SPQTREKCE00HG6 --- crates/mise-util/src/http/revalidate.rs | 11 +++++++---- docs/dev-tools/mise-lock.md | 4 +++- settings.toml | 10 ++++++---- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/crates/mise-util/src/http/revalidate.rs b/crates/mise-util/src/http/revalidate.rs index 7c845c69e1b..c3b024d3125 100644 --- a/crates/mise-util/src/http/revalidate.rs +++ b/crates/mise-util/src/http/revalidate.rs @@ -53,10 +53,13 @@ impl Cached { let _ = file::remove_all(&self.dir); return Ok(false); } - if let Some(parent) = dest.parent() { - file::create_dir_all(parent)?; - } - file::copy(&stored, dest)?; + let parent = dest.parent().unwrap_or(Path::new(".")); + file::create_dir_all(parent)?; + // Renamed into place so an interrupted copy never leaves a truncated + // file at the destination. + let tmp = tempfile::NamedTempFile::new_in(parent)?.into_temp_path(); + file::copy(&stored, &tmp)?; + tmp.persist(dest).map_err(|err| err.error)?; // Marks the entry as recently used for eviction. if let Ok(f) = std::fs::File::options() .write(true) diff --git a/docs/dev-tools/mise-lock.md b/docs/dev-tools/mise-lock.md index 5df2d3f1a53..ae174aa33e0 100644 --- a/docs/dev-tools/mise-lock.md +++ b/docs/dev-tools/mise-lock.md @@ -425,7 +425,9 @@ such as Python and Ruby, it covers only the source tarball, not the compiled output. Downloads are different. mise keeps them in a cache and reuses one only when -the server confirms it is unchanged, or when it hashes to a pinned checksum. See +the server confirms it is unchanged, or when it hashes to a pinned checksum. A +pinned checksum is always checked after the download; without one, a cache that +someone can write to is trusted. See [`download_cache`](/configuration/settings.html#download_cache). So treat a cache of installed files as trusted build input: diff --git a/settings.toml b/settings.toml index 0d59bfcd864..bd7d15e6bda 100644 --- a/settings.toml +++ b/settings.toml @@ -120,10 +120,12 @@ docs = """ mise keeps finished downloads in `$MISE_CACHE_DIR/downloads-cache`, with the `ETag` or `Last-Modified` value the server sent. The next request for the same URL asks the server whether the file changed; on `304 Not Modified` mise uses -the kept file once it hashes to the recorded value again, so a restored or -shared cache can save a download but a modified entry is dropped and fetched -again. Servers that send no validator are not cached. The cache holds up to -2 GiB; the least recently used entries go first. +the kept file once it hashes to the recorded value again, so a damaged entry is +dropped and fetched again. The recorded hash lives beside the file, so this does +not stop someone who can write the cache from replacing both: pin a checksum, +which is always checked after the download, if the cache is shared. Servers that +send no validator are not cached. The cache holds up to 2 GiB; the least +recently used entries go first. When an `http` tool pins a checksum, in its options or in `mise.lock`, mise also keeps the download in `$MISE_CACHE_DIR/downloads-cas`, named by that checksum, From 8c5702b15c59e71870ecb15ad9e107c2a43aaa90 Mon Sep 17 00:00:00 2001 From: jdx <216188+jdx@users.noreply.github.com> Date: Fri, 9 Oct 2026 11:58:13 +0000 Subject: [PATCH 11/17] fix(settings): sort download_cache alphabetically Co-Authored-By: Claude Sonnet 5.5 Entire-Checkpoint: 01M4G8EC5FZPMN1A6Q37537Y7K --- schema/mise.json | 10 +++++----- settings.toml | 48 ++++++++++++++++++++++++------------------------ 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/schema/mise.json b/schema/mise.json index 3a8761900ca..8330e7e4db5 100644 --- a/schema/mise.json +++ b/schema/mise.json @@ -640,11 +640,6 @@ "description": "Keep downloaded archives and sources after installing, for debugging.", "type": "boolean" }, - "download_cache": { - "default": true, - "description": "Reuse a finished download when the server says it hasn't changed.", - "type": "boolean" - }, "always_keep_install": { "description": "Keep a failed installation's partial install directory and downloads, for debugging.", "type": "boolean" @@ -913,6 +908,11 @@ } } }, + "download_cache": { + "default": true, + "description": "Reuse a finished download when the server says it hasn't changed.", + "type": "boolean" + }, "enable_tools": { "description": "Allowlist of configured tools mise uses; other tools are ignored.", "type": "array", diff --git a/settings.toml b/settings.toml index bd7d15e6bda..e948d17bc99 100644 --- a/settings.toml +++ b/settings.toml @@ -113,30 +113,6 @@ them while debugging a backend or plugin. This is not a download cache; see env = "MISE_ALWAYS_KEEP_DOWNLOAD" type = "Bool" -[download_cache] -default = true -description = "Reuse a finished download when the server says it hasn't changed." -docs = """ -mise keeps finished downloads in `$MISE_CACHE_DIR/downloads-cache`, with the -`ETag` or `Last-Modified` value the server sent. The next request for the same -URL asks the server whether the file changed; on `304 Not Modified` mise uses -the kept file once it hashes to the recorded value again, so a damaged entry is -dropped and fetched again. The recorded hash lives beside the file, so this does -not stop someone who can write the cache from replacing both: pin a checksum, -which is always checked after the download, if the cache is shared. Servers that -send no validator are not cached. The cache holds up to 2 GiB; the least -recently used entries go first. - -When an `http` tool pins a checksum, in its options or in `mise.lock`, mise also -keeps the download in `$MISE_CACHE_DIR/downloads-cas`, named by that checksum, -and reuses it without asking the server. - -Extracted files are not covered; see -[Restored caches and installed tools](/dev-tools/mise-lock.html#restored-caches). -""" -env = "MISE_DOWNLOAD_CACHE" -type = "Bool" - [always_keep_install] description = "Keep a failed installation's partial install directory and downloads, for debugging." env = "MISE_ALWAYS_KEEP_INSTALL" @@ -695,6 +671,30 @@ an installation source to the dotnet CLI. env = "MISE_DOTNET_REGISTRY_URL" type = "Url" +[download_cache] +default = true +description = "Reuse a finished download when the server says it hasn't changed." +docs = """ +mise keeps finished downloads in `$MISE_CACHE_DIR/downloads-cache`, with the +`ETag` or `Last-Modified` value the server sent. The next request for the same +URL asks the server whether the file changed; on `304 Not Modified` mise uses +the kept file once it hashes to the recorded value again, so a damaged entry is +dropped and fetched again. The recorded hash lives beside the file, so this does +not stop someone who can write the cache from replacing both: pin a checksum, +which is always checked after the download, if the cache is shared. Servers that +send no validator are not cached. The cache holds up to 2 GiB; the least +recently used entries go first. + +When an `http` tool pins a checksum, in its options or in `mise.lock`, mise also +keeps the download in `$MISE_CACHE_DIR/downloads-cas`, named by that checksum, +and reuses it without asking the server. + +Extracted files are not covered; see +[Restored caches and installed tools](/dev-tools/mise-lock.html#restored-caches). +""" +env = "MISE_DOWNLOAD_CACHE" +type = "Bool" + [enable_tools] description = "Allowlist of configured tools mise uses; other tools are ignored." docs = """ From 5d7ee3e305c94f849233ce3c5230ee6556752b86 Mon Sep 17 00:00:00 2001 From: jdx <216188+jdx@users.noreply.github.com> Date: Fri, 9 Oct 2026 12:21:53 +0000 Subject: [PATCH 12/17] wip: merge the download caches into one content-addressed store Co-Authored-By: Claude Sonnet 5.5 Entire-Checkpoint: 01M4G9SPX3Y7V77E558M0MP9TQ --- crates/mise-util/src/http.rs | 92 ++++++- crates/mise-util/src/http/cas.rs | 287 ++++++++++++++++++++ crates/mise-util/src/http/revalidate.rs | 160 ----------- crates/mise-util/src/http/tests/download.rs | 1 + crates/mise-util/src/http/tests/mod.rs | 59 +++- e2e/backend/test_http_download_cache | 11 +- schema/mise.json | 2 +- settings.toml | 38 +-- src/backend/aqua.rs | 6 +- src/backend/github.rs | 15 +- src/backend/http.rs | 51 +--- src/backend/packslip.rs | 11 +- src/backend/static_helpers.rs | 15 + src/download_cache.rs | 121 --------- src/lib.rs | 1 - src/packslip.rs | 17 +- 16 files changed, 517 insertions(+), 370 deletions(-) create mode 100644 crates/mise-util/src/http/cas.rs delete mode 100644 crates/mise-util/src/http/revalidate.rs delete mode 100644 src/download_cache.rs diff --git a/crates/mise-util/src/http.rs b/crates/mise-util/src/http.rs index ad5d9c77694..ccd726c63e8 100644 --- a/crates/mise-util/src/http.rs +++ b/crates/mise-util/src/http.rs @@ -29,7 +29,7 @@ use crate::time::format_duration; use crate::{env, file}; use mise_settings::Settings; -mod revalidate; +mod cas; pub static HTTP: Lazy = Lazy::new(|| { Client::new_shared( @@ -225,6 +225,14 @@ impl DownloadValidator { } } +/// How a download may use the store of finished downloads. +#[derive(Clone, Copy)] +struct CacheMode<'a> { + /// The checksum the tool pinned, as `algo:hex`. + pin: Option<&'a str>, + enabled: bool, +} + /// What an attempt learned beyond the bytes it wrote. #[derive(Default)] struct AttemptOutcome { @@ -1261,6 +1269,35 @@ impl Client { path: &Path, headers: &HeaderMap, pr: Option<&dyn SingleReport>, + ) -> Result { + self.download_file_with_pin(url, path, headers, None, pr) + .await + } + + /// Like [`Client::download_file`], for a tool that pinned `pin` + /// (`algo:hex`). A download kept earlier under that checksum is used + /// without a request once it hashes to the pin again. The caller still + /// verifies the result. + pub async fn download_file_pinned( + &self, + url: U, + path: &Path, + pin: Option<&str>, + pr: Option<&dyn SingleReport>, + ) -> Result { + let url = url.into_url()?; + let headers = host_auth_headers(&url)?; + self.download_file_with_pin(url, path, &headers, pin, pr) + .await + } + + pub async fn download_file_with_pin( + &self, + url: U, + path: &Path, + headers: &HeaderMap, + pin: Option<&str>, + pr: Option<&dyn SingleReport>, ) -> Result { let url = url.into_url()?; if Settings::get().generate_lockfiles() { @@ -1279,6 +1316,7 @@ impl Client { url.clone(), &directory.path().join("artifact"), headers, + pin, pr, crate::network::http_download_timeout(&Settings::get()), ) @@ -1311,6 +1349,7 @@ impl Client { url, path, headers, + pin, pr, crate::network::http_download_timeout(&Settings::get()), ) @@ -1322,25 +1361,44 @@ impl Client { url: U, path: &Path, headers: &HeaderMap, + pin: Option<&str>, pr: Option<&dyn SingleReport>, total_timeout: Duration, ) -> Result { let use_cache = Settings::get().download_cache; - self.download_file_inner(url, path, headers, pr, total_timeout, use_cache) + let cache = CacheMode { + pin, + enabled: use_cache, + }; + self.download_file_inner(url, path, headers, cache, pr, total_timeout) .await } - /// With `use_cache`, a finished download of the same request is offered to - /// the server for revalidation and reused on `304 Not Modified`. + /// With `use_cache`, a download kept under `pin` is reused outright, and one + /// kept for the same request is offered to the server for revalidation and + /// reused on `304 Not Modified`. async fn download_file_inner( &self, url: U, path: &Path, headers: &HeaderMap, + cache: CacheMode<'_>, pr: Option<&dyn SingleReport>, total_timeout: Duration, - use_cache: bool, ) -> Result { + let CacheMode { + pin, + enabled: use_cache, + } = cache; + // A pinned download kept earlier needs no request, so this comes before + // the offline check. + if use_cache && let Some(hit) = pin.and_then(cas::lookup_pin) { + let effective_filename = hit.effective_filename.clone(); + let dest = path.to_path_buf(); + if tokio::task::spawn_blocking(move || hit.restore(&dest)).await?? { + return Ok(DownloadFileMetadata { effective_filename }); + } + } ensure!( !crate::network::offline(&Settings::get()), "offline mode is enabled" @@ -1366,12 +1424,12 @@ impl Client { let progress = Arc::new(DownloadProgress::default()); let request_hash = download_request_hash(&url, headers); let cached = if use_cache { - revalidate::lookup(&request_hash) + cas::lookup_url(&request_hash) } else { None }; let outcome = Arc::new(AttemptOutcome { - cond: cached.as_ref().map(|c| c.validator().clone()), + cond: cached.as_ref().map(|c| c.validator.clone()), ..Default::default() }); @@ -1432,29 +1490,41 @@ impl Client { { let effective_filename = cached.effective_filename(); let dest = path.clone(); - let restored = tokio::task::spawn_blocking(move || cached.restore(&dest)).await??; + let hit = cached.into_hit(); + let restored = tokio::task::spawn_blocking(move || hit.restore(&dest)).await??; if restored { return Ok(DownloadFileMetadata { effective_filename }); } // The stored file no longer matches what was recorded: fetch it again. drop(_download_lock); + let cache = CacheMode { + pin, + enabled: false, + }; return Box::pin(self.download_file_inner( url, &path, headers, + cache, pr, total_timeout, - false, )) .await; } let persist_path = path.clone(); tokio::task::spawn_blocking(move || partial.persist(&persist_path)).await??; let validator = outcome.validator.lock().unwrap().take(); - if use_cache && let Some(validator) = validator { + if use_cache { let effective_filename = metadata.effective_filename.clone(); + let pin = pin.map(str::to_string); let stored = tokio::task::spawn_blocking(move || { - revalidate::store(&request_hash, &path, &validator, effective_filename) + cas::store( + &request_hash, + pin.as_deref(), + &path, + validator.as_ref(), + effective_filename, + ) }) .await; if let Ok(Err(err)) | Err(err) = stored.map_err(Report::from) { diff --git a/crates/mise-util/src/http/cas.rs b/crates/mise-util/src/http/cas.rs new file mode 100644 index 00000000000..f31e8687db4 --- /dev/null +++ b/crates/mise-util/src/http/cas.rs @@ -0,0 +1,287 @@ +//! A content-addressed store of finished downloads. +//! +//! Files are kept by their BLAKE3 hash under `blobs/`. Two small indexes point +//! into them: +//! +//! - `urls/`: a request (URL and headers) to the blob it last returned and the +//! validator (`ETag` or `Last-Modified`) the server sent with it, so a later +//! request can ask the server whether it changed and take `304 Not Modified` +//! as an answer. +//! - `pins/`: a checksum a tool pinned to a blob, so a download can be reused +//! with no request at all. +//! +//! Nothing in an index is trusted on its own. A blob is hashed again before +//! it is used, against the pin when there is one, and an entry that no longer +//! matches is dropped. + +use std::path::{Path, PathBuf}; +use std::time::SystemTime; + +use eyre::Result; +use serde::{Deserialize, Serialize}; + +use super::DownloadValidator; +use crate::file; + +/// Blobs are evicted, least recently used first, past this total size. +const MAX_CACHE_BYTES: u64 = 2 * 1024 * 1024 * 1024; + +/// Algorithms a pin may use. `md5` and `sha1` are not collision resistant +/// enough to name content by. +const PIN_ALGOS: &[&str] = &["sha256", "sha384", "sha512", "blake3"]; + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct UrlEntry { + validator: DownloadValidator, + blake3: String, + effective_filename: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct PinEntry { + blake3: String, + effective_filename: Option, +} + +/// Tests that don't ask for a directory never touch the user's cache. +#[cfg(test)] +pub(super) static TEST_ROOT: std::sync::Mutex> = std::sync::Mutex::new(None); + +fn root() -> Option { + #[cfg(test)] + { + TEST_ROOT.lock().unwrap().clone() + } + #[cfg(not(test))] + { + Some(crate::dirs::CACHE.join("downloads-cas")) + } +} + +fn is_hex(value: &str) -> bool { + !value.is_empty() && value.bytes().all(|b| b.is_ascii_hexdigit()) +} + +fn blob_path(root: &Path, blake3: &str) -> PathBuf { + root.join("blobs").join(blake3) +} + +fn url_path(root: &Path, request_hash: &str) -> PathBuf { + root.join("urls").join(format!("{request_hash}.json")) +} + +/// Split a pin into its algorithm and lowercase digest, or `None` when it +/// could not safely name a file. +fn parse_pin(pin: &str) -> Option<(&str, String)> { + let (algo, hex) = pin.split_once(':')?; + (PIN_ALGOS.contains(&algo) && is_hex(hex)).then(|| (algo, hex.to_lowercase())) +} + +fn pin_path(root: &Path, algo: &str, hex: &str) -> PathBuf { + root.join("pins").join(algo).join(format!("{hex}.json")) +} + +/// A stored download ready to be copied out. +pub(super) struct Hit { + blob: PathBuf, + pub(super) effective_filename: Option, + /// The index entry that led here, dropped if the blob doesn't verify. + entry: PathBuf, + verify: Verify, +} + +enum Verify { + /// The blob must hash to its own name. + Blake3(String), + /// The blob must hash to the tool's pinned checksum. + Pin { algo: String, hex: String }, +} + +/// What the server was last asked about, for a conditional request. +pub(super) struct Cached { + pub(super) validator: DownloadValidator, + hit: Hit, +} + +impl Cached { + pub(super) fn into_hit(self) -> Hit { + self.hit + } + + pub(super) fn effective_filename(&self) -> Option { + self.hit.effective_filename.clone() + } +} + +pub(super) fn lookup_url(request_hash: &str) -> Option { + let root = root()?; + let entry = url_path(&root, request_hash); + let url: UrlEntry = serde_json::from_slice(&std::fs::read(&entry).ok()?).ok()?; + if !url.validator.is_valid() || !is_hex(&url.blake3) { + return None; + } + let blob = blob_path(&root, &url.blake3); + if !blob.is_file() { + return None; + } + Some(Cached { + validator: url.validator, + hit: Hit { + blob, + effective_filename: url.effective_filename, + entry, + verify: Verify::Blake3(url.blake3), + }, + }) +} + +pub(super) fn lookup_pin(pin: &str) -> Option { + let root = root()?; + let (algo, hex) = parse_pin(pin)?; + let entry = pin_path(&root, algo, &hex); + let pinned: PinEntry = serde_json::from_slice(&std::fs::read(&entry).ok()?).ok()?; + if !is_hex(&pinned.blake3) { + return None; + } + let blob = blob_path(&root, &pinned.blake3); + if !blob.is_file() { + return None; + } + Some(Hit { + blob, + effective_filename: pinned.effective_filename, + entry, + verify: Verify::Pin { + algo: algo.to_string(), + hex, + }, + }) +} + +impl Hit { + /// Copy the blob to `dest` if it still hashes to what it is expected to. + /// Otherwise drop what pointed at it and return `false`. + pub(super) fn restore(&self, dest: &Path) -> Result { + let intact = match &self.verify { + Verify::Blake3(expected) => { + crate::hash::file_hash_blake3(&self.blob, None).is_ok_and(|hash| hash == *expected) + } + Verify::Pin { algo, hex } => { + crate::hash::ensure_checksum(&self.blob, hex, None, algo).is_ok() + } + }; + if !intact { + let _ = file::remove_file(&self.entry); + if matches!(self.verify, Verify::Blake3(_)) { + let _ = file::remove_file(&self.blob); + } + return Ok(false); + } + let parent = dest.parent().unwrap_or(Path::new(".")); + file::create_dir_all(parent)?; + // Renamed into place so an interrupted copy never leaves a truncated + // file at the destination. A `TempPath` holds no open handle, which + // Windows needs to copy over it. + let tmp = tempfile::NamedTempFile::new_in(parent)?.into_temp_path(); + file::copy(&self.blob, &tmp)?; + tmp.persist(dest).map_err(|err| err.error)?; + // Marks the blob as recently used for eviction. + if let Ok(f) = std::fs::File::options().write(true).open(&self.blob) { + let _ = f.set_modified(SystemTime::now()); + } + Ok(true) + } +} + +/// Write `contents` to `path` so a reader never sees a partial file. +fn write_atomic(path: &Path, contents: &[u8]) -> Result<()> { + let parent = path.parent().expect("index paths have a parent"); + file::create_dir_all(parent)?; + let tmp = tempfile::NamedTempFile::new_in(parent)?.into_temp_path(); + file::write(&tmp, contents)?; + tmp.persist(path).map_err(|err| err.error)?; + Ok(()) +} + +/// Keep `path`, just downloaded. `validator` lets the server confirm it later; +/// `pin` lets a tool that pinned the same checksum skip the request. +pub(super) fn store( + request_hash: &str, + pin: Option<&str>, + path: &Path, + validator: Option<&DownloadValidator>, + effective_filename: Option, +) -> Result<()> { + let Some(root) = root() else { + return Ok(()); + }; + // The caller checks the pin after the download. Checking here too keeps a + // wrong file from ever being indexed under it. + let pin = pin.and_then(parse_pin).filter(|(algo, hex)| { + let ok = crate::hash::ensure_checksum(path, hex, None, algo).is_ok(); + if !ok { + debug!("not indexing a download that doesn't match its pin"); + } + ok + }); + if validator.is_none() && pin.is_none() { + return Ok(()); + } + let blake3 = crate::hash::file_hash_blake3(path, None)?; + let blob = blob_path(&root, &blake3); + if !blob.is_file() { + let parent = blob.parent().expect("blob paths have a parent"); + file::create_dir_all(parent)?; + let tmp = tempfile::NamedTempFile::new_in(parent)?.into_temp_path(); + file::copy(path, &tmp)?; + tmp.persist(&blob).map_err(|err| err.error)?; + } + if let Some(validator) = validator { + let entry = UrlEntry { + validator: validator.clone(), + blake3: blake3.clone(), + effective_filename: effective_filename.clone(), + }; + write_atomic(&url_path(&root, request_hash), &serde_json::to_vec(&entry)?)?; + } + if let Some((algo, hex)) = pin { + let entry = PinEntry { + blake3, + effective_filename, + }; + write_atomic(&pin_path(&root, algo, &hex), &serde_json::to_vec(&entry)?)?; + } + evict(&root.join("blobs")); + Ok(()) +} + +fn evict(blobs: &Path) { + let Ok(read) = std::fs::read_dir(blobs) else { + return; + }; + let mut entries = Vec::new(); + let mut total = 0u64; + for entry in read.flatten() { + let Ok(meta) = entry.metadata() else { + continue; + }; + if !meta.is_file() || entry.file_name().to_string_lossy().starts_with('.') { + continue; + } + total += meta.len(); + entries.push(( + meta.modified().unwrap_or(SystemTime::UNIX_EPOCH), + meta.len(), + entry.path(), + )); + } + entries.sort(); + for (_, size, path) in entries { + if total <= MAX_CACHE_BYTES { + break; + } + if file::remove_file(&path).is_ok() { + total -= size; + } + } +} diff --git a/crates/mise-util/src/http/revalidate.rs b/crates/mise-util/src/http/revalidate.rs deleted file mode 100644 index c3b024d3125..00000000000 --- a/crates/mise-util/src/http/revalidate.rs +++ /dev/null @@ -1,160 +0,0 @@ -//! Reuse of a finished download when the server says it hasn't changed. -//! -//! Each entry holds the file, the validator (`ETag` or `Last-Modified`) the -//! server sent with it, and the file's BLAKE3 hash. The next request for the -//! same URL and headers carries that validator; a `304 Not Modified` answers -//! with no body, and the stored file is used after it hashes to the recorded -//! value again. - -use std::path::{Path, PathBuf}; -use std::time::SystemTime; - -use eyre::Result; -use serde::{Deserialize, Serialize}; - -use super::DownloadValidator; -use crate::file; - -/// Entries are evicted, oldest use first, past this total size. -const MAX_CACHE_BYTES: u64 = 2 * 1024 * 1024 * 1024; - -const FILE: &str = "file"; -const META: &str = "meta.json"; - -#[derive(Debug, Clone, Serialize, Deserialize)] -struct Meta { - validator: DownloadValidator, - blake3: String, - effective_filename: Option, -} - -pub(super) struct Cached { - dir: PathBuf, - meta: Meta, -} - -impl Cached { - pub(super) fn validator(&self) -> &DownloadValidator { - &self.meta.validator - } - - pub(super) fn effective_filename(&self) -> Option { - self.meta.effective_filename.clone() - } - - /// Copy the stored file to `dest` if it still hashes to what was recorded. - /// An entry that doesn't is removed. - pub(super) fn restore(&self, dest: &Path) -> Result { - let stored = self.dir.join(FILE); - let intact = stored.is_file() - && crate::hash::file_hash_blake3(&stored, None) - .is_ok_and(|hash| hash == self.meta.blake3); - if !intact { - let _ = file::remove_all(&self.dir); - return Ok(false); - } - let parent = dest.parent().unwrap_or(Path::new(".")); - file::create_dir_all(parent)?; - // Renamed into place so an interrupted copy never leaves a truncated - // file at the destination. - let tmp = tempfile::NamedTempFile::new_in(parent)?.into_temp_path(); - file::copy(&stored, &tmp)?; - tmp.persist(dest).map_err(|err| err.error)?; - // Marks the entry as recently used for eviction. - if let Ok(f) = std::fs::File::options() - .write(true) - .open(self.dir.join(META)) - { - let _ = f.set_modified(SystemTime::now()); - } - Ok(true) - } -} - -/// Tests that don't ask for a directory never touch the user's cache. -#[cfg(test)] -pub(super) static TEST_ROOT: std::sync::Mutex> = std::sync::Mutex::new(None); - -fn root() -> Option { - #[cfg(test)] - { - TEST_ROOT.lock().unwrap().clone() - } - #[cfg(not(test))] - { - Some(crate::dirs::CACHE.join("downloads-cache")) - } -} - -pub(super) fn lookup(request_hash: &str) -> Option { - let dir = root()?.join(request_hash); - let meta: Meta = serde_json::from_slice(&std::fs::read(dir.join(META)).ok()?).ok()?; - meta.validator.is_valid().then_some(Cached { dir, meta }) -} - -/// Keep `path`, just downloaded with `validator`. -pub(super) fn store( - request_hash: &str, - path: &Path, - validator: &DownloadValidator, - effective_filename: Option, -) -> Result<()> { - let Some(root) = root() else { - return Ok(()); - }; - let blake3 = crate::hash::file_hash_blake3(path, None)?; - let dir = root.join(request_hash); - let parent = dir.parent().expect("entry_dir joins a name"); - file::create_dir_all(parent)?; - // Built beside the entry under a name nothing else shares, then swapped in. - let staging = tempfile::Builder::new() - .prefix(".staging-") - .tempdir_in(parent)?; - file::copy(path, staging.path().join(FILE))?; - let meta = Meta { - validator: validator.clone(), - blake3, - effective_filename, - }; - file::write(staging.path().join(META), serde_json::to_vec(&meta)?)?; - let _ = file::remove_all(&dir); - let staged = staging.keep(); - if let Err(err) = std::fs::rename(&staged, &dir) { - let _ = file::remove_all(&staged); - // Another process stored the same download first. - if !dir.join(META).is_file() { - return Err(err.into()); - } - } - evict(parent); - Ok(()) -} - -fn evict(root: &Path) { - let Ok(read) = std::fs::read_dir(root) else { - return; - }; - let mut entries = Vec::new(); - let mut total = 0u64; - for entry in read.flatten() { - let path = entry.path(); - if entry.file_name().to_string_lossy().starts_with('.') { - continue; - } - let size = std::fs::metadata(path.join(FILE)).map_or(0, |m| m.len()); - let used = std::fs::metadata(path.join(META)) - .and_then(|m| m.modified()) - .unwrap_or(SystemTime::UNIX_EPOCH); - total += size; - entries.push((used, size, path)); - } - entries.sort(); - for (_, size, path) in entries { - if total <= MAX_CACHE_BYTES { - break; - } - if file::remove_all(&path).is_ok() { - total -= size; - } - } -} diff --git a/crates/mise-util/src/http/tests/download.rs b/crates/mise-util/src/http/tests/download.rs index 8df72050955..4ffba65e3ad 100644 --- a/crates/mise-util/src/http/tests/download.rs +++ b/crates/mise-util/src/http/tests/download.rs @@ -445,6 +445,7 @@ async fn test_download_total_timeout_bounds_trickling_response() { &path, &HeaderMap::new(), None, + None, Duration::from_millis(500), ), ) diff --git a/crates/mise-util/src/http/tests/mod.rs b/crates/mise-util/src/http/tests/mod.rs index e0eb11843fa..52cdaf50ae6 100644 --- a/crates/mise-util/src/http/tests/mod.rs +++ b/crates/mise-util/src/http/tests/mod.rs @@ -578,7 +578,7 @@ fn not_modified_response() -> &'static str { async fn test_download_reuses_stored_file_on_not_modified() { let _settings = set_test_http_retries(0); let cache = tempfile::tempdir().unwrap(); - *revalidate::TEST_ROOT.lock().unwrap() = Some(cache.path().to_path_buf()); + *cas::TEST_ROOT.lock().unwrap() = Some(cache.path().to_path_buf()); let (port, count, requests) = spawn_recording_server(vec![ cacheable_download_response(), not_modified_response(), @@ -606,18 +606,67 @@ async fn test_download_reuses_stored_file_on_not_modified() { // A stored file that no longer hashes to what was recorded is not used, // even though the server says it is current. - let stored = std::fs::read_dir(cache.path()) + let stored = std::fs::read_dir(cache.path().join("blobs")) .unwrap() .next() .unwrap() .unwrap() - .path() - .join("file"); + .path(); std::fs::write(&stored, b"tampered").unwrap(); std::fs::remove_file(&dest).unwrap(); client.download_file(&url, &dest, None).await.unwrap(); assert_eq!(std::fs::read(&dest).unwrap(), b"hello"); assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 4); - *revalidate::TEST_ROOT.lock().unwrap() = None; + *cas::TEST_ROOT.lock().unwrap() = None; +} + +#[tokio::test(flavor = "current_thread")] +async fn test_download_reuses_pinned_file_without_a_request() { + let _settings = set_test_http_retries(0); + let cache = tempfile::tempdir().unwrap(); + *cas::TEST_ROOT.lock().unwrap() = Some(cache.path().to_path_buf()); + let (port, count) = spawn_canned_server(vec![cacheable_download_response()]).await; + let url = format!("http://127.0.0.1:{port}/artifact"); + let client = Client::new(Duration::from_secs(2), ClientKind::Http).unwrap(); + let dir = tempfile::tempdir().unwrap(); + let dest = dir.path().join("artifact"); + let sha = crate::hash::file_hash_prog::; + let hello = dir.path().join("hello"); + std::fs::write(&hello, b"hello").unwrap(); + let pin = format!("sha256:{}", sha(&hello, None).unwrap()); + + client + .download_file_pinned(&url, &dest, Some(&pin), None) + .await + .unwrap(); + assert_eq!(std::fs::read(&dest).unwrap(), b"hello"); + + // A different URL with the same pin is served without any request. + std::fs::remove_file(&dest).unwrap(); + let other = "http://127.0.0.1:1/elsewhere"; + client + .download_file_pinned(other, &dest, Some(&pin), None) + .await + .unwrap(); + assert_eq!(std::fs::read(&dest).unwrap(), b"hello"); + assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 1); + + // A blob that no longer matches the pin is not used. + let blob = std::fs::read_dir(cache.path().join("blobs")) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + std::fs::write(&blob, b"tampered").unwrap(); + std::fs::remove_file(&dest).unwrap(); + assert!( + client + .download_file_pinned(other, &dest, Some(&pin), None) + .await + .is_err() + ); + + *cas::TEST_ROOT.lock().unwrap() = None; } diff --git a/e2e/backend/test_http_download_cache b/e2e/backend/test_http_download_cache index 4743ace8f26..366084af37d 100644 --- a/e2e/backend/test_http_download_cache +++ b/e2e/backend/test_http_download_cache @@ -21,8 +21,9 @@ EOF mise install assert_contains "mise x -- mytool" "cached tool ok" -ENTRY="$MISE_CACHE_DIR/downloads-cas/sha256/$SHA" +ENTRY="$MISE_CACHE_DIR/downloads-cas/pins/sha256/$SHA.json" assert "test -f $ENTRY && echo kept" "kept" +BLOB="$(find "$MISE_CACHE_DIR/downloads-cas/blobs" -type f | head -1)" # Without a pin, the server is asked whether the kept download is still current. printf '#!/bin/sh\necho unpinned tool ok\n' >"$SRV/mytool2" @@ -35,7 +36,7 @@ url = "http://127.0.0.1:${HTTP_PORT}/mytool2" EOF mise install assert_contains "mise x -- mytool2" "unpinned tool ok" -assert "ls $MISE_CACHE_DIR/downloads-cache | wc -l | tr -d ' '" "1" +assert "ls $MISE_CACHE_DIR/downloads-cas/urls | wc -l | tr -d ' '" "1" mise uninstall http:cas-unpinned@1.0.0 mise install assert_contains "mise x -- mytool2" "unpinned tool ok" @@ -54,9 +55,9 @@ assert_fail "test -d $MISE_DATA_DIR/installs/http-cas-tool/1.0.0" mise install assert_contains "mise x -- mytool" "cached tool ok" -# A kept file that no longer matches the pin is never installed. The server is -# down, so the install fails, and the bad entry is removed. -printf tampered >>"$ENTRY" +# A kept file that no longer matches is never installed. The server is down, so +# the install fails, and the pin entry that pointed at it is removed. +printf tampered >>"$BLOB" mise uninstall http:cas-tool@1.0.0 assert_fail "mise install" assert_fail "test -f $ENTRY" diff --git a/schema/mise.json b/schema/mise.json index 8330e7e4db5..a67a6c6e9e4 100644 --- a/schema/mise.json +++ b/schema/mise.json @@ -910,7 +910,7 @@ }, "download_cache": { "default": true, - "description": "Reuse a finished download when the server says it hasn't changed.", + "description": "Reuse finished downloads instead of fetching them again.", "type": "boolean" }, "enable_tools": { diff --git a/settings.toml b/settings.toml index e948d17bc99..c5f8aec395f 100644 --- a/settings.toml +++ b/settings.toml @@ -673,23 +673,27 @@ type = "Url" [download_cache] default = true -description = "Reuse a finished download when the server says it hasn't changed." -docs = """ -mise keeps finished downloads in `$MISE_CACHE_DIR/downloads-cache`, with the -`ETag` or `Last-Modified` value the server sent. The next request for the same -URL asks the server whether the file changed; on `304 Not Modified` mise uses -the kept file once it hashes to the recorded value again, so a damaged entry is -dropped and fetched again. The recorded hash lives beside the file, so this does -not stop someone who can write the cache from replacing both: pin a checksum, -which is always checked after the download, if the cache is shared. Servers that -send no validator are not cached. The cache holds up to 2 GiB; the least -recently used entries go first. - -When an `http` tool pins a checksum, in its options or in `mise.lock`, mise also -keeps the download in `$MISE_CACHE_DIR/downloads-cas`, named by that checksum, -and reuses it without asking the server. - -Extracted files are not covered; see +description = "Reuse finished downloads instead of fetching them again." +docs = """ +mise keeps finished downloads in `$MISE_CACHE_DIR/downloads-cas`, named by their +hash, and uses one again in two cases: + +- **The server confirms it is unchanged.** The next request for the same URL + carries the `ETag` or `Last-Modified` value the server sent, and a + `304 Not Modified` answer means the kept file is used instead of a new + download. Servers that send no validator are not cached. +- **A tool pinned its checksum**, in its options or in `mise.lock`. The kept + file is used with no request at all, even if the URL changed. + +The `http`, `github` and `aqua` backends pin; every other download that goes +through mise's HTTP client uses the first case. + +A kept file is hashed again before it is used, and one that no longer matches is +dropped and downloaded again. The recorded hash lives beside the file, so this +catches damage but does not stop someone who can write the cache from replacing +both. A pinned checksum is always checked after the download, so pin one if the +cache is shared. The cache holds up to 2 GiB; the least recently used entries go +first. Extracted files are not covered; see [Restored caches and installed tools](/dev-tools/mise-lock.html#restored-caches). """ env = "MISE_DOWNLOAD_CACHE" diff --git a/src/backend/aqua.rs b/src/backend/aqua.rs index aca12c45cf3..1d8174fa7d1 100644 --- a/src/backend/aqua.rs +++ b/src/backend/aqua.rs @@ -2824,7 +2824,11 @@ impl AquaBackend { return Ok(()); } ctx.pr.set_message(format!("download {filename}")); - HTTP.download_file(url, &tarball_path, Some(ctx.pr.as_ref())) + let pinned = tv + .lock_platforms + .get(&self.get_platform_key()) + .and_then(|platform| platform.checksum.clone()); + HTTP.download_file_pinned(url, &tarball_path, pinned.as_deref(), Some(ctx.pr.as_ref())) .await?; Ok(()) } diff --git a/src/backend/github.rs b/src/backend/github.rs index 6924981efeb..ba434dc10be 100644 --- a/src/backend/github.rs +++ b/src/backend/github.rs @@ -6,8 +6,8 @@ use crate::backend::options::{BackendOptions, VersionOrder}; use crate::backend::platform_target::PlatformTarget; use crate::backend::static_helpers::{ ArchiveLayout, get_filename_from_url, install_artifact, lookup_platform_key, - lookup_with_fallback, template_string, try_with_v_prefix, try_with_v_prefix_and_repo, - verify_artifact, + lookup_with_fallback, pinned_checksum, template_string, try_with_v_prefix, + try_with_v_prefix_and_repo, verify_artifact, }; use crate::backend::{MISE_BINS_DIR, SecurityFeature, runtime_path_for_install_path}; use crate::config::{Config, Settings, SettingsExt}; @@ -1715,8 +1715,15 @@ impl UnifiedGitBackend { }; ctx.pr.set_message(format!("download {filename}")); - HTTP.download_file_with_headers(url, &file_path, &headers, Some(ctx.pr.as_ref())) - .await?; + let pinned = pinned_checksum(tv, &self.get_platform_key(), opts.raw()); + HTTP.download_file_with_pin( + url, + &file_path, + &headers, + pinned.as_deref(), + Some(ctx.pr.as_ref()), + ) + .await?; // Verify and install ctx.pr.next_operation(); diff --git a/src/backend/http.rs b/src/backend/http.rs index 84b291b8404..2b181d1172b 100644 --- a/src/backend/http.rs +++ b/src/backend/http.rs @@ -9,13 +9,12 @@ use crate::backend::static_helpers::{ apply_rename_exe, bin_name_for_download, clean_binary_name, ensure_plain_bin_name, ensure_safe_relative_bin_path, eval_checksum_expr, fetch_checksum_from_file_with_headers, fetch_checksum_from_shasums_with_headers, get_filename_from_url, lookup_value_with_fallback, - rename_binary_name, shasums_has_entries, template_string, template_string_for_target, - template_string_strict, verify_artifact, + pinned_checksum, rename_binary_name, shasums_has_entries, template_string, + template_string_for_target, template_string_strict, verify_artifact, }; use crate::backend::version_list; use crate::config::Config; use crate::config::Settings; -use crate::download_cache; use crate::http::{DownloadFileMetadata, HTTP}; use crate::install_context::InstallContext; use crate::lockfile::PlatformInfo; @@ -1375,28 +1374,10 @@ impl Backend for HttpBackend { .and_then(|p| p.checksum.as_ref()) .is_some(); - // A checksum the tool pinned names the artifact, so a copy kept from an - // earlier install can stand in for the download once it hashes to it. - let pinned = opts.checksum().or_else(|| { - tv.lock_platforms - .get(&platform_key) - .and_then(|p| p.checksum.clone()) - }); - let local_src = local_artifact_path(&url)?; - let restored = match (&pinned, &local_src) { - (Some(checksum), None) => { - ctx.pr.set_message(format!("cached {filename}")); - download_cache::restore(checksum, &file_path, Some(ctx.pr.as_ref()))? - } - _ => None, - }; - let from_cache = restored.is_some(); + let pinned = pinned_checksum(&tv, &platform_key, opts.raw()); - let download = match (restored, local_src) { - (Some(restored), _) => DownloadFileMetadata { - effective_filename: restored.effective_filename, - }, - (None, Some(src)) => { + let download = match local_artifact_path(&url)? { + Some(src) => { ctx.pr.set_message(format!("copy {filename}")); if let Some(parent) = file_path.parent() { file::create_dir_all(parent)?; @@ -1404,20 +1385,26 @@ impl Backend for HttpBackend { file::run_blocking(|| file::copy(&src, &file_path))?; DownloadFileMetadata::default() } - (None, None) => { + None => { ctx.pr.set_message(format!("download {filename}")); let headers = opts.headers(Some(tv.version.as_str()))?; if headers.is_empty() { - HTTP.download_file_with_metadata(&url, &file_path, Some(ctx.pr.as_ref())) - .await? + HTTP.download_file_pinned( + &url, + &file_path, + pinned.as_deref(), + Some(ctx.pr.as_ref()), + ) + .await? } else { // Keep the automatic host token; configured headers override it. let headers = crate::http::with_host_auth(&reqwest::Url::parse(&url)?, &headers)?; - HTTP.download_file_with_headers_metadata( + HTTP.download_file_with_pin( &url, &file_path, &headers, + pinned.as_deref(), Some(ctx.pr.as_ref()), ) .await? @@ -1482,14 +1469,6 @@ impl Backend for HttpBackend { } self.verify_checksum(ctx, &mut tv, &file_path)?; - // Verified against the pin above, so it is safe to keep for next time. - if let Some(checksum) = pinned.as_ref().filter(|_| !from_cache) - && let Err(err) = - download_cache::store(checksum, &file_path, download.effective_filename.as_deref()) - { - debug!("could not keep {filename} in the download cache: {err}"); - } - Ok(tv) } diff --git a/src/backend/packslip.rs b/src/backend/packslip.rs index ecedf0b0bb5..1657e275f72 100644 --- a/src/backend/packslip.rs +++ b/src/backend/packslip.rs @@ -1656,7 +1656,8 @@ impl PackslipBackend { let bundle_path = tv.download_path().join(bundle_name(&project)); file::create_dir_all(tv.download_path())?; ctx.pr.set_message("download packslip".into()); - crate::packslip::download_file(&located.url, &bundle_path, Some(ctx.pr.as_ref())).await?; + crate::packslip::download_file(&located.url, &bundle_path, None, Some(ctx.pr.as_ref())) + .await?; let pinned: Vec<&String> = located.digest.iter().chain(vendor_digest.iter()).collect(); if !pinned.is_empty() { let (actual, _) = packslip::digest_file(&bundle_path)?; @@ -1813,7 +1814,13 @@ impl PackslipBackend { let file_path = tv.download_path().join(&artifact.name); ctx.pr.next_operation(); ctx.pr.set_message(format!("download {}", artifact.name)); - crate::packslip::download_file(&url, &file_path, Some(ctx.pr.as_ref())).await?; + // The signed digest names the artifact, so a copy kept earlier can stand in + // for the download; `verify_bundle` below still checks the result. + let signed = statement + .digest_of(&artifact.name) + .map(|sha256| format!("sha256:{sha256}")); + crate::packslip::download_file(&url, &file_path, signed.as_deref(), Some(ctx.pr.as_ref())) + .await?; // The signed digest and size first, then what the lockfile remembers: // a lock entry written from an earlier packslip keeps its checksum and diff --git a/src/backend/static_helpers.rs b/src/backend/static_helpers.rs index ff0358068ef..e8f080df682 100644 --- a/src/backend/static_helpers.rs +++ b/src/backend/static_helpers.rs @@ -932,6 +932,21 @@ fn make_configured_bin_executable(search_dir: &Path, bin_name: &str) -> Result<( Ok(()) } +/// The checksum a tool pinned for this platform, in its options or in the +/// lockfile. It names a kept download that can be reused, and is still checked +/// after the download either way. +pub(crate) fn pinned_checksum( + tv: &crate::toolset::ToolVersion, + platform_key: &str, + opts: &crate::toolset::ToolVersionOptions, +) -> Option { + lookup_with_fallback(opts, "checksum").or_else(|| { + tv.lock_platforms + .get(platform_key) + .and_then(|platform| platform.checksum.clone()) + }) +} + pub(crate) fn verify_artifact( _tv: &crate::toolset::ToolVersion, file_path: &Path, diff --git a/src/download_cache.rs b/src/download_cache.rs deleted file mode 100644 index 7332e18c563..00000000000 --- a/src/download_cache.rs +++ /dev/null @@ -1,121 +0,0 @@ -//! A content-addressed cache of verified download artifacts. -//! -//! Entries are keyed by the checksum a tool pinned (in its options or in -//! `mise.lock`), never by URL, and every entry is hashed again before it is -//! used. A cache that was restored, shared or edited can therefore cost a -//! download but never changes what is installed. - -use std::path::{Path, PathBuf}; - -use crate::config::Settings; -use crate::file; -use crate::hash; -use crate::result::Result; -use crate::ui::progress_report::SingleReport; - -/// Algorithms an entry may be keyed by. `md5` and `sha1` are not collision -/// resistant enough to address content by. -const ALGOS: &[&str] = &["sha256", "sha384", "sha512", "blake3"]; - -/// A cached artifact and the file name it was downloaded as. -pub struct Restored { - pub effective_filename: Option, -} - -fn root() -> PathBuf { - crate::dirs::CACHE.join("downloads-cas") -} - -/// The entry for a pinned `algo:hex` checksum, or `None` when the checksum -/// cannot safely name a file. -fn entry(checksum: &str) -> Option { - let (algo, hex) = checksum.split_once(':')?; - let hex = hex.to_lowercase(); - if !ALGOS.contains(&algo) || hex.is_empty() || !hex.bytes().all(|b| b.is_ascii_hexdigit()) { - return None; - } - Some(root().join(algo).join(hex)) -} - -fn name_path(entry: &Path) -> PathBuf { - entry.with_extension("name") -} - -/// Copy the artifact pinned by `checksum` to `dest` if the cache holds it and -/// it still hashes to `checksum`. An entry that does not is removed. -pub fn restore( - checksum: &str, - dest: &Path, - pr: Option<&dyn SingleReport>, -) -> Result> { - if !Settings::get().download_cache { - return Ok(None); - } - let Some(entry) = entry(checksum) else { - return Ok(None); - }; - if !entry.is_file() { - return Ok(None); - } - let (algo, hex) = checksum.split_once(':').expect("entry() checked the form"); - if let Err(err) = hash::ensure_checksum(&entry, hex, pr, algo) { - debug!("dropping download cache entry {}: {err}", entry.display()); - let _ = file::remove_file(&entry); - let _ = file::remove_file(name_path(&entry)); - return Ok(None); - } - if let Some(parent) = dest.parent() { - file::create_dir_all(parent)?; - } - file::copy(&entry, dest)?; - let effective_filename = std::fs::read_to_string(name_path(&entry)) - .ok() - .filter(|name| !name.is_empty()); - Ok(Some(Restored { effective_filename })) -} - -/// Keep `src`, which the caller has verified against `checksum`. -pub fn store(checksum: &str, src: &Path, effective_filename: Option<&str>) -> Result<()> { - if !Settings::get().download_cache { - return Ok(()); - } - let Some(entry) = entry(checksum) else { - return Ok(()); - }; - if entry.is_file() { - return Ok(()); - } - let dir = entry.parent().expect("entry() joins a file name"); - file::create_dir_all(dir)?; - // Copied to a name no other install shares and renamed into place, so a - // reader never sees a partial file and parallel stores can't truncate one. - // A `TempPath` holds no open handle, which Windows needs to copy over it. - let tmp = tempfile::NamedTempFile::new_in(dir)?.into_temp_path(); - std::fs::copy(src, &tmp)?; - if let Some(name) = effective_filename { - file::write(name_path(&entry), name)?; - } - tmp.persist(&entry).map_err(|err| err.error)?; - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn entry_accepts_pinned_hex_digests_only() { - let sha = format!("sha256:{}", "ab".repeat(32)); - assert!( - entry(&sha) - .unwrap() - .ends_with(format!("sha256/{}", "ab".repeat(32))) - ); - assert!(entry(&sha.to_uppercase().replace("SHA256", "sha256")).is_some()); - assert!(entry("md5:d41d8cd98f00b204e9800998ecf8427e").is_none()); - assert!(entry("sha1:da39a3ee5e6b4b0d3255bfef95601890afd80709").is_none()); - assert!(entry("sha256:../../etc/passwd").is_none()); - assert!(entry("sha256:").is_none()); - assert!(entry("deadbeef").is_none()); - } -} diff --git a/src/lib.rs b/src/lib.rs index 7d0e8e96518..acfff7290d0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -44,7 +44,6 @@ pub mod deps; pub(crate) mod deps_graph; pub mod direnv; pub mod dirs; -pub mod download_cache; pub mod duration; pub mod env; pub mod env_diff; diff --git a/src/packslip.rs b/src/packslip.rs index 85e0d80c272..ad1c355cfdc 100644 --- a/src/packslip.rs +++ b/src/packslip.rs @@ -263,21 +263,23 @@ fn downloaded_sign_in_page(url: &str, dest: &Path) -> bool { pub(crate) async fn download_file( url: &str, dest: &Path, + pin: Option<&str>, pr: Option<&dyn SingleReport>, ) -> Result<()> { let reason = match HTTP - .download_file_with_headers(url, dest, &headers_for(url)?, pr) + .download_file_with_pin(url, dest, &headers_for(url)?, pin, pr) .await { - Ok(()) if !downloaded_sign_in_page(url, dest) => return Ok(()), - Ok(()) => Unreachable::SignIn, + Ok(_) if !downloaded_sign_in_page(url, dest) => return Ok(()), + Ok(_) => Unreachable::SignIn, Err(err) => Unreachable::Failed(err), }; let Some(api_url) = retry_url(url, &reason).await else { return Err(reason.into_error(url)); }; - HTTP.download_file_with_headers(&api_url, dest, &headers_for(&api_url)?, pr) + HTTP.download_file_with_pin(&api_url, dest, &headers_for(&api_url)?, pin, pr) .await + .map(|_| ()) } /// Fetch a document a packslip names — a manifest or a signed release list — @@ -371,7 +373,10 @@ pub(crate) async fn fetch_files( // The tool is installed by now and the asset is an extra: // one that cannot be fetched is reported, not fatal. One // that arrives with the wrong digest is another matter. - if let Err(err) = download_file(url, &dest, Some(pr)).await { + let signed = statement + .digest_of(name) + .map(|sha256| format!("sha256:{sha256}")); + if let Err(err) = download_file(url, &dest, signed.as_deref(), Some(pr)).await { let _ = file::remove_all(&dest); warn!("{}: could not fetch {name}: {err}", tv.style()); continue; @@ -1987,7 +1992,7 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let dest = dir.path().join("tool.tar.gz"); let url = format!("{}/tool.tar.gz", server.url()); - download_file(&url, &dest, None).await.unwrap(); + download_file(&url, &dest, None, None).await.unwrap(); assert_eq!(file::read_to_string(&dest).unwrap(), "payload"); mock.assert_async().await; } From bd636f65b115c60d4127b180feae34e2aa4ba9a2 Mon Sep 17 00:00:00 2001 From: jdx <216188+jdx@users.noreply.github.com> Date: Fri, 9 Oct 2026 12:35:53 +0000 Subject: [PATCH 13/17] feat: install from the download store without copying, and deprecate always_keep_download Co-Authored-By: Claude Sonnet 5.5 Entire-Checkpoint: 01M4GAKC0QDNAHSJY38K62ZVK6 --- crates/mise-util/src/http.rs | 171 +++++++++++++++---- crates/mise-util/src/http/cas.rs | 222 +++++++++++++++++-------- crates/mise-util/src/http/tests/mod.rs | 70 ++++++-- docs/directories.md | 11 +- e2e/backend/test_aqua_resume_download | 8 +- schema/mise.json | 5 +- settings.toml | 17 +- src/backend/aqua.rs | 11 +- src/backend/github.rs | 2 +- src/backend/http.rs | 5 +- src/packslip.rs | 4 +- 11 files changed, 393 insertions(+), 133 deletions(-) diff --git a/crates/mise-util/src/http.rs b/crates/mise-util/src/http.rs index ccd726c63e8..8726b0ae921 100644 --- a/crates/mise-util/src/http.rs +++ b/crates/mise-util/src/http.rs @@ -233,6 +233,17 @@ struct CacheMode<'a> { enabled: bool, } +/// How a download was satisfied. +struct Fetched { + metadata: DownloadFileMetadata, + /// A stored file answered the request, so nothing was written to the + /// destination. + stored: Option, + /// The validator that came with a complete response, for keeping it. + validator: Option, + request_hash: String, +} + /// What an attempt learned beyond the bytes it wrote. #[derive(Default)] struct AttemptOutcome { @@ -1370,8 +1381,111 @@ impl Client { pin, enabled: use_cache, }; - self.download_file_inner(url, path, headers, cache, pr, total_timeout) - .await + let fetched = self + .download_file_inner(url, path, headers, cache, pr, total_timeout) + .await?; + if let Some(blob) = fetched.stored { + let dest = path.to_path_buf(); + tokio::task::spawn_blocking(move || cas::place(&blob, &dest, false)).await??; + return Ok(fetched.metadata); + } + if use_cache { + // Best effort: a download that can't be kept is still a download. + let path = path.to_path_buf(); + let pin = pin.map(str::to_string); + let effective_filename = fetched.metadata.effective_filename.clone(); + let kept = tokio::task::spawn_blocking(move || { + cas::store( + &path, + cas::Keep { + request_hash: &fetched.request_hash, + pin: pin.as_deref(), + validator: fetched.validator.as_ref(), + effective_filename, + }, + ) + }) + .await; + if let Ok(Err(err)) | Err(err) = kept.map_err(Report::from) { + debug!("could not keep the download: {err:#}"); + } + } + Ok(fetched.metadata) + } + + /// Like [`Client::download_file_pinned`], but the download is kept in the + /// store first and `dest` is a link to it where the filesystem allows one, + /// so the bytes exist once. Across devices, or without a store, it is a + /// copy. For callers that only read the file: a change through the link + /// would change the stored file, which is hashed again before it is reused. + pub async fn download_file_linked( + &self, + url: U, + dest: &Path, + headers: &HeaderMap, + pin: Option<&str>, + pr: Option<&dyn SingleReport>, + ) -> Result { + let url = url.into_url()?; + let name = dest + .file_name() + .ok_or_else(|| eyre!("download destination has no filename: {}", dest.display()))? + .to_string_lossy() + .to_string(); + let request_hash = download_request_hash(&url, headers); + let staging = (Settings::get().download_cache && !Settings::get().generate_lockfiles()) + .then(|| cas::staging_path(&request_hash, &name)) + .flatten(); + let Some(staging) = staging else { + return self + .download_file_with_pin(url, dest, headers, pin, pr) + .await; + }; + let cache = CacheMode { pin, enabled: true }; + let fetched = self + .download_file_inner( + url, + &staging, + headers, + cache, + pr, + crate::network::http_download_timeout(&Settings::get()), + ) + .await?; + let blob = match fetched.stored { + Some(blob) => Some(blob), + None => { + let staged = staging.clone(); + let pin = pin.map(str::to_string); + let effective_filename = fetched.metadata.effective_filename.clone(); + let validator = fetched.validator.clone(); + let hash = fetched.request_hash.clone(); + tokio::task::spawn_blocking(move || { + cas::ingest( + &staged, + cas::Keep { + request_hash: &hash, + pin: pin.as_deref(), + validator: validator.as_ref(), + effective_filename, + }, + ) + }) + .await?? + } + }; + let dest = dest.to_path_buf(); + // With nothing to index it by (no validator, no pin) the staged file is + // the download itself. + let (source, link) = match &blob { + Some(blob) => (blob.clone(), true), + None => (staging.clone(), false), + }; + tokio::task::spawn_blocking(move || cas::place(&source, &dest, link)).await??; + if blob.is_none() { + let _ = file::remove_file(&staging); + } + Ok(fetched.metadata) } /// With `use_cache`, a download kept under `pin` is reused outright, and one @@ -1385,25 +1499,30 @@ impl Client { cache: CacheMode<'_>, pr: Option<&dyn SingleReport>, total_timeout: Duration, - ) -> Result { + ) -> Result { let CacheMode { pin, enabled: use_cache, } = cache; + let url = url.into_url()?; + let request_hash = download_request_hash(&url, headers); // A pinned download kept earlier needs no request, so this comes before // the offline check. if use_cache && let Some(hit) = pin.and_then(cas::lookup_pin) { let effective_filename = hit.effective_filename.clone(); - let dest = path.to_path_buf(); - if tokio::task::spawn_blocking(move || hit.restore(&dest)).await?? { - return Ok(DownloadFileMetadata { effective_filename }); + if let Some(blob) = tokio::task::spawn_blocking(move || hit.verified()).await? { + return Ok(Fetched { + metadata: DownloadFileMetadata { effective_filename }, + stored: Some(blob), + validator: None, + request_hash, + }); } } ensure!( !crate::network::offline(&Settings::get()), "offline mode is enabled" ); - let url = url.into_url()?; debug!( "GET Downloading {} to {}", log_url(&url), @@ -1422,7 +1541,6 @@ impl Client { .await??; let attempt = Arc::new(AtomicUsize::new(0)); let progress = Arc::new(DownloadProgress::default()); - let request_hash = download_request_hash(&url, headers); let cached = if use_cache { cas::lookup_url(&request_hash) } else { @@ -1489,11 +1607,14 @@ impl Client { && let Some(cached) = cached { let effective_filename = cached.effective_filename(); - let dest = path.clone(); let hit = cached.into_hit(); - let restored = tokio::task::spawn_blocking(move || hit.restore(&dest)).await??; - if restored { - return Ok(DownloadFileMetadata { effective_filename }); + if let Some(blob) = tokio::task::spawn_blocking(move || hit.verified()).await? { + return Ok(Fetched { + metadata: DownloadFileMetadata { effective_filename }, + stored: Some(blob), + validator: None, + request_hash, + }); } // The stored file no longer matches what was recorded: fetch it again. drop(_download_lock); @@ -1514,24 +1635,12 @@ impl Client { let persist_path = path.clone(); tokio::task::spawn_blocking(move || partial.persist(&persist_path)).await??; let validator = outcome.validator.lock().unwrap().take(); - if use_cache { - let effective_filename = metadata.effective_filename.clone(); - let pin = pin.map(str::to_string); - let stored = tokio::task::spawn_blocking(move || { - cas::store( - &request_hash, - pin.as_deref(), - &path, - validator.as_ref(), - effective_filename, - ) - }) - .await; - if let Ok(Err(err)) | Err(err) = stored.map_err(Report::from) { - debug!("could not keep the download for revalidation: {err:#}"); - } - } - Ok(metadata) + Ok(Fetched { + metadata, + stored: None, + validator, + request_hash, + }) } async fn download_file_attempt( @@ -2432,7 +2541,7 @@ pub fn with_host_auth(url: &Url, headers: &HeaderMap) -> Result { Ok(merged) } -fn host_auth_headers(url: &Url) -> Result { +pub fn host_auth_headers(url: &Url) -> Result { // raw.githubusercontent.com is not an API host, but a private repository's // files are a 404 without the token, so it is routed here too. `get_headers` // decides what each host actually gets. diff --git a/crates/mise-util/src/http/cas.rs b/crates/mise-util/src/http/cas.rs index f31e8687db4..a6c541b9ba7 100644 --- a/crates/mise-util/src/http/cas.rs +++ b/crates/mise-util/src/http/cas.rs @@ -1,6 +1,8 @@ //! A content-addressed store of finished downloads. //! -//! Files are kept by their BLAKE3 hash under `blobs/`. Two small indexes point +//! Files are kept by their BLAKE3 hash under `blobs//`, under the +//! name they were downloaded as, since installers read the format and the +//! binary's name from it. Two small indexes point //! into them: //! //! - `urls/`: a request (URL and headers) to the blob it last returned and the @@ -62,10 +64,25 @@ fn is_hex(value: &str) -> bool { !value.is_empty() && value.bytes().all(|b| b.is_ascii_hexdigit()) } -fn blob_path(root: &Path, blake3: &str) -> PathBuf { +fn blob_dir(root: &Path, blake3: &str) -> PathBuf { root.join("blobs").join(blake3) } +/// The file a blob directory holds, whatever name it was stored under. +fn blob_file(dir: &Path) -> Option { + std::fs::read_dir(dir) + .ok()? + .flatten() + .map(|entry| entry.path()) + .find(|path| path.is_file()) +} + +/// Where an in-flight download is written: stable per request, so a partial +/// one can be resumed, and on the store's filesystem so it renames into place. +pub(super) fn staging_path(request_hash: &str, name: &str) -> Option { + Some(root()?.join("incoming").join(request_hash).join(name)) +} + fn url_path(root: &Path, request_hash: &str) -> PathBuf { root.join("urls").join(format!("{request_hash}.json")) } @@ -120,10 +137,7 @@ pub(super) fn lookup_url(request_hash: &str) -> Option { if !url.validator.is_valid() || !is_hex(&url.blake3) { return None; } - let blob = blob_path(&root, &url.blake3); - if !blob.is_file() { - return None; - } + let blob = blob_file(&blob_dir(&root, &url.blake3))?; Some(Cached { validator: url.validator, hit: Hit { @@ -143,10 +157,7 @@ pub(super) fn lookup_pin(pin: &str) -> Option { if !is_hex(&pinned.blake3) { return None; } - let blob = blob_path(&root, &pinned.blake3); - if !blob.is_file() { - return None; - } + let blob = blob_file(&blob_dir(&root, &pinned.blake3))?; Some(Hit { blob, effective_filename: pinned.effective_filename, @@ -159,9 +170,9 @@ pub(super) fn lookup_pin(pin: &str) -> Option { } impl Hit { - /// Copy the blob to `dest` if it still hashes to what it is expected to. - /// Otherwise drop what pointed at it and return `false`. - pub(super) fn restore(&self, dest: &Path) -> Result { + /// The stored file, once it hashes to what it is expected to. Otherwise + /// drop what pointed at it and return `None`. + pub(super) fn verified(&self) -> Option { let intact = match &self.verify { Verify::Blake3(expected) => { crate::hash::file_hash_blake3(&self.blob, None).is_ok_and(|hash| hash == *expected) @@ -172,25 +183,43 @@ impl Hit { }; if !intact { let _ = file::remove_file(&self.entry); - if matches!(self.verify, Verify::Blake3(_)) { - let _ = file::remove_file(&self.blob); + if matches!(self.verify, Verify::Blake3(_)) + && let Some(dir) = self.blob.parent() + { + let _ = file::remove_all(dir); } - return Ok(false); + return None; } - let parent = dest.parent().unwrap_or(Path::new(".")); - file::create_dir_all(parent)?; - // Renamed into place so an interrupted copy never leaves a truncated - // file at the destination. A `TempPath` holds no open handle, which - // Windows needs to copy over it. - let tmp = tempfile::NamedTempFile::new_in(parent)?.into_temp_path(); - file::copy(&self.blob, &tmp)?; - tmp.persist(dest).map_err(|err| err.error)?; // Marks the blob as recently used for eviction. if let Ok(f) = std::fs::File::options().write(true).open(&self.blob) { let _ = f.set_modified(SystemTime::now()); } - Ok(true) + Some(self.blob.clone()) + } +} + +/// Put `blob` at `dest` without ever leaving a partial file there. With `link` +/// the content is shared where the filesystem allows and copied where it +/// doesn't, such as across devices; without it the content is always copied. +pub(super) fn place(blob: &Path, dest: &Path, link: bool) -> Result<()> { + let parent = dest.parent().unwrap_or(Path::new(".")); + file::create_dir_all(parent)?; + // A directory of our own, so the name inside it is free to link to. A + // `TempPath` would exist already, and Windows can't copy over a file that + // is open. + let scratch = tempfile::Builder::new() + .prefix(".mise-place-") + .tempdir_in(parent)?; + let staged = scratch.path().join("file"); + if link { + file::hard_link_or_copy(blob, &staged)?; + } else { + file::copy(blob, &staged)?; } + std::fs::rename(&staged, dest).map_err(|err| { + eyre::Report::new(err).wrap_err(format!("failed to place {}", dest.display())) + })?; + Ok(()) } /// Write `contents` to `path` so a reader never sees a partial file. @@ -203,84 +232,145 @@ fn write_atomic(path: &Path, contents: &[u8]) -> Result<()> { Ok(()) } -/// Keep `path`, just downloaded. `validator` lets the server confirm it later; -/// `pin` lets a tool that pinned the same checksum skip the request. -pub(super) fn store( - request_hash: &str, - pin: Option<&str>, - path: &Path, - validator: Option<&DownloadValidator>, - effective_filename: Option, -) -> Result<()> { +/// What kept a download: how to find it again. +pub(super) struct Keep<'a> { + pub(super) request_hash: &'a str, + /// The checksum the tool pinned. + pub(super) pin: Option<&'a str>, + pub(super) validator: Option<&'a DownloadValidator>, + pub(super) effective_filename: Option, +} + +/// Keep `path`, just downloaded, by copying it into the store. Returns the +/// stored file, or `None` when there is nothing to index it by. +pub(super) fn store(path: &Path, keep: Keep<'_>) -> Result> { + keep_file(path, keep, false) +} + +/// Keep `path`, which sits in the store's staging area, by moving it into the +/// store. Returns the stored file. +pub(super) fn ingest(path: &Path, keep: Keep<'_>) -> Result> { + keep_file(path, keep, true) +} + +fn keep_file(path: &Path, keep: Keep<'_>, move_in: bool) -> Result> { let Some(root) = root() else { - return Ok(()); + return Ok(None); }; // The caller checks the pin after the download. Checking here too keeps a // wrong file from ever being indexed under it. - let pin = pin.and_then(parse_pin).filter(|(algo, hex)| { + let pin = keep.pin.and_then(parse_pin).filter(|(algo, hex)| { let ok = crate::hash::ensure_checksum(path, hex, None, algo).is_ok(); if !ok { debug!("not indexing a download that doesn't match its pin"); } ok }); - if validator.is_none() && pin.is_none() { - return Ok(()); + if keep.validator.is_none() && pin.is_none() { + return Ok(None); } + let name = path.file_name().expect("downloads have a file name"); let blake3 = crate::hash::file_hash_blake3(path, None)?; - let blob = blob_path(&root, &blake3); - if !blob.is_file() { - let parent = blob.parent().expect("blob paths have a parent"); - file::create_dir_all(parent)?; - let tmp = tempfile::NamedTempFile::new_in(parent)?.into_temp_path(); - file::copy(path, &tmp)?; - tmp.persist(&blob).map_err(|err| err.error)?; - } - if let Some(validator) = validator { + let dir = blob_dir(&root, &blake3); + let existing = blob_file(&dir); + let blob = match existing { + Some(blob) => { + if move_in { + let _ = file::remove_file(path); + } + blob + } + None => { + let blob = dir.join(name); + // Built beside its final place so the rename can't cross a device. + let blobs = root.join("blobs"); + file::create_dir_all(&blobs)?; + let building = tempfile::Builder::new() + .prefix(".building-") + .tempdir_in(&blobs)?; + let staged = building.path().join(name); + if move_in { + file::rename(path, &staged)?; + } else { + file::copy(path, &staged)?; + } + let building = building.keep(); + if let Err(err) = std::fs::rename(&building, &dir) { + let _ = file::remove_all(&building); + // Another process stored the same download first. + if blob_file(&dir).is_none() { + return Err(err.into()); + } + } + blob_file(&dir).unwrap_or(blob) + } + }; + if let Some(validator) = keep.validator { let entry = UrlEntry { validator: validator.clone(), blake3: blake3.clone(), - effective_filename: effective_filename.clone(), + effective_filename: keep.effective_filename.clone(), }; - write_atomic(&url_path(&root, request_hash), &serde_json::to_vec(&entry)?)?; + write_atomic( + &url_path(&root, keep.request_hash), + &serde_json::to_vec(&entry)?, + )?; } if let Some((algo, hex)) = pin { let entry = PinEntry { blake3, - effective_filename, + effective_filename: keep.effective_filename, }; write_atomic(&pin_path(&root, algo, &hex), &serde_json::to_vec(&entry)?)?; } - evict(&root.join("blobs")); - Ok(()) + evict(&root); + Ok(Some(blob)) } -fn evict(blobs: &Path) { - let Ok(read) = std::fs::read_dir(blobs) else { +/// Past `MAX_CACHE_BYTES` the least recently used blobs go first. Staged +/// downloads that were never finished go after a week. +fn evict(root: &Path) { + let week = std::time::Duration::from_secs(7 * 24 * 60 * 60); + if let Ok(read) = std::fs::read_dir(root.join("incoming")) { + for entry in read.flatten() { + let stale = entry + .metadata() + .and_then(|m| m.modified()) + .ok() + .and_then(|modified| modified.elapsed().ok()) + .is_some_and(|age| age > week); + if stale { + let _ = file::remove_all(entry.path()); + } + } + } + let Ok(read) = std::fs::read_dir(root.join("blobs")) else { return; }; let mut entries = Vec::new(); let mut total = 0u64; for entry in read.flatten() { - let Ok(meta) = entry.metadata() else { - continue; - }; - if !meta.is_file() || entry.file_name().to_string_lossy().starts_with('.') { + let dir = entry.path(); + if entry.file_name().to_string_lossy().starts_with('.') || !dir.is_dir() { continue; } - total += meta.len(); - entries.push(( - meta.modified().unwrap_or(SystemTime::UNIX_EPOCH), - meta.len(), - entry.path(), - )); + // Names of one blob share a file where they can, so count the largest. + let (mut size, mut used) = (0, SystemTime::UNIX_EPOCH); + for file in std::fs::read_dir(&dir).into_iter().flatten().flatten() { + if let Ok(meta) = file.metadata() { + size = size.max(meta.len()); + used = used.max(meta.modified().unwrap_or(SystemTime::UNIX_EPOCH)); + } + } + total += size; + entries.push((used, size, dir)); } entries.sort(); - for (_, size, path) in entries { + for (_, size, dir) in entries { if total <= MAX_CACHE_BYTES { break; } - if file::remove_file(&path).is_ok() { + if file::remove_all(&dir).is_ok() { total -= size; } } diff --git a/crates/mise-util/src/http/tests/mod.rs b/crates/mise-util/src/http/tests/mod.rs index 52cdaf50ae6..b8b38f2a010 100644 --- a/crates/mise-util/src/http/tests/mod.rs +++ b/crates/mise-util/src/http/tests/mod.rs @@ -606,12 +606,7 @@ async fn test_download_reuses_stored_file_on_not_modified() { // A stored file that no longer hashes to what was recorded is not used, // even though the server says it is current. - let stored = std::fs::read_dir(cache.path().join("blobs")) - .unwrap() - .next() - .unwrap() - .unwrap() - .path(); + let stored = stored_blob(cache.path()); std::fs::write(&stored, b"tampered").unwrap(); std::fs::remove_file(&dest).unwrap(); client.download_file(&url, &dest, None).await.unwrap(); @@ -653,12 +648,7 @@ async fn test_download_reuses_pinned_file_without_a_request() { assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 1); // A blob that no longer matches the pin is not used. - let blob = std::fs::read_dir(cache.path().join("blobs")) - .unwrap() - .next() - .unwrap() - .unwrap() - .path(); + let blob = stored_blob(cache.path()); std::fs::write(&blob, b"tampered").unwrap(); std::fs::remove_file(&dest).unwrap(); assert!( @@ -670,3 +660,59 @@ async fn test_download_reuses_pinned_file_without_a_request() { *cas::TEST_ROOT.lock().unwrap() = None; } + +fn stored_blob(cache: &std::path::Path) -> PathBuf { + let dir = std::fs::read_dir(cache.join("blobs")) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + std::fs::read_dir(dir) + .unwrap() + .next() + .unwrap() + .unwrap() + .path() +} + +#[tokio::test(flavor = "current_thread")] +async fn test_download_linked_shares_one_stored_file() { + let _settings = set_test_http_retries(0); + let cache = tempfile::tempdir().unwrap(); + *cas::TEST_ROOT.lock().unwrap() = Some(cache.path().to_path_buf()); + let (port, count) = spawn_canned_server(vec![cacheable_download_response()]).await; + let url = format!("http://127.0.0.1:{port}/artifact.bin"); + let client = Client::new(Duration::from_secs(2), ClientKind::Http).unwrap(); + let dir = tempfile::tempdir().unwrap(); + let dest = dir.path().join("artifact.bin"); + let hello = dir.path().join("hello"); + std::fs::write(&hello, b"hello").unwrap(); + let pin = format!( + "sha256:{}", + crate::hash::file_hash_prog::(&hello, None).unwrap() + ); + + client + .download_file_linked(&url, &dest, &HeaderMap::new(), Some(&pin), None) + .await + .unwrap(); + assert_eq!(std::fs::read(&dest).unwrap(), b"hello"); + // The stored file keeps the name it was downloaded as. + assert_eq!( + stored_blob(cache.path()).file_name().unwrap(), + "artifact.bin" + ); + + // A second install is served from the store with no request, and the + // destination holds the same bytes. + std::fs::remove_file(&dest).unwrap(); + client + .download_file_linked(&url, &dest, &HeaderMap::new(), Some(&pin), None) + .await + .unwrap(); + assert_eq!(std::fs::read(&dest).unwrap(), b"hello"); + assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 1); + + *cas::TEST_ROOT.lock().unwrap() = None; +} diff --git a/docs/directories.md b/docs/directories.md index d3eab322cf6..c0287a3f246 100644 --- a/docs/directories.md +++ b/docs/directories.md @@ -78,12 +78,11 @@ machines is not a supported way to install tools. For caching it in CI, see ### `downloads` {#local-share-mise-downloads} -Backends may write archives here while they install a tool. mise deletes them -afterwards unless -[`always_keep_download`](/configuration/settings.html#always_keep_download) is -set. This is not a download cache. To avoid reinstalling tools in CI, cache -`installs/` instead, together with the install store when it is a separate -directory. +Backends may write archives here while they install a tool, and mise deletes +them afterwards. Finished downloads are kept in the +[download cache](/configuration/settings.html#download_cache) instead. To avoid +reinstalling tools in CI, cache `installs/`, together with the install store +when it is a separate directory. ### `plugins` {#local-share-mise-plugins} diff --git a/e2e/backend/test_aqua_resume_download b/e2e/backend/test_aqua_resume_download index 4a5b05ee1d9..7cdc186812b 100644 --- a/e2e/backend/test_aqua_resume_download +++ b/e2e/backend/test_aqua_resume_download @@ -82,13 +82,13 @@ EOF_REGISTRY export MISE_AQUA_REGISTRY_URL="file://$PWD/registry" assert_fail "mise install aqua:example/resume-tool@1.0.0" -assert_succeed "find '$MISE_DATA_DIR/downloads' -name '*.mise-part' -type f | grep -q ." -assert_succeed "find '$MISE_DATA_DIR/downloads' -name '*.mise-part.json' -type f | grep -q ." +assert_succeed "find '$MISE_CACHE_DIR/downloads-cas/incoming' -name '*.mise-part' -type f | grep -q ." +assert_succeed "find '$MISE_CACHE_DIR/downloads-cas/incoming' -name '*.mise-part.json' -type f | grep -q ." mise install aqua:example/resume-tool@1.0.0 assert "mise exec aqua:example/resume-tool@1.0.0 -- resume-tool" "resumed download" assert_contains "sed -n '1p' requests.log" 'range= if-range=' assert_contains "cat requests.log" 'range=bytes=' assert_contains "cat requests.log" 'if-range="resume-v1"' -assert_fail "find '$MISE_DATA_DIR/downloads' -name '*.mise-part' -type f | grep -q ." -assert_fail "find '$MISE_DATA_DIR/downloads' -name '*.mise-part.json' -type f | grep -q ." +assert_fail "find '$MISE_CACHE_DIR/downloads-cas/incoming' -name '*.mise-part' -type f | grep -q ." +assert_fail "find '$MISE_CACHE_DIR/downloads-cas/incoming' -name '*.mise-part.json' -type f | grep -q ." diff --git a/schema/mise.json b/schema/mise.json index a67a6c6e9e4..ba21896d6f4 100644 --- a/schema/mise.json +++ b/schema/mise.json @@ -637,8 +637,9 @@ "type": "boolean" }, "always_keep_download": { - "description": "Keep downloaded archives and sources after installing, for debugging.", - "type": "boolean" + "description": "[deprecated] Keep downloaded archives and sources after installing, for debugging.", + "type": "boolean", + "deprecated": true }, "always_keep_install": { "description": "Keep a failed installation's partial install directory and downloads, for debugging.", diff --git a/settings.toml b/settings.toml index c5f8aec395f..6bc44d26e63 100644 --- a/settings.toml +++ b/settings.toml @@ -104,11 +104,15 @@ optional = true type = "Bool" [always_keep_download] -description = "Keep downloaded archives and sources after installing, for debugging." +deprecated = "Finished downloads are kept in the download cache instead; see download_cache." +deprecated_remove_at = "2027.11.0" +deprecated_warn_at = "2026.11.0" +description = "[deprecated] Keep downloaded archives and sources after installing, for debugging." docs = """ mise keeps the files under `~/.local/share/mise/downloads` so you can inspect -them while debugging a backend or plugin. This is not a download cache; see -[`download_cache`](#download_cache). +them while debugging a backend or plugin. Downloads that go through mise's HTTP +client are kept in [`download_cache`](#download_cache) instead, under their own +names, so this is only needed for files an asdf plugin script downloads itself. """ env = "MISE_ALWAYS_KEEP_DOWNLOAD" type = "Bool" @@ -685,8 +689,11 @@ hash, and uses one again in two cases: - **A tool pinned its checksum**, in its options or in `mise.lock`. The kept file is used with no request at all, even if the URL changed. -The `http`, `github` and `aqua` backends pin; every other download that goes -through mise's HTTP client uses the first case. +The `http`, `github`, `aqua` and `packslip` backends pin and install from the +stored file, linked into place where the filesystem allows, so the archive takes +no extra space and is not copied; across devices it is copied. Every other +download that goes through mise's HTTP client uses the first case and is copied +into place. A kept file is hashed again before it is used, and one that no longer matches is dropped and downloaded again. The recorded hash lives beside the file, so this diff --git a/src/backend/aqua.rs b/src/backend/aqua.rs index 1d8174fa7d1..96343733f84 100644 --- a/src/backend/aqua.rs +++ b/src/backend/aqua.rs @@ -2828,8 +2828,15 @@ impl AquaBackend { .lock_platforms .get(&self.get_platform_key()) .and_then(|platform| platform.checksum.clone()); - HTTP.download_file_pinned(url, &tarball_path, pinned.as_deref(), Some(ctx.pr.as_ref())) - .await?; + let headers = crate::http::host_auth_headers(&reqwest::Url::parse(url)?)?; + HTTP.download_file_linked( + url, + &tarball_path, + &headers, + pinned.as_deref(), + Some(ctx.pr.as_ref()), + ) + .await?; Ok(()) } diff --git a/src/backend/github.rs b/src/backend/github.rs index ba434dc10be..2c457ff4cb9 100644 --- a/src/backend/github.rs +++ b/src/backend/github.rs @@ -1716,7 +1716,7 @@ impl UnifiedGitBackend { ctx.pr.set_message(format!("download {filename}")); let pinned = pinned_checksum(tv, &self.get_platform_key(), opts.raw()); - HTTP.download_file_with_pin( + HTTP.download_file_linked( url, &file_path, &headers, diff --git a/src/backend/http.rs b/src/backend/http.rs index 2b181d1172b..98109e483b0 100644 --- a/src/backend/http.rs +++ b/src/backend/http.rs @@ -1389,9 +1389,10 @@ impl Backend for HttpBackend { ctx.pr.set_message(format!("download {filename}")); let headers = opts.headers(Some(tv.version.as_str()))?; if headers.is_empty() { - HTTP.download_file_pinned( + HTTP.download_file_linked( &url, &file_path, + &crate::http::host_auth_headers(&reqwest::Url::parse(&url)?)?, pinned.as_deref(), Some(ctx.pr.as_ref()), ) @@ -1400,7 +1401,7 @@ impl Backend for HttpBackend { // Keep the automatic host token; configured headers override it. let headers = crate::http::with_host_auth(&reqwest::Url::parse(&url)?, &headers)?; - HTTP.download_file_with_pin( + HTTP.download_file_linked( &url, &file_path, &headers, diff --git a/src/packslip.rs b/src/packslip.rs index ad1c355cfdc..a46a8087912 100644 --- a/src/packslip.rs +++ b/src/packslip.rs @@ -267,7 +267,7 @@ pub(crate) async fn download_file( pr: Option<&dyn SingleReport>, ) -> Result<()> { let reason = match HTTP - .download_file_with_pin(url, dest, &headers_for(url)?, pin, pr) + .download_file_linked(url, dest, &headers_for(url)?, pin, pr) .await { Ok(_) if !downloaded_sign_in_page(url, dest) => return Ok(()), @@ -277,7 +277,7 @@ pub(crate) async fn download_file( let Some(api_url) = retry_url(url, &reason).await else { return Err(reason.into_error(url)); }; - HTTP.download_file_with_pin(&api_url, dest, &headers_for(&api_url)?, pin, pr) + HTTP.download_file_linked(&api_url, dest, &headers_for(&api_url)?, pin, pr) .await .map(|_| ()) } From f0b2485c5745454e9592391a3f57168824fa1f20 Mon Sep 17 00:00:00 2001 From: jdx <216188+jdx@users.noreply.github.com> Date: Fri, 9 Oct 2026 12:38:21 +0000 Subject: [PATCH 14/17] fix(http): keep the download just stored out of eviction and correct the source-build docs Co-Authored-By: Claude Sonnet 5.5 Entire-Checkpoint: 01M4GAQW3FHB41Q7GHEDXHF34Y --- crates/mise-util/src/http/cas.rs | 14 +++++++++----- docs/dev-tools/mise-lock.md | 11 ++++++----- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/crates/mise-util/src/http/cas.rs b/crates/mise-util/src/http/cas.rs index a6c541b9ba7..6c942afb421 100644 --- a/crates/mise-util/src/http/cas.rs +++ b/crates/mise-util/src/http/cas.rs @@ -323,13 +323,14 @@ fn keep_file(path: &Path, keep: Keep<'_>, move_in: bool) -> Result) { let week = std::time::Duration::from_secs(7 * 24 * 60 * 60); if let Ok(read) = std::fs::read_dir(root.join("incoming")) { for entry in read.flatten() { @@ -351,7 +352,10 @@ fn evict(root: &Path) { let mut total = 0u64; for entry in read.flatten() { let dir = entry.path(); - if entry.file_name().to_string_lossy().starts_with('.') || !dir.is_dir() { + if entry.file_name().to_string_lossy().starts_with('.') + || !dir.is_dir() + || keep == Some(dir.as_path()) + { continue; } // Names of one blob share a file where they can, so count the largest. diff --git a/docs/dev-tools/mise-lock.md b/docs/dev-tools/mise-lock.md index ae174aa33e0..27fb3ed76f3 100644 --- a/docs/dev-tools/mise-lock.md +++ b/docs/dev-tools/mise-lock.md @@ -420,9 +420,9 @@ restored CI cache or a shared extraction store is trusted as is. mise can't validate an installed tree against the lockfile. The checksum covers the downloaded artifact, and installs are mutable on purpose: tools update -themselves and `postinstall` hooks change files. For tools built from source, -such as Python and Ruby, it covers only the source tarball, not the compiled -output. +themselves and `postinstall` hooks change files. For tools compiled from source, +such as Ruby, the lockfile doesn't vouch for the compiled output, and a tool +that downloads its own source may record no checksum for it at all. Downloads are different. mise keeps them in a cache and reuses one only when the server confirms it is unchanged, or when it hashes to a pinned checksum. A @@ -437,8 +437,9 @@ So treat a cache of installed files as trusted build input: the restored files against it. A manifest stored in the cache can be replaced along with it. That proves the cache is unchanged since CI saved it, not that it matches the lockfile. -- For a fully verified install, run `mise install --locked` with an empty data - directory. +- To check an install against the lockfile, run `mise install --locked` with an + empty data directory. Every artifact is then downloaded and checked against + the checksum the lockfile records, for the backends that record one. ## Provenance and verification {#provenance-and-security} From b4df2be6fd6e8170e3a47ab71a0cffee2272e67c Mon Sep 17 00:00:00 2001 From: jdx <216188+jdx@users.noreply.github.com> Date: Fri, 9 Oct 2026 12:48:54 +0000 Subject: [PATCH 15/17] fix(http): replace a damaged stored download, place over an existing file, and pin aqua by the API digest Co-Authored-By: Claude Sonnet 5.5 Entire-Checkpoint: 01M4GBB6BJ2W1Z1AXZJTMWNBFK --- crates/mise-util/src/http/cas.rs | 33 ++++++++++++---- crates/mise-util/src/http/tests/mod.rs | 55 ++++++++++++++++++++++++++ src/backend/aqua.rs | 9 ++++- 3 files changed, 87 insertions(+), 10 deletions(-) diff --git a/crates/mise-util/src/http/cas.rs b/crates/mise-util/src/http/cas.rs index 6c942afb421..c0616ef59ce 100644 --- a/crates/mise-util/src/http/cas.rs +++ b/crates/mise-util/src/http/cas.rs @@ -183,10 +183,15 @@ impl Hit { }; if !intact { let _ = file::remove_file(&self.entry); - if matches!(self.verify, Verify::Blake3(_)) - && let Some(dir) = self.blob.parent() - { - let _ = file::remove_all(dir); + // A blob that doesn't hash to its own name is damaged, whichever + // index led here. One that does merely isn't what the pin names. + if let Some(dir) = self.blob.parent() { + let name = dir.file_name().map(|n| n.to_string_lossy().to_string()); + let damaged = crate::hash::file_hash_blake3(&self.blob, None) + .is_ok_and(|hash| Some(hash) != name); + if damaged { + let _ = file::remove_all(dir); + } } return None; } @@ -216,9 +221,13 @@ pub(super) fn place(blob: &Path, dest: &Path, link: bool) -> Result<()> { } else { file::copy(blob, &staged)?; } - std::fs::rename(&staged, dest).map_err(|err| { - eyre::Report::new(err).wrap_err(format!("failed to place {}", dest.display())) - })?; + // `persist` replaces an existing file on Windows too, where a plain rename + // does not. + tempfile::TempPath::try_from_path(&staged)? + .persist(dest) + .map_err(|err| { + eyre::Report::new(err.error).wrap_err(format!("failed to place {}", dest.display())) + })?; Ok(()) } @@ -272,7 +281,15 @@ fn keep_file(path: &Path, keep: Keep<'_>, move_in: bool) -> Result { if move_in { diff --git a/crates/mise-util/src/http/tests/mod.rs b/crates/mise-util/src/http/tests/mod.rs index b8b38f2a010..e26cb41701d 100644 --- a/crates/mise-util/src/http/tests/mod.rs +++ b/crates/mise-util/src/http/tests/mod.rs @@ -716,3 +716,58 @@ async fn test_download_linked_shares_one_stored_file() { *cas::TEST_ROOT.lock().unwrap() = None; } + +#[tokio::test(flavor = "current_thread")] +async fn test_download_replaces_a_damaged_stored_file() { + let _settings = set_test_http_retries(0); + let cache = tempfile::tempdir().unwrap(); + *cas::TEST_ROOT.lock().unwrap() = Some(cache.path().to_path_buf()); + let (port, count) = spawn_canned_server(vec![ + cacheable_download_response(), + cacheable_download_response(), + ]) + .await; + let url = format!("http://127.0.0.1:{port}/artifact.bin"); + let client = Client::new(Duration::from_secs(2), ClientKind::Http).unwrap(); + let dir = tempfile::tempdir().unwrap(); + let dest = dir.path().join("artifact.bin"); + let hello = dir.path().join("hello"); + std::fs::write(&hello, b"hello").unwrap(); + let pin = format!( + "sha256:{}", + crate::hash::file_hash_prog::(&hello, None).unwrap() + ); + + client + .download_file_linked(&url, &dest, &HeaderMap::new(), Some(&pin), None) + .await + .unwrap(); + + // Damage the stored file in place. The next download must not install it, + // and must leave a good copy behind rather than the damaged one. + let blob = stored_blob(cache.path()); + std::fs::write(&blob, b"tampered").unwrap(); + std::fs::remove_file(&dest).unwrap(); + client + .download_file_linked(&url, &dest, &HeaderMap::new(), Some(&pin), None) + .await + .unwrap(); + assert_eq!(std::fs::read(&dest).unwrap(), b"hello"); + assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 2); + assert_eq!(std::fs::read(stored_blob(cache.path())).unwrap(), b"hello"); + + // Replacing a destination that already exists works too. + // Replace it with a new file: writing through the link would change the + // stored file itself. + std::fs::remove_file(&dest).unwrap(); + std::fs::write(&dest, b"stale").unwrap(); + let (port, _) = spawn_canned_server(vec![]).await; + let other = format!("http://127.0.0.1:{port}/artifact.bin"); + client + .download_file_linked(&other, &dest, &HeaderMap::new(), Some(&pin), None) + .await + .unwrap(); + assert_eq!(std::fs::read(&dest).unwrap(), b"hello"); + + *cas::TEST_ROOT.lock().unwrap() = None; +} diff --git a/src/backend/aqua.rs b/src/backend/aqua.rs index 96343733f84..6274fbe244e 100644 --- a/src/backend/aqua.rs +++ b/src/backend/aqua.rs @@ -571,7 +571,8 @@ impl Backend for AquaBackend { // Public repos download from the browser URL; private repos return 404/HTML there // and must be fetched from the API asset endpoint instead. let download_url = select_github_download_url(pkg.private, &url, url_api.as_deref()).await; - self.download(ctx, &tv, &download_url, &filename).await?; + self.download(ctx, &tv, &download_url, &filename, api_digest.as_deref()) + .await?; // Snapshot before any mutation so verify() knows whether the lockfile // originally contained a checksum (vs. one we just wrote from api_digest). @@ -2815,6 +2816,7 @@ impl AquaBackend { tv: &ToolVersion, url: &str, filename: &str, + api_digest: Option<&str>, ) -> Result<()> { let tarball_path = tv.download_path().join(filename); if tarball_path.exists() { @@ -2824,10 +2826,13 @@ impl AquaBackend { return Ok(()); } ctx.pr.set_message(format!("download {filename}")); + // The lockfile's checksum, or else the digest the GitHub API gave for the + // release asset, which verify() checks the same way. let pinned = tv .lock_platforms .get(&self.get_platform_key()) - .and_then(|platform| platform.checksum.clone()); + .and_then(|platform| platform.checksum.clone()) + .or_else(|| api_digest.map(str::to_string)); let headers = crate::http::host_auth_headers(&reqwest::Url::parse(url)?)?; HTTP.download_file_linked( url, From b59c20b8cecad3f55da75d5d838360b1e60d91de Mon Sep 17 00:00:00 2001 From: jdx <216188+jdx@users.noreply.github.com> Date: Fri, 9 Oct 2026 13:14:38 +0000 Subject: [PATCH 16/17] test(plugins): turn the download cache off where a corrupt download is the point Co-Authored-By: Claude Sonnet 5.5 Entire-Checkpoint: 01M4GCT9SXS2S5C8DPZD9R5ZR1 --- e2e/cli/test_plugins_packslip | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/e2e/cli/test_plugins_packslip b/e2e/cli/test_plugins_packslip index 08fb09e8c9d..04a12844703 100644 --- a/e2e/cli/test_plugins_packslip +++ b/e2e/cli/test_plugins_packslip @@ -54,12 +54,14 @@ wait "$second_install" mise plugins update bfs cp "$plugin/.mise-packslip-plugin.json" before.json -# A corrupt download must not remove or replace the working plugin. +# A corrupt download must not remove or replace the working plugin. The download +# cache would serve the verified copy kept earlier without asking the server, so +# it is off here to exercise the corrupt response. printf yes >server-state/corrupt -assert_fail "mise plugins update bfs" +assert_fail "MISE_DOWNLOAD_CACHE=0 mise plugins update bfs" assert "cmp before.json '$plugin/.mise-packslip-plugin.json'" assert "test -f '$plugin/metadata.lua'" -assert_fail "mise plugins install --force vfox:bfs '$source_url'" +assert_fail "MISE_DOWNLOAD_CACHE=0 mise plugins install --force vfox:bfs '$source_url'" assert "test -f '$plugin/metadata.lua'" # A signature from the pilot repository cannot authorize a different source. From 86d168375de5f7f41138121d1220de711559d8aa Mon Sep 17 00:00:00 2001 From: jdx <216188+jdx@users.noreply.github.com> Date: Fri, 9 Oct 2026 13:51:24 +0000 Subject: [PATCH 17/17] fix(http): name the path-length failure when placing a download, keep in-use blobs, and spend only the remaining timeout on a retry Co-Authored-By: Claude Sonnet 5.5 Entire-Checkpoint: 01M4GEXMDT046ESB926PRWBYEQ --- crates/mise-util/src/http.rs | 23 +++++++++++++-------- crates/mise-util/src/http/cas.rs | 35 ++++++++++++++++++++++++++------ 2 files changed, 43 insertions(+), 15 deletions(-) diff --git a/crates/mise-util/src/http.rs b/crates/mise-util/src/http.rs index 8726b0ae921..60c7a2ae479 100644 --- a/crates/mise-util/src/http.rs +++ b/crates/mise-util/src/http.rs @@ -1504,6 +1504,7 @@ impl Client { pin, enabled: use_cache, } = cache; + let started = Instant::now(); let url = url.into_url()?; let request_hash = download_request_hash(&url, headers); // A pinned download kept earlier needs no request, so this comes before @@ -1622,15 +1623,19 @@ impl Client { pin, enabled: false, }; - return Box::pin(self.download_file_inner( - url, - &path, - headers, - cache, - pr, - total_timeout, - )) - .await; + // What is left of the budget, not a fresh one. + let remaining = total_timeout + .checked_sub(started.elapsed()) + .filter(|left| !left.is_zero()) + .ok_or_else(|| { + eyre!( + "HTTP download timed out after {} for {}", + format_duration(total_timeout), + log_url(&url) + ) + })?; + return Box::pin(self.download_file_inner(url, &path, headers, cache, pr, remaining)) + .await; } let persist_path = path.clone(); tokio::task::spawn_blocking(move || partial.persist(&persist_path)).await??; diff --git a/crates/mise-util/src/http/cas.rs b/crates/mise-util/src/http/cas.rs index c0616ef59ce..c482e3c3bb3 100644 --- a/crates/mise-util/src/http/cas.rs +++ b/crates/mise-util/src/http/cas.rs @@ -28,6 +28,9 @@ use crate::file; /// Blobs are evicted, least recently used first, past this total size. const MAX_CACHE_BYTES: u64 = 2 * 1024 * 1024 * 1024; +/// How long after a blob was last used another install may still be placing it. +const IN_USE: std::time::Duration = std::time::Duration::from_secs(10 * 60); + /// Algorithms a pin may use. `md5` and `sha1` are not collision resistant /// enough to name content by. const PIN_ALGOS: &[&str] = &["sha256", "sha384", "sha512", "blake3"]; @@ -214,7 +217,8 @@ pub(super) fn place(blob: &Path, dest: &Path, link: bool) -> Result<()> { // is open. let scratch = tempfile::Builder::new() .prefix(".mise-place-") - .tempdir_in(parent)?; + .tempdir_in(parent) + .map_err(|err| placing(err, dest))?; let staged = scratch.path().join("file"); if link { file::hard_link_or_copy(blob, &staged)?; @@ -223,14 +227,28 @@ pub(super) fn place(blob: &Path, dest: &Path, link: bool) -> Result<()> { } // `persist` replaces an existing file on Windows too, where a plain rename // does not. - tempfile::TempPath::try_from_path(&staged)? + tempfile::TempPath::try_from_path(&staged) + .map_err(|err| placing(err, dest))? .persist(dest) - .map_err(|err| { - eyre::Report::new(err.error).wrap_err(format!("failed to place {}", dest.display())) - })?; + .map_err(|err| placing(err.error, dest))?; Ok(()) } +/// Name the step and the path, with the Windows hint for an over-long path: +/// `tempfile` doesn't get the extended-length handling `std::fs` does, so this +/// is the call that fails first as a directory approaches `MAX_PATH`. +fn placing(err: std::io::Error, dest: &Path) -> eyre::Report { + let msg = file::with_io_hint( + format!( + "failed to move the downloaded file into place: {}", + file::display_path(dest) + ), + dest, + &err, + ); + eyre::Report::new(err).wrap_err(msg) +} + /// Write `contents` to `path` so a reader never sees a partial file. fn write_atomic(path: &Path, contents: &[u8]) -> Result<()> { let parent = path.parent().expect("index paths have a parent"); @@ -345,7 +363,9 @@ fn keep_file(path: &Path, keep: Keep<'_>, move_in: bool) -> Result) { let week = std::time::Duration::from_secs(7 * 24 * 60 * 60); @@ -384,6 +404,9 @@ fn evict(root: &Path, keep: Option<&Path>) { } } total += size; + if used.elapsed().is_ok_and(|age| age < IN_USE) { + continue; + } entries.push((used, size, dir)); } entries.sort();