diff --git a/crates/mise-util/src/http.rs b/crates/mise-util/src/http.rs index dae0699fb81..60c7a2ae479 100644 --- a/crates/mise-util/src/http.rs +++ b/crates/mise-util/src/http.rs @@ -11,7 +11,8 @@ use regex::Regex; use reqwest::StatusCode; use reqwest::header::{ ACCEPT_ENCODING, AUTHORIZATION, CONTENT_RANGE, CONTENT_TYPE, COOKIE, DATE, ETAG, HeaderMap, - HeaderName, HeaderValue, IF_RANGE, LAST_MODIFIED, PROXY_AUTHORIZATION, RANGE, + HeaderName, HeaderValue, IF_MODIFIED_SINCE, IF_NONE_MATCH, IF_RANGE, LAST_MODIFIED, + PROXY_AUTHORIZATION, RANGE, }; use reqwest::{ClientBuilder, IntoUrl, Method, Response}; use serde::{Deserialize, Serialize}; @@ -28,6 +29,8 @@ use crate::time::format_duration; use crate::{env, file}; use mise_settings::Settings; +mod cas; + pub static HTTP: Lazy = Lazy::new(|| { Client::new_shared( crate::network::http_timeout(&Settings::get()), @@ -222,6 +225,36 @@ impl DownloadValidator { } } +/// How a download may use the store of finished downloads. +#[derive(Clone, Copy)] +struct CacheMode<'a> { + /// The checksum the tool pinned, as `algo:hex`. + pin: Option<&'a str>, + enabled: bool, +} + +/// How a download was satisfied. +struct Fetched { + metadata: DownloadFileMetadata, + /// A stored file answered the request, so nothing was written to the + /// destination. + stored: Option, + /// The validator that came with a complete response, for keeping it. + validator: Option, + request_hash: String, +} + +/// What an attempt learned beyond the bytes it wrote. +#[derive(Default)] +struct AttemptOutcome { + /// The validator of the stored copy, sent so the server can confirm it. + cond: Option, + /// The server confirmed the stored copy is current; nothing was written. + not_modified: std::sync::atomic::AtomicBool, + /// The validator that came with a complete response, for keeping it. + validator: Mutex>, +} + #[derive(Debug)] struct DownloadSizeMismatch { expected: u64, @@ -1247,6 +1280,35 @@ impl Client { path: &Path, headers: &HeaderMap, pr: Option<&dyn SingleReport>, + ) -> Result { + self.download_file_with_pin(url, path, headers, None, pr) + .await + } + + /// Like [`Client::download_file`], for a tool that pinned `pin` + /// (`algo:hex`). A download kept earlier under that checksum is used + /// without a request once it hashes to the pin again. The caller still + /// verifies the result. + pub async fn download_file_pinned( + &self, + url: U, + path: &Path, + pin: Option<&str>, + pr: Option<&dyn SingleReport>, + ) -> Result { + let url = url.into_url()?; + let headers = host_auth_headers(&url)?; + self.download_file_with_pin(url, path, &headers, pin, pr) + .await + } + + pub async fn download_file_with_pin( + &self, + url: U, + path: &Path, + headers: &HeaderMap, + pin: Option<&str>, + pr: Option<&dyn SingleReport>, ) -> Result { let url = url.into_url()?; if Settings::get().generate_lockfiles() { @@ -1265,6 +1327,7 @@ impl Client { url.clone(), &directory.path().join("artifact"), headers, + pin, pr, crate::network::http_download_timeout(&Settings::get()), ) @@ -1297,6 +1360,7 @@ impl Client { url, path, headers, + pin, pr, crate::network::http_download_timeout(&Settings::get()), ) @@ -1308,14 +1372,158 @@ impl Client { url: U, path: &Path, headers: &HeaderMap, + pin: Option<&str>, pr: Option<&dyn SingleReport>, total_timeout: Duration, ) -> Result { + let use_cache = Settings::get().download_cache; + let cache = CacheMode { + pin, + enabled: use_cache, + }; + let fetched = self + .download_file_inner(url, path, headers, cache, pr, total_timeout) + .await?; + if let Some(blob) = fetched.stored { + let dest = path.to_path_buf(); + tokio::task::spawn_blocking(move || cas::place(&blob, &dest, false)).await??; + return Ok(fetched.metadata); + } + if use_cache { + // Best effort: a download that can't be kept is still a download. + let path = path.to_path_buf(); + let pin = pin.map(str::to_string); + let effective_filename = fetched.metadata.effective_filename.clone(); + let kept = tokio::task::spawn_blocking(move || { + cas::store( + &path, + cas::Keep { + request_hash: &fetched.request_hash, + pin: pin.as_deref(), + validator: fetched.validator.as_ref(), + effective_filename, + }, + ) + }) + .await; + if let Ok(Err(err)) | Err(err) = kept.map_err(Report::from) { + debug!("could not keep the download: {err:#}"); + } + } + Ok(fetched.metadata) + } + + /// Like [`Client::download_file_pinned`], but the download is kept in the + /// store first and `dest` is a link to it where the filesystem allows one, + /// so the bytes exist once. Across devices, or without a store, it is a + /// copy. For callers that only read the file: a change through the link + /// would change the stored file, which is hashed again before it is reused. + pub async fn download_file_linked( + &self, + url: U, + dest: &Path, + headers: &HeaderMap, + pin: Option<&str>, + pr: Option<&dyn SingleReport>, + ) -> Result { + let url = url.into_url()?; + let name = dest + .file_name() + .ok_or_else(|| eyre!("download destination has no filename: {}", dest.display()))? + .to_string_lossy() + .to_string(); + let request_hash = download_request_hash(&url, headers); + let staging = (Settings::get().download_cache && !Settings::get().generate_lockfiles()) + .then(|| cas::staging_path(&request_hash, &name)) + .flatten(); + let Some(staging) = staging else { + return self + .download_file_with_pin(url, dest, headers, pin, pr) + .await; + }; + let cache = CacheMode { pin, enabled: true }; + let fetched = self + .download_file_inner( + url, + &staging, + headers, + cache, + pr, + crate::network::http_download_timeout(&Settings::get()), + ) + .await?; + let blob = match fetched.stored { + Some(blob) => Some(blob), + None => { + let staged = staging.clone(); + let pin = pin.map(str::to_string); + let effective_filename = fetched.metadata.effective_filename.clone(); + let validator = fetched.validator.clone(); + let hash = fetched.request_hash.clone(); + tokio::task::spawn_blocking(move || { + cas::ingest( + &staged, + cas::Keep { + request_hash: &hash, + pin: pin.as_deref(), + validator: validator.as_ref(), + effective_filename, + }, + ) + }) + .await?? + } + }; + let dest = dest.to_path_buf(); + // With nothing to index it by (no validator, no pin) the staged file is + // the download itself. + let (source, link) = match &blob { + Some(blob) => (blob.clone(), true), + None => (staging.clone(), false), + }; + tokio::task::spawn_blocking(move || cas::place(&source, &dest, link)).await??; + if blob.is_none() { + let _ = file::remove_file(&staging); + } + Ok(fetched.metadata) + } + + /// With `use_cache`, a download kept under `pin` is reused outright, and one + /// kept for the same request is offered to the server for revalidation and + /// reused on `304 Not Modified`. + async fn download_file_inner( + &self, + url: U, + path: &Path, + headers: &HeaderMap, + cache: CacheMode<'_>, + pr: Option<&dyn SingleReport>, + total_timeout: Duration, + ) -> Result { + let CacheMode { + pin, + enabled: use_cache, + } = cache; + let started = Instant::now(); + let url = url.into_url()?; + let request_hash = download_request_hash(&url, headers); + // A pinned download kept earlier needs no request, so this comes before + // the offline check. + if use_cache && let Some(hit) = pin.and_then(cas::lookup_pin) { + let effective_filename = hit.effective_filename.clone(); + if let Some(blob) = tokio::task::spawn_blocking(move || hit.verified()).await? { + return Ok(Fetched { + metadata: DownloadFileMetadata { effective_filename }, + stored: Some(blob), + validator: None, + request_hash, + }); + } + } ensure!( !crate::network::offline(&Settings::get()), "offline mode is enabled" ); - let url = url.into_url()?; debug!( "GET Downloading {} to {}", log_url(&url), @@ -1334,6 +1542,15 @@ impl Client { .await??; let attempt = Arc::new(AtomicUsize::new(0)); let progress = Arc::new(DownloadProgress::default()); + let cached = if use_cache { + cas::lookup_url(&request_hash) + } else { + None + }; + let outcome = Arc::new(AttemptOutcome { + cond: cached.as_ref().map(|c| c.validator.clone()), + ..Default::default() + }); // Retry the whole transfer, resuming a validated partial response when // possible. send_once_with_https_fallback_allow_416 (not @@ -1343,10 +1560,11 @@ impl Client { let progress = progress.clone(); let request_url = url.clone(); let partial = partial.clone(); + let outcome = outcome.clone(); async move { attempt.fetch_add(1, Ordering::Relaxed); progress.start_attempt(); - self.download_file_attempt(request_url, headers, &partial, pr, &progress) + self.download_file_attempt(request_url, headers, &partial, pr, &progress, &outcome) .await } }); @@ -1386,8 +1604,48 @@ impl Client { // of `timeout` prevents us from returning an error while it can still // install the destination in the background. let path = path.to_path_buf(); - tokio::task::spawn_blocking(move || partial.persist(&path)).await??; - Ok(metadata) + if outcome.not_modified.load(Ordering::Relaxed) + && let Some(cached) = cached + { + let effective_filename = cached.effective_filename(); + let hit = cached.into_hit(); + if let Some(blob) = tokio::task::spawn_blocking(move || hit.verified()).await? { + return Ok(Fetched { + metadata: DownloadFileMetadata { effective_filename }, + stored: Some(blob), + validator: None, + request_hash, + }); + } + // The stored file no longer matches what was recorded: fetch it again. + drop(_download_lock); + let cache = CacheMode { + pin, + enabled: false, + }; + // What is left of the budget, not a fresh one. + let remaining = total_timeout + .checked_sub(started.elapsed()) + .filter(|left| !left.is_zero()) + .ok_or_else(|| { + eyre!( + "HTTP download timed out after {} for {}", + format_duration(total_timeout), + log_url(&url) + ) + })?; + return Box::pin(self.download_file_inner(url, &path, headers, cache, pr, remaining)) + .await; + } + let persist_path = path.clone(); + tokio::task::spawn_blocking(move || partial.persist(&persist_path)).await??; + let validator = outcome.validator.lock().unwrap().take(); + Ok(Fetched { + metadata, + stored: None, + validator, + request_hash, + }) } async fn download_file_attempt( @@ -1397,6 +1655,7 @@ impl Client { partial: &PartialDownload, pr: Option<&dyn SingleReport>, progress: &DownloadProgress, + outcome: &AttemptOutcome, ) -> Result { let mut restarted_without_resume = false; loop { @@ -1420,6 +1679,20 @@ impl Client { let offset = resume.as_ref().map(|(_, size)| *size).unwrap_or(0); let mut request_headers = headers.clone(); request_headers.insert(ACCEPT_ENCODING, HeaderValue::from_static("identity")); + let mut conditional = false; + if resume.is_none() + && let Some(cond) = &outcome.cond + { + conditional = true; + match cond { + DownloadValidator::Etag(value) => { + request_headers.insert(IF_NONE_MATCH, HeaderValue::from_str(value)?); + } + DownloadValidator::LastModified { value, .. } => { + request_headers.insert(IF_MODIFIED_SINCE, HeaderValue::from_str(value)?); + } + } + } if let Some((state, _)) = &resume { request_headers.insert(RANGE, HeaderValue::from_str(&format!("bytes={offset}-"))?); request_headers.insert( @@ -1449,6 +1722,14 @@ impl Client { let served_by = resp.url(); progress.served_by(served_by); + if conditional && resp.status() == StatusCode::NOT_MODIFIED { + outcome.not_modified.store(true, Ordering::Relaxed); + return Ok(DownloadFileMetadata::default()); + } + *outcome.validator.lock().unwrap() = (resp.status() != StatusCode::PARTIAL_CONTENT) + .then(|| response_validator(resp.headers())) + .flatten(); + if resp.status() == StatusCode::RANGE_NOT_SATISFIABLE { if let Some(ParsedContentRange::Unsatisfied { total }) = resp .headers() @@ -2265,7 +2546,7 @@ pub fn with_host_auth(url: &Url, headers: &HeaderMap) -> Result { Ok(merged) } -fn host_auth_headers(url: &Url) -> Result { +pub fn host_auth_headers(url: &Url) -> Result { // raw.githubusercontent.com is not an API host, but a private repository's // files are a 404 without the token, so it is routed here too. `get_headers` // decides what each host actually gets. diff --git a/crates/mise-util/src/http/cas.rs b/crates/mise-util/src/http/cas.rs new file mode 100644 index 00000000000..c482e3c3bb3 --- /dev/null +++ b/crates/mise-util/src/http/cas.rs @@ -0,0 +1,421 @@ +//! A content-addressed store of finished downloads. +//! +//! Files are kept by their BLAKE3 hash under `blobs//`, under the +//! name they were downloaded as, since installers read the format and the +//! binary's name from it. Two small indexes point +//! into them: +//! +//! - `urls/`: a request (URL and headers) to the blob it last returned and the +//! validator (`ETag` or `Last-Modified`) the server sent with it, so a later +//! request can ask the server whether it changed and take `304 Not Modified` +//! as an answer. +//! - `pins/`: a checksum a tool pinned to a blob, so a download can be reused +//! with no request at all. +//! +//! Nothing in an index is trusted on its own. A blob is hashed again before +//! it is used, against the pin when there is one, and an entry that no longer +//! matches is dropped. + +use std::path::{Path, PathBuf}; +use std::time::SystemTime; + +use eyre::Result; +use serde::{Deserialize, Serialize}; + +use super::DownloadValidator; +use crate::file; + +/// Blobs are evicted, least recently used first, past this total size. +const MAX_CACHE_BYTES: u64 = 2 * 1024 * 1024 * 1024; + +/// How long after a blob was last used another install may still be placing it. +const IN_USE: std::time::Duration = std::time::Duration::from_secs(10 * 60); + +/// Algorithms a pin may use. `md5` and `sha1` are not collision resistant +/// enough to name content by. +const PIN_ALGOS: &[&str] = &["sha256", "sha384", "sha512", "blake3"]; + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct UrlEntry { + validator: DownloadValidator, + blake3: String, + effective_filename: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct PinEntry { + blake3: String, + effective_filename: Option, +} + +/// Tests that don't ask for a directory never touch the user's cache. +#[cfg(test)] +pub(super) static TEST_ROOT: std::sync::Mutex> = std::sync::Mutex::new(None); + +fn root() -> Option { + #[cfg(test)] + { + TEST_ROOT.lock().unwrap().clone() + } + #[cfg(not(test))] + { + Some(crate::dirs::CACHE.join("downloads-cas")) + } +} + +fn is_hex(value: &str) -> bool { + !value.is_empty() && value.bytes().all(|b| b.is_ascii_hexdigit()) +} + +fn blob_dir(root: &Path, blake3: &str) -> PathBuf { + root.join("blobs").join(blake3) +} + +/// The file a blob directory holds, whatever name it was stored under. +fn blob_file(dir: &Path) -> Option { + std::fs::read_dir(dir) + .ok()? + .flatten() + .map(|entry| entry.path()) + .find(|path| path.is_file()) +} + +/// Where an in-flight download is written: stable per request, so a partial +/// one can be resumed, and on the store's filesystem so it renames into place. +pub(super) fn staging_path(request_hash: &str, name: &str) -> Option { + Some(root()?.join("incoming").join(request_hash).join(name)) +} + +fn url_path(root: &Path, request_hash: &str) -> PathBuf { + root.join("urls").join(format!("{request_hash}.json")) +} + +/// Split a pin into its algorithm and lowercase digest, or `None` when it +/// could not safely name a file. +fn parse_pin(pin: &str) -> Option<(&str, String)> { + let (algo, hex) = pin.split_once(':')?; + (PIN_ALGOS.contains(&algo) && is_hex(hex)).then(|| (algo, hex.to_lowercase())) +} + +fn pin_path(root: &Path, algo: &str, hex: &str) -> PathBuf { + root.join("pins").join(algo).join(format!("{hex}.json")) +} + +/// A stored download ready to be copied out. +pub(super) struct Hit { + blob: PathBuf, + pub(super) effective_filename: Option, + /// The index entry that led here, dropped if the blob doesn't verify. + entry: PathBuf, + verify: Verify, +} + +enum Verify { + /// The blob must hash to its own name. + Blake3(String), + /// The blob must hash to the tool's pinned checksum. + Pin { algo: String, hex: String }, +} + +/// What the server was last asked about, for a conditional request. +pub(super) struct Cached { + pub(super) validator: DownloadValidator, + hit: Hit, +} + +impl Cached { + pub(super) fn into_hit(self) -> Hit { + self.hit + } + + pub(super) fn effective_filename(&self) -> Option { + self.hit.effective_filename.clone() + } +} + +pub(super) fn lookup_url(request_hash: &str) -> Option { + let root = root()?; + let entry = url_path(&root, request_hash); + let url: UrlEntry = serde_json::from_slice(&std::fs::read(&entry).ok()?).ok()?; + if !url.validator.is_valid() || !is_hex(&url.blake3) { + return None; + } + let blob = blob_file(&blob_dir(&root, &url.blake3))?; + Some(Cached { + validator: url.validator, + hit: Hit { + blob, + effective_filename: url.effective_filename, + entry, + verify: Verify::Blake3(url.blake3), + }, + }) +} + +pub(super) fn lookup_pin(pin: &str) -> Option { + let root = root()?; + let (algo, hex) = parse_pin(pin)?; + let entry = pin_path(&root, algo, &hex); + let pinned: PinEntry = serde_json::from_slice(&std::fs::read(&entry).ok()?).ok()?; + if !is_hex(&pinned.blake3) { + return None; + } + let blob = blob_file(&blob_dir(&root, &pinned.blake3))?; + Some(Hit { + blob, + effective_filename: pinned.effective_filename, + entry, + verify: Verify::Pin { + algo: algo.to_string(), + hex, + }, + }) +} + +impl Hit { + /// The stored file, once it hashes to what it is expected to. Otherwise + /// drop what pointed at it and return `None`. + pub(super) fn verified(&self) -> Option { + let intact = match &self.verify { + Verify::Blake3(expected) => { + crate::hash::file_hash_blake3(&self.blob, None).is_ok_and(|hash| hash == *expected) + } + Verify::Pin { algo, hex } => { + crate::hash::ensure_checksum(&self.blob, hex, None, algo).is_ok() + } + }; + if !intact { + let _ = file::remove_file(&self.entry); + // A blob that doesn't hash to its own name is damaged, whichever + // index led here. One that does merely isn't what the pin names. + if let Some(dir) = self.blob.parent() { + let name = dir.file_name().map(|n| n.to_string_lossy().to_string()); + let damaged = crate::hash::file_hash_blake3(&self.blob, None) + .is_ok_and(|hash| Some(hash) != name); + if damaged { + let _ = file::remove_all(dir); + } + } + return None; + } + // Marks the blob as recently used for eviction. + if let Ok(f) = std::fs::File::options().write(true).open(&self.blob) { + let _ = f.set_modified(SystemTime::now()); + } + Some(self.blob.clone()) + } +} + +/// Put `blob` at `dest` without ever leaving a partial file there. With `link` +/// the content is shared where the filesystem allows and copied where it +/// doesn't, such as across devices; without it the content is always copied. +pub(super) fn place(blob: &Path, dest: &Path, link: bool) -> Result<()> { + let parent = dest.parent().unwrap_or(Path::new(".")); + file::create_dir_all(parent)?; + // A directory of our own, so the name inside it is free to link to. A + // `TempPath` would exist already, and Windows can't copy over a file that + // is open. + let scratch = tempfile::Builder::new() + .prefix(".mise-place-") + .tempdir_in(parent) + .map_err(|err| placing(err, dest))?; + let staged = scratch.path().join("file"); + if link { + file::hard_link_or_copy(blob, &staged)?; + } else { + file::copy(blob, &staged)?; + } + // `persist` replaces an existing file on Windows too, where a plain rename + // does not. + tempfile::TempPath::try_from_path(&staged) + .map_err(|err| placing(err, dest))? + .persist(dest) + .map_err(|err| placing(err.error, dest))?; + Ok(()) +} + +/// Name the step and the path, with the Windows hint for an over-long path: +/// `tempfile` doesn't get the extended-length handling `std::fs` does, so this +/// is the call that fails first as a directory approaches `MAX_PATH`. +fn placing(err: std::io::Error, dest: &Path) -> eyre::Report { + let msg = file::with_io_hint( + format!( + "failed to move the downloaded file into place: {}", + file::display_path(dest) + ), + dest, + &err, + ); + eyre::Report::new(err).wrap_err(msg) +} + +/// Write `contents` to `path` so a reader never sees a partial file. +fn write_atomic(path: &Path, contents: &[u8]) -> Result<()> { + let parent = path.parent().expect("index paths have a parent"); + file::create_dir_all(parent)?; + let tmp = tempfile::NamedTempFile::new_in(parent)?.into_temp_path(); + file::write(&tmp, contents)?; + tmp.persist(path).map_err(|err| err.error)?; + Ok(()) +} + +/// What kept a download: how to find it again. +pub(super) struct Keep<'a> { + pub(super) request_hash: &'a str, + /// The checksum the tool pinned. + pub(super) pin: Option<&'a str>, + pub(super) validator: Option<&'a DownloadValidator>, + pub(super) effective_filename: Option, +} + +/// Keep `path`, just downloaded, by copying it into the store. Returns the +/// stored file, or `None` when there is nothing to index it by. +pub(super) fn store(path: &Path, keep: Keep<'_>) -> Result> { + keep_file(path, keep, false) +} + +/// Keep `path`, which sits in the store's staging area, by moving it into the +/// store. Returns the stored file. +pub(super) fn ingest(path: &Path, keep: Keep<'_>) -> Result> { + keep_file(path, keep, true) +} + +fn keep_file(path: &Path, keep: Keep<'_>, move_in: bool) -> Result> { + let Some(root) = root() else { + return Ok(None); + }; + // The caller checks the pin after the download. Checking here too keeps a + // wrong file from ever being indexed under it. + let pin = keep.pin.and_then(parse_pin).filter(|(algo, hex)| { + let ok = crate::hash::ensure_checksum(path, hex, None, algo).is_ok(); + if !ok { + debug!("not indexing a download that doesn't match its pin"); + } + ok + }); + if keep.validator.is_none() && pin.is_none() { + return Ok(None); + } + let name = path.file_name().expect("downloads have a file name"); + let blake3 = crate::hash::file_hash_blake3(path, None)?; + let dir = blob_dir(&root, &blake3); + // An existing blob is reused only while it still hashes to its own name; + // otherwise it is replaced by the file just downloaded. + let existing = blob_file(&dir).filter(|existing| { + let intact = crate::hash::file_hash_blake3(existing, None).is_ok_and(|h| h == blake3); + if !intact { + let _ = file::remove_all(&dir); + } + intact + }); + let blob = match existing { + Some(blob) => { + if move_in { + let _ = file::remove_file(path); + } + blob + } + None => { + let blob = dir.join(name); + // Built beside its final place so the rename can't cross a device. + let blobs = root.join("blobs"); + file::create_dir_all(&blobs)?; + let building = tempfile::Builder::new() + .prefix(".building-") + .tempdir_in(&blobs)?; + let staged = building.path().join(name); + if move_in { + file::rename(path, &staged)?; + } else { + file::copy(path, &staged)?; + } + let building = building.keep(); + if let Err(err) = std::fs::rename(&building, &dir) { + let _ = file::remove_all(&building); + // Another process stored the same download first. + if blob_file(&dir).is_none() { + return Err(err.into()); + } + } + blob_file(&dir).unwrap_or(blob) + } + }; + if let Some(validator) = keep.validator { + let entry = UrlEntry { + validator: validator.clone(), + blake3: blake3.clone(), + effective_filename: keep.effective_filename.clone(), + }; + write_atomic( + &url_path(&root, keep.request_hash), + &serde_json::to_vec(&entry)?, + )?; + } + if let Some((algo, hex)) = pin { + let entry = PinEntry { + blake3, + effective_filename: keep.effective_filename, + }; + write_atomic(&pin_path(&root, algo, &hex), &serde_json::to_vec(&entry)?)?; + } + evict(&root, blob.parent()); + Ok(Some(blob)) +} + +/// Past `MAX_CACHE_BYTES` the least recently used blobs go first, except +/// `keep`, which the caller is about to use, and any used within `IN_USE`: a +/// blob is verified, which refreshes its time, moments before another install +/// places it, so a recent one may be mid-use. Staged downloads that were never +/// finished go after a week. +fn evict(root: &Path, keep: Option<&Path>) { + let week = std::time::Duration::from_secs(7 * 24 * 60 * 60); + if let Ok(read) = std::fs::read_dir(root.join("incoming")) { + for entry in read.flatten() { + let stale = entry + .metadata() + .and_then(|m| m.modified()) + .ok() + .and_then(|modified| modified.elapsed().ok()) + .is_some_and(|age| age > week); + if stale { + let _ = file::remove_all(entry.path()); + } + } + } + let Ok(read) = std::fs::read_dir(root.join("blobs")) else { + return; + }; + let mut entries = Vec::new(); + let mut total = 0u64; + for entry in read.flatten() { + let dir = entry.path(); + if entry.file_name().to_string_lossy().starts_with('.') + || !dir.is_dir() + || keep == Some(dir.as_path()) + { + continue; + } + // Names of one blob share a file where they can, so count the largest. + let (mut size, mut used) = (0, SystemTime::UNIX_EPOCH); + for file in std::fs::read_dir(&dir).into_iter().flatten().flatten() { + if let Ok(meta) = file.metadata() { + size = size.max(meta.len()); + used = used.max(meta.modified().unwrap_or(SystemTime::UNIX_EPOCH)); + } + } + total += size; + if used.elapsed().is_ok_and(|age| age < IN_USE) { + continue; + } + entries.push((used, size, dir)); + } + entries.sort(); + for (_, size, dir) in entries { + if total <= MAX_CACHE_BYTES { + break; + } + if file::remove_all(&dir).is_ok() { + total -= size; + } + } +} diff --git a/crates/mise-util/src/http/tests/download.rs b/crates/mise-util/src/http/tests/download.rs index 8df72050955..4ffba65e3ad 100644 --- a/crates/mise-util/src/http/tests/download.rs +++ b/crates/mise-util/src/http/tests/download.rs @@ -445,6 +445,7 @@ async fn test_download_total_timeout_bounds_trickling_response() { &path, &HeaderMap::new(), None, + None, Duration::from_millis(500), ), ) diff --git a/crates/mise-util/src/http/tests/mod.rs b/crates/mise-util/src/http/tests/mod.rs index 960d1757856..e26cb41701d 100644 --- a/crates/mise-util/src/http/tests/mod.rs +++ b/crates/mise-util/src/http/tests/mod.rs @@ -559,3 +559,215 @@ refresh_expires_at = "2099-01-01T00:00:00Z" fn json_empty_array_response() -> &'static str { "HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\n[]" } + +fn cacheable_download_response() -> &'static str { + concat!( + "HTTP/1.1 200 OK\r\n", + "Content-Length: 5\r\n", + "ETag: \"artifact-v1\"\r\n", + "Connection: close\r\n", + "\r\n", + "hello" + ) +} +fn not_modified_response() -> &'static str { + "HTTP/1.1 304 Not Modified\r\nETag: \"artifact-v1\"\r\nConnection: close\r\n\r\n" +} + +#[tokio::test(flavor = "current_thread")] +async fn test_download_reuses_stored_file_on_not_modified() { + let _settings = set_test_http_retries(0); + let cache = tempfile::tempdir().unwrap(); + *cas::TEST_ROOT.lock().unwrap() = Some(cache.path().to_path_buf()); + let (port, count, requests) = spawn_recording_server(vec![ + cacheable_download_response(), + not_modified_response(), + not_modified_response(), + cacheable_download_response(), + ]) + .await; + let url = format!("http://127.0.0.1:{port}/artifact"); + let client = Client::new(Duration::from_secs(2), ClientKind::Http).unwrap(); + let dir = tempfile::tempdir().unwrap(); + let dest = dir.path().join("artifact"); + + client.download_file(&url, &dest, None).await.unwrap(); + assert_eq!(std::fs::read(&dest).unwrap(), b"hello"); + + // The second request is conditional, and a 304 serves the stored file. + std::fs::remove_file(&dest).unwrap(); + client.download_file(&url, &dest, None).await.unwrap(); + assert_eq!(std::fs::read(&dest).unwrap(), b"hello"); + assert!( + requests.lock().unwrap()[1] + .to_lowercase() + .contains("if-none-match: \"artifact-v1\"") + ); + + // A stored file that no longer hashes to what was recorded is not used, + // even though the server says it is current. + let stored = stored_blob(cache.path()); + std::fs::write(&stored, b"tampered").unwrap(); + std::fs::remove_file(&dest).unwrap(); + client.download_file(&url, &dest, None).await.unwrap(); + assert_eq!(std::fs::read(&dest).unwrap(), b"hello"); + assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 4); + + *cas::TEST_ROOT.lock().unwrap() = None; +} + +#[tokio::test(flavor = "current_thread")] +async fn test_download_reuses_pinned_file_without_a_request() { + let _settings = set_test_http_retries(0); + let cache = tempfile::tempdir().unwrap(); + *cas::TEST_ROOT.lock().unwrap() = Some(cache.path().to_path_buf()); + let (port, count) = spawn_canned_server(vec![cacheable_download_response()]).await; + let url = format!("http://127.0.0.1:{port}/artifact"); + let client = Client::new(Duration::from_secs(2), ClientKind::Http).unwrap(); + let dir = tempfile::tempdir().unwrap(); + let dest = dir.path().join("artifact"); + let sha = crate::hash::file_hash_prog::; + let hello = dir.path().join("hello"); + std::fs::write(&hello, b"hello").unwrap(); + let pin = format!("sha256:{}", sha(&hello, None).unwrap()); + + client + .download_file_pinned(&url, &dest, Some(&pin), None) + .await + .unwrap(); + assert_eq!(std::fs::read(&dest).unwrap(), b"hello"); + + // A different URL with the same pin is served without any request. + std::fs::remove_file(&dest).unwrap(); + let other = "http://127.0.0.1:1/elsewhere"; + client + .download_file_pinned(other, &dest, Some(&pin), None) + .await + .unwrap(); + assert_eq!(std::fs::read(&dest).unwrap(), b"hello"); + assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 1); + + // A blob that no longer matches the pin is not used. + let blob = stored_blob(cache.path()); + std::fs::write(&blob, b"tampered").unwrap(); + std::fs::remove_file(&dest).unwrap(); + assert!( + client + .download_file_pinned(other, &dest, Some(&pin), None) + .await + .is_err() + ); + + *cas::TEST_ROOT.lock().unwrap() = None; +} + +fn stored_blob(cache: &std::path::Path) -> PathBuf { + let dir = std::fs::read_dir(cache.join("blobs")) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + std::fs::read_dir(dir) + .unwrap() + .next() + .unwrap() + .unwrap() + .path() +} + +#[tokio::test(flavor = "current_thread")] +async fn test_download_linked_shares_one_stored_file() { + let _settings = set_test_http_retries(0); + let cache = tempfile::tempdir().unwrap(); + *cas::TEST_ROOT.lock().unwrap() = Some(cache.path().to_path_buf()); + let (port, count) = spawn_canned_server(vec![cacheable_download_response()]).await; + let url = format!("http://127.0.0.1:{port}/artifact.bin"); + let client = Client::new(Duration::from_secs(2), ClientKind::Http).unwrap(); + let dir = tempfile::tempdir().unwrap(); + let dest = dir.path().join("artifact.bin"); + let hello = dir.path().join("hello"); + std::fs::write(&hello, b"hello").unwrap(); + let pin = format!( + "sha256:{}", + crate::hash::file_hash_prog::(&hello, None).unwrap() + ); + + client + .download_file_linked(&url, &dest, &HeaderMap::new(), Some(&pin), None) + .await + .unwrap(); + assert_eq!(std::fs::read(&dest).unwrap(), b"hello"); + // The stored file keeps the name it was downloaded as. + assert_eq!( + stored_blob(cache.path()).file_name().unwrap(), + "artifact.bin" + ); + + // A second install is served from the store with no request, and the + // destination holds the same bytes. + std::fs::remove_file(&dest).unwrap(); + client + .download_file_linked(&url, &dest, &HeaderMap::new(), Some(&pin), None) + .await + .unwrap(); + assert_eq!(std::fs::read(&dest).unwrap(), b"hello"); + assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 1); + + *cas::TEST_ROOT.lock().unwrap() = None; +} + +#[tokio::test(flavor = "current_thread")] +async fn test_download_replaces_a_damaged_stored_file() { + let _settings = set_test_http_retries(0); + let cache = tempfile::tempdir().unwrap(); + *cas::TEST_ROOT.lock().unwrap() = Some(cache.path().to_path_buf()); + let (port, count) = spawn_canned_server(vec![ + cacheable_download_response(), + cacheable_download_response(), + ]) + .await; + let url = format!("http://127.0.0.1:{port}/artifact.bin"); + let client = Client::new(Duration::from_secs(2), ClientKind::Http).unwrap(); + let dir = tempfile::tempdir().unwrap(); + let dest = dir.path().join("artifact.bin"); + let hello = dir.path().join("hello"); + std::fs::write(&hello, b"hello").unwrap(); + let pin = format!( + "sha256:{}", + crate::hash::file_hash_prog::(&hello, None).unwrap() + ); + + client + .download_file_linked(&url, &dest, &HeaderMap::new(), Some(&pin), None) + .await + .unwrap(); + + // Damage the stored file in place. The next download must not install it, + // and must leave a good copy behind rather than the damaged one. + let blob = stored_blob(cache.path()); + std::fs::write(&blob, b"tampered").unwrap(); + std::fs::remove_file(&dest).unwrap(); + client + .download_file_linked(&url, &dest, &HeaderMap::new(), Some(&pin), None) + .await + .unwrap(); + assert_eq!(std::fs::read(&dest).unwrap(), b"hello"); + assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 2); + assert_eq!(std::fs::read(stored_blob(cache.path())).unwrap(), b"hello"); + + // Replacing a destination that already exists works too. + // Replace it with a new file: writing through the link would change the + // stored file itself. + std::fs::remove_file(&dest).unwrap(); + std::fs::write(&dest, b"stale").unwrap(); + let (port, _) = spawn_canned_server(vec![]).await; + let other = format!("http://127.0.0.1:{port}/artifact.bin"); + client + .download_file_linked(&other, &dest, &HeaderMap::new(), Some(&pin), None) + .await + .unwrap(); + assert_eq!(std::fs::read(&dest).unwrap(), b"hello"); + + *cas::TEST_ROOT.lock().unwrap() = None; +} diff --git a/docs/continuous-integration.md b/docs/continuous-integration.md index f9e73cc64d9..bf83f3afe6b 100644 --- a/docs/continuous-integration.md +++ b/docs/continuous-integration.md @@ -38,6 +38,11 @@ jobs: - run: mise run test ``` +::: warning +A restored tool cache is trusted as is: mise does not check installed files against +`mise.lock`. See [Restored caches and installed tools](/dev-tools/mise-lock.html#restored-caches). +::: + The example assumes a `test` task in `mise.toml`, such as `test = "npm ci && npm test"` under `[tasks]`. `mise exec -- npm test` works as well. diff --git a/docs/dev-tools/backends/http.md b/docs/dev-tools/backends/http.md index 933ddfe7d1f..60031692dec 100644 --- a/docs/dev-tools/backends/http.md +++ b/docs/dev-tools/backends/http.md @@ -552,6 +552,12 @@ Changes to shared files, including changes made by a `postinstall` hook, affect every install that uses them. `mise install --system`, `mise install --shared` and `mise install-into` destinations always get their own copy. +::: warning The shared store is trusted as is +mise does not re-check files already in `http-tarballs` against the lockfile +checksum. Don't let untrusted jobs or users write to a shared or restored store. +See [Restored caches and installed tools](/dev-tools/mise-lock.html#restored-caches). +::: + `mise prune`, `mise cache prune` and `mise cache clear` leave `http-tarballs` alone, because installs may still link to it. Installs made by older mise versions may also link into it. To give an install its own copy, remove diff --git a/docs/dev-tools/mise-lock.md b/docs/dev-tools/mise-lock.md index 6f32cf4f8d6..27fb3ed76f3 100644 --- a/docs/dev-tools/mise-lock.md +++ b/docs/dev-tools/mise-lock.md @@ -411,6 +411,36 @@ Lifecycle scripts can still produce different output on each machine. See for the layout and for editing sidecars, and the [npm](/dev-tools/backends/npm.html) and [PyPI](/dev-tools/backends/pypi.html) backend pages for installer limits. +## Restored caches and installed tools {#restored-caches} + +::: warning +`mise.lock` verifies what mise downloads, not what is already on disk. A +restored CI cache or a shared extraction store is trusted as is. +::: + +mise can't validate an installed tree against the lockfile. The checksum covers +the downloaded artifact, and installs are mutable on purpose: tools update +themselves and `postinstall` hooks change files. For tools compiled from source, +such as Ruby, the lockfile doesn't vouch for the compiled output, and a tool +that downloads its own source may record no checksum for it at all. + +Downloads are different. mise keeps them in a cache and reuses one only when +the server confirms it is unchanged, or when it hashes to a pinned checksum. A +pinned checksum is always checked after the download; without one, a cache that +someone can write to is trusted. See +[`download_cache`](/configuration/settings.html#download_cache). + +So treat a cache of installed files as trusted build input: + +- Don't restore caches written by untrusted refs or triggers. +- Record a manifest of the cache outside it, in trusted CI metadata, and check + the restored files against it. A manifest stored in the cache can be replaced + along with it. That proves the cache is unchanged since CI saved it, not that + it matches the lockfile. +- To check an install against the lockfile, run `mise install --locked` with an + empty data directory. Every artifact is then downloaded and checked against + the checksum the lockfile records, for the backends that record one. + ## Provenance and verification {#provenance-and-security} For supported backends, `mise lock` records provenance such as SLSA, Cosign, diff --git a/docs/directories.md b/docs/directories.md index d3eab322cf6..c0287a3f246 100644 --- a/docs/directories.md +++ b/docs/directories.md @@ -78,12 +78,11 @@ machines is not a supported way to install tools. For caching it in CI, see ### `downloads` {#local-share-mise-downloads} -Backends may write archives here while they install a tool. mise deletes them -afterwards unless -[`always_keep_download`](/configuration/settings.html#always_keep_download) is -set. This is not a download cache. To avoid reinstalling tools in CI, cache -`installs/` instead, together with the install store when it is a separate -directory. +Backends may write archives here while they install a tool, and mise deletes +them afterwards. Finished downloads are kept in the +[download cache](/configuration/settings.html#download_cache) instead. To avoid +reinstalling tools in CI, cache `installs/`, together with the install store +when it is a separate directory. ### `plugins` {#local-share-mise-plugins} diff --git a/e2e/backend/test_aqua_resume_download b/e2e/backend/test_aqua_resume_download index 4a5b05ee1d9..7cdc186812b 100644 --- a/e2e/backend/test_aqua_resume_download +++ b/e2e/backend/test_aqua_resume_download @@ -82,13 +82,13 @@ EOF_REGISTRY export MISE_AQUA_REGISTRY_URL="file://$PWD/registry" assert_fail "mise install aqua:example/resume-tool@1.0.0" -assert_succeed "find '$MISE_DATA_DIR/downloads' -name '*.mise-part' -type f | grep -q ." -assert_succeed "find '$MISE_DATA_DIR/downloads' -name '*.mise-part.json' -type f | grep -q ." +assert_succeed "find '$MISE_CACHE_DIR/downloads-cas/incoming' -name '*.mise-part' -type f | grep -q ." +assert_succeed "find '$MISE_CACHE_DIR/downloads-cas/incoming' -name '*.mise-part.json' -type f | grep -q ." mise install aqua:example/resume-tool@1.0.0 assert "mise exec aqua:example/resume-tool@1.0.0 -- resume-tool" "resumed download" assert_contains "sed -n '1p' requests.log" 'range= if-range=' assert_contains "cat requests.log" 'range=bytes=' assert_contains "cat requests.log" 'if-range="resume-v1"' -assert_fail "find '$MISE_DATA_DIR/downloads' -name '*.mise-part' -type f | grep -q ." -assert_fail "find '$MISE_DATA_DIR/downloads' -name '*.mise-part.json' -type f | grep -q ." +assert_fail "find '$MISE_CACHE_DIR/downloads-cas/incoming' -name '*.mise-part' -type f | grep -q ." +assert_fail "find '$MISE_CACHE_DIR/downloads-cas/incoming' -name '*.mise-part.json' -type f | grep -q ." diff --git a/e2e/backend/test_http_download_cache b/e2e/backend/test_http_download_cache new file mode 100644 index 00000000000..366084af37d --- /dev/null +++ b/e2e/backend/test_http_download_cache @@ -0,0 +1,63 @@ +#!/usr/bin/env bash + +# A download pinned by a checksum is kept by that checksum and reused, but only +# while it still hashes to the pin. + +SRV="$PWD/srv" +mkdir -p "$SRV" +printf '#!/bin/sh\necho cached tool ok\n' >"$SRV/mytool" +SHA="$(sha256sum "$SRV/mytool" | cut -d' ' -f1)" + +start_http_server --dir "$SRV" + +cat >mise.toml <"$SRV/mytool2" +touch -d '2020-01-01' "$SRV/mytool2" +cat >mise.toml <mise.toml <>"$BLOB" +mise uninstall http:cas-tool@1.0.0 +assert_fail "mise install" +assert_fail "test -f $ENTRY" diff --git a/e2e/cli/test_plugins_packslip b/e2e/cli/test_plugins_packslip index 08fb09e8c9d..04a12844703 100644 --- a/e2e/cli/test_plugins_packslip +++ b/e2e/cli/test_plugins_packslip @@ -54,12 +54,14 @@ wait "$second_install" mise plugins update bfs cp "$plugin/.mise-packslip-plugin.json" before.json -# A corrupt download must not remove or replace the working plugin. +# A corrupt download must not remove or replace the working plugin. The download +# cache would serve the verified copy kept earlier without asking the server, so +# it is off here to exercise the corrupt response. printf yes >server-state/corrupt -assert_fail "mise plugins update bfs" +assert_fail "MISE_DOWNLOAD_CACHE=0 mise plugins update bfs" assert "cmp before.json '$plugin/.mise-packslip-plugin.json'" assert "test -f '$plugin/metadata.lua'" -assert_fail "mise plugins install --force vfox:bfs '$source_url'" +assert_fail "MISE_DOWNLOAD_CACHE=0 mise plugins install --force vfox:bfs '$source_url'" assert "test -f '$plugin/metadata.lua'" # A signature from the pilot repository cannot authorize a different source. diff --git a/schema/mise.json b/schema/mise.json index 97a7f7df984..ba21896d6f4 100644 --- a/schema/mise.json +++ b/schema/mise.json @@ -637,8 +637,9 @@ "type": "boolean" }, "always_keep_download": { - "description": "Keep downloaded archives and sources after installing, for debugging.", - "type": "boolean" + "description": "[deprecated] Keep downloaded archives and sources after installing, for debugging.", + "type": "boolean", + "deprecated": true }, "always_keep_install": { "description": "Keep a failed installation's partial install directory and downloads, for debugging.", @@ -908,6 +909,11 @@ } } }, + "download_cache": { + "default": true, + "description": "Reuse finished downloads instead of fetching them again.", + "type": "boolean" + }, "enable_tools": { "description": "Allowlist of configured tools mise uses; other tools are ignored.", "type": "array", diff --git a/settings.toml b/settings.toml index 36a09c569be..6bc44d26e63 100644 --- a/settings.toml +++ b/settings.toml @@ -104,13 +104,15 @@ optional = true type = "Bool" [always_keep_download] -description = "Keep downloaded archives and sources after installing, for debugging." +deprecated = "Finished downloads are kept in the download cache instead; see download_cache." +deprecated_remove_at = "2027.11.0" +deprecated_warn_at = "2026.11.0" +description = "[deprecated] Keep downloaded archives and sources after installing, for debugging." docs = """ mise keeps the files under `~/.local/share/mise/downloads` so you can inspect -them while debugging a backend or plugin. This is not a download cache: some -backends skip a download when the file already exists, but that is not -guaranteed. To avoid reinstalling tools in CI, cache -`~/.local/share/mise/installs` instead. +them while debugging a backend or plugin. Downloads that go through mise's HTTP +client are kept in [`download_cache`](#download_cache) instead, under their own +names, so this is only needed for files an asdf plugin script downloads itself. """ env = "MISE_ALWAYS_KEEP_DOWNLOAD" type = "Bool" @@ -673,6 +675,37 @@ an installation source to the dotnet CLI. env = "MISE_DOTNET_REGISTRY_URL" type = "Url" +[download_cache] +default = true +description = "Reuse finished downloads instead of fetching them again." +docs = """ +mise keeps finished downloads in `$MISE_CACHE_DIR/downloads-cas`, named by their +hash, and uses one again in two cases: + +- **The server confirms it is unchanged.** The next request for the same URL + carries the `ETag` or `Last-Modified` value the server sent, and a + `304 Not Modified` answer means the kept file is used instead of a new + download. Servers that send no validator are not cached. +- **A tool pinned its checksum**, in its options or in `mise.lock`. The kept + file is used with no request at all, even if the URL changed. + +The `http`, `github`, `aqua` and `packslip` backends pin and install from the +stored file, linked into place where the filesystem allows, so the archive takes +no extra space and is not copied; across devices it is copied. Every other +download that goes through mise's HTTP client uses the first case and is copied +into place. + +A kept file is hashed again before it is used, and one that no longer matches is +dropped and downloaded again. The recorded hash lives beside the file, so this +catches damage but does not stop someone who can write the cache from replacing +both. A pinned checksum is always checked after the download, so pin one if the +cache is shared. The cache holds up to 2 GiB; the least recently used entries go +first. Extracted files are not covered; see +[Restored caches and installed tools](/dev-tools/mise-lock.html#restored-caches). +""" +env = "MISE_DOWNLOAD_CACHE" +type = "Bool" + [enable_tools] description = "Allowlist of configured tools mise uses; other tools are ignored." docs = """ diff --git a/src/backend/aqua.rs b/src/backend/aqua.rs index aca12c45cf3..6274fbe244e 100644 --- a/src/backend/aqua.rs +++ b/src/backend/aqua.rs @@ -571,7 +571,8 @@ impl Backend for AquaBackend { // Public repos download from the browser URL; private repos return 404/HTML there // and must be fetched from the API asset endpoint instead. let download_url = select_github_download_url(pkg.private, &url, url_api.as_deref()).await; - self.download(ctx, &tv, &download_url, &filename).await?; + self.download(ctx, &tv, &download_url, &filename, api_digest.as_deref()) + .await?; // Snapshot before any mutation so verify() knows whether the lockfile // originally contained a checksum (vs. one we just wrote from api_digest). @@ -2815,6 +2816,7 @@ impl AquaBackend { tv: &ToolVersion, url: &str, filename: &str, + api_digest: Option<&str>, ) -> Result<()> { let tarball_path = tv.download_path().join(filename); if tarball_path.exists() { @@ -2824,8 +2826,22 @@ impl AquaBackend { return Ok(()); } ctx.pr.set_message(format!("download {filename}")); - HTTP.download_file(url, &tarball_path, Some(ctx.pr.as_ref())) - .await?; + // The lockfile's checksum, or else the digest the GitHub API gave for the + // release asset, which verify() checks the same way. + let pinned = tv + .lock_platforms + .get(&self.get_platform_key()) + .and_then(|platform| platform.checksum.clone()) + .or_else(|| api_digest.map(str::to_string)); + let headers = crate::http::host_auth_headers(&reqwest::Url::parse(url)?)?; + HTTP.download_file_linked( + url, + &tarball_path, + &headers, + pinned.as_deref(), + Some(ctx.pr.as_ref()), + ) + .await?; Ok(()) } diff --git a/src/backend/github.rs b/src/backend/github.rs index 6924981efeb..2c457ff4cb9 100644 --- a/src/backend/github.rs +++ b/src/backend/github.rs @@ -6,8 +6,8 @@ use crate::backend::options::{BackendOptions, VersionOrder}; use crate::backend::platform_target::PlatformTarget; use crate::backend::static_helpers::{ ArchiveLayout, get_filename_from_url, install_artifact, lookup_platform_key, - lookup_with_fallback, template_string, try_with_v_prefix, try_with_v_prefix_and_repo, - verify_artifact, + lookup_with_fallback, pinned_checksum, template_string, try_with_v_prefix, + try_with_v_prefix_and_repo, verify_artifact, }; use crate::backend::{MISE_BINS_DIR, SecurityFeature, runtime_path_for_install_path}; use crate::config::{Config, Settings, SettingsExt}; @@ -1715,8 +1715,15 @@ impl UnifiedGitBackend { }; ctx.pr.set_message(format!("download {filename}")); - HTTP.download_file_with_headers(url, &file_path, &headers, Some(ctx.pr.as_ref())) - .await?; + let pinned = pinned_checksum(tv, &self.get_platform_key(), opts.raw()); + HTTP.download_file_linked( + url, + &file_path, + &headers, + pinned.as_deref(), + Some(ctx.pr.as_ref()), + ) + .await?; // Verify and install ctx.pr.next_operation(); diff --git a/src/backend/http.rs b/src/backend/http.rs index b65a6214d14..98109e483b0 100644 --- a/src/backend/http.rs +++ b/src/backend/http.rs @@ -9,8 +9,8 @@ use crate::backend::static_helpers::{ apply_rename_exe, bin_name_for_download, clean_binary_name, ensure_plain_bin_name, ensure_safe_relative_bin_path, eval_checksum_expr, fetch_checksum_from_file_with_headers, fetch_checksum_from_shasums_with_headers, get_filename_from_url, lookup_value_with_fallback, - rename_binary_name, shasums_has_entries, template_string, template_string_for_target, - template_string_strict, verify_artifact, + pinned_checksum, rename_binary_name, shasums_has_entries, template_string, + template_string_for_target, template_string_strict, verify_artifact, }; use crate::backend::version_list; use crate::config::Config; @@ -1374,6 +1374,8 @@ impl Backend for HttpBackend { .and_then(|p| p.checksum.as_ref()) .is_some(); + let pinned = pinned_checksum(&tv, &platform_key, opts.raw()); + let download = match local_artifact_path(&url)? { Some(src) => { ctx.pr.set_message(format!("copy {filename}")); @@ -1387,16 +1389,23 @@ impl Backend for HttpBackend { ctx.pr.set_message(format!("download {filename}")); let headers = opts.headers(Some(tv.version.as_str()))?; if headers.is_empty() { - HTTP.download_file_with_metadata(&url, &file_path, Some(ctx.pr.as_ref())) - .await? + HTTP.download_file_linked( + &url, + &file_path, + &crate::http::host_auth_headers(&reqwest::Url::parse(&url)?)?, + pinned.as_deref(), + Some(ctx.pr.as_ref()), + ) + .await? } else { // Keep the automatic host token; configured headers override it. let headers = crate::http::with_host_auth(&reqwest::Url::parse(&url)?, &headers)?; - HTTP.download_file_with_headers_metadata( + HTTP.download_file_linked( &url, &file_path, &headers, + pinned.as_deref(), Some(ctx.pr.as_ref()), ) .await? diff --git a/src/backend/packslip.rs b/src/backend/packslip.rs index ecedf0b0bb5..1657e275f72 100644 --- a/src/backend/packslip.rs +++ b/src/backend/packslip.rs @@ -1656,7 +1656,8 @@ impl PackslipBackend { let bundle_path = tv.download_path().join(bundle_name(&project)); file::create_dir_all(tv.download_path())?; ctx.pr.set_message("download packslip".into()); - crate::packslip::download_file(&located.url, &bundle_path, Some(ctx.pr.as_ref())).await?; + crate::packslip::download_file(&located.url, &bundle_path, None, Some(ctx.pr.as_ref())) + .await?; let pinned: Vec<&String> = located.digest.iter().chain(vendor_digest.iter()).collect(); if !pinned.is_empty() { let (actual, _) = packslip::digest_file(&bundle_path)?; @@ -1813,7 +1814,13 @@ impl PackslipBackend { let file_path = tv.download_path().join(&artifact.name); ctx.pr.next_operation(); ctx.pr.set_message(format!("download {}", artifact.name)); - crate::packslip::download_file(&url, &file_path, Some(ctx.pr.as_ref())).await?; + // The signed digest names the artifact, so a copy kept earlier can stand in + // for the download; `verify_bundle` below still checks the result. + let signed = statement + .digest_of(&artifact.name) + .map(|sha256| format!("sha256:{sha256}")); + crate::packslip::download_file(&url, &file_path, signed.as_deref(), Some(ctx.pr.as_ref())) + .await?; // The signed digest and size first, then what the lockfile remembers: // a lock entry written from an earlier packslip keeps its checksum and diff --git a/src/backend/static_helpers.rs b/src/backend/static_helpers.rs index ff0358068ef..e8f080df682 100644 --- a/src/backend/static_helpers.rs +++ b/src/backend/static_helpers.rs @@ -932,6 +932,21 @@ fn make_configured_bin_executable(search_dir: &Path, bin_name: &str) -> Result<( Ok(()) } +/// The checksum a tool pinned for this platform, in its options or in the +/// lockfile. It names a kept download that can be reused, and is still checked +/// after the download either way. +pub(crate) fn pinned_checksum( + tv: &crate::toolset::ToolVersion, + platform_key: &str, + opts: &crate::toolset::ToolVersionOptions, +) -> Option { + lookup_with_fallback(opts, "checksum").or_else(|| { + tv.lock_platforms + .get(platform_key) + .and_then(|platform| platform.checksum.clone()) + }) +} + pub(crate) fn verify_artifact( _tv: &crate::toolset::ToolVersion, file_path: &Path, diff --git a/src/packslip.rs b/src/packslip.rs index 85e0d80c272..a46a8087912 100644 --- a/src/packslip.rs +++ b/src/packslip.rs @@ -263,21 +263,23 @@ fn downloaded_sign_in_page(url: &str, dest: &Path) -> bool { pub(crate) async fn download_file( url: &str, dest: &Path, + pin: Option<&str>, pr: Option<&dyn SingleReport>, ) -> Result<()> { let reason = match HTTP - .download_file_with_headers(url, dest, &headers_for(url)?, pr) + .download_file_linked(url, dest, &headers_for(url)?, pin, pr) .await { - Ok(()) if !downloaded_sign_in_page(url, dest) => return Ok(()), - Ok(()) => Unreachable::SignIn, + Ok(_) if !downloaded_sign_in_page(url, dest) => return Ok(()), + Ok(_) => Unreachable::SignIn, Err(err) => Unreachable::Failed(err), }; let Some(api_url) = retry_url(url, &reason).await else { return Err(reason.into_error(url)); }; - HTTP.download_file_with_headers(&api_url, dest, &headers_for(&api_url)?, pr) + HTTP.download_file_linked(&api_url, dest, &headers_for(&api_url)?, pin, pr) .await + .map(|_| ()) } /// Fetch a document a packslip names — a manifest or a signed release list — @@ -371,7 +373,10 @@ pub(crate) async fn fetch_files( // The tool is installed by now and the asset is an extra: // one that cannot be fetched is reported, not fatal. One // that arrives with the wrong digest is another matter. - if let Err(err) = download_file(url, &dest, Some(pr)).await { + let signed = statement + .digest_of(name) + .map(|sha256| format!("sha256:{sha256}")); + if let Err(err) = download_file(url, &dest, signed.as_deref(), Some(pr)).await { let _ = file::remove_all(&dest); warn!("{}: could not fetch {name}: {err}", tv.style()); continue; @@ -1987,7 +1992,7 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let dest = dir.path().join("tool.tar.gz"); let url = format!("{}/tool.tar.gz", server.url()); - download_file(&url, &dest, None).await.unwrap(); + download_file(&url, &dest, None, None).await.unwrap(); assert_eq!(file::read_to_string(&dest).unwrap(), "payload"); mock.assert_async().await; }