Skip to content
Merged
Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
0ac040e
docs(http): say the shared extraction store is trusted as is
jdx Oct 9, 2026
b49bac0
docs(lock): explain why mise can't validate restored caches
jdx Oct 9, 2026
a5b5a54
docs(lock): note that entries without a checksum are generated, not v…
jdx Oct 9, 2026
6eb9b9d
docs(http): say how to discard an untrusted shared extraction store
jdx Oct 9, 2026
2311d94
docs(lock): shorten the restored cache warning
jdx Oct 9, 2026
6332926
feat(http): reuse verified downloads from a checksum-keyed cache
jdx Oct 9, 2026
76e7964
fix(http): give each download cache store its own temp file
jdx Oct 9, 2026
81e623e
feat: reuse downloads the server confirms are unchanged
jdx Oct 9, 2026
e04dbcf
fix(http): copy into the download cache through a closed temp path
jdx Oct 9, 2026
00a77aa
fix(http): restore cached downloads atomically and state what the cac…
jdx Oct 9, 2026
8c5702b
fix(settings): sort download_cache alphabetically
jdx Oct 9, 2026
5d7ee3e
wip: merge the download caches into one content-addressed store
jdx Oct 9, 2026
bd636f6
feat: install from the download store without copying, and deprecate …
jdx Oct 9, 2026
f0b2485
fix(http): keep the download just stored out of eviction and correct …
jdx Oct 9, 2026
b4df2be
fix(http): replace a damaged stored download, place over an existing …
jdx Oct 9, 2026
b59c20b
test(plugins): turn the download cache off where a corrupt download i…
jdx Oct 9, 2026
86d1683
fix(http): name the path-length failure when placing a download, keep…
jdx Oct 9, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
103 changes: 100 additions & 3 deletions crates/mise-util/src/http.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@ use regex::Regex;
use reqwest::StatusCode;
use reqwest::header::{
ACCEPT_ENCODING, AUTHORIZATION, CONTENT_RANGE, CONTENT_TYPE, COOKIE, DATE, ETAG, HeaderMap,
HeaderName, HeaderValue, IF_RANGE, LAST_MODIFIED, PROXY_AUTHORIZATION, RANGE,
HeaderName, HeaderValue, IF_MODIFIED_SINCE, IF_NONE_MATCH, IF_RANGE, LAST_MODIFIED,
PROXY_AUTHORIZATION, RANGE,
};
use reqwest::{ClientBuilder, IntoUrl, Method, Response};
use serde::{Deserialize, Serialize};
Expand All @@ -28,6 +29,8 @@ use crate::time::format_duration;
use crate::{env, file};
use mise_settings::Settings;

mod revalidate;

pub static HTTP: Lazy<Client> = Lazy::new(|| {
Client::new_shared(
crate::network::http_timeout(&Settings::get()),
Expand Down Expand Up @@ -222,6 +225,17 @@ impl DownloadValidator {
}
}

/// What an attempt learned beyond the bytes it wrote.
#[derive(Default)]
struct AttemptOutcome {
/// The validator of the stored copy, sent so the server can confirm it.
cond: Option<DownloadValidator>,
/// The server confirmed the stored copy is current; nothing was written.
not_modified: std::sync::atomic::AtomicBool,
/// The validator that came with a complete response, for keeping it.
validator: Mutex<Option<DownloadValidator>>,
}

#[derive(Debug)]
struct DownloadSizeMismatch {
expected: u64,
Expand Down Expand Up @@ -1310,6 +1324,22 @@ impl Client {
headers: &HeaderMap,
pr: Option<&dyn SingleReport>,
total_timeout: Duration,
) -> Result<DownloadFileMetadata> {
let use_cache = Settings::get().download_cache;
self.download_file_inner(url, path, headers, pr, total_timeout, use_cache)
.await
}

/// With `use_cache`, a finished download of the same request is offered to
/// the server for revalidation and reused on `304 Not Modified`.
async fn download_file_inner<U: IntoUrl>(
&self,
url: U,
path: &Path,
headers: &HeaderMap,
pr: Option<&dyn SingleReport>,
total_timeout: Duration,
use_cache: bool,
) -> Result<DownloadFileMetadata> {
ensure!(
!crate::network::offline(&Settings::get()),
Expand All @@ -1334,6 +1364,16 @@ impl Client {
.await??;
let attempt = Arc::new(AtomicUsize::new(0));
let progress = Arc::new(DownloadProgress::default());
let request_hash = download_request_hash(&url, headers);
let cached = if use_cache {
revalidate::lookup(&request_hash)
} else {
None
};
let outcome = Arc::new(AttemptOutcome {
cond: cached.as_ref().map(|c| c.validator().clone()),
..Default::default()
});

// Retry the whole transfer, resuming a validated partial response when
// possible. send_once_with_https_fallback_allow_416 (not
Expand All @@ -1343,10 +1383,11 @@ impl Client {
let progress = progress.clone();
let request_url = url.clone();
let partial = partial.clone();
let outcome = outcome.clone();
async move {
attempt.fetch_add(1, Ordering::Relaxed);
progress.start_attempt();
self.download_file_attempt(request_url, headers, &partial, pr, &progress)
self.download_file_attempt(request_url, headers, &partial, pr, &progress, &outcome)
.await
}
});
Expand Down Expand Up @@ -1386,7 +1427,40 @@ impl Client {
// of `timeout` prevents us from returning an error while it can still
// install the destination in the background.
let path = path.to_path_buf();
tokio::task::spawn_blocking(move || partial.persist(&path)).await??;
if outcome.not_modified.load(Ordering::Relaxed)
&& let Some(cached) = cached
{
let effective_filename = cached.effective_filename();
let dest = path.clone();
let restored = tokio::task::spawn_blocking(move || cached.restore(&dest)).await??;
if restored {
return Ok(DownloadFileMetadata { effective_filename });
}
// The stored file no longer matches what was recorded: fetch it again.
drop(_download_lock);
return Box::pin(self.download_file_inner(
url,
&path,
headers,
pr,
total_timeout,
false,
))
.await;
}
let persist_path = path.clone();
tokio::task::spawn_blocking(move || partial.persist(&persist_path)).await??;
let validator = outcome.validator.lock().unwrap().take();
if use_cache && let Some(validator) = validator {
let effective_filename = metadata.effective_filename.clone();
let stored = tokio::task::spawn_blocking(move || {
revalidate::store(&request_hash, &path, &validator, effective_filename)
})
.await;
if let Ok(Err(err)) | Err(err) = stored.map_err(Report::from) {
debug!("could not keep the download for revalidation: {err:#}");
}
}
Ok(metadata)
}

Expand All @@ -1397,6 +1471,7 @@ impl Client {
partial: &PartialDownload,
pr: Option<&dyn SingleReport>,
progress: &DownloadProgress,
outcome: &AttemptOutcome,
) -> Result<DownloadFileMetadata> {
let mut restarted_without_resume = false;
loop {
Expand All @@ -1420,6 +1495,20 @@ impl Client {
let offset = resume.as_ref().map(|(_, size)| *size).unwrap_or(0);
let mut request_headers = headers.clone();
request_headers.insert(ACCEPT_ENCODING, HeaderValue::from_static("identity"));
let mut conditional = false;
if resume.is_none()
&& let Some(cond) = &outcome.cond
{
conditional = true;
match cond {
DownloadValidator::Etag(value) => {
request_headers.insert(IF_NONE_MATCH, HeaderValue::from_str(value)?);
}
DownloadValidator::LastModified { value, .. } => {
request_headers.insert(IF_MODIFIED_SINCE, HeaderValue::from_str(value)?);
}
}
}
if let Some((state, _)) = &resume {
request_headers.insert(RANGE, HeaderValue::from_str(&format!("bytes={offset}-"))?);
request_headers.insert(
Expand Down Expand Up @@ -1449,6 +1538,14 @@ impl Client {
let served_by = resp.url();
progress.served_by(served_by);

if conditional && resp.status() == StatusCode::NOT_MODIFIED {
outcome.not_modified.store(true, Ordering::Relaxed);
return Ok(DownloadFileMetadata::default());
}
*outcome.validator.lock().unwrap() = (resp.status() != StatusCode::PARTIAL_CONTENT)
.then(|| response_validator(resp.headers()))
.flatten();

if resp.status() == StatusCode::RANGE_NOT_SATISFIABLE {
if let Some(ParsedContentRange::Unsatisfied { total }) = resp
.headers()
Expand Down
157 changes: 157 additions & 0 deletions crates/mise-util/src/http/revalidate.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@
//! Reuse of a finished download when the server says it hasn't changed.
//!
//! Each entry holds the file, the validator (`ETag` or `Last-Modified`) the
//! server sent with it, and the file's BLAKE3 hash. The next request for the
//! same URL and headers carries that validator; a `304 Not Modified` answers
//! with no body, and the stored file is used after it hashes to the recorded
//! value again.

use std::path::{Path, PathBuf};
use std::time::SystemTime;

use eyre::Result;
use serde::{Deserialize, Serialize};

use super::DownloadValidator;
use crate::file;

/// Entries are evicted, oldest use first, past this total size.
const MAX_CACHE_BYTES: u64 = 2 * 1024 * 1024 * 1024;

const FILE: &str = "file";
const META: &str = "meta.json";

#[derive(Debug, Clone, Serialize, Deserialize)]
struct Meta {
validator: DownloadValidator,
blake3: String,
effective_filename: Option<String>,
}

pub(super) struct Cached {
dir: PathBuf,
meta: Meta,
}

impl Cached {
pub(super) fn validator(&self) -> &DownloadValidator {
&self.meta.validator
}

pub(super) fn effective_filename(&self) -> Option<String> {
self.meta.effective_filename.clone()
}

/// Copy the stored file to `dest` if it still hashes to what was recorded.
/// An entry that doesn't is removed.
pub(super) fn restore(&self, dest: &Path) -> Result<bool> {
let stored = self.dir.join(FILE);
let intact = stored.is_file()
&& crate::hash::file_hash_blake3(&stored, None)
.is_ok_and(|hash| hash == self.meta.blake3);
if !intact {
let _ = file::remove_all(&self.dir);
return Ok(false);
}
Comment thread
cursor[bot] marked this conversation as resolved.
Resolved
if let Some(parent) = dest.parent() {
file::create_dir_all(parent)?;
}
file::copy(&stored, dest)?;
// Marks the entry as recently used for eviction.
if let Ok(f) = std::fs::File::options()
.write(true)
.open(self.dir.join(META))
{
let _ = f.set_modified(SystemTime::now());
}
Ok(true)
Comment thread
cursor[bot] marked this conversation as resolved.
Resolved
}
}

/// Tests that don't ask for a directory never touch the user's cache.
#[cfg(test)]
pub(super) static TEST_ROOT: std::sync::Mutex<Option<PathBuf>> = std::sync::Mutex::new(None);

fn root() -> Option<PathBuf> {
#[cfg(test)]
{
TEST_ROOT.lock().unwrap().clone()
}
#[cfg(not(test))]
{
Some(crate::dirs::CACHE.join("downloads-cache"))
}
}

pub(super) fn lookup(request_hash: &str) -> Option<Cached> {
let dir = root()?.join(request_hash);
let meta: Meta = serde_json::from_slice(&std::fs::read(dir.join(META)).ok()?).ok()?;
meta.validator.is_valid().then_some(Cached { dir, meta })
}

/// Keep `path`, just downloaded with `validator`.
pub(super) fn store(
request_hash: &str,
path: &Path,
validator: &DownloadValidator,
effective_filename: Option<String>,
) -> Result<()> {
let Some(root) = root() else {
return Ok(());
};
let blake3 = crate::hash::file_hash_blake3(path, None)?;
let dir = root.join(request_hash);
let parent = dir.parent().expect("entry_dir joins a name");
file::create_dir_all(parent)?;
// Built beside the entry under a name nothing else shares, then swapped in.
let staging = tempfile::Builder::new()
.prefix(".staging-")
.tempdir_in(parent)?;
file::copy(path, staging.path().join(FILE))?;
let meta = Meta {
validator: validator.clone(),
blake3,
effective_filename,
};
file::write(staging.path().join(META), serde_json::to_vec(&meta)?)?;
let _ = file::remove_all(&dir);
let staged = staging.keep();
if let Err(err) = std::fs::rename(&staged, &dir) {
let _ = file::remove_all(&staged);
// Another process stored the same download first.
if !dir.join(META).is_file() {
return Err(err.into());
}
}
evict(parent);
Ok(())
}

fn evict(root: &Path) {
let Ok(read) = std::fs::read_dir(root) else {
return;
};
let mut entries = Vec::new();
let mut total = 0u64;
for entry in read.flatten() {
let path = entry.path();
if entry.file_name().to_string_lossy().starts_with('.') {
continue;
}
let size = std::fs::metadata(path.join(FILE)).map_or(0, |m| m.len());
let used = std::fs::metadata(path.join(META))
.and_then(|m| m.modified())
.unwrap_or(SystemTime::UNIX_EPOCH);
total += size;
entries.push((used, size, path));
}
entries.sort();
for (_, size, path) in entries {
if total <= MAX_CACHE_BYTES {
break;
}
if file::remove_all(&path).is_ok() {
total -= size;
}
}
}
62 changes: 62 additions & 0 deletions crates/mise-util/src/http/tests/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -559,3 +559,65 @@ refresh_expires_at = "2099-01-01T00:00:00Z"
fn json_empty_array_response() -> &'static str {
"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\n[]"
}

fn cacheable_download_response() -> &'static str {
concat!(
"HTTP/1.1 200 OK\r\n",
"Content-Length: 5\r\n",
"ETag: \"artifact-v1\"\r\n",
"Connection: close\r\n",
"\r\n",
"hello"
)
}
fn not_modified_response() -> &'static str {
"HTTP/1.1 304 Not Modified\r\nETag: \"artifact-v1\"\r\nConnection: close\r\n\r\n"
}

#[tokio::test(flavor = "current_thread")]
async fn test_download_reuses_stored_file_on_not_modified() {
let _settings = set_test_http_retries(0);
let cache = tempfile::tempdir().unwrap();
*revalidate::TEST_ROOT.lock().unwrap() = Some(cache.path().to_path_buf());
let (port, count, requests) = spawn_recording_server(vec![
cacheable_download_response(),
not_modified_response(),
not_modified_response(),
cacheable_download_response(),
])
.await;
let url = format!("http://127.0.0.1:{port}/artifact");
let client = Client::new(Duration::from_secs(2), ClientKind::Http).unwrap();
let dir = tempfile::tempdir().unwrap();
let dest = dir.path().join("artifact");

client.download_file(&url, &dest, None).await.unwrap();
assert_eq!(std::fs::read(&dest).unwrap(), b"hello");

// The second request is conditional, and a 304 serves the stored file.
std::fs::remove_file(&dest).unwrap();
client.download_file(&url, &dest, None).await.unwrap();
assert_eq!(std::fs::read(&dest).unwrap(), b"hello");
assert!(
requests.lock().unwrap()[1]
.to_lowercase()
.contains("if-none-match: \"artifact-v1\"")
);

// A stored file that no longer hashes to what was recorded is not used,
// even though the server says it is current.
let stored = std::fs::read_dir(cache.path())
.unwrap()
.next()
.unwrap()
.unwrap()
.path()
.join("file");
std::fs::write(&stored, b"tampered").unwrap();
std::fs::remove_file(&dest).unwrap();
client.download_file(&url, &dest, None).await.unwrap();
assert_eq!(std::fs::read(&dest).unwrap(), b"hello");
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 4);

*revalidate::TEST_ROOT.lock().unwrap() = None;
}
Loading
Loading