Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions settings.toml
Original file line number Diff line number Diff line change
Expand Up @@ -748,6 +748,11 @@ type = "Duration"
aliases = ["fetch_remote_version_timeout"]
default = "20s"
description = "Timeout in seconds for HTTP requests to fetch new tool versions in mise."
docs = """
Fast commands using `prefer_offline` cap this timeout at 3 seconds so shims and
shell activation do not block for a long time when cached data is insufficient
and the network is unavailable.
"""
env = "MISE_FETCH_REMOTE_VERSIONS_TIMEOUT"
type = "Duration"

Expand Down Expand Up @@ -1206,6 +1211,10 @@ treated as permanent and not retried.
Backoff schedule with jitter: ~200ms / ~1s / ~4s / ~15s. Set to 0 to disable
retries entirely.

Retries are automatically disabled for fast commands using `prefer_offline`,
including shims and shell activation. These paths make one network attempt when
cached data is insufficient, then fail or fall back immediately.

When a retry rescues a request, a warning is logged with the original error so
flaky infrastructure doesn't silently mask itself.
"""
Expand Down
18 changes: 17 additions & 1 deletion src/config/settings.rs
Original file line number Diff line number Diff line change
Expand Up @@ -898,7 +898,12 @@ impl Settings {
}

pub fn fetch_remote_versions_timeout(&self) -> Duration {
duration::parse_duration(&self.fetch_remote_versions_timeout).unwrap()
let timeout = duration::parse_duration(&self.fetch_remote_versions_timeout).unwrap();
if self.prefer_offline() {
timeout.min(Duration::from_secs(3))
} else {
timeout
}
}

/// duration that remote version cache is kept for
Expand All @@ -922,6 +927,17 @@ impl Settings {
duration::parse_duration(&self.http_download_timeout).unwrap()
}

/// Fast-path commands should make at most one network attempt before falling
/// back to cached/local behavior. In particular, shims must not multiply a
/// stalled resolver timeout by the configured retry count.
pub fn http_retries(&self) -> i64 {
if self.prefer_offline() {
0
} else {
self.http_retries
}
}

/// Returns true if offline mode is enabled via setting or CLI flag/env var.
pub fn offline(&self) -> bool {
self.offline || *env::OFFLINE
Expand Down
2 changes: 1 addition & 1 deletion src/github/sigstore.rs
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ fn mise_retry_config() -> RetryConfig {
let settings = crate::config::Settings::get();
RetryConfig {
timeout: settings.http_timeout(),
retries: settings.http_retries.max(0) as usize,
retries: settings.http_retries().max(0) as usize,
..RetryConfig::default()
}
}
Expand Down
229 changes: 224 additions & 5 deletions src/http.rs
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,31 @@ pub static HTTP_FETCH: Lazy<Client> = Lazy::new(|| {
type CachedResult = Arc<OnceCell<Result<String, String>>>;
static HTTP_CACHE: Lazy<Mutex<HashMap<String, CachedResult>>> =
Lazy::new(|| Mutex::new(HashMap::new()));
/// Origins that returned a hard connection failure during a prefer-offline
Comment thread
jdx marked this conversation as resolved.
Resolved
/// process. Keep the original error text so a short-circuited request remains
/// actionable rather than hiding the reason the circuit opened.
static UNAVAILABLE_HTTP_HOSTS: Lazy<Mutex<HashMap<String, String>>> =
Lazy::new(|| Mutex::new(HashMap::new()));
type RetryStateHandle = Arc<Mutex<RetryState>>;

#[derive(Debug)]
struct UnavailableHttpHost {
origin: String,
cause: String,
}

impl std::fmt::Display for UnavailableHttpHost {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"HTTP host {} is unavailable after an earlier connection failure: {}",
self.origin, self.cause
)
}
}

impl std::error::Error for UnavailableHttpHost {}

struct RetryState {
headers: HeaderMap,
use_netrc: bool,
Expand Down Expand Up @@ -207,7 +230,7 @@ impl Client {
client: self,
url: url.into_url().unwrap(),
extra_headers: HeaderMap::new(),
retries: Settings::get().http_retries,
retries: Settings::get().http_retries(),
}
}

Expand Down Expand Up @@ -475,7 +498,7 @@ impl Client {
url,
headers,
verb_label,
Settings::get().http_retries,
Settings::get().http_retries(),
true,
)
.await
Expand All @@ -493,7 +516,7 @@ impl Client {
url,
headers,
verb_label,
Settings::get().http_retries,
Settings::get().http_retries(),
false,
)
.await
Expand Down Expand Up @@ -578,7 +601,10 @@ impl Client {
.await
{
Ok(resp) => Ok(resp),
Err(err) if url.scheme() == "http" && is_connection_error(&err) => {
Err(err)
if url.scheme() == "http"
&& (is_connection_error(&err) || is_unavailable_http_host_error(&err)) =>
{
let mut url = url;
url.set_scheme("https").unwrap();
self.send_once_with_retry_headers(method, url, headers, verb_label, options)
Expand Down Expand Up @@ -610,6 +636,17 @@ impl Client {
) -> Result<Response> {
let original_url = url.clone();
apply_url_replacements(&mut url);
let host_key = http_host_key(&url);
if Settings::get().prefer_offline()
&& let Some(host) = &host_key
&& let Some(cause) = UNAVAILABLE_HTTP_HOSTS.lock().unwrap().get(host).cloned()
{
return Err(UnavailableHttpHost {
origin: host.clone(),
cause,
}
.into());
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
debug!("{} {}", verb_label, &url);

// Apply netrc credentials after URL replacement.
Expand All @@ -635,6 +672,15 @@ impl Client {
let resp = match req.send().await {
Ok(resp) => resp,
Err(err) => {
if Settings::get().prefer_offline()
&& is_hard_connection_failure(&err)
&& let Some(host) = host_key
{
UNAVAILABLE_HTTP_HOSTS
.lock()
.unwrap()
.insert(host, err.to_string());
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
if err.is_timeout() {
let (setting, env_var) = match self.kind {
ClientKind::Http => ("http_timeout", "MISE_HTTP_TIMEOUT"),
Expand Down Expand Up @@ -1095,9 +1141,41 @@ fn is_connection_error(err: &Report) -> bool {
})
}

fn http_host_key(url: &Url) -> Option<String> {
let host = url.host_str()?;
let port = url.port_or_known_default()?;
Some(format!("{}://{host}:{port}", url.scheme()))
}

fn is_unavailable_http_host_error(err: &Report) -> bool {
err.chain()
.any(|err| err.downcast_ref::<UnavailableHttpHost>().is_some())
}

/// hyper-util exposes DNS failures in the error chain as a `dns error` source,
/// but reqwest intentionally erases the concrete connector type. Match that
/// stable connector error label rather than platform-specific getaddrinfo text.
fn is_dns_error(err: &(dyn std::error::Error + 'static)) -> bool {
let mut current = Some(err);
while let Some(source) = current {
if source.to_string() == "dns error" {
return true;
}
current = source.source();
}
false
Comment thread
jdx marked this conversation as resolved.
Resolved
}

fn is_hard_connection_failure(err: &reqwest::Error) -> bool {
is_dns_error(err) || (err.is_connect() && !err.is_timeout())
}

/// Classifies an error as transient (should retry) vs permanent.
/// Walks the error chain so wrapped errors (e.g. our timeout hint) still match.
pub(crate) fn is_transient(err: &Report) -> bool {
if is_dns_error(err.as_ref()) {
return false;
}
err.chain().any(|e| {
let Some(reqwest_err) = e.downcast_ref::<reqwest::Error>() else {
return false;
Expand Down Expand Up @@ -1126,7 +1204,7 @@ where
F: FnMut() -> Fut,
Fut: std::future::Future<Output = Result<T>>,
{
retry_async_with_retries(verb_label, url, Settings::get().http_retries, f).await
retry_async_with_retries(verb_label, url, Settings::get().http_retries(), f).await
}

pub(crate) async fn retry_async_with_retries<F, Fut, T>(
Expand Down Expand Up @@ -1266,6 +1344,14 @@ mod tests {
crate::config::Settings::reset(Some(settings));
SettingsGuard { _lock: lock }
}
fn set_test_prefer_offline(http_retries: i64) -> SettingsGuard {
let lock = TEST_SETTINGS_LOCK.lock().unwrap();
let mut settings = crate::config::settings::SettingsPartial::empty();
settings.prefer_offline = Some(true);
settings.http_retries = Some(http_retries);
crate::config::Settings::reset(Some(settings));
SettingsGuard { _lock: lock }
}
fn set_test_offline() -> SettingsGuard {
let lock = TEST_SETTINGS_LOCK.lock().unwrap();
let mut settings = crate::config::settings::SettingsPartial::empty();
Expand All @@ -1274,6 +1360,30 @@ mod tests {
SettingsGuard { _lock: lock }
}

struct UnavailableHostsGuard {
host_keys: Vec<String>,
}
impl UnavailableHostsGuard {
fn new(host_keys: Vec<String>) -> Self {
let mut unavailable = UNAVAILABLE_HTTP_HOSTS.lock().unwrap();
for host_key in &host_keys {
unavailable.remove(host_key);
}
drop(unavailable);
Self { host_keys }
}
}
impl Drop for UnavailableHostsGuard {
fn drop(&mut self) {
let mut unavailable = UNAVAILABLE_HTTP_HOSTS
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
for host_key in &self.host_keys {
unavailable.remove(host_key);
}
}
}

struct GithubOauthSettingsGuard {
_settings_lock: std::sync::MutexGuard<'static, ()>,
_github_env_lock: std::sync::MutexGuard<'static, ()>,
Expand Down Expand Up @@ -1663,6 +1773,115 @@ refresh_expires_at = "2099-01-01T00:00:00Z"
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 3);
}

#[tokio::test(flavor = "current_thread")]
async fn test_prefer_offline_disables_http_retries() {
let _guard = set_test_prefer_offline(3);
let (port, count) = spawn_canned_server(vec![bad_gateway_response(), ok_response()]).await;
let url: Url = format!("http://127.0.0.1:{port}/").parse().unwrap();
let client = Client::new(Duration::from_secs(2), ClientKind::Http).unwrap();
let err = client.get_async(url).await.unwrap_err();

assert!(format!("{err:?}").contains("502"));
assert_eq!(count.load(std::sync::atomic::Ordering::SeqCst), 1);
assert_eq!(
Settings::get().fetch_remote_versions_timeout(),
Duration::from_secs(3)
);
}

#[tokio::test(flavor = "current_thread")]
async fn test_reqwest_dns_error_is_not_transient_and_opens_circuit() {
let _settings_guard = set_test_prefer_offline(3);
let timeout = Duration::from_secs(3);
let client = Client {
reqwest: Client::_new()
.no_proxy()
.read_timeout(timeout)
.connect_timeout(timeout)
.build()
.unwrap(),
timeout,
kind: ClientKind::Fetch,
};
let url: Url = "https://mise-dns-regression.invalid/".parse().unwrap();
let host_key = http_host_key(&url).unwrap();
let _hosts_guard = UnavailableHostsGuard::new(vec![host_key.clone()]);

let err = client.get_async(url).await.unwrap_err();

assert!(is_dns_error(err.as_ref()), "unexpected error: {err:#}");
assert!(!is_transient(&err));
assert!(
UNAVAILABLE_HTTP_HOSTS
.lock()
.unwrap()
.contains_key(&host_key)
);
}

#[tokio::test(flavor = "current_thread")]
async fn test_circuit_broken_http_origin_falls_back_to_https() {
let _settings_guard = set_test_prefer_offline(3);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let port = listener.local_addr().unwrap().port();
let http_url: Url = format!("http://127.0.0.1:{port}/").parse().unwrap();
let https_url: Url = format!("https://127.0.0.1:{port}/").parse().unwrap();
let http_origin = http_host_key(&http_url).unwrap();
let https_origin = http_host_key(&https_url).unwrap();
let _hosts_guard = UnavailableHostsGuard::new(vec![http_origin.clone(), https_origin]);
UNAVAILABLE_HTTP_HOSTS
.lock()
.unwrap()
.insert(http_origin, "connection refused".to_string());

let accepted = Arc::new(AtomicUsize::new(0));
let accepted_inner = accepted.clone();
let server = tokio::spawn(async move {
if let Ok((mut socket, _)) = listener.accept().await {
accepted_inner.fetch_add(1, Ordering::SeqCst);
let _ = socket.shutdown().await;
}
});

let client = Client::new(Duration::from_secs(2), ClientKind::Http).unwrap();
let err = client.get_async(http_url).await.unwrap_err();
server.await.unwrap();

assert_eq!(accepted.load(Ordering::SeqCst), 1);
assert!(!is_unavailable_http_host_error(&err));
}

#[test]
fn test_unavailable_host_error_preserves_original_cause() {
let err: Report = UnavailableHttpHost {
origin: "https://example.com:443".to_string(),
cause: "connection refused".to_string(),
}
.into();

assert!(is_unavailable_http_host_error(&err));
assert!(err.to_string().contains("connection refused"));
}

#[tokio::test(flavor = "current_thread")]
async fn test_circuit_breaker_is_disabled_without_prefer_offline() {
let _settings_guard = set_test_http_retries(0);
let (port, count) = spawn_canned_server(vec![ok_response()]).await;
let url: Url = format!("http://127.0.0.1:{port}/").parse().unwrap();
let host_key = http_host_key(&url).unwrap();
let _hosts_guard = UnavailableHostsGuard::new(vec![host_key.clone()]);
UNAVAILABLE_HTTP_HOSTS
.lock()
.unwrap()
.insert(host_key, "connection refused".to_string());

let client = Client::new(Duration::from_secs(2), ClientKind::Http).unwrap();
let resp = client.get_async(url).await.unwrap();

assert!(resp.status().is_success());
assert_eq!(count.load(Ordering::SeqCst), 1);
}
Comment thread
jdx marked this conversation as resolved.
Resolved

#[tokio::test(flavor = "current_thread")]
async fn test_download_total_timeout_bounds_trickling_response() {
let port = spawn_trickling_server().await;
Expand Down
Loading