diff --git a/Cargo.lock b/Cargo.lock index 9b75e5b7a0..48d00e175b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5009,6 +5009,7 @@ dependencies = [ "bzip2 0.6.1", "calm_io", "cfg_aliases", + "chacha20poly1305", "chrono", "ci_info", "clap", @@ -5036,6 +5037,7 @@ dependencies = [ "glob", "globset", "heck", + "hex", "homedir", "humansize", "ignore", diff --git a/Cargo.toml b/Cargo.toml index 7f2066f2f4..4a69f9e118 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -101,6 +101,7 @@ glob = "0.3" globset = "0.4" ignore = { version = "0.4", features = [] } heck = "0.5" +hex = "0.4" humansize = "2" indenter = "0.3" indexmap = { version = "2", features = ["serde"] } @@ -149,6 +150,7 @@ serde_yaml = "0.9" sha1 = "0.10" sha2 = "0.10" blake3 = "1" +chacha20poly1305 = "0.10" shell-escape = "0.1" shell-words = "1" signal-hook = "0.3" diff --git a/crates/vfox/src/hooks/mise_env.rs b/crates/vfox/src/hooks/mise_env.rs index b39fd81df1..120ca85e93 100644 --- a/crates/vfox/src/hooks/mise_env.rs +++ b/crates/vfox/src/hooks/mise_env.rs @@ -1,4 +1,6 @@ -use mlua::{IntoLua, Lua, LuaSerdeExt, Value}; +use mlua::prelude::LuaError; +use mlua::{FromLua, IntoLua, Lua, LuaSerdeExt, Value}; +use std::path::PathBuf; use crate::Plugin; use crate::error::Result; @@ -10,20 +12,34 @@ pub struct MiseEnvContext { pub options: T, } +/// Result from a mise_env hook call +/// Supports both legacy format (just array of env keys) and extended format +/// with cache metadata +#[derive(Debug, Default)] +pub struct MiseEnvResult { + /// Environment variables to set + pub env: Vec, + /// Whether this module's output can be cached + /// Defaults to false for backward compatibility + pub cacheable: bool, + /// Files to watch for cache invalidation + pub watch_files: Vec, +} + impl Plugin { pub async fn mise_env( &self, ctx: MiseEnvContext, - ) -> Result> { + ) -> Result { debug!("[vfox:{}] mise_env", &self.name); - let env_keys = self + let result = self .eval_async(chunk! { require "hooks/mise_env" return PLUGIN:MiseEnv($ctx) }) .await?; - Ok(env_keys) + Ok(result) } } @@ -34,3 +50,72 @@ impl IntoLua for MiseEnvContext { Ok(Value::Table(table)) } } + +impl FromLua for MiseEnvResult { + fn from_lua(value: Value, lua: &Lua) -> std::result::Result { + match value { + // Extended format: { cacheable = true, watch_files = {...}, env = {...} } + Value::Table(table) => { + // Check if this is extended format by looking for 'env' or 'cacheable' key + let has_env = table.contains_key("env")?; + let has_cacheable = table.contains_key("cacheable")?; + let has_watch_files = table.contains_key("watch_files")?; + + if has_env || has_cacheable || has_watch_files { + // Extended format + let env: Vec = table + .get::>>("env") + .map_err(|e| { + LuaError::RuntimeError(format!( + "Invalid 'env' field in MiseEnv result: expected array of {{key, value}} pairs. Error: {e}" + )) + })? + .unwrap_or_default(); + let cacheable: bool = table + .get::>("cacheable") + .map_err(|e| { + LuaError::RuntimeError(format!( + "Invalid 'cacheable' field in MiseEnv result: expected boolean. Error: {e}" + )) + })? + .unwrap_or(false); + let watch_files: Vec = table + .get::>>("watch_files") + .map_err(|e| { + LuaError::RuntimeError(format!( + "Invalid 'watch_files' field in MiseEnv result: expected array of strings. Error: {e}" + )) + })? + .unwrap_or_default(); + + Ok(MiseEnvResult { + env, + cacheable, + watch_files: watch_files.into_iter().map(PathBuf::from).collect(), + }) + } else { + // Legacy format: table is actually an array of env keys + // Try to parse as array + let env: Vec = Vec::from_lua(Value::Table(table), lua).map_err(|e| { + LuaError::RuntimeError(format!( + "Failed to parse MiseEnv hook result. Expected either:\n\ + - Legacy format: array of {{key, value}} pairs like {{{{\"KEY\", \"VALUE\"}}, ...}}\n\ + - Extended format: table with 'env' field like {{env = {{}}, cacheable = true}}\n\ + Error: {e}" + )) + })?; + Ok(MiseEnvResult { + env, + cacheable: false, + watch_files: vec![], + }) + } + } + // Empty/nil result + Value::Nil => Ok(MiseEnvResult::default()), + _ => Err(LuaError::RuntimeError( + "Expected table or nil from MiseEnv hook".to_string(), + )), + } + } +} diff --git a/crates/vfox/src/vfox.rs b/crates/vfox/src/vfox.rs index 4fc8310eca..ae9e3a117b 100644 --- a/crates/vfox/src/vfox.rs +++ b/crates/vfox/src/vfox.rs @@ -13,7 +13,7 @@ use crate::hooks::backend_exec_env::BackendExecEnvContext; use crate::hooks::backend_install::BackendInstallContext; use crate::hooks::backend_list_versions::BackendListVersionsContext; use crate::hooks::env_keys::{EnvKey, EnvKeysContext}; -use crate::hooks::mise_env::MiseEnvContext; +use crate::hooks::mise_env::{MiseEnvContext, MiseEnvResult}; use crate::hooks::mise_path::MisePathContext; use crate::hooks::parse_legacy_file::ParseLegacyFileResponse; use crate::hooks::post_install::PostInstallContext; @@ -229,10 +229,10 @@ impl Vfox { sdk.env_keys(ctx).await } - pub async fn mise_env(&self, sdk: &str, opts: T) -> Result> { + pub async fn mise_env(&self, sdk: &str, opts: T) -> Result { let plugin = self.get_sdk(sdk)?; if !plugin.get_metadata()?.hooks.contains("mise_env") { - return Ok(vec![]); + return Ok(MiseEnvResult::default()); } let ctx = MiseEnvContext { args: vec![], diff --git a/docs/cli/exec.md b/docs/cli/exec.md index c123f4b5cd..cbd11df7b6 100644 --- a/docs/cli/exec.md +++ b/docs/cli/exec.md @@ -36,6 +36,10 @@ Command string to execute Number of jobs to run in parallel [default: 4] +### `--fresh-env` + +Bypass the environment cache and recompute the environment + ### `--no-prepare` Skip automatic dependency preparation diff --git a/docs/cli/run.md b/docs/cli/run.md index 19b1cfca24..9434e0a94e 100644 --- a/docs/cli/run.md +++ b/docs/cli/run.md @@ -97,6 +97,10 @@ Don't show any output except for errors Tool(s) to run in addition to what is in mise.toml files e.g.: node@20 python@3.10 +### `--fresh-env` + +Bypass the environment cache and recompute the environment + ### `--no-cache` Do not use cache on remote tasks diff --git a/docs/cli/tasks/run.md b/docs/cli/tasks/run.md index 00a8877e3f..e1b979d0be 100644 --- a/docs/cli/tasks/run.md +++ b/docs/cli/tasks/run.md @@ -111,6 +111,10 @@ Don't show any output except for errors Tool(s) to run in addition to what is in mise.toml files e.g.: node@20 python@3.10 +### `--fresh-env` + +Bypass the environment cache and recompute the environment + ### `--no-cache` Do not use cache on remote tasks diff --git a/docs/env-plugin-development.md b/docs/env-plugin-development.md index 74d451a243..436ffa6b06 100644 --- a/docs/env-plugin-development.md +++ b/docs/env-plugin-development.md @@ -89,11 +89,43 @@ function PLUGIN:MiseEnv(ctx) end ``` -**Return value**: Array of tables, each with: +**Return value**: Either a simple array of env keys, or a table with caching metadata. + +Simple format - array of tables, each with: - `key` (string, required): Environment variable name - `value` (string, required): Environment variable value +Extended format - table with: + +- `env` (array, required): Array of `{key, value}` tables (same as simple format) +- `cacheable` (boolean, optional): If `true`, mise can cache this plugin's output. Default: `false` +- `watch_files` (array of strings, optional): File paths to watch for changes. If any file's mtime changes, the cache is invalidated. + +Example using extended format with caching: + +```lua +function PLUGIN:MiseEnv(ctx) + local config_path = ctx.options.config_file or "config.json" + local config = load_config(config_path) + + return { + cacheable = true, + watch_files = {config_path}, + env = { + {key = "API_URL", value = config.api_url}, + {key = "API_KEY", value = config.api_key} + } + } +end +``` + +When `cacheable = true`, mise will cache the environment variables and only re-execute the plugin when: + +- Any file in `watch_files` changes +- The mise configuration changes +- The cache TTL expires (configured via `env_cache_ttl` setting) + ### hooks/mise_path.lua The `MisePath` hook returns directories to add to PATH (optional): @@ -256,27 +288,32 @@ function PLUGIN:MiseEnv(ctx) end ``` -### 4. Cache Expensive Operations +### 4. Use Built-in Caching for Expensive Operations -For plugins that fetch data from external services, consider caching: +For plugins that fetch data from external services, use mise's built-in caching by returning the extended format with `cacheable = true`: ```lua -local cache_file = os.getenv("HOME") .. "/.cache/my-plugin/secrets.json" - function PLUGIN:MiseEnv(ctx) - -- Check if cache is fresh - if is_cache_valid(cache_file, 300) then -- 5 minute cache - return load_from_cache(cache_file) - end + local config_file = ctx.options.config_file or "secrets.json" - -- Fetch fresh data + -- Fetch secrets (mise will cache the result) local secrets = fetch_secrets(ctx.options) - save_to_cache(cache_file, secrets) - return secrets + return { + cacheable = true, + watch_files = {config_file}, -- Re-fetch if config changes + env = secrets + } end ``` +This is preferred over manual caching because: + +- mise handles cache invalidation automatically +- Cache is encrypted with session-scoped keys +- Integrates with `mise cache clear` and `mise cache prune` +- Respects the `env_cache_ttl` setting + ### 5. Support Multiple Environments ```lua diff --git a/e2e/env/test_env_cache b/e2e/env/test_env_cache new file mode 100644 index 0000000000..00d7cfd03f --- /dev/null +++ b/e2e/env/test_env_cache @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +# Test env caching feature + +# Enable env caching with encryption key (as would be set by `mise activate`) +export MISE_ENV_CACHE=1 +export __MISE_ENV_CACHE_KEY="dGVzdGtleXRlc3RrZXl0ZXN0a2V5dGVzdGtleXRlc3Q=" + +# Create a basic config with env vars +cat >"$MISE_CONFIG_DIR/config.toml" <"$MISE_CONFIG_DIR/config.toml" <"$MISE_CONFIG_DIR/config.toml" <"$MISE_CONFIG_DIR/config.toml" <"$MISE_CONFIG_DIR/config.toml" <\fR Tool(s) to run in addition to what is in mise.toml files e.g.: node@20 python@3.10 .TP +\fB\-\-fresh\-env\fR +Bypass the environment cache and recompute the environment +.TP \fB\-\-no\-cache\fR Do not use cache on remote tasks .TP @@ -2550,6 +2556,9 @@ Don't show any output except for errors \fB\-t, \-\-tool\fR \fI\fR Tool(s) to run in addition to what is in mise.toml files e.g.: node@20 python@3.10 .TP +\fB\-\-fresh\-env\fR +Bypass the environment cache and recompute the environment +.TP \fB\-\-no\-cache\fR Do not use cache on remote tasks .TP diff --git a/mise.usage.kdl b/mise.usage.kdl index 39d35f13e7..e14c81569c 100644 --- a/mise.usage.kdl +++ b/mise.usage.kdl @@ -281,6 +281,7 @@ cmd exec help="Execute a command with tool(s) set" { flag "-j --jobs" help="Number of jobs to run in parallel\n[default: 4]" { arg } + flag --fresh-env help="Bypass the environment cache and recompute the environment" flag --no-prepare help="Skip automatic dependency preparation" flag --raw help="Directly pipe stdin/stdout/stderr from plugin to user Sets --jobs=1" arg "[TOOL@VERSION]…" help="Tool(s) to start e.g.: node@20 python@3.10" required=#false var=#true @@ -712,6 +713,7 @@ cmd run restart_token=::: help="Run task(s)" { flag "-t --tool" help="Tool(s) to run in addition to what is in mise.toml files e.g.: node@20 python@3.10" var=#true { arg } + flag --fresh-env help="Bypass the environment cache and recompute the environment" flag --no-cache help="Do not use cache on remote tasks" flag --no-prepare help="Skip automatic dependency preparation" flag --no-timings help="Hides elapsed time after each task completes" { @@ -1020,6 +1022,7 @@ cmd tasks help="Manage tasks" { flag "-t --tool" help="Tool(s) to run in addition to what is in mise.toml files e.g.: node@20 python@3.10" var=#true { arg } + flag --fresh-env help="Bypass the environment cache and recompute the environment" flag --no-cache help="Do not use cache on remote tasks" flag --no-prepare help="Skip automatic dependency preparation" flag --no-timings help="Hides elapsed time after each task completes" { diff --git a/schema/mise.json b/schema/mise.json index 19107c74c1..3a41766548 100644 --- a/schema/mise.json +++ b/schema/mise.json @@ -604,6 +604,16 @@ "type": "string" } }, + "env_cache": { + "default": false, + "description": "[experimental] Enable environment caching for nested mise invocations", + "type": "boolean" + }, + "env_cache_ttl": { + "default": "1h", + "description": "TTL for cached environments", + "type": "string" + }, "env_file": { "description": "Path to a file containing environment variables to automatically load.", "type": "string" diff --git a/settings.toml b/settings.toml index e08b12ed4c..d8dea1677b 100644 --- a/settings.toml +++ b/settings.toml @@ -404,6 +404,55 @@ parse_env = "list_by_comma" rc = true type = "ListString" +[env_cache] +default = false +description = "[experimental] Enable environment caching for nested mise invocations" +docs = """ +When enabled, mise will cache the computed environment (env vars and PATH) to disk. +This dramatically speeds up nested mise invocations (e.g., `mise x -- mise env`). + +The cache is encrypted using a session-scoped key (`__MISE_ENV_CACHE_KEY`) that is +generated when you run `mise activate` or `mise exec`. This means: +- Cache files are encrypted and unreadable by other processes +- When your shell session ends, the key is lost +- Old cache files become unreadable and will be regenerated + +Cache invalidation happens when: +- Any config file changes (mise.toml, .tool-versions, etc.) +- Tool versions change +- Settings change +- mise version changes +- TTL expires (configurable via `env_cache_ttl`) +- Any watched files change (from modules or _.source directives) + +Modules (vfox plugins) can declare themselves cacheable by returning +`{cacheable = true, watch_files = [...], env = [...]}` from their mise_env hook. +Modules that don't declare cacheability are treated as dynamic and will be +re-executed on each cache hit. + +Directives can opt out of caching by setting `cacheable = false`: +```toml +[env] +TIMESTAMP = { value = "{{ now() }}", cacheable = false } +_.source = { file = "dynamic.sh", cacheable = false } +``` +""" +env = "MISE_ENV_CACHE" +type = "Bool" + +[env_cache_ttl] +default = "1h" +description = "TTL for cached environments" +docs = """ +How long cached environments remain valid before being regenerated. +Accepts duration strings like "1h", "30m", "1d". + +Even with a valid TTL, caches are still invalidated when config files, +tool versions, settings, or watched files change. +""" +env = "MISE_ENV_CACHE_TTL" +type = "Duration" + [env_file] description = "Path to a file containing environment variables to automatically load." env = "MISE_ENV_FILE" diff --git a/src/cache.rs b/src/cache.rs index e8584f603a..291a214f09 100644 --- a/src/cache.rs +++ b/src/cache.rs @@ -19,6 +19,7 @@ use crate::config::Settings; use crate::file::{display_path, modified_duration}; use crate::hash::hash_to_str; use crate::rand::random_string; +use crate::toolset::env_cache::CachedEnv; use crate::{dirs, file}; #[derive(Debug)] @@ -253,14 +254,22 @@ pub(crate) fn auto_prune() -> Result<()> { debug!( "pruning old cache files, this behavior can be modified with the MISE_CACHE_PRUNE_AGE setting" ); - prune( - *dirs::CACHE, - &PruneOptions { + let opts = PruneOptions { + dry_run: false, + verbose: false, + age, + }; + prune(*dirs::CACHE, &opts)?; + // Also prune env cache using env_cache_ttl + let env_cache_dir = CachedEnv::cache_dir(); + if env_cache_dir.exists() { + let env_opts = PruneOptions { dry_run: false, verbose: false, - age, - }, - )?; + age: settings.env_cache_ttl(), + }; + prune(&env_cache_dir, &env_opts)?; + } Ok(()) } diff --git a/src/cli/activate.rs b/src/cli/activate.rs index fdf1fb6f8c..8e806b56bd 100644 --- a/src/cli/activate.rs +++ b/src/cli/activate.rs @@ -1,9 +1,11 @@ use std::path::{Path, PathBuf}; +use crate::config::Settings; use crate::env::PATH_KEY; use crate::file::touch_dir; use crate::path_env::PathEnv; use crate::shell::{ActivateOptions, ActivatePrelude, Shell, ShellType, get_shell}; +use crate::toolset::env_cache::CachedEnv; use crate::{dirs, env}; use eyre::Result; use itertools::Itertools; @@ -114,6 +116,17 @@ impl Activate { if let Some(prepend_path) = self.prepend_path(exe_dir) { prelude.push(prepend_path); } + + // Generate encryption key for env cache if caching is enabled + // This key is session-scoped and lost when the shell closes + if Settings::get().env_cache { + let key = CachedEnv::ensure_encryption_key(); + prelude.push(ActivatePrelude::SetEnv( + "__MISE_ENV_CACHE_KEY".to_string(), + key, + )); + } + miseprint!( "{}", shell.activate(ActivateOptions { diff --git a/src/cli/cache/clear.rs b/src/cli/cache/clear.rs index e9de287301..21bd47bf8b 100644 --- a/src/cli/cache/clear.rs +++ b/src/cli/cache/clear.rs @@ -1,5 +1,6 @@ use crate::dirs::CACHE; use crate::file::{display_path, remove_all}; +use crate::toolset::env_cache::CachedEnv; use eyre::Result; use filetime::set_file_times; use walkdir::WalkDir; @@ -47,6 +48,10 @@ impl CacheClear { remove_all(p)?; } } + // Also clear env cache when clearing all caches + if self.plugin.is_none() { + CachedEnv::clear()?; + } match &self.plugin { Some(plugins) => info!("cache cleared for {}", plugins.join(", ")), None => info!("cache cleared"), diff --git a/src/cli/cache/prune.rs b/src/cli/cache/prune.rs index 96ba673683..bd291ec88e 100644 --- a/src/cli/cache/prune.rs +++ b/src/cli/cache/prune.rs @@ -2,6 +2,7 @@ use crate::cache; use crate::cache::{PruneOptions, PruneResults}; use crate::config::Settings; use crate::dirs::CACHE; +use crate::toolset::env_cache::CachedEnv; use eyre::Result; use number_prefix::NumberPrefix; use std::time::Duration; @@ -29,7 +30,6 @@ pub struct CachePrune { impl CachePrune { pub fn run(self) -> Result<()> { let settings = Settings::get(); - let cache_dirs = vec![CACHE.to_path_buf()]; let opts = PruneOptions { dry_run: self.dry_run, verbose: self.verbose > 0, @@ -38,11 +38,25 @@ impl CachePrune { .unwrap_or(Duration::from_secs(30 * 24 * 60 * 60)), }; let mut results = PruneResults { size: 0, count: 0 }; - for p in &cache_dirs { - let r = cache::prune(p, &opts)?; + + // Prune main cache + let r = cache::prune(&CACHE.to_path_buf(), &opts)?; + results.size += r.size; + results.count += r.count; + + // Prune env cache using env_cache_ttl + let env_cache_dir = CachedEnv::cache_dir(); + if env_cache_dir.exists() { + let env_opts = PruneOptions { + dry_run: self.dry_run, + verbose: self.verbose > 0, + age: settings.env_cache_ttl(), + }; + let r = cache::prune(&env_cache_dir, &env_opts)?; results.size += r.size; results.count += r.count; } + let count = results.count; let size = bytes_str(results.size); info!("cache pruned {count} files, {size}"); diff --git a/src/cli/en.rs b/src/cli/en.rs index 19e3d4a895..333316c423 100644 --- a/src/cli/en.rs +++ b/src/cli/en.rs @@ -35,6 +35,7 @@ impl En { c: None, command: Some(command), no_prepare: false, + fresh_env: false, } .run() .await diff --git a/src/cli/exec.rs b/src/cli/exec.rs index 0c6efdbf10..4360a16e10 100644 --- a/src/cli/exec.rs +++ b/src/cli/exec.rs @@ -14,6 +14,7 @@ use crate::cmd; use crate::config::{Config, Settings}; use crate::env; use crate::prepare::{PrepareEngine, PrepareOptions}; +use crate::toolset::env_cache::CachedEnv; use crate::toolset::{InstallOptions, ResolveOptions, ToolsetBuilder}; /// Execute a command with tool(s) set @@ -46,6 +47,10 @@ pub struct Exec { #[clap(long, short, env = "MISE_JOBS", verbatim_doc_comment)] pub jobs: Option, + /// Bypass the environment cache and recompute the environment + #[clap(long)] + pub fresh_env: bool, + /// Skip automatic dependency preparation #[clap(long)] pub no_prepare: bool, @@ -59,6 +64,11 @@ pub struct Exec { impl Exec { #[async_backtrace::framed] pub async fn run(self) -> eyre::Result<()> { + // Temporarily unset cache key to force fresh env computation + if self.fresh_env { + env::reset_env_cache_key(); + } + let mut config = Config::get().await?; // Check if any tool arg explicitly specified @latest @@ -134,6 +144,12 @@ impl Exec { env.insert("MISE_ENV".to_string(), env::MISE_ENV.join(",")); } + // Ensure cache key is propagated to subprocesses for env caching + if Settings::get().env_cache && !self.fresh_env { + let key = CachedEnv::ensure_encryption_key(); + env.insert("__MISE_ENV_CACHE_KEY".to_string(), key); + } + if program.rsplit('/').next() == Some("fish") { let mut cmd = vec![]; for (k, v) in env.iter().filter(|(k, _)| *k != "PATH") { diff --git a/src/cli/mod.rs b/src/cli/mod.rs index bd6abd1397..f9d8287b2e 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -669,6 +669,7 @@ impl Cli { timeout: None, skip_deps: false, no_prepare: false, + fresh_env: false, }))); } else if let Some(cmd) = external::COMMANDS.get(&task) { external::execute( diff --git a/src/cli/run.rs b/src/cli/run.rs index 247bdf26eb..acde9b34d0 100644 --- a/src/cli/run.rs +++ b/src/cli/run.rs @@ -8,6 +8,7 @@ use super::args::ToolArg; use crate::cli::{Cli, unescape_task_args}; use crate::config::{Config, Settings}; use crate::duration; +use crate::env; use crate::prepare::{PrepareEngine, PrepareOptions}; use crate::task::has_any_args_defined; use crate::task::task_helpers::task_needs_permit; @@ -156,6 +157,10 @@ pub struct Run { #[clap(skip)] pub is_linear: bool, + /// Bypass the environment cache and recompute the environment + #[clap(long)] + pub fresh_env: bool, + /// Do not use cache on remote tasks #[clap(long, verbatim_doc_comment, env = "MISE_TASK_REMOTE_NO_CACHE")] pub no_cache: bool, @@ -213,6 +218,11 @@ impl Run { // Unescape task args early so we can check for help flags self.args = unescape_task_args(&self.args); + // Temporarily unset cache key to force fresh env computation + if self.fresh_env { + env::reset_env_cache_key(); + } + // Check if --help or -h is in the task args BEFORE toolset/prepare // NOTE: Only check self.args, not self.args_last, because args_last contains // arguments after explicit -- which should always be passed through to the task diff --git a/src/config/env_directive/mod.rs b/src/config/env_directive/mod.rs index ccdd6f4907..30dade27de 100644 --- a/src/config/env_directive/mod.rs +++ b/src/config/env_directive/mod.rs @@ -239,6 +239,10 @@ pub struct EnvResults { pub env_scripts: Vec, pub redactions: Vec, pub tool_add_paths: Vec, + /// Files to watch for cache invalidation (from modules and _.source directives) + pub watch_files: Vec, + /// True if any directive declared cacheable=false or is a dynamic module + pub has_uncacheable: bool, } #[derive(Debug, Clone, Default)] @@ -277,6 +281,8 @@ impl EnvResults { env_scripts: Vec::new(), redactions: Vec::new(), tool_add_paths: Vec::new(), + watch_files: Vec::new(), + has_uncacheable: false, }; let normalize_path = |config_root: &Path, p: PathBuf| { let p = p.strip_prefix("./").unwrap_or(&p); diff --git a/src/config/env_directive/module.rs b/src/config/env_directive/module.rs index 48f29c4eff..354f8b5573 100644 --- a/src/config/env_directive/module.rs +++ b/src/config/env_directive/module.rs @@ -15,9 +15,31 @@ impl EnvResults { redact: bool, ) -> Result<()> { let path = dirs::PLUGINS.join(name.to_kebab_case()); - let plugin = VfoxPlugin::new(name, path); - if let Some(env) = plugin.mise_env(value).await? { - for (k, v) in env { + let plugin = VfoxPlugin::new(name, path.clone()); + if let Some(response) = plugin.mise_env(value).await? { + // Track cacheability + if !response.cacheable { + r.has_uncacheable = true; + } + + // Add plugin directory to watch files for cache invalidation + // This ensures cache invalidates when plugin is updated + r.watch_files.push(path); + + // Add watch files for cache invalidation + // Absolutize relative paths to ensure consistent cache validation + // regardless of which directory mise is run from + let cwd = std::env::current_dir().unwrap_or_default(); + for watch_file in response.watch_files { + if watch_file.is_absolute() { + r.watch_files.push(watch_file); + } else { + r.watch_files.push(cwd.join(watch_file)); + } + } + + // Add env vars + for (k, v) in response.env { if redact { r.redactions.push(k.clone()); } diff --git a/src/config/settings.rs b/src/config/settings.rs index 29d29bb1a9..f3a88e3595 100644 --- a/src/config/settings.rs +++ b/src/config/settings.rs @@ -427,6 +427,10 @@ impl Settings { duration::parse_duration(&self.http_timeout).unwrap() } + pub fn env_cache_ttl(&self) -> Duration { + duration::parse_duration(&self.env_cache_ttl).unwrap() + } + pub fn task_timeout_duration(&self) -> Option { self.task_timeout .as_ref() diff --git a/src/env.rs b/src/env.rs index 43ef0694a4..d4b1196860 100644 --- a/src/env.rs +++ b/src/env.rs @@ -690,6 +690,11 @@ pub fn remove_var>(key: K) { } } +/// Remove the env cache encryption key to force fresh env computation +pub fn reset_env_cache_key() { + remove_var("__MISE_ENV_CACHE_KEY"); +} + /// Safe wrapper around std::env::vars() that handles invalid UTF-8 gracefully. /// This function uses vars_os() and converts OsString to String, skipping any /// environment variables that contain invalid UTF-8 sequences. diff --git a/src/plugins/vfox_plugin.rs b/src/plugins/vfox_plugin.rs index e3b820cc86..bfccd624bc 100644 --- a/src/plugins/vfox_plugin.rs +++ b/src/plugins/vfox_plugin.rs @@ -16,6 +16,17 @@ use std::sync::{Arc, Mutex, MutexGuard, mpsc}; use url::Url; use vfox::Vfox; use vfox::embedded_plugins; + +/// Result from a mise_env call with cache metadata +#[derive(Debug, Default)] +pub struct MiseEnvResponse { + /// Environment variables to set + pub env: IndexMap, + /// Whether this module's output can be cached + pub cacheable: bool, + /// Files to watch for cache invalidation + pub watch_files: Vec, +} use xx::regex; #[derive(Debug)] @@ -61,14 +72,18 @@ impl VfoxPlugin { vfox_to_url(url) } - pub async fn mise_env(&self, opts: &toml::Value) -> Result>> { + pub async fn mise_env(&self, opts: &toml::Value) -> Result> { let (vfox, _) = self.vfox(); - let mut out = indexmap!(); - let results = vfox.mise_env(&self.name, opts).await?; - for env in results { - out.insert(env.key, env.value); + let result = vfox.mise_env(&self.name, opts).await?; + let mut env = indexmap!(); + for ek in result.env { + env.insert(ek.key, ek.value); } - Ok(Some(out)) + Ok(Some(MiseEnvResponse { + env, + cacheable: result.cacheable, + watch_files: result.watch_files, + })) } pub async fn mise_path(&self, opts: &toml::Value) -> Result>> { diff --git a/src/shims.rs b/src/shims.rs index 953e64f8d9..3792bb33f2 100644 --- a/src/shims.rs +++ b/src/shims.rs @@ -44,6 +44,7 @@ pub async fn handle_shim() -> Result<()> { jobs: None, raw: false, no_prepare: true, // Skip prepare for shims to avoid performance impact + fresh_env: false, }; time!("shim exec"); exec.run().await?; diff --git a/src/task/task_executor.rs b/src/task/task_executor.rs index c00217e92b..244c006955 100644 --- a/src/task/task_executor.rs +++ b/src/task/task_executor.rs @@ -8,6 +8,7 @@ use crate::task::task_output::{TaskOutput, trunc}; use crate::task::task_output_handler::OutputHandler; use crate::task::task_source_checker::{save_checksum, sources_are_fresh, task_cwd}; use crate::task::{Deps, FailedTasks, GetMatchingExt, Task}; +use crate::toolset::env_cache::CachedEnv; use crate::ui::{style, time}; use duct::IntoExecutablePath; use eyre::{Report, Result, ensure, eyre}; @@ -201,6 +202,14 @@ impl TaskExecutor { if let Some(config_root) = &task.config_root { env.insert("MISE_CONFIG_ROOT".into(), config_root.display().to_string()); } + + // Ensure cache key exists for task subprocesses for nested mise invocations + // This matches exec.rs behavior - enables caching for subprocesses + if Settings::get().env_cache { + let key = CachedEnv::ensure_encryption_key(); + env.insert("__MISE_ENV_CACHE_KEY".into(), key); + } + let timer = std::time::Instant::now(); if let Some(file) = task.file_path(config).await? { diff --git a/src/toolset/env_cache.rs b/src/toolset/env_cache.rs new file mode 100644 index 0000000000..a6be5b70f9 --- /dev/null +++ b/src/toolset/env_cache.rs @@ -0,0 +1,351 @@ +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::time::SystemTime; + +use base64::Engine; +use base64::prelude::BASE64_STANDARD; +use blake3::Hasher; +use chacha20poly1305::{ + ChaCha20Poly1305, KeyInit, Nonce, + aead::{Aead, AeadCore, OsRng}, +}; +use eyre::{Result, bail}; +use serde::{Deserialize, Serialize}; + +use crate::config::Settings; +use crate::dirs; +use crate::file; + +/// Represents the cached environment data +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CachedEnv { + /// Cached environment variables + pub env: BTreeMap, + /// Cached PATH entries + pub paths: Vec, + /// Time when the cache was created + pub created_at: u64, + /// Files to watch for changes (from modules and _.source directives) + pub watch_files: Vec, + /// mtimes of watch files at cache creation time + pub watch_file_mtimes: Vec, + /// mise version when cache was created + pub mise_version: String, + /// SHA256 of the original cache key inputs (for debugging) + pub cache_key_debug: String, +} + +impl CachedEnv { + /// Returns the directory where env cache files are stored + pub fn cache_dir() -> PathBuf { + dirs::STATE.join("env-cache") + } + + /// Computes the cache key based on config files, settings, tool versions, etc. + pub fn compute_cache_key( + config_files: &[(PathBuf, u64)], // (path, mtime) + tool_versions: &[(String, String)], // (tool, version) + settings_hash: &str, + base_path: &str, + ) -> String { + let mut hasher = Hasher::new(); + + // mise version + hasher.update(env!("CARGO_PKG_VERSION").as_bytes()); + + // config files and their mtimes + for (path, mtime) in config_files { + hasher.update(path.to_string_lossy().as_bytes()); + hasher.update(&mtime.to_le_bytes()); + } + + // tool versions + for (tool, version) in tool_versions { + hasher.update(tool.as_bytes()); + hasher.update(version.as_bytes()); + } + + // settings hash + hasher.update(settings_hash.as_bytes()); + + // base PATH + hasher.update(base_path.as_bytes()); + + let hash = hasher.finalize(); + hex::encode(hash.as_bytes()) + } + + /// Gets the encryption key from the environment variable + fn get_encryption_key() -> Option<[u8; 32]> { + std::env::var("__MISE_ENV_CACHE_KEY").ok().and_then(|s| { + let bytes = BASE64_STANDARD.decode(&s).ok()?; + bytes.try_into().ok() + }) + } + + /// Generates a new encryption key and returns it as a base64 string + pub fn generate_encryption_key() -> String { + let key = ChaCha20Poly1305::generate_key(&mut OsRng); + BASE64_STANDARD.encode(key) + } + + /// Ensures an encryption key exists, returns one if not set + pub fn ensure_encryption_key() -> String { + std::env::var("__MISE_ENV_CACHE_KEY").unwrap_or_else(|_| Self::generate_encryption_key()) + } + + /// Encrypts data using ChaCha20-Poly1305 + fn encrypt(data: &[u8], key: &[u8; 32]) -> Result> { + let cipher = ChaCha20Poly1305::new_from_slice(key) + .map_err(|e| eyre::eyre!("failed to create cipher: {}", e))?; + let nonce = ChaCha20Poly1305::generate_nonce(&mut OsRng); + let ciphertext = cipher + .encrypt(&nonce, data) + .map_err(|e| eyre::eyre!("encryption failed: {}", e))?; + + // Format: nonce || ciphertext + let mut result = nonce.to_vec(); + result.extend(ciphertext); + Ok(result) + } + + /// Decrypts data using ChaCha20-Poly1305 + fn decrypt(data: &[u8], key: &[u8; 32]) -> Result> { + if data.len() < 12 { + bail!("data too short to contain nonce"); + } + + let nonce = Nonce::from_slice(&data[..12]); + let ciphertext = &data[12..]; + + let cipher = ChaCha20Poly1305::new_from_slice(key) + .map_err(|e| eyre::eyre!("failed to create cipher: {}", e))?; + let plaintext = cipher + .decrypt(nonce, ciphertext) + .map_err(|e| eyre::eyre!("decryption failed: {}", e))?; + + Ok(plaintext) + } + + /// Loads a cached environment from disk + pub fn load(cache_key: &str) -> Result> { + let key = match Self::get_encryption_key() { + Some(k) => k, + None => { + trace!("env_cache: no encryption key set, skipping cache load"); + return Ok(None); + } + }; + + let cache_file = Self::cache_dir().join(cache_key); + if !cache_file.exists() { + trace!( + "env_cache: cache file does not exist: {}", + cache_file.display() + ); + return Ok(None); + } + + let encrypted_data = file::read(&cache_file)?; + let decrypted_data = match Self::decrypt(&encrypted_data, &key) { + Ok(data) => data, + Err(e) => { + debug!("env_cache: decryption failed (key changed?): {}", e); + // Remove stale cache file + let _ = file::remove_file(&cache_file); + return Ok(None); + } + }; + + let cached: CachedEnv = match rmp_serde::from_slice(&decrypted_data) { + Ok(c) => c, + Err(e) => { + debug!("env_cache: deserialization failed: {}", e); + let _ = file::remove_file(&cache_file); + return Ok(None); + } + }; + + // Validate mise version + if cached.mise_version != env!("CARGO_PKG_VERSION") { + debug!( + "env_cache: mise version mismatch (cached: {}, current: {})", + cached.mise_version, + env!("CARGO_PKG_VERSION") + ); + let _ = file::remove_file(&cache_file); + return Ok(None); + } + + // Check TTL + let now = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + let ttl = Settings::get().env_cache_ttl().as_secs(); + let age = now.saturating_sub(cached.created_at); + if age > ttl { + debug!("env_cache: cache expired (age: {}s, ttl: {}s)", age, ttl); + let _ = file::remove_file(&cache_file); + return Ok(None); + } + + // Check watch files mtimes + if let Err(e) = Self::validate_watch_files(&cached.watch_files, &cached.watch_file_mtimes) { + debug!("env_cache: watch file validation failed: {}", e); + let _ = file::remove_file(&cache_file); + return Ok(None); + } + + trace!("env_cache: loaded cache for key {}", cache_key); + Ok(Some(cached)) + } + + /// Saves a cached environment to disk + pub fn save(&self, cache_key: &str) -> Result<()> { + let key = match Self::get_encryption_key() { + Some(k) => k, + None => { + trace!("env_cache: no encryption key set, skipping cache save"); + return Ok(()); + } + }; + + let cache_dir = Self::cache_dir(); + file::create_dir_all(&cache_dir)?; + + let serialized = rmp_serde::to_vec(self)?; + let encrypted = Self::encrypt(&serialized, &key)?; + + let cache_file = cache_dir.join(cache_key); + file::write(&cache_file, &encrypted)?; + + trace!("env_cache: saved cache for key {}", cache_key); + Ok(()) + } + + /// Validates that watch files haven't changed since cache creation + fn validate_watch_files(watch_files: &[PathBuf], expected_mtimes: &[u64]) -> Result<()> { + if watch_files.len() != expected_mtimes.len() { + bail!("watch file count mismatch"); + } + for (path, expected_mtime) in watch_files.iter().zip(expected_mtimes.iter()) { + if !path.exists() { + bail!("watch file no longer exists: {}", path.display()); + } + if let Some(current_mtime) = get_file_mtime(path) { + if current_mtime != *expected_mtime { + bail!( + "watch file mtime changed: {} (expected: {}, current: {})", + path.display(), + expected_mtime, + current_mtime + ); + } + } else { + bail!("could not get mtime for watch file: {}", path.display()); + } + } + Ok(()) + } + + /// Returns true if env caching is enabled and we have an encryption key + pub fn is_enabled() -> bool { + Settings::get().env_cache && Self::get_encryption_key().is_some() + } + + /// Clears all env cache files + pub fn clear() -> Result<()> { + let cache_dir = Self::cache_dir(); + if cache_dir.exists() { + file::remove_all(&cache_dir)?; + } + Ok(()) + } +} + +/// Helper to get the mtime of a file as seconds since UNIX epoch +pub fn get_file_mtime(path: &Path) -> Option { + std::fs::metadata(path) + .ok() + .and_then(|m| m.modified().ok()) + .and_then(|t| t.duration_since(SystemTime::UNIX_EPOCH).ok()) + .map(|d| d.as_secs()) +} + +/// Computes a hash of the current settings that affect env computation +pub fn compute_settings_hash() -> String { + let settings = Settings::get(); + let mut hasher = Hasher::new(); + + // Add settings that affect env computation + hasher.update(settings.experimental.to_string().as_bytes()); + hasher.update(settings.all_compile.to_string().as_bytes()); + + // Add any other relevant settings + if let Some(env_file) = &settings.env_file { + hasher.update(env_file.to_string_lossy().as_bytes()); + } + + let hash = hasher.finalize(); + hex::encode(&hash.as_bytes()[..8]) // Short hash for debugging +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_cache_key_computation() { + let config_files = vec![(PathBuf::from("/home/user/project/mise.toml"), 1234567890u64)]; + let tool_versions = vec![("node".to_string(), "20.0.0".to_string())]; + let settings_hash = "abc123"; + let base_path = "/usr/bin:/bin"; + + let key1 = + CachedEnv::compute_cache_key(&config_files, &tool_versions, settings_hash, base_path); + + // Same inputs should produce same key + let key2 = + CachedEnv::compute_cache_key(&config_files, &tool_versions, settings_hash, base_path); + assert_eq!(key1, key2); + + // Different mtime should produce different key + let config_files_changed = + vec![(PathBuf::from("/home/user/project/mise.toml"), 1234567891u64)]; + let key3 = CachedEnv::compute_cache_key( + &config_files_changed, + &tool_versions, + settings_hash, + base_path, + ); + assert_ne!(key1, key3); + } + + #[test] + fn test_encryption_roundtrip() { + let key: [u8; 32] = rand::random(); + let data = b"hello world"; + + let encrypted = CachedEnv::encrypt(data, &key).unwrap(); + let decrypted = CachedEnv::decrypt(&encrypted, &key).unwrap(); + + assert_eq!(data.as_slice(), decrypted.as_slice()); + } + + #[test] + fn test_generate_encryption_key() { + let key1 = CachedEnv::generate_encryption_key(); + let key2 = CachedEnv::generate_encryption_key(); + + // Keys should be different + assert_ne!(key1, key2); + + // Keys should be valid base64 + assert!(BASE64_STANDARD.decode(&key1).is_ok()); + assert!(BASE64_STANDARD.decode(&key2).is_ok()); + + // Decoded keys should be 32 bytes + assert_eq!(BASE64_STANDARD.decode(&key1).unwrap().len(), 32); + } +} diff --git a/src/toolset/mod.rs b/src/toolset/mod.rs index 0476cbde4c..4081b8fa5e 100644 --- a/src/toolset/mod.rs +++ b/src/toolset/mod.rs @@ -30,6 +30,7 @@ use helpers::TVTuple; pub use install_options::InstallOptions; mod builder; +pub mod env_cache; mod helpers; mod install_options; pub(crate) mod install_state; diff --git a/src/toolset/toolset_env.rs b/src/toolset/toolset_env.rs index 936d04e4ea..5a50ef1664 100644 --- a/src/toolset/toolset_env.rs +++ b/src/toolset/toolset_env.rs @@ -1,5 +1,7 @@ +use std::collections::BTreeMap; use std::path::PathBuf; use std::sync::Arc; +use std::time::SystemTime; use eyre::Result; @@ -9,6 +11,7 @@ use crate::env::{PATH_KEY, WARN_ON_MISSING_REQUIRED_ENV}; use crate::env_diff::EnvMap; use crate::path_env::PathEnv; use crate::toolset::Toolset; +use crate::toolset::env_cache::{CachedEnv, compute_settings_hash, get_file_mtime}; use crate::toolset::tool_request::ToolRequest; use crate::{env, parallel, uv}; @@ -21,15 +24,129 @@ impl Toolset { /// the full mise environment including all tool paths pub async fn env_with_path(&self, config: &Arc) -> Result { + // Try to load from cache if enabled + if CachedEnv::is_enabled() + && let Some(cached) = self.try_load_env_cache(config)? + { + trace!("env_cache: using cached environment"); + return Ok(cached); + } + let (mut env, env_results) = self.final_env(config).await?; let mut path_env = PathEnv::from_iter(env::PATH.clone()); - for p in self.list_final_paths(config, env_results).await? { + let paths = self.list_final_paths(config, env_results.clone()).await?; + for p in &paths { path_env.add(p.clone()); } env.insert(PATH_KEY.to_string(), path_env.to_string()); + + // Save to cache if enabled and no uncacheable directives + if CachedEnv::is_enabled() + && !env_results.has_uncacheable + && let Err(e) = self.save_env_cache(config, &env, &paths, &env_results) + { + debug!("env_cache: failed to save: {}", e); + } + Ok(env) } + /// Try to load environment from cache + fn try_load_env_cache(&self, config: &Arc) -> Result> { + let cache_key = self.compute_env_cache_key(config)?; + match CachedEnv::load(&cache_key)? { + Some(cached) => { + let mut env = cached.env; + // Reconstruct PATH from cached paths + let mut path_env = PathEnv::from_iter(env::PATH.clone()); + for p in cached.paths { + path_env.add(p); + } + env.insert(PATH_KEY.to_string(), path_env.to_string()); + Ok(Some(env)) + } + None => Ok(None), + } + } + + /// Save environment to cache + fn save_env_cache( + &self, + config: &Arc, + env: &EnvMap, + paths: &[PathBuf], + env_results: &EnvResults, + ) -> Result<()> { + let cache_key = self.compute_env_cache_key(config)?; + let now = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + + // Collect all files to watch (config files + module watch_files + env_files) + let mut watch_files: Vec = config.config_files.keys().cloned().collect(); + watch_files.extend(env_results.watch_files.clone()); + watch_files.extend(env_results.env_files.clone()); + watch_files.extend(env_results.env_scripts.clone()); + + // Get mtimes for watch files + let watch_file_mtimes: Vec = watch_files + .iter() + .map(|p| get_file_mtime(p).unwrap_or(0)) + .collect(); + + // Remove PATH from env before caching (we store paths separately) + let env_without_path: BTreeMap = env + .iter() + .filter(|(k, _)| k.as_str() != PATH_KEY.as_str()) + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + + let cached = CachedEnv { + env: env_without_path, + paths: paths.to_vec(), + created_at: now, + watch_files, + watch_file_mtimes, + mise_version: env!("CARGO_PKG_VERSION").to_string(), + cache_key_debug: cache_key.clone(), + }; + + cached.save(&cache_key) + } + + /// Compute the cache key for the current configuration + fn compute_env_cache_key(&self, config: &Arc) -> Result { + // Collect config files with their mtimes + let config_files: Vec<(PathBuf, u64)> = config + .config_files + .keys() + .map(|p| (p.clone(), get_file_mtime(p).unwrap_or(0))) + .collect(); + + // Collect tool versions + let tool_versions: Vec<(String, String)> = self + .list_current_versions() + .into_iter() + .map(|(b, tv)| (b.id().to_string(), tv.version.clone())) + .collect(); + + // Get settings hash + let settings_hash = compute_settings_hash(); + + // Get base PATH using platform-appropriate separator + let base_path = std::env::join_paths(env::PATH.iter()) + .map(|p| p.to_string_lossy().to_string()) + .unwrap_or_default(); + + Ok(CachedEnv::compute_cache_key( + &config_files, + &tool_versions, + &settings_hash, + &base_path, + )) + } + pub async fn env_from_tools(&self, config: &Arc) -> Vec<(String, String, String)> { let this = Arc::new(self.clone()); let items: Vec<_> = self diff --git a/xtasks/fig/src/mise.ts b/xtasks/fig/src/mise.ts index 511d91d793..f84fbdcecb 100644 --- a/xtasks/fig/src/mise.ts +++ b/xtasks/fig/src/mise.ts @@ -992,6 +992,12 @@ const completionSpec: Fig.Spec = { name: "jobs", }, }, + { + name: "--fresh-env", + description: + "Bypass the environment cache and recompute the environment", + isRepeatable: false, + }, { name: "--no-prepare", description: "Skip automatic dependency preparation", @@ -2042,6 +2048,12 @@ const completionSpec: Fig.Spec = { debounce: true, }, }, + { + name: "--fresh-env", + description: + "Bypass the environment cache and recompute the environment", + isRepeatable: false, + }, { name: "--no-cache", description: "Do not use cache on remote tasks", @@ -2860,6 +2872,12 @@ const completionSpec: Fig.Spec = { debounce: true, }, }, + { + name: "--fresh-env", + description: + "Bypass the environment cache and recompute the environment", + isRepeatable: false, + }, { name: "--no-cache", description: "Do not use cache on remote tasks",