Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion TASK_CACHE_PARITY.md
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ outside this tracker.
- [x] Report the reason for each cache miss
- [x] Show resolved cache inputs and outputs
- [x] Add machine-readable cache details to dry-run or task-info output
- [ ] Report hit rate, bytes restored, and time saved
- [x] Report hit rate, bytes restored, and time saved
- [ ] Add per-task cache inspection and deletion

## Integrity and portability
Expand Down
4 changes: 4 additions & 0 deletions docs/cli/run.md
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,10 @@ Explain the inputs that produced each task's output cache key

Output cache-key input details as JSON Lines without running tasks

### `--task-cache-stats`

Report task output cache hits, restored bytes, and time saved

### `--timeout <TIMEOUT>`

Timeout for the task to complete
Expand Down
4 changes: 4 additions & 0 deletions docs/cli/tasks/run.md
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,10 @@ Explain the inputs that produced each task's output cache key

Output cache-key input details as JSON Lines without running tasks

### `--task-cache-stats`

Report task output cache hits, restored bytes, and time saved

### `--timeout <TIMEOUT>`

Timeout for the task to complete
Expand Down
6 changes: 6 additions & 0 deletions docs/tasks/task-configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -716,6 +716,12 @@ values. This JSON Lines format remains streamable when a pattern selects multipl
command inputs still run so their presence can be reported accurately, but their output and hashes
are not included.

Use `mise run --task-cache-stats <task>` to print a run summary with the number and percentage of
artifact cache hits, the uncompressed output and log bytes restored, and the execution time recorded
when each restored entry was created. Entries written before this metadata was added remain readable
and contribute zero bytes and time when restored. Freshness skips that do not perform a cache lookup
are not counted as hits or misses.

`task_config.global_env` adds ambient variable names to every enabled task cache in the config
scope, including tasks with a task-local `cache` value. Unlike the default values under
`task_config.cache`, these names always compose with task-local `cache.env`.
Expand Down
25 changes: 22 additions & 3 deletions e2e/tasks/test_task_artifact_cache
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,11 @@ outputs = ["dist", "dist/result.txt"]
run = "true"
sources = ["input.txt"]
outputs = []

[tasks.fail]
run = "false"
sources = ["input.txt"]
outputs = []
EOF

printf 'one\n' >input.txt
Expand Down Expand Up @@ -64,21 +69,32 @@ assert_fail_contains \
assert "wc -l < runs.txt | tr -d ' '" "1"

# Existing fresh outputs use the normal fast freshness path.
assert_contains "PROFILE=debug mise run build 2>&1" "sources up-to-date, skipping"
cache_stats="$(PROFILE=debug mise run --task-cache-stats build 2>&1)"
assert_contains "echo \"$cache_stats\"" "sources up-to-date, skipping"
assert_contains "echo \"$cache_stats\"" "Task cache: no lookups"
assert "wc -l < runs.txt | tr -d ' '" "1"

# Missing outputs are restored from the local artifact cache without executing.
rm -rf dist
assert_contains "PROFILE=debug mise run build 2>&1" "restored outputs from cache"
cache_stats="$(PROFILE=debug mise run --task-cache-stats build 2>&1)"
assert_contains "echo \"$cache_stats\"" "restored outputs from cache"
assert_contains "echo \"$cache_stats\"" "Task cache: 1/1 hits (100%)"
assert_contains "echo \"$cache_stats\"" "10 B restored"
assert_not_contains "echo \"$cache_stats\"" "0ns saved"
assert "wc -l < runs.txt | tr -d ' '" "1"
assert "cat dist/result.txt" "debug:one"
assert_succeed "test -d dist/empty"

# Clearing the artifact cache invalidates the fast freshness skip even when
# outputs remain, so the task executes once to repopulate the cache.
mise cache clear
assert_not_contains "PROFILE=debug mise run build 2>&1" "sources up-to-date, skipping"
cache_stats="$(PROFILE=debug mise run --task-cache-stats build 2>&1)"
assert_not_contains "echo \"$cache_stats\"" "sources up-to-date, skipping"
assert_contains "echo \"$cache_stats\"" "Task cache: 0/1 hits (0%), 0 B restored, 0ns saved"
assert "wc -l < runs.txt | tr -d ' '" "2"
assert_fail_contains \
"mise run --task-cache-stats fail 2>&1" \
"Task cache: 0/1 hits (0%), 0 B restored, 0ns saved"
rm -rf dist
assert_contains "PROFILE=debug mise run build 2>&1" "restored outputs from cache"
assert "wc -l < runs.txt | tr -d ' '" "2"
Expand All @@ -102,6 +118,9 @@ assert "cat dist/result.txt" "release:two"

# Dry runs must not restore outputs or mutate artifact cache state.
rm -rf dist
assert_fail_contains \
"mise run --dry-run --task-cache-stats build 2>&1" \
"cannot be used with"
PROFILE=release mise run --dry-run build
assert_fail "test -e dist"
assert "wc -l < runs.txt | tr -d ' '" "4"
Expand Down
6 changes: 6 additions & 0 deletions man/man1/mise.1
Original file line number Diff line number Diff line change
Expand Up @@ -3160,6 +3160,9 @@ Explain the inputs that produced each task's output cache key
\fB\-\-task\-cache\-explain\-json\fR
Output cache\-key input details as JSON Lines without running tasks
.TP
\fB\-\-task\-cache\-stats\fR
Report task output cache hits, restored bytes, and time saved
.TP
\fB\-\-timeout\fR \fI<TIMEOUT>\fR
Timeout for the task to complete
e.g.: 30s, 5m
Expand Down Expand Up @@ -3978,6 +3981,9 @@ Explain the inputs that produced each task's output cache key
\fB\-\-task\-cache\-explain\-json\fR
Output cache\-key input details as JSON Lines without running tasks
.TP
\fB\-\-task\-cache\-stats\fR
Report task output cache hits, restored bytes, and time saved
.TP
\fB\-\-timeout\fR \fI<TIMEOUT>\fR
Timeout for the task to complete
e.g.: 30s, 5m
Expand Down
2 changes: 2 additions & 0 deletions mise.usage.kdl
Original file line number Diff line number Diff line change
Expand Up @@ -3162,6 +3162,7 @@ Set task output cache access for this run
}
flag --task-cache-explain help="Explain the inputs that produced each task's output cache key"
flag --task-cache-explain-json help="Output cache-key input details as JSON Lines without running tasks"
flag --task-cache-stats help="Report task output cache hits, restored bytes, and time saved"
flag --timeout help=#"""
Timeout for the task to complete
e.g.: 30s, 5m
Expand Down Expand Up @@ -4014,6 +4015,7 @@ Set task output cache access for this run
}
flag --task-cache-explain help="Explain the inputs that produced each task's output cache key"
flag --task-cache-explain-json help="Output cache-key input details as JSON Lines without running tasks"
flag --task-cache-stats help="Report task output cache hits, restored bytes, and time saved"
flag --timeout help=#"""
Timeout for the task to complete
e.g.: 30s, 5m
Expand Down
1 change: 1 addition & 0 deletions src/cli/bootstrap.rs
Original file line number Diff line number Diff line change
Expand Up @@ -926,6 +926,7 @@ impl Bootstrap {
task_cache: crate::task::TaskCacheMode::from_env()?,
task_cache_explain: false,
task_cache_explain_json: false,
task_cache_stats: false,
timeout: None,
skip_deps: false,
// a dry run must not auto-install tools before the (not actually
Expand Down
1 change: 1 addition & 0 deletions src/cli/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -802,6 +802,7 @@ impl Cli {
task_cache: crate::task::TaskCacheMode::from_env()?,
task_cache_explain: false,
task_cache_explain_json: false,
task_cache_stats: false,
timeout: None,
skip_deps: false,
skip_tools: false,
Expand Down
35 changes: 34 additions & 1 deletion src/cli/run.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ use crate::task::task_output_handler::OutputHandler;
use crate::task::{Deps, Task, TaskCacheMode};
use crate::toolset::{InstallOptions, ResolveOptions, ToolVersion, ToolsetBuilder};
use crate::ui::{ctrlc, info, style};
use bytesize::ByteSize;
use clap::{CommandFactory, ValueHint};
use eyre::{Result, bail, eyre};
use futures_util::FutureExt;
Expand Down Expand Up @@ -266,6 +267,10 @@ pub struct Run {
)]
pub task_cache_explain_json: bool,

/// Report task output cache hits, restored bytes, and time saved
#[clap(long, conflicts_with = "dry_run", verbatim_doc_comment)]
pub task_cache_stats: bool,

/// Timeout for the task to complete
/// e.g.: 30s, 5m
#[clap(long, verbatim_doc_comment)]
Expand Down Expand Up @@ -907,7 +912,11 @@ impl Run {
this.timings(),
this.is_interrupted(),
);
results_display.display_results(num_tasks, timer)?;
let result = results_display.display_results(num_tasks, timer);
if this.task_cache_stats {
this.display_task_cache_stats();
}
Comment thread
cursor[bot] marked this conversation as resolved.
Resolved
result?;
time!("parallelize_tasks done");

Ok(())
Expand Down Expand Up @@ -1316,6 +1325,30 @@ impl Run {
!self.quiet(None) && !self.no_timings
}

fn display_task_cache_stats(&self) {
let stats = *self
.executor
.as_ref()
.expect("executor must be initialized before displaying cache stats")
.cache_stats
.lock()
.unwrap();
let lookups = stats.hits.saturating_add(stats.misses);
if lookups == 0 {
safe_eprintln!("Task cache: no lookups");
return;
}
let hit_rate = stats.hits.saturating_mul(100) / lookups;
safe_eprintln!(
"Task cache: {}/{} hits ({}%), {} restored, {} saved",
stats.hits,
lookups,
hit_rate,
ByteSize::b(stats.restored_bytes).display().iec(),
crate::ui::time::format_duration(stats.time_saved),
);
}

fn quiet(&self, task: Option<&Task>) -> bool {
self.output_handler.as_ref().unwrap().quiet(task)
}
Expand Down
76 changes: 60 additions & 16 deletions src/task/task_cache.rs
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,10 @@ struct CacheManifest {
key: String,
roots: Vec<PathBuf>,
output: Vec<TaskCacheOutput>,
#[serde(default)]
restored_bytes: u64,
#[serde(default)]
execution_duration_ns: u64,
}

#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
Expand All @@ -122,10 +126,16 @@ pub(crate) enum TaskCacheOutput {
}

pub(crate) enum TaskCacheRestore {
Hit(Vec<TaskCacheOutput>),
Hit(TaskCacheHit),
Miss(TaskCacheMissReason),
}

pub(crate) struct TaskCacheHit {
pub(crate) output: Vec<TaskCacheOutput>,
pub(crate) restored_bytes: u64,
pub(crate) saved_duration: std::time::Duration,
}

pub(crate) enum TaskCacheMissReason {
CorruptEntry,
DependencyWithoutKey,
Expand Down Expand Up @@ -365,7 +375,7 @@ impl TaskArtifactCache {
if !manifest_path.is_file() {
return Ok(TaskCacheRestore::Miss(TaskCacheMissReason::EntryNotFound));
}
let restore = || -> Result<Vec<TaskCacheOutput>> {
let restore = || -> Result<TaskCacheHit> {
let manifest = self.read_manifest()?;
for root in &manifest.roots {
ensure_safe_relative(root)?;
Expand All @@ -377,7 +387,11 @@ impl TaskArtifactCache {
if let Err(err) = file::touch_file(&manifest_path) {
warn!("failed to update task cache manifest access time: {err}");
}
return Ok(manifest.output);
return Ok(TaskCacheHit {
output: manifest.output,
restored_bytes: manifest.restored_bytes,
saved_duration: std::time::Duration::from_nanos(manifest.execution_duration_ns),
});
}
// Serialize restores targeting the same working directory across
// mise processes so validation and renames form one cooperative
Expand Down Expand Up @@ -431,7 +445,11 @@ impl TaskArtifactCache {
if let Err(err) = file::touch_file(&manifest_path) {
warn!("failed to update task cache manifest access time: {err}");
}
Ok(manifest.output)
Ok(TaskCacheHit {
output: manifest.output,
restored_bytes: manifest.restored_bytes,
saved_duration: std::time::Duration::from_nanos(manifest.execution_duration_ns),
})
};

match restore() {
Expand All @@ -446,7 +464,12 @@ impl TaskArtifactCache {
}

/// Stores a successful task's declared outputs and captured logs.
pub(crate) fn store(&self, task: &Task, output: &[TaskCacheOutput]) -> Result<()> {
pub(crate) fn store(
&self,
task: &Task,
output: &[TaskCacheOutput],
execution_duration: std::time::Duration,
) -> Result<()> {
let roots = resolve_output_roots(task, &self.root, true)?;
let roots = remove_nested_roots(roots);
for root in &roots {
Expand All @@ -463,21 +486,26 @@ impl TaskArtifactCache {
.cache_dir
.join(format!("{}.part-{nonce}.json", self.key));

let output_bytes = output.iter().fold(0_u64, |total, line| {
let bytes = match line {
TaskCacheOutput::Stdout(line) | TaskCacheOutput::Stderr(line) => line.len() as u64,
};
total.saturating_add(bytes)
});
let archive_bytes = if !roots.is_empty() {
let output_matcher = build_output_matcher(&self.root, &task.outputs.patterns())?;
write_archive(&archive_partial, &self.root, &roots, &output_matcher)?
} else {
0
};
let manifest = CacheManifest {
format: CACHE_FORMAT_VERSION,
key: self.key.clone(),
roots,
output: output.to_vec(),
restored_bytes: archive_bytes.saturating_add(output_bytes),
execution_duration_ns: execution_duration.as_nanos().min(u64::MAX as u128) as u64,
};
if !manifest.roots.is_empty() {
let output_matcher = build_output_matcher(&self.root, &task.outputs.patterns())?;
write_archive(
&archive_partial,
&self.root,
&manifest.roots,
&output_matcher,
)?;
}
fs::write(&manifest_partial, serde_json::to_vec(&manifest)?)?;
if !manifest.roots.is_empty() {
file::rename(&archive_partial, &archive_path)?;
Expand Down Expand Up @@ -906,7 +934,7 @@ fn write_archive(
root: &Path,
roots: &[PathBuf],
output_matcher: &Override,
) -> Result<()> {
) -> Result<u64> {
let file = File::create(path)?;
let encoder = zstd::Encoder::new(file, 0)?;
let mut archive = Builder::new(encoder);
Expand All @@ -924,6 +952,7 @@ fn write_archive(
}
}

let mut restored_bytes = 0_u64;
for (rel, abs) in entries {
let metadata = fs::symlink_metadata(&abs)?;
let mut header = Header::new_gnu(if metadata.file_type().is_symlink() {
Expand Down Expand Up @@ -951,13 +980,14 @@ fn write_archive(
} else if metadata.is_file() {
header.set_size(metadata.len());
archive.append_data(&mut header, &rel, File::open(&abs)?)?;
restored_bytes = restored_bytes.saturating_add(metadata.len());
} else {
bail!("unsupported output file type: {}", rel.display());
}
}
let encoder = archive.into_inner()?;
encoder.finish()?;
Ok(())
Ok(restored_bytes)
}

#[cfg(unix)]
Expand Down Expand Up @@ -991,6 +1021,20 @@ mod tests {
assert!(toml::from_str::<TaskCacheConfig>("remote = true").is_err());
}

#[test]
fn older_manifests_default_stats_metadata() {
let manifest: CacheManifest = serde_json::from_value(serde_json::json!({
"format": 2,
"key": "cache-key",
"roots": [],
"output": [],
}))
.unwrap();

assert_eq!(manifest.restored_bytes, 0);
assert_eq!(manifest.execution_duration_ns, 0);
}

#[test]
fn cache_explanation_omits_environment_and_variable_values() {
let explanation = TaskCacheKeyExplanation {
Expand Down
Loading
Loading