From 8a25d5c83d6b55184635c016c5fbc06b939d2929 Mon Sep 17 00:00:00 2001 From: jdx <216188+jdx@users.noreply.github.com> Date: Sun, 13 Jul 2025 03:01:59 +0000 Subject: [PATCH] feat: blake3 support --- Cargo.lock | 34 +++++++++++++++++++++++++++------- Cargo.toml | 1 + e2e/backend/test_github | 2 +- e2e/backend/test_gitlab | 2 +- e2e/backend/test_http | 2 +- src/aqua/aqua_registry.rs | 1 + src/backend/github.rs | 6 ------ src/backend/http.rs | 6 ------ src/backend/mod.rs | 4 ++-- src/backend/ubi.rs | 4 ++-- src/hash.rs | 29 +++++++++++++++++++++++++++++ src/tera.rs | 10 +++++----- 12 files changed, 70 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7fe7981783..7ea5110803 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -228,6 +228,12 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + [[package]] name = "arrayvec" version = "0.7.6" @@ -394,6 +400,19 @@ version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" +[[package]] +name = "blake3" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3888aaa89e4b2a40fca9848e400f6a658a5a3978de7be858e209cafa8be9a4a0" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -720,7 +739,7 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fde0e0ec90c9dfb3b4b1a0891a7dcd0e2bffde2f7efed5fe7c9bb00e5bfb915e" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3365,7 +3384,7 @@ dependencies = [ "portable-atomic", "portable-atomic-util", "serde", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3755,6 +3774,7 @@ dependencies = [ "async-backtrace", "async-trait", "base64 0.22.1", + "blake3", "built", "bzip2 0.6.0", "calm_io", @@ -4616,7 +4636,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -4982,7 +5002,7 @@ dependencies = [ "errno 0.3.13", "libc", "linux-raw-sys 0.4.15", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -4995,7 +5015,7 @@ dependencies = [ "errno 0.3.13", "libc", "linux-raw-sys 0.9.4", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -5778,7 +5798,7 @@ dependencies = [ "getrandom 0.3.3", "once_cell", "rustix 1.0.7", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -6715,7 +6735,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index f2923acfd5..ab64f1f22d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -131,6 +131,7 @@ serde_json = "1" serde_yaml = "0.9" sha1 = "0.10" sha2 = "0.10" +blake3 = "1" shell-escape = "0.1" shell-words = "1" signal-hook = "0.3" diff --git a/e2e/backend/test_github b/e2e/backend/test_github index e4c70386c1..48605e14ec 100644 --- a/e2e/backend/test_github +++ b/e2e/backend/test_github @@ -50,4 +50,4 @@ assert_contains "cat mise.lock" '[tools."github:jdx/mise-test-fixtures"]' assert_contains "cat mise.lock" 'version = "1.0.0"' assert_contains "cat mise.lock" 'backend = "github:jdx/mise-test-fixtures"' assert_contains "cat mise.lock" '[tools."github:jdx/mise-test-fixtures".checksums]' -assert_contains "cat mise.lock" '"hello-world-1.0.0.tar.gz" = "sha256:dbca4f08377d70dc0828f5822fc47ecec3895cd9a6dacbc54cc984d346a571af"' +assert_contains "cat mise.lock" '"hello-world-1.0.0.tar.gz" = "blake3:71f774faa03daf1a58cc3339f8c73e6557348c8e0a2f3fb8148cc26e26bad83f"' diff --git a/e2e/backend/test_gitlab b/e2e/backend/test_gitlab index 82e61c8d25..1daf845f53 100644 --- a/e2e/backend/test_gitlab +++ b/e2e/backend/test_gitlab @@ -42,4 +42,4 @@ assert_contains "cat mise.lock" '[tools."gitlab:jdxcode/mise-test-fixtures"]' assert_contains "cat mise.lock" 'version = "1.0.0"' assert_contains "cat mise.lock" 'backend = "gitlab:jdxcode/mise-test-fixtures"' assert_contains "cat mise.lock" '[tools."gitlab:jdxcode/mise-test-fixtures".checksums]' -assert_contains "cat mise.lock" '"hello-world-1.0.0.tar.gz" = "sha256:dbca4f08377d70dc0828f5822fc47ecec3895cd9a6dacbc54cc984d346a571af"' +assert_contains "cat mise.lock" '"hello-world-1.0.0.tar.gz" = "blake3:71f774faa03daf1a58cc3339f8c73e6557348c8e0a2f3fb8148cc26e26bad83f"' diff --git a/e2e/backend/test_http b/e2e/backend/test_http index be4a91ce78..bdc0925230 100644 --- a/e2e/backend/test_http +++ b/e2e/backend/test_http @@ -103,4 +103,4 @@ assert_contains "cat mise.lock" '[tools."http:hello-lock"]' assert_contains "cat mise.lock" 'version = "1.0.0"' assert_contains "cat mise.lock" 'backend = "http:hello-lock"' assert_contains "cat mise.lock" '[tools."http:hello-lock".checksums]' -assert_contains "cat mise.lock" '"hello-world-1.0.0.tar.gz" = "sha256:dbca4f08377d70dc0828f5822fc47ecec3895cd9a6dacbc54cc984d346a571af"' +assert_contains "cat mise.lock" '"hello-world-1.0.0.tar.gz" = "blake3:71f774faa03daf1a58cc3339f8c73e6557348c8e0a2f3fb8148cc26e26bad83f"' diff --git a/src/aqua/aqua_registry.rs b/src/aqua/aqua_registry.rs index 2196db5ab2..95c4eeb43c 100644 --- a/src/aqua/aqua_registry.rs +++ b/src/aqua/aqua_registry.rs @@ -103,6 +103,7 @@ pub struct AquaFile { #[serde(rename_all = "lowercase")] #[strum(serialize_all = "lowercase")] pub enum AquaChecksumAlgorithm { + Blake3, Sha1, Sha256, Sha512, diff --git a/src/backend/github.rs b/src/backend/github.rs index 1596242f55..402c604d01 100644 --- a/src/backend/github.rs +++ b/src/backend/github.rs @@ -84,12 +84,6 @@ impl Backend for UnifiedGitBackend { HTTP.download_file(&asset_url, &file_path, Some(&ctx.pr)) .await?; - // Only add checksum if it doesn't already exist (for lockfile verification) - if let std::collections::btree_map::Entry::Vacant(e) = tv.checksums.entry(filename) { - let hash = hash::file_hash_sha256(&file_path, Some(&ctx.pr))?; - e.insert(format!("sha256:{hash}")); - } - // Verify self.verify_artifact(&tv, &file_path, &opts)?; diff --git a/src/backend/http.rs b/src/backend/http.rs index 8f73492048..e4a684d20a 100644 --- a/src/backend/http.rs +++ b/src/backend/http.rs @@ -55,12 +55,6 @@ impl Backend for HttpBackend { ctx.pr.set_message(format!("download {filename}")); HTTP.download_file(&url, &file_path, Some(&ctx.pr)).await?; - // Only add checksum if it doesn't already exist (for lockfile verification) - if let std::collections::btree_map::Entry::Vacant(e) = tv.checksums.entry(filename) { - let hash = hash::file_hash_sha256(&file_path, Some(&ctx.pr))?; - e.insert(format!("sha256:{hash}")); - } - // Verify self.verify_artifact(&tv, &file_path, &opts)?; diff --git a/src/backend/mod.rs b/src/backend/mod.rs index eb9d94685e..2457d2d8f7 100644 --- a/src/backend/mod.rs +++ b/src/backend/mod.rs @@ -738,8 +738,8 @@ pub trait Backend: Debug + Send + Sync { } } else if Settings::get().lockfile && Settings::get().experimental { ctx.pr.set_message(format!("generate checksum {filename}")); - let hash = hash::file_hash_sha256(file, Some(&ctx.pr))?; - tv.checksums.insert(filename, format!("sha256:{hash}")); + let hash = hash::file_hash_blake3(file, Some(&ctx.pr))?; + tv.checksums.insert(filename, format!("blake3:{hash}")); } Ok(()) } diff --git a/src/backend/ubi.rs b/src/backend/ubi.rs index 5e01888a60..37d3ba77e3 100644 --- a/src/backend/ubi.rs +++ b/src/backend/ubi.rs @@ -243,8 +243,8 @@ impl Backend for UbiBackend { } else if Settings::get().lockfile && Settings::get().experimental { ctx.pr .set_message(format!("checksum generate {checksum_key}")); - let hash = hash::file_hash_sha256(file, Some(&ctx.pr))?; - tv.checksums.insert(checksum_key, format!("sha256:{hash}")); + let hash = hash::file_hash_blake3(file, Some(&ctx.pr))?; + tv.checksums.insert(checksum_key, format!("blake3:{hash}")); } Ok(()) } diff --git a/src/hash.rs b/src/hash.rs index ada6cd9925..828ded09d8 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -6,6 +6,7 @@ use std::path::Path; use crate::file; use crate::file::display_path; use crate::ui::progress_report::SingleReport; +use blake3::Hasher as Blake3Hasher; use digest::Digest; use eyre::{Result, bail}; use md5::Md5; @@ -62,6 +63,33 @@ where Ok(format!("{hash:x}")) } +pub fn hash_blake3_to_str(s: &str) -> String { + let mut hasher = Blake3Hasher::new(); + hasher.update(s.as_bytes()); + hasher.finalize().to_hex().to_string() +} + +pub fn file_hash_blake3(path: &Path, pr: Option<&Box>) -> Result { + let mut file = file::open(path)?; + if let Some(pr) = pr { + pr.set_length(file.metadata()?.len()); + } + let mut hasher = Blake3Hasher::new(); + let mut buf = [0; 32 * 1024]; + loop { + let n = file.read(&mut buf)?; + if n == 0 { + break; + } + hasher.update(&buf[..n]); + if let Some(pr) = pr { + pr.inc(n as u64); + } + } + let hash = hasher.finalize(); + Ok(format!("{}", hash.to_hex())) +} + pub fn ensure_checksum( path: &Path, checksum: &str, @@ -70,6 +98,7 @@ pub fn ensure_checksum( ) -> Result<()> { let use_external_hasher = file::size(path).unwrap_or(u64::MAX) > 10 * 1024 * 1024; let actual = match algo { + "blake3" => file_hash_blake3(path, pr)?, "sha512" => { if use_external_hasher && file::which("sha512sum").is_some() { let out = cmd!("sha512sum", path).read()?; diff --git a/src/tera.rs b/src/tera.rs index 7050104eba..2c8e0a8751 100644 --- a/src/tera.rs +++ b/src/tera.rs @@ -95,7 +95,7 @@ static TERA: Lazy = Lazy::new(|| { move |input: &Value, args: &HashMap| match input { Value::String(s) => { let path = Path::new(s); - let mut hash = hash::file_hash_sha256(path, None).unwrap(); + let mut hash = hash::file_hash_blake3(path, None).unwrap(); if let Some(len) = args.get("len").and_then(Value::as_u64) { hash = hash.chars().take(len as usize).collect(); } @@ -108,7 +108,7 @@ static TERA: Lazy = Lazy::new(|| { "hash", move |input: &Value, args: &HashMap| match input { Value::String(s) => { - let mut hash = hash::hash_sha256_to_str(s); + let mut hash = hash::hash_blake3_to_str(s); if let Some(len) = args.get("len").and_then(Value::as_u64) { hash = hash.chars().take(len as usize).collect(); } @@ -343,7 +343,7 @@ pub fn tera_exec( cmd = cmd.dir(dir); } let result = if cache.is_some() || cache_duration.is_some() { - let cachehash = hash::hash_sha256_to_str( + let cachehash = hash::hash_blake3_to_str( &(dir .as_ref() .map(|d| d.to_string_lossy().to_string()) @@ -553,7 +553,7 @@ mod tests { async fn test_hash() { let _config = Config::get().await.unwrap(); let s = render("{{ \"foo\" | hash(len=8) }}"); - assert_eq!(s, "2c26b46b"); + assert_eq!(s, "04e0bb39"); } #[tokio::test] @@ -561,7 +561,7 @@ mod tests { async fn test_hash_file() { let _config = Config::get().await.unwrap(); let s = render("{{ \"../fixtures/shorthands.toml\" | hash_file(len=64) }}"); - insta::assert_snapshot!(s, @"518349c5734814ff9a21ab8d00ed2da6464b1699910246e763a4e6d5feb139fa"); + insta::assert_snapshot!(s, @"ce17f44735ea2083038e61c4b291ed31593e6cf4d93f5dc147e97e62962ac4e6"); } #[tokio::test]