diff --git a/.github/workflows/registry.yml b/.github/workflows/registry.yml index 21a8c862c5..0958200827 100644 --- a/.github/workflows/registry.yml +++ b/.github/workflows/registry.yml @@ -42,17 +42,18 @@ jobs: list-changed-tools: timeout-minutes: 10 runs-on: ubuntu-latest - if: ${{ github.event_name == 'pull_request' }} outputs: - tools: ${{ steps.diff.outputs.tools }} + tools: ${{ steps.diff.outputs.tools || steps.set-empty.outputs.tools }} steps: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 with: fetch-depth: 0 - uses: jdx/mise-action@c37c93293d6b742fc901e1406b8f764f6fb19dac # v2 + if: ${{ github.event_name == 'pull_request' }} with: install_args: jd yj - id: diff + if: ${{ github.event_name == 'pull_request' }} shell: bash # -eo pipefail run: | changed_tools=$( \ @@ -64,6 +65,9 @@ jobs: ) echo "$changed_tools" echo "tools=$changed_tools" >> $GITHUB_OUTPUT + - id: set-empty + if: ${{ github.event_name != 'pull_request' }} + run: echo "tools=" >> $GITHUB_OUTPUT test-tool: name: test-tool-${{ matrix.tranche }} diff --git a/Cargo.lock b/Cargo.lock index a0c9b4e7db..5b22540aa6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -207,6 +207,25 @@ version = "1.0.99" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100" +[[package]] +name = "aqua-registry" +version = "2025.9.10" +dependencies = [ + "expr-lang", + "eyre", + "heck", + "indexmap 2.11.0", + "itertools 0.14.0", + "log", + "serde", + "serde_derive", + "serde_yaml", + "strum", + "thiserror 2.0.16", + "tokio", + "versions 6.3.2", +] + [[package]] name = "arbitrary" version = "1.4.2" @@ -3858,6 +3877,7 @@ dependencies = [ name = "mise" version = "2025.9.10" dependencies = [ + "aqua-registry", "async-backtrace", "async-trait", "base64 0.22.1", @@ -3884,7 +3904,6 @@ dependencies = [ "duct 0.13.7", "either", "exec", - "expr-lang", "eyre", "filetime", "flate2", diff --git a/Cargo.toml b/Cargo.toml index 8b382dd7ab..4c00415208 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["crates/vfox"] +members = ["crates/vfox", "crates/aqua-registry"] [package] name = "mise" @@ -79,7 +79,6 @@ duct = "0.13" either = { version = "1", features = ["serde"] } homedir = "0.3" # expr-lang = { path = "../expr-lang" } -expr-lang = "0.3" eyre = "0.6" filetime = "0.2" flate2 = "1" @@ -157,6 +156,7 @@ urlencoding = "2.1.3" usage-lib = { version = "2", features = ["clap", "docs"] } versions = { version = "6", features = ["serde"] } vfox = { path = "crates/vfox", default-features = false } +aqua-registry = { path = "crates/aqua-registry" } walkdir = "2" which = "7" xx = { version = "2", features = ["glob"] } diff --git a/build.rs b/build.rs index 4183fe1d7e..62e3b9c98f 100644 --- a/build.rs +++ b/build.rs @@ -1,6 +1,6 @@ use heck::ToUpperCamelCase; use indexmap::IndexMap; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::{env, fs}; fn main() { @@ -14,7 +14,6 @@ fn main() { codegen_settings(); codegen_registry(); - codegen_aqua(); } fn codegen_registry() { @@ -329,68 +328,3 @@ pub static SETTINGS_META: Lazy> = Lazy::new fs::write(&dest_path, lines.join("\n")).unwrap(); } - -// pub static AQUA_STANDARD_REGISTRY_FILES: Lazy> = Lazy::new(|| { -// include!(concat!(env!("OUT_DIR"), "/aqua_standard_registry.rs")); -// }); - -fn codegen_aqua() { - let out_dir = env::var_os("OUT_DIR").unwrap(); - let dest_path = Path::new(&out_dir).join("aqua_standard_registry.rs"); - let mut lines = vec!["[".to_string()]; - for (k, v) in aqua_registries(®istry_dir()).unwrap_or_default() { - lines.push(format!(r####" ("{k}", r###"{v}"###),"####)); - } - lines.push("].into()".to_string()); - fs::write(&dest_path, lines.join("\n")).unwrap(); -} - -fn ls(path: &Path) -> Result, std::io::Error> { - fs::read_dir(path)? - .map(|entry| entry.map(|e| e.path())) - .collect() -} - -fn aqua_registries(d: &Path) -> Result, Box> { - let mut registries = vec![]; - for f in ls(d)? { - if f.is_dir() { - registries.extend(aqua_registries(&f)?); - } else if f.file_name() == Some("registry.yaml".as_ref()) { - let registry_yaml = fs::read_to_string(&f)?; - registries.push(( - f.parent() - .unwrap() - .strip_prefix(registry_dir()) - .unwrap() - .iter() - .map(|s| s.to_string_lossy().into_owned()) - .collect::>() - .join("/"), - registry_yaml.clone(), - )); - if registry_yaml.contains("aliases") { - let registry: serde_yaml::Value = serde_yaml::from_str(®istry_yaml)?; - if let Some(packages) = registry.get("packages").and_then(|p| p.as_sequence()) { - for package in packages { - if let Some(aliases) = package.get("aliases").and_then(|a| a.as_sequence()) - { - for alias in aliases { - if let Some(name) = alias.get("name").and_then(|n| n.as_str()) { - registries.push((name.to_string(), registry_yaml.clone())); - } - } - } - } - } - } - } - } - Ok(registries) -} - -fn registry_dir() -> PathBuf { - PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()) - .join("aqua-registry") - .join("pkgs") -} diff --git a/crates/aqua-registry/Cargo.toml b/crates/aqua-registry/Cargo.toml new file mode 100644 index 0000000000..0e87811b76 --- /dev/null +++ b/crates/aqua-registry/Cargo.toml @@ -0,0 +1,49 @@ +[package] +name = "aqua-registry" +version = "2025.9.10" +edition = "2024" +description = "Aqua registry backend for mise" +authors = ["Jeff Dickey (@jdx)"] +license = "MIT" +repository = "https://github.com/jdx/mise" +homepage = "https://mise.jdx.dev" +readme = "README.md" +keywords = ["mise", "aqua", "registry", "package-manager"] +categories = ["development-tools"] +build = "build.rs" +publish = false + +[package.metadata.cargo-machete] +ignored = ["serde"] + +[features] +default = [] + +[dependencies] +# Core dependencies +serde = { version = "1", features = ["derive"] } +serde_derive = "1" +serde_yaml = "0.9" +thiserror = "2" +eyre = "0.6" +indexmap = { version = "2", features = ["serde"] } +itertools = "0.14" +strum = { version = "0.27", features = ["derive"] } + +# Template parsing and evaluation +expr-lang = "0.3" +versions = { version = "6", features = ["serde"] } +heck = "0.5" + +# Async runtime +tokio = { version = "1", features = ["sync"] } + +# Logging +log = "0.4" + + +[dev-dependencies] +tokio = { version = "1", features = ["rt", "macros"] } + +[build-dependencies] +serde_yaml = "0.9" diff --git a/crates/aqua-registry/README.md b/crates/aqua-registry/README.md new file mode 100644 index 0000000000..ce2e9b88e2 --- /dev/null +++ b/crates/aqua-registry/README.md @@ -0,0 +1,21 @@ +# aqua-registry + +Aqua registry backend for [mise](https://mise.jdx.dev). + +This crate provides support for the [Aqua](https://aquaproj.github.io/) registry format, allowing mise to install tools from the Aqua ecosystem. + +## Features + +- Parse and validate Aqua registry YAML files +- Resolve package versions and platform-specific assets +- Template string evaluation for dynamic asset URLs +- Support for checksums, signatures, and provenance verification +- Platform-aware asset resolution for cross-platform tool installation + +## Usage + +This crate is primarily used internally by mise. For more information about mise, visit [mise.jdx.dev](https://mise.jdx.dev). + +## License + +MIT diff --git a/crates/aqua-registry/build.rs b/crates/aqua-registry/build.rs new file mode 100644 index 0000000000..5e21fc2e73 --- /dev/null +++ b/crates/aqua-registry/build.rs @@ -0,0 +1,121 @@ +use std::env; +use std::fs; +use std::path::Path; + +fn main() { + let out_dir = env::var("OUT_DIR").expect("OUT_DIR environment variable must be set"); + generate_baked_registry(&out_dir); +} +fn generate_baked_registry(out_dir: &str) { + let dest_path = Path::new(out_dir).join("aqua_standard_registry.rs"); + + // Look for the aqua-registry directory in the workspace root + let registry_dir = find_registry_dir() + .expect("Could not find aqua-registry directory in workspace root. Expected to find it at workspace_root/aqua-registry/"); + + let registries = + collect_aqua_registries(®istry_dir).expect("Failed to collect aqua registry files"); + + if registries.is_empty() { + panic!( + "No aqua registry files found in {}/pkgs/", + registry_dir.display() + ); + } + + let mut code = String::from("HashMap::from([\n"); + for (id, content) in registries { + code.push_str(&format!(" ({:?}, {:?}),\n", id, content)); + } + code.push_str("])"); + + fs::write(dest_path, code).expect("Failed to write baked registry file"); +} + +fn find_registry_dir() -> Option { + let current_dir = env::current_dir().ok()?; + + // Look for the workspace root by finding a Cargo.toml that contains [workspace] + let workspace_root = current_dir.ancestors().find(|dir| { + let cargo_toml = dir.join("Cargo.toml"); + if !cargo_toml.exists() { + return false; + } + // Check if this Cargo.toml defines a workspace + if let Ok(content) = fs::read_to_string(&cargo_toml) { + content.contains("[workspace]") + } else { + false + } + })?; + + let aqua_registry = workspace_root.join("aqua-registry"); + if aqua_registry.exists() { + return Some(aqua_registry); + } + + None +} + +fn collect_aqua_registries( + dir: &Path, +) -> Result, Box> { + let mut registries = Vec::new(); + + if !dir.exists() { + return Ok(registries); + } + + let pkgs_dir = dir.join("pkgs"); + if !pkgs_dir.exists() { + return Ok(registries); + } + + collect_registries_recursive(&pkgs_dir, &mut registries, String::new())?; + Ok(registries) +} + +fn collect_registries_recursive( + dir: &Path, + registries: &mut Vec<(String, String)>, + prefix: String, +) -> Result<(), Box> { + for entry in fs::read_dir(dir)? { + let entry = entry?; + let path = entry.path(); + + if path.is_dir() { + let dir_name = path.file_name().unwrap().to_string_lossy(); + let new_prefix = if prefix.is_empty() { + dir_name.to_string() + } else { + format!("{}/{}", prefix, dir_name) + }; + collect_registries_recursive(&path, registries, new_prefix)?; + } else if path.file_name() == Some(std::ffi::OsStr::new("registry.yaml")) { + let content = fs::read_to_string(&path)?; + registries.push((prefix.clone(), content.clone())); + + // Process aliases if they exist + #[allow(clippy::collapsible_if)] + if content.contains("aliases") { + if let Ok(registry) = serde_yaml::from_str::(&content) { + if let Some(packages) = registry.get("packages").and_then(|p| p.as_sequence()) { + for package in packages { + if let Some(aliases) = + package.get("aliases").and_then(|a| a.as_sequence()) + { + for alias in aliases { + if let Some(name) = alias.get("name").and_then(|n| n.as_str()) { + registries.push((name.to_string(), content.clone())); + } + } + } + } + } + } + } + } + } + Ok(()) +} diff --git a/crates/aqua-registry/src/lib.rs b/crates/aqua-registry/src/lib.rs new file mode 100644 index 0000000000..24aafcc59f --- /dev/null +++ b/crates/aqua-registry/src/lib.rs @@ -0,0 +1,77 @@ +//! Aqua Registry +//! +//! This crate provides functionality for working with Aqua package registry files. +//! It can load registry data from baked-in files, local repositories, or remote HTTP sources. + +mod registry; +mod template; +mod types; + +// Re-export only what's needed by the main mise crate +pub use registry::{ + AQUA_STANDARD_REGISTRY_FILES, AquaRegistry, DefaultRegistryFetcher, FileCacheStore, + NoOpCacheStore, +}; +pub use types::{AquaChecksumType, AquaMinisignType, AquaPackage, AquaPackageType, RegistryYaml}; + +use thiserror::Error; + +/// Errors that can occur when working with the Aqua registry +#[derive(Error, Debug)] +pub enum AquaRegistryError { + #[error("package not found: {0}")] + PackageNotFound(String), + #[error("registry not available: {0}")] + RegistryNotAvailable(String), + #[error("template error: {0}")] + TemplateError(#[from] eyre::Error), + #[error("yaml parse error: {0}")] + YamlError(#[from] serde_yaml::Error), + #[error("io error: {0}")] + IoError(#[from] std::io::Error), + #[error("expression error: {0}")] + ExpressionError(String), +} + +pub type Result = std::result::Result; + +/// Configuration for the Aqua registry +#[derive(Debug, Clone)] +pub struct AquaRegistryConfig { + /// Path to cache directory for cloned repositories + pub cache_dir: std::path::PathBuf, + /// URL of the registry repository (if None, only baked registry will be used) + pub registry_url: Option, + /// Whether to use the baked-in registry + pub use_baked_registry: bool, + /// Whether to skip network operations (prefer offline mode) + pub prefer_offline: bool, +} + +impl Default for AquaRegistryConfig { + fn default() -> Self { + Self { + cache_dir: std::env::temp_dir().join("aqua-registry"), + registry_url: Some("https://github.com/aquaproj/aqua-registry".to_string()), + use_baked_registry: true, + prefer_offline: false, + } + } +} + +/// Trait for fetching registry files from various sources +#[allow(async_fn_in_trait)] +pub trait RegistryFetcher { + /// Fetch and parse a registry YAML file for the given package ID + async fn fetch_registry(&self, package_id: &str) -> Result; +} + +/// Trait for caching registry data +pub trait CacheStore { + /// Check if cached data exists and is fresh + fn is_fresh(&self, key: &str) -> bool; + /// Store data in cache + fn store(&self, key: &str, data: &[u8]) -> std::io::Result<()>; + /// Retrieve data from cache + fn retrieve(&self, key: &str) -> std::io::Result>>; +} diff --git a/crates/aqua-registry/src/registry.rs b/crates/aqua-registry/src/registry.rs new file mode 100644 index 0000000000..7b138e45e3 --- /dev/null +++ b/crates/aqua-registry/src/registry.rs @@ -0,0 +1,229 @@ +use crate::types::{AquaPackage, RegistryYaml}; +use crate::{AquaRegistryConfig, AquaRegistryError, CacheStore, RegistryFetcher, Result}; +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::LazyLock; +use tokio::sync::Mutex; + +/// The main Aqua registry implementation +#[derive(Debug)] +pub struct AquaRegistry +where + F: RegistryFetcher, + C: CacheStore, +{ + #[allow(dead_code)] + config: AquaRegistryConfig, + fetcher: F, + #[allow(dead_code)] + cache_store: C, + #[allow(dead_code)] + repo_exists: bool, +} + +/// Default implementation of RegistryFetcher +#[derive(Debug, Clone)] +pub struct DefaultRegistryFetcher { + config: AquaRegistryConfig, +} + +/// No-op implementation of CacheStore +#[derive(Debug, Clone, Default)] +pub struct NoOpCacheStore; + +/// File-based cache store implementation +#[derive(Debug, Clone)] +pub struct FileCacheStore { + cache_dir: PathBuf, +} + +/// Baked registry files (compiled into binary) +pub static AQUA_STANDARD_REGISTRY_FILES: LazyLock> = + LazyLock::new(|| include!(concat!(env!("OUT_DIR"), "/aqua_standard_registry.rs"))); + +impl AquaRegistry { + /// Create a new AquaRegistry with the given configuration + pub fn new(config: AquaRegistryConfig) -> Self { + let repo_exists = Self::check_repo_exists(&config.cache_dir); + let fetcher = DefaultRegistryFetcher { + config: config.clone(), + }; + Self { + config, + fetcher, + cache_store: NoOpCacheStore, + repo_exists, + } + } + + /// Create a new AquaRegistry with custom fetcher and cache store + pub fn with_fetcher_and_cache( + config: AquaRegistryConfig, + fetcher: F, + cache_store: C, + ) -> AquaRegistry + where + F: RegistryFetcher, + C: CacheStore, + { + let repo_exists = Self::check_repo_exists(&config.cache_dir); + AquaRegistry { + config, + fetcher, + cache_store, + repo_exists, + } + } + + fn check_repo_exists(cache_dir: &std::path::Path) -> bool { + cache_dir.join(".git").exists() + } +} + +impl AquaRegistry +where + F: RegistryFetcher, + C: CacheStore, +{ + /// Get a package definition by ID + pub async fn package(&self, id: &str) -> Result { + static CACHE: LazyLock>> = + LazyLock::new(|| Mutex::new(HashMap::new())); + + if let Some(pkg) = CACHE.lock().await.get(id) { + return Ok(pkg.clone()); + } + + let registry = self.fetcher.fetch_registry(id).await?; + let mut pkg = registry + .packages + .into_iter() + .next() + .ok_or_else(|| AquaRegistryError::PackageNotFound(id.to_string()))?; + + pkg.setup_version_filter()?; + CACHE.lock().await.insert(id.to_string(), pkg.clone()); + Ok(pkg) + } + + /// Get a package definition configured for specific versions + pub async fn package_with_version( + &self, + id: &str, + versions: &[&str], + os: &str, + arch: &str, + ) -> Result { + Ok(self.package(id).await?.with_version(versions, os, arch)) + } +} + +impl RegistryFetcher for DefaultRegistryFetcher { + async fn fetch_registry(&self, package_id: &str) -> Result { + let path_id = package_id + .split('/') + .collect::>() + .join(std::path::MAIN_SEPARATOR_STR); + let path = self + .config + .cache_dir + .join("pkgs") + .join(&path_id) + .join("registry.yaml"); + + // Try to read from local repository first + if self.config.cache_dir.join(".git").exists() && path.exists() { + log::trace!("reading aqua-registry for {package_id} from repo at {path:?}"); + let contents = std::fs::read_to_string(&path)?; + return Ok(serde_yaml::from_str(&contents)?); + } + + // Fall back to baked registry if enabled + #[allow(clippy::collapsible_if)] + if self.config.use_baked_registry && AQUA_STANDARD_REGISTRY_FILES.contains_key(package_id) { + if let Some(content) = AQUA_STANDARD_REGISTRY_FILES.get(package_id) { + log::trace!("reading baked-in aqua-registry for {package_id}"); + return Ok(serde_yaml::from_str(content)?); + } + } + + Err(AquaRegistryError::RegistryNotAvailable(format!( + "no aqua-registry found for {package_id}" + ))) + } +} + +impl CacheStore for NoOpCacheStore { + fn is_fresh(&self, _key: &str) -> bool { + false + } + + fn store(&self, _key: &str, _data: &[u8]) -> std::io::Result<()> { + Ok(()) + } + + fn retrieve(&self, _key: &str) -> std::io::Result>> { + Ok(None) + } +} + +impl FileCacheStore { + pub fn new(cache_dir: PathBuf) -> Self { + Self { cache_dir } + } +} + +impl CacheStore for FileCacheStore { + fn is_fresh(&self, key: &str) -> bool { + // Check if cache entry exists and is less than a week old + #[allow(clippy::collapsible_if)] + if let Ok(metadata) = std::fs::metadata(self.cache_dir.join(key)) { + if let Ok(modified) = metadata.modified() { + let age = std::time::SystemTime::now() + .duration_since(modified) + .unwrap_or_default(); + return age < std::time::Duration::from_secs(7 * 24 * 60 * 60); // 1 week + } + } + false + } + + fn store(&self, key: &str, data: &[u8]) -> std::io::Result<()> { + let path = self.cache_dir.join(key); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::write(path, data) + } + + fn retrieve(&self, key: &str) -> std::io::Result>> { + let path = self.cache_dir.join(key); + match std::fs::read(path) { + Ok(data) => Ok(Some(data)), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(e), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_registry_creation() { + let config = AquaRegistryConfig::default(); + let registry = AquaRegistry::new(config); + + // This should not panic - registry should be created successfully + drop(registry); + } + + #[test] + fn test_cache_store() { + let cache = NoOpCacheStore; + assert!(!cache.is_fresh("test")); + assert!(cache.store("test", b"data").is_ok()); + assert!(cache.retrieve("test").unwrap().is_none()); + } +} diff --git a/src/aqua/aqua_template.rs b/crates/aqua-registry/src/template.rs similarity index 88% rename from src/aqua/aqua_template.rs rename to crates/aqua-registry/src/template.rs index e6e79d8e8e..c3dd7bae6d 100644 --- a/src/aqua/aqua_template.rs +++ b/crates/aqua-registry/src/template.rs @@ -202,28 +202,28 @@ impl Parser<'_> { #[cfg(test)] mod tests { - use crate::config::Config; - use super::*; - use crate::hashmap; - #[tokio::test] - async fn test_render() { - let _config = Config::get().await.unwrap(); + fn hashmap(data: Vec<(&str, &str)>) -> HashMap { + data.iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect() + } + + #[test] + fn test_render() { let tmpl = "Hello, {{.OS}}!"; - let mut ctx = HashMap::new(); - ctx.insert("OS".to_string(), "world".to_string()); + let ctx = hashmap(vec![("OS", "world")]); assert_eq!(render(tmpl, &ctx).unwrap(), "Hello, world!"); } macro_rules! parse_tests { ($($name:ident: $value:expr,)*) => { $( - #[tokio::test] - async fn $name() { - let _config = Config::get().await.unwrap(); - let (input, expected, ctx): (&str, &str, HashMap<&str, &str>) = $value; - let ctx = ctx.iter().map(|(k, v)| (k.to_string(), v.to_string())).collect(); + #[test] + fn $name() { + let (input, expected, ctx_data): (&str, &str, Vec<(&str, &str)>) = $value; + let ctx = hashmap(ctx_data); let parser = Parser { ctx: &ctx }; let tokens = lex(input).unwrap(); assert_eq!(expected, parser.parse(tokens.iter().collect()).unwrap()); @@ -232,25 +232,24 @@ mod tests { }} parse_tests!( - test_parse_key: (".OS", "world", hashmap!{"OS" => "world"}), - test_parse_string: ("\"world\"", "world", hashmap!{}), - test_parse_title: (r#"title "world""#, "World", hashmap!{}), - test_parse_trimv: (r#"trimV "v1.0.0""#, "1.0.0", hashmap!{}), - test_parse_trim_prefix: (r#"trimPrefix "v" "v1.0.0""#, "1.0.0", hashmap!{}), - test_parse_trim_prefix2: (r#"trimPrefix "v" "1.0.0""#, "1.0.0", hashmap!{}), - test_parse_trim_suffix: (r#"trimSuffix "-v1.0.0" "foo-v1.0.0""#, "foo", hashmap!{}), - test_parse_pipe: (r#"trimPrefix "foo-" "foo-v1.0.0" | trimV"#, "1.0.0", hashmap!{}), + test_parse_key: (".OS", "world", vec![("OS", "world")]), + test_parse_string: ("\"world\"", "world", vec![]), + test_parse_title: (r#"title "world""#, "World", vec![]), + test_parse_trimv: (r#"trimV "v1.0.0""#, "1.0.0", vec![]), + test_parse_trim_prefix: (r#"trimPrefix "v" "v1.0.0""#, "1.0.0", vec![]), + test_parse_trim_prefix2: (r#"trimPrefix "v" "1.0.0""#, "1.0.0", vec![]), + test_parse_trim_suffix: (r#"trimSuffix "-v1.0.0" "foo-v1.0.0""#, "foo", vec![]), + test_parse_pipe: (r#"trimPrefix "foo-" "foo-v1.0.0" | trimV"#, "1.0.0", vec![]), test_parse_multiple_pipes: ( r#"trimPrefix "foo-" "foo-v1.0.0-beta" | trimSuffix "-beta" | trimV"#, "1.0.0", - hashmap!{}, + vec![], ), - test_parse_replace: (r#"replace "foo" "bar" "foo-bar""#, "bar-bar", hashmap!{}), + test_parse_replace: (r#"replace "foo" "bar" "foo-bar""#, "bar-bar", vec![]), ); - #[tokio::test] - async fn test_parse_err() { - let _config = Config::get().await.unwrap(); + #[test] + fn test_parse_err() { let parser = Parser { ctx: &HashMap::new(), }; @@ -258,9 +257,8 @@ mod tests { assert!(parser.parse(tokens.iter().collect()).is_err()); } - #[tokio::test] - async fn test_lex() { - let _config = Config::get().await.unwrap(); + #[test] + fn test_lex() { assert_eq!( lex(r#"trimPrefix "foo-" "foo-v1.0.0" | trimV"#).unwrap(), vec![ diff --git a/src/aqua/aqua_registry.rs b/crates/aqua-registry/src/types.rs similarity index 70% rename from src/aqua/aqua_registry.rs rename to crates/aqua-registry/src/types.rs index 7ca5602c13..92890fdb2a 100644 --- a/src/aqua/aqua_registry.rs +++ b/crates/aqua-registry/src/types.rs @@ -1,42 +1,13 @@ -use crate::backend::aqua; -use crate::backend::aqua::{arch, os}; -use crate::duration::WEEKLY; -use crate::env; -use crate::git::{CloneOptions, Git}; -use crate::semver::split_version_prefix; -use crate::{aqua::aqua_template, config::Settings}; -use crate::{dirs, file, hashmap}; -use expr::{Context, Program, Value}; -use eyre::{ContextCompat, Result, bail, eyre}; +use expr::{Context, Environment, Program, Value}; +use eyre::{Result, eyre}; use indexmap::IndexSet; use itertools::Itertools; use serde_derive::Deserialize; use std::cmp::PartialEq; use std::collections::HashMap; -use std::path::PathBuf; -use std::sync::LazyLock as Lazy; -use std::sync::LazyLock; -use tokio::sync::Mutex; - -#[allow(clippy::invisible_characters)] -pub static AQUA_STANDARD_REGISTRY_FILES: Lazy> = - Lazy::new(|| include!(concat!(env!("OUT_DIR"), "/aqua_standard_registry.rs"))); - -pub static AQUA_REGISTRY: Lazy = Lazy::new(|| { - AquaRegistry::standard().unwrap_or_else(|err| { - warn!("failed to initialize aqua registry: {err:?}"); - AquaRegistry::default() - }) -}); -static AQUA_REGISTRY_PATH: Lazy = Lazy::new(|| dirs::CACHE.join("aqua-registry")); -static AQUA_DEFAULT_REGISTRY_URL: &str = "https://github.com/aquaproj/aqua-registry"; - -#[derive(Default)] -pub struct AquaRegistry { - path: PathBuf, - repo_exists: bool, -} +use versions::Versioning; +/// Type of Aqua package #[derive(Debug, Deserialize, Default, Clone, PartialEq, strum::Display)] #[strum(serialize_all = "snake_case")] #[serde(rename_all = "snake_case")] @@ -50,6 +21,7 @@ pub enum AquaPackageType { Cargo, } +/// Main Aqua package definition #[derive(Debug, Deserialize, Clone)] #[serde(default)] pub struct AquaPackage { @@ -83,6 +55,7 @@ pub struct AquaPackage { pub path: Option, } +/// Override configuration for specific OS/architecture combinations #[derive(Debug, Deserialize, Clone)] struct AquaOverride { #[serde(flatten)] @@ -91,12 +64,14 @@ struct AquaOverride { goarch: Option, } +/// File definition within a package #[derive(Debug, Deserialize, Clone)] pub struct AquaFile { pub name: String, pub src: Option, } +/// Checksum algorithm options #[derive(Debug, Deserialize, Clone, strum::AsRefStr, strum::Display)] #[serde(rename_all = "lowercase")] #[strum(serialize_all = "lowercase")] @@ -108,6 +83,7 @@ pub enum AquaChecksumAlgorithm { Md5, } +/// Type of checksum source #[derive(Debug, Deserialize, Clone)] #[serde(rename_all = "snake_case")] pub enum AquaChecksumType { @@ -115,6 +91,7 @@ pub enum AquaChecksumType { Http, } +/// Type of minisign source #[derive(Debug, Deserialize, Clone)] #[serde(rename_all = "snake_case")] pub enum AquaMinisignType { @@ -122,6 +99,7 @@ pub enum AquaMinisignType { Http, } +/// Cosign signature configuration #[derive(Debug, Deserialize, Clone)] pub struct AquaCosignSignature { pub r#type: Option, @@ -130,6 +108,8 @@ pub struct AquaCosignSignature { pub url: Option, pub asset: Option, } + +/// Cosign verification configuration #[derive(Debug, Deserialize, Clone)] pub struct AquaCosign { pub enabled: Option, @@ -142,6 +122,7 @@ pub struct AquaCosign { opts: Vec, } +/// SLSA provenance configuration #[derive(Debug, Deserialize, Clone)] pub struct AquaSlsaProvenance { pub enabled: Option, @@ -154,6 +135,7 @@ pub struct AquaSlsaProvenance { pub source_tag: Option, } +/// Minisign verification configuration #[derive(Debug, Deserialize, Clone)] pub struct AquaMinisign { pub enabled: Option, @@ -165,6 +147,7 @@ pub struct AquaMinisign { pub public_key: Option, } +/// Checksum verification configuration #[derive(Debug, Deserialize, Clone)] pub struct AquaChecksum { pub r#type: Option, @@ -177,115 +160,64 @@ pub struct AquaChecksum { url: Option, } +/// Checksum pattern configuration #[derive(Debug, Deserialize, Clone)] pub struct AquaChecksumPattern { pub checksum: String, pub file: Option, } +/// Registry YAML file structure #[derive(Debug, Deserialize)] -struct RegistryYaml { - packages: Vec, +pub struct RegistryYaml { + pub packages: Vec, } -impl AquaRegistry { - pub fn standard() -> Result { - let path = AQUA_REGISTRY_PATH.clone(); - let repo = Git::new(&path); - let settings = Settings::get(); - let registry_url = - settings - .aqua - .registry_url - .as_deref() - .or(if settings.aqua.baked_registry { - None - } else { - Some(AQUA_DEFAULT_REGISTRY_URL) - }); - if let Some(registry_url) = registry_url { - if repo.exists() { - // TODO: re-clone if remote url doesn't match - fetch_latest_repo(&repo)?; - } else { - info!("cloning aqua registry from {registry_url} to {path:?}"); - repo.clone(registry_url, CloneOptions::default())?; - } - } - Ok(Self { - path, - repo_exists: repo.exists(), - }) - } - - pub async fn package(&self, id: &str) -> Result { - static CACHE: LazyLock>> = - LazyLock::new(|| Mutex::new(HashMap::new())); - if let Some(pkg) = CACHE.lock().await.get(id) { - return Ok(pkg.clone()); - } - let path_id = id.split('/').join(std::path::MAIN_SEPARATOR_STR); - let path = self.path.join("pkgs").join(&path_id).join("registry.yaml"); - let mut pkg = self - .fetch_package_yaml(id, &path) - .await? - .packages - .into_iter() - .next() - .wrap_err(format!("no package found for {id} in {path:?}"))?; - if let Some(version_filter) = &pkg.version_filter { - pkg.version_filter_expr = Some(expr::compile(version_filter)?); +impl Default for AquaPackage { + fn default() -> Self { + Self { + r#type: AquaPackageType::GithubRelease, + repo_owner: String::new(), + repo_name: String::new(), + name: None, + asset: String::new(), + url: String::new(), + description: None, + format: String::new(), + rosetta2: false, + windows_arm_emulation: false, + complete_windows_ext: true, + supported_envs: Vec::new(), + files: Vec::new(), + replacements: HashMap::new(), + version_prefix: None, + version_filter: None, + version_filter_expr: None, + version_source: None, + checksum: None, + slsa_provenance: None, + minisign: None, + overrides: Vec::new(), + version_constraint: String::new(), + version_overrides: Vec::new(), + no_asset: false, + error_message: None, + path: None, } - CACHE.lock().await.insert(id.to_string(), pkg.clone()); - Ok(pkg) - } - - pub async fn package_with_version(&self, id: &str, versions: &[&str]) -> Result { - Ok(self.package(id).await?.with_version(versions)) - } - - async fn fetch_package_yaml(&self, id: &str, path: &PathBuf) -> Result { - let registry = if self.repo_exists { - trace!("reading aqua-registry for {id} from repo at {path:?}"); - serde_yaml::from_reader(file::open(path)?)? - } else if Settings::get().aqua.baked_registry - && AQUA_STANDARD_REGISTRY_FILES.contains_key(id) - { - trace!("reading baked-in aqua-registry for {id}"); - serde_yaml::from_str(AQUA_STANDARD_REGISTRY_FILES.get(id).unwrap())? - } else { - bail!("no aqua-registry found for {id}"); - }; - Ok(registry) - } -} - -fn fetch_latest_repo(repo: &Git) -> Result<()> { - if file::modified_duration(&repo.dir)? < WEEKLY { - return Ok(()); } - - // Don't update if PREFER_OFFLINE is set - if env::PREFER_OFFLINE.load(std::sync::atomic::Ordering::Relaxed) { - trace!("skipping aqua registry update due to PREFER_OFFLINE"); - return Ok(()); - } - - info!("updating aqua registry repo"); - repo.update(None)?; - Ok(()) } impl AquaPackage { - pub fn with_version(mut self, versions: &[&str]) -> AquaPackage { + /// Apply version-specific configurations and overrides + pub fn with_version(mut self, versions: &[&str], os: &str, arch: &str) -> AquaPackage { self = apply_override(self.clone(), self.version_override(versions)); if let Some(avo) = self.overrides.clone().into_iter().find(|o| { if let (Some(goos), Some(goarch)) = (&o.goos, &o.goarch) { - goos == aqua::os() && goarch == aqua::arch() + goos == os && goarch == arch } else if let Some(goos) = &o.goos { - goos == aqua::os() + goos == os } else if let Some(goarch) = &o.goarch { - goarch == aqua::arch() + goarch == arch } else { false } @@ -295,7 +227,6 @@ impl AquaPackage { self } - // all versions must refer to the same logical version. e.g. ["v1.2.3", "1.2.3"] fn version_override(&self, versions: &[&str]) -> &AquaPackage { let expressions = versions .iter() @@ -310,7 +241,9 @@ impl AquaPackage { } else { expressions.iter().any(|(expr, ctx)| { expr.eval(&vo.version_constraint, ctx) - .map_err(|e| debug!("error parsing {}: {e}", vo.version_constraint)) + .map_err(|e| { + log::debug!("error parsing {}: {e}", vo.version_constraint) + }) .unwrap_or(false.into()) .as_bool() .unwrap() @@ -320,6 +253,7 @@ impl AquaPackage { .unwrap_or(self) } + /// Detect the format of an archive based on its filename fn detect_format(&self, asset_name: &str) -> &'static str { let formats = [ "tar.br", "tar.bz2", "tar.gz", "tar.lz4", "tar.sz", "tar.xz", "tbr", "tbz", "tbz2", @@ -340,18 +274,19 @@ impl AquaPackage { "raw" } - pub fn format(&self, v: &str) -> Result<&str> { + /// Get the format for this package and version + pub fn format(&self, v: &str, os: &str, arch: &str) -> Result<&str> { if self.r#type == AquaPackageType::GithubArchive { return Ok("tar.gz"); } let format = if self.format.is_empty() { let asset = if !self.asset.is_empty() { - self.asset(v)? + self.asset(v, os, arch)? } else if !self.url.is_empty() { self.url.to_string() } else { - debug!("no asset or url for {}/{}", self.repo_owner, self.repo_name); - "".to_string() + log::debug!("no asset or url for {}/{}", self.repo_owner, self.repo_name); + String::new() }; self.detect_format(&asset) } else { @@ -365,36 +300,37 @@ impl AquaPackage { Ok(format) } - pub fn asset(&self, v: &str) -> Result { - // derive asset from url if not set and url contains a path + /// Get the asset name for this package and version + pub fn asset(&self, v: &str, os: &str, arch: &str) -> Result { if self.asset.is_empty() && self.url.split("/").count() > "//".len() { let asset = self.url.rsplit("/").next().unwrap_or(""); - self.parse_aqua_str(asset, v, &Default::default()) + self.parse_aqua_str(asset, v, &Default::default(), os, arch) } else { - self.parse_aqua_str(&self.asset, v, &Default::default()) + self.parse_aqua_str(&self.asset, v, &Default::default(), os, arch) } } - pub fn asset_strs(&self, v: &str) -> Result> { - let mut strs = IndexSet::from([self.asset(v)?]); - if cfg!(macos) { + /// Get all possible asset strings for this package, version and platform + pub fn asset_strs(&self, v: &str, os: &str, arch: &str) -> Result> { + let mut strs = + IndexSet::from([self.parse_aqua_str(&self.asset, v, &Default::default(), os, arch)?]); + if os == "darwin" { let mut ctx = HashMap::default(); ctx.insert("Arch".to_string(), "universal".to_string()); - strs.insert(self.parse_aqua_str(&self.asset, v, &ctx)?); - } else if cfg!(windows) { + strs.insert(self.parse_aqua_str(&self.asset, v, &ctx, os, arch)?); + } else if os == "windows" { let mut ctx = HashMap::default(); - let asset = self.parse_aqua_str(&self.asset, v, &ctx)?; - if self.complete_windows_ext && self.format(v)? == "raw" { + let asset = self.parse_aqua_str(&self.asset, v, &ctx, os, arch)?; + if self.complete_windows_ext && self.format(v, os, arch)? == "raw" { strs.insert(format!("{asset}.exe")); } else { strs.insert(asset); } - if cfg!(target_arch = "aarch64") { - // assume windows arm64 emulation is supported + if arch == "arm64" { ctx.insert("Arch".to_string(), "amd64".to_string()); - strs.insert(self.parse_aqua_str(&self.asset, v, &ctx)?); - let asset = self.parse_aqua_str(&self.asset, v, &ctx)?; - if self.complete_windows_ext && self.format(v)? == "raw" { + strs.insert(self.parse_aqua_str(&self.asset, v, &ctx, os, arch)?); + let asset = self.parse_aqua_str(&self.asset, v, &ctx, os, arch)?; + if self.complete_windows_ext && self.format(v, os, arch)? == "raw" { strs.insert(format!("{asset}.exe")); } else { strs.insert(asset); @@ -404,50 +340,81 @@ impl AquaPackage { Ok(strs) } - pub fn url(&self, v: &str) -> Result { + /// Get the URL for this package and version + pub fn url(&self, v: &str, os: &str, arch: &str) -> Result { let mut url = self.url.clone(); - if cfg!(windows) && self.complete_windows_ext && self.format(v)? == "raw" { + if os == "windows" && self.complete_windows_ext && self.format(v, os, arch)? == "raw" { url.push_str(".exe"); } - self.parse_aqua_str(&url, v, &Default::default()) + self.parse_aqua_str(&url, v, &Default::default(), os, arch) } - fn parse_aqua_str( + /// Parse an Aqua template string with variable substitution and platform info + pub fn parse_aqua_str( &self, s: &str, v: &str, overrides: &HashMap, + os: &str, + arch: &str, ) -> Result { - let os = os(); - let mut arch = arch(); + let mut actual_arch = arch; if os == "darwin" && arch == "arm64" && self.rosetta2 { - arch = "amd64"; + actual_arch = "amd64"; } if os == "windows" && arch == "arm64" && self.windows_arm_emulation { - arch = "amd64"; + actual_arch = "amd64"; } + let replace = |s: &str| { self.replacements .get(s) .map(|s| s.to_string()) .unwrap_or_else(|| s.to_string()) }; + let semver = if let Some(prefix) = &self.version_prefix { v.strip_prefix(prefix).unwrap_or(v) } else { v }; - let mut ctx = hashmap! { - "Version".to_string() => replace(v), - "SemVer".to_string() => replace(semver), - "OS".to_string() => replace(os), - "GOOS".to_string() => replace(os), - "GOARCH".to_string() => replace(arch), - "Arch".to_string() => replace(arch), - "Format".to_string() => replace(&self.format), - }; + + let mut ctx = HashMap::new(); + ctx.insert("Version".to_string(), replace(v)); + ctx.insert("SemVer".to_string(), replace(semver)); + ctx.insert("OS".to_string(), replace(os)); + ctx.insert("GOOS".to_string(), replace(os)); + ctx.insert("GOARCH".to_string(), replace(actual_arch)); + ctx.insert("Arch".to_string(), replace(actual_arch)); + ctx.insert("Format".to_string(), replace(&self.format)); ctx.extend(overrides.clone()); - aqua_template::render(s, &ctx) + + crate::template::render(s, &ctx) + } + + /// Set up version filter expression if configured + pub fn setup_version_filter(&mut self) -> Result<()> { + if let Some(version_filter) = &self.version_filter { + self.version_filter_expr = Some(expr::compile(version_filter)?); + } + Ok(()) + } + + /// Check if a version passes the version filter + pub fn version_filter_ok(&self, v: &str) -> Result { + if let Some(filter) = self.version_filter_expr.clone() { + if let Value::Bool(expr) = self.expr(v, filter)? { + Ok(expr) + } else { + log::warn!( + "invalid response from version filter: {}", + self.version_filter.as_ref().unwrap() + ); + Ok(true) + } + } else { + Ok(true) + } } fn expr(&self, v: &str, program: Program) -> Result { @@ -455,10 +422,10 @@ impl AquaPackage { expr.run(program, &self.expr_ctx(v)).map_err(|e| eyre!(e)) } - fn expr_parser(&self, v: &str) -> expr::Environment<'_> { + fn expr_parser(&self, v: &str) -> Environment<'_> { let (_, v) = split_version_prefix(v); - let ver = versions::Versioning::new(v); - let mut env = expr::Environment::new(); + let ver = Versioning::new(v); + let mut env = Environment::new(); env.add_function("semver", move |c| { if c.args.len() != 1 { return Err("semver() takes exactly one argument".to_string().into()); @@ -490,28 +457,38 @@ impl AquaPackage { ctx.insert("Version", v); ctx } +} - pub fn version_filter_ok(&self, v: &str) -> Result { - // TODO: precompile the expression - if let Some(filter) = self.version_filter_expr.clone() { - if let Value::Bool(expr) = self.expr(v, filter)? { - Ok(expr) - } else { - warn!( - "invalid response from version filter: {}", - self.version_filter.as_ref().unwrap() - ); - Ok(true) +/// splits a version number into an optional prefix and the remaining version string +fn split_version_prefix(version: &str) -> (String, String) { + version + .char_indices() + .find_map(|(i, c)| { + if c.is_ascii_digit() { + if i == 0 { + return Some(i); + } + // If the previous char is a delimiter or 'v', we found a split point. + let prev_char = version.chars().nth(i - 1).unwrap(); + if ['-', '_', '/', '.', 'v', 'V'].contains(&prev_char) { + return Some(i); + } } - } else { - Ok(true) - } - } + None + }) + .map_or_else( + || ("".into(), version.into()), + |i| { + let (prefix, version) = version.split_at(i); + (prefix.into(), version.into()) + }, + ) } impl AquaFile { - pub fn src(&self, pkg: &AquaPackage, v: &str) -> Result> { - let asset = pkg.asset(v)?; + /// Get the source path for this file within the package + pub fn src(&self, pkg: &AquaPackage, v: &str, os: &str, arch: &str) -> Result> { + let asset = pkg.asset(v, os, arch)?; let asset = asset.strip_suffix(".tar.gz").unwrap_or(&asset); let asset = asset.strip_suffix(".tar.xz").unwrap_or(asset); let asset = asset.strip_suffix(".tar.bz2").unwrap_or(asset); @@ -524,13 +501,14 @@ impl AquaFile { let asset = asset.strip_suffix(".txz").unwrap_or(asset); let asset = asset.strip_suffix(".tbz2").unwrap_or(asset); let asset = asset.strip_suffix(".tbz").unwrap_or(asset); - let ctx = hashmap! { - "AssetWithoutExt".to_string() => asset.to_string(), - "FileName".to_string() => self.name.to_string(), - }; + + let mut ctx = HashMap::new(); + ctx.insert("AssetWithoutExt".to_string(), asset.to_string()); + ctx.insert("FileName".to_string(), self.name.to_string()); + self.src .as_ref() - .map(|src| pkg.parse_aqua_str(src, v, &ctx)) + .map(|src| pkg.parse_aqua_str(src, v, &ctx, os, arch)) .transpose() } } @@ -590,11 +568,13 @@ fn apply_override(mut orig: AquaPackage, avo: &AquaPackage) -> AquaPackage { slsa_provenance.merge(avo_slsa_provenance); orig.slsa_provenance = Some(slsa_provenance); } + if let Some(avo_minisign) = avo.minisign.clone() { let mut minisign = orig.minisign.unwrap_or_else(|| avo_minisign.clone()); minisign.merge(avo_minisign); orig.minisign = Some(minisign); } + if avo.no_asset { orig.no_asset = true; } @@ -607,35 +587,47 @@ fn apply_override(mut orig: AquaPackage, avo: &AquaPackage) -> AquaPackage { orig } +// Implementation of merge methods for various types impl AquaChecksum { pub fn _type(&self) -> &AquaChecksumType { self.r#type.as_ref().unwrap() } + pub fn algorithm(&self) -> &AquaChecksumAlgorithm { self.algorithm.as_ref().unwrap() } - pub fn asset_strs(&self, pkg: &AquaPackage, v: &str) -> Result> { + + pub fn asset_strs( + &self, + pkg: &AquaPackage, + v: &str, + os: &str, + arch: &str, + ) -> Result> { let mut asset_strs = IndexSet::new(); - for asset in pkg.asset_strs(v)? { + for asset in pkg.asset_strs(v, os, arch)? { let checksum_asset = self.asset.as_ref().unwrap(); - let ctx = hashmap! { - "Asset".to_string() => asset.to_string(), - }; - asset_strs.insert(pkg.parse_aqua_str(checksum_asset, v, &ctx)?); + let mut ctx = HashMap::new(); + ctx.insert("Asset".to_string(), asset.to_string()); + asset_strs.insert(pkg.parse_aqua_str(checksum_asset, v, &ctx, os, arch)?); } Ok(asset_strs) } + pub fn pattern(&self) -> &AquaChecksumPattern { self.pattern.as_ref().unwrap() } + pub fn enabled(&self) -> bool { self.enabled.unwrap_or(true) } + pub fn file_format(&self) -> &str { self.file_format.as_deref().unwrap_or("raw") } - pub fn url(&self, pkg: &AquaPackage, v: &str) -> Result { - pkg.parse_aqua_str(self.url.as_ref().unwrap(), v, &Default::default()) + + pub fn url(&self, pkg: &AquaPackage, v: &str, os: &str, arch: &str) -> Result { + pkg.parse_aqua_str(self.url.as_ref().unwrap(), v, &Default::default(), os, arch) } fn merge(&mut self, other: Self) { @@ -670,10 +662,10 @@ impl AquaChecksum { } impl AquaCosign { - pub fn opts(&self, pkg: &AquaPackage, v: &str) -> Result> { + pub fn opts(&self, pkg: &AquaPackage, v: &str, os: &str, arch: &str) -> Result> { self.opts .iter() - .map(|opt| pkg.parse_aqua_str(opt, v, &Default::default())) + .map(|opt| pkg.parse_aqua_str(opt, v, &Default::default(), os, arch)) .collect() } @@ -715,16 +707,24 @@ impl AquaCosign { } impl AquaCosignSignature { - pub fn url(&self, pkg: &AquaPackage, v: &str) -> Result { - pkg.parse_aqua_str(self.url.as_ref().unwrap(), v, &Default::default()) + pub fn url(&self, pkg: &AquaPackage, v: &str, os: &str, arch: &str) -> Result { + pkg.parse_aqua_str(self.url.as_ref().unwrap(), v, &Default::default(), os, arch) } - pub fn asset(&self, pkg: &AquaPackage, v: &str) -> Result { - pkg.parse_aqua_str(self.asset.as_ref().unwrap(), v, &Default::default()) + + pub fn asset(&self, pkg: &AquaPackage, v: &str, os: &str, arch: &str) -> Result { + pkg.parse_aqua_str( + self.asset.as_ref().unwrap(), + v, + &Default::default(), + os, + arch, + ) } - pub fn arg(&self, pkg: &AquaPackage, v: &str) -> Result { + + pub fn arg(&self, pkg: &AquaPackage, v: &str, os: &str, arch: &str) -> Result { match self.r#type.as_deref().unwrap_or_default() { "github_release" => { - let asset = self.asset(pkg, v)?; + let asset = self.asset(pkg, v, os, arch)?; let repo_owner = self .repo_owner .clone() @@ -738,13 +738,14 @@ impl AquaCosignSignature { "https://github.com/{repo}/releases/download/{v}/{asset}" )) } - "http" => self.url(pkg, v), + "http" => self.url(pkg, v, os, arch), t => { - warn!( + log::warn!( "unsupported cosign signature type for {}/{}: {t}", - pkg.repo_owner, pkg.repo_name + pkg.repo_owner, + pkg.repo_name ); - Ok("".to_string()) + Ok(String::new()) } } } @@ -769,11 +770,18 @@ impl AquaCosignSignature { } impl AquaSlsaProvenance { - pub fn asset(&self, pkg: &AquaPackage, v: &str) -> Result { - pkg.parse_aqua_str(self.asset.as_ref().unwrap(), v, &Default::default()) + pub fn asset(&self, pkg: &AquaPackage, v: &str, os: &str, arch: &str) -> Result { + pkg.parse_aqua_str( + self.asset.as_ref().unwrap(), + v, + &Default::default(), + os, + arch, + ) } - pub fn url(&self, pkg: &AquaPackage, v: &str) -> Result { - pkg.parse_aqua_str(self.url.as_ref().unwrap(), v, &Default::default()) + + pub fn url(&self, pkg: &AquaPackage, v: &str, os: &str, arch: &str) -> Result { + pkg.parse_aqua_str(self.url.as_ref().unwrap(), v, &Default::default(), os, arch) } fn merge(&mut self, other: Self) { @@ -808,14 +816,29 @@ impl AquaMinisign { pub fn _type(&self) -> &AquaMinisignType { self.r#type.as_ref().unwrap() } - pub fn url(&self, pkg: &AquaPackage, v: &str) -> Result { - pkg.parse_aqua_str(self.url.as_ref().unwrap(), v, &Default::default()) + + pub fn url(&self, pkg: &AquaPackage, v: &str, os: &str, arch: &str) -> Result { + pkg.parse_aqua_str(self.url.as_ref().unwrap(), v, &Default::default(), os, arch) } - pub fn asset(&self, pkg: &AquaPackage, v: &str) -> Result { - pkg.parse_aqua_str(self.asset.as_ref().unwrap(), v, &Default::default()) + + pub fn asset(&self, pkg: &AquaPackage, v: &str, os: &str, arch: &str) -> Result { + pkg.parse_aqua_str( + self.asset.as_ref().unwrap(), + v, + &Default::default(), + os, + arch, + ) } - pub fn public_key(&self, pkg: &AquaPackage, v: &str) -> Result { - pkg.parse_aqua_str(self.public_key.as_ref().unwrap(), v, &Default::default()) + + pub fn public_key(&self, pkg: &AquaPackage, v: &str, os: &str, arch: &str) -> Result { + pkg.parse_aqua_str( + self.public_key.as_ref().unwrap(), + v, + &Default::default(), + os, + arch, + ) } fn merge(&mut self, other: Self) { @@ -842,37 +865,3 @@ impl AquaMinisign { } } } - -impl Default for AquaPackage { - fn default() -> Self { - Self { - r#type: AquaPackageType::GithubRelease, - repo_owner: "".to_string(), - repo_name: "".to_string(), - name: None, - asset: "".to_string(), - url: "".to_string(), - description: None, - format: "".to_string(), - rosetta2: false, - windows_arm_emulation: false, - complete_windows_ext: true, - supported_envs: vec![], - files: vec![], - replacements: HashMap::new(), - version_prefix: None, - version_filter: None, - version_filter_expr: None, - version_source: None, - checksum: None, - slsa_provenance: None, - minisign: None, - overrides: vec![], - version_constraint: "".to_string(), - version_overrides: vec![], - no_asset: false, - error_message: None, - path: None, - } - } -} diff --git a/src/aqua/aqua_registry_wrapper.rs b/src/aqua/aqua_registry_wrapper.rs new file mode 100644 index 0000000000..9fc1cb6eb1 --- /dev/null +++ b/src/aqua/aqua_registry_wrapper.rs @@ -0,0 +1,120 @@ +use crate::backend::aqua::{arch, os}; +use crate::config::Settings; +use crate::git::{CloneOptions, Git}; +use crate::{dirs, duration::WEEKLY, env, file}; +use aqua_registry::{AquaRegistry, AquaRegistryConfig}; +use eyre::Result; +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::LazyLock as Lazy; +use tokio::sync::Mutex; + +static AQUA_REGISTRY_PATH: Lazy = Lazy::new(|| dirs::CACHE.join("aqua-registry")); +static AQUA_DEFAULT_REGISTRY_URL: &str = "https://github.com/aquaproj/aqua-registry"; + +pub static AQUA_REGISTRY: Lazy = Lazy::new(|| { + MiseAquaRegistry::standard().unwrap_or_else(|err| { + warn!("failed to initialize aqua registry: {err:?}"); + MiseAquaRegistry::default() + }) +}); + +/// Wrapper around the aqua-registry crate that provides mise-specific functionality +#[derive(Debug)] +pub struct MiseAquaRegistry { + inner: AquaRegistry, + #[allow(dead_code)] + path: PathBuf, + #[allow(dead_code)] + repo_exists: bool, +} + +impl Default for MiseAquaRegistry { + fn default() -> Self { + let config = AquaRegistryConfig::default(); + let inner = AquaRegistry::new(config.clone()); + Self { + inner, + path: config.cache_dir, + repo_exists: false, + } + } +} + +impl MiseAquaRegistry { + pub fn standard() -> Result { + let path = AQUA_REGISTRY_PATH.clone(); + let repo = Git::new(&path); + let settings = Settings::get(); + let registry_url = + settings + .aqua + .registry_url + .as_deref() + .or(if settings.aqua.baked_registry { + None + } else { + Some(AQUA_DEFAULT_REGISTRY_URL) + }); + + if let Some(registry_url) = registry_url { + if repo.exists() { + fetch_latest_repo(&repo)?; + } else { + info!("cloning aqua registry from {registry_url} to {path:?}"); + repo.clone(registry_url, CloneOptions::default())?; + } + } + + let config = AquaRegistryConfig { + cache_dir: path.clone(), + registry_url: registry_url.map(|s| s.to_string()), + use_baked_registry: settings.aqua.baked_registry, + prefer_offline: env::PREFER_OFFLINE.load(std::sync::atomic::Ordering::Relaxed), + }; + + let inner = AquaRegistry::new(config); + + Ok(Self { + inner, + path, + repo_exists: repo.exists(), + }) + } + + pub async fn package(&self, id: &str) -> Result { + static CACHE: Lazy>> = + Lazy::new(|| Mutex::new(HashMap::new())); + + if let Some(pkg) = CACHE.lock().await.get(id) { + return Ok(pkg.clone()); + } + + let pkg = self.inner.package(id).await?; + CACHE.lock().await.insert(id.to_string(), pkg.clone()); + Ok(pkg) + } + + pub async fn package_with_version(&self, id: &str, versions: &[&str]) -> Result { + let pkg = self.package(id).await?; + Ok(pkg.with_version(versions, os(), arch())) + } +} + +fn fetch_latest_repo(repo: &Git) -> Result<()> { + if file::modified_duration(&repo.dir)? < WEEKLY { + return Ok(()); + } + + if env::PREFER_OFFLINE.load(std::sync::atomic::Ordering::Relaxed) { + trace!("skipping aqua registry update due to PREFER_OFFLINE"); + return Ok(()); + } + + info!("updating aqua registry repo"); + repo.update(None)?; + Ok(()) +} + +// Re-export types and static for compatibility +pub use aqua_registry::{AquaChecksumType, AquaMinisignType, AquaPackage, AquaPackageType}; diff --git a/src/aqua/mod.rs b/src/aqua/mod.rs index 9a3e71664d..e3432d4e11 100644 --- a/src/aqua/mod.rs +++ b/src/aqua/mod.rs @@ -1,2 +1 @@ -pub(crate) mod aqua_registry; -pub(crate) mod aqua_template; +pub(crate) mod aqua_registry_wrapper; diff --git a/src/backend/aqua.rs b/src/backend/aqua.rs index da176203e3..54095be847 100644 --- a/src/backend/aqua.rs +++ b/src/backend/aqua.rs @@ -11,7 +11,7 @@ use crate::plugins::VERSION_REGEX; use crate::registry::REGISTRY; use crate::toolset::ToolVersion; use crate::{ - aqua::aqua_registry::{ + aqua::aqua_registry_wrapper::{ AQUA_REGISTRY, AquaChecksumType, AquaMinisignType, AquaPackage, AquaPackageType, }, cache::{CacheManager, CacheManagerBuilder}, @@ -285,7 +285,7 @@ impl AquaBackend { continue; } } - let pkg = pkg.clone().with_version(&[version]); + let pkg = pkg.clone().with_version(&[version], os(), arch()); if let Some(prefix) = &pkg.version_prefix { if let Some(_v) = version.strip_prefix(prefix) { version = _v; @@ -315,13 +315,13 @@ impl AquaBackend { AquaPackageType::GithubArchive | AquaPackageType::GithubContent => { Ok((self.github_archive_url(pkg, v), false)) } - AquaPackageType::Http => pkg.url(v).map(|url| (url, false)), + AquaPackageType::Http => pkg.url(v, os(), arch()).map(|url| (url, false)), ref t => bail!("unsupported aqua package type: {t}"), } } async fn github_release_url(&self, pkg: &AquaPackage, v: &str) -> Result { - let asset_strs = pkg.asset_strs(v)?; + let asset_strs = pkg.asset_strs(v, os(), arch())?; self.github_release_asset(pkg, v, asset_strs).await } @@ -394,10 +394,10 @@ impl AquaBackend { if checksum.enabled() { let url = match checksum._type() { AquaChecksumType::GithubRelease => { - let asset_strs = checksum.asset_strs(pkg, v)?; + let asset_strs = checksum.asset_strs(pkg, v, os(), arch())?; self.github_release_asset(pkg, v, asset_strs).await? } - AquaChecksumType::Http => checksum.url(pkg, v)?, + AquaChecksumType::Http => checksum.url(pkg, v, os(), arch())?, }; let checksum_path = download_path.join(format!("{filename}.checksum")); HTTP.download_file(&url, &checksum_path, Some(&ctx.pr)) @@ -485,7 +485,7 @@ impl AquaBackend { debug!("minisign: {:?}", minisign); let sig_path = match minisign._type() { AquaMinisignType::GithubRelease => { - let asset = minisign.asset(pkg, v)?; + let asset = minisign.asset(pkg, v, os(), arch())?; let repo_owner = minisign .repo_owner .clone() @@ -510,7 +510,7 @@ impl AquaBackend { } } AquaMinisignType::Http => { - let url = minisign.url(pkg, v)?; + let url = minisign.url(pkg, v, os(), arch())?; let path = tv.download_path().join(filename).with_extension(".minisig"); HTTP.download_file(&url, &path, Some(&ctx.pr)).await?; path @@ -518,7 +518,7 @@ impl AquaBackend { }; let data = file::read(tv.download_path().join(filename))?; let sig = file::read_to_string(sig_path)?; - minisign::verify(&minisign.public_key(pkg, v)?, &data, &sig)?; + minisign::verify(&minisign.public_key(pkg, v, os(), arch())?, &data, &sig)?; } Ok(()) } @@ -552,7 +552,7 @@ impl AquaBackend { let repo = format!("{repo_owner}/{repo_name}"); let provenance_path = match slsa.r#type.as_deref().unwrap_or_default() { "github_release" => { - let asset = slsa.asset(pkg, v)?; + let asset = slsa.asset(pkg, v, os(), arch())?; let url = github::get_release(&repo, v) .await? .assets @@ -569,7 +569,7 @@ impl AquaBackend { } } "http" => { - let url = slsa.url(pkg, v)?; + let url = slsa.url(pkg, v, os(), arch())?; let path = tv.download_path().join(filename); HTTP.download_file(&url, &path, Some(&ctx.pr)).await?; path.to_string_lossy().to_string() @@ -635,25 +635,25 @@ impl AquaBackend { cmd = cmd.env("COSIGN_EXPERIMENTAL", "1"); } if let Some(signature) = &cosign.signature { - let arg = signature.arg(pkg, v)?; + let arg = signature.arg(pkg, v, os(), arch())?; if !arg.is_empty() { cmd = cmd.arg("--signature").arg(arg); } } if let Some(key) = &cosign.key { - let arg = key.arg(pkg, v)?; + let arg = key.arg(pkg, v, os(), arch())?; if !arg.is_empty() { cmd = cmd.arg("--key").arg(arg); } } if let Some(certificate) = &cosign.certificate { - let arg = certificate.arg(pkg, v)?; + let arg = certificate.arg(pkg, v, os(), arch())?; if !arg.is_empty() { cmd = cmd.arg("--certificate").arg(arg); } } if let Some(bundle) = &cosign.bundle { - let url = bundle.arg(pkg, v)?; + let url = bundle.arg(pkg, v, os(), arch())?; if !url.is_empty() { let filename = url.split('/').next_back().unwrap(); let bundle_path = download_path.join(filename); @@ -662,7 +662,7 @@ impl AquaBackend { cmd = cmd.arg("--bundle").arg(bundle_path); } } - for opt in cosign.opts(pkg, v)? { + for opt in cosign.opts(pkg, v, os(), arch())? { cmd = cmd.arg(opt); } for arg in Settings::get() @@ -694,11 +694,11 @@ impl AquaBackend { ctx.pr.set_message(format!("extract {filename}")); let install_path = tv.install_path(); file::remove_all(&install_path)?; - let format = pkg.format(v)?; + let format = pkg.format(v, os(), arch())?; let mut bin_names: Vec> = pkg .files .iter() - .filter_map(|file| match file.src(pkg, v) { + .filter_map(|file| match file.src(pkg, v, os(), arch()) { Ok(Some(s)) => Some(Cow::Owned(s)), Ok(None) => Some(Cow::Borrowed(file.name.as_str())), Err(_) => None, @@ -802,11 +802,11 @@ impl AquaBackend { .iter() .map(|f| { let srcs = if let Some(prefix) = &pkg.version_prefix { - vec![f.src(pkg, &format!("{}{}", prefix, tv.version))?] + vec![f.src(pkg, &format!("{}{}", prefix, tv.version), os(), arch())?] } else { vec![ - f.src(pkg, &tv.version)?, - f.src(pkg, &format!("v{}", tv.version))?, + f.src(pkg, &tv.version, os(), arch())?, + f.src(pkg, &format!("v{}", tv.version), os(), arch())?, ] }; Ok(srcs diff --git a/src/cli/doctor/mod.rs b/src/cli/doctor/mod.rs index 2e6c3a67c0..b4ea900ff7 100644 --- a/src/cli/doctor/mod.rs +++ b/src/cli/doctor/mod.rs @@ -545,7 +545,7 @@ fn shell() -> String { } fn aqua_registry_count() -> usize { - crate::aqua::aqua_registry::AQUA_STANDARD_REGISTRY_FILES.len() + aqua_registry::AQUA_STANDARD_REGISTRY_FILES.len() } fn aqua_registry_count_str() -> String { diff --git a/xtasks/release-plz b/xtasks/release-plz index c9a69de7de..83407338bb 100755 --- a/xtasks/release-plz +++ b/xtasks/release-plz @@ -19,6 +19,7 @@ if [[ $cur_version != "$latest_version" ]]; then cargo set-version "$cur_version" --workspace cargo publish --allow-dirty -p vfox cargo add "vfox@$cur_version" + # aqua-registry is an internal crate and should not be published cargo publish --allow-dirty -p mise changelog="$(git cliff --tag "v$cur_version" --strip all --unreleased)" changelog="$(echo "$changelog" | tail -n +3)"