diff --git a/.gitattributes b/.gitattributes index a3e5c95107..7bdc8b68b8 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,5 +1,5 @@ zipsign.pub binary -crates/aqua-registry/aqua-registry/** linguist-vendored +vendor/aqua-registry/** linguist-vendored crates/vfox/embedded-plugins/** linguist-vendored schema/mise-task.json linguist-generated=true schema/miserc.json linguist-generated=true diff --git a/.github/workflows/vendored-file-warning.yml b/.github/workflows/vendored-file-warning.yml index a0d4009dec..4077a092cc 100644 --- a/.github/workflows/vendored-file-warning.yml +++ b/.github/workflows/vendored-file-warning.yml @@ -3,7 +3,7 @@ name: vendored-file-warning on: pull_request: paths: - - "crates/aqua-registry/aqua-registry/**" + - "vendor/aqua-registry/**" - "crates/vfox/embedded-plugins/**" permissions: {} @@ -16,7 +16,7 @@ jobs: - name: Fail on vendored file changes run: | cat <<'EOF' - The aqua registry files under `crates/aqua-registry/aqua-registry/` are vendored + The aqua registry files under `vendor/aqua-registry/` are vendored from the upstream aqua-registry (https://github.com/aquaproj/aqua-registry) and should not be modified directly in this repo. Please submit package definition changes to the upstream aqua-registry instead, and they will be picked up here diff --git a/Cargo.lock b/Cargo.lock index d9f7def40f..d2c2d2c6bb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -248,6 +248,7 @@ dependencies = [ "indexmap 2.14.0", "itertools 0.14.0", "log", + "rkyv", "serde", "serde_yaml", "strum 0.28.0", @@ -1207,6 +1208,29 @@ version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" +[[package]] +name = "bytecheck" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0caa33a2c0edca0419d15ac723dff03f1956f7978329b1e3b5fdaaaed9d3ca8b" +dependencies = [ + "bytecheck_derive", + "ptr_meta", + "rancor", + "simdutf8", +] + +[[package]] +name = "bytecheck_derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89385e82b5d1821d2219e0b095efa2cc1f246cbf99080f3be46a1a85c0d392d9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "bytecount" version = "0.6.9" @@ -5880,6 +5904,26 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" +[[package]] +name = "munge" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e17401f259eba956ca16491461b6e8f72913a0a114e39736ce404410f915a0c" +dependencies = [ + "munge_macro", +] + +[[package]] +name = "munge_macro" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4568f25ccbd45ab5d5603dc34318c1ec56b117531781260002151b8530a9f931" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "native-tls" version = "0.2.18" @@ -6997,6 +7041,26 @@ dependencies = [ "prost", ] +[[package]] +name = "ptr_meta" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b9a0cf95a1196af61d4f1cbdab967179516d9a4a4312af1f31948f8f6224a79" +dependencies = [ + "ptr_meta_derive", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "purl" version = "0.1.6" @@ -7125,6 +7189,15 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" +[[package]] +name = "rancor" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a063ea72381527c2a0561da9c80000ef822bdd7c3241b1cc1b12100e3df081ee" +dependencies = [ + "ptr_meta", +] + [[package]] name = "rand" version = "0.8.6" @@ -7719,6 +7792,15 @@ version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +[[package]] +name = "rend" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cadadef317c2f20755a64d7fdc48f9e7178ee6b0e1f7fce33fa60f1d68a276e6" +dependencies = [ + "bytecheck", +] + [[package]] name = "reqwest" version = "0.12.28" @@ -7863,6 +7945,36 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rkyv" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73389e0c99e664f919275ab5b5b0471391fe9a8de61e1dff9b1eaf56a90f16e3" +dependencies = [ + "bytecheck", + "bytes", + "hashbrown 0.17.0", + "indexmap 2.14.0", + "munge", + "ptr_meta", + "rancor", + "rend", + "rkyv_derive", + "tinyvec", + "uuid", +] + +[[package]] +name = "rkyv_derive" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d2ed0b54125315fb36bd021e82d314d1c126548f871634b483f46b31d13cac6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "rmcp" version = "1.6.0" diff --git a/Cargo.toml b/Cargo.toml index 159712d0c2..65ca6ec25a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,6 +32,9 @@ include = [ "/src/**/*.rs", "/src/assets/**", "/src/plugins/core/assets/**", + "/vendor/aqua-registry/LICENSE", + "/vendor/aqua-registry/metadata.json", + "/vendor/aqua-registry/registry.yml", ] build = "build.rs" rust-version = "1.91" @@ -226,13 +229,15 @@ sevenz-rust = "0.6" winapi = { version = "0.3.9", features = ["consoleapi", "minwindef"] } [build-dependencies] +aqua-registry = { path = "crates/aqua-registry" } built = { version = "0.8", features = ["chrono"] } cfg_aliases = "0.2" +eyre = "0.6" heck = "0.5" -toml = "1.0" indexmap = "2" serde = "1" serde_yaml = "0.9" +toml = "1.0" [dev-dependencies] clap-sort = "1" diff --git a/build.rs b/build.rs index 77aa6f08d0..562b22866b 100644 --- a/build.rs +++ b/build.rs @@ -1,20 +1,35 @@ use heck::ToUpperCamelCase; use indexmap::IndexMap; use serde::Serialize as _; -use std::path::Path; +use std::collections::{HashMap, HashSet}; +use std::path::{Path, PathBuf}; use std::{env, fs}; -fn main() { +use aqua_registry::encode_package_rkyv; +use aqua_registry::types::{AquaPackage, RegistryPackageRow, RegistryYaml}; +use eyre::{Result, eyre}; +use serde_yaml::Value; + +fn main() -> Result<()> { cfg_aliases::cfg_aliases! { asdf: { any(feature = "asdf", not(target_os = "windows")) }, macos: { target_os = "macos" }, linux: { target_os = "linux" }, vfox: { any(feature = "vfox", target_os = "windows") }, } - built::write_built_file().expect("Failed to acquire build-time information"); + built::write_built_file()?; codegen_settings(); codegen_registry(); + codegen_aqua_standard_registry()?; + Ok(()) +} + +#[derive(Debug)] +struct AquaPackageRegistry { + id: String, + content: Vec, + aliases: Vec, } /// Generate a raw string literal that safely contains the given content. @@ -305,6 +320,172 @@ fn codegen_registry() { fs::write(&dest_path, lines.join("\n")).unwrap(); } +fn codegen_aqua_standard_registry() -> Result<()> { + let out_dir = env::var("OUT_DIR")?; + let files_dest_path = Path::new(&out_dir).join("aqua_standard_registry_files.rs"); + let aliases_dest_path = Path::new(&out_dir).join("aqua_standard_registry_aliases.rs"); + let metadata_dest_path = Path::new(&out_dir).join("aqua_standard_registry_metadata.rs"); + let packages_dir = Path::new(&out_dir).join("aqua_standard_registry_packages"); + + let registry_file = Path::new("vendor/aqua-registry/registry.yml"); + let metadata_file = Path::new("vendor/aqua-registry/metadata.json"); + + println!("cargo:rerun-if-changed={}", registry_file.display()); + println!("cargo:rerun-if-changed={}", metadata_file.display()); + + let registry_yaml = serde_yaml::from_str::(&fs::read_to_string(registry_file)?)?; + + let registries = aqua_package_registries(®istry_yaml.packages)?; + if registries.is_empty() { + return Err(eyre!( + "Aqua registry file {} contains no packages", + registry_file.display() + )); + } + + fs::create_dir_all(&packages_dir)?; + fs::write( + files_dest_path, + aqua_registry_files_code(®istries, &packages_dir)?, + )?; + fs::write(aliases_dest_path, aqua_registry_aliases_code(®istries))?; + + let metadata = serde_yaml::from_str::(&fs::read_to_string(metadata_file)?)?; + let repository = yaml_string_field(&metadata, "repository").ok_or_else(|| { + eyre!( + "Aqua registry metadata file {} does not contain a repository", + metadata_file.display() + ) + })?; + let tag = yaml_string_field(&metadata, "tag").ok_or_else(|| { + eyre!( + "Aqua registry metadata file {} does not contain a tag", + metadata_file.display() + ) + })?; + + fs::write( + metadata_dest_path, + format!("AquaRegistryMetadata {{ repository: {repository:?}, tag: {tag:?} }}"), + )?; + + Ok(()) +} + +fn aqua_package_registries(rows: &[RegistryPackageRow]) -> Result> { + let mut registries = Vec::new(); + for row in rows { + let package = &row.package; + let Some(id) = aqua_canonical_package_id(package) else { + continue; + }; + let content = encode_package_rkyv(package)?; + registries.push(AquaPackageRegistry { + id, + content, + aliases: row.aliases.clone(), + }); + } + Ok(registries) +} + +fn aqua_registry_files_code( + registries: &[AquaPackageRegistry], + packages_dir: &Path, +) -> Result { + let mut used_stems = HashMap::new(); + let mut entries = registries + .iter() + .map(|registry| { + let stem = aqua_package_file_stem(®istry.id); + if let Some(other_id) = used_stems.insert(stem.clone(), registry.id.as_str()) { + return Err(eyre!( + "baked aqua registry package filename collision for {other_id:?} and {:?}: {stem}", + registry.id + )); + } + let filename = format!("{stem}.rkyv"); + let path = packages_dir.join(filename); + fs::write(&path, ®istry.content)?; + Ok((registry.id.clone(), path)) + }) + .collect::>>()?; + entries.sort_by(|a, b| a.0.cmp(&b.0)); + + Ok(aqua_registry_bytes_map_code(&entries)) +} + +fn aqua_registry_bytes_map_code(entries: &[(String, PathBuf)]) -> String { + let mut code = String::from("HashMap::from([\n"); + for (key, path) in entries { + code.push_str(&format!( + " ({key:?}, include_bytes!({:?}).as_slice()),\n", + path.display().to_string() + )); + } + code.push_str("])"); + code +} + +/// Hashes the canonical package ID with FNV-1a 64-bit to generate compact, +/// deterministic baked package filenames without leaking path separators. +fn aqua_package_file_stem(id: &str) -> String { + let mut hash = 0xcbf29ce484222325u64; + for byte in id.as_bytes() { + hash ^= u64::from(*byte); + hash = hash.wrapping_mul(0x100000001b3); + } + format!("{hash:016x}") +} + +fn aqua_registry_aliases_code(registries: &[AquaPackageRegistry]) -> String { + let canonical_ids = registries + .iter() + .map(|registry| registry.id.as_str()) + .collect::>(); + let mut aliases = HashMap::new(); + + for registry in registries { + for alias in ®istry.aliases { + if alias != ®istry.id && !canonical_ids.contains(alias.as_str()) { + aliases.insert(alias.clone(), registry.id.clone()); + } + } + } + + let mut entries = aliases.into_iter().collect::>(); + entries.sort_by(|a, b| a.0.cmp(&b.0)); + + aqua_registry_string_map_code(&entries) +} + +fn aqua_registry_string_map_code(entries: &[(String, String)]) -> String { + let mut code = String::from("HashMap::from([\n"); + for (key, value) in entries { + code.push_str(&format!(" ({key:?}, {value:?}),\n")); + } + code.push_str("])"); + code +} + +fn aqua_canonical_package_id(package: &AquaPackage) -> Option { + package + .name + .clone() + .or_else(|| { + if package.repo_owner.is_empty() || package.repo_name.is_empty() { + None + } else { + Some(format!("{}/{}", package.repo_owner, package.repo_name)) + } + }) + .or_else(|| package.path.clone()) +} + +fn yaml_string_field(value: &Value, key: &str) -> Option { + value.get(key)?.as_str().map(str::to_string) +} + fn codegen_settings() { let out_dir = env::var_os("OUT_DIR").unwrap(); let dest_path = Path::new(&out_dir).join("settings.rs"); diff --git a/crates/aqua-registry/Cargo.toml b/crates/aqua-registry/Cargo.toml index 8a4f644276..67d30923ec 100644 --- a/crates/aqua-registry/Cargo.toml +++ b/crates/aqua-registry/Cargo.toml @@ -10,15 +10,7 @@ homepage = "https://mise.en.dev" readme = "README.md" keywords = ["mise", "aqua", "registry", "package-manager"] categories = ["development-tools"] -build = "build.rs" -include = [ - "/README.md", - "/build.rs", - "/src/**/*.rs", - "/aqua-registry/LICENSE", - "/aqua-registry/metadata.json", - "/aqua-registry/registry.yaml", -] +include = ["/README.md", "/src/**/*.rs"] [package.metadata.cargo-machete] ignored = ["serde"] @@ -34,12 +26,13 @@ thiserror = "2" eyre = "0.6" indexmap = { version = "2", features = ["serde"] } itertools = "0.14" +rkyv = { version = "0.8", features = ["unaligned"] } strum = { version = "0.28", features = ["derive"] } # Template parsing and evaluation expr-lang = "1" -versions = { version = "7", features = ["serde"] } heck = "0.5" +versions = { version = "7", features = ["serde"] } # Async runtime tokio = { version = "1", features = ["sync"] } @@ -50,7 +43,3 @@ log = "0.4" [dev-dependencies] tokio = { version = "1", features = ["rt", "macros"] } - -[build-dependencies] -eyre = "0.6" -serde_yaml = "0.9" diff --git a/crates/aqua-registry/build.rs b/crates/aqua-registry/build.rs deleted file mode 100644 index f4197c115b..0000000000 --- a/crates/aqua-registry/build.rs +++ /dev/null @@ -1,219 +0,0 @@ -use std::collections::{HashMap, HashSet}; -use std::env; -use std::fs; -use std::path::{Path, PathBuf}; - -use eyre::{Result, WrapErr, eyre}; -use serde_yaml::Value; - -fn main() -> Result<()> { - let out_dir = env::var("OUT_DIR").wrap_err("OUT_DIR environment variable must be set")?; - generate_baked_registry(&out_dir)?; - generate_registry_metadata(&out_dir)?; - Ok(()) -} - -#[derive(Debug)] -struct PackageRegistry { - id: String, - content: String, - aliases: Vec, -} - -fn generate_baked_registry(out_dir: &str) -> Result<()> { - let files_dest_path = Path::new(out_dir).join("aqua_standard_registry_files.rs"); - let aliases_dest_path = Path::new(out_dir).join("aqua_standard_registry_aliases.rs"); - - let registry_file = find_registry_file()?; - - println!("cargo:rerun-if-changed={}", registry_file.display()); - - let content = fs::read_to_string(®istry_file).wrap_err_with(|| { - format!( - "Failed to read aqua registry file {}", - registry_file.display() - ) - })?; - - let registry = serde_yaml::from_str::(&content).wrap_err_with(|| { - format!( - "Failed to parse aqua registry file {}", - registry_file.display() - ) - })?; - let packages = registry - .get("packages") - .and_then(|packages| packages.as_sequence()) - .ok_or_else(|| { - eyre!( - "Aqua registry file {} does not contain a packages list", - registry_file.display() - ) - })?; - let registries = package_registries(packages)?; - if registries.is_empty() { - return Err(eyre!( - "Aqua registry file {} contains no packages", - registry_file.display() - )); - } - - fs::write(files_dest_path, registry_files_code(®istries)) - .wrap_err("Failed to write baked registry files")?; - fs::write(aliases_dest_path, registry_aliases_code(®istries)) - .wrap_err("Failed to write baked registry aliases")?; - Ok(()) -} - -fn generate_registry_metadata(out_dir: &str) -> Result<()> { - let metadata_dest_path = Path::new(out_dir).join("aqua_standard_registry_metadata.rs"); - let metadata_file = find_registry_metadata_file()?; - - println!("cargo:rerun-if-changed={}", metadata_file.display()); - - let content = fs::read_to_string(&metadata_file).wrap_err_with(|| { - format!( - "Failed to read aqua registry metadata file {}", - metadata_file.display() - ) - })?; - let metadata = serde_yaml::from_str::(&content).wrap_err_with(|| { - format!( - "Failed to parse aqua registry metadata file {}", - metadata_file.display() - ) - })?; - let repository = string_field(&metadata, "repository").ok_or_else(|| { - eyre!( - "Aqua registry metadata file {} does not contain a repository", - metadata_file.display() - ) - })?; - let tag = string_field(&metadata, "tag").ok_or_else(|| { - eyre!( - "Aqua registry metadata file {} does not contain a tag", - metadata_file.display() - ) - })?; - - fs::write( - metadata_dest_path, - format!("AquaRegistryMetadata {{ repository: {repository:?}, tag: {tag:?} }}"), - ) - .wrap_err("Failed to write baked registry metadata")?; - Ok(()) -} - -fn package_registries(packages: &[Value]) -> Result> { - let mut registries = Vec::new(); - for package in packages { - let Some(id) = canonical_package_id(package) else { - continue; - }; - let content = package_registry_yaml(package)?; - let aliases = package_aliases(package); - registries.push(PackageRegistry { - id, - content, - aliases, - }); - } - Ok(registries) -} - -fn registry_files_code(registries: &[PackageRegistry]) -> String { - let mut entries = registries - .iter() - .map(|registry| (registry.id.clone(), registry.content.clone())) - .collect::>(); - entries.sort_by(|a, b| a.0.cmp(&b.0)); - - registry_map_code(&entries) -} - -fn registry_aliases_code(registries: &[PackageRegistry]) -> String { - let canonical_ids = registries - .iter() - .map(|registry| registry.id.as_str()) - .collect::>(); - let mut aliases = HashMap::new(); - - for registry in registries { - for alias in ®istry.aliases { - if alias != ®istry.id && !canonical_ids.contains(alias.as_str()) { - aliases.insert(alias.clone(), registry.id.clone()); - } - } - } - - let mut entries = aliases.into_iter().collect::>(); - entries.sort_by(|a, b| a.0.cmp(&b.0)); - - registry_map_code(&entries) -} - -fn registry_map_code(entries: &[(String, String)]) -> String { - let mut code = String::from("HashMap::from([\n"); - for (key, value) in entries { - code.push_str(&format!(" ({key:?}, {value:?}),\n")); - } - code.push_str("])"); - code -} - -fn package_registry_yaml(package: &Value) -> Result { - let mut registry = serde_yaml::Mapping::new(); - registry.insert( - Value::String("packages".to_string()), - Value::Sequence(vec![package.clone()]), - ); - serde_yaml::to_string(&Value::Mapping(registry)) - .wrap_err("Failed to serialize aqua package registry") -} - -fn canonical_package_id(package: &Value) -> Option { - string_field(package, "name") - .or_else(|| { - let repo_owner = string_field(package, "repo_owner")?; - let repo_name = string_field(package, "repo_name")?; - Some(format!("{repo_owner}/{repo_name}")) - }) - .or_else(|| string_field(package, "path")) -} - -fn package_aliases(package: &Value) -> Vec { - package - .get("aliases") - .and_then(|aliases| aliases.as_sequence()) - .map(|aliases| { - aliases - .iter() - .filter_map(|alias| string_field(alias, "name")) - .collect() - }) - .unwrap_or_default() -} - -fn string_field(value: &Value, key: &str) -> Option { - value.get(key)?.as_str().map(str::to_string) -} - -fn find_registry_file() -> Result { - registry_file("registry.yaml", "Registry file") -} - -fn find_registry_metadata_file() -> Result { - registry_file("metadata.json", "Registry metadata file") -} - -fn registry_file(file_name: &str, description: &str) -> Result { - let manifest_dir = env::var("CARGO_MANIFEST_DIR") - .wrap_err("CARGO_MANIFEST_DIR environment variable must be set")?; - let embedded = Path::new(&manifest_dir) - .join("aqua-registry") - .join(file_name); - if embedded.exists() { - return Ok(embedded); - } - Err(eyre!("{description} not found at {}", embedded.display())) -} diff --git a/crates/aqua-registry/src/codec.rs b/crates/aqua-registry/src/codec.rs new file mode 100644 index 0000000000..9dae75df5e --- /dev/null +++ b/crates/aqua-registry/src/codec.rs @@ -0,0 +1,46 @@ +use crate::types::AquaPackage; +use crate::{AquaRegistryError, Result}; +use rkyv::rancor::Error as RkyvError; + +pub fn encode_package_rkyv(package: &AquaPackage) -> Result> { + rkyv::to_bytes::(package) + .map(|bytes| bytes.to_vec()) + .map_err(|err| { + AquaRegistryError::RegistryNotAvailable(format!( + "failed to encode aqua package as rkyv: {err}" + )) + }) +} + +pub fn decode_package_rkyv(package_id: &str, bytes: &[u8]) -> Result { + rkyv::from_bytes::(bytes).map_err(|err| { + AquaRegistryError::RegistryNotAvailable(format!( + "failed to decode aqua package {package_id} from rkyv: {err}" + )) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::AquaVar; + + #[test] + fn test_rkyv_package_roundtrip_preserves_var_default() { + let mut package = AquaPackage::default(); + package.repo_owner = "owner".into(); + package.repo_name = "repo".into(); + package.vars = vec![AquaVar { + name: "channel".into(), + default: Some("beta".into()), + required: false, + }]; + + let bytes = encode_package_rkyv(&package).unwrap(); + let decoded = decode_package_rkyv("owner/repo", &bytes).unwrap(); + + assert_eq!(decoded.repo_owner, "owner"); + assert_eq!(decoded.repo_name, "repo"); + assert_eq!(decoded.vars[0].default.as_deref(), Some("beta")); + } +} diff --git a/crates/aqua-registry/src/lib.rs b/crates/aqua-registry/src/lib.rs index 9595150bd0..0616bb404e 100644 --- a/crates/aqua-registry/src/lib.rs +++ b/crates/aqua-registry/src/lib.rs @@ -3,15 +3,14 @@ //! This crate provides functionality for working with Aqua package registry files. //! It can load registry data from baked-in files, local repositories, or remote HTTP sources. +mod codec; mod registry; mod template; -mod types; +pub mod types; // Re-export only what's needed by the main mise crate -pub use registry::{ - AQUA_STANDARD_REGISTRY_FILES, AQUA_STANDARD_REGISTRY_METADATA, AquaRegistry, - AquaRegistryMetadata, DefaultRegistryFetcher, FileCacheStore, NoOpCacheStore, package_ids, -}; +pub use codec::{decode_package_rkyv, encode_package_rkyv}; +pub use registry::{AquaRegistry, DefaultRegistryFetcher, FileCacheStore, NoOpCacheStore}; pub use types::{ AquaChecksum, AquaChecksumType, AquaCosign, AquaFile, AquaMinisignType, AquaPackage, AquaPackageType, AquaVar, RegistryYaml, @@ -62,11 +61,11 @@ impl Default for AquaRegistryConfig { } } -/// Trait for fetching registry files from various sources +/// Trait for fetching aqua packages from various sources #[allow(async_fn_in_trait)] pub trait RegistryFetcher { - /// Fetch and parse a registry YAML file for the given package ID - async fn fetch_registry(&self, package_id: &str) -> Result; + /// Fetch and parse a package definition for the given package ID. + async fn fetch_package(&self, package_id: &str) -> Result; } /// Trait for caching registry data diff --git a/crates/aqua-registry/src/registry.rs b/crates/aqua-registry/src/registry.rs index 7f01c66c1d..475a18401d 100644 --- a/crates/aqua-registry/src/registry.rs +++ b/crates/aqua-registry/src/registry.rs @@ -37,48 +37,6 @@ pub struct FileCacheStore { cache_dir: PathBuf, } -/// Metadata for the baked aqua registry snapshot. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct AquaRegistryMetadata { - pub repository: &'static str, - pub tag: &'static str, -} - -/// Baked canonical registry files (compiled into binary). -pub static AQUA_STANDARD_REGISTRY_FILES: LazyLock> = - LazyLock::new(|| include!(concat!(env!("OUT_DIR"), "/aqua_standard_registry_files.rs"))); - -/// Baked aqua registry snapshot metadata (compiled into binary). -pub static AQUA_STANDARD_REGISTRY_METADATA: AquaRegistryMetadata = include!(concat!( - env!("OUT_DIR"), - "/aqua_standard_registry_metadata.rs" -)); - -/// Baked alias-to-canonical package ID map (compiled into binary). -static AQUA_STANDARD_REGISTRY_ALIASES: LazyLock> = - LazyLock::new(|| { - include!(concat!( - env!("OUT_DIR"), - "/aqua_standard_registry_aliases.rs" - )) - }); - -/// Returns all package IDs from the baked-in aqua registry. -pub fn package_ids() -> Vec<&'static str> { - AQUA_STANDARD_REGISTRY_FILES.keys().copied().collect() -} - -fn baked_registry_file(package_id: &str) -> Option<&'static str> { - if let Some(content) = AQUA_STANDARD_REGISTRY_FILES.get(package_id) { - return Some(*content); - } - - AQUA_STANDARD_REGISTRY_ALIASES - .get(package_id) - .and_then(|canonical| AQUA_STANDARD_REGISTRY_FILES.get(*canonical)) - .copied() -} - impl AquaRegistry { /// Create a new AquaRegistry with the given configuration pub fn new(config: AquaRegistryConfig) -> Self { @@ -132,12 +90,7 @@ where return Ok(pkg.clone()); } - let registry = self.fetcher.fetch_registry(id).await?; - let mut pkg = registry - .packages - .into_iter() - .next() - .ok_or_else(|| AquaRegistryError::PackageNotFound(id.to_string()))?; + let mut pkg = self.fetcher.fetch_package(id).await?; pkg.setup_version_filter()?; CACHE.lock().await.insert(id.to_string(), pkg.clone()); @@ -146,7 +99,7 @@ where } impl RegistryFetcher for DefaultRegistryFetcher { - async fn fetch_registry(&self, package_id: &str) -> Result { + async fn fetch_package(&self, package_id: &str) -> Result { let path_id = package_id .split('/') .collect::>() @@ -162,15 +115,13 @@ impl RegistryFetcher for DefaultRegistryFetcher { if self.config.cache_dir.join(".git").exists() && path.exists() { log::trace!("reading aqua-registry for {package_id} from repo at {path:?}"); let contents = std::fs::read_to_string(&path)?; - return Ok(serde_yaml::from_str(&contents)?); - } - - // Fall back to baked registry if enabled - if self.config.use_baked_registry - && let Some(content) = baked_registry_file(package_id) - { - log::trace!("reading baked-in aqua-registry for {package_id}"); - return Ok(serde_yaml::from_str(content)?); + let registry = serde_yaml::from_str::(&contents)?; + return registry + .packages + .into_iter() + .next() + .map(|row| row.package) + .ok_or_else(|| AquaRegistryError::PackageNotFound(package_id.to_string())); } Err(AquaRegistryError::RegistryNotAvailable(format!( @@ -252,53 +203,4 @@ mod tests { assert!(cache.store("test", b"data").is_ok()); assert!(cache.retrieve("test").unwrap().is_none()); } - - #[test] - fn test_baked_registry_package_lookup() { - let registry = baked_registry_file("01mf02/jaq").unwrap(); - let registry = serde_yaml::from_str::(registry).unwrap(); - - let package = registry.packages.into_iter().next().unwrap(); - assert_eq!(package.repo_owner, "01mf02"); - assert_eq!(package.repo_name, "jaq"); - } - - #[test] - fn test_baked_registry_path_only_package_lookup() { - let registry = baked_registry_file("golang.org/x/perf/cmd/benchstat").unwrap(); - let registry = serde_yaml::from_str::(registry).unwrap(); - - let package = registry.packages.into_iter().next().unwrap(); - assert_eq!( - package.path.as_deref(), - Some("golang.org/x/perf/cmd/benchstat") - ); - } - - #[test] - fn test_baked_registry_metadata() { - assert_eq!( - AQUA_STANDARD_REGISTRY_METADATA.repository, - "aquaproj/aqua-registry" - ); - assert!(!AQUA_STANDARD_REGISTRY_METADATA.tag.is_empty()); - assert!(AQUA_STANDARD_REGISTRY_METADATA.tag.starts_with('v')); - } - - #[test] - fn test_baked_registry_alias_lookup() { - let alias = "elijah-potter/harper/harper-ls"; - - assert!(!AQUA_STANDARD_REGISTRY_FILES.contains_key(alias)); - assert_eq!( - AQUA_STANDARD_REGISTRY_ALIASES.get(alias).copied(), - Some("Automattic/harper/harper-ls") - ); - - let registry = baked_registry_file(alias).unwrap(); - let registry = serde_yaml::from_str::(registry).unwrap(); - - let package = registry.packages.into_iter().next().unwrap(); - assert_eq!(package.name.as_deref(), Some("Automattic/harper/harper-ls")); - } } diff --git a/crates/aqua-registry/src/types.rs b/crates/aqua-registry/src/types.rs index b3817eb911..06a5a366cf 100644 --- a/crates/aqua-registry/src/types.rs +++ b/crates/aqua-registry/src/types.rs @@ -2,13 +2,24 @@ use expr::{Context, Environment, Program, Value}; use eyre::{Result, eyre}; use indexmap::IndexSet; use itertools::Itertools; -use serde::Deserialize; +use rkyv::{Archive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize}; +use serde::{Deserialize, Deserializer}; use std::cmp::PartialEq; use std::collections::HashMap; use versions::Versioning; /// Type of Aqua package -#[derive(Debug, Deserialize, Default, Clone, PartialEq, strum::Display)] +#[derive( + Debug, + Deserialize, + Archive, + RkyvDeserialize, + RkyvSerialize, + Default, + Clone, + PartialEq, + strum::Display, +)] #[strum(serialize_all = "snake_case")] #[serde(rename_all = "snake_case")] pub enum AquaPackageType { @@ -18,11 +29,26 @@ pub enum AquaPackageType { GithubRelease, Http, GoInstall, + GoBuild, Cargo, } /// Main Aqua package definition -#[derive(Debug, Deserialize, Clone)] +/// +/// rkyv archives parsed package data only. Runtime-only fields mirror serde's +/// skipped behavior with `rkyv::with::Skip`. +#[derive(Debug, Deserialize, Archive, RkyvDeserialize, RkyvSerialize, Clone)] +#[rkyv(serialize_bounds( + __S: rkyv::ser::Writer + rkyv::ser::Allocator, + __S::Error: rkyv::rancor::Source, +))] +#[rkyv(deserialize_bounds(__D::Error: rkyv::rancor::Source))] +#[rkyv(bytecheck( + bounds( + __C: rkyv::validation::ArchiveContext, + __C::Error: rkyv::rancor::Source, + ) +))] #[serde(default)] pub struct AquaPackage { pub r#type: AquaPackageType, @@ -39,10 +65,12 @@ pub struct AquaPackage { pub supported_envs: Vec, pub files: Vec, pub vars: Vec, + #[serde(default, deserialize_with = "deserialize_string_map")] pub replacements: HashMap, pub version_prefix: Option, version_filter: Option, #[serde(skip)] + #[rkyv(with = rkyv::with::Skip)] version_filter_expr: Option, pub version_source: Option, pub cosign: Option, @@ -50,18 +78,21 @@ pub struct AquaPackage { pub slsa_provenance: Option, pub minisign: Option, pub github_artifact_attestations: Option, + #[rkyv(omit_bounds)] overrides: Vec, version_constraint: String, + #[rkyv(omit_bounds)] pub version_overrides: Vec, pub no_asset: bool, pub error_message: Option, pub path: Option, #[serde(skip)] + #[rkyv(with = rkyv::with::Skip)] var_values: HashMap, } /// Override configuration for specific OS/architecture combinations -#[derive(Debug, Deserialize, Clone)] +#[derive(Debug, Deserialize, Archive, RkyvDeserialize, RkyvSerialize, Clone)] struct AquaOverride { #[serde(flatten)] pkg: AquaPackage, @@ -72,7 +103,7 @@ struct AquaOverride { } /// Runtime variant selector for an override. -#[derive(Debug, Deserialize, Clone)] +#[derive(Debug, Deserialize, Archive, RkyvDeserialize, RkyvSerialize, Clone)] struct AquaVariant { key: String, value: String, @@ -84,19 +115,19 @@ struct AquaRuntime<'a> { } /// Variable definition for Aqua templates -#[derive(Debug, Deserialize, Clone, Default)] +#[derive(Debug, Deserialize, Archive, RkyvDeserialize, RkyvSerialize, Clone, Default)] pub struct AquaVar { pub name: String, /// Aqua's schema allows arbitrary YAML defaults, but mise intentionally /// supports only string defaults to keep variable resolution simple. - #[serde(default)] + #[serde(default, deserialize_with = "deserialize_optional_scalar_string")] pub default: Option, #[serde(default)] pub required: bool, } /// File definition within a package -#[derive(Debug, Deserialize, Clone, Default)] +#[derive(Debug, Deserialize, Archive, RkyvDeserialize, RkyvSerialize, Clone, Default)] pub struct AquaFile { pub name: String, pub src: Option, @@ -106,7 +137,16 @@ pub struct AquaFile { } /// Checksum algorithm options -#[derive(Debug, Deserialize, Clone, strum::AsRefStr, strum::Display)] +#[derive( + Debug, + Deserialize, + Archive, + RkyvDeserialize, + RkyvSerialize, + Clone, + strum::AsRefStr, + strum::Display, +)] #[serde(rename_all = "lowercase")] #[strum(serialize_all = "lowercase")] pub enum AquaChecksumAlgorithm { @@ -117,7 +157,7 @@ pub enum AquaChecksumAlgorithm { } /// Type of checksum source -#[derive(Debug, Deserialize, Clone)] +#[derive(Debug, Deserialize, Archive, RkyvDeserialize, RkyvSerialize, Clone)] #[serde(rename_all = "snake_case")] pub enum AquaChecksumType { GithubRelease, @@ -125,7 +165,7 @@ pub enum AquaChecksumType { } /// Type of minisign source -#[derive(Debug, Deserialize, Clone)] +#[derive(Debug, Deserialize, Archive, RkyvDeserialize, RkyvSerialize, Clone)] #[serde(rename_all = "snake_case")] pub enum AquaMinisignType { GithubRelease, @@ -133,7 +173,7 @@ pub enum AquaMinisignType { } /// Cosign signature configuration -#[derive(Debug, Deserialize, Clone)] +#[derive(Debug, Deserialize, Archive, RkyvDeserialize, RkyvSerialize, Clone)] pub struct AquaCosignSignature { pub r#type: Option, pub repo_owner: Option, @@ -143,7 +183,7 @@ pub struct AquaCosignSignature { } /// Cosign verification configuration -#[derive(Debug, Deserialize, Clone)] +#[derive(Debug, Deserialize, Archive, RkyvDeserialize, RkyvSerialize, Clone)] pub struct AquaCosign { pub enabled: Option, pub signature: Option, @@ -155,7 +195,7 @@ pub struct AquaCosign { } /// SLSA provenance configuration -#[derive(Debug, Deserialize, Clone)] +#[derive(Debug, Deserialize, Archive, RkyvDeserialize, RkyvSerialize, Clone)] pub struct AquaSlsaProvenance { pub enabled: Option, pub r#type: Option, @@ -168,7 +208,7 @@ pub struct AquaSlsaProvenance { } /// Minisign verification configuration -#[derive(Debug, Deserialize, Clone)] +#[derive(Debug, Deserialize, Archive, RkyvDeserialize, RkyvSerialize, Clone)] pub struct AquaMinisign { pub enabled: Option, pub r#type: Option, @@ -180,14 +220,14 @@ pub struct AquaMinisign { } /// GitHub artifact attestations configuration -#[derive(Debug, Deserialize, Clone)] +#[derive(Debug, Deserialize, Archive, RkyvDeserialize, RkyvSerialize, Clone)] pub struct AquaGithubArtifactAttestations { pub enabled: Option, pub signer_workflow: Option, } /// Checksum verification configuration -#[derive(Debug, Deserialize, Clone)] +#[derive(Debug, Deserialize, Archive, RkyvDeserialize, RkyvSerialize, Clone)] pub struct AquaChecksum { pub r#type: Option, pub algorithm: Option, @@ -200,7 +240,7 @@ pub struct AquaChecksum { } /// Checksum pattern configuration -#[derive(Debug, Deserialize, Clone)] +#[derive(Debug, Deserialize, Archive, RkyvDeserialize, RkyvSerialize, Clone)] pub struct AquaChecksumPattern { pub checksum: String, pub file: Option, @@ -209,7 +249,94 @@ pub struct AquaChecksumPattern { /// Registry YAML file structure #[derive(Debug, Deserialize)] pub struct RegistryYaml { - pub packages: Vec, + pub packages: Vec, +} + +/// Top-level package row in a merged aqua registry YAML file. +#[derive(Debug, Deserialize)] +pub struct RegistryPackageRow { + #[serde(flatten)] + pub package: AquaPackage, + #[serde(default, deserialize_with = "deserialize_registry_aliases")] + pub aliases: Vec, +} + +fn deserialize_registry_aliases<'de, D>( + deserializer: D, +) -> std::result::Result, D::Error> +where + D: Deserializer<'de>, +{ + let aliases = Option::::deserialize(deserializer)?; + Ok(aliases + .and_then(|aliases| { + aliases + .as_sequence() + .map(|aliases| aliases.iter().filter_map(registry_alias_name).collect()) + }) + .unwrap_or_default()) +} + +fn registry_alias_name(alias: &serde_yaml::Value) -> Option { + alias.get("name")?.as_str().map(str::to_string) +} + +fn deserialize_optional_scalar_string<'de, D>( + deserializer: D, +) -> std::result::Result, D::Error> +where + D: Deserializer<'de>, +{ + let value = Option::::deserialize(deserializer)?; + match value { + None | Some(serde_yaml::Value::Null) => Ok(None), + Some(value) => yaml_scalar_to_string(value).map(Some).ok_or_else(|| { + ::custom("invalid type: expected a scalar string default") + }), + } +} + +fn deserialize_string_map<'de, D>( + deserializer: D, +) -> std::result::Result, D::Error> +where + D: Deserializer<'de>, +{ + let value = Option::::deserialize(deserializer)?; + let Some(value) = value else { + return Ok(HashMap::new()); + }; + let serde_yaml::Value::Mapping(mapping) = value else { + return Err(::custom( + "invalid type: expected a string map", + )); + }; + + mapping + .into_iter() + .map(|(key, value)| { + let key = yaml_scalar_to_string(key).ok_or_else(|| { + ::custom( + "invalid type: expected a scalar string map key", + ) + })?; + let value = yaml_scalar_to_string(value).ok_or_else(|| { + ::custom( + "invalid type: expected a scalar string map value", + ) + })?; + Ok((key, value)) + }) + .collect() +} + +fn yaml_scalar_to_string(value: serde_yaml::Value) -> Option { + match value { + serde_yaml::Value::String(value) => Some(value), + serde_yaml::Value::Bool(value) => Some(value.to_string()), + serde_yaml::Value::Number(value) => Some(value.to_string()), + _ => None, + } } impl Default for AquaPackage { @@ -1076,6 +1203,54 @@ mod tests { Some(value.to_string()) } + fn first_registry_package(yml: &str) -> AquaPackage { + serde_yaml::from_str::(yml) + .unwrap() + .packages + .into_iter() + .next() + .unwrap() + .package + } + + #[test] + fn test_registry_package_row_aliases_are_top_level_only() { + let yml = r#" +packages: + - name: example/canonical + aliases: + - name: example/alias + - name: 123 + - other: ignored + unsupported_field: ignored + version_overrides: + - aliases: + - name: example/nested-alias +"#; + let registry = serde_yaml::from_str::(yml).unwrap(); + let row = registry.packages.into_iter().next().unwrap(); + + assert_eq!(row.package.name.as_deref(), Some("example/canonical")); + assert_eq!(row.aliases, vec!["example/alias"]); + assert_eq!(row.package.version_overrides.len(), 1); + } + + #[test] + fn test_registry_package_row_preserves_yaml_scalar_coercions() { + let yml = r#" +packages: + - replacements: + 386: i686 + vars: + - name: enabled + default: true +"#; + let pkg = first_registry_package(yml); + + assert_eq!(pkg.replacements.get("386"), Some(&"i686".to_string())); + assert_eq!(pkg.vars[0].default.as_deref(), Some("true")); + } + #[test] fn test_aqua_file_src_gradle() { // Test the gradle package src template: {{.AssetWithoutExt | trimSuffix "-bin"}}/bin/gradle @@ -1266,12 +1441,7 @@ packages: - name: channel default: stable "#; - let pkg = serde_yaml::from_str::(yml) - .unwrap() - .packages - .into_iter() - .next() - .unwrap(); + let pkg = first_registry_package(yml); let asset = pkg.asset("1.0.0", "linux", "amd64").unwrap(); assert_eq!(asset, "tool-stable-1.0.0.tar.gz"); } @@ -1287,16 +1457,23 @@ packages: default: {yaml_default} "# ); - let pkg = serde_yaml::from_str::(&yml) - .unwrap() - .packages - .into_iter() - .next() - .unwrap(); + let pkg = first_registry_package(&yml); assert_eq!(pkg.vars[0].default.as_deref(), Some(expected)); } } + #[test] + fn test_vars_null_default_deserializes_as_none() { + let yml = r#" +packages: + - vars: + - name: channel + default: null +"#; + let pkg = first_registry_package(yml); + assert_eq!(pkg.vars[0].default, None); + } + #[test] fn test_vars_sequence_and_mapping_defaults_fail_yaml_parse() { for yaml_default in ["[stable, beta]", "{channel: stable}"] { @@ -1316,24 +1493,6 @@ packages: } } - #[test] - fn test_vars_null_default_deserializes_as_none() { - let yml = r#" -packages: - - vars: - - name: channel - default: null -"#; - let pkg = serde_yaml::from_str::(yml) - .unwrap() - .packages - .into_iter() - .next() - .unwrap(); - - assert_eq!(pkg.vars[0].default, None); - } - #[test] fn test_vars_required_missing() { let pkg = AquaPackage { @@ -1397,12 +1556,7 @@ packages: type: github_release asset: "{{.Asset}}.sigstore.json" "#; - let pkg = serde_yaml::from_str::(yml) - .unwrap() - .packages - .into_iter() - .next() - .unwrap(); + let pkg = first_registry_package(yml); assert!(pkg.cosign.is_some()); assert!(pkg.checksum.is_none()); } @@ -1425,13 +1579,7 @@ packages: type: github_release asset: cosign.pub "#; - let pkg = serde_yaml::from_str::(yml) - .unwrap() - .packages - .into_iter() - .next() - .unwrap() - .with_version(&["v1.0.0"], "linux", "amd64"); + let pkg = first_registry_package(yml).with_version(&["v1.0.0"], "linux", "amd64"); let cosign = pkg.cosign.unwrap(); assert!(cosign.bundle.is_some()); assert!(cosign.key.is_some()); @@ -1454,12 +1602,7 @@ packages: - key: libc value: musl "#; - let pkg = serde_yaml::from_str::(yml) - .unwrap() - .packages - .into_iter() - .next() - .unwrap(); + let pkg = first_registry_package(yml); let gnu = pkg .clone() @@ -1494,13 +1637,12 @@ packages: - key: libc value: musl "#; - let pkg = serde_yaml::from_str::(yml) - .unwrap() - .packages - .into_iter() - .next() - .unwrap() - .with_version_libc(&["1.0.0"], "linux", "amd64", Some("musl")); + let pkg = first_registry_package(yml).with_version_libc( + &["1.0.0"], + "linux", + "amd64", + Some("musl"), + ); assert_eq!( pkg.url("1.0.0", "linux", "amd64").unwrap(), @@ -1521,13 +1663,7 @@ packages: - key: libc value: musl "#; - let pkg = serde_yaml::from_str::(yml) - .unwrap() - .packages - .into_iter() - .next() - .unwrap() - .with_version(&["1.0.0"], "linux", "amd64"); + let pkg = first_registry_package(yml).with_version(&["1.0.0"], "linux", "amd64"); assert_eq!( pkg.url("1.0.0", "linux", "amd64").unwrap(), diff --git a/hk.pkl b/hk.pkl index 43f7222929..94606947e3 100644 --- a/hk.pkl +++ b/hk.pkl @@ -15,7 +15,7 @@ local linters = new Mapping { // PATH (e.g. a Homebrew prettier), keeping local and CI in sync. ["prettier"] = (Builtins.prettier) { batch = false - exclude = "crates/aqua-registry/aqua-registry/**" + exclude = "vendor/aqua-registry/**" check = "mise x prettier -- prettier --check {{ files }}" check_list_files = "mise x prettier -- prettier --list-different {{ files }}" fix = "mise x prettier -- prettier --write {{ files }}" diff --git a/scripts/gen-aqua-changelog.sh b/scripts/gen-aqua-changelog.sh index 4ffb04b437..efa677daec 100755 --- a/scripts/gen-aqua-changelog.sh +++ b/scripts/gen-aqua-changelog.sh @@ -10,7 +10,7 @@ OLD_TAG="${1:-}" NEW_TAG="${2:-}" HEADING_LEVEL="${3:-###}" # Default to ### for CHANGELOG.md sections REPO="aquaproj/aqua-registry" -NEW_REGISTRY="crates/aqua-registry/aqua-registry/registry.yaml" +NEW_REGISTRY="vendor/aqua-registry/registry.yml" if [[ -z $OLD_TAG ]] || [[ -z $NEW_TAG ]] || [[ $OLD_TAG == "$NEW_TAG" ]]; then exit 0 diff --git a/src/aqua/aqua_registry_wrapper.rs b/src/aqua/aqua_registry_wrapper.rs index c2231f9447..f0e132a6a6 100644 --- a/src/aqua/aqua_registry_wrapper.rs +++ b/src/aqua/aqua_registry_wrapper.rs @@ -1,7 +1,9 @@ use crate::config::Settings; use crate::git::{CloneOptions, Git}; use crate::{dirs, duration::WEEKLY, file}; -use aqua_registry::{AquaRegistry, AquaRegistryConfig}; +use aqua_registry::{ + AquaRegistry, AquaRegistryConfig, AquaRegistryError, NoOpCacheStore, RegistryFetcher, +}; use eyre::Result; use std::collections::HashMap; use std::path::PathBuf; @@ -21,7 +23,7 @@ pub static AQUA_REGISTRY: Lazy = Lazy::new(|| { /// Wrapper around the aqua-registry crate that provides mise-specific functionality #[derive(Debug)] pub struct MiseAquaRegistry { - inner: AquaRegistry, + inner: AquaRegistry, #[allow(dead_code)] path: PathBuf, #[allow(dead_code)] @@ -31,7 +33,7 @@ pub struct MiseAquaRegistry { impl Default for MiseAquaRegistry { fn default() -> Self { let config = AquaRegistryConfig::default(); - let inner = AquaRegistry::new(config.clone()); + let inner = aqua_registry(config.clone()); Self { inner, path: config.cache_dir, @@ -72,7 +74,7 @@ impl MiseAquaRegistry { prefer_offline: settings.prefer_offline(), }; - let inner = AquaRegistry::new(config); + let inner = aqua_registry(config); Ok(Self { inner, @@ -95,6 +97,65 @@ impl MiseAquaRegistry { } } +#[derive(Debug, Clone)] +struct MiseRegistryFetcher { + config: AquaRegistryConfig, +} + +fn aqua_registry(config: AquaRegistryConfig) -> AquaRegistry { + AquaRegistry::with_fetcher_and_cache( + config.clone(), + MiseRegistryFetcher { config }, + NoOpCacheStore, + ) +} + +impl RegistryFetcher for MiseRegistryFetcher { + async fn fetch_package(&self, package_id: &str) -> aqua_registry::Result { + if self.config.use_baked_registry + && !self.config.cache_dir.join(".git").exists() + && let Some(package) = super::standard_registry::package(package_id) + { + log::trace!("reading baked-in aqua package for {package_id}"); + return package; + } + + let path_id = package_id + .split('/') + .collect::>() + .join(std::path::MAIN_SEPARATOR_STR); + let path = self + .config + .cache_dir + .join("pkgs") + .join(&path_id) + .join("registry.yaml"); + + if self.config.cache_dir.join(".git").exists() && path.exists() { + log::trace!("reading aqua-registry for {package_id} from repo at {path:?}"); + let contents = std::fs::read_to_string(&path)?; + let registry = serde_yaml::from_str::(&contents)?; + return registry + .packages + .into_iter() + .next() + .map(|row| row.package) + .ok_or_else(|| AquaRegistryError::PackageNotFound(package_id.to_string())); + } + + if self.config.use_baked_registry + && let Some(package) = super::standard_registry::package(package_id) + { + log::trace!("reading baked-in aqua package for {package_id}"); + return package; + } + + Err(AquaRegistryError::RegistryNotAvailable(format!( + "no aqua-registry found for {package_id}" + ))) + } +} + fn fetch_latest_repo(repo: &Git) -> Result<()> { if file::modified_duration(&repo.dir)? < WEEKLY { return Ok(()); @@ -116,7 +177,7 @@ struct AquaSuggestionsCache { } static AQUA_SUGGESTIONS_CACHE: Lazy = Lazy::new(|| { - let ids = aqua_registry::package_ids(); + let ids = super::standard_registry::package_ids(); let mut name_to_ids: HashMap<&'static str, Vec<&'static str>> = HashMap::new(); for id in ids { if let Some((_, name)) = id.rsplit_once('/') { @@ -154,3 +215,47 @@ pub fn aqua_suggest(query: &str) -> Vec { pub use aqua_registry::{ AquaChecksum, AquaChecksumType, AquaCosign, AquaMinisignType, AquaPackage, AquaPackageType, }; + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + + fn test_fetcher(cache_dir: PathBuf, use_baked_registry: bool) -> MiseRegistryFetcher { + MiseRegistryFetcher { + config: AquaRegistryConfig { + cache_dir, + registry_url: Some("https://example.com/custom-aqua-registry".to_string()), + use_baked_registry, + prefer_offline: false, + }, + } + } + + #[tokio::test] + async fn test_custom_registry_falls_back_to_baked_registry_when_enabled() { + let temp = tempfile::tempdir().unwrap(); + std::fs::create_dir(temp.path().join(".git")).unwrap(); + + let package = test_fetcher(temp.path().to_path_buf(), true) + .fetch_package("01mf02/jaq") + .await + .unwrap(); + + assert_eq!(package.repo_owner, "01mf02"); + assert_eq!(package.repo_name, "jaq"); + } + + #[tokio::test] + async fn test_custom_registry_does_not_fall_back_when_baked_registry_disabled() { + let temp = tempfile::tempdir().unwrap(); + std::fs::create_dir(temp.path().join(".git")).unwrap(); + + let err = test_fetcher(temp.path().to_path_buf(), false) + .fetch_package("01mf02/jaq") + .await + .unwrap_err(); + + assert!(matches!(err, AquaRegistryError::RegistryNotAvailable(_))); + } +} diff --git a/src/aqua/mod.rs b/src/aqua/mod.rs index e3432d4e11..a625046864 100644 --- a/src/aqua/mod.rs +++ b/src/aqua/mod.rs @@ -1 +1,2 @@ pub(crate) mod aqua_registry_wrapper; +pub(crate) mod standard_registry; diff --git a/src/aqua/standard_registry.rs b/src/aqua/standard_registry.rs new file mode 100644 index 0000000000..5a5482174c --- /dev/null +++ b/src/aqua/standard_registry.rs @@ -0,0 +1,114 @@ +use aqua_registry::{AquaPackage, Result, decode_package_rkyv}; +use std::collections::HashMap; +use std::sync::LazyLock; + +/// Metadata for the baked aqua registry snapshot. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct AquaRegistryMetadata { + pub repository: &'static str, + pub tag: &'static str, +} + +/// Baked canonical registry packages (compiled into the mise binary). +pub static AQUA_STANDARD_REGISTRY_FILES: LazyLock> = + LazyLock::new(|| include!(concat!(env!("OUT_DIR"), "/aqua_standard_registry_files.rs"))); + +/// Baked aqua registry snapshot metadata (compiled into the mise binary). +pub static AQUA_STANDARD_REGISTRY_METADATA: AquaRegistryMetadata = include!(concat!( + env!("OUT_DIR"), + "/aqua_standard_registry_metadata.rs" +)); + +/// Baked alias-to-canonical package ID map (compiled into the mise binary). +static AQUA_STANDARD_REGISTRY_ALIASES: LazyLock> = + LazyLock::new(|| { + include!(concat!( + env!("OUT_DIR"), + "/aqua_standard_registry_aliases.rs" + )) + }); + +/// Returns all package IDs from the baked-in aqua registry. +pub fn package_ids() -> Vec<&'static str> { + AQUA_STANDARD_REGISTRY_FILES.keys().copied().collect() +} + +pub fn package(package_id: &str) -> Option> { + baked_registry_file(package_id).map(|content| decode_package_rkyv(package_id, content)) +} + +fn baked_registry_file(package_id: &str) -> Option<&'static [u8]> { + if let Some(content) = AQUA_STANDARD_REGISTRY_FILES.get(package_id) { + return Some(*content); + } + + AQUA_STANDARD_REGISTRY_ALIASES + .get(package_id) + .and_then(|canonical| AQUA_STANDARD_REGISTRY_FILES.get(*canonical)) + .copied() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_baked_registry_package_lookup() { + let package = package("01mf02/jaq").unwrap().unwrap(); + + assert_eq!(package.repo_owner, "01mf02"); + assert_eq!(package.repo_name, "jaq"); + } + + #[test] + fn test_baked_registry_path_only_package_lookup() { + let package = package("golang.org/x/perf/cmd/benchstat").unwrap().unwrap(); + + assert_eq!( + package.path.as_deref(), + Some("golang.org/x/perf/cmd/benchstat") + ); + } + + #[test] + fn test_baked_registry_metadata() { + assert_eq!( + AQUA_STANDARD_REGISTRY_METADATA.repository, + "aquaproj/aqua-registry" + ); + assert!(!AQUA_STANDARD_REGISTRY_METADATA.tag.is_empty()); + assert!(AQUA_STANDARD_REGISTRY_METADATA.tag.starts_with('v')); + } + + #[test] + fn test_baked_registry_alias_lookup() { + let alias = "elijah-potter/harper/harper-ls"; + + assert!(!AQUA_STANDARD_REGISTRY_FILES.contains_key(alias)); + assert_eq!( + AQUA_STANDARD_REGISTRY_ALIASES.get(alias).copied(), + Some("Automattic/harper/harper-ls") + ); + + let alias_package = package(alias).unwrap().unwrap(); + let canonical_package = package("Automattic/harper/harper-ls").unwrap().unwrap(); + + assert_eq!( + alias_package.name.as_deref(), + Some("Automattic/harper/harper-ls") + ); + assert_eq!( + alias_package.name.as_deref(), + canonical_package.name.as_deref() + ); + assert_eq!(alias_package.repo_owner, canonical_package.repo_owner); + assert_eq!(alias_package.repo_name, canonical_package.repo_name); + } + + #[test] + fn test_baked_registry_numeric_replacement_keys() { + let package = package("sharkdp/hyperfine").unwrap().unwrap(); + + assert_eq!(package.replacements.get("386"), Some(&"i686".to_string())); + } +} diff --git a/src/backend/aqua.rs b/src/backend/aqua.rs index 97de9bbf3d..a55859fece 100644 --- a/src/backend/aqua.rs +++ b/src/backend/aqua.rs @@ -3006,9 +3006,10 @@ fn validate(pkg: &AquaPackage) -> Result<()> { .unwrap_or_default() ) } - AquaPackageType::GoInstall => { + AquaPackageType::GoInstall | AquaPackageType::GoBuild => { bail!( - "package type `go_install` is not supported in the aqua backend. Use the go backend instead{}.", + "package type `{}` is not supported in the aqua backend. Use the go backend instead{}.", + pkg.r#type, pkg.path .as_ref() .map(|path| format!(": go:{path}")) diff --git a/src/cli/doctor/mod.rs b/src/cli/doctor/mod.rs index e55c9db03a..3b7f1385cd 100644 --- a/src/cli/doctor/mod.rs +++ b/src/cli/doctor/mod.rs @@ -104,7 +104,8 @@ impl Doctor { .collect(), ); let mut aqua = serde_json::Map::new(); - let aqua_registry_metadata = aqua_registry::AQUA_STANDARD_REGISTRY_METADATA; + let aqua_registry_metadata = + crate::aqua::standard_registry::AQUA_STANDARD_REGISTRY_METADATA; aqua.insert( "baked_in_registry_repository".into(), aqua_registry_metadata.repository.into(), @@ -711,11 +712,11 @@ fn shell() -> String { } fn aqua_registry_count() -> usize { - aqua_registry::AQUA_STANDARD_REGISTRY_FILES.len() + crate::aqua::standard_registry::AQUA_STANDARD_REGISTRY_FILES.len() } fn aqua_registry_count_str() -> String { - let metadata = aqua_registry::AQUA_STANDARD_REGISTRY_METADATA; + let metadata = crate::aqua::standard_registry::AQUA_STANDARD_REGISTRY_METADATA; format!( "baked in registry: {}@{}\nbaked in registry tools: {}", metadata.repository, diff --git a/crates/aqua-registry/aqua-registry/LICENSE b/vendor/aqua-registry/LICENSE similarity index 100% rename from crates/aqua-registry/aqua-registry/LICENSE rename to vendor/aqua-registry/LICENSE diff --git a/crates/aqua-registry/aqua-registry/metadata.json b/vendor/aqua-registry/metadata.json similarity index 100% rename from crates/aqua-registry/aqua-registry/metadata.json rename to vendor/aqua-registry/metadata.json diff --git a/crates/aqua-registry/aqua-registry/registry.yaml b/vendor/aqua-registry/registry.yml similarity index 100% rename from crates/aqua-registry/aqua-registry/registry.yaml rename to vendor/aqua-registry/registry.yml diff --git a/xtasks/lint-fix.sh b/xtasks/lint-fix.sh index f38c097d25..b382c91615 100755 --- a/xtasks/lint-fix.sh +++ b/xtasks/lint-fix.sh @@ -4,4 +4,4 @@ #MISE description="Automatically fix lint issues" set -euxo pipefail -hk fix --all --exclude crates/aqua-registry/aqua-registry +hk fix --all --exclude vendor/aqua-registry diff --git a/xtasks/release-plz b/xtasks/release-plz index e7636b16d5..8217703082 100755 --- a/xtasks/release-plz +++ b/xtasks/release-plz @@ -264,8 +264,9 @@ fi mise run fetch-gpg-keys mise run render ::: lint-fix -AQUA_REGISTRY_DIR="crates/aqua-registry/aqua-registry" +AQUA_REGISTRY_DIR="vendor/aqua-registry" AQUA_REGISTRY_METADATA="$AQUA_REGISTRY_DIR/metadata.json" +AQUA_REGISTRY_FILE="$AQUA_REGISTRY_DIR/registry.yml" AQUA_REGISTRY_REPO="aquaproj/aqua-registry" # Capture current aqua-registry tag before updating @@ -278,7 +279,7 @@ NEW_AQUA_REGISTRY_TAG="$(gh release view --repo "$AQUA_REGISTRY_REPO" --json tag rm -rf "$AQUA_REGISTRY_DIR" mkdir -p "$AQUA_REGISTRY_DIR" curl -fsSL "https://raw.githubusercontent.com/$AQUA_REGISTRY_REPO/$NEW_AQUA_REGISTRY_TAG/registry.yaml" \ - -o "$AQUA_REGISTRY_DIR/registry.yaml" + -o "$AQUA_REGISTRY_FILE" curl -fsSL "https://raw.githubusercontent.com/$AQUA_REGISTRY_REPO/$NEW_AQUA_REGISTRY_TAG/LICENSE" \ -o "$AQUA_REGISTRY_DIR/LICENSE" cat >"$AQUA_REGISTRY_METADATA" <