From ea007c5e3b52eb96fa9d5651f36124c977f78eb4 Mon Sep 17 00:00:00 2001 From: eri Date: Thu, 5 Dec 2024 13:38:43 +0100 Subject: [PATCH] feat: mayor refactor of the metadata reading --- Cargo.toml | 13 +- binary/Cargo.toml | 23 --- binary/build.rs | 345 --------------------------------- binary/src/lib.rs | 1 - build.rs | 35 ++++ meta/Cargo.toml | 19 ++ meta/src/binary.rs | 319 ++++++++++++++++++++++++++++++ meta/src/lib.rs | 158 +++++---------- meta/src/parse.rs | 173 +++++++++++++++++ meta/src/test.rs | 17 ++ meta/src/tests/a/Cargo.toml | 8 + meta/src/tests/a/src/lib.rs | 0 meta/src/tests/main/Cargo.toml | 9 + meta/src/tests/main/src/lib.rs | 0 src/lib.rs | 9 +- 15 files changed, 637 insertions(+), 492 deletions(-) delete mode 100644 binary/Cargo.toml delete mode 100644 binary/build.rs delete mode 100644 binary/src/lib.rs create mode 100644 build.rs create mode 100644 meta/src/binary.rs create mode 100644 meta/src/parse.rs create mode 100644 meta/src/test.rs create mode 100644 meta/src/tests/a/Cargo.toml create mode 100644 meta/src/tests/a/src/lib.rs create mode 100644 meta/src/tests/main/Cargo.toml create mode 100644 meta/src/tests/main/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index 2fb3eea..4a733af 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,14 +19,15 @@ edition = "2018" documentation = "https://docs.rs/system-deps/" readme = "README.md" +[build-dependencies] +system-deps-meta = { path = "./meta", optional = true } + [dependencies] pkg-config = "0.3.25" toml = { version = "0.8", default-features = false, features = ["parse"] } version-compare = "0.2" heck = "0.5" cfg-expr = { version = "0.17", features = ["targets"] } -system-deps-binary = { path = "./binary", optional = true } -cc = { version = "1.2", optional = true } [dev-dependencies] lazy_static = "1" @@ -35,7 +36,7 @@ assert_matches = "1.5" [features] default = [ "binary" ] -binary = [ "dep:system-deps-binary", "dep:cc" ] -gx = [ "system-deps-binary/gz" ] -xz = [ "system-deps-binary/xz" ] -zip = [ "system-deps-binary/zip" ] +binary = [ "system-deps-meta/binary" ] +gx = [ "system-deps-meta/gz" ] +xz = [ "system-deps-meta/xz" ] +zip = [ "system-deps-meta/zip" ] diff --git a/binary/Cargo.toml b/binary/Cargo.toml deleted file mode 100644 index 3c0af0b..0000000 --- a/binary/Cargo.toml +++ /dev/null @@ -1,23 +0,0 @@ -[package] -name = "system-deps-binary" -version = "0.1.0" -edition = "2021" - -[build-dependencies] -system-deps-meta = { path = "../meta" } -serde = { version = "1.0", features = ["derive"] } -sha256 = { version = "1.5" } -reqwest = { version = "0.12", features = ["blocking"], optional = true } -flate2 = { version = "1.0", optional = true } -xz = { version = "0.1", optional = true } -tar = { version = "0.4", optional = true } -zip = { version = "2.2", optional = true } -apple-flat-package = { version = "0.19", optional = true } - -[features] -default = [ "web" ] -web = [ "dep:reqwest" ] -gz = [ "dep:flate2", "dep:tar" ] -xz = [ "dep:xz", "dep:tar" ] -zip = [ "dep:zip" ] -pkg = [ "dep:apple-flat-package" ] diff --git a/binary/build.rs b/binary/build.rs deleted file mode 100644 index 3481f81..0000000 --- a/binary/build.rs +++ /dev/null @@ -1,345 +0,0 @@ -use std::{ - collections::HashMap, - fs, io, - path::{Path, PathBuf}, -}; - -use serde::Deserialize; - -/// The extension of the binary archive. -/// Support for different extensions is enabled using features. -#[derive(Debug)] -enum Extension { - /// A `.tar.gz` archive. - #[cfg(feature = "gz")] - TarGz, - /// A `.tar.xz` archive. - #[cfg(feature = "xz")] - TarXz, - /// A `.zip` archive. - #[cfg(feature = "zip")] - Zip, - /// An Apple package archive. Untested. - #[cfg(feature = "pkg")] - Pkg, -} - -/// Represents one location from where to download library binaries. -#[derive(Debug, Deserialize)] -struct UrlBinary { - /// The url from which to download the archived binaries. It suppports: - /// - /// - Web urls, in the form `http[s]://website/archive.ext`. - /// This must directly download an archive with a known `Extension`. - /// - Local files, in the form `file:///path/to/archive.ext`. - /// Note that this is made of the url descriptor `file://`, and then an absolute path, that - /// starts with `/`, so three total slashes are needed. - /// The path can point at an archive with a known `Extension`, or to a folder containing the - /// uncompressed binaries. - url: String, - /// Optionally, a checksum of the downloaded archive. When set, it is used to correctly cache - /// the result. If this is not specified, it will still be cached by cargo, but redownloads - /// might happen more often. It has no effect if `url` is a local folder. - checksum: Option, - /// A list of relative paths inside the binary archive that point to a folder containing - /// package config files. These directories will be prepended to the `PKG_CONFIG_PATH` when - /// compiling the affected libraries. - pkg_paths: Vec, - /// Controls if the paths from this binary apply to all packages or just to this one. - global: Option, -} - -/// Represents a binary that follows another. -#[derive(Debug, Deserialize)] -struct FollowBinary { - /// The `system-deps` formatted name of another library which has binaries specified. - /// This library will alias the configuration of the followed one. If `url` is specified - /// alongside this field, it will no longer follow the original configuration. - follows: String, -} - -/// Deserializes the correct binary type. `Url` has precedence. -#[derive(Debug, Deserialize)] -#[serde(untagged)] -enum Binary { - Url(UrlBinary), - Follow(FollowBinary), -} - -pub fn main() { - // Add pkg-config paths to the overrides - // TODO: This should probably follow some deterministic ordering to avoid issues - - let dest_path = Path::new(system_deps_meta::BUILD_TARGET_DIR).join("binary_config.rs"); - println!("cargo:rustc-env=BINARY_CONFIG={}", dest_path.display()); - - let options = paths() - .into_iter() - .map(|(name, paths)| format!(r#""{}" => &{:?},"#, name, paths)) - .collect::>() - .join("\n "); - - let config = format!( - r#" -pub fn get_path(name: &str) -> &[&'static str] {{ - match name {{ - {} - _ => &[], - }} -}} -"#, - options - ); - - fs::write(dest_path, config).expect("Error when writing binary config"); -} - -/// Looks up an environment variable and adds it to the rerun flags. -fn env(name: &str) -> Option { - println!("cargo:rerun-if-env-changed={}", name); - std::env::var(name).ok() -} - -/// Uses the metadata from the cargo manifests and the environment to build a list of urls -/// from where to download binaries for dependencies and adds them to their `PKG_CONFIG_PATH`. -fn paths() -> HashMap> { - let values = system_deps_meta::read_metadata("system-deps"); - - // Read metadata from the crate graph - let mut binaries = values - .into_iter() - .filter_map(|(n, v)| Some((n, system_deps_meta::from_value(v).ok()?))) - .collect::>(); - - let mut paths = HashMap::>::new(); - let mut follow_list = HashMap::new(); - - // Global overrides from environment - // TODO: Change this so the env set global url always is first in the list of paths - if let Some(url) = env("SYSTEM_DEPS_BINARY_URL") { - let checksum = env("SYSTEM_DEPS_BINARY_CHECKSUM"); - let pkg_paths = env("SYSTEM_DEPS_BINARY_PKG_PATHS"); - binaries.insert( - ".from_env".into(), - Binary::Url(UrlBinary { - url, - checksum, - pkg_paths: pkg_paths - .map(|p| { - std::env::split_paths(&p) - .map(|v| { - v.to_str() - .expect("Error with global binary package path") - .to_string() - }) - .collect() - }) - .unwrap_or_default(), - global: Some(true), - }), - ); - } - - for (name, bin) in binaries { - match bin { - Binary::Follow(FollowBinary { follows }) => { - follow_list.insert(name.clone(), follows); - } - Binary::Url(bin) => { - // The binaries are stored in the target dir set by `system_deps_meta`. - // If they are specific to a dependency, they live in a subfolder. - let mut dst = PathBuf::from(&system_deps_meta::BUILD_TARGET_DIR); - if !name.is_empty() { - dst.push(name.clone()); - }; - - // Only download the binaries if there isn't already a valid copy - if !check_valid_dir(&dst, bin.checksum) - .expect("Error when checking the download directory") - { - download(&bin.url, &dst).expect("Error when getting binaries"); - } - - // Add pkg config paths to the overrides - if bin.global.unwrap_or_default() { - paths - .entry("".into()) - .or_default() - .extend(bin.pkg_paths.iter().map(|p| dst.join(p))); - } - paths - .entry(name) - .or_default() - .extend(bin.pkg_paths.iter().map(|p| dst.join(p))); - } - } - } - - // Go through the list of follows and if they don't already have binaries, - // link them to the followed one. - for (from, to) in follow_list { - if !paths.contains_key(&from) { - let followed = paths - .get(&to) - .unwrap_or_else(|| { - panic!( - "The library `{}` tried to follow `{}` but it doesn't exist", - from, to, - ) - }) - .clone(); - paths.insert(from, followed); - }; - } - - paths -} - -/// Checks if the target directory is valid and if binaries need to be redownloaded. -/// On an `Ok` result, if the value is true it means that the directory is correct. -fn check_valid_dir(dst: &Path, checksum: Option) -> io::Result { - // If it doesn't exist yet the download will need to happen - if !dst.try_exists()? { - return Ok(false); - } - - // Raise an error if it is a file - if dst.is_file() { - return Err(io::Error::new( - io::ErrorKind::Other, - format!("The target directory is a file {:?}", dst), - )); - } - - // If a checksum is not specified, assume the directory is invalid - let Some(checksum) = checksum else { - return Ok(false); - }; - - // Check if the checksum is valid - let valid = dst - .read_dir()? - .find(|f| f.as_ref().is_ok_and(|f| f.file_name() == "checksum")) - .and_then(|s| s.ok()) - .and_then(|s| fs::read_to_string(s.path()).ok()) - .and_then(|s| (checksum == s).then_some(())) - .is_some(); - - Ok(valid) -} - -/// Retrieve a binary archive from the specified `url` and decompress it in the target directory. -/// "Download" is used as an umbrella term, since this can also be a local file. -fn download(url: &str, dst: &Path) -> io::Result<()> { - let ext = match url { - #[cfg(feature = "gz")] - u if u.ends_with(".tar.gz") => Ok(Extension::TarGz), - #[cfg(feature = "xz")] - u if u.ends_with(".tar.xz") => Ok(Extension::TarXz), - #[cfg(feature = "zip")] - u if u.ends_with(".zip") => Ok(Extension::Zip), - #[cfg(feature = "pkg")] - u if u.ends_with(".pkg") => Ok(Extension::Pkg), - u => Err(io::Error::new( - io::ErrorKind::Other, - format!("Unsuppported binary extension, {:?}", u.split(".").last()), - )), - }; - - // Local file - if let Some(file_path) = url.strip_prefix("file://") { - let path = Path::new(file_path); - match ext { - Ok(ext) => { - let file = fs::read(path)?; - decompress(&file, dst, ext)?; - } - Err(e) => { - // If it is a folder it can be symlinked - if !path.is_dir() { - return Err(e); - } - if !dst.read_link().is_ok_and(|l| l == path) { - #[cfg(unix)] - std::os::unix::fs::symlink(file_path, dst)?; - #[cfg(windows)] - std::os::windows::fs::symlink_dir(file_path, dst)?; - } - } - }; - } - // Download from the web - else { - #[cfg(not(feature = "web"))] - panic!("To download a binary file you must enable the `web` feature"); - #[cfg(feature = "web")] - { - let ext = ext?; - let file = reqwest::blocking::get(url) - .and_then(|req| req.bytes()) - .map_err(|e| { - io::Error::new(io::ErrorKind::Other, format!("Download error: {:?}", e)) - })?; - decompress(&file, dst, ext)?; - } - } - - Ok(()) -} - -/// Extract a binary archive to the target directory. The methods for unpacking are -/// different depending on the extension. Each file type is gated behind a feature to -/// avoid having too many dependencies. -#[allow(unused)] -fn decompress(file: &[u8], dst: &Path, ext: Extension) -> io::Result<()> { - #[cfg(not(any(feature = "gz", feature = "xz", feature = "zip", feature = "pkg")))] - unreachable!(); - - match ext { - #[cfg(feature = "gz")] - Extension::TarGz => { - let reader = flate2::read::GzDecoder::new(file); - let mut archive = tar::Archive::new(reader); - archive.unpack(dst)?; - } - #[cfg(feature = "xz")] - Extension::TarXz => { - let reader = xz::read::XzDecoder::new(file); - let mut archive = tar::Archive::new(reader); - archive.unpack(dst)?; - } - #[cfg(feature = "zip")] - Extension::Zip => { - let reader = io::Cursor::new(file); - let mut archive = zip::ZipArchive::new(reader)?; - archive.extract(dst)?; - } - #[cfg(feature = "pkg")] - Extension::Pkg => { - // TODO: Test this with actual pkg files, do they have pc files inside? - // TODO: Error handling - let reader = io::Cursor::new(file); - let mut archive = apple_flat_package::PkgReader::new(reader).unwrap(); - let pkgs = archive.component_packages().unwrap(); - let mut cpio = pkgs.first().unwrap().payload_reader().unwrap().unwrap(); - while let Some(next) = cpio.next() { - let entry = next.unwrap(); - let mut file = Vec::new(); - cpio.read_to_end(&mut file).unwrap(); - if entry.file_size() != 0 { - let dst = dst.join(entry.name()); - fs::create_dir_all(dst.parent().unwrap())?; - fs::write(&dst, file)?; - } - } - } - }; - - // Update the checksum - let checksum = sha256::digest(file); - let mut path = dst.to_path_buf(); - path.push("checksum"); - fs::write(path, checksum)?; - - Ok(()) -} diff --git a/binary/src/lib.rs b/binary/src/lib.rs deleted file mode 100644 index ac31115..0000000 --- a/binary/src/lib.rs +++ /dev/null @@ -1 +0,0 @@ -include!(env!("BINARY_CONFIG")); diff --git a/build.rs b/build.rs new file mode 100644 index 0000000..be45206 --- /dev/null +++ b/build.rs @@ -0,0 +1,35 @@ +pub fn main() { + #[cfg(feature = "binary")] + binary::build(); +} + +#[cfg(feature = "binary")] +mod binary { + use std::{ + collections::HashMap, + fs, + path::{Path, PathBuf}, + }; + + use system_deps_meta::{read_metadata, Binary, BinaryPaths, BUILD_TARGET_DIR}; + + pub fn build() { + // Add pkg-config paths to the overrides + // TODO: This should probably follow some deterministic ordering to avoid issues + + let dest_path = Path::new(BUILD_TARGET_DIR).join("binary_config.rs"); + println!("cargo:rustc-env=BINARY_CONFIG={}", dest_path.display()); + + // Read metadata from the crate graph + let manifest = PathBuf::from(system_deps_meta::BUILD_MANIFEST); + let metadata = read_metadata(&manifest, "system-deps"); + + // Download the binaries and get their pkg_config paths + let mut binaries: HashMap = metadata.get(|| true).unwrap_or_default(); + println!("BINARIES {:?}", binaries); + let paths = BinaryPaths::from(binaries.drain()); + println!("PATHS {:?}", paths); + + fs::write(dest_path, paths.build()).expect("Error when writing binary config"); + } +} diff --git a/meta/Cargo.toml b/meta/Cargo.toml index 3181bb6..5640785 100644 --- a/meta/Cargo.toml +++ b/meta/Cargo.toml @@ -5,5 +5,24 @@ edition = "2021" [dependencies] cargo_metadata = "0.18" +# Metadata serde_json = "1.0" +serde = "1.0" cfg-expr = { version = "0.17", features = ["targets"] } +# Binary +sha256 = { version = "1.5" } +reqwest = { version = "0.12", features = ["blocking"], optional = true } +flate2 = { version = "1.0", optional = true } +xz = { version = "0.1", optional = true } +tar = { version = "0.4", optional = true } +zip = { version = "2.2", optional = true } +apple-flat-package = { version = "0.19", optional = true } + +[features] +default = [ "web" ] +binary = [ ] +web = [ "dep:reqwest" ] +gz = [ "dep:flate2", "dep:tar" ] +xz = [ "dep:xz", "dep:tar" ] +zip = [ "dep:zip" ] +pkg = [ "dep:apple-flat-package" ] diff --git a/meta/src/binary.rs b/meta/src/binary.rs new file mode 100644 index 0000000..15c9f6b --- /dev/null +++ b/meta/src/binary.rs @@ -0,0 +1,319 @@ +use std::{ + collections::{HashMap, HashSet}, + fs, + path::{Path, PathBuf}, +}; + +use serde::Deserialize; + +use crate::Error; + +/// The extension of the binary archive. +/// Support for different extensions is enabled using features. +#[derive(Debug)] +enum Extension { + /// A `.tar.gz` archive. + #[cfg(feature = "gz")] + TarGz, + /// A `.tar.xz` archive. + #[cfg(feature = "xz")] + TarXz, + /// A `.zip` archive. + #[cfg(feature = "zip")] + Zip, + /// An Apple package archive. Untested. + #[cfg(feature = "pkg")] + Pkg, +} + +/// Represents one location from where to download library binaries. +#[derive(Debug, Deserialize)] +pub struct Binary { + /// The url from which to download the archived binaries. It suppports: + /// + /// - Web urls, in the form `http[s]://website/archive.ext`. + /// This must directly download an archive with a known `Extension`. + /// - Local files, in the form `file:///path/to/archive.ext`. + /// Note that this is made of the url descriptor `file://`, and then an absolute path, that + /// starts with `/`, so three total slashes are needed. + /// The path can point at an archive with a known `Extension`, or to a folder containing the + /// uncompressed binaries. + url: String, + /// Optionally, a checksum of the downloaded archive. When set, it is used to correctly cache + /// the result. If this is not specified, it will still be cached by cargo, but redownloads + /// might happen more often. It has no effect if `url` is a local folder. + checksum: Option, + /// A list of relative paths inside the binary archive that point to a folder containing + /// package config files. These directories will be prepended to the `PKG_CONFIG_PATH` when + /// compiling the affected libraries. + pkg_paths: Vec, + /// Controls if the paths from this binary apply to all packages or just to this one. + global: Option, + /// The `system-deps` formatted name of another library which has binaries specified. + /// This library will alias the configuration of the followed one. If `url` is specified + /// alongside this field, it will no longer follow the original configuration. + follows: Option, +} + +impl Binary { + pub fn paths(&self, name: &str) -> Result, Error> { + // Set this binary to follow + //if let Some(follows) = self.follows { + // follow_list.insert(name.clone(), follows); + //} + + // The binaries are stored in the target dir set by `system_deps_meta`. + // If they are specific to a dependency, they live in a subfolder. + let mut dst = PathBuf::from(&crate::BUILD_TARGET_DIR); + if !name.is_empty() { + dst.push(name); + }; + + // Only download the binaries if there isn't already a valid copy + if !check_valid_dir(&dst, self.checksum.as_deref())? { + download(&self.url, &dst)?; + } + + Ok(self.pkg_paths.iter().map(|p| dst.join(p)).collect()) + } + + pub fn is_global(&self) -> bool { + self.global.unwrap_or_default() + } +} + +#[derive(Debug)] +pub struct BinaryPaths(HashMap>); + +impl> From for BinaryPaths { + /// Uses the metadata from the cargo manifests and the environment to build a list of urls + /// from where to download binaries for dependencies and adds them to their `PKG_CONFIG_PATH`. + fn from(binaries: I) -> Self { + let mut paths: HashMap> = HashMap::new(); + + for (name, bin) in binaries { + let p = bin.paths(&name).unwrap(); + if bin.is_global() { + paths + .entry("".into()) + .or_default() + .extend(p.iter().cloned()) + } + paths.entry(name).or_default().extend(p.into_iter()); + } + + Self(paths) + } +} + +impl BinaryPaths { + pub fn build(self) -> String { + let options = self + .0 + .into_iter() + .map(|(name, paths)| format!(r#""{}" => &{:?},"#, name, paths)) + .collect::>() + .join("\n "); + + format!( + r#" +/// TODO: +pub fn get_path(name: &str) -> &[&'static str] {{ + match name {{ + {} + _ => &[], + }} +}} +"#, + options + ) + } + + // Global overrides from environment + // TODO: Change this so the env set global url always is first in the list of paths + //if let Some(url) = env("SYSTEM_DEPS_BINARY_URL") { + // let checksum = env("SYSTEM_DEPS_BINARY_CHECKSUM"); + // let pkg_paths = env("SYSTEM_DEPS_BINARY_PKG_PATHS"); + // binaries.insert( + // ".from_env".into(), + // Binary::Url(UrlBinary { + // url, + // checksum, + // pkg_paths: pkg_paths + // .map(|p| { + // std::env::split_paths(&p) + // .map(|v| { + // v.to_str() + // .expect("Error with global binary package path") + // .to_string() + // }) + // .collect() + // }) + // .unwrap_or_default(), + // global: Some(true), + // }), + // ); + //} + + // Go through the list of follows and if they don't already have binaries, + // link them to the followed one. + //for (from, to) in follow_list { + // if !paths.contains_key(&from) { + // let followed = paths + // .get(to.as_str()) + // .unwrap_or_else(|| { + // panic!( + // "The library `{}` tried to follow `{}` but it doesn't exist", + // from, to, + // ) + // }) + // .clone(); + // paths.insert(from, followed); + // }; + //} +} + +/// Checks if the target directory is valid and if binaries need to be redownloaded. +/// On an `Ok` result, if the value is true it means that the directory is correct. +fn check_valid_dir(dst: &Path, checksum: Option<&str>) -> Result { + let e = |e| Error::InvalidDirectory(e); + + // If it doesn't exist yet the download will need to happen + if !dst.try_exists().map_err(e)? { + return Ok(false); + } + + // Raise an error if it is a file + if dst.is_file() { + return Err(Error::DirectoryIsFile(dst.display().to_string())); + } + + // If a checksum is not specified, assume the directory is invalid + let Some(checksum) = checksum else { + return Ok(false); + }; + + // Check if the checksum is valid + for f in dst.read_dir().map_err(e)? { + let f = f.map_err(e)?; + if f.file_name() != "checksum" { + continue; + } + return Ok(checksum == fs::read_to_string(f.path()).map_err(e)?); + } + Ok(false) +} + +/// Retrieve a binary archive from the specified `url` and decompress it in the target directory. +/// "Download" is used as an umbrella term, since this can also be a local file. +fn download(url: &str, dst: &Path) -> Result<(), Error> { + let ext = match url { + #[cfg(feature = "gz")] + u if u.ends_with(".tar.gz") => Ok(Extension::TarGz), + #[cfg(feature = "xz")] + u if u.ends_with(".tar.xz") => Ok(Extension::TarXz), + #[cfg(feature = "zip")] + u if u.ends_with(".zip") => Ok(Extension::Zip), + #[cfg(feature = "pkg")] + u if u.ends_with(".pkg") => Ok(Extension::Pkg), + u => Err(Error::InvalidExtension(u.into())), + }; + + // Local file + if let Some(file_path) = url.strip_prefix("file://") { + let path = Path::new(file_path); + match ext { + Ok(ext) => { + let file = fs::read(path).map_err(|e| Error::LocalFileError(e))?; + decompress(&file, dst, ext)?; + } + Err(e) => { + // If it is a folder it can be symlinked + if !path.is_dir() { + return Err(e); + } + if !dst.read_link().is_ok_and(|l| l == path) { + #[cfg(unix)] + std::os::unix::fs::symlink(file_path, dst) + .map_err(|e| Error::SymlinkError(e))?; + #[cfg(windows)] + std::os::windows::fs::symlink_dir(file_path, dst) + .map_err(|e| Error::SymlinkError(e))?; + } + } + }; + } + // Download from the web + else { + #[cfg(not(feature = "web"))] + panic!("To download a binary file you must enable the `web` feature"); + #[cfg(feature = "web")] + { + let ext = ext?; + let file = reqwest::blocking::get(url).and_then(|req| req.bytes())?; + decompress(&file, dst, ext)?; + } + } + + Ok(()) +} + +/// Extract a binary archive to the target directory. The methods for unpacking are +/// different depending on the extension. Each file type is gated behind a feature to +/// avoid having too many dependencies. +#[allow(unused)] +fn decompress(file: &[u8], dst: &Path, ext: Extension) -> Result<(), Error> { + #[cfg(any(feature = "gz", feature = "xz", feature = "zip", feature = "pkg"))] + { + let e = |e| Error::DecompressError(e); + + match ext { + #[cfg(feature = "gz")] + Extension::TarGz => { + let reader = flate2::read::GzDecoder::new(file); + let mut archive = tar::Archive::new(reader); + archive.unpack(dst).map_err(e)?; + } + #[cfg(feature = "xz")] + Extension::TarXz => { + let reader = xz::read::XzDecoder::new(file); + let mut archive = tar::Archive::new(reader); + archive.unpack(dst).map_err(e)?; + } + #[cfg(feature = "zip")] + Extension::Zip => { + let reader = io::Cursor::new(file); + let mut archive = zip::ZipArchive::new(reader).map_err(e)?; + archive.extract(dst).map_err(e)?; + } + #[cfg(feature = "pkg")] + Extension::Pkg => { + // TODO: Test this with actual pkg files, do they have pc files inside? + // TODO: Error handling + let reader = io::Cursor::new(file); + let mut archive = apple_flat_package::PkgReader::new(reader).unwrap(); + let pkgs = archive.component_packages().unwrap(); + let mut cpio = pkgs.first().unwrap().payload_reader().unwrap().unwrap(); + while let Some(next) = cpio.next() { + let entry = next.unwrap(); + let mut file = Vec::new(); + cpio.read_to_end(&mut file).unwrap(); + if entry.file_size() != 0 { + let dst = dst.join(entry.name()); + fs::create_dir_all(dst.parent().unwrap())?; + fs::write(&dst, file).map_err(e)?; + } + } + } + }; + + // Update the checksum + let checksum = sha256::digest(file); + let mut path = dst.to_path_buf(); + path.push("checksum"); + if let Err(e) = fs::write(path, checksum) { + println!("cargo:warning=Couldn't write the binary checksum {:?}", e); + }; + } + Ok(()) +} diff --git a/meta/src/lib.rs b/meta/src/lib.rs index 1c30a74..d68483c 100644 --- a/meta/src/lib.rs +++ b/meta/src/lib.rs @@ -1,16 +1,15 @@ -use std::{ - collections::{HashSet, VecDeque}, - path::PathBuf, - sync::OnceLock, -}; +use std::{fmt, io}; -use cargo_metadata::{DependencyKind, MetadataCommand}; -use cfg_expr::{targets::get_builtin_target_by_triple, Expression, Predicate}; -use serde_json::{Map, Value}; +mod parse; +pub use parse::*; -pub use cargo_metadata::Metadata; -pub use serde_json::from_value; -pub type Values = Map; +#[cfg(feature = "binary")] +mod binary; +#[cfg(feature = "binary")] +pub use binary::*; + +#[cfg(test)] +mod test; /// Path to the top level Cargo.toml. pub const BUILD_MANIFEST: &str = env!("BUILD_MANIFEST"); @@ -18,115 +17,50 @@ pub const BUILD_MANIFEST: &str = env!("BUILD_MANIFEST"); /// Directory where `system-deps` related build products will be stored. pub const BUILD_TARGET_DIR: &str = env!("BUILD_TARGET_DIR"); -/// Get metadata from every crate in the project. -fn metadata() -> &'static Metadata { - static CACHED: OnceLock = OnceLock::new(); - CACHED.get_or_init(|| { - MetadataCommand::new() - .manifest_path(BUILD_MANIFEST) - .exec() - .unwrap() - }) -} - -fn check_cfg(lit: &str) -> Option { - let cfg = Expression::parse(lit).ok()?; - let target = get_builtin_target_by_triple(&std::env::var("TARGET").ok()?)?; - cfg.eval(|pred| match pred { - Predicate::Target(tp) => Some(tp.matches(target)), - _ => None, - }) +/// Metadata related errors. +#[derive(Debug)] +pub enum Error { + // Binary + DecompressError(io::Error), + DirectoryIsFile(String), + InvalidDirectory(io::Error), + InvalidExtension(String), + LocalFileError(io::Error), + SymlinkError(io::Error), + // Web + DownloadError(reqwest::Error), } -/// Inserts values from b into a only if they don't already exist. -/// TODO: This function could be a lot cleaner and it needs better error handling. -/// The logic for merging values needs to handle more cases so during testing this will have to be rewritten. -/// Additionally, make sure that only downstream crates can override the metadata. -fn merge(a: &mut Value, b: Value) { - match (a, b) { - (a @ &mut Value::Object(_), Value::Object(b)) => { - for (k, v) in b { - // Check the cfg expressions on the tree to see if they apply - if k.starts_with("cfg(") { - if check_cfg(&k).unwrap_or_default() { - merge(a, v); - } - continue; - } - let a = a.as_object_mut().unwrap(); - if let Some(e) = a.get_mut(&k) { - if e.is_object() { - merge(e, v); - } - } else { - a.insert(k, v); - } - } - } - (a, b) => *a = b, +impl From for Error { + fn from(e: reqwest::Error) -> Self { + Self::DownloadError(e) } } -/// Recursively read dependency manifests to find metadata matching a key. -/// The matching metadata is aggregated in a list, with downstream crates having priority -/// for overwriting values. It will only read from the metadata sections matching the -/// provided key. -/// -/// ```toml -/// [package.metadata.key] -/// some_value = ... -/// other_value = ... -/// ``` -pub fn read_metadata(key: &str) -> Values { - let metadata = metadata(); - let project_root = PathBuf::from(BUILD_MANIFEST); - let project_root = project_root.parent().unwrap(); - - // Depending on if we are on a workspace or not, use the root package or all the - // workspace packages as a starting point - let mut packages = if let Some(root) = metadata.root_package() { - VecDeque::from([root]) - } else { - metadata.workspace_packages().into() - }; - - // Add the workspace metadata (if it exists) first - let mut res = metadata - .workspace_metadata - .as_object() - .and_then(|meta| meta.get(key)) - .cloned() - .unwrap_or(Value::Object(Map::new())); - - // Iterate through the dependency tree to visit all packages - let mut visited: HashSet<&str> = packages.iter().map(|p| p.name.as_str()).collect(); - while let Some(pkg) = packages.pop_front() { - // TODO: Optional packages - - for dep in &pkg.dependencies { - match dep.kind { - DependencyKind::Normal | DependencyKind::Build => {} - _ => continue, +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Error::DecompressError(e) => { + write!(f, "Error while decompressing the binaries: {:?}", e) + } + Error::DirectoryIsFile(dir) => write!(f, "The target directory is a file {:?}", dir), + Error::InvalidDirectory(e) => write!(f, "The binary directory is not valid: {:?}", e), + Error::InvalidExtension(url) => { + write!(f, "Unsuppported binary extension for {:?}", url) + } + Error::LocalFileError(e) => { + write!(f, "Error reading the local binary file: {:?}", e) } - if !visited.insert(&dep.name) { - continue; + Error::SymlinkError(e) => { + write!(f, "Error creating symlink to local binary folder: {:?}", e) } - if let Some(dep_pkg) = metadata.packages.iter().find(|p| p.name == dep.name) { - packages.push_back(dep_pkg); - }; + Error::DownloadError(e) => write!(f, "Error while downloading: {:?}", e), } - - // Keep track of the local manifests to see if they change - if pkg.manifest_path.starts_with(project_root) { - println!("cargo:rerun-if-changed={}", pkg.manifest_path); - }; - - // Get the `package.metadata.key` and merge it - let Some(meta) = pkg.metadata.as_object().and_then(|meta| meta.get(key)) else { - continue; - }; - merge(&mut res, meta.clone()); } +} - res.as_object().cloned().unwrap_or_default() +/// Looks up an environment variable and adds it to the rerun flags. +fn env(name: &str) -> Option { + println!("cargo:rerun-if-env-changed={}", name); + std::env::var(name).ok() } diff --git a/meta/src/parse.rs b/meta/src/parse.rs new file mode 100644 index 0000000..2337fce --- /dev/null +++ b/meta/src/parse.rs @@ -0,0 +1,173 @@ +use std::{ + collections::{HashMap, HashSet, VecDeque}, + path::Path, + sync::OnceLock, +}; + +use cargo_metadata::{DependencyKind, Metadata, MetadataCommand}; +use serde::de::DeserializeOwned; +use serde_json::{from_value, Map, Value}; + +/// Stores a section of metadata found in one package. +/// `next` indexes the to packages downstream from this. +#[derive(Debug, Default)] +pub struct MetadataNode { + value: Option, + next: HashSet, +} + +/// Graph like structure that stores the package nodes that have a metadata entry. +#[derive(Debug)] +pub struct MetadataList { + nodes: HashMap<&'static str, MetadataNode>, +} + +impl MetadataList { + fn new() -> Self { + Self { + nodes: HashMap::from([("", MetadataNode::default())]), + } + } + + fn insert(&mut self, name: &'static str, parent: &str, values: Option) { + match values { + Some(v) => { + // Create a new node if it doesn't exist + self.nodes.entry(name).or_insert_with(|| MetadataNode { + value: Some(v), + ..Default::default() + }); + } + None => { + if !self.nodes.contains_key(name) { + // If no value is provided only add the node to the parent if it already exists + return; + } + } + }; + + // Add child to the parent node + let parent_node = self + .nodes + .get_mut(parent) + .expect("Error creating metadata graph"); + parent_node.next.insert(name.into()); + } + + /// Applies the reducing rules to the tree and returns the final value transformed to the desired type + pub fn get(&self, f: impl Fn() -> bool) -> Option { + let base = self.nodes.get("").unwrap(); + let mut stack = VecDeque::from([base]); + + let mut values = Map::new(); + while let Some(node) = stack.pop_front() { + stack.extend(node.next.iter().filter_map(|n| self.nodes.get(n.as_str()))); + //if let Some(ref v) = node.value { + // println!("VALUES {:?}", v); + //} + } + + from_value::(Value::Object(values)).ok() + } + + // /// Uses `cfg_expr` to evaluate a conditional expression in a toml key. + // /// At the moment it only supports target expressions. + // /// + // /// ```toml + // /// [package.metadata.'cfg(target = "unix")'] + // /// value = ... + // /// ``` + //fn check_cfg(lit: &str) -> Option { + // let cfg = Expression::parse(lit).ok()?; + // let target = get_builtin_target_by_triple(&std::env::var("TARGET").ok()?)?; + // cfg.eval(|pred| match pred { + // Predicate::Target(tp) => Some(tp.matches(target)), + // _ => None, + // }) + //} +} + +/// Recursively read dependency manifests to find metadata matching a key. +/// The matching metadata is aggregated in a list, with downstream crates having priority +/// for overwriting values. It will only read from the metadata sections matching the +/// provided section key. +/// +/// ```toml +/// [package.metadata.section] +/// some_value = ... +/// other_value = ... +/// ``` +pub fn read_metadata(manifest: &Path, section: &str) -> MetadataList { + static CACHED: OnceLock = OnceLock::new(); + let metadata = CACHED.get_or_init(|| { + MetadataCommand::new() + .manifest_path(manifest) + .exec() + .unwrap() + }); + + let project_root = manifest.parent().unwrap(); + + // Depending on if we are on a workspace or not, use the root package or all the + // workspace packages as a starting point + let mut packages = if let Some(root) = metadata.root_package() { + VecDeque::from([(root, None)]) + } else { + metadata + .workspace_packages() + .into_iter() + .map(|p| (p, None)) + .collect() + }; + + // Add the workspace metadata (if it exists) first + //let mut res = metadata + // .workspace_metadata + // .as_object() + // .and_then(|meta| meta.get(key)) + // .cloned() + // .unwrap_or(Value::Object(Map::new())); + + let mut res = MetadataList::new(); + + // Iterate through the dependency tree to visit all packages + let mut visited: HashSet<&str> = packages.iter().map(|(p, _)| p.name.as_str()).collect(); + while let Some((pkg, parent)) = packages.pop_front() { + // TODO: Optional packages + + // Keep track of the local manifests to see if they change + if pkg.manifest_path.starts_with(project_root) { + println!("cargo:rerun-if-changed={}", pkg.manifest_path); + }; + + // Get `package.metadata.section` and add it to the metadata graph + let section = pkg.metadata.as_object().and_then(|meta| meta.get(section)); + res.insert( + pkg.name.as_str(), + parent.unwrap_or_default(), + section.cloned(), + ); + + // TODO: If this is the last element, don't keep going + + // Add dependencies to the queue + for dep in &pkg.dependencies { + match dep.kind { + DependencyKind::Normal | DependencyKind::Build => {} + _ => continue, + } + + // If visited, don't keep going, but add dependencies to graph + if !visited.insert(&dep.name) { + res.insert(pkg.name.as_str(), parent.unwrap_or_default(), None); + continue; + } + + if let Some(dep_pkg) = metadata.packages.iter().find(|p| p.name == dep.name) { + packages.push_back((dep_pkg, Some(pkg.name.as_str()))); + }; + } + } + + res +} diff --git a/meta/src/test.rs b/meta/src/test.rs new file mode 100644 index 0000000..567ad8f --- /dev/null +++ b/meta/src/test.rs @@ -0,0 +1,17 @@ +use std::path::Path; + +use crate::*; + +#[test] +fn metadata() { + let manifest = Path::new(BUILD_MANIFEST) + .parent() + .unwrap() + .join("src/tests/main/Cargo.toml"); + let values = read_metadata(&manifest, "system-deps"); + + println!("manifest: {}\n", manifest.display()); + println!("values: {:?}\n", values); + + panic!(); +} diff --git a/meta/src/tests/a/Cargo.toml b/meta/src/tests/a/Cargo.toml new file mode 100644 index 0000000..2e48b5a --- /dev/null +++ b/meta/src/tests/a/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "a" + +[package.metadata.system-deps.a] +version = "1.0" +name = "a" +url = "file:///tmp/test" +checksum = "abcd" diff --git a/meta/src/tests/a/src/lib.rs b/meta/src/tests/a/src/lib.rs new file mode 100644 index 0000000..e69de29 diff --git a/meta/src/tests/main/Cargo.toml b/meta/src/tests/main/Cargo.toml new file mode 100644 index 0000000..fae53c3 --- /dev/null +++ b/meta/src/tests/main/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "main" + +[dependencies] +a = { path = "../a" } + +[package.metadata.system-deps.a] +url = "file:///tmp/other" +checksum = "wxyz" diff --git a/meta/src/tests/main/src/lib.rs b/meta/src/tests/main/src/lib.rs new file mode 100644 index 0000000..e69de29 diff --git a/src/lib.rs b/src/lib.rs index b4dd4aa..4ef604a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -296,6 +296,9 @@ use std::str::FromStr; mod metadata; use metadata::MetaData; +#[cfg(feature = "binary")] +include!(env!("BINARY_CONFIG")); + /// system-deps errors #[derive(Debug)] pub enum Error { @@ -883,11 +886,7 @@ impl Config { #[cfg(not(feature = "binary"))] let pkg_config_paths: [&str; 0] = []; #[cfg(feature = "binary")] - let pkg_config_paths = [ - system_deps_binary::get_path(name.as_str()), - system_deps_binary::get_path(""), - ] - .concat(); + let pkg_config_paths = [get_path(name.as_str()), get_path("")].concat(); // should the lib be statically linked? let statik = cfg!(feature = "binary")