diff --git a/src/build.rs b/src/build.rs index 277200981..3b368613d 100644 --- a/src/build.rs +++ b/src/build.rs @@ -6,13 +6,9 @@ use miette::{Context, IntoDiagnostic}; use rattler_conda_types::{Channel, MatchSpec, Platform, package::PathsJson}; use crate::{ - apply_patch_custom, - metadata::{Output, build_reindexed_channels}, - recipe::parser::TestType, - render::resolved_dependencies::RunExportsDownload, - render::solver::load_repodatas, - script::InterpreterError, - tool_configuration, + apply_patch_custom, metadata::Output, metadata::build_reindexed_channels, + recipe::parser::TestType, render::resolved_dependencies::RunExportsDownload, + render::solver::load_repodatas, script::InterpreterError, tool_configuration, }; /// Behavior for handling the working directory during the build process diff --git a/src/cache.rs b/src/cache.rs index ad27a4791..22f1b093e 100644 --- a/src/cache.rs +++ b/src/cache.rs @@ -11,7 +11,8 @@ use sha2::{Digest, Sha256}; use crate::{ env_vars, - metadata::{Output, build_reindexed_channels}, + metadata::Output, + metadata::build_reindexed_channels, packaging::Files, recipe::{ Jinja, diff --git a/src/lib.rs b/src/lib.rs index 367b20270..b30a73714 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -20,6 +20,7 @@ pub mod system_tools; pub mod tool_configuration; #[cfg(feature = "tui")] pub mod tui; +pub mod types; pub mod used_variables; pub mod utils; pub mod variant_config; @@ -40,7 +41,6 @@ mod windows; mod package_cache_reporter; pub mod source_code; -use crate::render::resolved_dependencies::RunExportsDownload; use std::{ collections::{BTreeMap, HashMap}, path::{Path, PathBuf}, @@ -55,10 +55,6 @@ use dialoguer::Confirm; use dunce::canonicalize; use fs_err as fs; use futures::FutureExt; -use metadata::{ - BuildConfiguration, BuildSummary, Directories, Output, PackageIdentifier, PackagingSettings, - build_reindexed_channels, -}; use miette::{Context, IntoDiagnostic}; pub use normalized_key::NormalizedKey; use opt::*; @@ -73,15 +69,20 @@ use rattler_solve::SolveStrategy; use rattler_virtual_packages::VirtualPackageOverrides; use recipe::parser::{Dependency, TestType, find_outputs_from_src}; use recipe::variable::Variable; +use render::resolved_dependencies::RunExportsDownload; use selectors::SelectorConfig; use source::patch::apply_patch_custom; use source_code::Source; use system_tools::SystemTools; use tool_configuration::{Configuration, ContinueOnFailure, SkipExisting, TestStrategy}; +use types::Directories; +use types::{ + BuildConfiguration, BuildSummary, PackageIdentifier, PackagingSettings, + build_reindexed_channels, +}; use variant_config::VariantConfig; -use crate::metadata::Debug; -use crate::metadata::PlatformWithVirtualPackages; +use crate::metadata::{Debug, Output, PlatformWithVirtualPackages}; /// Returns the recipe path. pub fn get_recipe_path(path: &Path) -> miette::Result { @@ -354,7 +355,7 @@ pub async fn get_build_output( let timestamp = chrono::Utc::now(); let virtual_package_override = VirtualPackageOverrides::from_env(); - let output = metadata::Output { + let output = Output { recipe: recipe.clone(), build_configuration: BuildConfiguration { target_platform: discovered_output.target_platform, @@ -821,7 +822,7 @@ pub async fn rebuild( let rendered_recipe = fs::read_to_string(temp_dir.join("rendered_recipe.yaml")).into_diagnostic()?; - let mut output: metadata::Output = serde_yaml::from_str(&rendered_recipe).into_diagnostic()?; + let mut output: Output = serde_yaml::from_str(&rendered_recipe).into_diagnostic()?; // set recipe dir to the temp folder output.build_configuration.directories.recipe_dir = temp_dir; diff --git a/src/metadata.rs b/src/metadata.rs index fa4839d74..d7a480319 100644 --- a/src/metadata.rs +++ b/src/metadata.rs @@ -1,821 +1,8 @@ //! All the metadata that makes up a recipe file -use std::{ - borrow::Cow, - collections::BTreeMap, - fmt::{self, Display, Formatter}, - io::Write, - iter, - path::{Path, PathBuf}, - str::FromStr, - sync::{Arc, Mutex}, +pub use crate::types::{ + BuildConfiguration, Debug, Output, PlatformWithVirtualPackages, build_reindexed_channels, }; -use chrono::{DateTime, Utc}; -use dunce::canonicalize; -use fs_err as fs; -use indicatif::HumanBytes; -use rattler_conda_types::{ - Channel, ChannelUrl, GenericVirtualPackage, PackageName, Platform, RepoDataRecord, - VersionWithSource, - compression_level::CompressionLevel, - package::{ArchiveType, PathType, PathsEntry, PathsJson}, -}; -use rattler_index::{IndexFsConfig, index_fs}; -use rattler_repodata_gateway::SubdirSelection; -use rattler_solve::{ChannelPriority, SolveStrategy}; -use rattler_virtual_packages::{ - DetectVirtualPackageError, VirtualPackageOverrides, VirtualPackages, -}; -use serde::{Deserialize, Deserializer, Serialize}; -use serde_json::Value; - -use crate::{ - console_utils::github_integration_enabled, - hash::HashInfo, - normalized_key::NormalizedKey, - recipe::{ - jinja::SelectorConfig, - parser::{Recipe, Source}, - variable::Variable, - }, - render::resolved_dependencies::FinalizedDependencies, - script::SandboxConfiguration, - system_tools::SystemTools, - tool_configuration, - utils::remove_dir_all_force, -}; -/// A Git revision -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct GitRev(String); - -impl FromStr for GitRev { - type Err = (); - - fn from_str(s: &str) -> Result { - Ok(GitRev(s.to_string())) - } -} - -impl Default for GitRev { - fn default() -> Self { - Self(String::from("HEAD")) - } -} - -impl Display for GitRev { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.0) - } -} - -/// Directories used during the build process -#[derive(Debug, Clone, Serialize, Deserialize, Default)] -pub struct Directories { - /// The directory where the recipe is located - #[serde(skip)] - pub recipe_dir: PathBuf, - /// The path where the recipe is located - #[serde(skip)] - pub recipe_path: PathBuf, - /// The folder where the cache is located - #[serde(skip)] - pub cache_dir: PathBuf, - /// The host prefix is the directory where host dependencies are installed - /// Exposed as `$PREFIX` (or `%PREFIX%` on Windows) in the build script - pub host_prefix: PathBuf, - /// The build prefix is the directory where build dependencies are installed - /// Exposed as `$BUILD_PREFIX` (or `%BUILD_PREFIX%` on Windows) in the build - /// script - pub build_prefix: PathBuf, - /// The work directory is the directory where the source code is copied to - pub work_dir: PathBuf, - /// The parent directory of host, build and work directories - pub build_dir: PathBuf, - /// The output directory or local channel directory - #[serde(skip)] - pub output_dir: PathBuf, -} - -fn get_build_dir( - output_dir: &Path, - name: &str, - no_build_id: bool, - timestamp: &DateTime, -) -> Result { - let since_the_epoch = timestamp.timestamp(); - - let dirname = if no_build_id { - format!("rattler-build_{}", name) - } else { - format!("rattler-build_{}_{:?}", name, since_the_epoch) - }; - Ok(output_dir.join("bld").join(dirname)) -} - -impl Directories { - /// Create all directories needed for the building of a package - pub fn setup( - name: &str, - recipe_path: &Path, - output_dir: &Path, - no_build_id: bool, - timestamp: &DateTime, - merge_build_and_host: bool, - ) -> Result { - if !output_dir.exists() { - fs::create_dir_all(output_dir)?; - } - let output_dir = canonicalize(output_dir)?; - - let build_dir = get_build_dir(&output_dir, name, no_build_id, timestamp) - .expect("Could not create build directory"); - // TODO move this into build_dir, and keep build_dir consistent. - let cache_dir = output_dir.join("build_cache"); - let recipe_dir = recipe_path - .parent() - .ok_or_else(|| { - std::io::Error::new(std::io::ErrorKind::NotFound, "Parent directory not found") - })? - .to_path_buf(); - - let host_prefix = if cfg!(target_os = "windows") { - build_dir.join("h_env") - } else { - let placeholder_template = "_placehold"; - let mut placeholder = String::new(); - let placeholder_length: usize = 255; - - while placeholder.len() < placeholder_length { - placeholder.push_str(placeholder_template); - } - - let placeholder = placeholder - [0..placeholder_length - build_dir.join("host_env").as_os_str().len()] - .to_string(); - - build_dir.join(format!("host_env{}", placeholder)) - }; - - let directories = Directories { - build_dir: build_dir.clone(), - build_prefix: if merge_build_and_host { - host_prefix.clone() - } else { - build_dir.join("build_env") - }, - cache_dir, - host_prefix, - work_dir: build_dir.join("work"), - recipe_dir, - recipe_path: recipe_path.to_path_buf(), - output_dir, - }; - - Ok(directories) - } - - /// Remove all directories except for the cache directory - pub fn clean(&self) -> Result<(), std::io::Error> { - if self.build_dir.exists() { - let folders = self.build_dir.read_dir()?; - for folder in folders { - let folder = folder?; - - if folder.path() == self.cache_dir { - continue; - } - - if folder.file_type()?.is_dir() { - remove_dir_all_force(&folder.path())?; - } - } - } - Ok(()) - } - - /// Creates the build directory. - pub fn create_build_dir(&self, remove_existing_work_dir: bool) -> Result<(), std::io::Error> { - if remove_existing_work_dir && self.work_dir.exists() { - fs::remove_dir_all(&self.work_dir)?; - } - - fs::create_dir_all(&self.work_dir)?; - - Ok(()) - } - - /// create all directories - pub fn recreate_directories(&self) -> Result<(), std::io::Error> { - if self.build_dir.exists() { - fs::remove_dir_all(&self.build_dir)?; - } - - if !self.output_dir.exists() { - fs::create_dir_all(&self.output_dir)?; - } - - fs::create_dir_all(&self.build_dir)?; - fs::create_dir_all(&self.work_dir)?; - fs::create_dir_all(&self.build_prefix)?; - fs::create_dir_all(&self.host_prefix)?; - - Ok(()) - } -} - -/// Default value for store recipe for backwards compatibility -fn default_true() -> bool { - true -} - -/// Settings when creating the package (compression etc.) -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PackagingSettings { - /// The archive type, currently supported are `tar.bz2` and `conda` - pub archive_type: ArchiveType, - /// The compression level from 1-9 or -7-22 for `tar.bz2` and `conda` - /// archives - pub compression_level: i32, -} - -impl PackagingSettings { - /// Create a new `PackagingSettings` from the command line arguments - /// and the selected archive type. - pub fn from_args(archive_type: ArchiveType, compression_level: CompressionLevel) -> Self { - let compression_level: i32 = match archive_type { - ArchiveType::TarBz2 => compression_level.to_bzip2_level().unwrap() as i32, - ArchiveType::Conda => compression_level.to_zstd_level().unwrap(), - }; - - Self { - archive_type, - compression_level, - } - } -} - -/// Defines both a platform and the virtual packages that describe the -/// capabilities of the platform. -#[derive(Debug, Clone, Serialize)] -pub struct PlatformWithVirtualPackages { - /// The platform - pub platform: Platform, - - /// The virtual packages for the platform - pub virtual_packages: Vec, -} - -impl PlatformWithVirtualPackages { - /// Returns the current platform and the virtual packages available on the - /// current system. - pub fn detect(overrides: &VirtualPackageOverrides) -> Result { - let platform = Platform::current(); - Self::detect_for_platform(platform, overrides) - } - - /// Detect the virtual packages for the given platform, filling in defaults where appropriate - pub fn detect_for_platform( - platform: Platform, - overrides: &VirtualPackageOverrides, - ) -> Result { - let virtual_packages = VirtualPackages::detect_for_platform(platform, overrides)? - .into_generic_virtual_packages() - .collect(); - Ok(Self { - platform, - virtual_packages, - }) - } -} - -impl<'de> Deserialize<'de> for PlatformWithVirtualPackages { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - #[derive(Deserialize)] - pub struct Object { - pub platform: Platform, - pub virtual_packages: Vec, - } - - serde_untagged::UntaggedEnumVisitor::new() - .string(|s| { - Ok(Self { - platform: Platform::from_str(s).map_err(serde::de::Error::custom)?, - virtual_packages: vec![], - }) - }) - .map(|m| { - let object: Object = m.deserialize()?; - Ok(Self { - platform: object.platform, - virtual_packages: object.virtual_packages, - }) - }) - .deserialize(deserializer) - } -} - -/// A newtype wrapper around a boolean indicating whether debug output is enabled -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] -pub struct Debug(bool); - -impl Debug { - /// Create a new Debug instance - pub fn new(debug: bool) -> Self { - Self(debug) - } - - /// Returns true if debug output is enabled - pub fn is_enabled(&self) -> bool { - self.0 - } -} - -/// The configuration for a build of a package -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct BuildConfiguration { - /// The target platform for the build - pub target_platform: Platform, - /// The host platform (usually target platform, but for `noarch` it's the - /// build platform) - pub host_platform: PlatformWithVirtualPackages, - /// The build platform (the platform that the build is running on) - pub build_platform: PlatformWithVirtualPackages, - /// The selected variant for this build - pub variant: BTreeMap, - /// THe computed hash of the variant - pub hash: HashInfo, - /// The directories for the build (work, source, build, host, ...) - pub directories: Directories, - /// The channels to use when resolving environments - pub channels: Vec, - /// The channel priority that is used to resolve dependencies - pub channel_priority: ChannelPriority, - /// The solve strategy to use when resolving dependencies - pub solve_strategy: SolveStrategy, - /// The timestamp to use for the build - pub timestamp: chrono::DateTime, - /// All subpackages coming from this output or other outputs from the same - /// recipe - pub subpackages: BTreeMap, - /// Package format (.tar.bz2 or .conda) - pub packaging_settings: PackagingSettings, - /// Whether to store the recipe and build instructions in the final package - /// or not - #[serde(skip_serializing, default = "default_true")] - pub store_recipe: bool, - /// Whether to set additional environment variables to force colors in the - /// build script or not - #[serde(skip_serializing, default = "default_true")] - pub force_colors: bool, - - /// The configuration for the sandbox - #[serde(skip_serializing, default)] - pub sandbox_config: Option, - /// Whether to enable debug output in build scripts - #[serde(skip_serializing, default)] - pub debug: Debug, - /// Exclude packages newer than this date from the solver - #[serde(skip_serializing, default)] - pub exclude_newer: Option>, -} - -impl BuildConfiguration { - /// true if the build is cross-compiling - pub fn cross_compilation(&self) -> bool { - self.target_platform != self.build_platform.platform - } - - /// Retrieve the sandbox configuration for this output - pub fn sandbox_config(&self) -> Option<&SandboxConfiguration> { - self.sandbox_config.as_ref() - } - - /// Construct a `SelectorConfig` from the given `BuildConfiguration` - pub fn selector_config(&self) -> SelectorConfig { - SelectorConfig { - target_platform: self.target_platform, - host_platform: self.host_platform.platform, - build_platform: self.build_platform.platform, - variant: self.variant.clone(), - hash: Some(self.hash.clone()), - experimental: false, - allow_undefined: false, - recipe_path: None, - } - } -} - -/// A package identifier -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct PackageIdentifier { - /// The name of the package - pub name: PackageName, - /// The version of the package - pub version: VersionWithSource, - /// The build string of the package - pub build_string: String, -} - -/// The summary of a build -#[derive(Debug, Clone, Default)] -pub struct BuildSummary { - /// The start time of the build - pub build_start: Option>, - /// The end time of the build - pub build_end: Option>, - - /// The path to the artifact - pub artifact: Option, - /// Any warnings that were recorded during the build - pub warnings: Vec, - /// The paths that are packaged in the artifact - pub paths: Option, - /// Whether the build was successful or not - pub failed: bool, -} - -/// A output. This is the central element that is passed to the `run_build` -/// function and fully specifies all the options and settings to run the build. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Output { - /// The rendered recipe that is used to build this output - pub recipe: Recipe, - /// The build configuration for this output (e.g. target_platform, channels, - /// and other settings) - pub build_configuration: BuildConfiguration, - /// The finalized dependencies for this output. If this is `None`, the - /// dependencies have not been resolved yet. During the `run_build` - /// functions, the dependencies are resolved and this field is filled. - pub finalized_dependencies: Option, - /// The finalized sources for this output. Contain the exact git hashes for - /// the sources that are used to build this output. - pub finalized_sources: Option>, - - /// The finalized dependencies from the cache (if there is a cache - /// instruction) - #[serde(skip_serializing_if = "Option::is_none")] - pub finalized_cache_dependencies: Option, - /// The finalized sources from the cache (if there is a cache instruction) - #[serde(skip_serializing_if = "Option::is_none")] - pub finalized_cache_sources: Option>, - - /// Summary of the build - #[serde(skip)] - pub build_summary: Arc>, - /// The system tools that are used to build this output - pub system_tools: SystemTools, - /// Some extra metadata that should be recorded additionally in about.json - /// Usually it is used during the CI build to record link to the CI job - /// that created this artifact - #[serde(skip_serializing_if = "Option::is_none")] - pub extra_meta: Option>, -} - -impl Output { - /// The name of the package - pub fn name(&self) -> &PackageName { - self.recipe.package().name() - } - - /// The version of the package - pub fn version(&self) -> &VersionWithSource { - self.recipe.package().version() - } - - /// The build string is either the build string from the recipe or computed - /// from the hash and build number. - pub fn build_string(&self) -> Cow<'_, str> { - self.recipe - .build() - .string - .as_resolved() - .expect("Build string is not resolved") - .into() - } - - /// retrieve an identifier for this output ({name}-{version}-{build_string}) - pub fn identifier(&self) -> String { - format!( - "{}-{}-{}", - self.name().as_normalized(), - self.version(), - &self.build_string() - ) - } - - /// Record a warning during the build - pub fn record_warning(&self, warning: &str) { - self.build_summary - .lock() - .unwrap() - .warnings - .push(warning.to_string()); - } - - /// Record the start of the build - pub fn record_build_start(&self) { - self.build_summary.lock().unwrap().build_start = Some(chrono::Utc::now()); - } - - /// Record the artifact that was created during the build - pub fn record_artifact(&self, artifact: &Path, paths: &PathsJson) { - let mut summary = self.build_summary.lock().unwrap(); - summary.artifact = Some(artifact.to_path_buf()); - summary.paths = Some(paths.clone()); - } - - /// Record the end of the build - pub fn record_build_end(&self) { - let mut summary = self.build_summary.lock().unwrap(); - summary.build_end = Some(chrono::Utc::now()); - } - - /// Shorthand to retrieve the variant configuration for this output - pub fn variant(&self) -> &BTreeMap { - &self.build_configuration.variant - } - - /// Shorthand to retrieve the host prefix for this output - pub fn prefix(&self) -> &Path { - &self.build_configuration.directories.host_prefix - } - - /// Shorthand to retrieve the build prefix for this output - pub fn build_prefix(&self) -> &Path { - &self.build_configuration.directories.build_prefix - } - - /// Shorthand to retrieve the target platform for this output - pub fn target_platform(&self) -> &Platform { - &self.build_configuration.target_platform - } - - /// Shorthand to retrieve the target platform for this output - pub fn host_platform(&self) -> &PlatformWithVirtualPackages { - &self.build_configuration.host_platform - } - - /// Search for the resolved package with the given name in the host prefix - /// Returns a tuple of the package and a boolean indicating whether the - /// package is directly requested - pub fn find_resolved_package(&self, name: &str) -> Option<(&RepoDataRecord, bool)> { - let host = self.finalized_dependencies.as_ref()?.host.as_ref()?; - let record = host - .resolved - .iter() - .find(|p| p.package_record.name.as_normalized() == name); - - let is_requested = host.specs.iter().any(|s| { - s.spec() - .name - .as_ref() - .map(|n| n.as_normalized() == name) - .unwrap_or(false) - }); - - record.map(|r| (r, is_requested)) - } - - /// Print a nice summary of the build - pub fn log_build_summary(&self) -> Result<(), std::io::Error> { - let summary = self.build_summary.lock().unwrap(); - let identifier = self.identifier(); - let span = tracing::info_span!("Build summary for", recipe = identifier); - let _enter = span.enter(); - - if let Some(artifact) = &summary.artifact { - let bytes = HumanBytes(fs::metadata(artifact).map(|m| m.len()).unwrap_or(0)); - tracing::info!("Artifact: {} ({})", artifact.display(), bytes); - } else { - tracing::info!("No artifact was created"); - } - tracing::info!("{}", self); - - if !summary.warnings.is_empty() { - tracing::warn!("Warnings:"); - for warning in &summary.warnings { - tracing::warn!("{}", warning); - } - } - - if let Ok(github_summary) = std::env::var("GITHUB_STEP_SUMMARY") { - if !github_integration_enabled() { - return Ok(()); - } - // append to the summary file - let mut summary_file = fs::OpenOptions::new() - .append(true) - .create(true) - .open(github_summary)?; - - writeln!(summary_file, "### Build summary for {}", identifier)?; - if let Some(article) = &summary.artifact { - let bytes = HumanBytes(fs::metadata(article).map(|m| m.len()).unwrap_or(0)); - writeln!( - summary_file, - "**Artifact**: {} ({})", - article.display(), - bytes - )?; - } else { - writeln!(summary_file, "**No artifact was created**")?; - } - - if let Some(paths) = &summary.paths { - if paths.paths.is_empty() { - writeln!(summary_file, "Included files: **No files included**")?; - } else { - /// Github detail expander - fn format_entry(entry: &PathsEntry) -> String { - let mut extra_info = Vec::new(); - if entry.prefix_placeholder.is_some() { - extra_info.push("contains prefix"); - } - if entry.no_link { - extra_info.push("no link"); - } - match entry.path_type { - PathType::SoftLink => extra_info.push("soft link"), - // skip default - PathType::HardLink => {} - PathType::Directory => extra_info.push("directory"), - } - let bytes = entry.size_in_bytes.unwrap_or(0); - - format!( - "| `{}` | {} | {} |", - entry.relative_path.to_string_lossy(), - HumanBytes(bytes), - extra_info.join(", ") - ) - } - - writeln!(summary_file, "
")?; - writeln!( - summary_file, - "Included files ({} files)\n", - paths.paths.len() - )?; - writeln!(summary_file, "| Path | Size | Extra info |")?; - writeln!(summary_file, "| --- | --- | --- |")?; - for path in &paths.paths { - writeln!(summary_file, "{}", format_entry(path))?; - } - writeln!(summary_file, "\n
\n")?; - } - } - - if !summary.warnings.is_empty() { - writeln!(summary_file, "> [!WARNING]")?; - writeln!(summary_file, "> **Warnings during build:**\n>")?; - for warning in &summary.warnings { - writeln!(summary_file, "> - {}", warning)?; - } - writeln!(summary_file)?; - } - - writeln!( - summary_file, - "
Resolved dependencies\n\n{}\n
\n", - self.format_as_markdown() - )?; - } - Ok(()) - } -} - -impl Output { - /// Format the output as a markdown table - pub fn format_as_markdown(&self) -> String { - let mut output = String::new(); - self.format_table_with_option(&mut output, comfy_table::presets::ASCII_MARKDOWN, true) - .expect("Could not format table"); - output - } - - fn format_table_with_option( - &self, - f: &mut impl fmt::Write, - table_format: &str, - long: bool, - ) -> std::fmt::Result { - let template = || -> comfy_table::Table { - let mut table = comfy_table::Table::new(); - if table_format == comfy_table::presets::UTF8_FULL { - table - .load_preset(comfy_table::presets::UTF8_FULL_CONDENSED) - .apply_modifier(comfy_table::modifiers::UTF8_ROUND_CORNERS); - } else { - table.load_preset(table_format); - } - table - }; - - writeln!(f, "Variant configuration (hash: {}):", self.build_string())?; - let mut table = template(); - if table_format != comfy_table::presets::UTF8_FULL { - table.set_header(["Key", "Value"]); - } - self.build_configuration.variant.iter().for_each(|(k, v)| { - table.add_row([k.normalize(), format!("{:?}", v)]); - }); - writeln!(f, "{}\n", table)?; - - if let Some(finalized_dependencies) = &self.finalized_dependencies { - if let Some(build) = &finalized_dependencies.build { - writeln!(f, "Build dependencies:")?; - writeln!(f, "{}\n", build.to_table(template(), long))?; - } - - if let Some(host) = &finalized_dependencies.host { - writeln!(f, "Host dependencies:")?; - writeln!(f, "{}\n", host.to_table(template(), long))?; - } - - if !finalized_dependencies.run.depends.is_empty() { - writeln!(f, "Run dependencies:")?; - writeln!( - f, - "{}\n", - finalized_dependencies.run.to_table(template(), long) - )?; - } - } - - Ok(()) - } -} - -impl Display for Output { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - self.format_table_with_option(f, comfy_table::presets::UTF8_FULL, false) - } -} - -/// Builds the channel list and reindexes the output channel. -pub async fn build_reindexed_channels( - build_configuration: &BuildConfiguration, - tool_configuration: &tool_configuration::Configuration, -) -> Result, std::io::Error> { - let output_dir = &build_configuration.directories.output_dir; - let output_channel = Channel::from_directory(output_dir); - - // Clear the repodata gateway of any cached values for the output channel. - tool_configuration.repodata_gateway.clear_repodata_cache( - &output_channel, - SubdirSelection::Some( - [build_configuration.target_platform] - .iter() - .map(ToString::to_string) - .collect(), - ), - ); - - let index_config = IndexFsConfig { - channel: output_dir.clone(), - target_platform: Some(build_configuration.target_platform), - repodata_patch: None, - write_zst: false, - write_shards: false, - force: false, - max_parallel: num_cpus::get_physical(), - multi_progress: None, - }; - - // Reindex the output channel from the files on disk - index_fs(index_config) - .await - .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?; - - Ok(iter::once(output_channel.base_url) - .chain(build_configuration.channels.iter().cloned()) - .collect()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn setup_build_dir_test() { - // without build_id (aka timestamp) - let dir = tempfile::tempdir().unwrap(); - let p1 = get_build_dir(dir.path(), "name", true, &Utc::now()).unwrap(); - let f1 = p1.file_name().unwrap(); - assert!(f1.eq("rattler-build_name")); - - // with build_id (aka timestamp) - let timestamp = &Utc::now(); - let p2 = get_build_dir(dir.path(), "name", false, timestamp).unwrap(); - let f2 = p2.file_name().unwrap(); - let epoch = timestamp.timestamp(); - assert!(f2.eq(format!("rattler-build_name_{epoch}").as_str())); - } -} - #[cfg(test)] mod test { use chrono::TimeZone; @@ -830,32 +17,9 @@ mod test { use std::str::FromStr; use url::Url; - use super::{Directories, Output}; + use super::Output; use crate::render::resolved_dependencies::{self, SourceDependency}; - #[test] - fn test_directories_yaml_rendering() { - let tempdir = tempfile::tempdir().unwrap(); - - let directories = Directories::setup( - "name", - &tempdir.path().join("recipe"), - &tempdir.path().join("output"), - false, - &chrono::Utc::now(), - false, - ) - .unwrap(); - directories.create_build_dir(false).unwrap(); - - // test yaml roundtrip - let yaml = serde_yaml::to_string(&directories).unwrap(); - let directories2: Directories = serde_yaml::from_str(&yaml).unwrap(); - assert_eq!(directories.build_dir, directories2.build_dir); - assert_eq!(directories.build_prefix, directories2.build_prefix); - assert_eq!(directories.host_prefix, directories2.host_prefix); - } - #[test] fn test_resolved_dependencies_rendering() { let resolved_dependencies = resolved_dependencies::ResolvedDependencies { diff --git a/src/package_test/content_test.rs b/src/package_test/content_test.rs index ff75f21fd..f09ad8139 100644 --- a/src/package_test/content_test.rs +++ b/src/package_test/content_test.rs @@ -1,8 +1,9 @@ use std::collections::HashSet; use std::path::PathBuf; +use crate::metadata::Output; +use crate::package_test::TestError; use crate::recipe::parser::PackageContentsTest; -use crate::{metadata::Output, package_test::TestError}; use globset::{Glob, GlobBuilder, GlobSet}; use rattler_conda_types::{Platform, package::PathsJson}; diff --git a/src/post_process/relink.rs b/src/post_process/relink.rs index e48e7be1f..7e841a6b1 100644 --- a/src/post_process/relink.rs +++ b/src/post_process/relink.rs @@ -1,10 +1,10 @@ use fs_err as fs; -use crate::metadata::Output; use crate::packaging::TempFiles; use crate::linux::link::SharedObject; use crate::macos::link::Dylib; +use crate::metadata::Output; use crate::recipe::parser::GlobVec; use crate::system_tools::{SystemTools, ToolError}; use crate::windows::link::Dll; diff --git a/src/render/resolved_dependencies.rs b/src/render/resolved_dependencies.rs index d95778276..fa06cdbbb 100644 --- a/src/render/resolved_dependencies.rs +++ b/src/render/resolved_dependencies.rs @@ -18,15 +18,15 @@ use thiserror::Error; use super::pin::PinError; use crate::{ - metadata::{BuildConfiguration, Output, build_reindexed_channels}, + metadata::Output, + metadata::{BuildConfiguration, build_reindexed_channels}, package_cache_reporter::PackageCacheReporter, recipe::parser::{Dependency, Requirements}, render::{ pin::PinArgs, solver::{install_packages, solve_environment}, }, - tool_configuration, - tool_configuration::Configuration, + tool_configuration::{self, Configuration}, }; use super::reporters::GatewayReporter; diff --git a/src/script/mod.rs b/src/script/mod.rs index d45801454..e5a4cee27 100644 --- a/src/script/mod.rs +++ b/src/script/mod.rs @@ -31,7 +31,8 @@ use tokio_util::{ use crate::{ env_vars::{self}, - metadata::{Debug, Output}, + metadata::Debug, + metadata::Output, recipe::{ Jinja, parser::{Script, ScriptContent}, diff --git a/src/source/mod.rs b/src/source/mod.rs index ea44912e7..cc5229d58 100644 --- a/src/source/mod.rs +++ b/src/source/mod.rs @@ -6,7 +6,6 @@ use std::{ }; use crate::{ - metadata::{Directories, Output}, recipe::parser::{GitRev, GitSource, Source}, source::{ checksum::Checksum, @@ -14,6 +13,7 @@ use crate::{ }, system_tools::ToolError, tool_configuration, + types::{Directories, Output}, }; use fs_err as fs; diff --git a/src/types/build_configuration.rs b/src/types/build_configuration.rs new file mode 100644 index 000000000..791dfff2a --- /dev/null +++ b/src/types/build_configuration.rs @@ -0,0 +1,95 @@ +//! All the metadata that makes up a recipe file +use std::collections::BTreeMap; + +use rattler_conda_types::{ChannelUrl, PackageName, Platform}; +use rattler_solve::{ChannelPriority, SolveStrategy}; +use serde::{Deserialize, Serialize}; + +use crate::{ + hash::HashInfo, + normalized_key::NormalizedKey, + recipe::{jinja::SelectorConfig, variable::Variable}, + script::SandboxConfiguration, + types::{ + Debug, Directories, PackageIdentifier, PackagingSettings, PlatformWithVirtualPackages, + }, +}; + +/// Default value for store recipe for backwards compatibility +fn default_true() -> bool { + true +} +/// The configuration for a build of a package +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BuildConfiguration { + /// The target platform for the build + pub target_platform: Platform, + /// The host platform (usually target platform, but for `noarch` it's the + /// build platform) + pub host_platform: PlatformWithVirtualPackages, + /// The build platform (the platform that the build is running on) + pub build_platform: PlatformWithVirtualPackages, + /// The selected variant for this build + pub variant: BTreeMap, + /// THe computed hash of the variant + pub hash: HashInfo, + /// The directories for the build (work, source, build, host, ...) + pub directories: Directories, + /// The channels to use when resolving environments + pub channels: Vec, + /// The channel priority that is used to resolve dependencies + pub channel_priority: ChannelPriority, + /// The solve strategy to use when resolving dependencies + pub solve_strategy: SolveStrategy, + /// The timestamp to use for the build + pub timestamp: chrono::DateTime, + /// All subpackages coming from this output or other outputs from the same + /// recipe + pub subpackages: BTreeMap, + /// Package format (.tar.bz2 or .conda) + pub packaging_settings: PackagingSettings, + /// Whether to store the recipe and build instructions in the final package + /// or not + #[serde(skip_serializing, default = "default_true")] + pub store_recipe: bool, + /// Whether to set additional environment variables to force colors in the + /// build script or not + #[serde(skip_serializing, default = "default_true")] + pub force_colors: bool, + + /// The configuration for the sandbox + #[serde(skip_serializing, default)] + pub sandbox_config: Option, + /// Whether to enable debug output in build scripts + #[serde(skip_serializing, default)] + pub debug: Debug, + /// Exclude packages newer than this date from the solver + #[serde(skip_serializing, default)] + pub exclude_newer: Option>, +} + +impl BuildConfiguration { + /// true if the build is cross-compiling + pub fn cross_compilation(&self) -> bool { + self.target_platform != self.build_platform.platform + } + + /// Retrieve the sandbox configuration for this output + pub fn sandbox_config(&self) -> Option<&SandboxConfiguration> { + self.sandbox_config.as_ref() + } + + /// Construct a `SelectorConfig` from the given `BuildConfiguration` + pub fn selector_config(&self) -> SelectorConfig { + SelectorConfig { + target_platform: self.target_platform, + host_platform: self.host_platform.platform, + build_platform: self.build_platform.platform, + variant: self.variant.clone(), + hash: Some(self.hash.clone()), + experimental: false, + allow_undefined: false, + recipe_path: None, + } + } +} diff --git a/src/types/build_output.rs b/src/types/build_output.rs new file mode 100644 index 000000000..c083fe28a --- /dev/null +++ b/src/types/build_output.rs @@ -0,0 +1,341 @@ +use fs_err as fs; +use indicatif::HumanBytes; +use rattler_conda_types::{ + PackageName, Platform, RepoDataRecord, VersionWithSource, + package::{PathType, PathsEntry, PathsJson}, +}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::{ + borrow::Cow, + collections::BTreeMap, + fmt::{self, Display, Formatter}, + io::Write, + path::Path, + sync::{Arc, Mutex}, +}; + +use crate::{ + NormalizedKey, + console_utils::github_integration_enabled, + recipe::{Recipe, parser::Source, variable::Variable}, + render::resolved_dependencies::FinalizedDependencies, + system_tools::SystemTools, + types::{BuildConfiguration, BuildSummary, PlatformWithVirtualPackages}, +}; + +/// A output. This is the central element that is passed to the `run_build` +/// function and fully specifies all the options and settings to run the build. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BuildOutput { + /// The rendered recipe that is used to build this output + pub recipe: Recipe, + /// The build configuration for this output (e.g. target_platform, channels, + /// and other settings) + pub build_configuration: BuildConfiguration, + /// The finalized dependencies for this output. If this is `None`, the + /// dependencies have not been resolved yet. During the `run_build` + /// functions, the dependencies are resolved and this field is filled. + pub finalized_dependencies: Option, + /// The finalized sources for this output. Contain the exact git hashes for + /// the sources that are used to build this output. + pub finalized_sources: Option>, + + /// The finalized dependencies from the cache (if there is a cache + /// instruction) + #[serde(skip_serializing_if = "Option::is_none")] + pub finalized_cache_dependencies: Option, + /// The finalized sources from the cache (if there is a cache instruction) + #[serde(skip_serializing_if = "Option::is_none")] + pub finalized_cache_sources: Option>, + + /// Summary of the build + #[serde(skip)] + pub build_summary: Arc>, + /// The system tools that are used to build this output + pub system_tools: SystemTools, + /// Some extra metadata that should be recorded additionally in about.json + /// Usually it is used during the CI build to record link to the CI job + /// that created this artifact + #[serde(skip_serializing_if = "Option::is_none")] + pub extra_meta: Option>, +} + +impl BuildOutput { + /// The name of the package + pub fn name(&self) -> &PackageName { + self.recipe.package().name() + } + + /// The version of the package + pub fn version(&self) -> &VersionWithSource { + self.recipe.package().version() + } + + /// The build string is either the build string from the recipe or computed + /// from the hash and build number. + pub fn build_string(&self) -> Cow<'_, str> { + self.recipe + .build() + .string + .as_resolved() + .expect("Build string is not resolved") + .into() + } + + /// retrieve an identifier for this output ({name}-{version}-{build_string}) + pub fn identifier(&self) -> String { + format!( + "{}-{}-{}", + self.name().as_normalized(), + self.version(), + &self.build_string() + ) + } + + /// Record a warning during the build + pub fn record_warning(&self, warning: &str) { + self.build_summary + .lock() + .unwrap() + .warnings + .push(warning.to_string()); + } + + /// Record the start of the build + pub fn record_build_start(&self) { + self.build_summary.lock().unwrap().build_start = Some(chrono::Utc::now()); + } + + /// Record the artifact that was created during the build + pub fn record_artifact(&self, artifact: &Path, paths: &PathsJson) { + let mut summary = self.build_summary.lock().unwrap(); + summary.artifact = Some(artifact.to_path_buf()); + summary.paths = Some(paths.clone()); + } + + /// Record the end of the build + pub fn record_build_end(&self) { + let mut summary = self.build_summary.lock().unwrap(); + summary.build_end = Some(chrono::Utc::now()); + } + + /// Shorthand to retrieve the variant configuration for this output + pub fn variant(&self) -> &BTreeMap { + &self.build_configuration.variant + } + + /// Shorthand to retrieve the host prefix for this output + pub fn prefix(&self) -> &Path { + &self.build_configuration.directories.host_prefix + } + + /// Shorthand to retrieve the build prefix for this output + pub fn build_prefix(&self) -> &Path { + &self.build_configuration.directories.build_prefix + } + + /// Shorthand to retrieve the target platform for this output + pub fn target_platform(&self) -> &Platform { + &self.build_configuration.target_platform + } + + /// Shorthand to retrieve the target platform for this output + pub fn host_platform(&self) -> &PlatformWithVirtualPackages { + &self.build_configuration.host_platform + } + + /// Search for the resolved package with the given name in the host prefix + /// Returns a tuple of the package and a boolean indicating whether the + /// package is directly requested + pub fn find_resolved_package(&self, name: &str) -> Option<(&RepoDataRecord, bool)> { + let host = self.finalized_dependencies.as_ref()?.host.as_ref()?; + let record = host + .resolved + .iter() + .find(|p| p.package_record.name.as_normalized() == name); + + let is_requested = host.specs.iter().any(|s| { + s.spec() + .name + .as_ref() + .map(|n| n.as_normalized() == name) + .unwrap_or(false) + }); + + record.map(|r| (r, is_requested)) + } + + /// Print a nice summary of the build + pub fn log_build_summary(&self) -> Result<(), std::io::Error> { + let summary = self.build_summary.lock().unwrap(); + let identifier = self.identifier(); + let span = tracing::info_span!("Build summary for", recipe = identifier); + let _enter = span.enter(); + + if let Some(artifact) = &summary.artifact { + let bytes = HumanBytes(fs::metadata(artifact).map(|m| m.len()).unwrap_or(0)); + tracing::info!("Artifact: {} ({})", artifact.display(), bytes); + } else { + tracing::info!("No artifact was created"); + } + tracing::info!("{}", self); + + if !summary.warnings.is_empty() { + tracing::warn!("Warnings:"); + for warning in &summary.warnings { + tracing::warn!("{}", warning); + } + } + + if let Ok(github_summary) = std::env::var("GITHUB_STEP_SUMMARY") { + if !github_integration_enabled() { + return Ok(()); + } + // append to the summary file + let mut summary_file = fs::OpenOptions::new() + .append(true) + .create(true) + .open(github_summary)?; + + writeln!(summary_file, "### Build summary for {}", identifier)?; + if let Some(article) = &summary.artifact { + let bytes = HumanBytes(fs::metadata(article).map(|m| m.len()).unwrap_or(0)); + writeln!( + summary_file, + "**Artifact**: {} ({})", + article.display(), + bytes + )?; + } else { + writeln!(summary_file, "**No artifact was created**")?; + } + + if let Some(paths) = &summary.paths { + if paths.paths.is_empty() { + writeln!(summary_file, "Included files: **No files included**")?; + } else { + /// Github detail expander + fn format_entry(entry: &PathsEntry) -> String { + let mut extra_info = Vec::new(); + if entry.prefix_placeholder.is_some() { + extra_info.push("contains prefix"); + } + if entry.no_link { + extra_info.push("no link"); + } + match entry.path_type { + PathType::SoftLink => extra_info.push("soft link"), + // skip default + PathType::HardLink => {} + PathType::Directory => extra_info.push("directory"), + } + let bytes = entry.size_in_bytes.unwrap_or(0); + + format!( + "| `{}` | {} | {} |", + entry.relative_path.to_string_lossy(), + HumanBytes(bytes), + extra_info.join(", ") + ) + } + + writeln!(summary_file, "
")?; + writeln!( + summary_file, + "Included files ({} files)\n", + paths.paths.len() + )?; + writeln!(summary_file, "| Path | Size | Extra info |")?; + writeln!(summary_file, "| --- | --- | --- |")?; + for path in &paths.paths { + writeln!(summary_file, "{}", format_entry(path))?; + } + writeln!(summary_file, "\n
\n")?; + } + } + + if !summary.warnings.is_empty() { + writeln!(summary_file, "> [!WARNING]")?; + writeln!(summary_file, "> **Warnings during build:**\n>")?; + for warning in &summary.warnings { + writeln!(summary_file, "> - {}", warning)?; + } + writeln!(summary_file)?; + } + + writeln!( + summary_file, + "
Resolved dependencies\n\n{}\n
\n", + self.format_as_markdown() + )?; + } + Ok(()) + } + + /// Format the output as a markdown table + pub fn format_as_markdown(&self) -> String { + let mut output = String::new(); + self.format_table_with_option(&mut output, comfy_table::presets::ASCII_MARKDOWN, true) + .expect("Could not format table"); + output + } + + fn format_table_with_option( + &self, + f: &mut impl fmt::Write, + table_format: &str, + long: bool, + ) -> std::fmt::Result { + let template = || -> comfy_table::Table { + let mut table = comfy_table::Table::new(); + if table_format == comfy_table::presets::UTF8_FULL { + table + .load_preset(comfy_table::presets::UTF8_FULL_CONDENSED) + .apply_modifier(comfy_table::modifiers::UTF8_ROUND_CORNERS); + } else { + table.load_preset(table_format); + } + table + }; + + writeln!(f, "Variant configuration (hash: {}):", self.build_string())?; + let mut table = template(); + if table_format != comfy_table::presets::UTF8_FULL { + table.set_header(["Key", "Value"]); + } + self.build_configuration.variant.iter().for_each(|(k, v)| { + table.add_row([k.normalize(), format!("{:?}", v)]); + }); + writeln!(f, "{}\n", table)?; + + if let Some(finalized_dependencies) = &self.finalized_dependencies { + if let Some(build) = &finalized_dependencies.build { + writeln!(f, "Build dependencies:")?; + writeln!(f, "{}\n", build.to_table(template(), long))?; + } + + if let Some(host) = &finalized_dependencies.host { + writeln!(f, "Host dependencies:")?; + writeln!(f, "{}\n", host.to_table(template(), long))?; + } + + if !finalized_dependencies.run.depends.is_empty() { + writeln!(f, "Run dependencies:")?; + writeln!( + f, + "{}\n", + finalized_dependencies.run.to_table(template(), long) + )?; + } + } + + Ok(()) + } +} + +impl Display for BuildOutput { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + self.format_table_with_option(f, comfy_table::presets::UTF8_FULL, false) + } +} diff --git a/src/types/directories.rs b/src/types/directories.rs new file mode 100644 index 000000000..cf976d8a8 --- /dev/null +++ b/src/types/directories.rs @@ -0,0 +1,208 @@ +use std::path::{Path, PathBuf}; + +use chrono::{DateTime, Utc}; +use fs_err as fs; +use serde::{Deserialize, Serialize}; + +use dunce::canonicalize; + +use crate::utils::remove_dir_all_force; + +/// Directories used during the build process +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct Directories { + /// The directory where the recipe is located + #[serde(skip)] + pub recipe_dir: PathBuf, + /// The path where the recipe is located + #[serde(skip)] + pub recipe_path: PathBuf, + /// The folder where the cache is located + #[serde(skip)] + pub cache_dir: PathBuf, + /// The host prefix is the directory where host dependencies are installed + /// Exposed as `$PREFIX` (or `%PREFIX%` on Windows) in the build script + pub host_prefix: PathBuf, + /// The build prefix is the directory where build dependencies are installed + /// Exposed as `$BUILD_PREFIX` (or `%BUILD_PREFIX%` on Windows) in the build + /// script + pub build_prefix: PathBuf, + /// The work directory is the directory where the source code is copied to + pub work_dir: PathBuf, + /// The parent directory of host, build and work directories + pub build_dir: PathBuf, + /// The output directory or local channel directory + #[serde(skip)] + pub output_dir: PathBuf, +} + +fn get_build_dir( + output_dir: &Path, + name: &str, + no_build_id: bool, + timestamp: &DateTime, +) -> Result { + let since_the_epoch = timestamp.timestamp(); + + let dirname = if no_build_id { + format!("rattler-build_{}", name) + } else { + format!("rattler-build_{}_{:?}", name, since_the_epoch) + }; + Ok(output_dir.join("bld").join(dirname)) +} + +impl Directories { + /// Create all directories needed for the building of a package + pub fn setup( + name: &str, + recipe_path: &Path, + output_dir: &Path, + no_build_id: bool, + timestamp: &DateTime, + merge_build_and_host: bool, + ) -> Result { + if !output_dir.exists() { + fs::create_dir_all(output_dir)?; + } + let output_dir = canonicalize(output_dir)?; + + let build_dir = get_build_dir(&output_dir, name, no_build_id, timestamp) + .expect("Could not create build directory"); + // TODO move this into build_dir, and keep build_dir consistent. + let cache_dir = output_dir.join("build_cache"); + let recipe_dir = recipe_path + .parent() + .ok_or_else(|| { + std::io::Error::new(std::io::ErrorKind::NotFound, "Parent directory not found") + })? + .to_path_buf(); + + let host_prefix = if cfg!(target_os = "windows") { + build_dir.join("h_env") + } else { + let placeholder_template = "_placehold"; + let mut placeholder = String::new(); + let placeholder_length: usize = 255; + + while placeholder.len() < placeholder_length { + placeholder.push_str(placeholder_template); + } + + let placeholder = placeholder + [0..placeholder_length - build_dir.join("host_env").as_os_str().len()] + .to_string(); + + build_dir.join(format!("host_env{}", placeholder)) + }; + + let directories = Directories { + build_dir: build_dir.clone(), + build_prefix: if merge_build_and_host { + host_prefix.clone() + } else { + build_dir.join("build_env") + }, + cache_dir, + host_prefix, + work_dir: build_dir.join("work"), + recipe_dir, + recipe_path: recipe_path.to_path_buf(), + output_dir, + }; + + Ok(directories) + } + + /// Remove all directories except for the cache directory + pub fn clean(&self) -> Result<(), std::io::Error> { + if self.build_dir.exists() { + let folders = self.build_dir.read_dir()?; + for folder in folders { + let folder = folder?; + + if folder.path() == self.cache_dir { + continue; + } + + if folder.file_type()?.is_dir() { + remove_dir_all_force(&folder.path())?; + } + } + } + Ok(()) + } + + /// Creates the build directory. + pub fn create_build_dir(&self, remove_existing_work_dir: bool) -> Result<(), std::io::Error> { + if remove_existing_work_dir && self.work_dir.exists() { + fs::remove_dir_all(&self.work_dir)?; + } + + fs::create_dir_all(&self.work_dir)?; + + Ok(()) + } + + /// create all directories + pub fn recreate_directories(&self) -> Result<(), std::io::Error> { + if self.build_dir.exists() { + fs::remove_dir_all(&self.build_dir)?; + } + + if !self.output_dir.exists() { + fs::create_dir_all(&self.output_dir)?; + } + + fs::create_dir_all(&self.build_dir)?; + fs::create_dir_all(&self.work_dir)?; + fs::create_dir_all(&self.build_prefix)?; + fs::create_dir_all(&self.host_prefix)?; + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn setup_build_dir_test() { + // without build_id (aka timestamp) + let dir = tempfile::tempdir().unwrap(); + let p1 = get_build_dir(dir.path(), "name", true, &Utc::now()).unwrap(); + let f1 = p1.file_name().unwrap(); + assert!(f1.eq("rattler-build_name")); + + // with build_id (aka timestamp) + let timestamp = &Utc::now(); + let p2 = get_build_dir(dir.path(), "name", false, timestamp).unwrap(); + let f2 = p2.file_name().unwrap(); + let epoch = timestamp.timestamp(); + assert!(f2.eq(format!("rattler-build_name_{epoch}").as_str())); + } + + #[test] + fn test_directories_yaml_rendering() { + let tempdir = tempfile::tempdir().unwrap(); + + let directories = Directories::setup( + "name", + &tempdir.path().join("recipe"), + &tempdir.path().join("output"), + false, + &chrono::Utc::now(), + false, + ) + .unwrap(); + directories.create_build_dir(false).unwrap(); + + // test yaml roundtrip + let yaml = serde_yaml::to_string(&directories).unwrap(); + let directories2: Directories = serde_yaml::from_str(&yaml).unwrap(); + assert_eq!(directories.build_dir, directories2.build_dir); + assert_eq!(directories.build_prefix, directories2.build_prefix); + assert_eq!(directories.host_prefix, directories2.host_prefix); + } +} diff --git a/src/types/mod.rs b/src/types/mod.rs new file mode 100644 index 000000000..33f9a05ac --- /dev/null +++ b/src/types/mod.rs @@ -0,0 +1,200 @@ +//! Common types used throughout rattler-build +//! All the metadata that makes up a recipe file +use std::{iter, path::PathBuf, str::FromStr}; + +use chrono::{DateTime, Utc}; +use rattler_conda_types::{ + Channel, ChannelUrl, GenericVirtualPackage, PackageName, Platform, VersionWithSource, + compression_level::CompressionLevel, + package::{ArchiveType, PathsJson}, +}; +use rattler_index::{IndexFsConfig, index_fs}; +use rattler_repodata_gateway::SubdirSelection; +use rattler_virtual_packages::{ + DetectVirtualPackageError, VirtualPackageOverrides, VirtualPackages, +}; +use serde::{Deserialize, Deserializer, Serialize}; + +use crate::tool_configuration; + +mod build_configuration; +mod build_output; +mod directories; + +pub use build_configuration::BuildConfiguration; +pub use build_output::BuildOutput as Output; +pub use directories::Directories; + +/// Settings when creating the package (compression etc.) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PackagingSettings { + /// The archive type, currently supported are `tar.bz2` and `conda` + pub archive_type: ArchiveType, + /// The compression level from 1-9 or -7-22 for `tar.bz2` and `conda` + /// archives + pub compression_level: i32, +} + +impl PackagingSettings { + /// Create a new `PackagingSettings` from the command line arguments + /// and the selected archive type. + pub fn from_args(archive_type: ArchiveType, compression_level: CompressionLevel) -> Self { + let compression_level: i32 = match archive_type { + ArchiveType::TarBz2 => compression_level.to_bzip2_level().unwrap() as i32, + ArchiveType::Conda => compression_level.to_zstd_level().unwrap(), + }; + + Self { + archive_type, + compression_level, + } + } +} + +/// Defines both a platform and the virtual packages that describe the +/// capabilities of the platform. +#[derive(Debug, Clone, Serialize)] +pub struct PlatformWithVirtualPackages { + /// The platform + pub platform: Platform, + + /// The virtual packages for the platform + pub virtual_packages: Vec, +} + +impl PlatformWithVirtualPackages { + /// Returns the current platform and the virtual packages available on the + /// current system. + pub fn detect(overrides: &VirtualPackageOverrides) -> Result { + let platform = Platform::current(); + Self::detect_for_platform(platform, overrides) + } + + /// Detect the virtual packages for the given platform, filling in defaults where appropriate + pub fn detect_for_platform( + platform: Platform, + overrides: &VirtualPackageOverrides, + ) -> Result { + let virtual_packages = VirtualPackages::detect_for_platform(platform, overrides)? + .into_generic_virtual_packages() + .collect(); + Ok(Self { + platform, + virtual_packages, + }) + } +} + +impl<'de> Deserialize<'de> for PlatformWithVirtualPackages { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + pub struct Object { + pub platform: Platform, + pub virtual_packages: Vec, + } + + serde_untagged::UntaggedEnumVisitor::new() + .string(|s| { + Ok(Self { + platform: Platform::from_str(s).map_err(serde::de::Error::custom)?, + virtual_packages: vec![], + }) + }) + .map(|m| { + let object: Object = m.deserialize()?; + Ok(Self { + platform: object.platform, + virtual_packages: object.virtual_packages, + }) + }) + .deserialize(deserializer) + } +} + +/// A newtype wrapper around a boolean indicating whether debug output is enabled +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct Debug(bool); + +impl Debug { + /// Create a new Debug instance + pub fn new(debug: bool) -> Self { + Self(debug) + } + + /// Returns true if debug output is enabled + pub fn is_enabled(&self) -> bool { + self.0 + } +} + +/// A package identifier +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct PackageIdentifier { + /// The name of the package + pub name: PackageName, + /// The version of the package + pub version: VersionWithSource, + /// The build string of the package + pub build_string: String, +} + +/// The summary of a build +#[derive(Debug, Clone, Default)] +pub struct BuildSummary { + /// The start time of the build + pub build_start: Option>, + /// The end time of the build + pub build_end: Option>, + + /// The path to the artifact + pub artifact: Option, + /// Any warnings that were recorded during the build + pub warnings: Vec, + /// The paths that are packaged in the artifact + pub paths: Option, + /// Whether the build was successful or not + pub failed: bool, +} + +/// Builds the channel list and reindexes the output channel. +pub async fn build_reindexed_channels( + build_configuration: &BuildConfiguration, + tool_configuration: &tool_configuration::Configuration, +) -> Result, std::io::Error> { + let output_dir = &build_configuration.directories.output_dir; + let output_channel = Channel::from_directory(output_dir); + + // Clear the repodata gateway of any cached values for the output channel. + tool_configuration.repodata_gateway.clear_repodata_cache( + &output_channel, + SubdirSelection::Some( + [build_configuration.target_platform] + .iter() + .map(ToString::to_string) + .collect(), + ), + ); + + let index_config = IndexFsConfig { + channel: output_dir.clone(), + target_platform: Some(build_configuration.target_platform), + repodata_patch: None, + write_zst: false, + write_shards: false, + force: false, + max_parallel: num_cpus::get_physical(), + multi_progress: None, + }; + + // Reindex the output channel from the files on disk + index_fs(index_config) + .await + .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?; + + Ok(iter::once(output_channel.base_url) + .chain(build_configuration.channels.iter().cloned()) + .collect()) +}