Add recursive dependency download functionality

- Track and display download results in an ASCII table

- Make `source already exists` not an error, just a result type

- Introduced a `--recursive` flag to the `download` subcommand to
  enable downloading of both the main packages and all their
  dependencies.

- Added `--image` and `--env` options to specify the Docker image
  and environment variables, ensuring the correct dependency tree
  is resolved based on the build environment. Took this approach
  from the tree-of command.

- Used a `HashSet` to avoid duplicate processing of packages and
  their dependencies.

This update improves clarity and tracking of the download outcomes
and allows for more comprehensive package management by
ensuring that all necessary dependencies are downloaded alongside
the requested packages.

Signed-off-by: Nico Steinle <[email protected]>

src/cli.rs

@@ -843,6 +843,42 @@ pub fn cli() -> Command {
                     .help("Set timeout for download in seconds")
                     .value_parser(clap::value_parser!(u64))
                 )
+
+                .arg(Arg::new("recursive")
+                    .action(ArgAction::SetTrue)
+                    .required(false)
+                    .long("recursive")
+                    .help("Download the sources and all the dependency sources")
+                )
+
+                .arg(Arg::new("image")
+                    .required(false)
+                    .value_name("IMAGE NAME")
+                    .short('I')
+                    .long("image")
+                    .help("Name of the Docker image to use")
+                    .long_help(indoc::indoc!(r#"
+                        Name of the Docker image to use.
+
+                        Required because tree might look different on different images because of
+                        conditions on dependencies.
+                    "#))
+                )
+
+                .arg(Arg::new("env")
+                    .required(false)
+                    .action(ArgAction::Append)
+                    .short('E')
+                    .long("env")
+                    .value_parser(env_pass_validator)
+                    .help("Additional env to be passed when building packages")
+                    .long_help(indoc::indoc!(r#"
+                        Additional env to be passed when building packages.
+
+                        Required because tree might look different on different images because of
+                        conditions on dependencies.
+                    "#))
+                )
             )
             .subcommand(Command::new("of")
                 .about("Get the paths of the sources of a package")

src/commands/source/download.rs

@@ -8,26 +8,35 @@
 // SPDX-License-Identifier: EPL-2.0
 //
 
+use std::collections::HashSet;
 use std::concat;
+use std::fmt;
+use std::fmt::Display;
 use std::path::PathBuf;
 use std::sync::Arc;
 
 use anyhow::anyhow;
 use anyhow::Context;
 use anyhow::Error;
 use anyhow::Result;
+use ascii_table::{Align, AsciiTable};
 use clap::ArgMatches;
 use futures::stream::{FuturesUnordered, StreamExt};
 use regex::Regex;
 use tokio::io::AsyncWriteExt;
 use tokio::sync::{Mutex, Semaphore};
-use tracing::{info, trace, warn};
+use tracing::{debug, error, info, trace, warn};
 
-use crate::config::*;
-use crate::package::Package;
+use crate::config::Configuration;
+use crate::package::condition::ConditionData;
+use crate::package::Dag;
 use crate::package::PackageName;
 use crate::package::PackageVersionConstraint;
 use crate::repository::Repository;
+use crate::util::docker::ImageNameLookup;
+use crate::util::EnvironmentVariableName;
+
+use crate::package::Package;
 use crate::source::*;
 use crate::util::progress::ProgressBars;
 
@@ -42,6 +51,17 @@ enum DownloadResult {
     MarkedManual,
 }
 
+impl fmt::Display for DownloadResult {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            DownloadResult::Forced => write!(f, "forced"),
+            DownloadResult::Skipped => write!(f, "skipped"),
+            DownloadResult::Succeeded => write!(f, "succeeded"),
+            DownloadResult::MarkedManual => write!(f, "marked manual"),
+        }
+    }
+}
+
 /// A wrapper around the indicatif::ProgressBar
 ///
 /// A wrapper around the indicatif::ProgressBar that is used to synchronize status information from
@@ -270,6 +290,7 @@ pub async fn download(
     progressbars: ProgressBars,
 ) -> Result<()> {
     let force = matches.get_flag("force");
+    let recursive = matches.get_flag("recursive");
     let timeout = matches.get_one::<u64>("timeout").copied();
     let cache = PathBuf::from(config.source_cache_root());
     let sc = SourceCache::new(cache);
@@ -294,9 +315,45 @@ pub async fn download(
         NUMBER_OF_MAX_CONCURRENT_DOWNLOADS,
     ));
 
-    let r = find_packages(&repo, pname, pvers, matching_regexp)?;
+    let found_packages = find_packages(&repo, pname, pvers, matching_regexp)?;
+
+    let packages_to_download: HashSet<Package> = match recursive {
+        true => {
+            debug!("Finding package dependencies recursively");
+
+            let image_name_lookup = ImageNameLookup::create(config.docker().images())?;
+            let image_name = matches
+                .get_one::<String>("image")
+                .map(|s| image_name_lookup.expand(s))
+                .transpose()?;
+
+            let additional_env = matches
+                .get_many::<String>("env")
+                .unwrap_or_default()
+                .map(AsRef::as_ref)
+                .map(crate::util::env::parse_to_env)
+                .collect::<Result<Vec<(EnvironmentVariableName, String)>>>()?;
+
+            let condition_data = ConditionData {
+                image_name: image_name.as_ref(),
+                env: &additional_env,
+            };
+
+            let dependencies: Vec<Package> = found_packages
+                .iter()
+                .flat_map(|package| {
+                    Dag::for_root_package((*package).clone(), &repo, None, &condition_data)
+                        .map(|d| d.dag().graph().node_weights().cloned().collect::<Vec<_>>())
+                        .unwrap()
+                })
+                .collect();
+
+            HashSet::from_iter(dependencies)
+        }
+        false => HashSet::from_iter(found_packages.into_iter().cloned()),
+    };
 
-    let r: Vec<(SourceEntry, Result<DownloadResult>)> = r
+    let mut r: Vec<(SourceEntry, Result<DownloadResult>)> = packages_to_download
         .iter()
         .flat_map(|p| {
             sc.sources_for(p).into_iter().map(|source| {
@@ -314,6 +371,71 @@ pub async fn download(
         .collect()
         .await;
 
+    {
+        let mut ascii_table = AsciiTable::default();
+        ascii_table.set_max_width(
+            terminal_size::terminal_size()
+                .map(|tpl| tpl.0 .0 as usize)
+                .unwrap_or(80),
+        );
+        ascii_table.column(0).set_header("#").set_align(Align::Left);
+        ascii_table
+            .column(1)
+            .set_header("Package name")
+            .set_align(Align::Left);
+        ascii_table
+            .column(2)
+            .set_header("Version")
+            .set_align(Align::Left);
+        ascii_table
+            .column(3)
+            .set_header("Source name")
+            .set_align(Align::Left);
+        ascii_table
+            .column(4)
+            .set_header("Status")
+            .set_align(Align::Left);
+        ascii_table
+            .column(5)
+            .set_header("Path")
+            .set_align(Align::Left);
+
+        let numbers: Vec<usize> = (0..r.len()).map(|n| n + 1).collect();
+        r.sort_by(|(a, _), (b, _)| a.package_name().partial_cmp(b.package_name()).unwrap());
+        let source_paths: Vec<String> = r.iter().map(|v| v.0.path_as_string()).collect();
+
+        let data: Vec<Vec<&dyn Display>> = r
+            .iter()
+            .enumerate()
+            .map(|(i, v)| {
+                debug!("source_entry: {:#?}", v.0);
+                let n = &numbers[i];
+                let mut row: Vec<&dyn Display> = vec![
+                    n,
+                    v.0.package_name(),
+                    v.0.package_version(),
+                    v.0.package_source_name(),
+                ];
+                if v.1.is_ok() {
+                    let result = v.1.as_ref().unwrap() as &dyn Display;
+                    row.push(result);
+                } else {
+                    row.push(&"failed");
+                }
+                row.push(&source_paths[i]);
+                row
+            })
+            .collect();
+
+        ascii_table.print(data);
+    }
+
+    for p in r {
+        if p.1.is_err() {
+            error!("{}: {:?}", p.0.package_name(), p.1);
+        }
+    }
+
     super::verify(matches, config, repo, progressbars).await?;
 
     Ok(())

src/source/mod.rs

@@ -14,6 +14,7 @@ use anyhow::anyhow;
 use anyhow::Context;
 use anyhow::Error;
 use anyhow::Result;
+use getset::Getters;
 use tracing::trace;
 use url::Url;
 
@@ -37,11 +38,14 @@ impl SourceCache {
     }
 }
 
-#[derive(Debug)]
+#[derive(Debug, Getters)]
 pub struct SourceEntry {
     cache_root: PathBuf,
+    #[getset(get = "pub")]
     package_name: PackageName,
+    #[getset(get = "pub")]
     package_version: PackageVersion,
+    #[getset(get = "pub")]
     package_source_name: String,
     package_source: Source,
 }
@@ -73,6 +77,10 @@ impl SourceEntry {
         })
     }
 
+    pub fn path_as_string(&self) -> String {
+        self.path().to_string_lossy().to_string()
+    }
+
     pub fn url(&self) -> &Url {
         self.package_source.url()
     }