Auto merge of #5203 - matklad:out-dir, r=alexcrichton

cargo build --out-dir

This is intended to fix #4875. Very much work in progress, just to figure out how exactly `--out-dir` should work :)

The path of least resistance for implementing `out-dir` would be to just link everything from `target/debug` to the `out-dir`. However, the problem with this approach is that we link *too much* to the `target/debug`. For example, if you run `cargo build --bin foo`, you'll find not only the `foo` itself there, but also rlibs from the same workspace.

I think it's rather important not to copy extra stuff to out-dir, so it is implemented as a completely new parameter, which is threaded through various config structs and applies *only* to the top-level units (i.e, to the stuff user explicitly requested to compile on the command line).

I also plan to add platform specific tests here, to check that we get .pdb on windows and .dSYM on macs for various crate-types.

Because, in general, a single target may produce multiple files, `out-dir` has to be a directory, you can't directly specify the output *file*.

Note that artifacts are still generated to the `target` dir as well.

Also cc @nirbheek, I hope that this might be useful for Meson integration, because `--out-dir` should be more straightforward to use than `--message-format=json`.

The end result, for `cargo build --bin rustraytracer --out-dir out` on windows looks like this:

![image](https://user-images.githubusercontent.com/1711539/37605115-941c0b22-2ba3-11e8-9685-9756a10fdfac.png)

Note how we have fewer files in the `out` :)

r? @alexcrichton

Author: bors

src/bin/command_prelude.rs

@@ -288,6 +288,7 @@ pub trait ArgMatchesExt {
             message_format,
             target_rustdoc_args: None,
             target_rustc_args: None,
+            export_dir: None,
         };
         Ok(opts)
     }

src/bin/commands/build.rs

@@ -27,6 +27,7 @@ pub fn cli() -> App {
         .arg_release("Build artifacts in release mode, with optimizations")
         .arg_features()
         .arg_target_triple("Build for the target triple")
+        .arg(opt("out-dir", "Copy final artifacts to this directory").value_name("PATH"))
         .arg_manifest_path()
         .arg_message_format()
         .after_help(
@@ -49,7 +50,13 @@ the --release flag will use the `release` profile instead.
 
 pub fn exec(config: &mut Config, args: &ArgMatches) -> CliResult {
     let ws = args.workspace(config)?;
-    let compile_opts = args.compile_options(config, CompileMode::Build)?;
+    let mut compile_opts = args.compile_options(config, CompileMode::Build)?;
+    compile_opts.export_dir = args.value_of_path("out-dir", config);
+    if compile_opts.export_dir.is_some() && !config.cli_unstable().out_dir {
+        Err(format_err!(
+            "`--out-dir` flag is unstable, pass `-Z out-dir` to enable it"
+        ))?;
+    };
     ops::compile(&ws, &compile_opts)?;
     Ok(())
 }

src/cargo/core/features.rs

@@ -286,6 +286,7 @@ pub struct CliUnstable {
     pub no_index_update: bool,
     pub avoid_dev_deps: bool,
     pub minimal_versions: bool,
+    pub out_dir: bool,
 }
 
 impl CliUnstable {
@@ -319,6 +320,7 @@ impl CliUnstable {
             "no-index-update" => self.no_index_update = true,
             "avoid-dev-deps" => self.avoid_dev_deps = true,
             "minimal-versions" => self.minimal_versions = true,
+            "out-dir" => self.out_dir = true,
             _ => bail!("unknown `-Z` flag specified: {}", k),
         }
 

src/cargo/ops/cargo_clean.rs

@@ -98,6 +98,7 @@ pub fn clean(ws: &Workspace, opts: &CleanOptions) -> CargoResult<()> {
             ..BuildConfig::default()
         },
         profiles,
+        None,
         &units,
     )?;
 

src/cargo/ops/cargo_compile.rs

@@ -64,6 +64,12 @@ pub struct CompileOptions<'a> {
     /// The specified target will be compiled with all the available arguments,
     /// note that this only accounts for the *final* invocation of rustc
     pub target_rustc_args: Option<Vec<String>>,
+    /// The directory to copy final artifacts to. Note that even if `out_dir` is
+    /// set, a copy of artifacts still could be found a `target/(debug\release)`
+    /// as usual.
+    // Note that, although the cmd-line flag name is `out-dir`, in code we use
+    // `export_dir`, to avoid confusion with out dir at `target/debug/deps`.
+    pub export_dir: Option<PathBuf>,
 }
 
 impl<'a> CompileOptions<'a> {
@@ -84,6 +90,7 @@ impl<'a> CompileOptions<'a> {
             message_format: MessageFormat::Human,
             target_rustdoc_args: None,
             target_rustc_args: None,
+            export_dir: None,
         }
     }
 }
@@ -233,6 +240,7 @@ pub fn compile_ws<'a>(
         ref filter,
         ref target_rustdoc_args,
         ref target_rustc_args,
+        ref export_dir,
     } = *options;
 
     let target = match target {
@@ -330,7 +338,6 @@ pub fn compile_ws<'a>(
             package_targets.push((to_build, vec![(target, profile)]));
         }
     }
-
     let mut ret = {
         let _p = profile::start("compiling");
         let mut build_config = scrape_build_config(config, jobs, target)?;
@@ -349,6 +356,7 @@ pub fn compile_ws<'a>(
             config,
             build_config,
             profiles,
+            export_dir.clone(),
             &exec,
         )?
     };

src/cargo/ops/cargo_package.rs

@@ -350,6 +350,7 @@ fn run_verify(ws: &Workspace, tar: &FileLock, opts: &PackageOpts) -> CargoResult
             mode: ops::CompileMode::Build,
             target_rustdoc_args: None,
             target_rustc_args: None,
+            export_dir: None,
         },
         Arc::new(DefaultExecutor),
     )?;

src/cargo/ops/cargo_rustc/context/compilation_files.rs

@@ -27,6 +27,7 @@ pub struct CompilationFiles<'a, 'cfg: 'a> {
     pub(super) host: Layout,
     /// The target directory layout for the target (if different from then host)
     pub(super) target: Option<Layout>,
+    export_dir: Option<(PathBuf, Vec<Unit<'a>>)>,
     ws: &'a Workspace<'cfg>,
     metas: HashMap<Unit<'a>, Option<Metadata>>,
     /// For each Unit, a list all files produced.
@@ -49,6 +50,7 @@ impl<'a, 'cfg: 'a> CompilationFiles<'a, 'cfg> {
         roots: &[Unit<'a>],
         host: Layout,
         target: Option<Layout>,
+        export_dir: Option<PathBuf>,
         ws: &'a Workspace<'cfg>,
         cx: &Context<'a, 'cfg>,
     ) -> CompilationFiles<'a, 'cfg> {
@@ -65,6 +67,7 @@ impl<'a, 'cfg: 'a> CompilationFiles<'a, 'cfg> {
             ws,
             host,
             target,
+            export_dir: export_dir.map(|dir| (dir, roots.to_vec())),
             metas,
             outputs,
         }
@@ -107,6 +110,15 @@ impl<'a, 'cfg: 'a> CompilationFiles<'a, 'cfg> {
         }
     }
 
+    pub fn export_dir(&self, unit: &Unit<'a>) -> Option<PathBuf> {
+        let &(ref dir, ref roots) = self.export_dir.as_ref()?;
+        if roots.contains(unit) {
+            Some(dir.clone())
+        } else {
+            None
+        }
+    }
+
     pub fn pkg_dir(&self, unit: &Unit<'a>) -> String {
         let name = unit.pkg.package_id().name();
         match self.metas[unit] {

src/cargo/ops/cargo_rustc/context/mod.rs

@@ -2,7 +2,7 @@
 
 use std::collections::{HashMap, HashSet};
 use std::env;
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use std::str::{self, FromStr};
 use std::sync::Arc;
 
@@ -105,6 +105,7 @@ impl<'a, 'cfg> Context<'a, 'cfg> {
         config: &'cfg Config,
         build_config: BuildConfig,
         profiles: &'a Profiles,
+        export_dir: Option<PathBuf>,
         units: &[Unit<'a>],
     ) -> CargoResult<Context<'a, 'cfg>> {
         let dest = if build_config.release {
@@ -164,7 +165,7 @@ impl<'a, 'cfg> Context<'a, 'cfg> {
         cx.probe_target_info()?;
         let deps = build_unit_dependencies(units, &cx)?;
         cx.unit_dependencies = deps;
-        let files = CompilationFiles::new(units, host_layout, target_layout, ws, &cx);
+        let files = CompilationFiles::new(units, host_layout, target_layout, export_dir, ws, &cx);
         cx.files = Some(files);
         Ok(cx)
     }

src/cargo/ops/cargo_rustc/mod.rs

@@ -3,7 +3,7 @@ use std::env;
 use std::ffi::{OsStr, OsString};
 use std::fs;
 use std::io::{self, Write};
-use std::path::{self, PathBuf};
+use std::path::{self, Path, PathBuf};
 use std::sync::Arc;
 
 use same_file::is_same_file;
@@ -141,6 +141,7 @@ pub fn compile_targets<'a, 'cfg: 'a>(
     config: &'cfg Config,
     build_config: BuildConfig,
     profiles: &'a Profiles,
+    export_dir: Option<PathBuf>,
     exec: &Arc<Executor>,
 ) -> CargoResult<Compilation<'cfg>> {
     let units = pkg_targets
@@ -171,6 +172,7 @@ pub fn compile_targets<'a, 'cfg: 'a>(
         config,
         build_config,
         profiles,
+        export_dir,
         &units,
     )?;
 
@@ -569,6 +571,7 @@ fn link_targets<'a, 'cfg>(
     fresh: bool,
 ) -> CargoResult<Work> {
     let outputs = cx.outputs(unit)?;
+    let export_dir = cx.files().export_dir(unit);
     let package_id = unit.pkg.package_id().clone();
     let target = unit.target.clone();
     let profile = unit.profile.clone();
@@ -600,41 +603,14 @@ fn link_targets<'a, 'cfg>(
                 }
             };
             destinations.push(dst.display().to_string());
+            hardlink_or_copy(&src, &dst)?;
+            if let Some(ref path) = export_dir {
+                if !path.exists() {
+                    fs::create_dir_all(path)?;
+                }
 
-            debug!("linking {} to {}", src.display(), dst.display());
-            if is_same_file(src, dst).unwrap_or(false) {
-                continue;
-            }
-            if dst.exists() {
-                paths::remove_file(&dst)?;
+                hardlink_or_copy(&src, &path.join(dst.file_name().unwrap()))?;
             }
-
-            let link_result = if src.is_dir() {
-                #[cfg(unix)]
-                use std::os::unix::fs::symlink;
-                #[cfg(target_os = "redox")]
-                use std::os::redox::fs::symlink;
-                #[cfg(windows)]
-                use std::os::windows::fs::symlink_dir as symlink;
-
-                let dst_dir = dst.parent().unwrap();
-                assert!(src.starts_with(dst_dir));
-                symlink(src.strip_prefix(dst_dir).unwrap(), dst)
-            } else {
-                fs::hard_link(src, dst)
-            };
-            link_result
-                .or_else(|err| {
-                    debug!("link failed {}. falling back to fs::copy", err);
-                    fs::copy(src, dst).map(|_| ())
-                })
-                .chain_err(|| {
-                    format!(
-                        "failed to link or copy `{}` to `{}`",
-                        src.display(),
-                        dst.display()
-                    )
-                })?;
         }
 
         if json_messages {
@@ -651,6 +627,48 @@ fn link_targets<'a, 'cfg>(
     }))
 }
 
+fn hardlink_or_copy(src: &Path, dst: &Path) -> CargoResult<()> {
+    debug!("linking {} to {}", src.display(), dst.display());
+    if is_same_file(src, dst).unwrap_or(false) {
+        return Ok(());
+    }
+    if dst.exists() {
+        paths::remove_file(&dst)?;
+    }
+
+    let link_result = if src.is_dir() {
+        #[cfg(unix)]
+        use std::os::unix::fs::symlink;
+        #[cfg(target_os = "redox")]
+        use std::os::redox::fs::symlink;
+        #[cfg(windows)]
+        use std::os::windows::fs::symlink_dir as symlink;
+
+        let dst_dir = dst.parent().unwrap();
+        let src = if src.starts_with(dst_dir) {
+            src.strip_prefix(dst_dir).unwrap()
+        } else {
+            src
+        };
+        symlink(src, dst)
+    } else {
+        fs::hard_link(src, dst)
+    };
+    link_result
+        .or_else(|err| {
+            debug!("link failed {}. falling back to fs::copy", err);
+            fs::copy(src, dst).map(|_| ())
+        })
+        .chain_err(|| {
+            format!(
+                "failed to link or copy `{}` to `{}`",
+                src.display(),
+                dst.display()
+            )
+        })?;
+    Ok(())
+}
+
 fn load_build_deps(cx: &Context, unit: &Unit) -> Option<Arc<BuildScripts>> {
     cx.build_scripts.get(unit).cloned()
 }

tests/testsuite/main.rs

@@ -62,6 +62,7 @@ mod login;
 mod metadata;
 mod net_config;
 mod new;
+mod out_dir;
 mod overrides;
 mod package;
 mod patch;