Auto merge of #51207 - michaelwoerister:build-lld-on-linux, r=<try>

[experimental]: Build LLVM with ThinLTO enabled

This is a work in progress. Let's see how it goes.

src/bootstrap/builder.rs

@@ -17,10 +17,10 @@ use std::fs;
 use std::hash::Hash;
 use std::ops::Deref;
 use std::path::{Path, PathBuf};
-use std::process::Command;
 use std::time::{Instant, Duration};
 use std::collections::HashMap;
 
+use build_helper::command_ext::Command;
 use compile;
 use install;
 use dist;
@@ -658,7 +658,7 @@ impl<'a> Builder<'a> {
         if let Some(ref error_format) = self.config.rustc_error_format {
             cargo.env("RUSTC_ERROR_FORMAT", error_format);
         }
-        if cmd != "build" && cmd != "check" && want_rustdoc {
+        if cmd != "build" && cmd != "check" && cmd != "rustc" && want_rustdoc {
             cargo.env("RUSTDOC_LIBDIR", self.rustc_libdir(self.compiler(2, self.config.build)));
         }
 
@@ -804,7 +804,7 @@ impl<'a> Builder<'a> {
             }
         }
 
-        if cmd == "build" && mode == Mode::Libstd
+        if (cmd == "build" || cmd == "rustc") && mode == Mode::Libstd
             && self.config.extended && compiler.is_final_stage(self)
         {
             cargo.env("RUSTC_SAVE_ANALYSIS", "api".to_string());

src/bootstrap/cc_detect.rs

@@ -34,9 +34,9 @@
 use std::collections::HashSet;
 use std::{env, iter};
 use std::path::{Path, PathBuf};
-use std::process::Command;
 
 use build_helper::output;
+use build_helper::command_ext::Command;
 use cc;
 
 use Build;
@@ -158,7 +158,9 @@ fn set_compiler(cfg: &mut cc::Build,
                 return
             }
 
-            let output = output(c.to_command().arg("--version"));
+
+            let output = output(Command::from_std_command(c.to_command())
+                                .arg("--version"));
             let i = match output.find(" 4.") {
                 Some(i) => i,
                 None => return,

src/bootstrap/channel.rs

@@ -16,9 +16,9 @@
 //! print out as part of its version information.
 
 use std::path::Path;
-use std::process::Command;
 
 use build_helper::output;
+use build_helper::command_ext::Command;
 
 use Build;
 use config::Config;

src/bootstrap/compile.rs

@@ -22,19 +22,19 @@ use std::fs::{self, File};
 use std::io::BufReader;
 use std::io::prelude::*;
 use std::path::{Path, PathBuf};
-use std::process::{Command, Stdio};
+use std::process::Stdio;
 use std::str;
 use std::cmp::min;
 
 use build_helper::{output, mtime, up_to_date};
+use build_helper::command_ext::Command;
 use filetime::FileTime;
 use serde_json;
 
 use util::{exe, libdir, is_dylib, CiEnv};
 use {Compiler, Mode};
 use native;
 use tool;
-
 use cache::{INTERNER, Interned};
 use builder::{Step, RunConfig, ShouldRun, Builder};
 
@@ -634,20 +634,28 @@ impl Step for CodegenBackend {
             return;
         }
 
-        let mut cargo = builder.cargo(compiler, Mode::Librustc, target, "build");
+        let mut cargo = builder.cargo(compiler, Mode::Librustc, target, "rustc");
         let mut features = builder.rustc_features().to_string();
         cargo.arg("--manifest-path")
             .arg(builder.src.join("src/librustc_codegen_llvm/Cargo.toml"));
         rustc_cargo_env(builder, &mut cargo);
 
         features += &build_codegen_backend(&builder, &mut cargo, &compiler, target, backend);
+        cargo.arg("--features").arg(features);
 
         let tmp_stamp = builder.cargo_out(compiler, Mode::Librustc, target)
             .join(".tmp.stamp");
 
+        if builder.config.llvm_thin_lto {
+            cargo.deferred_arg("--")
+                 .deferred_arg("-Clink-arg=-fuse-ld=lld")
+                 .deferred_arg("-Clink-arg=-flto=thin")
+                 .deferred_arg("-Clink-arg=-O2");
+        }
+
         let _folder = builder.fold_output(|| format!("stage{}-rustc_codegen_llvm", compiler.stage));
         let files = run_cargo(builder,
-                              cargo.arg("--features").arg(features),
+                              &mut cargo,
                               &tmp_stamp,
                               false);
         if builder.config.dry_run {

src/bootstrap/config.rs

@@ -79,6 +79,7 @@ pub struct Config {
     pub llvm_assertions: bool,
     pub llvm_optimize: bool,
     pub llvm_release_debuginfo: bool,
+    pub llvm_thin_lto: bool,
     pub llvm_version_check: bool,
     pub llvm_static_stdcpp: bool,
     pub llvm_link_shared: bool,
@@ -161,6 +162,7 @@ pub struct Target {
     pub cc: Option<PathBuf>,
     pub cxx: Option<PathBuf>,
     pub ar: Option<PathBuf>,
+    pub ranlib: Option<PathBuf>,
     pub linker: Option<PathBuf>,
     pub ndk: Option<PathBuf>,
     pub crt_static: Option<bool>,
@@ -245,6 +247,7 @@ struct Llvm {
     assertions: Option<bool>,
     optimize: Option<bool>,
     release_debuginfo: Option<bool>,
+    thin_lto: Option<bool>,
     version_check: Option<bool>,
     static_libstdcpp: Option<bool>,
     targets: Option<String>,
@@ -321,6 +324,7 @@ struct TomlTarget {
     cc: Option<String>,
     cxx: Option<String>,
     ar: Option<String>,
+    ranlib: Option<String>,
     linker: Option<String>,
     android_ndk: Option<String>,
     crt_static: Option<bool>,
@@ -499,6 +503,7 @@ impl Config {
             llvm_assertions = llvm.assertions;
             set(&mut config.llvm_optimize, llvm.optimize);
             set(&mut config.llvm_release_debuginfo, llvm.release_debuginfo);
+            set(&mut config.llvm_thin_lto, llvm.thin_lto);
             set(&mut config.llvm_version_check, llvm.version_check);
             set(&mut config.llvm_static_stdcpp, llvm.static_libstdcpp);
             set(&mut config.llvm_link_shared, llvm.link_shared);
@@ -569,6 +574,7 @@ impl Config {
                 target.cc = cfg.cc.clone().map(PathBuf::from);
                 target.cxx = cfg.cxx.clone().map(PathBuf::from);
                 target.ar = cfg.ar.clone().map(PathBuf::from);
+                target.ranlib = cfg.ranlib.clone().map(PathBuf::from);
                 target.linker = cfg.linker.clone().map(PathBuf::from);
                 target.crt_static = cfg.crt_static.clone();
                 target.musl_root = cfg.musl_root.clone().map(PathBuf::from);

src/bootstrap/dist.rs

@@ -22,9 +22,10 @@ use std::env;
 use std::fs::{self, File};
 use std::io::{Read, Write};
 use std::path::{PathBuf, Path};
-use std::process::{Command, Stdio};
+use std::process::Stdio;
 
 use build_helper::output;
+use build_helper::command_ext::Command;
 
 use {Compiler, Mode};
 use channel;

src/bootstrap/install.rs

@@ -16,11 +16,11 @@
 use std::env;
 use std::fs;
 use std::path::{Path, PathBuf, Component};
-use std::process::Command;
 
 use dist::{self, pkgname, sanitize_sh, tmpdir};
 
 use builder::{Builder, RunConfig, ShouldRun, Step};
+use build_helper::command_ext::Command;
 use cache::Interned;
 use config::Config;
 

src/bootstrap/lib.rs

@@ -145,11 +145,12 @@ use std::env;
 use std::fs::{self, OpenOptions, File};
 use std::io::{self, Seek, SeekFrom, Write, Read};
 use std::path::{PathBuf, Path};
-use std::process::{self, Command};
+use std::process;
 use std::slice;
 use std::str;
 
 use build_helper::{run_silent, run_suppressed, try_run_silent, try_run_suppressed, output, mtime};
+use build_helper::command_ext::Command;
 use filetime::FileTime;
 
 use util::{exe, libdir, OutputFolder, CiEnv};
@@ -264,6 +265,7 @@ pub struct Build {
     cc: HashMap<Interned<String>, cc::Tool>,
     cxx: HashMap<Interned<String>, cc::Tool>,
     ar: HashMap<Interned<String>, PathBuf>,
+    ranlib: HashMap<Interned<String>, PathBuf>,
     // Misc
     crates: HashMap<Interned<String>, Crate>,
     is_sudo: bool,
@@ -365,6 +367,7 @@ impl Build {
             cc: HashMap::new(),
             cxx: HashMap::new(),
             ar: HashMap::new(),
+            ranlib: HashMap::new(),
             crates: HashMap::new(),
             lldb_version: None,
             lldb_python_dir: None,
@@ -724,6 +727,11 @@ impl Build {
         self.ar.get(&target).map(|p| &**p)
     }
 
+    /// Returns the path to the `ranlib` utility for the target specified.
+    fn ranlib(&self, target: Interned<String>) -> Option<&Path> {
+        self.ranlib.get(&target).map(|p| &**p)
+    }
+
     /// Returns the path to the C++ compiler for the target specified.
     fn cxx(&self, target: Interned<String>) -> Result<&Path, String> {
         match self.cxx.get(&target) {

src/bootstrap/metadata.rs

@@ -9,10 +9,10 @@
 // except according to those terms.
 
 use std::collections::HashMap;
-use std::process::Command;
 use std::path::PathBuf;
 
 use build_helper::output;
+use build_helper::command_ext::Command;
 use serde_json;
 
 use {Build, Crate};

src/bootstrap/native.rs

@@ -23,9 +23,9 @@ use std::ffi::OsString;
 use std::fs::{self, File};
 use std::io::{Read, Write};
 use std::path::{Path, PathBuf};
-use std::process::Command;
 
 use build_helper::output;
+use build_helper::command_ext::Command;
 use cmake;
 use cc;
 
@@ -154,6 +154,11 @@ impl Step for Llvm {
            .define("LLVM_TARGET_ARCH", target.split('-').next().unwrap())
            .define("LLVM_DEFAULT_TARGET_TRIPLE", target);
 
+        if builder.config.llvm_thin_lto {
+            cfg.define("LLVM_ENABLE_LTO", "Thin")
+               .define("LLVM_ENABLE_LLD", "ON");
+        }
+
         // By default, LLVM will automatically find OCaml and, if it finds it,
         // install the LLVM bindings in LLVM_OCAML_INSTALL_PATH, which defaults
         // to /usr/bin/ocaml.
@@ -352,6 +357,14 @@ fn configure_cmake(builder: &Builder,
         }
     }
 
+    if let Some(ranlib) = builder.ranlib(target) {
+        if ranlib.is_absolute() {
+            // LLVM build breaks if `CMAKE_RANLIB` is a relative path, for some reason it
+            // tries to resolve this path in the LLVM build directory.
+            cfg.define("CMAKE_RANLIB", sanitize_cc(ranlib));
+        }
+    }
+
     if env::var_os("SCCACHE_ERROR_LOG").is_some() {
         cfg.env("RUST_LOG", "sccache=warn");
     }

src/bootstrap/sanity.rs

@@ -24,9 +24,9 @@ use std::ffi::{OsString, OsStr};
 use std::fs::{self, File};
 use std::io::Read;
 use std::path::PathBuf;
-use std::process::Command;
 
 use build_helper::output;
+use build_helper::command_ext::Command;
 
 use Build;
 

src/bootstrap/test.rs

@@ -19,10 +19,10 @@ use std::iter;
 use std::fmt;
 use std::fs::{self, File};
 use std::path::{PathBuf, Path};
-use std::process::Command;
 use std::io::Read;
 
 use build_helper::{self, output};
+use build_helper::command_ext::Command;
 
 use builder::{Kind, RunConfig, ShouldRun, Builder, Compiler, Step};
 use Crate as CargoCrate;

src/bootstrap/tool.rs

@@ -12,11 +12,12 @@ use std::fs;
 use std::env;
 use std::iter;
 use std::path::PathBuf;
-use std::process::{Command, exit};
+use std::process::exit;
 
 use Mode;
 use Compiler;
 use builder::{Step, RunConfig, ShouldRun, Builder};
+use build_helper::command_ext::Command;
 use util::{exe, add_lib_path};
 use compile::{self, libtest_stamp, libstd_stamp, librustc_stamp};
 use native;

src/bootstrap/util.rs

@@ -18,11 +18,11 @@ use std::str;
 use std::fs;
 use std::io::{self, Write};
 use std::path::{Path, PathBuf};
-use std::process::Command;
 use std::time::{SystemTime, Instant};
 
 use config::Config;
 use builder::Builder;
+use build_helper::command_ext::Command;
 
 /// Returns the `name` as the filename of a static library for `target`.
 pub fn staticlib(name: &str, target: &str) -> String {