Skip to content

Commit

Permalink
Auto merge of #95170 - jyn514:ci-llvm, r=Mark-Simulacrum
Browse files Browse the repository at this point in the history
Move `download-ci-llvm` out of bootstrap.py

This is ready for review. It has been tested on Windows, Linux, and NixOS.

The second commit ports the changes from #95234 to Rust; I can remove it if desired.

Helps with #94829.

As a follow-up, this makes it possible to avoid downloading llvm until it's needed for building `rustc_llvm`; it would be nice to do that, but it shouldn't go in the first draft. It might also be possible to avoid requiring python until tests run (currently there's a check in `sanity.rs`), but I haven't looked too much into that.

`@rustbot` label +A-rustbuild
  • Loading branch information
bors committed Apr 27, 2022
2 parents 2799141 + 7885ade commit ff18038
Show file tree
Hide file tree
Showing 10 changed files with 369 additions and 184 deletions.
2 changes: 2 additions & 0 deletions Cargo.lock
Original file line number Diff line number Diff line change
Expand Up @@ -225,8 +225,10 @@ dependencies = [
"pretty_assertions",
"serde",
"serde_json",
"tar",
"toml",
"winapi",
"xz2",
]

[[package]]
Expand Down
2 changes: 2 additions & 0 deletions src/bootstrap/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -42,10 +42,12 @@ cc = "1.0.69"
libc = "0.2"
serde = { version = "1.0.8", features = ["derive"] }
serde_json = "1.0.2"
tar = "0.4"
toml = "0.5"
ignore = "0.4.10"
opener = "0.5"
once_cell = "1.7.2"
xz2 = "0.1"

[target.'cfg(windows)'.dependencies.winapi]
version = "0.3"
Expand Down
149 changes: 0 additions & 149 deletions src/bootstrap/bootstrap.py
Original file line number Diff line number Diff line change
Expand Up @@ -500,81 +500,6 @@ def download_toolchain(self, stage0=True, rustc_channel=None):
with output(self.rustfmt_stamp()) as rustfmt_stamp:
rustfmt_stamp.write(self.stage0_rustfmt.channel())

# Avoid downloading LLVM twice (once for stage0 and once for the master rustc)
if self.downloading_llvm() and stage0:
# We want the most recent LLVM submodule update to avoid downloading
# LLVM more often than necessary.
#
# This git command finds that commit SHA, looking for bors-authored
# commits that modified src/llvm-project or other relevant version
# stamp files.
#
# This works even in a repository that has not yet initialized
# submodules.
top_level = subprocess.check_output([
"git", "rev-parse", "--show-toplevel",
]).decode(sys.getdefaultencoding()).strip()
llvm_sha = subprocess.check_output([
"git", "rev-list", "[email protected]", "-n1",
"--first-parent", "HEAD",
"--",
"{}/src/llvm-project".format(top_level),
"{}/src/bootstrap/download-ci-llvm-stamp".format(top_level),
# the LLVM shared object file is named `LLVM-12-rust-{version}-nightly`
"{}/src/version".format(top_level)
]).decode(sys.getdefaultencoding()).strip()
llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
llvm_root = self.llvm_root()
llvm_lib = os.path.join(llvm_root, "lib")
if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
self._download_ci_llvm(llvm_sha, llvm_assertions)
for binary in ["llvm-config", "FileCheck"]:
self.fix_bin_or_dylib(os.path.join(llvm_root, "bin", binary))
for lib in os.listdir(llvm_lib):
if lib.endswith(".so"):
self.fix_bin_or_dylib(os.path.join(llvm_lib, lib))
with output(self.llvm_stamp()) as llvm_stamp:
llvm_stamp.write(llvm_sha + str(llvm_assertions))

def downloading_llvm(self):
opt = self.get_toml('download-ci-llvm', 'llvm')
# This is currently all tier 1 targets and tier 2 targets with host tools
# (since others may not have CI artifacts)
# https://doc.rust-lang.org/rustc/platform-support.html#tier-1
supported_platforms = [
# tier 1
"aarch64-unknown-linux-gnu",
"i686-pc-windows-gnu",
"i686-pc-windows-msvc",
"i686-unknown-linux-gnu",
"x86_64-unknown-linux-gnu",
"x86_64-apple-darwin",
"x86_64-pc-windows-gnu",
"x86_64-pc-windows-msvc",
# tier 2 with host tools
"aarch64-apple-darwin",
"aarch64-pc-windows-msvc",
"aarch64-unknown-linux-musl",
"arm-unknown-linux-gnueabi",
"arm-unknown-linux-gnueabihf",
"armv7-unknown-linux-gnueabihf",
"mips-unknown-linux-gnu",
"mips64-unknown-linux-gnuabi64",
"mips64el-unknown-linux-gnuabi64",
"mipsel-unknown-linux-gnu",
"powerpc-unknown-linux-gnu",
"powerpc64-unknown-linux-gnu",
"powerpc64le-unknown-linux-gnu",
"riscv64gc-unknown-linux-gnu",
"s390x-unknown-linux-gnu",
"x86_64-unknown-freebsd",
"x86_64-unknown-illumos",
"x86_64-unknown-linux-musl",
"x86_64-unknown-netbsd",
]
return opt == "true" \
or (opt == "if-available" and self.build in supported_platforms)

def _download_component_helper(
self, filename, pattern, tarball_suffix, stage0=True, key=None
):
Expand Down Expand Up @@ -606,53 +531,6 @@ def _download_component_helper(
)
unpack(tarball, tarball_suffix, self.bin_root(stage0), match=pattern, verbose=self.verbose)

def _download_ci_llvm(self, llvm_sha, llvm_assertions):
if not llvm_sha:
print("error: could not find commit hash for downloading LLVM")
print("help: maybe your repository history is too shallow?")
print("help: consider disabling `download-ci-llvm`")
print("help: or fetch enough history to include one upstream commit")
exit(1)
cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
cache_dst = os.path.join(self.build_dir, "cache")
rustc_cache = os.path.join(cache_dst, cache_prefix)
if not os.path.exists(rustc_cache):
os.makedirs(rustc_cache)

base = "https://ci-artifacts.rust-lang.org"
url = "rustc-builds/{}".format(llvm_sha)
if llvm_assertions:
url = url.replace('rustc-builds', 'rustc-builds-alt')
# ci-artifacts are only stored as .xz, not .gz
if not support_xz():
print("error: XZ support is required to download LLVM")
print("help: consider disabling `download-ci-llvm` or using python3")
exit(1)
tarball_suffix = '.tar.xz'
filename = "rust-dev-nightly-" + self.build + tarball_suffix
tarball = os.path.join(rustc_cache, filename)
if not os.path.exists(tarball):
help_on_error = "error: failed to download llvm from ci"
help_on_error += "\nhelp: old builds get deleted after a certain time"
help_on_error += "\nhelp: if trying to compile an old commit of rustc,"
help_on_error += " disable `download-ci-llvm` in config.toml:"
help_on_error += "\n"
help_on_error += "\n[llvm]"
help_on_error += "\ndownload-ci-llvm = false"
help_on_error += "\n"
get(
base,
"{}/{}".format(url, filename),
tarball,
self.checksums_sha256,
verbose=self.verbose,
do_verify=False,
help_on_error=help_on_error,
)
unpack(tarball, tarball_suffix, self.llvm_root(),
match="rust-dev",
verbose=self.verbose)

def fix_bin_or_dylib(self, fname):
"""Modifies the interpreter section of 'fname' to fix the dynamic linker,
or the RPATH section, to fix the dynamic library search path
Expand Down Expand Up @@ -816,17 +694,6 @@ def rustfmt_stamp(self):
"""
return os.path.join(self.bin_root(True), '.rustfmt-stamp')

def llvm_stamp(self):
"""Return the path for .llvm-stamp
>>> rb = RustBuild()
>>> rb.build_dir = "build"
>>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
True
"""
return os.path.join(self.llvm_root(), '.llvm-stamp')


def program_out_of_date(self, stamp_path, key):
"""Check if the given program stamp is out of date"""
if not os.path.exists(stamp_path) or self.clean:
Expand Down Expand Up @@ -856,22 +723,6 @@ def bin_root(self, stage0):
subdir = "ci-rustc"
return os.path.join(self.build_dir, self.build, subdir)

def llvm_root(self):
"""Return the CI LLVM root directory
>>> rb = RustBuild()
>>> rb.build_dir = "build"
>>> rb.llvm_root() == os.path.join("build", "ci-llvm")
True
When the 'build' property is given should be a nested directory:
>>> rb.build = "devel"
>>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
True
"""
return os.path.join(self.build_dir, self.build, "ci-llvm")

def get_toml(self, key, section=None):
"""Returns the value of the given key in config.toml, otherwise returns None
Expand Down
7 changes: 6 additions & 1 deletion src/bootstrap/builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ use std::process::Command;
use std::time::{Duration, Instant};

use crate::cache::{Cache, Interned, INTERNER};
use crate::check;
use crate::compile;
use crate::config::{SplitDebuginfo, TargetSelection};
use crate::dist;
Expand All @@ -25,6 +24,7 @@ use crate::test;
use crate::tool::{self, SourceType};
use crate::util::{self, add_dylib_path, add_link_lib_path, exe, libdir, output, t};
use crate::EXTRA_CHECK_CFGS;
use crate::{check, Config};
use crate::{Build, CLang, DocTests, GitRepo, Mode};

pub use crate::Compiler;
Expand Down Expand Up @@ -960,6 +960,11 @@ impl<'a> Builder<'a> {
None
}

/// Convenience wrapper to allow `builder.llvm_link_shared()` instead of `builder.config.llvm_link_shared(&builder)`.
pub(crate) fn llvm_link_shared(&self) -> bool {
Config::llvm_link_shared(self)
}

/// Prepares an invocation of `cargo` to be run.
///
/// This will create a `Command` that represents a pending execution of
Expand Down
2 changes: 1 addition & 1 deletion src/bootstrap/compile.rs
Original file line number Diff line number Diff line change
Expand Up @@ -737,7 +737,7 @@ pub fn rustc_cargo_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetS
);
cargo.env("LLVM_STATIC_STDCPP", file);
}
if builder.config.llvm_link_shared {
if builder.llvm_link_shared() {
cargo.env("LLVM_LINK_SHARED", "1");
}
if builder.config.llvm_use_libcxx {
Expand Down
70 changes: 52 additions & 18 deletions src/bootstrap/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
//! This module implements parsing `config.toml` configuration files to tweak
//! how the build runs.

use std::cell::Cell;
use std::cmp;
use std::collections::{HashMap, HashSet};
use std::env;
Expand All @@ -11,7 +12,7 @@ use std::fs;
use std::path::{Path, PathBuf};
use std::str::FromStr;

use crate::builder::TaskPath;
use crate::builder::{Builder, TaskPath};
use crate::cache::{Interned, INTERNER};
use crate::channel::GitInfo;
pub use crate::flags::Subcommand;
Expand Down Expand Up @@ -69,13 +70,14 @@ pub struct Config {
pub test_compare_mode: bool,
pub llvm_libunwind: LlvmLibunwind,
pub color: Color,
pub patch_binaries_for_nix: bool,

pub on_fail: Option<String>,
pub stage: u32,
pub keep_stage: Vec<u32>,
pub keep_stage_std: Vec<u32>,
pub src: PathBuf,
// defaults to `config.toml`
/// defaults to `config.toml`
pub config: PathBuf,
pub jobs: Option<u32>,
pub cmd: Subcommand,
Expand All @@ -96,7 +98,11 @@ pub struct Config {
pub llvm_release_debuginfo: bool,
pub llvm_version_check: bool,
pub llvm_static_stdcpp: bool,
pub llvm_link_shared: bool,
/// `None` if `llvm_from_ci` is true and we haven't yet downloaded llvm.
#[cfg(not(test))]
llvm_link_shared: Cell<Option<bool>>,
#[cfg(test)]
pub llvm_link_shared: Cell<Option<bool>>,
pub llvm_clang_cl: Option<String>,
pub llvm_targets: Option<String>,
pub llvm_experimental_targets: Option<String>,
Expand Down Expand Up @@ -858,6 +864,7 @@ impl Config {
set(&mut config.local_rebuild, build.local_rebuild);
set(&mut config.print_step_timings, build.print_step_timings);
set(&mut config.print_step_rusage, build.print_step_rusage);
set(&mut config.patch_binaries_for_nix, build.patch_binaries_for_nix);

config.verbose = cmp::max(config.verbose, flags.verbose);

Expand Down Expand Up @@ -913,7 +920,9 @@ impl Config {
set(&mut config.llvm_release_debuginfo, llvm.release_debuginfo);
set(&mut config.llvm_version_check, llvm.version_check);
set(&mut config.llvm_static_stdcpp, llvm.static_libstdcpp);
set(&mut config.llvm_link_shared, llvm.link_shared);
if let Some(v) = llvm.link_shared {
config.llvm_link_shared.set(Some(v));
}
config.llvm_targets = llvm.targets.clone();
config.llvm_experimental_targets = llvm.experimental_targets.clone();
config.llvm_link_jobs = llvm.link_jobs;
Expand Down Expand Up @@ -983,6 +992,7 @@ impl Config {
check_ci_llvm!(llvm.optimize);
check_ci_llvm!(llvm.thin_lto);
check_ci_llvm!(llvm.release_debuginfo);
// CI-built LLVM can be either dynamic or static. We won't know until we download it.
check_ci_llvm!(llvm.link_shared);
check_ci_llvm!(llvm.static_libstdcpp);
check_ci_llvm!(llvm.targets);
Expand All @@ -1000,26 +1010,14 @@ impl Config {
check_ci_llvm!(llvm.clang);
check_ci_llvm!(llvm.build_config);
check_ci_llvm!(llvm.plugins);

// CI-built LLVM can be either dynamic or static.
let ci_llvm = config.out.join(&*config.build.triple).join("ci-llvm");
config.llvm_link_shared = if config.dry_run {
// just assume dynamic for now
true
} else {
let link_type = t!(
std::fs::read_to_string(ci_llvm.join("link-type.txt")),
format!("CI llvm missing: {}", ci_llvm.display())
);
link_type == "dynamic"
};
}

// NOTE: can never be hit when downloading from CI, since we call `check_ci_llvm!(thin_lto)` above.
if config.llvm_thin_lto && llvm.link_shared.is_none() {
// If we're building with ThinLTO on, by default we want to link
// to LLVM shared, to avoid re-doing ThinLTO (which happens in
// the link step) with each stage.
config.llvm_link_shared = true;
config.llvm_link_shared.set(Some(true));
}
}

Expand Down Expand Up @@ -1274,6 +1272,42 @@ impl Config {
}
}

/// The absolute path to the downloaded LLVM artifacts.
pub(crate) fn ci_llvm_root(&self) -> PathBuf {
assert!(self.llvm_from_ci);
self.out.join(&*self.build.triple).join("ci-llvm")
}

/// Determine whether llvm should be linked dynamically.
///
/// If `false`, llvm should be linked statically.
/// This is computed on demand since LLVM might have to first be downloaded from CI.
pub(crate) fn llvm_link_shared(builder: &Builder<'_>) -> bool {
let mut opt = builder.config.llvm_link_shared.get();
if opt.is_none() && builder.config.dry_run {
// just assume static for now - dynamic linking isn't supported on all platforms
return false;
}

let llvm_link_shared = *opt.get_or_insert_with(|| {
if builder.config.llvm_from_ci {
crate::native::maybe_download_ci_llvm(builder);
let ci_llvm = builder.config.ci_llvm_root();
let link_type = t!(
std::fs::read_to_string(ci_llvm.join("link-type.txt")),
format!("CI llvm missing: {}", ci_llvm.display())
);
link_type == "dynamic"
} else {
// unclear how thought-through this default is, but it maintains compatibility with
// previous behavior
false
}
});
builder.config.llvm_link_shared.set(opt);
llvm_link_shared
}

pub fn verbose(&self) -> bool {
self.verbose > 0
}
Expand Down
Loading

0 comments on commit ff18038

Please sign in to comment.