Skip to content

Commit

Permalink
Auto merge of #77145 - pietroalbini:refactor-build-manifest-versions,…
Browse files Browse the repository at this point in the history 
… r=Mark-Simulacrum

Refactor versions detection in build-manifest

This PR refactors how `build-manifest` handles versions, making the following changes:

* `build-manifest` now detects the "package releases" on its own, without relying on rustbuild providing them through CLI arguments. This drastically simplifies calling the tool outside of `x.py`, and will allow to ship the prebuilt tool in a tarball in the future, with the goal of stopping to invoke `x.py` during `promote-release`.
* The `tar` command is not used to extract the version and the git hash from tarballs anymore. The `flate2` and `tar` crates are used instead. This makes detecting those pieces of data way faster, as the archive is decompressed just once and we stop parsing the archive once all the information is retrieved.
* The code to extract the version and the git hash now stores all the collected data dynamically, without requiring to add new fields to the `Builder` struct every time.

I tested the changes locally and it should behave the same as before.

r? `@Mark-Simulacrum`
  • Loading branch information
@bors
bors committed Sep 29, 2020
2 parents 4d52dc4 + 0749ad0 commit 9b77a6a
Show file tree
Hide file tree
Showing 6 changed files with 293 additions and 217 deletions.
3 changes: 3 additions & 0 deletions Cargo.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -231,8 +231,11 @@ dependencies = [
name = "build-manifest"
version = "0.1.0"
dependencies = [
"anyhow",
"flate2",
"serde",
"serde_json",
"tar",
"toml",
]

Expand Down
10 changes: 2 additions & 8 deletions src/bootstrap/dist.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2368,15 +2368,9 @@ impl Step for HashSign {
cmd.arg(sign);
cmd.arg(distdir(builder));
cmd.arg(today.trim());
cmd.arg(builder.rust_package_vers());
cmd.arg(addr);
cmd.arg(builder.package_vers(&builder.release_num("cargo")));
cmd.arg(builder.package_vers(&builder.release_num("rls")));
cmd.arg(builder.package_vers(&builder.release_num("rust-analyzer/crates/rust-analyzer")));
cmd.arg(builder.package_vers(&builder.release_num("clippy")));
cmd.arg(builder.package_vers(&builder.release_num("miri")));
cmd.arg(builder.package_vers(&builder.release_num("rustfmt")));
cmd.arg(builder.llvm_tools_package_vers());
cmd.arg(&builder.config.channel);
cmd.arg(&builder.src);

builder.create_dir(&distdir(builder));

Expand Down
3 changes: 3 additions & 0 deletions src/tools/build-manifest/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,6 @@ edition = "2018"
toml = "0.5"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
anyhow = "1.0.32"
flate2 = "1.0.16"
tar = "0.4.29"
9 changes: 4 additions & 5 deletions src/tools/build-manifest/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,9 @@ Then, you can generate the manifest and all the packages from `path/to/dist` to

```
$ BUILD_MANIFEST_DISABLE_SIGNING=1 cargo +nightly run \
path/to/dist path/to/output 1970-01-01 \
nightly nightly nightly nightly nightly nightly nightly nightly \
http://example.com
path/to/dist path/to/output 1970-01-01 http://example.com \
CHANNEL path/to/rust/repo
```

In the future, if the tool complains about missing arguments just add more
`nightly`s in the middle.
Remember to replace `CHANNEL` with the channel you produced dist artifacts of
and `path/to/rust/repo` with the path to your checkout of the Rust repository.
233 changes: 29 additions & 204 deletions src/tools/build-manifest/src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,10 @@
//! via `x.py dist hash-and-sign`; the cmdline arguments are set up
//! by rustbuild (in `src/bootstrap/dist.rs`).
use serde::Serialize;
mod versions;

use crate::versions::{PkgType, Versions};
use serde::Serialize;
use std::collections::BTreeMap;
use std::collections::HashMap;
use std::env;
Expand Down Expand Up @@ -226,14 +228,7 @@ macro_rules! t {
}

struct Builder {
rust_release: String,
cargo_release: String,
rls_release: String,
rust_analyzer_release: String,
clippy_release: String,
rustfmt_release: String,
llvm_tools_release: String,
miri_release: String,
versions: Versions,

input: PathBuf,
output: PathBuf,
Expand All @@ -242,24 +237,6 @@ struct Builder {
s3_address: String,
date: String,

rust_version: Option<String>,
cargo_version: Option<String>,
rls_version: Option<String>,
rust_analyzer_version: Option<String>,
clippy_version: Option<String>,
rustfmt_version: Option<String>,
llvm_tools_version: Option<String>,
miri_version: Option<String>,

rust_git_commit_hash: Option<String>,
cargo_git_commit_hash: Option<String>,
rls_git_commit_hash: Option<String>,
rust_analyzer_git_commit_hash: Option<String>,
clippy_git_commit_hash: Option<String>,
rustfmt_git_commit_hash: Option<String>,
llvm_tools_git_commit_hash: Option<String>,
miri_git_commit_hash: Option<String>,

should_sign: bool,
}

Expand All @@ -280,15 +257,9 @@ fn main() {
let input = PathBuf::from(args.next().unwrap());
let output = PathBuf::from(args.next().unwrap());
let date = args.next().unwrap();
let rust_release = args.next().unwrap();
let s3_address = args.next().unwrap();
let cargo_release = args.next().unwrap();
let rls_release = args.next().unwrap();
let rust_analyzer_release = args.next().unwrap();
let clippy_release = args.next().unwrap();
let miri_release = args.next().unwrap();
let rustfmt_release = args.next().unwrap();
let llvm_tools_release = args.next().unwrap();
let channel = args.next().unwrap();
let monorepo_path = args.next().unwrap();

// Do not ask for a passphrase while manually testing
let mut passphrase = String::new();
Expand All @@ -298,14 +269,7 @@ fn main() {
}

Builder {
rust_release,
cargo_release,
rls_release,
rust_analyzer_release,
clippy_release,
rustfmt_release,
llvm_tools_release,
miri_release,
versions: Versions::new(&channel, &input, Path::new(&monorepo_path)).unwrap(),

input,
output,
Expand All @@ -314,87 +278,21 @@ fn main() {
s3_address,
date,

rust_version: None,
cargo_version: None,
rls_version: None,
rust_analyzer_version: None,
clippy_version: None,
rustfmt_version: None,
llvm_tools_version: None,
miri_version: None,

rust_git_commit_hash: None,
cargo_git_commit_hash: None,
rls_git_commit_hash: None,
rust_analyzer_git_commit_hash: None,
clippy_git_commit_hash: None,
rustfmt_git_commit_hash: None,
llvm_tools_git_commit_hash: None,
miri_git_commit_hash: None,

should_sign,
}
.build();
}

enum PkgType {
RustSrc,
Cargo,
Rls,
RustAnalyzer,
Clippy,
Rustfmt,
LlvmTools,
Miri,
Other,
}

impl PkgType {
fn from_component(component: &str) -> Self {
use PkgType::*;
match component {
"rust-src" => RustSrc,
"cargo" => Cargo,
"rls" | "rls-preview" => Rls,
"rust-analyzer" | "rust-analyzer-preview" => RustAnalyzer,
"clippy" | "clippy-preview" => Clippy,
"rustfmt" | "rustfmt-preview" => Rustfmt,
"llvm-tools" | "llvm-tools-preview" => LlvmTools,
"miri" | "miri-preview" => Miri,
_ => Other,
}
}
}

impl Builder {
fn build(&mut self) {
self.rust_version = self.version("rust", "x86_64-unknown-linux-gnu");
self.cargo_version = self.version("cargo", "x86_64-unknown-linux-gnu");
self.rls_version = self.version("rls", "x86_64-unknown-linux-gnu");
self.rust_analyzer_version = self.version("rust-analyzer", "x86_64-unknown-linux-gnu");
self.clippy_version = self.version("clippy", "x86_64-unknown-linux-gnu");
self.rustfmt_version = self.version("rustfmt", "x86_64-unknown-linux-gnu");
self.llvm_tools_version = self.version("llvm-tools", "x86_64-unknown-linux-gnu");
self.miri_version = self.version("miri", "x86_64-unknown-linux-gnu");

self.rust_git_commit_hash = self.git_commit_hash("rust", "x86_64-unknown-linux-gnu");
self.cargo_git_commit_hash = self.git_commit_hash("cargo", "x86_64-unknown-linux-gnu");
self.rls_git_commit_hash = self.git_commit_hash("rls", "x86_64-unknown-linux-gnu");
self.rust_analyzer_git_commit_hash =
self.git_commit_hash("rust-analyzer", "x86_64-unknown-linux-gnu");
self.clippy_git_commit_hash = self.git_commit_hash("clippy", "x86_64-unknown-linux-gnu");
self.rustfmt_git_commit_hash = self.git_commit_hash("rustfmt", "x86_64-unknown-linux-gnu");
self.llvm_tools_git_commit_hash =
self.git_commit_hash("llvm-tools", "x86_64-unknown-linux-gnu");
self.miri_git_commit_hash = self.git_commit_hash("miri", "x86_64-unknown-linux-gnu");

self.check_toolstate();
self.digest_and_sign();
let manifest = self.build_manifest();
self.write_channel_files(&self.rust_release, &manifest);

if self.rust_release != "beta" && self.rust_release != "nightly" {
self.write_channel_files("stable", &manifest);
let rust_version = self.versions.package_version(&PkgType::Rust).unwrap();
self.write_channel_files(self.versions.channel(), &manifest);
if self.versions.channel() != rust_version {
self.write_channel_files(&rust_version, &manifest);
}
}

Expand All @@ -415,8 +313,7 @@ impl Builder {
// Mark some tools as missing based on toolstate.
if toolstates.get("miri").map(|s| &*s as &str) != Some("test-pass") {
println!("Miri tests are not passing, removing component");
self.miri_version = None;
self.miri_git_commit_hash = None;
self.versions.disable_version(&PkgType::Miri);
}
}

Expand Down Expand Up @@ -501,7 +398,7 @@ impl Builder {
// The compiler libraries are not stable for end users, and they're also huge, so we only
// `rustc-dev` for nightly users, and only in the "complete" profile. It's still possible
// for users to install the additional component manually, if needed.
if self.rust_release == "nightly" {
if self.versions.channel() == "nightly" {
self.extend_profile("complete", &mut manifest.profiles, &["rustc-dev"]);
self.extend_profile("complete", &mut manifest.profiles, &["rustc-docs"]);
}
Expand All @@ -518,13 +415,10 @@ impl Builder {
}

fn rust_package(&mut self, manifest: &Manifest) -> Package {
let version_info = self.versions.version(&PkgType::Rust).expect("missing Rust tarball");
let mut pkg = Package {
version: self
.cached_version("rust")
.as_ref()
.expect("Couldn't find Rust version")
.clone(),
git_commit_hash: self.cached_git_commit_hash("rust").clone(),
version: version_info.version.expect("missing Rust version"),
git_commit_hash: version_info.git_commit,
target: BTreeMap::new(),
};
for host in HOSTS {
Expand All @@ -539,7 +433,7 @@ impl Builder {
}

fn target_host_combination(&mut self, host: &str, manifest: &Manifest) -> Option<Target> {
let filename = self.filename("rust", host);
let filename = self.versions.tarball_name(&PkgType::Rust, host).unwrap();
let digest = self.digests.remove(&filename)?;
let xz_filename = filename.replace(".tar.gz", ".tar.xz");
let xz_digest = self.digests.remove(&xz_filename);
Expand Down Expand Up @@ -630,15 +524,14 @@ impl Builder {
}

fn package(&mut self, pkgname: &str, dst: &mut BTreeMap<String, Package>, targets: &[&str]) {
let (version, mut is_present) = self
.cached_version(pkgname)
.as_ref()
.cloned()
.map(|version| (version, true))
.unwrap_or_default(); // `is_present` defaults to `false` here.
let version_info = self
.versions
.version(&PkgType::from_component(pkgname))
.expect("failed to load package version");
let mut is_present = version_info.present;

// Never ship nightly-only components for other trains.
if self.rust_release != "nightly" && NIGHTLY_ONLY_COMPONENTS.contains(&pkgname) {
if self.versions.channel() != "nightly" && NIGHTLY_ONLY_COMPONENTS.contains(&pkgname) {
is_present = false; // Pretend the component is entirely missing.
}

Expand All @@ -647,7 +540,10 @@ impl Builder {
.map(|name| {
if is_present {
// The component generally exists, but it might still be missing for this target.
let filename = self.filename(pkgname, name);
let filename = self
.versions
.tarball_name(&PkgType::from_component(pkgname), name)
.unwrap();
let digest = match self.digests.remove(&filename) {
Some(digest) => digest,
// This component does not exist for this target -- skip it.
Expand Down Expand Up @@ -679,8 +575,8 @@ impl Builder {
dst.insert(
pkgname.to_string(),
Package {
version,
git_commit_hash: self.cached_git_commit_hash(pkgname).clone(),
version: version_info.version.unwrap_or_default(),
git_commit_hash: version_info.git_commit,
target: targets,
},
);
Expand All @@ -690,77 +586,6 @@ impl Builder {
format!("{}/{}/{}", self.s3_address, self.date, filename)
}

fn filename(&self, component: &str, target: &str) -> String {
use PkgType::*;
match PkgType::from_component(component) {
RustSrc => format!("rust-src-{}.tar.gz", self.rust_release),
Cargo => format!("cargo-{}-{}.tar.gz", self.cargo_release, target),
Rls => format!("rls-{}-{}.tar.gz", self.rls_release, target),
RustAnalyzer => {
format!("rust-analyzer-{}-{}.tar.gz", self.rust_analyzer_release, target)
}
Clippy => format!("clippy-{}-{}.tar.gz", self.clippy_release, target),
Rustfmt => format!("rustfmt-{}-{}.tar.gz", self.rustfmt_release, target),
LlvmTools => format!("llvm-tools-{}-{}.tar.gz", self.llvm_tools_release, target),
Miri => format!("miri-{}-{}.tar.gz", self.miri_release, target),
Other => format!("{}-{}-{}.tar.gz", component, self.rust_release, target),
}
}

fn cached_version(&self, component: &str) -> &Option<String> {
use PkgType::*;
match PkgType::from_component(component) {
Cargo => &self.cargo_version,
Rls => &self.rls_version,
RustAnalyzer => &self.rust_analyzer_version,
Clippy => &self.clippy_version,
Rustfmt => &self.rustfmt_version,
LlvmTools => &self.llvm_tools_version,
Miri => &self.miri_version,
_ => &self.rust_version,
}
}

fn cached_git_commit_hash(&self, component: &str) -> &Option<String> {
use PkgType::*;
match PkgType::from_component(component) {
Cargo => &self.cargo_git_commit_hash,
Rls => &self.rls_git_commit_hash,
RustAnalyzer => &self.rust_analyzer_git_commit_hash,
Clippy => &self.clippy_git_commit_hash,
Rustfmt => &self.rustfmt_git_commit_hash,
LlvmTools => &self.llvm_tools_git_commit_hash,
Miri => &self.miri_git_commit_hash,
_ => &self.rust_git_commit_hash,
}
}

fn version(&self, component: &str, target: &str) -> Option<String> {
self.untar(component, target, |filename| format!("{}/version", filename))
}

fn git_commit_hash(&self, component: &str, target: &str) -> Option<String> {
self.untar(component, target, |filename| format!("{}/git-commit-hash", filename))
}

fn untar<F>(&self, component: &str, target: &str, dir: F) -> Option<String>
where
F: FnOnce(String) -> String,
{
let mut cmd = Command::new("tar");
let filename = self.filename(component, target);
cmd.arg("xf")
.arg(self.input.join(&filename))
.arg(dir(filename.replace(".tar.gz", "")))
.arg("-O");
let output = t!(cmd.output());
if output.status.success() {
Some(String::from_utf8_lossy(&output.stdout).trim().to_string())
} else {
None
}
}

fn hash(&self, path: &Path) -> String {
let sha = t!(Command::new("shasum")
.arg("-a")
Expand Down
Loading

0 comments on commit 9b77a6a

Please sign in to comment.