Rollup merge of #97828 - ferrocene:pa-config-artifacts, r=jyn514

Allow configuring where artifacts are downloaded from

Bootstrap has support for downloading prebuilt LLVM and rustc artifacts to speed up local builds, but that currently works only for users working on `rust-lang/rust`. Forks of the repository (for example Ferrocene) might have different URLs to download artifacts from, or might use a different email address on merge commits, breaking both LLVM and rustc artifact downloads.

This PR refactors bootstrap to load the download URLs and other constants from `src/stage0.json`, allowing downstream forks to tweak those values. It also future-proofs the download code to easily allow forks to add their own custom protocols (like `s3://`).

This PR is best reviewed commit-by-commit.

Author: JohnTitor

src/bootstrap/bootstrap.py

@@ -1043,7 +1043,7 @@ def bootstrap(help_triggered):
     build.checksums_sha256 = data["checksums_sha256"]
     build.stage0_compiler = Stage0Toolchain(data["compiler"])
 
-    build.set_dist_environment(data["dist_server"])
+    build.set_dist_environment(data["config"]["dist_server"])
 
     build.build = args.build or build.build_triple()
 

src/bootstrap/builder.rs

@@ -870,20 +870,23 @@ impl<'a> Builder<'a> {
         self.try_run(patchelf.arg(fname));
     }
 
-    pub(crate) fn download_component(
-        &self,
-        base: &str,
-        url: &str,
-        dest_path: &Path,
-        help_on_error: &str,
-    ) {
+    pub(crate) fn download_component(&self, url: &str, dest_path: &Path, help_on_error: &str) {
         // Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/.
         let tempfile = self.tempdir().join(dest_path.file_name().unwrap());
-        self.download_with_retries(&tempfile, &format!("{}/{}", base, url), help_on_error);
+        // While bootstrap itself only supports http and https downloads, downstream forks might
+        // need to download components from other protocols. The match allows them adding more
+        // protocols without worrying about merge conficts if we change the HTTP implementation.
+        match url.split_once("://").map(|(proto, _)| proto) {
+            Some("http") | Some("https") => {
+                self.download_http_with_retries(&tempfile, url, help_on_error)
+            }
+            Some(other) => panic!("unsupported protocol {other} in {url}"),
+            None => panic!("no protocol in {url}"),
+        }
         t!(std::fs::rename(&tempfile, dest_path));
     }
 
-    fn download_with_retries(&self, tempfile: &Path, url: &str, help_on_error: &str) {
+    fn download_http_with_retries(&self, tempfile: &Path, url: &str, help_on_error: &str) {
         println!("downloading {}", url);
         // Try curl. If that fails and we are on windows, fallback to PowerShell.
         let mut curl = Command::new("curl");

src/bootstrap/config.rs

@@ -20,7 +20,6 @@ use crate::channel::GitInfo;
 pub use crate::flags::Subcommand;
 use crate::flags::{Color, Flags};
 use crate::util::{exe, output, program_out_of_date, t};
-use crate::RustfmtMetadata;
 use once_cell::sync::OnceCell;
 use serde::{Deserialize, Deserializer};
 
@@ -73,6 +72,7 @@ pub struct Config {
     pub test_compare_mode: bool,
     pub color: Color,
     pub patch_binaries_for_nix: bool,
+    pub stage0_metadata: Stage0Metadata,
 
     pub on_fail: Option<String>,
     pub stage: u32,
@@ -212,6 +212,28 @@ pub struct Config {
     pub out: PathBuf,
 }
 
+#[derive(Default, Deserialize)]
+#[cfg_attr(test, derive(Clone))]
+pub struct Stage0Metadata {
+    pub config: Stage0Config,
+    pub checksums_sha256: HashMap<String, String>,
+    pub rustfmt: Option<RustfmtMetadata>,
+}
+#[derive(Default, Deserialize)]
+#[cfg_attr(test, derive(Clone))]
+pub struct Stage0Config {
+    pub dist_server: String,
+    pub artifacts_server: String,
+    pub artifacts_with_llvm_assertions_server: String,
+    pub git_merge_commit_email: String,
+}
+#[derive(Default, Deserialize)]
+#[cfg_attr(test, derive(Clone))]
+pub struct RustfmtMetadata {
+    pub date: String,
+    pub version: String,
+}
+
 #[derive(Clone, Debug)]
 pub enum RustfmtState {
     SystemToolchain(PathBuf),
@@ -776,6 +798,9 @@ impl Config {
         config.llvm_profile_use = flags.llvm_profile_use;
         config.llvm_profile_generate = flags.llvm_profile_generate;
 
+        let stage0_json = t!(std::fs::read(&config.src.join("src").join("stage0.json")));
+        config.stage0_metadata = t!(serde_json::from_slice::<Stage0Metadata>(&stage0_json));
+
         #[cfg(test)]
         let get_toml = |_| TomlConfig::default();
         #[cfg(not(test))]
@@ -1103,8 +1128,11 @@ impl Config {
             config.rust_codegen_units_std = rust.codegen_units_std.map(threads_from_config);
             config.rust_profile_use = flags.rust_profile_use.or(rust.profile_use);
             config.rust_profile_generate = flags.rust_profile_generate.or(rust.profile_generate);
-            config.download_rustc_commit =
-                download_ci_rustc_commit(rust.download_rustc, config.verbose > 0);
+            config.download_rustc_commit = download_ci_rustc_commit(
+                &config.stage0_metadata,
+                rust.download_rustc,
+                config.verbose > 0,
+            );
         } else {
             config.rust_profile_use = flags.rust_profile_use;
             config.rust_profile_generate = flags.rust_profile_generate;
@@ -1424,7 +1452,11 @@ fn threads_from_config(v: u32) -> u32 {
 }
 
 /// Returns the commit to download, or `None` if we shouldn't download CI artifacts.
-fn download_ci_rustc_commit(download_rustc: Option<StringOrBool>, verbose: bool) -> Option<String> {
+fn download_ci_rustc_commit(
+    stage0_metadata: &Stage0Metadata,
+    download_rustc: Option<StringOrBool>,
+    verbose: bool,
+) -> Option<String> {
     // If `download-rustc` is not set, default to rebuilding.
     let if_unchanged = match download_rustc {
         None | Some(StringOrBool::Bool(false)) => return None,
@@ -1443,13 +1475,12 @@ fn download_ci_rustc_commit(download_rustc: Option<StringOrBool>, verbose: bool)
 
     // Look for a version to compare to based on the current commit.
     // Only commits merged by bors will have CI artifacts.
-    let merge_base = output(Command::new("git").args(&[
-        "rev-list",
-        "[email protected]",
-        "-n1",
-        "--first-parent",
-        "HEAD",
-    ]));
+    let merge_base = output(
+        Command::new("git")
+            .arg("rev-list")
+            .arg(format!("--author={}", stage0_metadata.config.git_merge_commit_email))
+            .args(&["-n1", "--first-parent", "HEAD"]),
+    );
     let commit = merge_base.trim_end();
     if commit.is_empty() {
         println!("error: could not find commit hash for downloading rustc");
@@ -1484,7 +1515,7 @@ fn download_ci_rustc_commit(download_rustc: Option<StringOrBool>, verbose: bool)
 }
 
 fn maybe_download_rustfmt(builder: &Builder<'_>) -> Option<PathBuf> {
-    let RustfmtMetadata { date, version } = builder.stage0_metadata.rustfmt.as_ref()?;
+    let RustfmtMetadata { date, version } = builder.config.stage0_metadata.rustfmt.as_ref()?;
     let channel = format!("{version}-{date}");
 
     let host = builder.config.build;
@@ -1568,13 +1599,13 @@ fn download_component(
     let tarball = cache_dir.join(&filename);
     let (base_url, url, should_verify) = match mode {
         DownloadSource::CI => (
-            "https://ci-artifacts.rust-lang.org/rustc-builds".to_string(),
+            builder.config.stage0_metadata.config.artifacts_server.clone(),
             format!("{key}/{filename}"),
             false,
         ),
         DownloadSource::Dist => {
             let dist_server = env::var("RUSTUP_DIST_SERVER")
-                .unwrap_or(builder.stage0_metadata.dist_server.to_string());
+                .unwrap_or(builder.config.stage0_metadata.config.dist_server.to_string());
             // NOTE: make `dist` part of the URL because that's how it's stored in src/stage0.json
             (dist_server, format!("dist/{key}/{filename}"), true)
         }
@@ -1590,7 +1621,7 @@ fn download_component(
             target at this time, see https://doc.rust-lang.org/nightly\
             /rustc/platform-support.html for more information."
         );
-        let sha256 = builder.stage0_metadata.checksums_sha256.get(&url).expect(&error);
+        let sha256 = builder.config.stage0_metadata.checksums_sha256.get(&url).expect(&error);
         if tarball.exists() {
             if builder.verify(&tarball, sha256) {
                 builder.unpack(&tarball, &bin_root, prefix);
@@ -1610,7 +1641,7 @@ fn download_component(
         None
     };
 
-    builder.download_component(&base_url, &url, &tarball, "");
+    builder.download_component(&format!("{base_url}/{url}"), &tarball, "");
     if let Some(sha256) = checksum {
         if !builder.verify(&tarball, sha256) {
             panic!("failed to verify {}", tarball.display());

src/bootstrap/lib.rs

@@ -118,7 +118,6 @@ use std::os::windows::fs::symlink_file;
 
 use filetime::FileTime;
 use once_cell::sync::OnceCell;
-use serde::Deserialize;
 
 use crate::builder::Kind;
 use crate::config::{LlvmLibunwind, TargetSelection};
@@ -294,8 +293,6 @@ pub struct Build {
     hosts: Vec<TargetSelection>,
     targets: Vec<TargetSelection>,
 
-    // Stage 0 (downloaded) compiler, lld and cargo or their local rust equivalents
-    stage0_metadata: Stage0Metadata,
     initial_rustc: PathBuf,
     initial_cargo: PathBuf,
     initial_lld: PathBuf,
@@ -322,18 +319,6 @@ pub struct Build {
     metrics: metrics::BuildMetrics,
 }
 
-#[derive(Deserialize)]
-struct Stage0Metadata {
-    dist_server: String,
-    checksums_sha256: HashMap<String, String>,
-    rustfmt: Option<RustfmtMetadata>,
-}
-#[derive(Deserialize)]
-struct RustfmtMetadata {
-    date: String,
-    version: String,
-}
-
 #[derive(Debug)]
 struct Crate {
     name: Interned<String>,
@@ -482,11 +467,7 @@ impl Build {
             bootstrap_out
         };
 
-        let stage0_json = t!(std::fs::read_to_string(&src.join("src").join("stage0.json")));
-        let stage0_metadata = t!(serde_json::from_str::<Stage0Metadata>(&stage0_json));
-
         let mut build = Build {
-            stage0_metadata,
             initial_rustc: config.initial_rustc.clone(),
             initial_cargo: config.initial_cargo.clone(),
             initial_lld,

src/bootstrap/native.rs

@@ -121,7 +121,7 @@ pub(crate) fn maybe_download_ci_llvm(builder: &Builder<'_>) {
     let mut rev_list = Command::new("git");
     rev_list.args(&[
         PathBuf::from("rev-list"),
-        "--author=[email protected]".into(),
+        format!("--author={}", builder.config.stage0_metadata.config.git_merge_commit_email).into(),
         "-n1".into(),
         "--first-parent".into(),
         "HEAD".into(),
@@ -170,11 +170,10 @@ fn download_ci_llvm(builder: &Builder<'_>, llvm_sha: &str) {
     if !rustc_cache.exists() {
         t!(fs::create_dir_all(&rustc_cache));
     }
-    let base = "https://ci-artifacts.rust-lang.org";
-    let url = if llvm_assertions {
-        format!("rustc-builds-alt/{}", llvm_sha)
+    let base = if llvm_assertions {
+        &builder.config.stage0_metadata.config.artifacts_with_llvm_assertions_server
     } else {
-        format!("rustc-builds/{}", llvm_sha)
+        &builder.config.stage0_metadata.config.artifacts_server
     };
     let filename = format!("rust-dev-nightly-{}.tar.xz", builder.build.build.triple);
     let tarball = rustc_cache.join(&filename);
@@ -187,7 +186,11 @@ help: if trying to compile an old commit of rustc, disable `download-ci-llvm` in
 [llvm]
 download-ci-llvm = false
 ";
-        builder.download_component(base, &format!("{}/{}", url, filename), &tarball, help_on_error);
+        builder.download_component(
+            &format!("{base}/{llvm_sha}/{filename}"),
+            &tarball,
+            help_on_error,
+        );
     }
     let llvm_root = builder.config.ci_llvm_root();
     builder.unpack(&tarball, &llvm_root, "rust-dev");

src/stage0.json

@@ -1,6 +1,20 @@
 {
-  "__comment": "Generated by `./x.py run src/tools/bump-stage0`. Run that command again to update the bootstrap compiler.",
-  "dist_server": "https://static.rust-lang.org",
+  "config": {
+    "dist_server": "https://static.rust-lang.org",
+    "artifacts_server": "https://ci-artifacts.rust-lang.org/rustc-builds",
+    "artifacts_with_llvm_assertions_server": "https://ci-artifacts.rust-lang.org/rustc-builds-alt",
+    "git_merge_commit_email": "[email protected]"
+  },
+  "__comments": [
+    "The configuration above this comment is editable, and can be changed",
+    "by forks of the repository if they have alternate values.",
+    "",
+    "The section below is generated by `./x.py run src/tools/bump-stage0`,",
+    "run that command again to update the bootstrap compiler.",
+    "",
+    "All changes below this comment will be overridden the next time the",
+    "tool is executed."
+  ],
   "compiler": {
     "date": "2022-05-20",
     "version": "beta"