From 00e1a90e569e8a76bab96877dac840955cbea9c0 Mon Sep 17 00:00:00 2001 From: Jan David Date: Mon, 20 May 2024 12:56:39 +0200 Subject: [PATCH 01/27] Fix error message when parsing actions --- src/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.rs b/src/config.rs index 070ce46..fe046aa 100644 --- a/src/config.rs +++ b/src/config.rs @@ -73,7 +73,7 @@ impl FromStr for Action { match input { "promote-release" => Ok(Action::PromoteRelease), "promote-branches" => Ok(Action::PromoteBranches), - _ => anyhow::bail!("unknown channel: {}", input), + _ => anyhow::bail!("unknown action: {}", input), } } } From 0a64d2565e4d8cf6b78a91707f2ab4f20731fe5b Mon Sep 17 00:00:00 2001 From: Jan David Date: Mon, 20 May 2024 13:17:09 +0200 Subject: [PATCH 02/27] Create action for rustup --- src/config.rs | 10 ++++++++++ src/main.rs | 2 ++ src/rustup.rs | 7 +++++++ 3 files changed, 19 insertions(+) create mode 100644 src/rustup.rs diff --git a/src/config.rs b/src/config.rs index fe046aa..fc3f0fb 100644 --- a/src/config.rs +++ b/src/config.rs @@ -48,6 +48,8 @@ impl std::fmt::Display for Channel { } } +// Allow all variant names to start with `Promote` +#[allow(clippy::enum_variant_names)] #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub(crate) enum Action { /// This is the default action, what we'll do if the environment variable @@ -64,6 +66,13 @@ pub(crate) enum Action { /// * Create a rust-lang/cargo branch for the appropriate beta commit. /// * Post a PR against the newly created beta branch bump src/ci/channel to `beta`. PromoteBranches, + + /// This promotes a new rustup release: + /// + /// * Copy binaries into archives + /// * Copy binaries from dev-static to production + /// * Update dev release number + PromoteRustup, } impl FromStr for Action { @@ -73,6 +82,7 @@ impl FromStr for Action { match input { "promote-release" => Ok(Action::PromoteRelease), "promote-branches" => Ok(Action::PromoteBranches), + "promote-rustup" => Ok(Action::PromoteRustup), _ => anyhow::bail!("unknown action: {}", input), } } diff --git a/src/main.rs b/src/main.rs index 5dd7aea..aed9722 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,6 +8,7 @@ mod discourse; mod fastly; mod github; mod recompress; +mod rustup; mod sign; mod smoke_test; @@ -76,6 +77,7 @@ impl Context { match self.config.action { config::Action::PromoteRelease => self.do_release()?, config::Action::PromoteBranches => self.do_branching()?, + config::Action::PromoteRustup => self.do_rustup()?, } Ok(()) } diff --git a/src/rustup.rs b/src/rustup.rs new file mode 100644 index 0000000..016adae --- /dev/null +++ b/src/rustup.rs @@ -0,0 +1,7 @@ +use crate::Context; + +impl Context { + pub fn do_rustup(&mut self) -> anyhow::Result<()> { + Ok(()) + } +} From 8d4a4476cbeb9ab6b3153df55738921fa2a9b83d Mon Sep 17 00:00:00 2001 From: Jan David Date: Wed, 22 May 2024 16:55:12 +0200 Subject: [PATCH 03/27] Implement release process for rustup --- src/main.rs | 2 +- src/rustup.rs | 139 +++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 138 insertions(+), 3 deletions(-) diff --git a/src/main.rs b/src/main.rs index aed9722..ad12f93 100644 --- a/src/main.rs +++ b/src/main.rs @@ -77,7 +77,7 @@ impl Context { match self.config.action { config::Action::PromoteRelease => self.do_release()?, config::Action::PromoteBranches => self.do_branching()?, - config::Action::PromoteRustup => self.do_rustup()?, + config::Action::PromoteRustup => self.promote_rustup()?, } Ok(()) } diff --git a/src/rustup.rs b/src/rustup.rs index 016adae..723ff7c 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -1,7 +1,142 @@ -use crate::Context; +use std::fs; +use std::path::{Path, PathBuf}; + +use anyhow::{anyhow, Error}; + +use crate::config::Channel; +use crate::{run, Context}; impl Context { - pub fn do_rustup(&mut self) -> anyhow::Result<()> { + /// Promote a `rustup` release + /// + /// The [release process] for `rustup` involves copying existing artifacts from one S3 bucket to + /// another, updating the manifest, and archiving the artifacts for long-term storage. + /// + /// `rustup` uses different branches to manage releases. Whenever a commit is pushed to the + /// `stable` branch in [rust-lang/rustup], GitHub Actions workflows build release artifacts and + /// copy them into `s3://dev-static-rust-lang-org/rustup/dist/`. + /// + /// When a new release is done and this method is invoked, it downloads the artifacts from that + /// bucket (which must always be set as the `DOWNLOAD_BUCKET` variable). A copy of the artifacts + /// is archived in `s3://${UPLOAD_BUCKET}/rustup/archive/${version}/`, where `version` is passed + /// to this program as a command-line argument. `UPLOAD_BUCKET` can either be the `dev-static` + /// or the `static` bucket. + /// + /// If the release is for the `stable` channel, the artifacts are also copied to the `dist/` + /// path in the `UPLOAD_BUCKET` bucket. The `dist/` path is used by the `rustup` installer to + /// download the latest release. + /// + /// Then, the `release-stable.toml` manifest is updated with the new version and copied to + /// `s3://${UPLOAD_BUCKET}/rustup/release-stable.toml`. + /// + /// [release process]: https://rust-lang.github.io/rustup/dev-guide/release-process.html + /// [rust-lang/rustup]: https://github.com/rust-lang/rustup + pub fn promote_rustup(&mut self) -> anyhow::Result<()> { + println!("Checking channel..."); + if self.config.channel != Channel::Stable && self.config.channel != Channel::Beta { + return Err(anyhow!( + "promoting rustup is only supported for the stable and beta channels" + )); + } + + // Download the rustup artifacts from S3 + println!("Downloading artifacts from dev-static..."); + let dist_dir = self.download_rustup_artifacts()?; + + // Archive the artifacts + println!("Archiving artifacts..."); + self.archive_rustup_artifacts(&dist_dir)?; + + if self.config.channel == Channel::Stable { + // Promote the artifacts to the release bucket + println!("Promoting artifacts to dist/..."); + self.promote_rustup_artifacts(&dist_dir)?; + } + + // Update the release number + println!("Updating version and manifest..."); + self.update_rustup_release()?; + Ok(()) } + + fn download_rustup_artifacts(&mut self) -> Result { + let dl = self.dl_dir().join("dist"); + // Remove the directory if it exists, otherwise just ignore. + let _ = fs::remove_dir_all(&dl); + fs::create_dir_all(&dl)?; + + run(self + .aws_s3() + .arg("cp") + .arg("--recursive") + .arg("--only-show-errors") + .arg(&self.s3_artifacts_url("dist/")) + .arg(format!("{}/", dl.display())))?; + + Ok(dl) + } + + fn archive_rustup_artifacts(&mut self, dist_dir: &Path) -> Result<(), Error> { + let version = self + .current_version + .as_ref() + .ok_or_else(|| anyhow!("failed to get current version for rustup release"))?; + + let path = format!("archive/{}/", version); + + self.upload_rustup_artifacts(dist_dir, &path) + } + + fn promote_rustup_artifacts(&mut self, dist_dir: &Path) -> Result<(), Error> { + let release_bucket_url = format!( + "s3://{}/{}/{}", + self.config.upload_bucket, + self.config.download_dir, + dist_dir.display(), + ); + + run(self + .aws_s3() + .arg("cp") + .arg("--recursive") + .arg("--only-show-errors") + .arg(format!("{}/", dist_dir.display())) + .arg(&release_bucket_url)) + } + + fn upload_rustup_artifacts(&mut self, dist_dir: &Path, target_path: &str) -> Result<(), Error> { + run(self + .aws_s3() + .arg("cp") + .arg("--recursive") + .arg("--only-show-errors") + .arg(format!("{}/", dist_dir.display())) + .arg(&self.s3_artifacts_url(target_path))) + } + + fn update_rustup_release(&mut self) -> Result<(), Error> { + let version = self + .current_version + .as_ref() + .ok_or_else(|| anyhow!("failed to get current version for rustup release"))?; + + let manifest_path = self.dl_dir().join("release-stable.toml"); + let manifest = format!( + r#" +schema-version = '1' +version = '{}' + "#, + version + ); + + fs::write(&manifest_path, manifest)?; + + run(self + .aws_s3() + .arg("cp") + .arg("--only-show-errors") + .arg(manifest_path) + .arg(&self.s3_artifacts_url("release-stable.toml"))) + } } From d40ff454acab7dc28d30661fde32ab085edcc278 Mon Sep 17 00:00:00 2001 From: Jan David Date: Thu, 27 Jun 2024 12:06:11 +0200 Subject: [PATCH 04/27] Extract channel check for Rustup into function --- src/rustup.rs | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/rustup.rs b/src/rustup.rs index 723ff7c..e88d187 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -32,12 +32,8 @@ impl Context { /// [release process]: https://rust-lang.github.io/rustup/dev-guide/release-process.html /// [rust-lang/rustup]: https://github.com/rust-lang/rustup pub fn promote_rustup(&mut self) -> anyhow::Result<()> { - println!("Checking channel..."); - if self.config.channel != Channel::Stable && self.config.channel != Channel::Beta { - return Err(anyhow!( - "promoting rustup is only supported for the stable and beta channels" - )); - } + // Rustup only has beta and stable releases, so we fail fast when trying to promote nightly + self.enforce_rustup_channel()?; // Download the rustup artifacts from S3 println!("Downloading artifacts from dev-static..."); @@ -60,6 +56,18 @@ impl Context { Ok(()) } + fn enforce_rustup_channel(&self) -> anyhow::Result<()> { + println!("Checking channel..."); + + if self.config.channel != Channel::Stable && self.config.channel != Channel::Beta { + return Err(anyhow!( + "promoting rustup is only supported for the stable and beta channels" + )); + } + + Ok(()) + } + fn download_rustup_artifacts(&mut self) -> Result { let dl = self.dl_dir().join("dist"); // Remove the directory if it exists, otherwise just ignore. From b803170598e8f2066e54f240e16c8a4fccdd5cad Mon Sep 17 00:00:00 2001 From: Jan David Date: Thu, 27 Jun 2024 12:07:41 +0200 Subject: [PATCH 05/27] Get latest commit from Rustup's stable branch --- src/rustup.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/rustup.rs b/src/rustup.rs index e88d187..e318a74 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -1,7 +1,7 @@ use std::fs; use std::path::{Path, PathBuf}; -use anyhow::{anyhow, Error}; +use anyhow::{anyhow, Context as AnyhowContext, Error}; use crate::config::Channel; use crate::{run, Context}; @@ -35,6 +35,9 @@ impl Context { // Rustup only has beta and stable releases, so we fail fast when trying to promote nightly self.enforce_rustup_channel()?; + // The latest commit on the `stable` branch is used to determine the version number + let head_sha = self.get_head_sha_for_rustup()?; + // Download the rustup artifacts from S3 println!("Downloading artifacts from dev-static..."); let dist_dir = self.download_rustup_artifacts()?; @@ -68,6 +71,14 @@ impl Context { Ok(()) } + fn get_head_sha_for_rustup(&self) -> anyhow::Result { + self.config + .github() + .context("failed to get HEAD SHA from GitHub - credentials not configured")? + .token("rust-lang/rustup")? + .get_ref("heads/stable") + } + fn download_rustup_artifacts(&mut self) -> Result { let dl = self.dl_dir().join("dist"); // Remove the directory if it exists, otherwise just ignore. From da3164d6bcf1a271e8e7162b8bbe00a91415e28e Mon Sep 17 00:00:00 2001 From: Jan David Date: Thu, 27 Jun 2024 14:22:55 +0200 Subject: [PATCH 06/27] Fetch next Rustup version from GitHub --- src/rustup.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/rustup.rs b/src/rustup.rs index e318a74..8c5dd50 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -2,6 +2,7 @@ use std::fs; use std::path::{Path, PathBuf}; use anyhow::{anyhow, Context as AnyhowContext, Error}; +use serde::Deserialize; use crate::config::Channel; use crate::{run, Context}; @@ -37,6 +38,7 @@ impl Context { // The latest commit on the `stable` branch is used to determine the version number let head_sha = self.get_head_sha_for_rustup()?; + let version = self.get_next_rustup_version(&head_sha)?; // Download the rustup artifacts from S3 println!("Downloading artifacts from dev-static..."); @@ -79,6 +81,26 @@ impl Context { .get_ref("heads/stable") } + fn get_next_rustup_version(&self, sha: &str) -> anyhow::Result { + println!("Getting next Rustup version from Cargo.toml..."); + + #[derive(Debug, Deserialize)] + struct CargoToml { + version: String, + } + + let cargo_toml = self + .config + .github() + .context("failed to get new rustup version from GitHub - credentials not configured")? + .token("rust-lang/rustup")? + .read_file(Some(sha), "Cargo.toml")?; + + let toml: CargoToml = toml::from_str(&cargo_toml.content()?)?; + + Ok(toml.version) + } + fn download_rustup_artifacts(&mut self) -> Result { let dl = self.dl_dir().join("dist"); // Remove the directory if it exists, otherwise just ignore. From 49980a339104f6ee41e6881ba55ff2f555b33cbf Mon Sep 17 00:00:00 2001 From: Jan David Date: Thu, 27 Jun 2024 14:24:59 +0200 Subject: [PATCH 07/27] Download Rustup artifacts for a given commit --- src/rustup.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/rustup.rs b/src/rustup.rs index 8c5dd50..ba53aa0 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -40,9 +40,8 @@ impl Context { let head_sha = self.get_head_sha_for_rustup()?; let version = self.get_next_rustup_version(&head_sha)?; - // Download the rustup artifacts from S3 - println!("Downloading artifacts from dev-static..."); - let dist_dir = self.download_rustup_artifacts()?; + // Download the Rustup artifacts from S3 + let dist_dir = self.download_rustup_artifacts(&head_sha)?; // Archive the artifacts println!("Archiving artifacts..."); @@ -101,7 +100,9 @@ impl Context { Ok(toml.version) } - fn download_rustup_artifacts(&mut self) -> Result { + fn download_rustup_artifacts(&mut self, sha: &str) -> Result { + println!("Downloading artifacts from dev-static..."); + let dl = self.dl_dir().join("dist"); // Remove the directory if it exists, otherwise just ignore. let _ = fs::remove_dir_all(&dl); @@ -112,7 +113,7 @@ impl Context { .arg("cp") .arg("--recursive") .arg("--only-show-errors") - .arg(&self.s3_artifacts_url("dist/")) + .arg(&self.s3_artifacts_url(&format!("builds/{sha}"))) .arg(format!("{}/", dl.display())))?; Ok(dl) From c4cff77d94b64d3d619452379a1e2195bbb88138 Mon Sep 17 00:00:00 2001 From: Jan David Date: Thu, 27 Jun 2024 14:27:23 +0200 Subject: [PATCH 08/27] Pass Rustup version to archive and manifest --- src/rustup.rs | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/src/rustup.rs b/src/rustup.rs index ba53aa0..04aa4fc 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -44,18 +44,15 @@ impl Context { let dist_dir = self.download_rustup_artifacts(&head_sha)?; // Archive the artifacts - println!("Archiving artifacts..."); - self.archive_rustup_artifacts(&dist_dir)?; + self.archive_rustup_artifacts(&dist_dir, &version)?; if self.config.channel == Channel::Stable { // Promote the artifacts to the release bucket - println!("Promoting artifacts to dist/..."); self.promote_rustup_artifacts(&dist_dir)?; } // Update the release number - println!("Updating version and manifest..."); - self.update_rustup_release()?; + self.update_rustup_release(&version)?; Ok(()) } @@ -119,11 +116,8 @@ impl Context { Ok(dl) } - fn archive_rustup_artifacts(&mut self, dist_dir: &Path) -> Result<(), Error> { - let version = self - .current_version - .as_ref() - .ok_or_else(|| anyhow!("failed to get current version for rustup release"))?; + fn archive_rustup_artifacts(&mut self, dist_dir: &Path, version: &str) -> Result<(), Error> { + println!("Archiving artifacts for version {version}..."); let path = format!("archive/{}/", version); @@ -131,6 +125,8 @@ impl Context { } fn promote_rustup_artifacts(&mut self, dist_dir: &Path) -> Result<(), Error> { + println!("Promoting artifacts to dist/..."); + let release_bucket_url = format!( "s3://{}/{}/{}", self.config.upload_bucket, @@ -157,11 +153,8 @@ impl Context { .arg(&self.s3_artifacts_url(target_path))) } - fn update_rustup_release(&mut self) -> Result<(), Error> { - let version = self - .current_version - .as_ref() - .ok_or_else(|| anyhow!("failed to get current version for rustup release"))?; + fn update_rustup_release(&mut self, version: &str) -> Result<(), Error> { + println!("Updating version and manifest..."); let manifest_path = self.dl_dir().join("release-stable.toml"); let manifest = format!( From 7ccff04c8bfdba76c05befea50b528a4adfa8458 Mon Sep 17 00:00:00 2001 From: Jan David Date: Thu, 27 Jun 2024 14:32:02 +0200 Subject: [PATCH 09/27] Update documentation for Rustup release process --- src/rustup.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/rustup.rs b/src/rustup.rs index 04aa4fc..493b6f9 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -15,13 +15,13 @@ impl Context { /// /// `rustup` uses different branches to manage releases. Whenever a commit is pushed to the /// `stable` branch in [rust-lang/rustup], GitHub Actions workflows build release artifacts and - /// copy them into `s3://dev-static-rust-lang-org/rustup/dist/`. + /// copy them into `s3://rustup-builds/builds/${commit-sha}/`. /// - /// When a new release is done and this method is invoked, it downloads the artifacts from that + /// When a new release is cut and this method is invoked, it downloads the artifacts from that /// bucket (which must always be set as the `DOWNLOAD_BUCKET` variable). A copy of the artifacts - /// is archived in `s3://${UPLOAD_BUCKET}/rustup/archive/${version}/`, where `version` is passed - /// to this program as a command-line argument. `UPLOAD_BUCKET` can either be the `dev-static` - /// or the `static` bucket. + /// is archived in `s3://${UPLOAD_BUCKET}/rustup/archive/${version}/`, where `version` is + /// derived from the Cargo.toml file in the `stable` branch. `UPLOAD_BUCKET` can either be the + /// `dev-static` or the `static` bucket. /// /// If the release is for the `stable` channel, the artifacts are also copied to the `dist/` /// path in the `UPLOAD_BUCKET` bucket. The `dist/` path is used by the `rustup` installer to From ede844c60b774613c5660c8b07d83500d5e70fdb Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 24 Sep 2024 10:50:23 +0200 Subject: [PATCH 10/27] Refactor run script to execute different local releases --- local/{run.sh => release.sh} | 0 local/rustup.sh | 8 ++++++++ local/setup.sh | 6 +++--- run.sh | 26 +++++++++++++++++++++----- 4 files changed, 32 insertions(+), 8 deletions(-) rename local/{run.sh => release.sh} (100%) create mode 100755 local/rustup.sh diff --git a/local/run.sh b/local/release.sh similarity index 100% rename from local/run.sh rename to local/release.sh diff --git a/local/rustup.sh b/local/rustup.sh new file mode 100755 index 0000000..f889d54 --- /dev/null +++ b/local/rustup.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# This script is executed at the start of each local release for Rustup, and +# prepares the environment by copying the artifacts built by CI onto the MinIO +# instance. Then, it starts promote-release with the right flags. + +set -euo pipefail +IFS=$'\n\t' diff --git a/local/setup.sh b/local/setup.sh index 2e7e1e0..14b2f45 100755 --- a/local/setup.sh +++ b/local/setup.sh @@ -77,9 +77,9 @@ cat < [commit]" +if [[ "$#" -lt 1 ]]; then + echo "Usage: $0 " + exit 1 +fi +command="$1" + +if [[ "${command}" == "release" ]]; then + if [[ "$#" -lt 2 ]] || [[ "$#" -gt 3 ]]; then + echo "Usage: $0 release [commit]" + exit 1 + fi +fi + +if [[ "${command}" == "rustup" ]]; then + if [[ "$#" -ne 2 ]]; then + echo "Usage: $0 rustup [commit]" exit 1 + fi fi -channel="$1" -override_commit="${2-}" + +channel="$2" +override_commit="${3-}" container_id="$(docker-compose ps -q local)" if [[ "${container_id}" == "" ]]; then @@ -31,4 +47,4 @@ fi cargo build --release # Run the command inside the docker environment. -docker-compose exec -T local /src/local/run.sh "${channel}" "${override_commit}" +docker-compose exec -T local "/src/local/${command}.sh" "${channel}" "${override_commit}" From 1e516af93b00ea66d005d70c2725fe9ff9020650 Mon Sep 17 00:00:00 2001 From: Jan David Date: Wed, 25 Sep 2024 13:11:45 +0200 Subject: [PATCH 11/27] Support other platforms than Linux --- local/Dockerfile | 8 +++++--- local/release.sh | 8 ++++++++ run.sh | 6 ++++-- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/local/Dockerfile b/local/Dockerfile index 0087840..406b20c 100644 --- a/local/Dockerfile +++ b/local/Dockerfile @@ -16,13 +16,15 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ git \ gnupg \ jq \ + libssl-dev \ + openssl \ + pkg-config \ python3 \ socat # Install rustup while removing the pre-installed stable toolchain. -RUN curl https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init >/tmp/rustup-init && \ - chmod +x /tmp/rustup-init && \ - /tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain stable && \ +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ + sh -s -- -y --no-modify-path --profile minimal --default-toolchain stable && \ /root/.cargo/bin/rustup toolchain remove stable ENV PATH=/root/.cargo/bin:$PATH diff --git a/local/release.sh b/local/release.sh index 45f7b98..7564352 100755 --- a/local/release.sh +++ b/local/release.sh @@ -101,6 +101,14 @@ for file in "${DOWNLOAD_STANDALONE[@]}"; do download "${file}" done +# Build the promote-release binary if it hasn't been pre-built +if [[ ! -f "/src/target/release/promote-release" ]]; then + echo "==> building promote-release" + cd /src + cargo build --release + cd .. +fi + echo "==> configuring the environment" # Point to the right GnuPG environment export GNUPGHOME=/persistent/gpg-home diff --git a/run.sh b/run.sh index 217f4ab..2a4e0f3 100755 --- a/run.sh +++ b/run.sh @@ -43,8 +43,10 @@ if [[ "${container_status}" != "running" ]]; then exit 1 fi -# Ensure the release build is done -cargo build --release +# Pre-built the binary if the host and Docker environments match +if [[ "$(uname)" == "Linux" ]]; then + cargo build --release +fi # Run the command inside the docker environment. docker-compose exec -T local "/src/local/${command}.sh" "${channel}" "${override_commit}" From b1dd2a2c21f86cde5a6877d1bb35e0dea9b8ee34 Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 1 Oct 2024 12:29:16 +0200 Subject: [PATCH 12/27] Download Rustup files from CDN --- local/rustup.sh | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/local/rustup.sh b/local/rustup.sh index f889d54..1c35379 100755 --- a/local/rustup.sh +++ b/local/rustup.sh @@ -6,3 +6,51 @@ set -euo pipefail IFS=$'\n\t' + +RUSTUP_REPO="https://github.com/rust-lang/rustup" +RUSTUP_DEFAULT_BRANCH="master" + +# S3 bucket from which to download the Rustup artifacts +S3_BUCKET="rustup-builds" + +# CDN from which to download the CI artifacts +DOWNLOAD_BASE="https://rustup-builds.rust-lang.org" + +# The artifacts for the following targets will be downloaded and copied during +# the release process. At least one target is required. +DOWNLOAD_TARGETS=( + "x86_64-unknown-linux-gnu" +) + +# The following files will be downloaded and put into the local MinIO instance. +DOWNLOAD_FILES=( + "rustup-init" + "rustup-init.sha256" + "rustup-setup" + "rustup-setup.sha256" +) + +channel="$1" +override_commit="$2" + +if [[ "${override_commit}" = "" ]]; then + echo "==> detecting the last Rustup commit on the default branch" + commit="$(git ls-remote "${RUSTUP_REPO}" | grep "refs/heads/${RUSTUP_DEFAULT_BRANCH}" | awk '{print($1)}')" +else + echo "=>> using overridden commit ${override_commit}" + commit="${override_commit}" +fi + +for target in "${DOWNLOAD_TARGETS[@]}"; do + if ! mc stat "local/artifacts/builds/${commit}/dist/${target}" >/dev/null 2>&1; then + echo "==> copying ${target} from S3" + + for file in "${DOWNLOAD_FILES[@]}"; do + if curl -Lo /tmp/component "${DOWNLOAD_BASE}/${commit}/dist/${target}/${file}" --fail; then + mc cp /tmp/component "local/artifacts/builds/${commit}/dist/${target}/${file}" >/dev/null + fi + done + else + echo "==> reusing cached ${target} target" + fi +done From b843e516c513e1fd7a0f701a7d549bc472d2684e Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 1 Oct 2024 12:29:49 +0200 Subject: [PATCH 13/27] Build promote-release inside the container if necessary --- local/rustup.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/local/rustup.sh b/local/rustup.sh index 1c35379..e0f0658 100755 --- a/local/rustup.sh +++ b/local/rustup.sh @@ -54,3 +54,11 @@ for target in "${DOWNLOAD_TARGETS[@]}"; do echo "==> reusing cached ${target} target" fi done + +# Build the promote-release binary if it hasn't been pre-built +if [[ ! -f "/src/target/release/promote-release" ]]; then + echo "==> building promote-release" + cd /src + cargo build --release + cd .. +fi From f5c3c5e104364b17b098d54fa09fb6f50ef0917d Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 1 Oct 2024 12:41:39 +0200 Subject: [PATCH 14/27] Get download path from config --- src/rustup.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/rustup.rs b/src/rustup.rs index 493b6f9..0a64d9b 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -105,12 +105,14 @@ impl Context { let _ = fs::remove_dir_all(&dl); fs::create_dir_all(&dl)?; + let download_path = format!("{}/{}", self.config.download_dir, sha); + run(self .aws_s3() .arg("cp") .arg("--recursive") .arg("--only-show-errors") - .arg(&self.s3_artifacts_url(&format!("builds/{sha}"))) + .arg(&self.s3_artifacts_url(&download_path)) .arg(format!("{}/", dl.display())))?; Ok(dl) From a947033cc61807d71d919d731252f7c8bd9b3237 Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 1 Oct 2024 12:51:54 +0200 Subject: [PATCH 15/27] Fix upload directory --- src/rustup.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rustup.rs b/src/rustup.rs index 0a64d9b..a0c3e21 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -132,7 +132,7 @@ impl Context { let release_bucket_url = format!( "s3://{}/{}/{}", self.config.upload_bucket, - self.config.download_dir, + self.config.upload_dir, dist_dir.display(), ); From 883b5ede798214ceed0755d550ae5b5cd76fb534 Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 1 Oct 2024 13:03:55 +0200 Subject: [PATCH 16/27] Get version from user-provided commit --- src/rustup.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/rustup.rs b/src/rustup.rs index a0c3e21..9a68d12 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -36,8 +36,10 @@ impl Context { // Rustup only has beta and stable releases, so we fail fast when trying to promote nightly self.enforce_rustup_channel()?; - // The latest commit on the `stable` branch is used to determine the version number - let head_sha = self.get_head_sha_for_rustup()?; + // Get the latest commit from the `stable` branch or use the user-provided override + let head_sha = self.get_commit_sha_for_rustup_release()?; + + // The commit on the `stable` branch is used to determine the version number let version = self.get_next_rustup_version(&head_sha)?; // Download the Rustup artifacts from S3 @@ -69,6 +71,13 @@ impl Context { Ok(()) } + fn get_commit_sha_for_rustup_release(&self) -> anyhow::Result { + match &self.config.override_commit { + Some(sha) => Ok(sha.clone()), + None => self.get_head_sha_for_rustup(), + } + } + fn get_head_sha_for_rustup(&self) -> anyhow::Result { self.config .github() From ad86e13e8361ab9750f49c4c9122ae374cac6288 Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 1 Oct 2024 15:05:30 +0200 Subject: [PATCH 17/27] Don't require GitHub credentials to fetch Rustup metadata --- src/rustup.rs | 58 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 43 insertions(+), 15 deletions(-) diff --git a/src/rustup.rs b/src/rustup.rs index 9a68d12..4cbf7f1 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -1,10 +1,12 @@ use std::fs; use std::path::{Path, PathBuf}; -use anyhow::{anyhow, Context as AnyhowContext, Error}; +use anyhow::{anyhow, Error}; +use curl::easy::Easy; use serde::Deserialize; use crate::config::Channel; +use crate::curl_helper::BodyExt; use crate::{run, Context}; impl Context { @@ -79,31 +81,57 @@ impl Context { } fn get_head_sha_for_rustup(&self) -> anyhow::Result { - self.config - .github() - .context("failed to get HEAD SHA from GitHub - credentials not configured")? - .token("rust-lang/rustup")? - .get_ref("heads/stable") + #[derive(Deserialize)] + struct Commit { + sha: String, + } + + let url = format!( + "https://api.github.com/repos/rust-lang/rustup/commits/{}", + self.config.channel + ); + + let mut client = Easy::new(); + client.url(&url)?; + client.useragent("rust-lang/promote-release")?; + + let commit: Commit = client.without_body().send_with_response()?; + + Ok(commit.sha) } fn get_next_rustup_version(&self, sha: &str) -> anyhow::Result { println!("Getting next Rustup version from Cargo.toml..."); - #[derive(Debug, Deserialize)] + #[derive(Deserialize)] + struct Content { + content: String, + } + + #[derive(Deserialize)] struct CargoToml { + package: Package, + } + + #[derive(Deserialize)] + struct Package { version: String, } - let cargo_toml = self - .config - .github() - .context("failed to get new rustup version from GitHub - credentials not configured")? - .token("rust-lang/rustup")? - .read_file(Some(sha), "Cargo.toml")?; + let url = + format!("https://api.github.com/repos/rust-lang/rustup/contents/Cargo.toml?ref={sha}"); + + let mut client = Easy::new(); + client.url(&url)?; + client.useragent("rust-lang/promote-release")?; + + let content: Content = client.without_body().send_with_response()?; + let decoded_content = base64::decode(&content.content.replace('\n', ""))?; + let cargo_toml = String::from_utf8(decoded_content)?; - let toml: CargoToml = toml::from_str(&cargo_toml.content()?)?; + let toml: CargoToml = toml::from_str(&cargo_toml)?; - Ok(toml.version) + Ok(toml.package.version) } fn download_rustup_artifacts(&mut self, sha: &str) -> Result { From 694e3a48924cf983035ea4504c1bbe416dc31d57 Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 1 Oct 2024 15:19:47 +0200 Subject: [PATCH 18/27] Create rustup-builds bucket in local environment --- local/setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/local/setup.sh b/local/setup.sh index 14b2f45..97219b4 100755 --- a/local/setup.sh +++ b/local/setup.sh @@ -11,7 +11,7 @@ MINIO_URL="http://${MINIO_HOST}:${MINIO_PORT}" MINIO_ACCESS_KEY="access_key" MINIO_SECRET_KEY="secret_key" -MINIO_BUCKETS=( "static" "artifacts" ) +MINIO_BUCKETS=( "static" "artifacts" "rustup-builds" ) # Quit immediately when docker-compose receives a Ctrl+C trap exit EXIT From 8132d6928e4071f40b27573faed06ceb9c8afbb6 Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 1 Oct 2024 15:20:22 +0200 Subject: [PATCH 19/27] Fix upload destination for Rustup archives --- src/rustup.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/rustup.rs b/src/rustup.rs index 4cbf7f1..757ca20 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -189,7 +189,10 @@ impl Context { .arg("--recursive") .arg("--only-show-errors") .arg(format!("{}/", dist_dir.display())) - .arg(&self.s3_artifacts_url(target_path))) + .arg(format!( + "s3://{}/{}/{}", + self.config.upload_bucket, self.config.upload_dir, target_path + ))) } fn update_rustup_release(&mut self, version: &str) -> Result<(), Error> { From 89f7324e653aa5b4ff91311ce734b41acfc48942 Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 1 Oct 2024 15:24:50 +0200 Subject: [PATCH 20/27] Fix upload destination for stable Rustup release --- src/rustup.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/rustup.rs b/src/rustup.rs index 757ca20..7f3d4da 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -167,10 +167,8 @@ impl Context { println!("Promoting artifacts to dist/..."); let release_bucket_url = format!( - "s3://{}/{}/{}", - self.config.upload_bucket, - self.config.upload_dir, - dist_dir.display(), + "s3://{}/{}/dist/", + self.config.upload_bucket, self.config.upload_dir, ); run(self From f82d531655049e6d588b7561b69c708f3b7ef203 Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 1 Oct 2024 15:29:13 +0200 Subject: [PATCH 21/27] Fix upload destination for Rustup manifest --- src/rustup.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/rustup.rs b/src/rustup.rs index 7f3d4da..163a6d2 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -212,6 +212,9 @@ version = '{}' .arg("cp") .arg("--only-show-errors") .arg(manifest_path) - .arg(&self.s3_artifacts_url("release-stable.toml"))) + .arg(format!( + "s3://{}/{}/release-stable.toml", + self.config.upload_bucket, self.config.upload_dir + ))) } } From 9b7dba8ee92c94fc493ef588ecd2f3fc773e3443 Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 1 Oct 2024 15:35:14 +0200 Subject: [PATCH 22/27] Configure environment to run Rustup releases locally --- local/rustup.sh | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/local/rustup.sh b/local/rustup.sh index e0f0658..5d7ff8c 100755 --- a/local/rustup.sh +++ b/local/rustup.sh @@ -62,3 +62,37 @@ if [[ ! -f "/src/target/release/promote-release" ]]; then cargo build --release cd .. fi + +echo "==> configuring the environment" + +# Release Rustup +export PROMOTE_RELEASE_ACTION="promote-rustup" + +# Point to the right GnuPG environment +export GNUPGHOME=/persistent/gpg-home + +## Environment variables also used in prod releases +export AWS_ACCESS_KEY_ID="access_key" +export AWS_SECRET_ACCESS_KEY="secret_key" +export PROMOTE_RELEASE_CHANNEL="${channel}" +export PROMOTE_RELEASE_CLOUDFRONT_DOC_ID="" +export PROMOTE_RELEASE_CLOUDFRONT_STATIC_ID="" +export PROMOTE_RELEASE_DOWNLOAD_BUCKET="rustup-builds" +export PROMOTE_RELEASE_DOWNLOAD_DIR="builds" +export PROMOTE_RELEASE_GPG_KEY_FILE="" +export PROMOTE_RELEASE_GPG_PASSWORD_FILE="" +export PROMOTE_RELEASE_UPLOAD_ADDR="" +export PROMOTE_RELEASE_UPLOAD_BUCKET="static" +export PROMOTE_RELEASE_UPLOAD_STORAGE_CLASS="STANDARD" +export PROMOTE_RELEASE_UPLOAD_DIR="rustup" + +## Environment variables used only by local releases +export PROMOTE_RELEASE_S3_ENDPOINT_URL="http://minio:9000" + +# Conditional environment variables +if [[ "${override_commit}" != "" ]]; then + export PROMOTE_RELEASE_OVERRIDE_COMMIT="${override_commit}" +fi + +echo "==> starting promote-release" +/src/target/release/promote-release /persistent/release "${channel}" From 5aad90dedf5dc6b9034031b5fc67ee943c993170 Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 1 Oct 2024 15:41:48 +0200 Subject: [PATCH 23/27] Set up CI workflows for refactored actions --- .github/workflows/ci.yml | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 71178f6..0724cc3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,4 @@ --- - name: CI on: push: @@ -29,7 +28,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - local: + release: name: Local release runs-on: ubuntu-latest @@ -49,7 +48,7 @@ jobs: run: docker-compose up -d - name: Run the local release process for channel ${{ matrix.channel }} - run: ./run.sh ${{ matrix.channel }} + run: ./run.sh release ${{ matrix.channel }} - name: Validate the generated signatures run: docker-compose exec -T local /src/local/check-signature.sh ${{ matrix.channel }} @@ -62,6 +61,33 @@ jobs: env: RUSTUP_DIST_SERVER: http://localhost:9000/static + rustup: + name: Local rustup + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + channel: [stable, beta] + + steps: + - name: Clone the source code + uses: actions/checkout@v3 + + - name: Ensure Rust Stable is up to date + run: rustup self update && rustup update stable + + - name: Start the local environment + run: docker-compose up -d + + - name: Run the local release process for channel ${{ matrix.channel }} + run: ./run.sh rustup ${{ matrix.channel }} + + - name: Update Rustup from the local environment + run: rustup self update + env: + RUSTUP_UPDATE_ROOT: http://localhost:9000/static/rustup + docker: name: Build Docker image runs-on: ubuntu-latest From a52f9753bd2f53a9d230631a30ca0e4f6d5ed840 Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 1 Oct 2024 15:43:34 +0200 Subject: [PATCH 24/27] Fix required jobs for deployments --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0724cc3..e9f18d9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -117,7 +117,7 @@ jobs: permissions: id-token: write - needs: [test, local, docker] + needs: [test, release, rustup, docker] if: github.event_name == 'push' && github.repository == 'rust-lang/promote-release' && github.ref == 'refs/heads/master' steps: From ad0c357a235638e9b84b9e4c0df77e713bdbba60 Mon Sep 17 00:00:00 2001 From: Jan David Date: Thu, 3 Oct 2024 11:42:28 +0200 Subject: [PATCH 25/27] Fix unnecessary borrow warnings --- src/rustup.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rustup.rs b/src/rustup.rs index 163a6d2..a2eaaf4 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -126,7 +126,7 @@ impl Context { client.useragent("rust-lang/promote-release")?; let content: Content = client.without_body().send_with_response()?; - let decoded_content = base64::decode(&content.content.replace('\n', ""))?; + let decoded_content = base64::decode(content.content.replace('\n', ""))?; let cargo_toml = String::from_utf8(decoded_content)?; let toml: CargoToml = toml::from_str(&cargo_toml)?; @@ -149,7 +149,7 @@ impl Context { .arg("cp") .arg("--recursive") .arg("--only-show-errors") - .arg(&self.s3_artifacts_url(&download_path)) + .arg(self.s3_artifacts_url(&download_path)) .arg(format!("{}/", dl.display())))?; Ok(dl) From fd3f06d87f511e2ce3bae8ce956e0d49b27acf85 Mon Sep 17 00:00:00 2001 From: Jan David Date: Thu, 3 Oct 2024 11:44:35 +0200 Subject: [PATCH 26/27] Cut beta releases for Rustup from the stable branch --- src/rustup.rs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/rustup.rs b/src/rustup.rs index a2eaaf4..6c045c9 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -86,13 +86,8 @@ impl Context { sha: String, } - let url = format!( - "https://api.github.com/repos/rust-lang/rustup/commits/{}", - self.config.channel - ); - let mut client = Easy::new(); - client.url(&url)?; + client.url("https://api.github.com/repos/rust-lang/rustup/commits/stable")?; client.useragent("rust-lang/promote-release")?; let commit: Commit = client.without_body().send_with_response()?; From a3a0d897f5280185a3e5dca49433d07e17ad12f9 Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 8 Oct 2024 15:48:59 +0200 Subject: [PATCH 27/27] Allow Rustup version to be overridden in tests --- .github/workflows/ci.yml | 3 +++ src/rustup.rs | 10 ++++++++++ 2 files changed, 13 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 05c8dbd..f8966ad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,6 +65,9 @@ jobs: name: Local rustup runs-on: ubuntu-latest + env: + PROMOTE_RELEASE_RUSTUP_OVERRIDE_VERSION: 99.0.0 + strategy: fail-fast: false matrix: diff --git a/src/rustup.rs b/src/rustup.rs index 6c045c9..5c065fb 100644 --- a/src/rustup.rs +++ b/src/rustup.rs @@ -96,6 +96,16 @@ impl Context { } fn get_next_rustup_version(&self, sha: &str) -> anyhow::Result { + // Allow the version to be overridden manually, for example to test the release process + if let Ok(version) = std::env::var("PROMOTE_RELEASE_RUSTUP_OVERRIDE_VERSION") { + println!("Using override version: {}", version); + Ok(version) + } else { + self.get_next_rustup_version_from_github(sha) + } + } + + fn get_next_rustup_version_from_github(&self, sha: &str) -> anyhow::Result { println!("Getting next Rustup version from Cargo.toml..."); #[derive(Deserialize)]