diff --git a/.github/scripts/get-or-create-release-branch.sh b/.github/scripts/get-or-create-release-branch.sh new file mode 100755 index 0000000000..a32c5afe60 --- /dev/null +++ b/.github/scripts/get-or-create-release-branch.sh @@ -0,0 +1,82 @@ +#!/bin/bash +# +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# +set -e + +# Compute the name of the release branch starting from the version that needs to be released ($SEMANTIC_VERSION). +# If it's the beginning of a new release series, the branch is created and pushed to the remote (chosen according to +# the value $DRY_RUN). +# If it isn't the beginning of a new release series, the script makes sure that the commit that will be tagged is at +# the tip of the (pre-existing) release branch. +# +# The script populates an output file with key-value pairs that are needed in the release CI workflow to carry out +# the next steps in the release flow: the name of the release branch and a boolean flag that is set to 'true' if this +# is the beginning of a new release series. + +if [ -z "$SEMANTIC_VERSION" ]; then + echo "'SEMANTIC_VERSION' must be populated." + exit 1 +fi + +if [ -z "$1" ]; then + echo "You need to specify the path of the file where you want to collect the output" + exit 1 +else + output_file="$1" +fi + +# Split on the dots +version_array=(${SEMANTIC_VERSION//./ }) +major=${version_array[0]} +minor=${version_array[1]} +patch=${version_array[2]} +if [[ "${major}" == "" || "${minor}" == "" || "${patch}" == "" ]]; then + echo "'${SEMANTIC_VERSION}' is not a valid semver tag" + exit 1 +fi +if [[ $major == 0 ]]; then + branch_name="smithy-rs-release-${major}.${minor}.x" + if [[ $patch == 0 ]]; then + echo "new_release_series=true" >"${output_file}" + fi +else + branch_name="smithy-rs-release-${major}.x.y" + if [[ $minor == 0 && $patch == 0 ]]; then + echo "new_release_series=true" >"${output_file}" + fi +fi + +if [[ "${DRY_RUN}" == "true" ]]; then + branch_name="${branch_name}-preview" +fi +echo "release_branch=${branch_name}" >"${output_file}" + +if [[ "${DRY_RUN}" == "true" ]]; then + git push --force origin "HEAD:${branch_name}" +else + commit_sha=$(git rev-parse --short HEAD) + if git ls-remote --exit-code --heads origin "${branch_name}"; then + # The release branch already exists, we need to make sure that our commit is its current tip + branch_head_sha=$(git rev-parse --verify --short "refs/heads/${branch_name}") + if [[ "${branch_head_sha}" != "${commit_sha}" ]]; then + echo "The release branch - ${branch_name} - already exists. ${commit_sha}, the commit you chose when " + echo "launching this release, is not its current HEAD (${branch_head_sha}). This is not allowed: you " + echo "MUST release from the HEAD of the release branch if it already exists." + exit 1 + fi + else + # The release branch does not exist. + # We need to make sure that the commit SHA that we are releasing is on `main`. + git fetch origin main + if git branch --contains "${commit_sha}" | grep main; then + # We can then create the release branch and set the current commit as its tip + git checkout -b "${branch_name}" + git push origin "${branch_name}" + else + echo "You must choose a commit from main to create a new release series!" + exit 1 + fi + fi +fi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 94c0ed4681..7c674e613e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,6 +14,13 @@ on: required: false default: false type: boolean + git_ref: + description: | + The git reference that all checks should be run against. It can be a branch, a tag or a commit SHA. + If unspecified, it will default to the git reference or SHA that triggered the execution of this workflow. + required: false + type: string + default: "" env: rust_version: 1.62.1 @@ -39,6 +46,7 @@ jobs: - uses: actions/checkout@v3 with: path: smithy-rs + ref: ${{ inputs.git_ref }} # The models from aws-sdk-rust are needed to generate the full SDK for CI - uses: actions/checkout@v3 with: @@ -84,6 +92,7 @@ jobs: - uses: actions/checkout@v3 with: path: smithy-rs + ref: ${{ inputs.git_ref }} - name: Run ${{ matrix.test.action }} uses: ./smithy-rs/.github/actions/docker-build with: @@ -113,6 +122,7 @@ jobs: - uses: actions/checkout@v3 with: path: smithy-rs + ref: ${{ inputs.git_ref }} - name: Run ${{ matrix.test.action }} uses: ./smithy-rs/.github/actions/docker-build with: @@ -128,6 +138,8 @@ jobs: RUSTFLAGS: -D warnings steps: - uses: actions/checkout@v3 + with: + ref: ${{ inputs.git_ref }} # Pinned to the commit hash of v2.1.0 - uses: Swatinem/rust-cache@b894d59a8d236e2979b247b80dac8d053ab340dd with: @@ -190,6 +202,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 + with: + ref: ${{ inputs.git_ref }} # Pinned to the commit hash of v2.1.0 - uses: Swatinem/rust-cache@b894d59a8d236e2979b247b80dac8d053ab340dd with: @@ -258,6 +272,7 @@ jobs: - uses: actions/checkout@v3 with: path: smithy-rs + ref: ${{ inputs.git_ref }} - name: Run ${{ matrix.actions.action }} uses: ./smithy-rs/.github/actions/docker-build with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b4ac6c289c..e33c355b9d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,48 +6,44 @@ # Allow only one release to run at a time concurrency: - group: release-smithy-rs + group: release-smithy-rs-${{ inputs.dry_run }} cancel-in-progress: true env: rust_version: 1.62.1 name: Release smithy-rs -run-name: ${{ github.workflow }} - ${{ inputs.dry_run && 'Dry run' || 'Production run' }} +run-name: ${{ github.workflow }} ${{ inputs.semantic_version }} (${{ inputs.commit_sha }}) - ${{ inputs.dry_run && 'Dry run' || 'Production run' }} on: workflow_dispatch: inputs: + commit_sha: + description: The SHA of the git commit that you want to release (e.g. b2318b0) + required: true + type: string + semantic_version: + description: The semver tag that you want to release (e.g. 0.52.1) + required: true + type: string dry_run: - description: Dry runs will only produce release artifacts, but will not cut a release tag in GitHub nor publish to crates.io + description: Dry runs will only produce release artifacts, but they will not cut a release tag in GitHub nor publish to crates.io required: true type: boolean default: true jobs: - main-branch-check: - name: Check that workflow is running in main - runs-on: ubuntu-latest - steps: - - name: Main branch check - if: ${{ github.ref_name != 'main' }} - uses: actions/github-script@v6 - with: - script: | - core.setFailed("The release workflow can only be ran on main (current branch: ${{ github.ref_name }})") - - # If a release is kicked off before an image is built after push to main, - # or if a dry-run release is kicked off against a non-main branch to test - # automation changes, we'll need to build a base image to work against. - # This job will be a no-op if an image was already built on main. + # We'll need to build a base image to work against if: + # - a release was kicked off before the image build step triggered by a push to the release branch/main completed + # - a dry-run release was kicked off against a feature branch to test automation changes + # This job will be a no-op if an image had already been built. acquire-base-image: name: Acquire Base Image - needs: - - main-branch-check runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 with: path: smithy-rs + ref: ${{ inputs.commit_sha }} fetch-depth: 0 - name: Acquire base image id: acquire @@ -61,17 +57,74 @@ jobs: release-ci: name: Prerelease checks + if: inputs.dry_run == false needs: - acquire-base-image uses: ./.github/workflows/ci.yml with: run_sdk_examples: false + git_ref: ${{ inputs.commit_sha }} + + get-or-create-release-branch: + name: Get or create a release branch + needs: + - release-ci + outputs: + release_branch: ${{ steps.branch-push.outputs.release_branch }} + new_release_series: ${{ steps.branch-push.outputs.new_release_series }} + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ inputs.commit_sha }} + token: ${{ secrets.RELEASE_AUTOMATION_BOT_PAT }} + - name: Get or create release branch + id: branch-push + shell: bash + env: + SEMANTIC_VERSION: ${{ inputs.semantic_version }} + DRY_RUN: ${{ inputs.dry_run }} + run: | + set -e + + ./.github/scripts/get-or-create-release-branch.sh output + cat output > $GITHUB_OUTPUT + + upgrade-gradle-properties: + name: Upgrade gradle.properties + if: inputs.dry_run == false + needs: + - get-or-create-release-branch + outputs: + commit_sha: ${{ steps.gradle-push.outputs.commit_sha }} + release_branch: ${{ needs.get-or-create-release-branch.outputs.release_branch }} + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ inputs.release_branch }} + token: ${{ secrets.RELEASE_AUTOMATION_BOT_PAT }} + - name: Upgrade gradle.properties + uses: ./smithy-rs/.github/actions/docker-build + with: + action: upgrade-gradle-smithy-rs-release ${{ inputs.semantic_version }} + - name: Push gradle.properties changes + id: gradle-push + shell: bash + env: + SEMANTIC_VERSION: ${{ inputs.semantic_version }} + DRY_RUN: ${{ inputs.dry_run }} + run: | + if [[ git diff-index --quiet HEAD ]]; then + # The file was actually changed, we need to commit and push the changes + git commit gradle.properties --message "Upgrade the smithy-rs runtime crates version to ${SEMANTIC_VERSION}" + echo "Pushing upgraded gradle.properties commit..." + git push origin + fi + echo "commit_sha=$(git rev-parse --short HEAD)" > $GITHUB_OUTPUT release: name: Release needs: - - acquire-base-image - - release-ci + - upgrade-gradle-properties runs-on: ubuntu-latest steps: - name: Install Rust @@ -81,6 +134,7 @@ jobs: - name: Checkout smithy-rs uses: actions/checkout@v3 with: + ref: ${{ needs.upgrade-gradle-properties.outputs.release_branch }} path: smithy-rs token: ${{ secrets.RELEASE_AUTOMATION_BOT_PAT }} - name: Generate release artifacts @@ -93,13 +147,8 @@ jobs: shell: bash working-directory: smithy-rs-release/smithy-rs run: | - if [[ "${{ inputs.dry_run }}" == "true" ]]; then - echo "Pushing a preview of the release to the smithy-rs-release-preview branch" - git push --force origin HEAD:smithy-rs-release-preview - else - echo "Pushing release commits..." - git push origin - fi + echo "Pushing release commits..." + git push origin - name: Tag release uses: actions/github-script@v6 with: @@ -133,3 +182,31 @@ jobs: else publisher publish -y --location . fi + + trim-changelog-next-on-main: + name: Remove released entries from CHANGELOG.next.toml on the main branch + if: inputs.dry_run == false && needs.get-or-create-release-branch.outputs.new_release_series == true + needs: + - get-or-create-release-branch + - release + runs-on: ubuntu-latest + steps: + - name: Checkout smithy-rs + uses: actions/checkout@v3 + with: + ref: ${{ inputs.commit_sha }} + token: ${{ secrets.RELEASE_AUTOMATION_BOT_PAT }} + - name: Empty CHANGELOG.next.toml + uses: ./smithy-rs/.github/actions/docker-build + with: + action: generate-new-changelog-next-toml + - name: Push smithy-rs changes + shell: bash + run: | + # This will fail if other commits have been pushed to `main` after `commit_sha` + # In particular, this will ALWAYS fail if you are creating a new release series from + # a commit that is not the current tip of `main`. + # We can build more refined automation to handle this case in the future - until then, it'll require + # a manual PR to edit the current CHANGELOG.next.toml file and remove the released entries. + git commit CHANGELOG.next.toml --message "Remove released entries from \`CHANGELOG.next.toml\`" + git push origin main diff --git a/gradle.properties b/gradle.properties index 7ffa3792ec..9857dcc895 100644 --- a/gradle.properties +++ b/gradle.properties @@ -9,7 +9,7 @@ rust.msrv=1.62.1 org.gradle.jvmargs=-Xmx1024M # Version number to use for the generated runtime crates -smithy.rs.runtime.crate.version=0.54.1 +smithy.rs.runtime.crate.version=0.0.0-smithy-rs-head kotlin.code.style=official diff --git a/tools/changelogger/src/init.rs b/tools/changelogger/src/init.rs new file mode 100644 index 0000000000..90377e4029 --- /dev/null +++ b/tools/changelogger/src/init.rs @@ -0,0 +1,16 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use crate::render::EXAMPLE_ENTRY; +use clap::Parser; +use std::io::Write; + +#[derive(Parser, Debug, Eq, PartialEq)] +pub struct InitArgs {} + +pub fn subcommand_init(_args: &InitArgs) -> anyhow::Result<()> { + writeln!(std::io::stdout(), "{}", EXAMPLE_ENTRY)?; + Ok(()) +} diff --git a/tools/changelogger/src/lib.rs b/tools/changelogger/src/lib.rs index 8eb12613f0..483dfd2013 100644 --- a/tools/changelogger/src/lib.rs +++ b/tools/changelogger/src/lib.rs @@ -4,5 +4,6 @@ */ pub mod entry; +pub mod init; pub mod render; pub mod split; diff --git a/tools/changelogger/src/main.rs b/tools/changelogger/src/main.rs index 5e3655ad67..db36023ff3 100644 --- a/tools/changelogger/src/main.rs +++ b/tools/changelogger/src/main.rs @@ -4,36 +4,36 @@ */ use anyhow::Result; +use changelogger::init::subcommand_init; +use changelogger::render::subcommand_render; +use changelogger::split::subcommand_split; use clap::Parser; -use render::subcommand_render; -use split::subcommand_split; - -mod entry; -mod render; -mod split; #[derive(Parser, Debug, Eq, PartialEq)] #[clap(name = "changelogger", author, version, about)] pub enum Args { /// Split SDK changelog entries into a separate file - Split(split::SplitArgs), + Split(changelogger::split::SplitArgs), /// Render a TOML/JSON changelog into GitHub-flavored Markdown - Render(render::RenderArgs), + Render(changelogger::render::RenderArgs), + /// Print to stdout the empty "next" CHANGELOG template. + Init(changelogger::init::InitArgs), } fn main() -> Result<()> { match Args::parse() { Args::Split(split) => subcommand_split(&split), Args::Render(render) => subcommand_render(&render), + Args::Init(init) => subcommand_init(&init), } } #[cfg(test)] mod tests { use super::Args; - use crate::entry::ChangeSet; - use crate::render::RenderArgs; - use crate::split::SplitArgs; + use changelogger::entry::ChangeSet; + use changelogger::render::RenderArgs; + use changelogger::split::SplitArgs; use clap::Parser; use std::path::PathBuf; diff --git a/tools/ci-build/scripts/generate-new-changelog-next-toml b/tools/ci-build/scripts/generate-new-changelog-next-toml new file mode 100755 index 0000000000..e318c01e7f --- /dev/null +++ b/tools/ci-build/scripts/generate-new-changelog-next-toml @@ -0,0 +1,9 @@ +#!/bin/bash +# +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# + +set -eux + +changelogger init > CHANGELOG.next.toml diff --git a/tools/ci-build/scripts/upgrade-gradle-properties b/tools/ci-build/scripts/upgrade-gradle-properties new file mode 100755 index 0000000000..3402ae23e3 --- /dev/null +++ b/tools/ci-build/scripts/upgrade-gradle-properties @@ -0,0 +1,9 @@ +#!/bin/bash +# +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# + +set -eux + +publisher upgrade-runtime-crates-version --version ${1} diff --git a/tools/publisher/src/fs.rs b/tools/publisher/src/fs.rs index c2b529c738..1ab53fa778 100644 --- a/tools/publisher/src/fs.rs +++ b/tools/publisher/src/fs.rs @@ -9,7 +9,7 @@ use tokio::fs::File; use tokio::io::{AsyncReadExt, AsyncWriteExt}; /// Abstraction of the filesystem to allow for more tests to be added in the future. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Copy)] pub enum Fs { Real, } diff --git a/tools/publisher/src/main.rs b/tools/publisher/src/main.rs index 273601a3c6..176f04a9ce 100644 --- a/tools/publisher/src/main.rs +++ b/tools/publisher/src/main.rs @@ -16,6 +16,8 @@ use publisher::subcommand::publish::subcommand_publish; use publisher::subcommand::publish::PublishArgs; use publisher::subcommand::tag_versions_manifest::subcommand_tag_versions_manifest; use publisher::subcommand::tag_versions_manifest::TagVersionsManifestArgs; +use publisher::subcommand::upgrade_runtime_crates_version::subcommand_upgrade_runtime_crates_version; +use publisher::subcommand::upgrade_runtime_crates_version::UpgradeRuntimeCratesVersionArgs; use publisher::subcommand::yank_release::{subcommand_yank_release, YankReleaseArgs}; use tracing_subscriber::fmt::format::FmtSpan; @@ -24,6 +26,11 @@ use tracing_subscriber::fmt::format::FmtSpan; enum Args { /// Fixes path dependencies in manifests to also have version numbers FixManifests(FixManifestsArgs), + /// Upgrade the version of the runtime crates used by the code generator (via `gradle.properties`). + /// + /// The command will fail if you try to perform a downgrade - e.g. change the version from + /// `0.53.1` to `0.52.0` or `0.53.0`. + UpgradeRuntimeCratesVersion(UpgradeRuntimeCratesVersionArgs), /// Publishes crates to crates.io Publish(PublishArgs), /// Publishes an empty library crate to crates.io when a new runtime crate is introduced. @@ -52,6 +59,9 @@ async fn main() -> Result<()> { match Args::parse() { Args::ClaimCrateNames(args) => subcommand_claim_crate_names(&args).await?, + Args::UpgradeRuntimeCratesVersion(args) => { + subcommand_upgrade_runtime_crates_version(&args).await? + } Args::Publish(args) => subcommand_publish(&args).await?, Args::FixManifests(args) => subcommand_fix_manifests(&args).await?, Args::YankRelease(args) => subcommand_yank_release(&args).await?, diff --git a/tools/publisher/src/subcommand/generate_version_manifest.rs b/tools/publisher/src/subcommand/generate_version_manifest.rs index be2d791599..719cb167f7 100644 --- a/tools/publisher/src/subcommand/generate_version_manifest.rs +++ b/tools/publisher/src/subcommand/generate_version_manifest.rs @@ -142,16 +142,6 @@ fn find_released_versions( if let Some(unrecent_version) = unrecent_versions.crates.get(crate_name) { let unrecent_version = parse_version(crate_name, &unrecent_version.version)?; if unrecent_version != recent_version { - // Sanity check: version numbers shouldn't decrease - if unrecent_version > recent_version { - bail!( - "Version number for `{}` decreased between releases (from `{}` to `{}`)", - crate_name, - unrecent_version, - recent_version - ); - } - // If the crate is in both version manifests with differing version // numbers, then it is part of the release released_versions.insert(crate_name.clone(), recent_version.to_string()); @@ -318,35 +308,6 @@ mod tests { assert!(result.is_ok()); } - #[test] - fn test_find_released_versions_decreased_version_number_sanity_check() { - let result = find_released_versions( - &fake_manifest( - &[ - ("aws-config", "0.11.0"), - ("aws-sdk-s3", "0.13.0"), - ("aws-sdk-dynamodb", "0.12.0"), - ], - None, - ), - &fake_manifest( - &[ - ("aws-config", "0.11.0"), - ("aws-sdk-s3", "0.12.0"), // oops, S3 went backwards - ("aws-sdk-dynamodb", "0.12.0"), - ], - None, - ), - ); - assert!(result.is_err()); - let error = format!("{}", result.err().unwrap()); - assert!( - error.starts_with("Version number for `aws-sdk-s3` decreased"), - "Unexpected error: {}", - error - ); - } - #[test] fn test_find_released_versions() { let result = find_released_versions( diff --git a/tools/publisher/src/subcommand/mod.rs b/tools/publisher/src/subcommand/mod.rs index 9462cf58ca..256993e7b0 100644 --- a/tools/publisher/src/subcommand/mod.rs +++ b/tools/publisher/src/subcommand/mod.rs @@ -9,4 +9,5 @@ pub mod generate_version_manifest; pub mod hydrate_readme; pub mod publish; pub mod tag_versions_manifest; +pub mod upgrade_runtime_crates_version; pub mod yank_release; diff --git a/tools/publisher/src/subcommand/upgrade_runtime_crates_version.rs b/tools/publisher/src/subcommand/upgrade_runtime_crates_version.rs new file mode 100644 index 0000000000..0bf17e3c01 --- /dev/null +++ b/tools/publisher/src/subcommand/upgrade_runtime_crates_version.rs @@ -0,0 +1,75 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use crate::fs::Fs; +use anyhow::{anyhow, bail, Context}; +use clap::Parser; +use regex::Regex; +use std::path::{Path, PathBuf}; + +#[derive(Parser, Debug)] +pub struct UpgradeRuntimeCratesVersionArgs { + /// The version of runtime crates you want the code generator to use (e.g. `0.52.0`). + #[clap(long)] + version: String, + /// The path to the `gradle.properties` file. It will default to `gradle.properties` if + /// left unspecified. + #[clap(long, default_value = "gradle.properties")] + gradle_properties_path: PathBuf, +} + +pub async fn subcommand_upgrade_runtime_crates_version( + args: &UpgradeRuntimeCratesVersionArgs, +) -> Result<(), anyhow::Error> { + let upgraded_version = semver::Version::parse(args.version.as_str()) + .with_context(|| format!("{} is not a valid semver version", &args.version))?; + let fs = Fs::Real; + let gradle_properties = read_gradle_properties(fs, &args.gradle_properties_path).await?; + let version_regex = + Regex::new(r"(?Psmithy\.rs\.runtime\.crate\.version=)(?P\d+\.\d+\.\d+-.*)") + .unwrap(); + let current_version = version_regex.captures(&gradle_properties).ok_or_else(|| { + anyhow!( + "Failed to extract the expected runtime crates version from `{:?}`", + &args.gradle_properties_path + ) + })?; + let current_version = current_version.name("version").unwrap(); + let current_version = semver::Version::parse(current_version.as_str()) + .with_context(|| format!("{} is not a valid semver version", current_version.as_str()))?; + if current_version > upgraded_version + // Special version tag used on the `main` branch + && current_version != semver::Version::parse("0.0.0-smithy-rs-head").unwrap() + { + bail!("Moving from {current_version} to {upgraded_version} would be a *downgrade*. This command doesn't allow it!"); + } + let updated_gradle_properties = version_regex.replace( + &gradle_properties, + format!("${{field}}{}", upgraded_version), + ); + update_gradle_properties( + fs, + &args.gradle_properties_path, + updated_gradle_properties.as_ref(), + ) + .await?; + Ok(()) +} + +async fn read_gradle_properties(fs: Fs, path: &Path) -> Result { + let bytes = fs.read_file(path).await?; + let contents = String::from_utf8(bytes) + .with_context(|| format!("`{:?}` contained non-UTF8 data", path))?; + Ok(contents) +} + +async fn update_gradle_properties( + fs: Fs, + path: &Path, + contents: &str, +) -> Result<(), anyhow::Error> { + fs.write_file(path, contents.as_bytes()).await?; + Ok(()) +}