diff --git a/.cargo/config.toml b/.cargo/config.toml index 4304b7c8cdfecf..18a2f71838eb8e 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,3 +1,9 @@ [target.x86_64-unknown-linux-gnu] # When building for linux, target the minimal supported CPU rustflags = ["-Ctarget-cpu=x86-64-v2"] + +# registry configuration for crate publish test +[registries.kellnr] +index = "sparse+http://127.0.0.1:8000/api/v1/crates/" +credential-provider = ["cargo:token"] +token = "Zy9HhJ02RJmg0GCrgLfaCVfU6IwDfhXD" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bf8a9ce0f78e27..f86fa79614d118 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,11 +6,11 @@ on: - "*" jobs: - trigger-buildkite-pipeline: + crate-publish-test: if: github.repository == 'anza-xyz/agave' runs-on: ubuntu-latest steps: - - name: Trigger a Buildkite Build + - name: Trigger crate publish test in buildkite uses: "buildkite/trigger-pipeline-action@v2.3.0" with: buildkite_api_access_token: ${{ secrets.TRIGGER_BK_BUILD_TOKEN }} @@ -18,10 +18,14 @@ jobs: branch: "${{ github.ref_name }}" build_env_vars: '{"TRIGGERED_BUILDKITE_TAG": "${{ github.ref_name }}"}' commit: "HEAD" - message: ":github: Triggered from a GitHub Action" + message: "Crate publish test" + wait: true + wait_interval: '10' + wait_timeout: '7200' draft-release: if: github.repository == 'anza-xyz/agave' + needs: crate-publish-test runs-on: ubuntu-latest steps: - name: Create Release @@ -39,8 +43,27 @@ jobs: prerelease: true }) + trigger-buildkite-pipeline: + if: github.repository == 'anza-xyz/agave' + needs: [ crate-publish-test, draft-release ] + runs-on: ubuntu-latest + steps: + - name: Trigger a Buildkite Build + uses: "buildkite/trigger-pipeline-action@v2.3.0" + with: + buildkite_api_access_token: ${{ secrets.TRIGGER_BK_BUILD_TOKEN }} + pipeline: "anza/agave-secondary" + branch: "${{ github.ref_name }}" + build_env_vars: '{"TRIGGERED_BUILDKITE_TAG": "${{ github.ref_name }}"}' + commit: "HEAD" + message: ":github: Triggered from a GitHub Action" + wait: true + wait_interval: '10' + wait_timeout: '7200' + version-bump: if: github.repository == 'anza-xyz/agave' + needs: [ trigger-buildkite-pipeline ] runs-on: ubuntu-latest steps: - name: Checkout code diff --git a/ci/buildkite-secondary.yml b/ci/buildkite-secondary.yml index af119d7a9977f6..d3fcb48e781c9f 100644 --- a/ci/buildkite-secondary.yml +++ b/ci/buildkite-secondary.yml @@ -7,23 +7,36 @@ steps: agents: queue: "release-build" timeout_in_minutes: 10 + if: build.message != "Crate publish test" + - name: "publish dry-run" + command: "ci/publish-crate.sh --dry-run" + timeout_in_minutes: 180 + artifact_paths: "log-*.txt" + env: + CRATE_PUBLISH_TEST: "true" + agents: + queue: "release-build" + if: build.message == "Crate publish test" - wait - name: "publish tarball (x86_64-unknown-linux-gnu)" command: "ci/publish-tarball.sh" agents: queue: "release-build" timeout_in_minutes: 60 + if: build.message != "Crate publish test" - name: "publish installer" command: "ci/publish-installer.sh" agents: queue: "release-build" timeout_in_minutes: 5 + if: build.message != "Crate publish test" - wait - name: "publish docker" command: "docker-solana/build.sh" agents: queue: "release-build" timeout_in_minutes: 60 + if: build.message != "Crate publish test" - name: "publish crate" command: "ci/publish-crate.sh" agents: @@ -33,6 +46,7 @@ steps: permit_on_passed: true timeout_in_minutes: 240 branches: "!master" + if: build.message != "Crate publish test" - name: "trigger github actions windows build" command: ".buildkite/scripts/trigger-github-actions-windows-build.sh" agents: @@ -41,6 +55,7 @@ steps: manual: permit_on_passed: true timeout_in_minutes: 5 + if: build.message != "Crate publish test" - name: "publish tarball (aarch64-apple-darwin)" command: "ci/publish-tarball.sh" agents: @@ -49,6 +64,7 @@ steps: manual: permit_on_passed: true timeout_in_minutes: 60 + if: build.message != "Crate publish test" - name: "publish tarball (x86_64-apple-darwin)" command: "ci/publish-tarball.sh" agents: @@ -57,3 +73,4 @@ steps: manual: permit_on_passed: true timeout_in_minutes: 60 + if: build.message != "Crate publish test" diff --git a/ci/change-crate-deps.py b/ci/change-crate-deps.py new file mode 100755 index 00000000000000..780c59d6d68de5 --- /dev/null +++ b/ci/change-crate-deps.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 + +# This script: +# - searches given manifest file for solana / agave dependencies +# - modifies each found line in the file by adding a custom registry parameter +# - writes the output back to the given manifest file +# + +import json +import os +import subprocess +import sys + +if len(sys.argv) != 3: + print('Usage: %s ' % sys.argv[0]) + +real_file = os.path.realpath(__file__) +ci_path = os.path.dirname(real_file) +src_root = os.path.dirname(ci_path) +cargo_toml = os.path.join(src_root, sys.argv[1]) +pkg_name = sys.argv[2] +version = os.environ.get('CI_TAG') + + +def load_metadata(): + cmd = f'{src_root}/cargo metadata --no-deps --format-version=1' + return json.loads(subprocess.Popen( + [f"{src_root}/cargo", "metadata", "--no-deps", "--format-version=1"], stdout=subprocess.PIPE).communicate()[0]) + + +def get_pkg_deps(package_name): + metadata = load_metadata() + dependency_graph = dict() + + for pkg in metadata['packages']: + dependency_graph[pkg['name']] = [ + x['name'] for x in pkg['dependencies'] if x['name'].startswith(('solana', 'agave')) and x['source'] is None + ] + return dependency_graph[package_name] + + +with open(cargo_toml, 'rb', 0) as file: + data = file.readlines() + deps = get_pkg_deps(pkg_name) + + for idx, line in enumerate(data): + split_line = line.decode().split(" =", 1) + if split_line[0] in deps and not split_line[1].startswith(' { registry'): + tmp = split_line[1].replace("workspace = true", "version = \"" + version + "\"") + tmp = tmp.replace("{", '').lstrip() + data[idx] = bytes(split_line[0] + ' = { registry = "kellnr", ' + tmp, 'utf-8') + +with open(cargo_toml, 'wb') as file: + file.writelines(data) diff --git a/ci/docker-run.sh b/ci/docker-run.sh index 503f8f51790505..a041d7007b08e5 100755 --- a/ci/docker-run.sh +++ b/ci/docker-run.sh @@ -99,6 +99,12 @@ if [[ -z "$SOLANA_DOCKER_RUN_NOSETUID" ]]; then ARGS+=(--user "$(id -u):$(id -g)") fi +if [[ -n "$CRATE_PUBLISH_TEST" ]]; then + ARGS+=( + --network container:kellnr + ) +fi + if [[ -n $SOLANA_ALLOCATE_TTY ]]; then # Colored output, progress bar and Ctrl-C: # https://stackoverflow.com/a/41099052/10242004 diff --git a/ci/env.sh b/ci/env.sh index 953e880e399b28..1fc62a1bd786de 100644 --- a/ci/env.sh +++ b/ci/env.sh @@ -50,6 +50,9 @@ if [[ -n $CI ]]; then else export CI_TAG=$BUILDKITE_TAG fi + if [[ -n $CRATE_PUBLISH_TEST ]]; then + export CRATE_PUBLISH_TEST=$CRATE_PUBLISH_TEST + fi elif [[ -n $APPVEYOR ]]; then export CI_BRANCH=$APPVEYOR_REPO_BRANCH export CI_BUILD_ID=$APPVEYOR_BUILD_ID diff --git a/ci/order-crates-for-publishing.py b/ci/order-crates-for-publishing.py index 02a9fa016bae24..1480a4ed0e516d 100755 --- a/ci/order-crates-for-publishing.py +++ b/ci/order-crates-for-publishing.py @@ -10,7 +10,7 @@ import os import json import subprocess -import sys; +import sys real_file = os.path.realpath(__file__) ci_path = os.path.dirname(real_file) @@ -102,10 +102,10 @@ def get_packages(): wrong_self_dev_dependencies = list() for pkg in metadata['packages']: - manifest_path[pkg['name']] = pkg['manifest_path']; + manifest_path[pkg['name']] = pkg['manifest_path'] dependency_graph[pkg['name']] = [ x['name'] for x in pkg['dependencies'] if should_add(pkg, x, wrong_self_dev_dependencies) - ]; + ] # Check for direct circular dependencies circular_dependencies = set() diff --git a/ci/publish-crate.sh b/ci/publish-crate.sh index 5d7f3b1e1e1c50..adb49995110802 100755 --- a/ci/publish-crate.sh +++ b/ci/publish-crate.sh @@ -1,21 +1,57 @@ #!/usr/bin/env bash -set -e + +# shellcheck disable=SC2317 +cleanup() { + ec=$? + docker container stop kellnr || true + docker container prune -f + exit "$ec" +} + +err_handler() { + ec=$? + echo "ERROR line $1: $BASH_COMMAND" + exit "$ec" +} + +trap cleanup EXIT +trap err_handler ERR SIGINT + +set -Eeuo pipefail + cd "$(dirname "$0")/.." source ci/semver_bash/semver.sh source ci/rust-version.sh stable +DRY_RUN=false +if [[ $1 = --dry-run ]]; then + DRY_RUN=true + export CRATE_PUBLISH_TEST=true + shift +fi + # shellcheck disable=SC2086 is_crate_version_uploaded() { name=$1 version=$2 - curl https://crates.io/api/v1/crates/${name}/${version} | \ - python3 -c "import sys,json; print('version' in json.load(sys.stdin));" + if $DRY_RUN; then + curl http://127.0.0.1:8000/api/v1/crates/${name}/crate_versions | \ + python3 -c "import sys,json; print(len(json.load(sys.stdin)['versions']) > 0);" + else + curl https://crates.io/api/v1/crates/${name}/${version} | \ + python3 -c "import sys,json; print('version' in json.load(sys.stdin));" + fi } # Only package/publish if this is a tagged release [[ -n $CI_TAG ]] || { +if $DRY_RUN; then + CI_TAG=$(grep '^version = "' Cargo.toml | cut -d "=" -f2 | xargs) + CRATES_IO_TOKEN="test" +else echo CI_TAG unset, skipped exit 0 +fi } semverParseInto "$CI_TAG" MAJOR MINOR PATCH SPECIAL @@ -37,10 +73,16 @@ done Cargo_tomls=$(ci/order-crates-for-publishing.py) +if $DRY_RUN; then + docker run -p 8000:8000 --name kellnr -d ghcr.io/kellnr/kellnr:5 +fi + +sleep 5 + for Cargo_toml in $Cargo_tomls; do echo "--- $Cargo_toml" - # check the version which doesn't inherit from worksapce + # check the version which doesn't inherit from workspace if ! grep -q "^version = { workspace = true }$" "$Cargo_toml"; then echo "Warn: $Cargo_toml doesn't use the inherited version" grep -q "^version = \"$expectedCrateVersion\"$" "$Cargo_toml" || { @@ -62,10 +104,16 @@ for Cargo_toml in $Cargo_tomls; do fi ( - set -x - crate=$(dirname "$Cargo_toml") - cargoCommand="cargo publish --token $CRATES_IO_TOKEN" + if $DRY_RUN; then + ci/change-crate-deps.py "$Cargo_toml" "$crate_name" + + # token is a default value from the kellnr image https://kellnr.io/documentation#config-values + # registry value is defined in docker-run.sh script + cargoCommand="cargo publish --registry kellnr --token Zy9HhJ02RJmg0GCrgLfaCVfU6IwDfhXD --allow-dirty" + else + cargoCommand="cargo publish --token $CRATES_IO_TOKEN" + fi numRetries=10 for ((i = 1; i <= numRetries; i++)); do