Ci/crate publish test

.cargo/config.toml

@@ -1,3 +1,9 @@
 [target.x86_64-unknown-linux-gnu]
 # When building for linux, target the minimal supported CPU
 rustflags = ["-Ctarget-cpu=x86-64-v2"]
+
+# registry configuration for crate publish test
+[registries.kellnr]
+index = "sparse+http://127.0.0.1:8000/api/v1/crates/"
+credential-provider = ["cargo:token"]
+token = "Zy9HhJ02RJmg0GCrgLfaCVfU6IwDfhXD"

.github/workflows/release.yml

@@ -6,22 +6,26 @@ on:
       - "*"
 
 jobs:
-  trigger-buildkite-pipeline:
+  crate-publish-test:
     if: github.repository == 'anza-xyz/agave'
     runs-on: ubuntu-latest
     steps:
-      - name: Trigger a Buildkite Build
+      - name: Trigger crate publish test in buildkite
         uses: "buildkite/trigger-pipeline-action@v2.3.0"
         with:
           buildkite_api_access_token: ${{ secrets.TRIGGER_BK_BUILD_TOKEN }}
           pipeline: "anza/agave-secondary"
           branch: "${{ github.ref_name }}"
           build_env_vars: '{"TRIGGERED_BUILDKITE_TAG": "${{ github.ref_name }}"}'
           commit: "HEAD"
-          message: ":github: Triggered from a GitHub Action"
+          message: "Crate publish test"
+          wait: true
+          wait_interval: '10'
+          wait_timeout: '7200'
 
   draft-release:
     if: github.repository == 'anza-xyz/agave'
+    needs: crate-publish-test
     runs-on: ubuntu-latest
     steps:
       - name: Create Release
@@ -39,8 +43,27 @@ jobs:
               prerelease: true
             })
 
+  trigger-buildkite-pipeline:
+    if: github.repository == 'anza-xyz/agave'
+    needs: [ crate-publish-test, draft-release ]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Trigger a Buildkite Build
+        uses: "buildkite/trigger-pipeline-action@v2.3.0"
+        with:
+          buildkite_api_access_token: ${{ secrets.TRIGGER_BK_BUILD_TOKEN }}
+          pipeline: "anza/agave-secondary"
+          branch: "${{ github.ref_name }}"
+          build_env_vars: '{"TRIGGERED_BUILDKITE_TAG": "${{ github.ref_name }}"}'
+          commit: "HEAD"
+          message: ":github: Triggered from a GitHub Action"
+          wait: true
+          wait_interval: '10'
+          wait_timeout: '7200'
+
   version-bump:
     if: github.repository == 'anza-xyz/agave'
+    needs: [ trigger-buildkite-pipeline ]
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code

ci/buildkite-secondary.yml

@@ -7,23 +7,36 @@ steps:
     agents:
       queue: "release-build"
     timeout_in_minutes: 10
+    if: build.message != "Crate publish test"
+  - name: "publish dry-run"
+    command: "ci/publish-crate.sh --dry-run"
+    timeout_in_minutes: 180
+    artifact_paths: "log-*.txt"
+    env:
+      CRATE_PUBLISH_TEST: "true"
+    agents:
+      queue: "release-build"
+    if: build.message == "Crate publish test"
   - wait
   - name: "publish tarball (x86_64-unknown-linux-gnu)"
     command: "ci/publish-tarball.sh"
     agents:
       queue: "release-build"
     timeout_in_minutes: 60
+    if: build.message != "Crate publish test"
   - name: "publish installer"
     command: "ci/publish-installer.sh"
     agents:
       queue: "release-build"
     timeout_in_minutes: 5
+    if: build.message != "Crate publish test"
   - wait
   - name: "publish docker"
     command: "docker-solana/build.sh"
     agents:
       queue: "release-build"
     timeout_in_minutes: 60
+    if: build.message != "Crate publish test"
   - name: "publish crate"
     command: "ci/publish-crate.sh"
     agents:
@@ -33,6 +46,7 @@ steps:
         permit_on_passed: true
     timeout_in_minutes: 240
     branches: "!master"
+    if: build.message != "Crate publish test"
   - name: "trigger github actions windows build"
     command: ".buildkite/scripts/trigger-github-actions-windows-build.sh"
     agents:
@@ -41,6 +55,7 @@ steps:
       manual:
         permit_on_passed: true
     timeout_in_minutes: 5
+    if: build.message != "Crate publish test"
   - name: "publish tarball (aarch64-apple-darwin)"
     command: "ci/publish-tarball.sh"
     agents:
@@ -49,6 +64,7 @@ steps:
       manual:
         permit_on_passed: true
     timeout_in_minutes: 60
+    if: build.message != "Crate publish test"
   - name: "publish tarball (x86_64-apple-darwin)"
     command: "ci/publish-tarball.sh"
     agents:
@@ -57,3 +73,4 @@ steps:
       manual:
         permit_on_passed: true
     timeout_in_minutes: 60
+    if: build.message != "Crate publish test"

ci/change-crate-deps.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+
+# This script:
+# - searches given manifest file for solana / agave dependencies
+# - modifies each found line in the file by adding a custom registry parameter
+# - writes the output back to the given manifest file
+#
+
+import json
+import os
+import subprocess
+import sys
+
+if len(sys.argv) != 3:
+    print('Usage: %s <manifest path> <crate-name>' % sys.argv[0])
+
+real_file = os.path.realpath(__file__)
+ci_path = os.path.dirname(real_file)
+src_root = os.path.dirname(ci_path)
+cargo_toml = os.path.join(src_root, sys.argv[1])
+pkg_name = sys.argv[2]
+version = os.environ.get('CI_TAG')
+
+
+def load_metadata():
+    cmd = f'{src_root}/cargo metadata --no-deps --format-version=1'
+    return json.loads(subprocess.Popen(
+        [f"{src_root}/cargo", "metadata", "--no-deps", "--format-version=1"], stdout=subprocess.PIPE).communicate()[0])
+
+
+def get_pkg_deps(package_name):
+    metadata = load_metadata()
+    dependency_graph = dict()
+
+    for pkg in metadata['packages']:
+        dependency_graph[pkg['name']] = [
+            x['name'] for x in pkg['dependencies'] if x['name'].startswith(('solana', 'agave')) and x['source'] is None
+        ]
+    return dependency_graph[package_name]
+
+
+with open(cargo_toml, 'rb', 0) as file:
+    data = file.readlines()
+    deps = get_pkg_deps(pkg_name)
+
+    for idx, line in enumerate(data):
+        split_line = line.decode().split(" =", 1)
+        if split_line[0] in deps and not split_line[1].startswith(' { registry'):
+            tmp = split_line[1].replace("workspace = true", "version = \"" + version + "\"")
+            tmp = tmp.replace("{", '').lstrip()
+            data[idx] = bytes(split_line[0] + ' = { registry = "kellnr", ' + tmp, 'utf-8')
+
+with open(cargo_toml, 'wb') as file:
+    file.writelines(data)

ci/docker-run.sh

@@ -99,6 +99,12 @@ if [[ -z "$SOLANA_DOCKER_RUN_NOSETUID" ]]; then
   ARGS+=(--user "$(id -u):$(id -g)")
 fi
 
+if [[ -n "$CRATE_PUBLISH_TEST" ]]; then
+  ARGS+=(
+    --network container:kellnr
+  )
+fi
+
 if [[ -n $SOLANA_ALLOCATE_TTY ]]; then
   # Colored output, progress bar and Ctrl-C:
   # https://stackoverflow.com/a/41099052/10242004

ci/env.sh

@@ -50,6 +50,9 @@ if [[ -n $CI ]]; then
     else
       export CI_TAG=$BUILDKITE_TAG
     fi
+    if [[ -n $CRATE_PUBLISH_TEST ]]; then
+      export CRATE_PUBLISH_TEST=$CRATE_PUBLISH_TEST
+    fi
   elif [[ -n $APPVEYOR ]]; then
     export CI_BRANCH=$APPVEYOR_REPO_BRANCH
     export CI_BUILD_ID=$APPVEYOR_BUILD_ID

ci/order-crates-for-publishing.py

@@ -10,7 +10,7 @@
 import os
 import json
 import subprocess
-import sys;
+import sys
 
 real_file = os.path.realpath(__file__)
 ci_path = os.path.dirname(real_file)
@@ -102,10 +102,10 @@ def get_packages():
     wrong_self_dev_dependencies = list()
 
     for pkg in metadata['packages']:
-        manifest_path[pkg['name']] = pkg['manifest_path'];
+        manifest_path[pkg['name']] = pkg['manifest_path']
         dependency_graph[pkg['name']] = [
             x['name'] for x in pkg['dependencies'] if should_add(pkg, x, wrong_self_dev_dependencies)
-        ];
+        ]
 
     # Check for direct circular dependencies
     circular_dependencies = set()

ci/publish-crate.sh

@@ -1,21 +1,57 @@
 #!/usr/bin/env bash
-set -e
+
+# shellcheck disable=SC2317
+cleanup() {
+  ec=$?
+  docker container stop kellnr || true
+  docker container prune -f
+  exit "$ec"
+}
+
+err_handler() {
+    ec=$?
+    echo "ERROR  line $1: $BASH_COMMAND"
+    exit "$ec"
+}
+
+trap cleanup EXIT
+trap err_handler ERR SIGINT
+
+set -Eeuo pipefail
+
 cd "$(dirname "$0")/.."
 source ci/semver_bash/semver.sh
 source ci/rust-version.sh stable
 
+DRY_RUN=false
+if [[ $1 = --dry-run ]]; then
+  DRY_RUN=true
+  export CRATE_PUBLISH_TEST=true
+  shift
+fi
+
 # shellcheck disable=SC2086
 is_crate_version_uploaded() {
   name=$1
   version=$2
-  curl https://crates.io/api/v1/crates/${name}/${version} | \
-  python3 -c "import sys,json; print('version' in json.load(sys.stdin));"
+  if $DRY_RUN; then
+    curl http://127.0.0.1:8000/api/v1/crates/${name}/crate_versions | \
+    python3 -c "import sys,json; print(len(json.load(sys.stdin)['versions']) > 0);"
+  else
+    curl https://crates.io/api/v1/crates/${name}/${version} | \
+    python3 -c "import sys,json; print('version' in json.load(sys.stdin));"
+  fi
 }
 
 # Only package/publish if this is a tagged release
 [[ -n $CI_TAG ]] || {
+if $DRY_RUN; then
+  CI_TAG=$(grep '^version = "' Cargo.toml | cut -d "=" -f2 | xargs)
+  CRATES_IO_TOKEN="test"
+else
   echo CI_TAG unset, skipped
   exit 0
+fi
 }
 
 semverParseInto "$CI_TAG" MAJOR MINOR PATCH SPECIAL
@@ -37,10 +73,16 @@ done
 
 Cargo_tomls=$(ci/order-crates-for-publishing.py)
 
+if $DRY_RUN; then
+  docker run -p 8000:8000 --name kellnr -d ghcr.io/kellnr/kellnr:5
+fi
+
+sleep 5
+
 for Cargo_toml in $Cargo_tomls; do
   echo "--- $Cargo_toml"
 
-  # check the version which doesn't inherit from worksapce
+  # check the version which doesn't inherit from workspace
   if ! grep -q "^version = { workspace = true }$" "$Cargo_toml"; then
     echo "Warn: $Cargo_toml doesn't use the inherited version"
     grep -q "^version = \"$expectedCrateVersion\"$" "$Cargo_toml" || {
@@ -62,10 +104,16 @@ for Cargo_toml in $Cargo_tomls; do
   fi
 
   (
-    set -x
-
     crate=$(dirname "$Cargo_toml")
-    cargoCommand="cargo publish --token $CRATES_IO_TOKEN"
+    if $DRY_RUN; then
+      ci/change-crate-deps.py "$Cargo_toml" "$crate_name"
+
+      # token is a default value from the kellnr image https://kellnr.io/documentation#config-values
+      # registry value is defined in docker-run.sh script
+      cargoCommand="cargo publish --registry kellnr --token Zy9HhJ02RJmg0GCrgLfaCVfU6IwDfhXD --allow-dirty"
+    else
+      cargoCommand="cargo publish --token $CRATES_IO_TOKEN"
+    fi
 
     numRetries=10
     for ((i = 1; i <= numRetries; i++)); do