From 935551842450258df90ccd8c747cfb4f37722221 Mon Sep 17 00:00:00 2001 From: Yihau Chen Date: Mon, 12 Feb 2024 15:17:51 +0800 Subject: [PATCH 01/84] [anza migration] replace binaries backend (#6) * ci: add upload-gcs-artifact * ci: publish release binaries to GCS * ci: redirect github repo to anza-xyz * ci: publish windows binaries to GCS * replace release.solana.com with release.anza.xyz * use a explicit name for credential --- .github/workflows/release-artifacts-auto.yml | 6 ++-- .../workflows/release-artifacts-manually.yml | 4 +-- .github/workflows/release-artifacts.yml | 33 +++++++++++-------- ci/publish-installer.sh | 12 +++---- ci/publish-tarball.sh | 6 ++-- ci/upload-ci-artifact.sh | 10 ++++++ ci/upload-github-release-asset.sh | 2 +- install/solana-install-init.sh | 4 +-- install/src/command.rs | 8 ++--- 9 files changed, 48 insertions(+), 37 deletions(-) diff --git a/.github/workflows/release-artifacts-auto.yml b/.github/workflows/release-artifacts-auto.yml index a8309cdffc8a72..0cdd176e04396c 100644 --- a/.github/workflows/release-artifacts-auto.yml +++ b/.github/workflows/release-artifacts-auto.yml @@ -14,14 +14,12 @@ concurrency: jobs: release-artifacts: - if: github.repository == 'solana-labs/solana' + if: github.repository == 'anza-xyz/agave' uses: ./.github/workflows/release-artifacts.yml with: commit: ${{ github.sha }} secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} + GCS_RELEASE_BUCKET_WRITER_CREDIENTIAL: ${{ secrets.GCS_RELEASE_BUCKET_WRITER_CREDIENTIAL }} error_reporting: needs: diff --git a/.github/workflows/release-artifacts-manually.yml b/.github/workflows/release-artifacts-manually.yml index 35de72922c32c8..fe5c1b03b638b3 100644 --- a/.github/workflows/release-artifacts-manually.yml +++ b/.github/workflows/release-artifacts-manually.yml @@ -14,6 +14,4 @@ jobs: with: commit: ${{ github.event.inputs.commit }} secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} + GCS_RELEASE_BUCKET_WRITER_CREDIENTIAL: ${{ secrets.GCS_RELEASE_BUCKET_WRITER_CREDIENTIAL }} diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 98dc697920262c..14760837ba0dbd 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -7,11 +7,7 @@ on: required: false type: string secrets: - AWS_ACCESS_KEY_ID: - required: true - AWS_SECRET_ACCESS_KEY: - required: true - AWS_S3_BUCKET: + GCS_RELEASE_BUCKET_WRITER_CREDIENTIAL: required: true jobs: @@ -71,17 +67,17 @@ jobs: shell: bash run: | FOLDER_NAME=${{ steps.build.outputs.tag || steps.build.outputs.channel }} - mkdir -p "github-action-s3-upload/$FOLDER_NAME" - cp -v "solana-release-x86_64-pc-windows-msvc.tar.bz2" "github-action-s3-upload/$FOLDER_NAME/" - cp -v "solana-release-x86_64-pc-windows-msvc.yml" "github-action-s3-upload/$FOLDER_NAME/" - cp -v "solana-install-init-x86_64-pc-windows-msvc"* "github-action-s3-upload/$FOLDER_NAME" + mkdir -p "windows-release/$FOLDER_NAME" + cp -v "solana-release-x86_64-pc-windows-msvc.tar.bz2" "windows-release/$FOLDER_NAME/" + cp -v "solana-release-x86_64-pc-windows-msvc.yml" "windows-release/$FOLDER_NAME/" + cp -v "solana-install-init-x86_64-pc-windows-msvc"* "windows-release/$FOLDER_NAME" - name: Upload Artifacts if: ${{ steps.build.outputs.channel != '' || steps.build.outputs.tag != '' }} uses: actions/upload-artifact@v3 with: name: windows-artifact - path: github-action-s3-upload/ + path: windows-release/ windows-s3-upload: if: ${{ needs.windows-build.outputs.channel != '' || needs.windows-build.outputs.tag != '' }} @@ -92,7 +88,16 @@ jobs: uses: actions/download-artifact@v3 with: name: windows-artifact - path: ./github-action-s3-upload + path: .windows-release/ + + - name: Setup crediential + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_RELEASE_BUCKET_WRITER_CREDIENTIAL }}" + + - name: Upload files to GCS + run: | + gcloud storage cp --recursive windows-release/* gs://anza-release/ - name: Upload uses: jakejarvis/s3-sync-action@master @@ -103,7 +108,7 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} AWS_REGION: "us-west-1" - SOURCE_DIR: "github-action-s3-upload" + SOURCE_DIR: "windows-release" windows-gh-release: if: ${{ needs.windows-build.outputs.tag != '' }} @@ -114,7 +119,7 @@ jobs: uses: actions/download-artifact@v3 with: name: windows-artifact - path: ./github-action-s3-upload + path: .windows-release/ - name: Release uses: softprops/action-gh-release@v1 @@ -122,4 +127,4 @@ jobs: tag_name: ${{ needs.windows-build.outputs.tag }} draft: true files: | - github-action-s3-upload/${{ needs.windows-build.outputs.tag }}/* + windows-release/${{ needs.windows-build.outputs.tag }}/* diff --git a/ci/publish-installer.sh b/ci/publish-installer.sh index 4b5345ae0d26fe..e58fd939dd1a40 100755 --- a/ci/publish-installer.sh +++ b/ci/publish-installer.sh @@ -26,14 +26,14 @@ fi # upload install script source ci/upload-ci-artifact.sh -cat >release.solana.com-install <release.anza.xyz-install <>release.solana.com-install +cat install/solana-install-init.sh >>release.anza.xyz-install -echo --- AWS S3 Store: "install" -upload-s3-artifact "/solana/release.solana.com-install" "s3://release.solana.com/$CHANNEL_OR_TAG/install" +echo --- GCS: "install" +upload-gcs-artifact "/solana/release.anza.xyz-install" "gs://anza-release/$CHANNEL_OR_TAG/install" echo Published to: -ci/format-url.sh https://release.solana.com/"$CHANNEL_OR_TAG"/install +ci/format-url.sh https://release.anza.xyz/"$CHANNEL_OR_TAG"/install diff --git a/ci/publish-tarball.sh b/ci/publish-tarball.sh index ff72bb7da2d066..5c64f09564fe9f 100755 --- a/ci/publish-tarball.sh +++ b/ci/publish-tarball.sh @@ -118,11 +118,11 @@ for file in "${TARBALL_BASENAME}"-$TARGET.tar.bz2 "${TARBALL_BASENAME}"-$TARGET. fi if [[ -n $BUILDKITE ]]; then - echo --- AWS S3 Store: "$file" - upload-s3-artifact "/solana/$file" s3://release.solana.com/"$CHANNEL_OR_TAG"/"$file" + echo --- GCS Store: "$file" + upload-gcs-artifact "/solana/$file" gs://anza-release/"$CHANNEL_OR_TAG"/"$file" echo Published to: - $DRYRUN ci/format-url.sh https://release.solana.com/"$CHANNEL_OR_TAG"/"$file" + $DRYRUN ci/format-url.sh https://release.anza.xyz/"$CHANNEL_OR_TAG"/"$file" if [[ -n $TAG ]]; then ci/upload-github-release-asset.sh "$file" diff --git a/ci/upload-ci-artifact.sh b/ci/upload-ci-artifact.sh index 1236da9f27114a..e7cc34ab2b2d8c 100644 --- a/ci/upload-ci-artifact.sh +++ b/ci/upload-ci-artifact.sh @@ -40,3 +40,13 @@ upload-s3-artifact() { docker run "${args[@]}" ) } + +upload-gcs-artifact() { + echo "--- artifact: $1 to $2" + docker run --rm \ + -v "$GCS_RELEASE_BUCKET_WRITER_CREDIENTIAL:/application_default_credentials.json" \ + -v "$PWD:/solana" \ + -e CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=/application_default_credentials.json \ + gcr.io/google.com/cloudsdktool/google-cloud-cli:latest \ + gcloud storage cp "$1" "$2" +} diff --git a/ci/upload-github-release-asset.sh b/ci/upload-github-release-asset.sh index ca2ae2a8f60443..229fb8993edafd 100755 --- a/ci/upload-github-release-asset.sh +++ b/ci/upload-github-release-asset.sh @@ -26,7 +26,7 @@ fi # Force CI_REPO_SLUG since sometimes # BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG is not set correctly, causing the # artifact upload to fail -CI_REPO_SLUG=solana-labs/solana +CI_REPO_SLUG=anza-xyz/agave #if [[ -z $CI_REPO_SLUG ]]; then # echo Error: CI_REPO_SLUG not defined # exit 1 diff --git a/install/solana-install-init.sh b/install/solana-install-init.sh index db36dc61e2ff30..4f28e300be52ab 100755 --- a/install/solana-install-init.sh +++ b/install/solana-install-init.sh @@ -16,9 +16,9 @@ { # this ensures the entire script is downloaded # if [ -z "$SOLANA_DOWNLOAD_ROOT" ]; then - SOLANA_DOWNLOAD_ROOT="https://github.com/solana-labs/solana/releases/download/" + SOLANA_DOWNLOAD_ROOT="https://github.com/anza-xyz/agave/releases/download/" fi -GH_LATEST_RELEASE="https://api.github.com/repos/solana-labs/solana/releases/latest" +GH_LATEST_RELEASE="https://api.github.com/repos/anza-xyz/agave/releases/latest" set -e diff --git a/install/src/command.rs b/install/src/command.rs index d7b92c17690bda..218e815467e9a9 100644 --- a/install/src/command.rs +++ b/install/src/command.rs @@ -572,7 +572,7 @@ pub fn init( fn github_release_download_url(release_semver: &str) -> String { format!( - "https://github.com/solana-labs/solana/releases/download/v{}/solana-release-{}.tar.bz2", + "https://github.com/anza-xyz/agave/releases/download/v{}/solana-release-{}.tar.bz2", release_semver, crate::build_env::TARGET ) @@ -580,7 +580,7 @@ fn github_release_download_url(release_semver: &str) -> String { fn release_channel_download_url(release_channel: &str) -> String { format!( - "https://release.solana.com/{}/solana-release-{}.tar.bz2", + "https://release.anza.xyz/{}/solana-release-{}.tar.bz2", release_channel, crate::build_env::TARGET ) @@ -588,7 +588,7 @@ fn release_channel_download_url(release_channel: &str) -> String { fn release_channel_version_url(release_channel: &str) -> String { format!( - "https://release.solana.com/{}/solana-release-{}.yml", + "https://release.anza.xyz/{}/solana-release-{}.yml", release_channel, crate::build_env::TARGET ) @@ -905,7 +905,7 @@ fn check_for_newer_github_release( while page == 1 || releases.len() == PER_PAGE { let url = reqwest::Url::parse_with_params( - "https://api.github.com/repos/solana-labs/solana/releases", + "https://api.github.com/repos/anza-xyz/agave/releases", &[ ("per_page", &format!("{PER_PAGE}")), ("page", &format!("{page}")), From b0022d73eadacf9097afffb80aeda6c10596b310 Mon Sep 17 00:00:00 2001 From: Yihau Chen Date: Mon, 12 Feb 2024 15:18:11 +0800 Subject: [PATCH 02/84] [anza migration] ci (#5) * Update README.md * ci: update CodeCov report link * ci: update github pr link * ci: rename secondary pipeline * replace org name in .mergify * update channel info link * update dependabot pr link * use anza docker image * delete travis --------- Co-authored-by: Will Hickey --- .mergify.yml | 4 +- .travis.yml | 94 -------------------------------- README.md | 34 +----------- ci/buildkite-pipeline-in-disk.sh | 4 +- ci/buildkite-pipeline.sh | 8 +-- ci/buildkite-solana-private.sh | 2 +- ci/channel-info.sh | 2 +- ci/dependabot-pr.sh | 2 +- ci/rust-version.sh | 2 +- ci/test-coverage.sh | 2 +- 10 files changed, 14 insertions(+), 140 deletions(-) delete mode 100644 .travis.yml diff --git a/.mergify.yml b/.mergify.yml index ef576943d5d635..166f59a5f365d1 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -33,7 +33,7 @@ pull_request_rules: actions: request_reviews: teams: - - "@solana-labs/community-pr-subscribers" + - "@anza-xyz/community-pr-subscribers" - name: label changes from monorepo-triage conditions: - author≠@core-contributors @@ -102,7 +102,7 @@ pull_request_rules: actions: backport: assignees: &BackportAssignee - - "{{ merged_by|replace('mergify[bot]', label|select('equalto', 'community')|first|default(author)|replace('community', '@solana-labs/community-pr-subscribers')) }}" + - "{{ merged_by|replace('mergify[bot]', label|select('equalto', 'community')|first|default(author)|replace('community', '@anza-xyz/community-pr-subscribers')) }}" title: "{{ destination_branch }}: {{ title }} (backport of #{{ number }})" ignore_conflicts: true labels: diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index c2dd13e32551ba..00000000000000 --- a/.travis.yml +++ /dev/null @@ -1,94 +0,0 @@ -branches: - only: - - master - - /^v\d+\.\d+/ - -notifications: - email: false - slack: - on_success: change - if: NOT type = pull_request - secure: F4IjOE05MyaMOdPRL+r8qhs7jBvv4yDM3RmFKE1zNXnfUOqV4X38oQM1EI+YVsgpMQLj/pxnEB7wcTE4Bf86N6moLssEULCpvAuMVoXj4QbWdomLX+01WbFa6fLVeNQIg45NHrz2XzVBhoKOrMNnl+QI5mbR2AlS5oqsudHsXDnyLzZtd4Y5SDMdYG1zVWM01+oNNjgNfjcCGmOE/K0CnOMl6GPi3X9C34tJ19P2XT7MTDsz1/IfEF7fro2Q8DHEYL9dchJMoisXSkem5z7IDQkGzXsWdWT4NnndUvmd1MlTCE9qgoXDqRf95Qh8sB1Dz08HtvgfaosP2XjtNTfDI9BBYS15Ibw9y7PchAJE1luteNjF35EOy6OgmCLw/YpnweqfuNViBZz+yOPWXVC0kxnPIXKZ1wyH9ibeH6E4hr7a8o9SV/6SiWIlbYF+IR9jPXyTCLP/cc3sYljPWxDnhWFwFdRVIi3PbVAhVu7uWtVUO17Oc9gtGPgs/GrhOMkJfwQPXaudRJDpVZowxTX4x9kefNotlMAMRgq+Drbmgt4eEBiCNp0ITWgh17BiE1U09WS3myuduhoct85+FoVeaUkp1sxzHVtGsNQH0hcz7WcpZyOM+AwistJA/qzeEDQao5zi1eKWPbO2xAhi2rV1bDH6bPf/4lDBwLRqSiwvlWU= - -os: linux -dist: bionic -language: minimal - -jobs: - include: - - &release-artifacts - if: type IN (api, cron) OR tag IS present - name: "macOS release artifacts" - os: osx - osx_image: xcode12 - language: rust - rust: - - stable - install: - - source ci/rust-version.sh - - PATH="/usr/local/opt/coreutils/libexec/gnubin:$PATH" - - readlink -f . - - brew install gnu-tar - - PATH="/usr/local/opt/gnu-tar/libexec/gnubin:$PATH" - - tar --version - script: - - source ci/env.sh - - rustup set profile default - - ci/publish-tarball.sh - deploy: - - provider: s3 - access_key_id: $AWS_ACCESS_KEY_ID - secret_access_key: $AWS_SECRET_ACCESS_KEY - bucket: release.solana.com - region: us-west-1 - skip_cleanup: true - acl: public_read - local_dir: travis-s3-upload - on: - all_branches: true - - provider: releases - token: $GITHUB_TOKEN - skip_cleanup: true - file_glob: true - file: travis-release-upload/* - on: - tags: true - - <<: *release-artifacts - name: "Windows release artifacts" - os: windows - install: - - choco install openssl - - export OPENSSL_DIR="C:\Program Files\OpenSSL-Win64" - - source ci/rust-version.sh - - PATH="/usr/local/opt/coreutils/libexec/gnubin:$PATH" - - readlink -f . - # Linux release artifacts are still built by ci/buildkite-secondary.yml - #- <<: *release-artifacts - # name: "Linux release artifacts" - # os: linux - # before_install: - # - sudo apt-get install libssl-dev libudev-dev - - # docs pull request - - name: "docs" - if: type IN (push, pull_request) OR tag IS present - language: node_js - node_js: - - "lts/*" - - services: - - docker - - cache: - directories: - - ~/.npm - - before_install: - - source ci/env.sh - - .travis/channel_restriction.sh edge beta || travis_terminate 0 - - .travis/affects.sh docs/ .travis || travis_terminate 0 - - cd docs/ - - source .travis/before_install.sh - - script: - - source .travis/script.sh diff --git a/README.md b/README.md index c6183f6ab6183e..bbaeb3d019a658 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@

- Solana + Solana

@@ -113,35 +113,3 @@ problem is solved by this code?" On the other hand, if a test does fail and you better way to solve the same problem, a Pull Request with your solution would most certainly be welcome! Likewise, if rewriting a test can better communicate what code it's protecting, please send us that patch! - -# Disclaimer - -All claims, content, designs, algorithms, estimates, roadmaps, -specifications, and performance measurements described in this project -are done with the Solana Labs, Inc. (“SL”) good faith efforts. It is up to -the reader to check and validate their accuracy and truthfulness. -Furthermore, nothing in this project constitutes a solicitation for -investment. - -Any content produced by SL or developer resources that SL provides are -for educational and inspirational purposes only. SL does not encourage, -induce or sanction the deployment, integration or use of any such -applications (including the code comprising the Solana blockchain -protocol) in violation of applicable laws or regulations and hereby -prohibits any such deployment, integration or use. This includes the use of -any such applications by the reader (a) in violation of export control -or sanctions laws of the United States or any other applicable -jurisdiction, (b) if the reader is located in or ordinarily resident in -a country or territory subject to comprehensive sanctions administered -by the U.S. Office of Foreign Assets Control (OFAC), or (c) if the -reader is or is working on behalf of a Specially Designated National -(SDN) or a person subject to similar blocking or denied party -prohibitions. - -The reader should be aware that U.S. export control and sanctions laws prohibit -U.S. persons (and other persons that are subject to such laws) from transacting -with persons in certain countries and territories or that are on the SDN list. -Accordingly, there is a risk to individuals that other persons using any of the -code contained in this repo, or a derivation thereof, may be sanctioned persons -and that transactions with such persons would be a violation of U.S. export -controls and sanctions law. diff --git a/ci/buildkite-pipeline-in-disk.sh b/ci/buildkite-pipeline-in-disk.sh index ad12e1fc000a89..2ce8af0432106b 100755 --- a/ci/buildkite-pipeline-in-disk.sh +++ b/ci/buildkite-pipeline-in-disk.sh @@ -289,7 +289,7 @@ if [[ -n $BUILDKITE_TAG ]]; then start_pipeline "Tag pipeline for $BUILDKITE_TAG" annotate --style info --context release-tag \ - "https://github.com/solana-labs/solana/releases/$BUILDKITE_TAG" + "https://github.com/anza-xyz/agave/releases/$BUILDKITE_TAG" # Jump directly to the secondary build to publish release artifacts quickly trigger_secondary_step @@ -307,7 +307,7 @@ if [[ $BUILDKITE_BRANCH =~ ^pull ]]; then # Add helpful link back to the corresponding Github Pull Request annotate --style info --context pr-backlink \ - "Github Pull Request: https://github.com/solana-labs/solana/$BUILDKITE_BRANCH" + "Github Pull Request: https://github.com/anza-xyz/agave/$BUILDKITE_BRANCH" if [[ $GITHUB_USER = "dependabot[bot]" ]]; then command_step dependabot "ci/dependabot-pr.sh" 5 diff --git a/ci/buildkite-pipeline.sh b/ci/buildkite-pipeline.sh index d40273863cc7a3..4ae00c9feab586 100755 --- a/ci/buildkite-pipeline.sh +++ b/ci/buildkite-pipeline.sh @@ -121,8 +121,8 @@ EOF trigger_secondary_step() { cat >> "$output_file" <<"EOF" - - name: "Trigger Build on solana-secondary" - trigger: "solana-secondary" + - name: "Trigger Build on agave-secondary" + trigger: "agave-secondary" branches: "!pull/*" async: true soft_fail: true @@ -315,7 +315,7 @@ if [[ -n $BUILDKITE_TAG ]]; then start_pipeline "Tag pipeline for $BUILDKITE_TAG" annotate --style info --context release-tag \ - "https://github.com/solana-labs/solana/releases/$BUILDKITE_TAG" + "https://github.com/anza-xyz/agave/releases/$BUILDKITE_TAG" # Jump directly to the secondary build to publish release artifacts quickly trigger_secondary_step @@ -333,7 +333,7 @@ if [[ $BUILDKITE_BRANCH =~ ^pull ]]; then # Add helpful link back to the corresponding Github Pull Request annotate --style info --context pr-backlink \ - "Github Pull Request: https://github.com/solana-labs/solana/$BUILDKITE_BRANCH" + "Github Pull Request: https://github.com/anza-xyz/agave/$BUILDKITE_BRANCH" if [[ $GITHUB_USER = "dependabot[bot]" ]]; then command_step dependabot "ci/dependabot-pr.sh" 5 diff --git a/ci/buildkite-solana-private.sh b/ci/buildkite-solana-private.sh index 70d8e4bfe4f59f..e5886a314eb27c 100755 --- a/ci/buildkite-solana-private.sh +++ b/ci/buildkite-solana-private.sh @@ -287,7 +287,7 @@ if [[ $BUILDKITE_BRANCH =~ ^pull ]]; then # Add helpful link back to the corresponding Github Pull Request annotate --style info --context pr-backlink \ - "Github Pull Request: https://github.com/solana-labs/solana/$BUILDKITE_BRANCH" + "Github Pull Request: https://github.com/anza-xyz/agave/$BUILDKITE_BRANCH" if [[ $GITHUB_USER = "dependabot[bot]" ]]; then command_step dependabot "ci/dependabot-pr.sh" 5 diff --git a/ci/channel-info.sh b/ci/channel-info.sh index c82806454d012c..2bb808365653c6 100755 --- a/ci/channel-info.sh +++ b/ci/channel-info.sh @@ -11,7 +11,7 @@ here="$(dirname "$0")" # shellcheck source=ci/semver_bash/semver.sh source "$here"/semver_bash/semver.sh -remote=https://github.com/solana-labs/solana.git +remote=https://github.com/anza-xyz/agave.git # Fetch all vX.Y.Z tags # diff --git a/ci/dependabot-pr.sh b/ci/dependabot-pr.sh index 91ecd5948c9a43..bb019001a0bcfa 100755 --- a/ci/dependabot-pr.sh +++ b/ci/dependabot-pr.sh @@ -21,7 +21,7 @@ fi echo --- "(FAILING) Backpropagating dependabot-triggered Cargo.lock updates" name="dependabot-buildkite" -api_base="https://api.github.com/repos/solana-labs/solana/pulls" +api_base="https://api.github.com/repos/anza-xyz/agave/pulls" pr_num=$(echo "$BUILDKITE_BRANCH" | grep -Eo '[0-9]+') branch=$(curl -s "$api_base/$pr_num" | python3 -c 'import json,sys;print(json.load(sys.stdin)["head"]["ref"])') diff --git a/ci/rust-version.sh b/ci/rust-version.sh index 3321f1d5ecb6a1..97ebb1c7935006 100644 --- a/ci/rust-version.sh +++ b/ci/rust-version.sh @@ -37,7 +37,7 @@ export rust_stable="$stable_version" export rust_nightly=nightly-"$nightly_version" -export ci_docker_image="solanalabs/ci:rust_${rust_stable}_${rust_nightly}" +export ci_docker_image="anzaxyz/ci:rust_${rust_stable}_${rust_nightly}" [[ -z $1 ]] || ( diff --git a/ci/test-coverage.sh b/ci/test-coverage.sh index 44231cd338a13e..ffd362acd287b8 100755 --- a/ci/test-coverage.sh +++ b/ci/test-coverage.sh @@ -32,5 +32,5 @@ else codecov -t "${CODECOV_TOKEN}" annotate --style success --context codecov.io \ - "CodeCov report: https://codecov.io/github/solana-labs/solana/commit/${CI_COMMIT:0:9}" + "CodeCov report: https://codecov.io/github/anza-xyz/agave/commit/${CI_COMMIT:0:9}" fi From 58e9a19f11822c93fdece4567d5250cd0b671ad1 Mon Sep 17 00:00:00 2001 From: Yihau Chen Date: Tue, 13 Feb 2024 00:00:58 +0800 Subject: [PATCH 03/84] [anza migration] ci: fix path (#8) * ci: fix windows build * ci: publish sdk docker image with the new name * update automerge status --- .github/workflows/release-artifacts.yml | 4 ++-- .mergify.yml | 2 +- sdk/docker-solana/build.sh | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 14760837ba0dbd..a77fd672d8b4b6 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -79,7 +79,7 @@ jobs: name: windows-artifact path: windows-release/ - windows-s3-upload: + windows-gcs-upload: if: ${{ needs.windows-build.outputs.channel != '' || needs.windows-build.outputs.tag != '' }} needs: [windows-build] runs-on: ubuntu-20.04 @@ -88,7 +88,7 @@ jobs: uses: actions/download-artifact@v3 with: name: windows-artifact - path: .windows-release/ + path: ./windows-release - name: Setup crediential uses: "google-github-actions/auth@v2" diff --git a/.mergify.yml b/.mergify.yml index 166f59a5f365d1..19f9b8f116a78a 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -50,7 +50,7 @@ pull_request_rules: - name: automatic merge (squash) on CI success conditions: - and: - - status-success=buildkite/solana + - status-success=buildkite/agave - status-success=ci-gate - label=automerge - label!=no-automerge diff --git a/sdk/docker-solana/build.sh b/sdk/docker-solana/build.sh index f1c8ee265d6d56..70e3d0d23e44de 100755 --- a/sdk/docker-solana/build.sh +++ b/sdk/docker-solana/build.sh @@ -29,7 +29,7 @@ cp -f ../../fetch-spl.sh usr/bin/ ./fetch-spl.sh ) -docker build -t solanalabs/solana:"$CHANNEL_OR_TAG" . +docker build -t anzaxyz/agave:"$CHANNEL_OR_TAG" . maybeEcho= if [[ -z $CI ]]; then @@ -43,4 +43,4 @@ else fi ) fi -$maybeEcho docker push solanalabs/solana:"$CHANNEL_OR_TAG" +$maybeEcho docker push anzaxyz/agave:"$CHANNEL_OR_TAG" From 91e3dd225004927f84638410f32cb98717ffb1de Mon Sep 17 00:00:00 2001 From: Yihau Chen Date: Thu, 15 Feb 2024 22:06:51 +0800 Subject: [PATCH 04/84] [anza migration] ci: removed unused s3 upload in Windows build (#9) ci: removed unused s3 upload in Windows build --- .github/workflows/release-artifacts.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index a77fd672d8b4b6..d8e22c42ce5da8 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -99,17 +99,6 @@ jobs: run: | gcloud storage cp --recursive windows-release/* gs://anza-release/ - - name: Upload - uses: jakejarvis/s3-sync-action@master - with: - args: --acl public-read --follow-symlinks - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} - AWS_REGION: "us-west-1" - SOURCE_DIR: "windows-release" - windows-gh-release: if: ${{ needs.windows-build.outputs.tag != '' }} needs: [windows-build] From 3f9a7a52eac0da3a8feb48fd83c939b222c66bc4 Mon Sep 17 00:00:00 2001 From: Yihau Chen Date: Thu, 22 Feb 2024 11:44:01 +0800 Subject: [PATCH 05/84] [anza migration] rename crates (#10) * rename geyser-plugin-interface * rename cargo registry * rename watchtower * rename ledger tool * rename validator * rename install * rename geyser plugin interface when patch --- .../scripts/downstream-project-spl-common.sh | 3 + .github/workflows/release-artifacts.yml | 2 +- Cargo.lock | 424 +++++++++--------- Cargo.toml | 4 +- cargo-registry/Cargo.toml | 4 +- ci/localnet-sanity.sh | 4 +- ci/publish-installer.sh | 2 +- ci/publish-tarball.sh | 4 +- ci/run-sanity.sh | 2 +- docs/src/backwards-compatibility.md | 2 +- docs/src/cli/install.md | 12 +- docs/src/clusters/available.md | 12 +- docs/src/clusters/benchmark.md | 2 +- docs/src/implemented-proposals/installer.md | 58 +-- .../rpc-transaction-history.md | 2 +- docs/src/operations/best-practices/general.md | 28 +- .../operations/best-practices/monitoring.md | 28 +- docs/src/operations/guides/restart-cluster.md | 14 +- .../operations/guides/validator-failover.md | 12 +- docs/src/operations/guides/validator-start.md | 32 +- docs/src/operations/guides/vote-accounts.md | 8 +- docs/src/operations/setup-a-validator.md | 24 +- docs/src/operations/setup-an-rpc-node.md | 4 +- docs/src/validator/geyser.md | 8 +- geyser-plugin-interface/Cargo.toml | 4 +- .../src/geyser_plugin_interface.rs | 2 +- geyser-plugin-manager/Cargo.toml | 2 +- .../src/accounts_update_notifier.rs | 6 +- .../src/block_metadata_notifier.rs | 4 +- geyser-plugin-manager/src/entry_notifier.rs | 6 +- .../src/geyser_plugin_manager.rs | 4 +- .../src/slot_status_notifier.rs | 2 +- .../src/transaction_notifier.rs | 4 +- install/Cargo.toml | 4 +- ...-install-init.sh => agave-install-init.sh} | 16 +- install/install-help.sh | 6 +- ...-install-init.rs => agave-install-init.rs} | 2 +- install/src/command.rs | 4 +- install/src/lib.rs | 2 +- install/src/main.rs | 2 +- ledger-tool/Cargo.toml | 4 +- ledger-tool/src/blockstore.rs | 2 +- ledger-tool/src/ledger_utils.rs | 8 +- ledger/src/blockstore_db.rs | 2 +- local-cluster/tests/local_cluster.rs | 4 +- multinode-demo/bootstrap-validator.sh | 4 +- multinode-demo/common.sh | 6 +- multinode-demo/validator.sh | 6 +- net/net.sh | 8 +- net/remote/remote-deploy-update.sh | 2 +- net/remote/remote-node.sh | 8 +- net/remote/remote-sanity.sh | 4 +- notifier/src/lib.rs | 4 +- programs/sbf/Cargo.lock | 152 +++---- programs/sbf/Cargo.toml | 2 +- programs/sbf/rust/simulation/Cargo.toml | 2 +- .../sbf/rust/simulation/tests/validator.rs | 2 +- .../src/nonblocking/pubsub_client.rs | 6 +- pubsub-client/src/pubsub_client.rs | 6 +- rbpf-cli/src/main.rs | 4 +- rpc/src/rpc.rs | 8 +- ...tall-deploy.sh => agave-install-deploy.sh} | 4 +- scripts/cargo-install-all.sh | 14 +- scripts/check-dev-context-only-utils.sh | 2 +- scripts/run.sh | 6 +- .../abi-testcases/mixed-validator-test.sh | 6 +- .../stability-testcases/gossip-dos-test.sh | 6 +- validator/Cargo.toml | 8 +- validator/src/bin/solana-test-validator.rs | 8 +- validator/src/bootstrap.rs | 2 +- validator/src/main.rs | 20 +- watchtower/Cargo.toml | 4 +- watchtower/README.md | 2 +- watchtower/src/main.rs | 10 +- 74 files changed, 554 insertions(+), 547 deletions(-) rename install/{solana-install-init.sh => agave-install-init.sh} (89%) rename install/src/bin/{solana-install-init.rs => agave-install-init.rs} (92%) rename scripts/{solana-install-deploy.sh => agave-install-deploy.sh} (90%) diff --git a/.github/scripts/downstream-project-spl-common.sh b/.github/scripts/downstream-project-spl-common.sh index c6dcfaca007867..861be12c7d1a45 100644 --- a/.github/scripts/downstream-project-spl-common.sh +++ b/.github/scripts/downstream-project-spl-common.sh @@ -22,3 +22,6 @@ if semverGT "$project_used_solana_version" "$SOLANA_VER"; then fi ./patch.crates-io.sh "$SOLANA_DIR" + +# anza migration stopgap. can be removed when agave is fully recommended for public usage. +sed -i 's/solana-geyser-plugin-interface/agave-geyser-plugin-interface/g' ./Cargo.toml diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index d8e22c42ce5da8..45be181c3ce9e1 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -70,7 +70,7 @@ jobs: mkdir -p "windows-release/$FOLDER_NAME" cp -v "solana-release-x86_64-pc-windows-msvc.tar.bz2" "windows-release/$FOLDER_NAME/" cp -v "solana-release-x86_64-pc-windows-msvc.yml" "windows-release/$FOLDER_NAME/" - cp -v "solana-install-init-x86_64-pc-windows-msvc"* "windows-release/$FOLDER_NAME" + cp -v "agave-install-init-x86_64-pc-windows-msvc"* "windows-release/$FOLDER_NAME" - name: Upload Artifacts if: ${{ steps.build.outputs.channel != '' || steps.build.outputs.tag != '' }} diff --git a/Cargo.lock b/Cargo.lock index 650b369d205c5c..c72b90930d7cf9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -62,6 +62,217 @@ dependencies = [ "zeroize", ] +[[package]] +name = "agave-cargo-registry" +version = "1.19.0" +dependencies = [ + "clap 2.33.3", + "flate2", + "hex", + "hyper", + "log", + "rustc_version 0.4.0", + "serde", + "serde_json", + "sha2 0.10.8", + "solana-clap-utils", + "solana-cli", + "solana-cli-config", + "solana-cli-output", + "solana-logger", + "solana-remote-wallet", + "solana-rpc-client", + "solana-rpc-client-api", + "solana-sdk", + "solana-version", + "tar", + "tempfile", + "tokio", + "toml 0.8.10", +] + +[[package]] +name = "agave-geyser-plugin-interface" +version = "1.19.0" +dependencies = [ + "log", + "solana-sdk", + "solana-transaction-status", + "thiserror", +] + +[[package]] +name = "agave-install" +version = "1.19.0" +dependencies = [ + "atty", + "bincode", + "bzip2", + "chrono", + "clap 2.33.3", + "console", + "crossbeam-channel", + "ctrlc", + "dirs-next", + "indicatif", + "lazy_static", + "nix 0.26.4", + "reqwest", + "scopeguard", + "semver 1.0.22", + "serde", + "serde_yaml 0.8.26", + "serde_yaml 0.9.32", + "solana-clap-utils", + "solana-config-program", + "solana-logger", + "solana-rpc-client", + "solana-sdk", + "solana-version", + "tar", + "tempfile", + "url 2.5.0", + "winapi 0.3.9", + "winreg", +] + +[[package]] +name = "agave-ledger-tool" +version = "1.19.0" +dependencies = [ + "assert_cmd", + "bs58", + "bytecount", + "chrono", + "clap 2.33.3", + "crossbeam-channel", + "csv", + "dashmap", + "futures 0.3.30", + "histogram", + "itertools", + "log", + "num_cpus", + "regex", + "serde", + "serde_json", + "signal-hook", + "solana-account-decoder", + "solana-accounts-db", + "solana-bpf-loader-program", + "solana-clap-utils", + "solana-cli-output", + "solana-core", + "solana-cost-model", + "solana-entry", + "solana-geyser-plugin-manager", + "solana-gossip", + "solana-ledger", + "solana-logger", + "solana-measure", + "solana-program-runtime", + "solana-rpc", + "solana-runtime", + "solana-sdk", + "solana-stake-program", + "solana-storage-bigtable", + "solana-streamer", + "solana-svm", + "solana-transaction-status", + "solana-unified-scheduler-pool", + "solana-version", + "solana-vote-program", + "solana_rbpf", + "thiserror", + "tikv-jemallocator", + "tokio", +] + +[[package]] +name = "agave-validator" +version = "1.19.0" +dependencies = [ + "agave-geyser-plugin-interface", + "chrono", + "clap 2.33.3", + "console", + "core_affinity", + "crossbeam-channel", + "fd-lock", + "indicatif", + "itertools", + "jsonrpc-core", + "jsonrpc-core-client", + "jsonrpc-derive", + "jsonrpc-ipc-server", + "jsonrpc-server-utils", + "lazy_static", + "libc", + "libloading", + "log", + "num_cpus", + "rand 0.8.5", + "rayon", + "serde", + "serde_json", + "serde_yaml 0.9.32", + "signal-hook", + "solana-account-decoder", + "solana-accounts-db", + "solana-clap-utils", + "solana-cli-config", + "solana-core", + "solana-download-utils", + "solana-entry", + "solana-faucet", + "solana-genesis-utils", + "solana-geyser-plugin-manager", + "solana-gossip", + "solana-ledger", + "solana-logger", + "solana-metrics", + "solana-net-utils", + "solana-perf", + "solana-poh", + "solana-rpc", + "solana-rpc-client", + "solana-rpc-client-api", + "solana-runtime", + "solana-sdk", + "solana-send-transaction-service", + "solana-storage-bigtable", + "solana-streamer", + "solana-svm", + "solana-test-validator", + "solana-tpu-client", + "solana-unified-scheduler-pool", + "solana-version", + "solana-vote-program", + "spl-token-2022", + "symlink", + "thiserror", + "tikv-jemallocator", +] + +[[package]] +name = "agave-watchtower" +version = "1.19.0" +dependencies = [ + "clap 2.33.3", + "humantime", + "log", + "solana-clap-utils", + "solana-cli-config", + "solana-cli-output", + "solana-logger", + "solana-metrics", + "solana-notifier", + "solana-rpc-client", + "solana-rpc-client-api", + "solana-sdk", + "solana-version", +] + [[package]] name = "ahash" version = "0.7.6" @@ -5482,35 +5693,6 @@ dependencies = [ "tar", ] -[[package]] -name = "solana-cargo-registry" -version = "1.19.0" -dependencies = [ - "clap 2.33.3", - "flate2", - "hex", - "hyper", - "log", - "rustc_version 0.4.0", - "serde", - "serde_json", - "sha2 0.10.8", - "solana-clap-utils", - "solana-cli", - "solana-cli-config", - "solana-cli-output", - "solana-logger", - "solana-remote-wallet", - "solana-rpc-client", - "solana-rpc-client-api", - "solana-sdk", - "solana-version", - "tar", - "tempfile", - "tokio", - "toml 0.8.10", -] - [[package]] name = "solana-cargo-test-bpf" version = "1.19.0" @@ -6040,20 +6222,11 @@ dependencies = [ "solana-sdk", ] -[[package]] -name = "solana-geyser-plugin-interface" -version = "1.19.0" -dependencies = [ - "log", - "solana-sdk", - "solana-transaction-status", - "thiserror", -] - [[package]] name = "solana-geyser-plugin-manager" version = "1.19.0" dependencies = [ + "agave-geyser-plugin-interface", "bs58", "crossbeam-channel", "json5", @@ -6064,7 +6237,6 @@ dependencies = [ "serde_json", "solana-accounts-db", "solana-entry", - "solana-geyser-plugin-interface", "solana-ledger", "solana-measure", "solana-metrics", @@ -6126,41 +6298,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "solana-install" -version = "1.19.0" -dependencies = [ - "atty", - "bincode", - "bzip2", - "chrono", - "clap 2.33.3", - "console", - "crossbeam-channel", - "ctrlc", - "dirs-next", - "indicatif", - "lazy_static", - "nix 0.26.4", - "reqwest", - "scopeguard", - "semver 1.0.22", - "serde", - "serde_yaml 0.8.26", - "serde_yaml 0.9.32", - "solana-clap-utils", - "solana-config-program", - "solana-logger", - "solana-rpc-client", - "solana-sdk", - "solana-version", - "tar", - "tempfile", - "url 2.5.0", - "winapi 0.3.9", - "winreg", -] - [[package]] name = "solana-keygen" version = "1.19.0" @@ -6248,58 +6385,6 @@ dependencies = [ "trees", ] -[[package]] -name = "solana-ledger-tool" -version = "1.19.0" -dependencies = [ - "assert_cmd", - "bs58", - "bytecount", - "chrono", - "clap 2.33.3", - "crossbeam-channel", - "csv", - "dashmap", - "futures 0.3.30", - "histogram", - "itertools", - "log", - "num_cpus", - "regex", - "serde", - "serde_json", - "signal-hook", - "solana-account-decoder", - "solana-accounts-db", - "solana-bpf-loader-program", - "solana-clap-utils", - "solana-cli-output", - "solana-core", - "solana-cost-model", - "solana-entry", - "solana-geyser-plugin-manager", - "solana-gossip", - "solana-ledger", - "solana-logger", - "solana-measure", - "solana-program-runtime", - "solana-rpc", - "solana-runtime", - "solana-sdk", - "solana-stake-program", - "solana-storage-bigtable", - "solana-streamer", - "solana-svm", - "solana-transaction-status", - "solana-unified-scheduler-pool", - "solana-version", - "solana-vote-program", - "solana_rbpf", - "thiserror", - "tikv-jemallocator", - "tokio", -] - [[package]] name = "solana-loader-v4-program" version = "1.19.0" @@ -7463,72 +7548,6 @@ dependencies = [ "solana-metrics", ] -[[package]] -name = "solana-validator" -version = "1.19.0" -dependencies = [ - "chrono", - "clap 2.33.3", - "console", - "core_affinity", - "crossbeam-channel", - "fd-lock", - "indicatif", - "itertools", - "jsonrpc-core", - "jsonrpc-core-client", - "jsonrpc-derive", - "jsonrpc-ipc-server", - "jsonrpc-server-utils", - "lazy_static", - "libc", - "libloading", - "log", - "num_cpus", - "rand 0.8.5", - "rayon", - "serde", - "serde_json", - "serde_yaml 0.9.32", - "signal-hook", - "solana-account-decoder", - "solana-accounts-db", - "solana-clap-utils", - "solana-cli-config", - "solana-core", - "solana-download-utils", - "solana-entry", - "solana-faucet", - "solana-genesis-utils", - "solana-geyser-plugin-interface", - "solana-geyser-plugin-manager", - "solana-gossip", - "solana-ledger", - "solana-logger", - "solana-metrics", - "solana-net-utils", - "solana-perf", - "solana-poh", - "solana-rpc", - "solana-rpc-client", - "solana-rpc-client-api", - "solana-runtime", - "solana-sdk", - "solana-send-transaction-service", - "solana-storage-bigtable", - "solana-streamer", - "solana-svm", - "solana-test-validator", - "solana-tpu-client", - "solana-unified-scheduler-pool", - "solana-version", - "solana-vote-program", - "spl-token-2022", - "symlink", - "thiserror", - "tikv-jemallocator", -] - [[package]] name = "solana-version" version = "1.19.0" @@ -7585,25 +7604,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "solana-watchtower" -version = "1.19.0" -dependencies = [ - "clap 2.33.3", - "humantime", - "log", - "solana-clap-utils", - "solana-cli-config", - "solana-cli-output", - "solana-logger", - "solana-metrics", - "solana-notifier", - "solana-rpc-client", - "solana-rpc-client-api", - "solana-sdk", - "solana-version", -] - [[package]] name = "solana-wen-restart" version = "1.19.0" diff --git a/Cargo.toml b/Cargo.toml index 66436c9cfb3fd8..27376370297e26 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -317,7 +317,7 @@ solana-bench-tps = { path = "bench-tps", version = "=1.19.0" } solana-bloom = { path = "bloom", version = "=1.19.0" } solana-bpf-loader-program = { path = "programs/bpf_loader", version = "=1.19.0" } solana-bucket-map = { path = "bucket_map", version = "=1.19.0" } -solana-cargo-registry = { path = "cargo-registry", version = "=1.19.0" } +agave-cargo-registry = { path = "cargo-registry", version = "=1.19.0" } solana-clap-utils = { path = "clap-utils", version = "=1.19.0" } solana-clap-v3-utils = { path = "clap-v3-utils", version = "=1.19.0" } solana-cli = { path = "cli", version = "=1.19.0" } @@ -336,7 +336,7 @@ solana-frozen-abi = { path = "frozen-abi", version = "=1.19.0" } solana-frozen-abi-macro = { path = "frozen-abi/macro", version = "=1.19.0" } solana-genesis = { path = "genesis", version = "=1.19.0" } solana-genesis-utils = { path = "genesis-utils", version = "=1.19.0" } -solana-geyser-plugin-interface = { path = "geyser-plugin-interface", version = "=1.19.0" } +agave-geyser-plugin-interface = { path = "geyser-plugin-interface", version = "=1.19.0" } solana-geyser-plugin-manager = { path = "geyser-plugin-manager", version = "=1.19.0" } solana-gossip = { path = "gossip", version = "=1.19.0" } solana-ledger = { path = "ledger", version = "=1.19.0" } diff --git a/cargo-registry/Cargo.toml b/cargo-registry/Cargo.toml index 4e13f477ee5e2d..395493a8e85f00 100644 --- a/cargo-registry/Cargo.toml +++ b/cargo-registry/Cargo.toml @@ -1,7 +1,7 @@ [package] -name = "solana-cargo-registry" +name = "agave-cargo-registry" description = "Solana cargo registry" -documentation = "https://docs.rs/solana-cargo-registry" +documentation = "https://docs.rs/agave-cargo-registry" version = { workspace = true } authors = { workspace = true } repository = { workspace = true } diff --git a/ci/localnet-sanity.sh b/ci/localnet-sanity.sh index e6734e180aa2da..b01eca31d50d81 100755 --- a/ci/localnet-sanity.sh +++ b/ci/localnet-sanity.sh @@ -202,8 +202,8 @@ killNodes() { # Try to use the RPC exit API to cleanly exit the first two nodes # (dynamic nodes, -x, are just killed) echo "--- RPC exit" - $solana_validator --ledger "$SOLANA_CONFIG_DIR"/bootstrap-validator exit --force || true - $solana_validator --ledger "$SOLANA_CONFIG_DIR"/validator exit --force || true + $agave_validator --ledger "$SOLANA_CONFIG_DIR"/bootstrap-validator exit --force || true + $agave_validator --ledger "$SOLANA_CONFIG_DIR"/validator exit --force || true # Give the nodes a splash of time to cleanly exit before killing them sleep 2 diff --git a/ci/publish-installer.sh b/ci/publish-installer.sh index e58fd939dd1a40..f7d98ffd5ddcf9 100755 --- a/ci/publish-installer.sh +++ b/ci/publish-installer.sh @@ -31,7 +31,7 @@ SOLANA_RELEASE=$CHANNEL_OR_TAG SOLANA_INSTALL_INIT_ARGS=$CHANNEL_OR_TAG SOLANA_DOWNLOAD_ROOT=https://release.anza.xyz EOF -cat install/solana-install-init.sh >>release.anza.xyz-install +cat install/agave-install-init.sh >>release.anza.xyz-install echo --- GCS: "install" upload-gcs-artifact "/solana/release.anza.xyz-install" "gs://anza-release/$CHANNEL_OR_TAG/install" diff --git a/ci/publish-tarball.sh b/ci/publish-tarball.sh index 5c64f09564fe9f..da5862fb3de1d2 100755 --- a/ci/publish-tarball.sh +++ b/ci/publish-tarball.sh @@ -93,7 +93,7 @@ echo --- Creating release tarball tar cvf "${TARBALL_BASENAME}"-$TARGET.tar "${RELEASE_BASENAME}" bzip2 "${TARBALL_BASENAME}"-$TARGET.tar - cp "${RELEASE_BASENAME}"/bin/solana-install-init solana-install-init-$TARGET + cp "${RELEASE_BASENAME}"/bin/agave-install-init agave-install-init-$TARGET cp "${RELEASE_BASENAME}"/version.yml "${TARBALL_BASENAME}"-$TARGET.yml ) @@ -110,7 +110,7 @@ fi source ci/upload-ci-artifact.sh -for file in "${TARBALL_BASENAME}"-$TARGET.tar.bz2 "${TARBALL_BASENAME}"-$TARGET.yml solana-install-init-"$TARGET"* $MAYBE_TARBALLS; do +for file in "${TARBALL_BASENAME}"-$TARGET.tar.bz2 "${TARBALL_BASENAME}"-$TARGET.yml agave-install-init-"$TARGET"* $MAYBE_TARBALLS; do if [[ -n $DO_NOT_PUBLISH_TAR ]]; then upload-ci-artifact "$file" echo "Skipped $file due to DO_NOT_PUBLISH_TAR" diff --git a/ci/run-sanity.sh b/ci/run-sanity.sh index 8108d13a061fd5..88a6f40b1adf28 100755 --- a/ci/run-sanity.sh +++ b/ci/run-sanity.sh @@ -31,7 +31,7 @@ while [[ $latest_slot -le $((snapshot_slot + 1)) ]]; do latest_slot=$($solana_cli --url http://localhost:8899 slot --commitment processed) done -$solana_validator --ledger config/ledger exit --force || true +$agave_validator --ledger config/ledger exit --force || true wait $pid diff --git a/docs/src/backwards-compatibility.md b/docs/src/backwards-compatibility.md index 4a3c60b8e129bd..0fdc388ea2dbae 100644 --- a/docs/src/backwards-compatibility.md +++ b/docs/src/backwards-compatibility.md @@ -76,7 +76,7 @@ Major releases: - [`solana-program`](https://docs.rs/solana-program/) - Rust SDK for writing programs - [`solana-client`](https://docs.rs/solana-client/) - Rust client for connecting to RPC API - [`solana-cli-config`](https://docs.rs/solana-cli-config/) - Rust client for managing Solana CLI config files -- [`solana-geyser-plugin-interface`](https://docs.rs/solana-geyser-plugin-interface/) - Rust interface for developing Solana Geyser plugins. +- [`agave-geyser-plugin-interface`](https://docs.rs/agave-geyser-plugin-interface/) - Rust interface for developing Solana Geyser plugins. Patch releases: diff --git a/docs/src/cli/install.md b/docs/src/cli/install.md index 3667c733e3f4d4..20f6516314fb02 100644 --- a/docs/src/cli/install.md +++ b/docs/src/cli/install.md @@ -56,7 +56,7 @@ Please update your PATH environment variable to include the solana programs: solana --version ``` -- After a successful install, `solana-install update` may be used to easily +- After a successful install, `agave-install update` may be used to easily update the Solana software to a newer version at any time. --- @@ -74,7 +74,7 @@ solana --version installer into a temporary directory: ```bash -cmd /c "curl https://release.solana.com/LATEST_SOLANA_RELEASE_VERSION/solana-install-init-x86_64-pc-windows-msvc.exe --output C:\solana-install-tmp\solana-install-init.exe --create-dirs" +cmd /c "curl https://release.solana.com/LATEST_SOLANA_RELEASE_VERSION/agave-install-init-x86_64-pc-windows-msvc.exe --output C:\agave-install-tmp\agave-install-init.exe --create-dirs" ``` - Copy and paste the following command, then press Enter to install the latest @@ -82,7 +82,7 @@ cmd /c "curl https://release.solana.com/LATEST_SOLANA_RELEASE_VERSION/solana-ins to allow the program to run. ```bash -C:\solana-install-tmp\solana-install-init.exe LATEST_SOLANA_RELEASE_VERSION +C:\agave-install-tmp\agave-install-init.exe LATEST_SOLANA_RELEASE_VERSION ``` - When the installer is finished, press Enter. @@ -97,12 +97,12 @@ C:\solana-install-tmp\solana-install-init.exe LATEST_SOLANA_RELEASE_VERSION solana --version ``` -- After a successful install, `solana-install update` may be used to easily +- After a successful install, `agave-install update` may be used to easily update the Solana software to a newer version at any time. ## Download Prebuilt Binaries -If you would rather not use `solana-install` to manage the install, you can +If you would rather not use `agave-install` to manage the install, you can manually download and install the binaries. ### Linux @@ -255,7 +255,7 @@ You can then run the following command to obtain the same result as with prebuilt binaries: ```bash -solana-install init +agave-install init ``` ## Use Homebrew diff --git a/docs/src/clusters/available.md b/docs/src/clusters/available.md index dfbca41672b499..52a7d469ad0cc5 100644 --- a/docs/src/clusters/available.md +++ b/docs/src/clusters/available.md @@ -41,10 +41,10 @@ export SOLANA_METRICS_CONFIG="host=https://metrics.solana.com:8086,db=devnet,u=s solana config set --url https://api.devnet.solana.com ``` -##### Example `solana-validator` command-line +##### Example `agave-validator` command-line ```bash -$ solana-validator \ +$ agave-validator \ --identity validator-keypair.json \ --vote-account vote-account-keypair.json \ --known-validator dv1ZAGvdsz5hHLwWXsVnM94hWf1pjbKVau1QVkaMJ92 \ @@ -93,10 +93,10 @@ export SOLANA_METRICS_CONFIG="host=https://metrics.solana.com:8086,db=tds,u=test solana config set --url https://api.testnet.solana.com ``` -##### Example `solana-validator` command-line +##### Example `agave-validator` command-line ```bash -$ solana-validator \ +$ agave-validator \ --identity validator-keypair.json \ --vote-account vote-account-keypair.json \ --known-validator 5D1fNXzvv5NjV1ysLjirC4WY92RNsVH18vjmcszZd8on \ @@ -145,10 +145,10 @@ export SOLANA_METRICS_CONFIG="host=https://metrics.solana.com:8086,db=mainnet-be solana config set --url https://api.mainnet-beta.solana.com ``` -##### Example `solana-validator` command-line +##### Example `agave-validator` command-line ```bash -$ solana-validator \ +$ agave-validator \ --identity ~/validator-keypair.json \ --vote-account ~/vote-account-keypair.json \ --known-validator 7Np41oeYqPefeNQEHSv1UDhYrehxin3NStELsSKCT4K2 \ diff --git a/docs/src/clusters/benchmark.md b/docs/src/clusters/benchmark.md index d913f9e5f16392..35978cdd0967dd 100644 --- a/docs/src/clusters/benchmark.md +++ b/docs/src/clusters/benchmark.md @@ -108,7 +108,7 @@ For example Generally we are using `debug` for infrequent debug messages, `trace` for potentially frequent messages and `info` for performance-related logging. -You can also attach to a running process with GDB. The leader's process is named _solana-validator_: +You can also attach to a running process with GDB. The leader's process is named _agave-validator_: ```bash sudo gdb diff --git a/docs/src/implemented-proposals/installer.md b/docs/src/implemented-proposals/installer.md index a3ad797171c5b8..c052aa7b4e54e5 100644 --- a/docs/src/implemented-proposals/installer.md +++ b/docs/src/implemented-proposals/installer.md @@ -13,16 +13,16 @@ This document proposes an easy to use software install and updater that can be u The easiest install method for supported platforms: ```bash -$ curl -sSf https://raw.githubusercontent.com/solana-labs/solana/v1.0.0/install/solana-install-init.sh | sh +$ curl -sSf https://raw.githubusercontent.com/solana-labs/solana/v1.0.0/install/agave-install-init.sh | sh ``` -This script will check github for the latest tagged release and download and run the `solana-install-init` binary from there. +This script will check github for the latest tagged release and download and run the `agave-install-init` binary from there. If additional arguments need to be specified during the installation, the following shell syntax is used: ```bash -$ init_args=.... # arguments for `solana-install-init ...` -$ curl -sSf https://raw.githubusercontent.com/solana-labs/solana/v1.0.0/install/solana-install-init.sh | sh -s - ${init_args} +$ init_args=.... # arguments for `agave-install-init ...` +$ curl -sSf https://raw.githubusercontent.com/solana-labs/solana/v1.0.0/install/agave-install-init.sh | sh -s - ${init_args} ``` ### Fetch and run a pre-built installer from a Github release @@ -30,9 +30,9 @@ $ curl -sSf https://raw.githubusercontent.com/solana-labs/solana/v1.0.0/install/ With a well-known release URL, a pre-built binary can be obtained for supported platforms: ```bash -$ curl -o solana-install-init https://github.com/solana-labs/solana/releases/download/v1.0.0/solana-install-init-x86_64-apple-darwin -$ chmod +x ./solana-install-init -$ ./solana-install-init --help +$ curl -o agave-install-init https://github.com/solana-labs/solana/releases/download/v1.0.0/agave-install-init-x86_64-apple-darwin +$ chmod +x ./agave-install-init +$ ./agave-install-init --help ``` ### Build and run the installer from source @@ -51,16 +51,16 @@ Given a solana release tarball \(as created by `ci/publish-tarball.sh`\) that ha ```bash $ solana-keygen new -o update-manifest.json # <-- only generated once, the public key is shared with users -$ solana-install deploy http://example.com/path/to/solana-release.tar.bz2 update-manifest.json +$ agave-install deploy http://example.com/path/to/solana-release.tar.bz2 update-manifest.json ``` ### Run a validator node that auto updates itself ```bash -$ solana-install init --pubkey 92DMonmBYXwEMHJ99c9ceRSpAmk9v6i3RdvDdXaVcrfj # <-- pubkey is obtained from whoever is deploying the updates -$ export PATH=~/.local/share/solana-install/bin:$PATH +$ agave-install init --pubkey 92DMonmBYXwEMHJ99c9ceRSpAmk9v6i3RdvDdXaVcrfj # <-- pubkey is obtained from whoever is deploying the updates +$ export PATH=~/.local/share/agave-install/bin:$PATH $ solana-keygen ... # <-- runs the latest solana-keygen -$ solana-install run solana-validator ... # <-- runs a validator, restarting it as necessary when an update is applied +$ agave-install run agave-validator ... # <-- runs a validator, restarting it as necessary when an update is applied ``` ## On-chain Update Manifest @@ -87,9 +87,9 @@ pub struct SignedUpdateManifest { } ``` -Note that the `manifest` field itself contains a corresponding signature \(`manifest_signature`\) to guard against man-in-the-middle attacks between the `solana-install` tool and the solana cluster RPC API. +Note that the `manifest` field itself contains a corresponding signature \(`manifest_signature`\) to guard against man-in-the-middle attacks between the `agave-install` tool and the solana cluster RPC API. -To guard against rollback attacks, `solana-install` will refuse to install an update with an older `timestamp_secs` than what is currently installed. +To guard against rollback attacks, `agave-install` will refuse to install an update with an older `timestamp_secs` than what is currently installed. ## Release Archive Contents @@ -101,17 +101,17 @@ A release archive is expected to be a tar file compressed with bzip2 with the fo - `/bin/` -- directory containing available programs in the release. - `solana-install` will symlink this directory to + `agave-install` will symlink this directory to - `~/.local/share/solana-install/bin` for use by the `PATH` environment + `~/.local/share/agave-install/bin` for use by the `PATH` environment variable. - `...` -- any additional files and directories are permitted -## solana-install Tool +## agave-install Tool -The `solana-install` tool is used by the user to install and update their cluster software. +The `agave-install` tool is used by the user to install and update their cluster software. It manages the following files and directories in the user's home directory: @@ -122,11 +122,11 @@ It manages the following files and directories in the user's home directory: ### Command-line Interface ```text -solana-install 0.16.0 +agave-install 0.16.0 The solana cluster software installer USAGE: - solana-install [OPTIONS] + agave-install [OPTIONS] FLAGS: -h, --help Prints help information @@ -145,11 +145,11 @@ SUBCOMMANDS: ``` ```text -solana-install-init +agave-install-init initializes a new installation USAGE: - solana-install init [OPTIONS] + agave-install init [OPTIONS] FLAGS: -h, --help Prints help information @@ -161,11 +161,11 @@ OPTIONS: ``` ```text -solana-install info +agave-install info displays information about the current installation USAGE: - solana-install info [FLAGS] + agave-install info [FLAGS] FLAGS: -h, --help Prints help information @@ -173,11 +173,11 @@ FLAGS: ``` ```text -solana-install deploy +agave-install deploy deploys a new update USAGE: - solana-install deploy + agave-install deploy FLAGS: -h, --help Prints help information @@ -188,22 +188,22 @@ ARGS: ``` ```text -solana-install update +agave-install update checks for an update, and if available downloads and applies it USAGE: - solana-install update + agave-install update FLAGS: -h, --help Prints help information ``` ```text -solana-install run +agave-install run Runs a program while periodically checking and applying software updates USAGE: - solana-install run [program_arguments]... + agave-install run [program_arguments]... FLAGS: -h, --help Prints help information diff --git a/docs/src/implemented-proposals/rpc-transaction-history.md b/docs/src/implemented-proposals/rpc-transaction-history.md index 54288ad9659bd7..607a79ce658b98 100644 --- a/docs/src/implemented-proposals/rpc-transaction-history.md +++ b/docs/src/implemented-proposals/rpc-transaction-history.md @@ -68,7 +68,7 @@ the results of BigTable queries more complicated but is not a significant issue. ## Data Population The ongoing population of instance data will occur on an epoch cadence through -the use of a new `solana-ledger-tool` command that will convert rocksdb data for +the use of a new `agave-ledger-tool` command that will convert rocksdb data for a given slot range into the instance schema. The same process will be run once, manually, to backfill the existing ledger diff --git a/docs/src/operations/best-practices/general.md b/docs/src/operations/best-practices/general.md index 29ef42c81b7f5f..3e531b0160c571 100644 --- a/docs/src/operations/best-practices/general.md +++ b/docs/src/operations/best-practices/general.md @@ -23,12 +23,12 @@ watch past workshops through the ## Help with the validator command line -From within the Solana CLI, you can execute the `solana-validator` command with +From within the Solana CLI, you can execute the `agave-validator` command with the `--help` flag to get a better understanding of the flags and sub commands available. ``` -solana-validator --help +agave-validator --help ``` ## Restarting your validator @@ -49,14 +49,14 @@ solana leader-schedule Based on the current slot and the leader schedule, you can calculate open time windows where your validator is not expected to produce blocks. -Assuming you are ready to restart, you may use the `solana-validator exit` +Assuming you are ready to restart, you may use the `agave-validator exit` command. The command exits your validator process when an appropriate idle time window is reached. Assuming that you have systemd implemented for your validator process, the validator should restart automatically after the exit. See the below help command for details: ``` -solana-validator exit --help +agave-validator exit --help ``` ## Upgrading @@ -75,28 +75,28 @@ process. It is a best practice to always build your Solana binaries from source. If you build from source, you are certain that the code you are building has not been tampered with before the binary was created. You may also be able to optimize -your `solana-validator` binary to your specific hardware. +your `agave-validator` binary to your specific hardware. If you build from source on the validator machine (or a machine with the same CPU), you can target your specific architecture using the `-march` flag. Refer to the following doc for [instructions on building from source](../../cli/install.md#build-from-source). -### solana-install +### agave-install If you are not comfortable building from source, or you need to quickly install a new version to test something out, you could instead try using the -`solana-install` command. +`agave-install` command. Assuming you want to install Solana version `1.14.17`, you would execute the following: ``` -solana-install init 1.14.17 +agave-install init 1.14.17 ``` This command downloads the executable for `1.14.17` and installs it into a -`.local` directory. You can also look at `solana-install --help` for more +`.local` directory. You can also look at `agave-install --help` for more options. > **Note** this command only works if you already have the solana cli installed. @@ -106,7 +106,7 @@ options. ### Restart For all install methods, the validator process will need to be restarted before -the newly installed version is in use. Use `solana-validator exit` to restart +the newly installed version is in use. Use `agave-validator exit` to restart your validator process. ### Verifying version @@ -132,13 +132,13 @@ have state locally. In other cases such as restarts for upgrades, a snapshot download should be avoided. To avoid downloading a snapshot on restart, add the following flag to the -`solana-validator` command: +`agave-validator` command: ``` --no-snapshot-fetch ``` -If you use this flag with the `solana-validator` command, make sure that you run +If you use this flag with the `agave-validator` command, make sure that you run `solana catchup ` after your validator starts to make sure that the validator is catching up in a reasonable time. After some time (potentially a few hours), if it appears that your validator continues to fall behind, then you @@ -199,7 +199,7 @@ It is important that you do not accidentally run out of funds in your identity account, as your node will stop voting. It is also important to note that this account keypair is the most vulnerable of the three keypairs in a vote account because the keypair for the identity account is stored on your validator when -running the `solana-validator` software. How much SOL you should store there is +running the `agave-validator` software. How much SOL you should store there is up to you. As a best practice, make sure to check the account regularly and refill or deduct from it as needed. To check the account balance do: @@ -207,7 +207,7 @@ refill or deduct from it as needed. To check the account balance do: solana balance validator-keypair.json ``` -> **Note** `solana-watchtower` can monitor for a minimum validator identity +> **Note** `agave-watchtower` can monitor for a minimum validator identity > balance. See [monitoring best practices](./monitoring.md) for details. ## Withdrawing From The Vote Account diff --git a/docs/src/operations/best-practices/monitoring.md b/docs/src/operations/best-practices/monitoring.md index 6d04fc38487be7..a0f2ef9df9fa22 100644 --- a/docs/src/operations/best-practices/monitoring.md +++ b/docs/src/operations/best-practices/monitoring.md @@ -4,34 +4,34 @@ sidebar_label: Monitoring pagination_label: "Best Practices: Validator Monitoring" --- -It is essential that you have monitoring in place on your validator. In the event that your validator is delinquent (behind the rest of the network) you want to respond immediately to fix the issue. One very useful tool to monitor your validator is [`solana-watchtower`](#solana-watchtower). +It is essential that you have monitoring in place on your validator. In the event that your validator is delinquent (behind the rest of the network) you want to respond immediately to fix the issue. One very useful tool to monitor your validator is [`agave-watchtower`](#agave-watchtower). ## Solana Watchtower -Solana Watchtower is an extremely useful monitoring tool that will regularly monitor the health of your validator. It can monitor your validator for delinquency then notify you on your application of choice: Slack, Discord, Telegram or Twilio. Additionally, `solana-watchtower` has the ability to monitor the health of the entire cluster so that you can be aware of any cluster wide problems. +Solana Watchtower is an extremely useful monitoring tool that will regularly monitor the health of your validator. It can monitor your validator for delinquency then notify you on your application of choice: Slack, Discord, Telegram or Twilio. Additionally, `agave-watchtower` has the ability to monitor the health of the entire cluster so that you can be aware of any cluster wide problems. ### Getting Started -To get started with Solana Watchtower, run `solana-watchtower --help`. From the help menu, you can see the optional flags and an explanation of the command. +To get started with Solana Watchtower, run `agave-watchtower --help`. From the help menu, you can see the optional flags and an explanation of the command. Here is a sample command that will monitor a validator node with an identity public key of `2uTk98rqqwENevkPH2AHHzGHXgeGc1h6ku8hQUqWeXZp`: ``` -solana-watchtower --monitor-active-stake --validator-identity \ +agave-watchtower --monitor-active-stake --validator-identity \ 2uTk98rqqwENevkPH2AHHzGHXgeGc1h6ku8hQUqWeXZp ``` -The command will monitor your validator, but you will not get notifications unless you added the environment variables mentioned in `solana-watchtower --help`. Since getting each of these services setup for notifications is not straight forward, the next section will walk through [setting up watchtower notifications on Telegram](#setup-telegram-notifications). +The command will monitor your validator, but you will not get notifications unless you added the environment variables mentioned in `agave-watchtower --help`. Since getting each of these services setup for notifications is not straight forward, the next section will walk through [setting up watchtower notifications on Telegram](#setup-telegram-notifications). ### Best Practices -It is a best practice to run the `solana-watchtower` command on a separate server from your validator. +It is a best practice to run the `agave-watchtower` command on a separate server from your validator. -In the case that you run `solana-watchtower` on the same computer as your `solana-validator` process, then during catastrophic events like a power outage, you will not be aware of the issue, because your `solana-watchtower` process will stop at the same time as your `solana-validator` process. +In the case that you run `agave-watchtower` on the same computer as your `agave-validator` process, then during catastrophic events like a power outage, you will not be aware of the issue, because your `agave-watchtower` process will stop at the same time as your `agave-validator` process. -Additionally, while running the `solana-watchtower` process manually with environment variables set in the terminal is a good way to test out the command, it is not operationally sound because the process will not be restarted when the terminal closes or during a system restart. +Additionally, while running the `agave-watchtower` process manually with environment variables set in the terminal is a good way to test out the command, it is not operationally sound because the process will not be restarted when the terminal closes or during a system restart. -Instead, you could run your `solana-watchtower` command as a system process similar to `solana-validator`. In the system process file, you can specify the environment variables for your bot. +Instead, you could run your `agave-watchtower` command as a system process similar to `agave-validator`. In the system process file, you can specify the environment variables for your bot. ### Setup Telegram Notifications @@ -41,7 +41,7 @@ To send validator health notifications to your Telegram account, we are going to 2. Send a message to the bot 3. Create a Telegram group that will get the watchtower notifications 4. Add the environment variables to your command line environment -5. Restart the `solana-watchtower` command +5. Restart the `agave-watchtower` command #### Create a Bot Using BotFather @@ -61,7 +61,7 @@ In Telegram, click on the new message icon and then select new group. Find your Now that you have a bot setup, you will need to set the environment variables for the bot so that watchtower can send notifications. -First, recall the chat message that you got from _@BotFather_. In the message, there was an HTTP API token for your bot. The token will have this format: `389178471:MMTKMrnZB4ErUzJmuFIXTKE6DupLSgoa7h4o`. You will use that token to set the `TELEGRAM_BOT_TOKEN` environment variable. In the terminal where you plan to run `solana-watchtower`, run the following: +First, recall the chat message that you got from _@BotFather_. In the message, there was an HTTP API token for your bot. The token will have this format: `389178471:MMTKMrnZB4ErUzJmuFIXTKE6DupLSgoa7h4o`. You will use that token to set the `TELEGRAM_BOT_TOKEN` environment variable. In the terminal where you plan to run `agave-watchtower`, run the following: ``` export TELEGRAM_BOT_TOKEN= @@ -73,14 +73,14 @@ Next, in your browser, go to `https://api.telegram.org/bot/getUp The response should be in JSON. Search for the string `"chat":` in the JSON. The `id` value of that chat is your `TELEGRAM_CHAT_ID`. It will be a negative number like: `-781559558`. Remember to include the negative sign! If you cannot find `"chat":` in the JSON, then you may have to remove the bot from your chat group and add it again. -With your Telegram chat id in hand, export the environment variable where you plan to run `solana-watchtower`: +With your Telegram chat id in hand, export the environment variable where you plan to run `agave-watchtower`: ``` export TELEGRAM_CHAT_ID= ``` -#### Restart solana-watchtower +#### Restart agave-watchtower -Once your environment variables are set, restart `solana-watchtower`. You should see output about your validator. +Once your environment variables are set, restart `agave-watchtower`. You should see output about your validator. To test that your Telegram configuration is working properly, you could stop your validator briefly until it is labeled as delinquent. Up to a minute after the validator is delinquent, you should receive a message in the Telegram group from your bot. Start the validator again and verify that you get another message in your Telegram group from the bot. The message should say `all clear`. \ No newline at end of file diff --git a/docs/src/operations/guides/restart-cluster.md b/docs/src/operations/guides/restart-cluster.md index 85d4731d604c65..cda3f30a5a016d 100644 --- a/docs/src/operations/guides/restart-cluster.md +++ b/docs/src/operations/guides/restart-cluster.md @@ -11,7 +11,7 @@ pagination_label: "Validator Guides: Restart a Cluster" In Solana 1.14 or greater, run the following command to output the latest optimistically confirmed slot your validator observed: ```bash -solana-ledger-tool -l ledger latest-optimistic-slots +agave-ledger-tool -l ledger latest-optimistic-slots ``` In Solana 1.13 or less, the latest optimistically confirmed can be found by looking for the more recent occurrence of @@ -34,11 +34,11 @@ instead. ### Step 4. Create a new snapshot for slot `SLOT_X` with a hard fork at slot `SLOT_X` ```bash -$ solana-ledger-tool -l --snapshot-archive-path --incremental-snapshot-archive-path create-snapshot SLOT_X --hard-fork SLOT_X +$ agave-ledger-tool -l --snapshot-archive-path --incremental-snapshot-archive-path create-snapshot SLOT_X --hard-fork SLOT_X ``` The snapshots directory should now contain the new snapshot. -`solana-ledger-tool create-snapshot` will also output the new shred version, and bank hash value, +`agave-ledger-tool create-snapshot` will also output the new shred version, and bank hash value, call this NEW_SHRED_VERSION and NEW_BANK_HASH respectively. Adjust your validator's arguments: @@ -68,7 +68,7 @@ Post something like the following to #announcements (adjusting the text as appro > 2. a. Preferred method, start from your local ledger with: > > ```bash -> solana-validator +> agave-validator > --wait-for-supermajority SLOT_X # <-- NEW! IMPORTANT! REMOVE AFTER THIS RESTART > --expected-bank-hash NEW_BANK_HASH # <-- NEW! IMPORTANT! REMOVE AFTER THIS RESTART > --hard-fork SLOT_X # <-- NEW! IMPORTANT! REMOVE AFTER THIS RESTART @@ -84,7 +84,7 @@ Post something like the following to #announcements (adjusting the text as appro > b. If your validator doesn't have ledger up to slot SLOT_X or if you have deleted your ledger, have it instead download a snapshot with: > > ```bash -> solana-validator +> agave-validator > --wait-for-supermajority SLOT_X # <-- NEW! IMPORTANT! REMOVE AFTER THIS RESTART > --expected-bank-hash NEW_BANK_HASH # <-- NEW! IMPORTANT! REMOVE AFTER THIS RESTART > --entrypoint entrypoint.testnet.solana.com:8001 @@ -95,7 +95,7 @@ Post something like the following to #announcements (adjusting the text as appro > ... # <-- your other --identity/--vote-account/etc arguments > ``` > -> You can check for which slots your ledger has with: `solana-ledger-tool -l path/to/ledger bounds` +> You can check for which slots your ledger has with: `agave-ledger-tool -l path/to/ledger bounds` > > 3. Wait until 80% of the stake comes online > @@ -122,7 +122,7 @@ and create a new snapshot with additional `--destake-vote-account ` arguments for each of the non-responsive validator's vote account address ```bash -$ solana-ledger-tool -l ledger create-snapshot SLOT_X ledger --hard-fork SLOT_X \ +$ agave-ledger-tool -l ledger create-snapshot SLOT_X ledger --hard-fork SLOT_X \ --destake-vote-account \ --destake-vote-account \ . diff --git a/docs/src/operations/guides/validator-failover.md b/docs/src/operations/guides/validator-failover.md index 168a1a4312cec0..b7b3fea568194b 100644 --- a/docs/src/operations/guides/validator-failover.md +++ b/docs/src/operations/guides/validator-failover.md @@ -85,11 +85,11 @@ For more information on etcd TLS setup, please refer to https://etcd.io/docs/v3.5/op-guide/security/#example-2-client-to-server-authentication-with-https-client-certificates ### Primary Validator -The following additional `solana-validator` parameters are required to enable +The following additional `agave-validator` parameters are required to enable tower storage into etcd: ``` -solana-validator ... \ +agave-validator ... \ --tower-storage etcd \ --etcd-cacert-file certs/etcd-ca.pem \ --etcd-cert-file certs/validator.pem \ @@ -103,7 +103,7 @@ that your etcd endpoint remain accessible at all times. ### Secondary Validator Configure the secondary validator like the primary with the exception of the -following `solana-validator` command-line argument changes: +following `agave-validator` command-line argument changes: * Generate and use a secondary validator identity: `--identity secondary-validator-keypair.json` * Add `--no-check-vote-account` * Add `--authorized-voter validator-keypair.json` (where @@ -114,8 +114,8 @@ When both validators are running normally and caught up to the cluster, a failover from primary to secondary can be triggered by running the following command on the secondary validator: ```bash -$ solana-validator wait-for-restart-window --identity validator-keypair.json \ - && solana-validator set-identity validator-keypair.json +$ agave-validator wait-for-restart-window --identity validator-keypair.json \ + && agave-validator set-identity validator-keypair.json ``` The secondary validator will acquire a lock on the tower in etcd to ensure @@ -131,7 +131,7 @@ exit. However if/when the secondary validator restarts, it will do so using the secondary validator identity and thus the restart cycle is broken. ## Triggering a failover via monitoring -Monitoring of your choosing can invoke the `solana-validator set-identity +Monitoring of your choosing can invoke the `agave-validator set-identity validator-keypair.json` command mentioned in the previous section. It is not necessary to guarantee the primary validator has halted before failing diff --git a/docs/src/operations/guides/validator-start.md b/docs/src/operations/guides/validator-start.md index 378783798b3ce8..d86c714be4e6a6 100644 --- a/docs/src/operations/guides/validator-start.md +++ b/docs/src/operations/guides/validator-start.md @@ -32,7 +32,7 @@ detail on cluster activity. ## Enabling CUDA If your machine has a GPU with CUDA installed \(Linux-only currently\), include -the `--cuda` argument to `solana-validator`. +the `--cuda` argument to `agave-validator`. When your validator is started look for the following log message to indicate that CUDA is enabled: `"[ solana::validator] CUDA is enabled"` @@ -47,7 +47,7 @@ the following commands. #### **Optimize sysctl knobs** ```bash -sudo bash -c "cat >/etc/sysctl.d/21-solana-validator.conf </etc/sysctl.d/21-agave-validator.conf <` -argument to `solana-validator`. You can specify multiple ones by repeating the argument `--known-validator --known-validator `. +argument to `agave-validator`. You can specify multiple ones by repeating the argument `--known-validator --known-validator `. This has two effects, one is when the validator is booting with `--only-known-rpc`, it will only ask that set of known nodes for downloading genesis and snapshot data. Another is that in combination with the `--halt-on-known-validators-accounts-hash-mismatch` option, it will monitor the merkle root hash of the entire accounts state of other known nodes on gossip and if the hashes produce any mismatch, @@ -277,13 +277,13 @@ account state divergence. Connect to the cluster by running: ```bash -solana-validator \ +agave-validator \ --identity ~/validator-keypair.json \ --vote-account ~/vote-account-keypair.json \ --rpc-port 8899 \ --entrypoint entrypoint.devnet.solana.com:8001 \ --limit-ledger-size \ - --log ~/solana-validator.log + --log ~/agave-validator.log ``` To force validator logging to the console add a `--log -` argument, otherwise @@ -296,7 +296,7 @@ The ledger will be placed in the `ledger/` directory by default, use the > [paper wallet seed phrase](../../cli/wallets/paper.md) > for your `--identity` and/or > `--authorized-voter` keypairs. To use these, pass the respective argument as -> `solana-validator --identity ASK ... --authorized-voter ASK ...` +> `agave-validator --identity ASK ... --authorized-voter ASK ...` > and you will be prompted to enter your seed phrases and optional passphrase. Confirm your validator is connected to the network by opening a new terminal and @@ -312,7 +312,7 @@ If your validator is connected, its public key and IP address will appear in the By default the validator will dynamically select available network ports in the 8000-10000 range, and may be overridden with `--dynamic-port-range`. For -example, `solana-validator --dynamic-port-range 11000-11020 ...` will restrict +example, `agave-validator --dynamic-port-range 11000-11020 ...` will restrict the validator to ports 11000-11020. ### Limiting ledger size to conserve disk space @@ -366,8 +366,8 @@ WantedBy=multi-user.target ``` Now create `/home/sol/bin/validator.sh` to include the desired -`solana-validator` command-line. Ensure that the 'exec' command is used to -start the validator process (i.e. "exec solana-validator ..."). This is +`agave-validator` command-line. Ensure that the 'exec' command is used to +start the validator process (i.e. "exec agave-validator ..."). This is important because without it, logrotate will end up killing the validator every time the logs are rotated. @@ -394,14 +394,14 @@ to be reverted and the issue reproduced before help can be provided. #### Log rotation -The validator log file, as specified by `--log ~/solana-validator.log`, can get +The validator log file, as specified by `--log ~/agave-validator.log`, can get very large over time and it's recommended that log rotation be configured. The validator will re-open its log file when it receives the `USR1` signal, which is the basic primitive that enables log rotation. If the validator is being started by a wrapper shell script, it is important to -launch the process with `exec` (`exec solana-validator ...`) when using logrotate. +launch the process with `exec` (`exec agave-validator ...`) when using logrotate. This will prevent the `USR1` signal from being sent to the script's process instead of the validator's, which will kill them both. @@ -409,13 +409,13 @@ instead of the validator's, which will kill them both. An example setup for the `logrotate`, which assumes that the validator is running as a systemd service called `sol.service` and writes a log file at -/home/sol/solana-validator.log: +/home/sol/agave-validator.log: ```bash # Setup log rotation cat > logrotate.sol </etc/sysctl.d/21-solana-validator.conf </etc/sysctl.d/21-agave-validator.conf < For more explanation on the flags used in the command, refer to the `solana-validator --help` command +> For more explanation on the flags used in the command, refer to the `agave-validator --help` command ``` #!/bin/bash -exec solana-validator \ +exec agave-validator \ --identity /home/sol/validator-keypair.json \ --known-validator 5D1fNXzvv5NjV1ysLjirC4WY92RNsVH18vjmcszZd8on \ --known-validator dDzy5SR3AXdYWVqbDEkVFdvSPCtS9ihF5kJkHCtXoFs \ diff --git a/docs/src/validator/geyser.md b/docs/src/validator/geyser.md index 769856303767d6..efea2e18e30269 100644 --- a/docs/src/validator/geyser.md +++ b/docs/src/validator/geyser.md @@ -24,20 +24,20 @@ implementation for the PostgreSQL database. ### Important Crates: -- [`solana-geyser-plugin-interface`] — This crate defines the plugin +- [`agave-geyser-plugin-interface`] — This crate defines the plugin interfaces. - [`solana-accountsdb-plugin-postgres`] — The crate for the referential plugin implementation for the PostgreSQL database. -[`solana-geyser-plugin-interface`]: https://docs.rs/solana-geyser-plugin-interface +[`agave-geyser-plugin-interface`]: https://docs.rs/agave-geyser-plugin-interface [`solana-accountsdb-plugin-postgres`]: https://docs.rs/solana-accountsdb-plugin-postgres [`solana-sdk`]: https://docs.rs/solana-sdk [`solana-transaction-status`]: https://docs.rs/solana-transaction-status ## The Plugin Interface -The Plugin interface is declared in [`solana-geyser-plugin-interface`]. It +The Plugin interface is declared in [`agave-geyser-plugin-interface`]. It is defined by the trait `GeyserPlugin`. The plugin should implement the trait and expose a "C" function `_create_plugin` to return the pointer to this trait. For example, in the referential implementation, the following code @@ -166,7 +166,7 @@ please refer to [`solana-sdk`] and [`solana-transaction-status`] The `slot` points to the slot the transaction is executed at. For more details, please refer to the Rust documentation in -[`solana-geyser-plugin-interface`]. +[`agave-geyser-plugin-interface`]. ## Example PostgreSQL Plugin diff --git a/geyser-plugin-interface/Cargo.toml b/geyser-plugin-interface/Cargo.toml index af99758b47d630..56f42fd4612cec 100644 --- a/geyser-plugin-interface/Cargo.toml +++ b/geyser-plugin-interface/Cargo.toml @@ -1,7 +1,7 @@ [package] -name = "solana-geyser-plugin-interface" +name = "agave-geyser-plugin-interface" description = "The Solana Geyser plugin interface." -documentation = "https://docs.rs/solana-geyser-plugin-interface" +documentation = "https://docs.rs/agave-geyser-plugin-interface" version = { workspace = true } authors = { workspace = true } repository = { workspace = true } diff --git a/geyser-plugin-interface/src/geyser_plugin_interface.rs b/geyser-plugin-interface/src/geyser_plugin_interface.rs index 037aedf8b87e89..d9a3b00f8dc4c8 100644 --- a/geyser-plugin-interface/src/geyser_plugin_interface.rs +++ b/geyser-plugin-interface/src/geyser_plugin_interface.rs @@ -327,7 +327,7 @@ pub trait GeyserPlugin: Any + Send + Sync + std::fmt::Debug { /// # Examples /// /// ``` - /// use solana_geyser_plugin_interface::geyser_plugin_interface::{GeyserPlugin, + /// use agave_geyser_plugin_interface::geyser_plugin_interface::{GeyserPlugin, /// GeyserPluginError, Result}; /// /// #[derive(Debug)] diff --git a/geyser-plugin-manager/Cargo.toml b/geyser-plugin-manager/Cargo.toml index d905248150b717..ebef2f637f642d 100644 --- a/geyser-plugin-manager/Cargo.toml +++ b/geyser-plugin-manager/Cargo.toml @@ -10,6 +10,7 @@ license = { workspace = true } edition = { workspace = true } [dependencies] +agave-geyser-plugin-interface = { workspace = true } bs58 = { workspace = true } crossbeam-channel = { workspace = true } json5 = { workspace = true } @@ -20,7 +21,6 @@ log = { workspace = true } serde_json = { workspace = true } solana-accounts-db = { workspace = true } solana-entry = { workspace = true } -solana-geyser-plugin-interface = { workspace = true } solana-ledger = { workspace = true } solana-measure = { workspace = true } solana-metrics = { workspace = true } diff --git a/geyser-plugin-manager/src/accounts_update_notifier.rs b/geyser-plugin-manager/src/accounts_update_notifier.rs index 7c7e3370fc00eb..90ab0b7998a35c 100644 --- a/geyser-plugin-manager/src/accounts_update_notifier.rs +++ b/geyser-plugin-manager/src/accounts_update_notifier.rs @@ -1,14 +1,14 @@ /// Module responsible for notifying plugins of account updates use { crate::geyser_plugin_manager::GeyserPluginManager, + agave_geyser_plugin_interface::geyser_plugin_interface::{ + ReplicaAccountInfoV3, ReplicaAccountInfoVersions, + }, log::*, solana_accounts_db::{ account_storage::meta::StoredAccountMeta, accounts_update_notifier_interface::AccountsUpdateNotifierInterface, }, - solana_geyser_plugin_interface::geyser_plugin_interface::{ - ReplicaAccountInfoV3, ReplicaAccountInfoVersions, - }, solana_measure::measure::Measure, solana_metrics::*, solana_sdk::{ diff --git a/geyser-plugin-manager/src/block_metadata_notifier.rs b/geyser-plugin-manager/src/block_metadata_notifier.rs index 76d203c5e0ed44..87f15f41fc0ae0 100644 --- a/geyser-plugin-manager/src/block_metadata_notifier.rs +++ b/geyser-plugin-manager/src/block_metadata_notifier.rs @@ -3,10 +3,10 @@ use { block_metadata_notifier_interface::BlockMetadataNotifier, geyser_plugin_manager::GeyserPluginManager, }, - log::*, - solana_geyser_plugin_interface::geyser_plugin_interface::{ + agave_geyser_plugin_interface::geyser_plugin_interface::{ ReplicaBlockInfoV3, ReplicaBlockInfoVersions, }, + log::*, solana_measure::measure::Measure, solana_metrics::*, solana_sdk::{clock::UnixTimestamp, pubkey::Pubkey, reward_info::RewardInfo}, diff --git a/geyser-plugin-manager/src/entry_notifier.rs b/geyser-plugin-manager/src/entry_notifier.rs index ea14592b615db8..da9a9698ed1540 100644 --- a/geyser-plugin-manager/src/entry_notifier.rs +++ b/geyser-plugin-manager/src/entry_notifier.rs @@ -1,11 +1,11 @@ /// Module responsible for notifying plugins about entries use { crate::geyser_plugin_manager::GeyserPluginManager, - log::*, - solana_entry::entry::EntrySummary, - solana_geyser_plugin_interface::geyser_plugin_interface::{ + agave_geyser_plugin_interface::geyser_plugin_interface::{ ReplicaEntryInfoV2, ReplicaEntryInfoVersions, }, + log::*, + solana_entry::entry::EntrySummary, solana_ledger::entry_notifier_interface::EntryNotifier, solana_measure::measure::Measure, solana_metrics::*, diff --git a/geyser-plugin-manager/src/geyser_plugin_manager.rs b/geyser-plugin-manager/src/geyser_plugin_manager.rs index a15f9e1318075d..3d0abe16899637 100644 --- a/geyser-plugin-manager/src/geyser_plugin_manager.rs +++ b/geyser-plugin-manager/src/geyser_plugin_manager.rs @@ -1,9 +1,9 @@ use { + agave_geyser_plugin_interface::geyser_plugin_interface::GeyserPlugin, jsonrpc_core::{ErrorCode, Result as JsonRpcResult}, jsonrpc_server_utils::tokio::sync::oneshot::Sender as OneShotSender, libloading::Library, log::*, - solana_geyser_plugin_interface::geyser_plugin_interface::GeyserPlugin, std::{ ops::{Deref, DerefMut}, path::Path, @@ -442,8 +442,8 @@ mod tests { crate::geyser_plugin_manager::{ GeyserPluginManager, LoadedGeyserPlugin, TESTPLUGIN2_CONFIG, TESTPLUGIN_CONFIG, }, + agave_geyser_plugin_interface::geyser_plugin_interface::GeyserPlugin, libloading::Library, - solana_geyser_plugin_interface::geyser_plugin_interface::GeyserPlugin, std::sync::{Arc, RwLock}, }; diff --git a/geyser-plugin-manager/src/slot_status_notifier.rs b/geyser-plugin-manager/src/slot_status_notifier.rs index 587abe2f79d4de..1557bb2d4d8c36 100644 --- a/geyser-plugin-manager/src/slot_status_notifier.rs +++ b/geyser-plugin-manager/src/slot_status_notifier.rs @@ -1,7 +1,7 @@ use { crate::geyser_plugin_manager::GeyserPluginManager, + agave_geyser_plugin_interface::geyser_plugin_interface::SlotStatus, log::*, - solana_geyser_plugin_interface::geyser_plugin_interface::SlotStatus, solana_measure::measure::Measure, solana_metrics::*, solana_sdk::clock::Slot, diff --git a/geyser-plugin-manager/src/transaction_notifier.rs b/geyser-plugin-manager/src/transaction_notifier.rs index ab821e811047d2..b757c1202b377d 100644 --- a/geyser-plugin-manager/src/transaction_notifier.rs +++ b/geyser-plugin-manager/src/transaction_notifier.rs @@ -1,10 +1,10 @@ /// Module responsible for notifying plugins of transactions use { crate::geyser_plugin_manager::GeyserPluginManager, - log::*, - solana_geyser_plugin_interface::geyser_plugin_interface::{ + agave_geyser_plugin_interface::geyser_plugin_interface::{ ReplicaTransactionInfoV2, ReplicaTransactionInfoVersions, }, + log::*, solana_measure::measure::Measure, solana_metrics::*, solana_rpc::transaction_notifier_interface::TransactionNotifier, diff --git a/install/Cargo.toml b/install/Cargo.toml index 588d4315df5f35..c40a0ee6e9eee3 100644 --- a/install/Cargo.toml +++ b/install/Cargo.toml @@ -1,7 +1,7 @@ [package] -name = "solana-install" +name = "agave-install" description = "The solana cluster software installer" -documentation = "https://docs.rs/solana-install" +documentation = "https://docs.rs/agave-install" version = { workspace = true } authors = { workspace = true } repository = { workspace = true } diff --git a/install/solana-install-init.sh b/install/agave-install-init.sh similarity index 89% rename from install/solana-install-init.sh rename to install/agave-install-init.sh index 4f28e300be52ab..cf2d1babf3c306 100755 --- a/install/solana-install-init.sh +++ b/install/agave-install-init.sh @@ -10,7 +10,7 @@ # except according to those terms. # This is just a little script that can be downloaded from the internet to -# install solana-install. It just does platform detection, downloads the installer +# install agave-install. It just does platform detection, downloads the installer # and runs it. { # this ensures the entire script is downloaded # @@ -24,11 +24,11 @@ set -e usage() { cat 1>&2 < --pubkey + agave-install-init [FLAGS] [OPTIONS] --data_dir --pubkey FLAGS: -h, --help Prints help information @@ -81,7 +81,7 @@ main() { esac TARGET="${_cputype}-${_ostype}" - temp_dir="$(mktemp -d 2>/dev/null || ensure mktemp -d -t solana-install-init)" + temp_dir="$(mktemp -d 2>/dev/null || ensure mktemp -d -t agave-install-init)" ensure mkdir -p "$temp_dir" # Check for SOLANA_RELEASE environment variable override. Otherwise fetch @@ -101,8 +101,8 @@ main() { fi fi - download_url="$SOLANA_DOWNLOAD_ROOT/$release/solana-install-init-$TARGET" - solana_install_init="$temp_dir/solana-install-init" + download_url="$SOLANA_DOWNLOAD_ROOT/$release/agave-install-init-$TARGET" + solana_install_init="$temp_dir/agave-install-init" printf 'downloading %s installer\n' "$release" 1>&2 @@ -111,7 +111,7 @@ main() { ensure chmod u+x "$solana_install_init" if [ ! -x "$solana_install_init" ]; then printf '%s\n' "Cannot execute $solana_install_init (likely because of mounting /tmp as noexec)." 1>&2 - printf '%s\n' "Please copy the file to a location where you can execute binaries and run ./solana-install-init." 1>&2 + printf '%s\n' "Please copy the file to a location where you can execute binaries and run ./agave-install-init." 1>&2 exit 1 fi @@ -130,7 +130,7 @@ main() { } err() { - printf 'solana-install-init: %s\n' "$1" >&2 + printf 'agave-install-init: %s\n' "$1" >&2 exit 1 } diff --git a/install/install-help.sh b/install/install-help.sh index 9fb08afa6d14c9..7604777e378677 100755 --- a/install/install-help.sh +++ b/install/install-help.sh @@ -4,11 +4,11 @@ set -e cd "$(dirname "$0")"/.. cargo="$(readlink -f "./cargo")" -"$cargo" build --package solana-install +"$cargo" build --package agave-install export PATH=$PWD/target/debug:$PATH echo "\`\`\`manpage" -solana-install --help +agave-install --help echo "\`\`\`" echo "" @@ -16,7 +16,7 @@ commands=(init info deploy update run) for x in "${commands[@]}"; do echo "\`\`\`manpage" - solana-install "${x}" --help + agave-install "${x}" --help echo "\`\`\`" echo "" done diff --git a/install/src/bin/solana-install-init.rs b/install/src/bin/agave-install-init.rs similarity index 92% rename from install/src/bin/solana-install-init.rs rename to install/src/bin/agave-install-init.rs index ec888d8f452090..84c154ac12b35e 100644 --- a/install/src/bin/solana-install-init.rs +++ b/install/src/bin/agave-install-init.rs @@ -16,7 +16,7 @@ fn press_enter() { } fn main() { - solana_install::main_init().unwrap_or_else(|err| { + agave_install::main_init().unwrap_or_else(|err| { println!("Error: {err}"); press_enter(); exit(1); diff --git a/install/src/command.rs b/install/src/command.rs index 218e815467e9a9..4ae9e7ee38cedd 100644 --- a/install/src/command.rs +++ b/install/src/command.rs @@ -540,7 +540,7 @@ pub fn init( explicit_release: Option, ) -> Result<(), String> { let config = { - // Write new config file only if different, so that running |solana-install init| + // Write new config file only if different, so that running |agave-install init| // repeatedly doesn't unnecessarily re-download let mut current_config = Config::load(config_file).unwrap_or_default(); current_config.current_update_manifest = None; @@ -870,7 +870,7 @@ fn check_for_newer_github_release( prerelease_allowed: bool, ) -> Result, String> { let client = reqwest::blocking::Client::builder() - .user_agent("solana-install") + .user_agent("agave-install") .build() .map_err(|err| err.to_string())?; diff --git a/install/src/lib.rs b/install/src/lib.rs index 159317edd2e5a8..a28b963d65f825 100644 --- a/install/src/lib.rs +++ b/install/src/lib.rs @@ -281,7 +281,7 @@ pub fn main() -> Result<(), String> { pub fn main_init() -> Result<(), String> { solana_logger::setup(); - let matches = App::new("solana-install-init") + let matches = App::new("agave-install-init") .about("Initializes a new installation") .version(solana_version::version!()) .arg({ diff --git a/install/src/main.rs b/install/src/main.rs index c7b15aa6a67206..245f09825ddc6a 100644 --- a/install/src/main.rs +++ b/install/src/main.rs @@ -1,3 +1,3 @@ fn main() -> Result<(), String> { - solana_install::main() + agave_install::main() } diff --git a/ledger-tool/Cargo.toml b/ledger-tool/Cargo.toml index 6da42940a4ba7f..cb87a0e16f4a36 100644 --- a/ledger-tool/Cargo.toml +++ b/ledger-tool/Cargo.toml @@ -1,7 +1,7 @@ [package] -name = "solana-ledger-tool" +name = "agave-ledger-tool" description = "Blockchain, Rebuilt for Scale" -documentation = "https://docs.rs/solana-ledger-tool" +documentation = "https://docs.rs/agave-ledger-tool" version = { workspace = true } authors = { workspace = true } repository = { workspace = true } diff --git a/ledger-tool/src/blockstore.rs b/ledger-tool/src/blockstore.rs index 453a801702f864..fed6abde2f2d08 100644 --- a/ledger-tool/src/blockstore.rs +++ b/ledger-tool/src/blockstore.rs @@ -359,7 +359,7 @@ pub fn blockstore_subcommands<'a, 'b>(hidden: bool) -> Vec> { and timestamps.", ) // This command is important in cluster restart scenarios, so do not hide it ever - // such that the subcommand will be visible as the top level of solana-ledger-tool + // such that the subcommand will be visible as the top level of agave-ledger-tool .arg( Arg::with_name("num_slots") .long("num-slots") diff --git a/ledger-tool/src/ledger_utils.rs b/ledger-tool/src/ledger_utils.rs index 116b21527ae4d8..c05cc6c2d64cd0 100644 --- a/ledger-tool/src/ledger_utils.rs +++ b/ledger-tool/src/ledger_utils.rs @@ -187,14 +187,14 @@ pub fn load_and_process_ledger( } let account_paths = if let Some(account_paths) = arg_matches.value_of("account_paths") { - // If this blockstore access is Primary, no other process (solana-validator) can hold + // If this blockstore access is Primary, no other process (agave-validator) can hold // Primary access. So, allow a custom accounts path without worry of wiping the accounts - // of solana-validator. + // of agave-validator. if !blockstore.is_primary_access() { // Attempt to open the Blockstore in Primary access; if successful, no other process // was holding Primary so allow things to proceed with custom accounts path. Release - // the Primary access instead of holding it to give priority to solana-validator over - // solana-ledger-tool should solana-validator start before we've finished. + // the Primary access instead of holding it to give priority to agave-validator over + // agave-ledger-tool should agave-validator start before we've finished. info!( "Checking if another process currently holding Primary access to {:?}", blockstore.ledger_path() diff --git a/ledger/src/blockstore_db.rs b/ledger/src/blockstore_db.rs index 18ba491ea34bd1..8b6b44edae61f6 100644 --- a/ledger/src/blockstore_db.rs +++ b/ledger/src/blockstore_db.rs @@ -431,7 +431,7 @@ impl Rocks { info!( "Opening Rocks with secondary (read only) access at: {secondary_path:?}. \ This secondary access could temporarily degrade other accesses, such as \ - by solana-validator" + by agave-validator" ); DB::open_cf_descriptors_as_secondary( &db_options, diff --git a/local-cluster/tests/local_cluster.rs b/local-cluster/tests/local_cluster.rs index 3b18ba44bf2d03..20eef0bb0e3e2d 100644 --- a/local-cluster/tests/local_cluster.rs +++ b/local-cluster/tests/local_cluster.rs @@ -2321,13 +2321,13 @@ fn test_hard_fork_with_gap_in_roots() { ); // create hard-forked snapshot only for validator a, emulating the manual cluster restart - // procedure with `solana-ledger-tool create-snapshot` + // procedure with `agave-ledger-tool create-snapshot` let genesis_slot = 0; { let blockstore_a = Blockstore::open(&val_a_ledger_path).unwrap(); create_snapshot_to_hard_fork(&blockstore_a, hard_fork_slot, vec![hard_fork_slot]); - // Intentionally make solana-validator unbootable by replaying blocks from the genesis to + // Intentionally make agave-validator unbootable by replaying blocks from the genesis to // ensure the hard-forked snapshot is used always. Otherwise, we couldn't create a gap // in the ledger roots column family reliably. // There was a bug which caused the hard-forked snapshot at an unrooted slot to forget diff --git a/multinode-demo/bootstrap-validator.sh b/multinode-demo/bootstrap-validator.sh index 5afc543b2f0032..2872af5cc426af 100755 --- a/multinode-demo/bootstrap-validator.sh +++ b/multinode-demo/bootstrap-validator.sh @@ -14,9 +14,9 @@ if [[ "$SOLANA_GPU_MISSING" -eq 1 ]]; then fi if [[ -n $SOLANA_CUDA ]]; then - program=$solana_validator_cuda + program=$agave_validator_cuda else - program=$solana_validator + program=$agave_validator fi no_restart=0 diff --git a/multinode-demo/common.sh b/multinode-demo/common.sh index 9ae9331cb7a11d..1643208947b643 100644 --- a/multinode-demo/common.sh +++ b/multinode-demo/common.sh @@ -40,6 +40,8 @@ else if [[ -z $program ]]; then crate="cli" program="solana" + elif [[ $program == "validator" || $program == "ledger-tool" || $program == "watchtower" || $program == "install" ]]; then + program="agave-$program" else program="solana-$program" fi @@ -63,8 +65,8 @@ fi solana_bench_tps=$(solana_program bench-tps) solana_faucet=$(solana_program faucet) -solana_validator=$(solana_program validator) -solana_validator_cuda="$solana_validator --cuda" +agave_validator=$(solana_program validator) +agave_validator_cuda="$agave_validator --cuda" solana_genesis=$(solana_program genesis) solana_gossip=$(solana_program gossip) solana_keygen=$(solana_program keygen) diff --git a/multinode-demo/validator.sh b/multinode-demo/validator.sh index 487154101ac979..efb7a6afd56ea0 100755 --- a/multinode-demo/validator.sh +++ b/multinode-demo/validator.sh @@ -64,7 +64,7 @@ while [[ -n $1 ]]; do elif [[ $1 = --no-airdrop ]]; then airdrops_enabled=0 shift - # solana-validator options + # agave-validator options elif [[ $1 = --expected-genesis-hash ]]; then args+=("$1" "$2") shift 2 @@ -270,9 +270,9 @@ if [[ $maybeRequireTower = true ]]; then fi if [[ -n $SOLANA_CUDA ]]; then - program=$solana_validator_cuda + program=$agave_validator_cuda else - program=$solana_validator + program=$agave_validator fi set -e diff --git a/net/net.sh b/net/net.sh index a2d16cef20f417..36bc48efdb7861 100755 --- a/net/net.sh +++ b/net/net.sh @@ -122,7 +122,7 @@ Operate a configured testnet sanity/start-specific options: -F - Discard validator nodes that didn't bootup successfully - -o noInstallCheck - Skip solana-install sanity + -o noInstallCheck - Skip agave-install sanity -o rejectExtraNodes - Require the exact number of nodes stop-specific options: @@ -138,7 +138,7 @@ Operate a configured testnet --netem-cmd - Optional command argument to netem. Default is "add". Use "cleanup" to remove rules. update-specific options: - --platform linux|osx|windows - Deploy the tarball using 'solana-install deploy ...' for the + --platform linux|osx|windows - Deploy the tarball using 'agave-install deploy ...' for the given platform (multiple platforms may be specified) (-t option must be supplied as well) @@ -514,11 +514,11 @@ deployUpdate() { declare bootstrapLeader=${validatorIpList[0]} for updatePlatform in $updatePlatforms; do - echo "--- Deploying solana-install update: $updatePlatform" + echo "--- Deploying agave-install update: $updatePlatform" ( set -x - scripts/solana-install-update-manifest-keypair.sh "$updatePlatform" + scripts/agave-install-update-manifest-keypair.sh "$updatePlatform" timeout 30s scp "${sshOptions[@]}" \ update_manifest_keypair.json "$bootstrapLeader:solana/update_manifest_keypair.json" diff --git a/net/remote/remote-deploy-update.sh b/net/remote/remote-deploy-update.sh index dd772927c0e119..3a71cf5725123e 100755 --- a/net/remote/remote-deploy-update.sh +++ b/net/remote/remote-deploy-update.sh @@ -35,6 +35,6 @@ loadConfigFile PATH="$HOME"/.cargo/bin:"$PATH" set -x -scripts/solana-install-deploy.sh \ +scripts/agave-install-deploy.sh \ --keypair config/faucet.json \ localhost "$releaseChannel" "$updatePlatform" diff --git a/net/remote/remote-node.sh b/net/remote/remote-node.sh index aeb920bd50bab0..b7d224088da9f9 100755 --- a/net/remote/remote-node.sh +++ b/net/remote/remote-node.sh @@ -121,7 +121,7 @@ cat >> ~/solana/on-reboot < system-stats.pid if ${GPU_CUDA_OK} && [[ -e /dev/nvidia0 ]]; then - echo Selecting solana-validator-cuda + echo Selecting agave-validator-cuda export SOLANA_CUDA=1 elif ${GPU_FAIL_IF_NONE} ; then echo "Expected GPU, found none!" @@ -257,13 +257,13 @@ EOF if [[ -n "$maybeWarpSlot" ]]; then # shellcheck disable=SC2086 # Do not want to quote $maybeWarSlot - solana-ledger-tool -l config/bootstrap-validator create-snapshot 0 config/bootstrap-validator $maybeWarpSlot + agave-ledger-tool -l config/bootstrap-validator create-snapshot 0 config/bootstrap-validator $maybeWarpSlot fi - solana-ledger-tool -l config/bootstrap-validator shred-version --max-genesis-archive-unpacked-size 1073741824 | tee config/shred-version + agave-ledger-tool -l config/bootstrap-validator shred-version --max-genesis-archive-unpacked-size 1073741824 | tee config/shred-version if [[ -n "$maybeWaitForSupermajority" ]]; then - bankHash=$(solana-ledger-tool -l config/bootstrap-validator bank-hash --halt-at-slot 0) + bankHash=$(agave-ledger-tool -l config/bootstrap-validator bank-hash --halt-at-slot 0) extraNodeArgs="$extraNodeArgs --expected-bank-hash $bankHash" echo "$bankHash" > config/bank-hash fi diff --git a/net/remote/remote-sanity.sh b/net/remote/remote-sanity.sh index 8c36e99ffdf936..91dae4b57336fa 100755 --- a/net/remote/remote-sanity.sh +++ b/net/remote/remote-sanity.sh @@ -65,7 +65,7 @@ local|tar|skip) export USE_INSTALL=1 solana_cli=solana solana_gossip=solana-gossip - solana_install=solana-install + solana_install=agave-install ;; *) echo "Unknown deployment method: $deployMethod" @@ -122,7 +122,7 @@ else fi if $installCheck && [[ -r update_manifest_keypair.json ]]; then - echo "--- $sanityTargetIp: solana-install test" + echo "--- $sanityTargetIp: agave-install test" ( set -x diff --git a/notifier/src/lib.rs b/notifier/src/lib.rs index a369225772492c..75406d2fbdae33 100644 --- a/notifier/src/lib.rs +++ b/notifier/src/lib.rs @@ -19,7 +19,7 @@ /// /// To receive a Twilio SMS notification on failure, having a Twilio account, /// and a sending number owned by that account, -/// define environment variable before running `solana-watchtower`: +/// define environment variable before running `agave-watchtower`: /// ```bash /// export TWILIO_CONFIG='ACCOUNT=,TOKEN=,TO=,FROM=' /// ``` @@ -208,7 +208,7 @@ impl Notifier { NotificationType::Resolve { ref incident } => incident.clone().to_string(), }; - let data = json!({"payload":{"summary":msg,"source":"solana-watchtower","severity":"critical"},"routing_key":routing_key,"event_action":event_action,"dedup_key":dedup_key}); + let data = json!({"payload":{"summary":msg,"source":"agave-watchtower","severity":"critical"},"routing_key":routing_key,"event_action":event_action,"dedup_key":dedup_key}); let url = "https://events.pagerduty.com/v2/enqueue"; if let Err(err) = self.client.post(url).json(&data).send() { diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index 2829cf27b6da6f..7546c56bd2b26a 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -63,6 +63,80 @@ dependencies = [ "zeroize", ] +[[package]] +name = "agave-geyser-plugin-interface" +version = "1.19.0" +dependencies = [ + "log", + "solana-sdk", + "solana-transaction-status", + "thiserror", +] + +[[package]] +name = "agave-validator" +version = "1.19.0" +dependencies = [ + "agave-geyser-plugin-interface", + "chrono", + "clap 2.33.3", + "console", + "core_affinity", + "crossbeam-channel", + "fd-lock", + "indicatif", + "itertools", + "jsonrpc-core", + "jsonrpc-core-client", + "jsonrpc-derive", + "jsonrpc-ipc-server", + "jsonrpc-server-utils", + "lazy_static", + "libc", + "libloading", + "log", + "num_cpus", + "rand 0.8.5", + "rayon", + "serde", + "serde_json", + "serde_yaml", + "signal-hook", + "solana-accounts-db", + "solana-clap-utils", + "solana-cli-config", + "solana-core", + "solana-download-utils", + "solana-entry", + "solana-faucet", + "solana-genesis-utils", + "solana-geyser-plugin-manager", + "solana-gossip", + "solana-ledger", + "solana-logger", + "solana-metrics", + "solana-net-utils", + "solana-perf", + "solana-poh", + "solana-rpc", + "solana-rpc-client", + "solana-rpc-client-api", + "solana-runtime", + "solana-sdk", + "solana-send-transaction-service", + "solana-storage-bigtable", + "solana-streamer", + "solana-svm", + "solana-test-validator", + "solana-tpu-client", + "solana-unified-scheduler-pool", + "solana-version", + "solana-vote-program", + "symlink", + "thiserror", + "tikv-jemallocator", +] + [[package]] name = "ahash" version = "0.7.6" @@ -5044,20 +5118,11 @@ dependencies = [ "solana-sdk", ] -[[package]] -name = "solana-geyser-plugin-interface" -version = "1.19.0" -dependencies = [ - "log", - "solana-sdk", - "solana-transaction-status", - "thiserror", -] - [[package]] name = "solana-geyser-plugin-manager" version = "1.19.0" dependencies = [ + "agave-geyser-plugin-interface", "bs58", "crossbeam-channel", "json5", @@ -5068,7 +5133,6 @@ dependencies = [ "serde_json", "solana-accounts-db", "solana-entry", - "solana-geyser-plugin-interface", "solana-ledger", "solana-measure", "solana-metrics", @@ -6055,11 +6119,11 @@ dependencies = [ name = "solana-sbf-rust-simulation" version = "1.19.0" dependencies = [ + "agave-validator", "solana-logger", "solana-program", "solana-program-test", "solana-sdk", - "solana-validator", ] [[package]] @@ -6462,70 +6526,6 @@ dependencies = [ "solana-vote", ] -[[package]] -name = "solana-validator" -version = "1.19.0" -dependencies = [ - "chrono", - "clap 2.33.3", - "console", - "core_affinity", - "crossbeam-channel", - "fd-lock", - "indicatif", - "itertools", - "jsonrpc-core", - "jsonrpc-core-client", - "jsonrpc-derive", - "jsonrpc-ipc-server", - "jsonrpc-server-utils", - "lazy_static", - "libc", - "libloading", - "log", - "num_cpus", - "rand 0.8.5", - "rayon", - "serde", - "serde_json", - "serde_yaml", - "signal-hook", - "solana-accounts-db", - "solana-clap-utils", - "solana-cli-config", - "solana-core", - "solana-download-utils", - "solana-entry", - "solana-faucet", - "solana-genesis-utils", - "solana-geyser-plugin-interface", - "solana-geyser-plugin-manager", - "solana-gossip", - "solana-ledger", - "solana-logger", - "solana-metrics", - "solana-net-utils", - "solana-perf", - "solana-poh", - "solana-rpc", - "solana-rpc-client", - "solana-rpc-client-api", - "solana-runtime", - "solana-sdk", - "solana-send-transaction-service", - "solana-storage-bigtable", - "solana-streamer", - "solana-svm", - "solana-test-validator", - "solana-tpu-client", - "solana-unified-scheduler-pool", - "solana-version", - "solana-vote-program", - "symlink", - "thiserror", - "tikv-jemallocator", -] - [[package]] name = "solana-version" version = "1.19.0" diff --git a/programs/sbf/Cargo.toml b/programs/sbf/Cargo.toml index 8a99a0f005471a..dee6a947b1965d 100644 --- a/programs/sbf/Cargo.toml +++ b/programs/sbf/Cargo.toml @@ -46,7 +46,7 @@ solana-sbf-rust-realloc = { path = "rust/realloc", version = "=1.19.0", default- solana-sbf-rust-realloc-invoke = { path = "rust/realloc_invoke", version = "=1.19.0" } solana-sdk = { path = "../../sdk", version = "=1.19.0" } solana-transaction-status = { path = "../../transaction-status", version = "=1.19.0" } -solana-validator = { path = "../../validator", version = "=1.19.0" } +agave-validator = { path = "../../validator", version = "=1.19.0" } solana-zk-token-sdk = { path = "../../zk-token-sdk", version = "=1.19.0" } solana-svm = { path = "../../svm", version = "=1.19.0" } solana_rbpf = "=0.8.0" diff --git a/programs/sbf/rust/simulation/Cargo.toml b/programs/sbf/rust/simulation/Cargo.toml index 7091ef9d5ade0c..e9728e5916b801 100644 --- a/programs/sbf/rust/simulation/Cargo.toml +++ b/programs/sbf/rust/simulation/Cargo.toml @@ -16,10 +16,10 @@ test-bpf = [] solana-program = { workspace = true } [dev-dependencies] +agave-validator = { workspace = true } solana-logger = { workspace = true } solana-program-test = { workspace = true } solana-sdk = { workspace = true } -solana-validator = { workspace = true } [lib] crate-type = ["cdylib", "lib"] diff --git a/programs/sbf/rust/simulation/tests/validator.rs b/programs/sbf/rust/simulation/tests/validator.rs index 3044ad9a642629..17de51e665e3ec 100644 --- a/programs/sbf/rust/simulation/tests/validator.rs +++ b/programs/sbf/rust/simulation/tests/validator.rs @@ -1,13 +1,13 @@ #![cfg(feature = "test-bpf")] use { + agave_validator::test_validator::*, solana_program::{ instruction::{AccountMeta, Instruction}, pubkey::Pubkey, sysvar, }, solana_sdk::{signature::Signer, transaction::Transaction}, - solana_validator::test_validator::*, }; #[test] diff --git a/pubsub-client/src/nonblocking/pubsub_client.rs b/pubsub-client/src/nonblocking/pubsub_client.rs index 408df60454e4e1..b79e91f681b97f 100644 --- a/pubsub-client/src/nonblocking/pubsub_client.rs +++ b/pubsub-client/src/nonblocking/pubsub_client.rs @@ -33,7 +33,7 @@ //! By default the [`block_subscribe`] and [`vote_subscribe`] events are //! disabled on RPC nodes. They can be enabled by passing //! `--rpc-pubsub-enable-block-subscription` and -//! `--rpc-pubsub-enable-vote-subscription` to `solana-validator`. When these +//! `--rpc-pubsub-enable-vote-subscription` to `agave-validator`. When these //! methods are disabled, the RPC server will return a "Method not found" error //! message. //! @@ -381,7 +381,7 @@ impl PubsubClient { /// Receives messages of type [`RpcBlockUpdate`] when a block is confirmed or finalized. /// /// This method is disabled by default. It can be enabled by passing - /// `--rpc-pubsub-enable-block-subscription` to `solana-validator`. + /// `--rpc-pubsub-enable-block-subscription` to `agave-validator`. /// /// # RPC Reference /// @@ -452,7 +452,7 @@ impl PubsubClient { /// votes are observed prior to confirmation and may never be confirmed. /// /// This method is disabled by default. It can be enabled by passing - /// `--rpc-pubsub-enable-vote-subscription` to `solana-validator`. + /// `--rpc-pubsub-enable-vote-subscription` to `agave-validator`. /// /// # RPC Reference /// diff --git a/pubsub-client/src/pubsub_client.rs b/pubsub-client/src/pubsub_client.rs index e1a2dd34546528..70769619db1f4d 100644 --- a/pubsub-client/src/pubsub_client.rs +++ b/pubsub-client/src/pubsub_client.rs @@ -32,7 +32,7 @@ //! By default the [`block_subscribe`] and [`vote_subscribe`] events are //! disabled on RPC nodes. They can be enabled by passing //! `--rpc-pubsub-enable-block-subscription` and -//! `--rpc-pubsub-enable-vote-subscription` to `solana-validator`. When these +//! `--rpc-pubsub-enable-vote-subscription` to `agave-validator`. When these //! methods are disabled, the RPC server will return a "Method not found" error //! message. //! @@ -416,7 +416,7 @@ impl PubsubClient { /// Receives messages of type [`RpcBlockUpdate`] when a block is confirmed or finalized. /// /// This method is disabled by default. It can be enabled by passing - /// `--rpc-pubsub-enable-block-subscription` to `solana-validator`. + /// `--rpc-pubsub-enable-block-subscription` to `agave-validator`. /// /// # RPC Reference /// @@ -578,7 +578,7 @@ impl PubsubClient { /// votes are observed prior to confirmation and may never be confirmed. /// /// This method is disabled by default. It can be enabled by passing - /// `--rpc-pubsub-enable-vote-subscription` to `solana-validator`. + /// `--rpc-pubsub-enable-vote-subscription` to `agave-validator`. /// /// # RPC Reference /// diff --git a/rbpf-cli/src/main.rs b/rbpf-cli/src/main.rs index e7db982026f82a..9e243f0836aa0f 100644 --- a/rbpf-cli/src/main.rs +++ b/rbpf-cli/src/main.rs @@ -1,6 +1,6 @@ fn main() { println!( - r##"rbpf-cli is replaced by solana-ledger-tool program run subcommand. -Please, use 'solana-ledger-tool program run --help' for more information."## + r##"rbpf-cli is replaced by agave-ledger-tool program run subcommand. +Please, use 'agave-ledger-tool program run --help' for more information."## ); } diff --git a/rpc/src/rpc.rs b/rpc/src/rpc.rs index caeb0953109fbb..01f623dccdc108 100644 --- a/rpc/src/rpc.rs +++ b/rpc/src/rpc.rs @@ -2561,7 +2561,7 @@ pub mod rpc_minimal { #[rpc(meta, name = "getVersion")] fn get_version(&self, meta: Self::Metadata) -> Result; - // TODO: Refactor `solana-validator wait-for-restart-window` to not require this method, so + // TODO: Refactor `agave-validator wait-for-restart-window` to not require this method, so // it can be removed from rpc_minimal #[rpc(meta, name = "getVoteAccounts")] fn get_vote_accounts( @@ -2570,7 +2570,7 @@ pub mod rpc_minimal { config: Option, ) -> Result; - // TODO: Refactor `solana-validator wait-for-restart-window` to not require this method, so + // TODO: Refactor `agave-validator wait-for-restart-window` to not require this method, so // it can be removed from rpc_minimal #[rpc(meta, name = "getLeaderSchedule")] fn get_leader_schedule( @@ -2696,7 +2696,7 @@ pub mod rpc_minimal { }) } - // TODO: Refactor `solana-validator wait-for-restart-window` to not require this method, so + // TODO: Refactor `agave-validator wait-for-restart-window` to not require this method, so // it can be removed from rpc_minimal fn get_vote_accounts( &self, @@ -2707,7 +2707,7 @@ pub mod rpc_minimal { meta.get_vote_accounts(config) } - // TODO: Refactor `solana-validator wait-for-restart-window` to not require this method, so + // TODO: Refactor `agave-validator wait-for-restart-window` to not require this method, so // it can be removed from rpc_minimal fn get_leader_schedule( &self, diff --git a/scripts/solana-install-deploy.sh b/scripts/agave-install-deploy.sh similarity index 90% rename from scripts/solana-install-deploy.sh rename to scripts/agave-install-deploy.sh index ea77ca34bc9ea3..a8f8eeb65b3857 100755 --- a/scripts/solana-install-deploy.sh +++ b/scripts/agave-install-deploy.sh @@ -26,7 +26,7 @@ if [[ -z $URL || -z $TAG ]]; then fi if [[ ! -f update_manifest_keypair.json ]]; then - "$SOLANA_ROOT"/scripts/solana-install-update-manifest-keypair.sh "$OS" + "$SOLANA_ROOT"/scripts/agave-install-update-manifest-keypair.sh "$OS" fi case "$OS" in @@ -76,4 +76,4 @@ if [[ $balance = "0 lamports" ]]; then fi # shellcheck disable=SC2086 # Don't want to double quote $maybeKeypair -solana-install deploy $maybeKeypair --url "$URL" "$DOWNLOAD_URL" update_manifest_keypair.json +agave-install deploy $maybeKeypair --url "$URL" "$DOWNLOAD_URL" update_manifest_keypair.json diff --git a/scripts/cargo-install-all.sh b/scripts/cargo-install-all.sh index 549aa15550b0eb..029b1fbf27943d 100755 --- a/scripts/cargo-install-all.sh +++ b/scripts/cargo-install-all.sh @@ -91,8 +91,8 @@ if [[ $CI_OS_NAME = windows ]]; then cargo-test-bpf cargo-test-sbf solana - solana-install - solana-install-init + agave-install + agave-install-init solana-keygen solana-stake-accounts solana-test-validator @@ -106,12 +106,12 @@ else solana-bench-tps solana-faucet solana-gossip - solana-install + agave-install solana-keygen - solana-ledger-tool + agave-ledger-tool solana-log-analyzer solana-net-shaper - solana-validator + agave-validator rbpf-cli ) @@ -123,11 +123,11 @@ else cargo-test-bpf cargo-test-sbf solana-dos - solana-install-init + agave-install-init solana-stake-accounts solana-test-validator solana-tokens - solana-watchtower + agave-watchtower ) fi diff --git a/scripts/check-dev-context-only-utils.sh b/scripts/check-dev-context-only-utils.sh index 8719af96a212e4..6a4f798c633e26 100755 --- a/scripts/check-dev-context-only-utils.sh +++ b/scripts/check-dev-context-only-utils.sh @@ -31,7 +31,7 @@ source ci/rust-version.sh nightly declare tainted_packages=( solana-accounts-bench solana-banking-bench - solana-ledger-tool + agave-ledger-tool ) # convert to comma separeted (ref: https://stackoverflow.com/a/53839433) diff --git a/scripts/run.sh b/scripts/run.sh index 699bfce3e253e3..2d8e1ec88ac450 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -23,9 +23,11 @@ fi PATH=$PWD/target/$profile:$PATH ok=true -for program in solana-{faucet,genesis,keygen,validator}; do +for program in solana-{faucet,genesis,keygen}; do $program -V || ok=false done +agave-validator -V || ok=false + $ok || { echo echo "Unable to locate required programs. Try building them first with:" @@ -115,7 +117,7 @@ args=( --no-os-network-limits-test ) # shellcheck disable=SC2086 -solana-validator "${args[@]}" $SOLANA_RUN_SH_VALIDATOR_ARGS & +agave-validator "${args[@]}" $SOLANA_RUN_SH_VALIDATOR_ARGS & validator=$! wait "$validator" diff --git a/system-test/abi-testcases/mixed-validator-test.sh b/system-test/abi-testcases/mixed-validator-test.sh index 8ab673b26a3d21..c0400560dc519e 100755 --- a/system-test/abi-testcases/mixed-validator-test.sh +++ b/system-test/abi-testcases/mixed-validator-test.sh @@ -30,14 +30,14 @@ solanaInstallGlobalOpts=( bootstrapInstall() { declare v=$1 if [[ ! -h $solanaInstallDataDir/active_release ]]; then - sh "$SOLANA_ROOT"/install/solana-install-init.sh "$v" "${solanaInstallGlobalOpts[@]}" + sh "$SOLANA_ROOT"/install/agave-install-init.sh "$v" "${solanaInstallGlobalOpts[@]}" fi export PATH="$solanaInstallDataDir/active_release/bin/:$PATH" } bootstrapInstall "$baselineVersion" for v in "${otherVersions[@]}"; do - solana-install-init "${solanaInstallGlobalOpts[@]}" "$v" + agave-install-init "${solanaInstallGlobalOpts[@]}" "$v" solana -V done @@ -113,7 +113,7 @@ for v in "${otherVersions[@]}"; do ( set -x tmux new-window -t abi -n "$v" " \ - $SOLANA_BIN/solana-validator \ + $SOLANA_BIN/agave-validator \ --ledger $ledger \ --no-snapshot-fetch \ --entrypoint 127.0.0.1:8001 \ diff --git a/system-test/stability-testcases/gossip-dos-test.sh b/system-test/stability-testcases/gossip-dos-test.sh index f8afade75dc847..68c3c540d5948c 100755 --- a/system-test/stability-testcases/gossip-dos-test.sh +++ b/system-test/stability-testcases/gossip-dos-test.sh @@ -19,14 +19,14 @@ solanaInstallGlobalOpts=( bootstrapInstall() { declare v=$1 if [[ ! -h $solanaInstallDataDir/active_release ]]; then - sh "$SOLANA_ROOT"/install/solana-install-init.sh "$v" "${solanaInstallGlobalOpts[@]}" + sh "$SOLANA_ROOT"/install/agave-install-init.sh "$v" "${solanaInstallGlobalOpts[@]}" fi export PATH="$solanaInstallDataDir/active_release/bin/:$PATH" } bootstrapInstall "edge" -solana-install-init --version -solana-install-init edge +agave-install-init --version +agave-install-init edge solana-gossip --version solana-dos --version diff --git a/validator/Cargo.toml b/validator/Cargo.toml index 5cc76a810116b3..362a07343b5e4a 100644 --- a/validator/Cargo.toml +++ b/validator/Cargo.toml @@ -1,8 +1,8 @@ [package] -name = "solana-validator" +name = "agave-validator" description = "Blockchain, Rebuilt for Scale" -documentation = "https://docs.rs/solana-validator" -default-run = "solana-validator" +documentation = "https://docs.rs/agave-validator" +default-run = "agave-validator" version = { workspace = true } authors = { workspace = true } repository = { workspace = true } @@ -11,6 +11,7 @@ license = { workspace = true } edition = { workspace = true } [dependencies] +agave-geyser-plugin-interface = { workspace = true } chrono = { workspace = true, features = ["default", "serde"] } clap = { workspace = true } console = { workspace = true } @@ -41,7 +42,6 @@ solana-download-utils = { workspace = true } solana-entry = { workspace = true } solana-faucet = { workspace = true } solana-genesis-utils = { workspace = true } -solana-geyser-plugin-interface = { workspace = true } solana-geyser-plugin-manager = { workspace = true } solana-gossip = { workspace = true } solana-ledger = { workspace = true } diff --git a/validator/src/bin/solana-test-validator.rs b/validator/src/bin/solana-test-validator.rs index 42f5a0634c0cfa..68e6bcca4fd96f 100644 --- a/validator/src/bin/solana-test-validator.rs +++ b/validator/src/bin/solana-test-validator.rs @@ -1,4 +1,8 @@ use { + agave_validator::{ + admin_rpc_service, cli, dashboard::Dashboard, ledger_lockfile, lock_ledger, + println_name_value, redirect_stderr_to_file, + }, clap::{crate_name, value_t, value_t_or_exit, values_t_or_exit}, crossbeam_channel::unbounded, itertools::Itertools, @@ -28,10 +32,6 @@ use { }, solana_streamer::socket::SocketAddrSpace, solana_test_validator::*, - solana_validator::{ - admin_rpc_service, cli, dashboard::Dashboard, ledger_lockfile, lock_ledger, - println_name_value, redirect_stderr_to_file, - }, std::{ collections::HashSet, fs, io, diff --git a/validator/src/bootstrap.rs b/validator/src/bootstrap.rs index 8d5457744a23b8..12bbd0b21001c9 100644 --- a/validator/src/bootstrap.rs +++ b/validator/src/bootstrap.rs @@ -447,7 +447,7 @@ pub fn attempt_download_genesis_and_snapshot( ) .unwrap_or_else(|err| { // Consider failures here to be more likely due to user error (eg, - // incorrect `solana-validator` command-line arguments) rather than the + // incorrect `agave-validator` command-line arguments) rather than the // RPC node failing. // // Power users can always use the `--no-check-vote-account` option to diff --git a/validator/src/main.rs b/validator/src/main.rs index ec70796130e7d2..9741a2aecd68a8 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -2,6 +2,15 @@ #[cfg(not(target_env = "msvc"))] use jemallocator::Jemalloc; use { + agave_validator::{ + admin_rpc_service, + admin_rpc_service::{load_staked_nodes_overrides, StakedNodesOverrides}, + bootstrap, + cli::{app, warn_for_deprecated_arguments, DefaultArgs}, + dashboard::Dashboard, + ledger_lockfile, lock_ledger, new_spinner_progress_bar, println_name_value, + redirect_stderr_to_file, + }, clap::{crate_name, value_t, value_t_or_exit, values_t, values_t_or_exit, ArgMatches}, console::style, crossbeam_channel::unbounded, @@ -60,15 +69,6 @@ use { solana_streamer::socket::SocketAddrSpace, solana_svm::runtime_config::RuntimeConfig, solana_tpu_client::tpu_client::DEFAULT_TPU_ENABLE_UDP, - solana_validator::{ - admin_rpc_service, - admin_rpc_service::{load_staked_nodes_overrides, StakedNodesOverrides}, - bootstrap, - cli::{app, warn_for_deprecated_arguments, DefaultArgs}, - dashboard::Dashboard, - ledger_lockfile, lock_ledger, new_spinner_progress_bar, println_name_value, - redirect_stderr_to_file, - }, std::{ collections::{HashSet, VecDeque}, env, @@ -917,7 +917,7 @@ pub fn main() { let logfile = matches .value_of("logfile") .map(|s| s.into()) - .unwrap_or_else(|| format!("solana-validator-{}.log", identity_keypair.pubkey())); + .unwrap_or_else(|| format!("agave-validator-{}.log", identity_keypair.pubkey())); if logfile == "-" { None diff --git a/watchtower/Cargo.toml b/watchtower/Cargo.toml index d8bad3cf4d18f0..4088ee7d9b51ab 100644 --- a/watchtower/Cargo.toml +++ b/watchtower/Cargo.toml @@ -1,7 +1,7 @@ [package] -name = "solana-watchtower" +name = "agave-watchtower" description = "Blockchain, Rebuilt for Scale" -documentation = "https://docs.rs/solana-watchtower" +documentation = "https://docs.rs/agave-watchtower" version = { workspace = true } authors = { workspace = true } repository = { workspace = true } diff --git a/watchtower/README.md b/watchtower/README.md index 33a13939cd260c..ab219be67575eb 100644 --- a/watchtower/README.md +++ b/watchtower/README.md @@ -1,4 +1,4 @@ -The `solana-watchtower` program is used to monitor the health of a cluster. It +The `agave-watchtower` program is used to monitor the health of a cluster. It periodically polls the cluster over an RPC API to confirm that the transaction count is advancing, new blockhashes are available, and no validators are delinquent. Results are reported as InfluxDB metrics, with an optional push diff --git a/watchtower/src/main.rs b/watchtower/src/main.rs index f42acdaadaabc6..341b7903c0a33e 100644 --- a/watchtower/src/main.rs +++ b/watchtower/src/main.rs @@ -47,7 +47,7 @@ fn get_config() -> Config { .version(solana_version::version!()) .after_help("ADDITIONAL HELP: To receive a Slack, Discord, PagerDuty and/or Telegram notification on sanity failure, - define environment variables before running `solana-watchtower`: + define environment variables before running `agave-watchtower`: export SLACK_WEBHOOK=... export DISCORD_WEBHOOK=... @@ -63,7 +63,7 @@ fn get_config() -> Config { To receive a Twilio SMS notification on failure, having a Twilio account, and a sending number owned by that account, - define environment variable before running `solana-watchtower`: + define environment variable before running `agave-watchtower`: export TWILIO_CONFIG='ACCOUNT=,TOKEN=,TO=,FROM='") .arg({ @@ -166,7 +166,7 @@ fn get_config() -> Config { .value_name("SUFFIX") .takes_value(true) .default_value("") - .help("Add this string into all notification messages after \"solana-watchtower\"") + .help("Add this string into all notification messages after \"agave-watchtower\"") ) .get_matches(); @@ -381,7 +381,7 @@ fn main() -> Result<(), Box> { if let Some((failure_test_name, failure_error_message)) = &failure { let notification_msg = format!( - "solana-watchtower{}: Error: {}: {}", + "agave-watchtower{}: Error: {}: {}", config.name_suffix, failure_test_name, failure_error_message ); num_consecutive_failures += 1; @@ -415,7 +415,7 @@ fn main() -> Result<(), Box> { ); info!("{}", all_clear_msg); notifier.send( - &format!("solana-watchtower{}: {}", config.name_suffix, all_clear_msg), + &format!("agave-watchtower{}: {}", config.name_suffix, all_clear_msg), &NotificationType::Resolve { incident }, ); } From 570d1a919729741f062ae0ad0b1f9bf52fbb83f2 Mon Sep 17 00:00:00 2001 From: Brooks Date: Mon, 4 Mar 2024 13:59:35 -0500 Subject: [PATCH 06/84] Adds a bench for hash_account() (#47) --- Cargo.lock | 1 + accounts-db/Cargo.toml | 5 ++++ accounts-db/benches/bench_hashing.rs | 43 ++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+) create mode 100644 accounts-db/benches/bench_hashing.rs diff --git a/Cargo.lock b/Cargo.lock index c72b90930d7cf9..a67b7aba0d6b64 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5390,6 +5390,7 @@ dependencies = [ "bytemuck", "byteorder", "bzip2", + "criterion", "crossbeam-channel", "dashmap", "ed25519-dalek", diff --git a/accounts-db/Cargo.toml b/accounts-db/Cargo.toml index b986c17de0636b..702f14f9f3b07d 100644 --- a/accounts-db/Cargo.toml +++ b/accounts-db/Cargo.toml @@ -70,6 +70,7 @@ name = "solana_accounts_db" [dev-dependencies] assert_matches = { workspace = true } +criterion = { workspace = true } ed25519-dalek = { workspace = true } libsecp256k1 = { workspace = true } memoffset = { workspace = true } @@ -89,3 +90,7 @@ rustc_version = { workspace = true } [features] dev-context-only-utils = [] + +[[bench]] +name = "bench_hashing" +harness = false diff --git a/accounts-db/benches/bench_hashing.rs b/accounts-db/benches/bench_hashing.rs new file mode 100644 index 00000000000000..3158f78c7a938f --- /dev/null +++ b/accounts-db/benches/bench_hashing.rs @@ -0,0 +1,43 @@ +use { + criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}, + solana_accounts_db::accounts_db::AccountsDb, + solana_sdk::{account::AccountSharedData, pubkey::Pubkey}, +}; + +const KB: usize = 1024; +const MB: usize = KB * KB; + +const DATA_SIZES: [usize; 6] = [ + 0, // the smallest account + 165, // the size of an spl token account + 200, // the size of a stake account + KB, // a medium sized account + MB, // a large sized account + 10 * MB, // the largest account +]; + +/// The number of bytes of *non account data* that are also hashed as +/// part of computing an account's hash. +/// +/// Ensure this constant stays in sync with the value of `META_SIZE` in +/// AccountsDb::hash_account_data(). +const META_SIZE: usize = 81; + +fn bench_hash_account(c: &mut Criterion) { + let lamports = 123_456_789; + let owner = Pubkey::default(); + let address = Pubkey::default(); + + let mut group = c.benchmark_group("hash_account"); + for data_size in DATA_SIZES { + let num_bytes = META_SIZE.checked_add(data_size).unwrap(); + group.throughput(Throughput::Bytes(num_bytes as u64)); + let account = AccountSharedData::new(lamports, data_size, &owner); + group.bench_function(BenchmarkId::new("data_size", data_size), |b| { + b.iter(|| AccountsDb::hash_account(&account, &address)); + }); + } +} + +criterion_group!(benches, bench_hash_account,); +criterion_main!(benches); From f5912104d0230b601e2b3696a94d376ad8f7c73c Mon Sep 17 00:00:00 2001 From: Brennan Date: Mon, 4 Mar 2024 13:17:34 -0800 Subject: [PATCH 07/84] update mio to 0.8.11 (#60) --- Cargo.lock | 4 ++-- programs/sbf/Cargo.lock | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a67b7aba0d6b64..78ff40111ee0b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3394,9 +3394,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.8" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", "wasi 0.11.0+wasi-snapshot-preview1", diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index 7546c56bd2b26a..cb0ad6f1ee448c 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -2892,9 +2892,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.8" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", "wasi 0.11.0+wasi-snapshot-preview1", From 93f5b514fa410b0c94a7ce134bed2fc871400890 Mon Sep 17 00:00:00 2001 From: Brooks Date: Mon, 4 Mar 2024 16:32:51 -0500 Subject: [PATCH 08/84] Adds StartingSnapshotStorages to AccountsHashVerifier (#58) --- accounts-db/src/lib.rs | 1 + accounts-db/src/starting_snapshot_storages.rs | 19 +++++ core/src/accounts_hash_verifier.rs | 8 +- core/src/validator.rs | 43 ++++++---- core/tests/epoch_accounts_hash.rs | 2 + core/tests/snapshots.rs | 2 + ledger-tool/src/ledger_utils.rs | 32 +++++--- ledger/src/bank_forks_utils.rs | 81 ++++++++++++++----- 8 files changed, 135 insertions(+), 53 deletions(-) create mode 100644 accounts-db/src/starting_snapshot_storages.rs diff --git a/accounts-db/src/lib.rs b/accounts-db/src/lib.rs index 7883f852d1e3f2..b7994fe4354118 100644 --- a/accounts-db/src/lib.rs +++ b/accounts-db/src/lib.rs @@ -37,6 +37,7 @@ pub mod secondary_index; pub mod shared_buffer_reader; pub mod sorted_storages; pub mod stake_rewards; +pub mod starting_snapshot_storages; pub mod storable_accounts; pub mod tiered_storage; pub mod utils; diff --git a/accounts-db/src/starting_snapshot_storages.rs b/accounts-db/src/starting_snapshot_storages.rs new file mode 100644 index 00000000000000..cc5e26c61872b7 --- /dev/null +++ b/accounts-db/src/starting_snapshot_storages.rs @@ -0,0 +1,19 @@ +use {crate::accounts_db::AccountStorageEntry, std::sync::Arc}; + +/// Snapshot storages that the node loaded from +/// +/// This is used to support fastboot. Since fastboot reuses existing storages, we must carefully +/// handle the storages used to load at startup. If we do not handle these storages properly, +/// restarting from the same local state (i.e. bank snapshot) may fail. +#[derive(Debug)] +pub enum StartingSnapshotStorages { + /// Starting from genesis has no storages yet + Genesis, + /// Starting from a snapshot archive always extracts the storages from the archive, so no + /// special handling is necessary to preserve them. + Archive, + /// Starting from local state must preserve the loaded storages. These storages must *not* be + /// recycled or removed prior to taking the next snapshot, otherwise restarting from the same + /// bank snapshot may fail. + Fastboot(Vec>), +} diff --git a/core/src/accounts_hash_verifier.rs b/core/src/accounts_hash_verifier.rs index 0e427d0675a2b1..f5572d94a3c7d1 100644 --- a/core/src/accounts_hash_verifier.rs +++ b/core/src/accounts_hash_verifier.rs @@ -9,6 +9,7 @@ use { IncrementalAccountsHash, }, sorted_storages::SortedStorages, + starting_snapshot_storages::StartingSnapshotStorages, }, solana_measure::measure_us, solana_runtime::{ @@ -42,6 +43,7 @@ impl AccountsHashVerifier { accounts_package_sender: Sender, accounts_package_receiver: Receiver, snapshot_package_sender: Option>, + starting_snapshot_storages: StartingSnapshotStorages, exit: Arc, snapshot_config: SnapshotConfig, ) -> Self { @@ -54,7 +56,11 @@ impl AccountsHashVerifier { // To support fastboot, we must ensure the storages used in the latest POST snapshot are // not recycled nor removed early. Hold an Arc of their AppendVecs to prevent them from // expiring. - let mut fastboot_storages = None; + let mut fastboot_storages = match starting_snapshot_storages { + StartingSnapshotStorages::Genesis => None, + StartingSnapshotStorages::Archive => None, + StartingSnapshotStorages::Fastboot(storages) => Some(storages), + }; loop { if exit.load(Ordering::Relaxed) { break; diff --git a/core/src/validator.rs b/core/src/validator.rs index a6d5921bcef5c9..196dad5f25d17a 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -35,6 +35,7 @@ use { accounts_index::AccountSecondaryIndexes, accounts_update_notifier_interface::AccountsUpdateNotifier, hardened_unpack::{open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE}, + starting_snapshot_storages::StartingSnapshotStorages, utils::{move_and_async_delete_path, move_and_async_delete_path_contents}, }, solana_client::connection_cache::{ConnectionCache, Protocol}, @@ -690,6 +691,7 @@ impl Validator { completed_slots_receiver, leader_schedule_cache, starting_snapshot_hashes, + starting_snapshot_storages, TransactionHistoryServices { transaction_status_sender, transaction_status_service, @@ -779,6 +781,7 @@ impl Validator { accounts_package_sender.clone(), accounts_package_receiver, snapshot_package_sender, + starting_snapshot_storages, exit.clone(), config.snapshot_config.clone(), ); @@ -1767,6 +1770,7 @@ fn load_blockstore( CompletedSlotsReceiver, LeaderScheduleCache, Option, + StartingSnapshotStorages, TransactionHistoryServices, blockstore_processor::ProcessOptions, BlockstoreRootScan, @@ -1856,23 +1860,27 @@ fn load_blockstore( let entry_notifier_service = entry_notifier .map(|entry_notifier| EntryNotifierService::new(entry_notifier, exit.clone())); - let (bank_forks, mut leader_schedule_cache, starting_snapshot_hashes) = - bank_forks_utils::load_bank_forks( - &genesis_config, - &blockstore, - config.account_paths.clone(), - Some(&config.snapshot_config), - &process_options, - transaction_history_services - .cache_block_meta_sender - .as_ref(), - entry_notifier_service - .as_ref() - .map(|service| service.sender()), - accounts_update_notifier, - exit, - ) - .map_err(|err| err.to_string())?; + let ( + bank_forks, + mut leader_schedule_cache, + starting_snapshot_hashes, + starting_snapshot_storages, + ) = bank_forks_utils::load_bank_forks( + &genesis_config, + &blockstore, + config.account_paths.clone(), + Some(&config.snapshot_config), + &process_options, + transaction_history_services + .cache_block_meta_sender + .as_ref(), + entry_notifier_service + .as_ref() + .map(|service| service.sender()), + accounts_update_notifier, + exit, + ) + .map_err(|err| err.to_string())?; // Before replay starts, set the callbacks in each of the banks in BankForks so that // all dropped banks come through the `pruned_banks_receiver` channel. This way all bank @@ -1898,6 +1906,7 @@ fn load_blockstore( completed_slots_receiver, leader_schedule_cache, starting_snapshot_hashes, + starting_snapshot_storages, transaction_history_services, process_options, blockstore_root_scan, diff --git a/core/tests/epoch_accounts_hash.rs b/core/tests/epoch_accounts_hash.rs index b0dd111676af79..62e31f0a88b766 100755 --- a/core/tests/epoch_accounts_hash.rs +++ b/core/tests/epoch_accounts_hash.rs @@ -9,6 +9,7 @@ use { accounts_hash::CalcAccountsHashConfig, accounts_index::AccountSecondaryIndexes, epoch_accounts_hash::EpochAccountsHash, + starting_snapshot_storages::StartingSnapshotStorages, }, solana_core::{ accounts_hash_verifier::AccountsHashVerifier, @@ -196,6 +197,7 @@ impl BackgroundServices { accounts_package_sender.clone(), accounts_package_receiver, Some(snapshot_package_sender), + StartingSnapshotStorages::Genesis, exit.clone(), snapshot_config.clone(), ); diff --git a/core/tests/snapshots.rs b/core/tests/snapshots.rs index 2694f7294a7217..e67c942f07ab0b 100644 --- a/core/tests/snapshots.rs +++ b/core/tests/snapshots.rs @@ -11,6 +11,7 @@ use { accounts_hash::AccountsHash, accounts_index::AccountSecondaryIndexes, epoch_accounts_hash::EpochAccountsHash, + starting_snapshot_storages::StartingSnapshotStorages, }, solana_core::{ accounts_hash_verifier::AccountsHashVerifier, @@ -1043,6 +1044,7 @@ fn test_snapshots_with_background_services( accounts_package_sender, accounts_package_receiver, Some(snapshot_package_sender), + StartingSnapshotStorages::Genesis, exit.clone(), snapshot_test_config.snapshot_config.clone(), ); diff --git a/ledger-tool/src/ledger_utils.rs b/ledger-tool/src/ledger_utils.rs index c05cc6c2d64cd0..8a8302d7e4e94b 100644 --- a/ledger-tool/src/ledger_utils.rs +++ b/ledger-tool/src/ledger_utils.rs @@ -268,19 +268,24 @@ pub fn load_and_process_ledger( }; let exit = Arc::new(AtomicBool::new(false)); - let (bank_forks, leader_schedule_cache, starting_snapshot_hashes, ..) = - bank_forks_utils::load_bank_forks( - genesis_config, - blockstore.as_ref(), - account_paths, - snapshot_config.as_ref(), - &process_options, - None, - None, // Maybe support this later, though - accounts_update_notifier, - exit.clone(), - ) - .map_err(LoadAndProcessLedgerError::LoadBankForks)?; + let ( + bank_forks, + leader_schedule_cache, + starting_snapshot_hashes, + starting_snapshot_storages, + .., + ) = bank_forks_utils::load_bank_forks( + genesis_config, + blockstore.as_ref(), + account_paths, + snapshot_config.as_ref(), + &process_options, + None, + None, // Maybe support this later, though + accounts_update_notifier, + exit.clone(), + ) + .map_err(LoadAndProcessLedgerError::LoadBankForks)?; let block_verification_method = value_t!( arg_matches, "block_verification_method", @@ -325,6 +330,7 @@ pub fn load_and_process_ledger( accounts_package_sender.clone(), accounts_package_receiver, None, + starting_snapshot_storages, exit.clone(), SnapshotConfig::new_load_only(), ); diff --git a/ledger/src/bank_forks_utils.rs b/ledger/src/bank_forks_utils.rs index 17412c1801ac68..b30f90986bb9c2 100644 --- a/ledger/src/bank_forks_utils.rs +++ b/ledger/src/bank_forks_utils.rs @@ -10,7 +10,10 @@ use { use_snapshot_archives_at_startup::{self, UseSnapshotArchivesAtStartup}, }, log::*, - solana_accounts_db::accounts_update_notifier_interface::AccountsUpdateNotifier, + solana_accounts_db::{ + accounts_update_notifier_interface::AccountsUpdateNotifier, + starting_snapshot_storages::StartingSnapshotStorages, + }, solana_runtime::{ accounts_background_service::AbsRequestSender, bank_forks::BankForks, @@ -67,6 +70,7 @@ pub type LoadResult = result::Result< Arc>, LeaderScheduleCache, Option, + StartingSnapshotStorages, ), BankForksUtilsError, >; @@ -88,7 +92,13 @@ pub fn load( accounts_update_notifier: Option, exit: Arc, ) -> LoadResult { - let (bank_forks, leader_schedule_cache, starting_snapshot_hashes, ..) = load_bank_forks( + let ( + bank_forks, + leader_schedule_cache, + starting_snapshot_hashes, + starting_snapshot_storages, + .., + ) = load_bank_forks( genesis_config, blockstore, account_paths, @@ -111,7 +121,12 @@ pub fn load( ) .map_err(BankForksUtilsError::ProcessBlockstoreFromRoot)?; - Ok((bank_forks, leader_schedule_cache, starting_snapshot_hashes)) + Ok(( + bank_forks, + leader_schedule_cache, + starting_snapshot_hashes, + starting_snapshot_storages, + )) } #[allow(clippy::too_many_arguments)] @@ -161,7 +176,7 @@ pub fn load_bank_forks( )) } - let (bank_forks, starting_snapshot_hashes) = + let (bank_forks, starting_snapshot_hashes, starting_snapshot_storages) = if let Some((full_snapshot_archive_info, incremental_snapshot_archive_info)) = get_snapshots_to_load(snapshot_config) { @@ -173,17 +188,22 @@ pub fn load_bank_forks( ); std::fs::create_dir_all(&snapshot_config.bank_snapshots_dir) .expect("create bank snapshots dir"); - let (bank_forks, starting_snapshot_hashes) = bank_forks_from_snapshot( - full_snapshot_archive_info, - incremental_snapshot_archive_info, - genesis_config, - account_paths, - snapshot_config, - process_options, - accounts_update_notifier, - exit, - )?; - (bank_forks, Some(starting_snapshot_hashes)) + let (bank_forks, starting_snapshot_hashes, starting_snapshot_storages) = + bank_forks_from_snapshot( + full_snapshot_archive_info, + incremental_snapshot_archive_info, + genesis_config, + account_paths, + snapshot_config, + process_options, + accounts_update_notifier, + exit, + )?; + ( + bank_forks, + Some(starting_snapshot_hashes), + starting_snapshot_storages, + ) } else { info!("Processing ledger from genesis"); let bank_forks = blockstore_processor::process_blockstore_for_bank_0( @@ -202,7 +222,7 @@ pub fn load_bank_forks( .root_bank() .set_startup_verification_complete(); - (bank_forks, None) + (bank_forks, None, StartingSnapshotStorages::Genesis) }; let mut leader_schedule_cache = @@ -218,7 +238,12 @@ pub fn load_bank_forks( .for_each(|hard_fork_slot| root_bank.register_hard_fork(*hard_fork_slot)); } - Ok((bank_forks, leader_schedule_cache, starting_snapshot_hashes)) + Ok(( + bank_forks, + leader_schedule_cache, + starting_snapshot_hashes, + starting_snapshot_storages, + )) } #[allow(clippy::too_many_arguments)] @@ -231,7 +256,14 @@ fn bank_forks_from_snapshot( process_options: &ProcessOptions, accounts_update_notifier: Option, exit: Arc, -) -> Result<(Arc>, StartingSnapshotHashes), BankForksUtilsError> { +) -> Result< + ( + Arc>, + StartingSnapshotHashes, + StartingSnapshotStorages, + ), + BankForksUtilsError, +> { // Fail hard here if snapshot fails to load, don't silently continue if account_paths.is_empty() { return Err(BankForksUtilsError::AccountPathsNotPresent); @@ -257,7 +289,7 @@ fn bank_forks_from_snapshot( .unwrap_or(true), }; - let bank = if will_startup_from_snapshot_archives { + let (bank, starting_snapshot_storages) = if will_startup_from_snapshot_archives { // Given that we are going to boot from an archive, the append vecs held in the snapshot dirs for fast-boot should // be released. They will be released by the account_background_service anyway. But in the case of the account_paths // using memory-mounted file system, they are not released early enough to give space for the new append-vecs from @@ -292,7 +324,7 @@ fn bank_forks_from_snapshot( .map(|archive| archive.path().display().to_string()) .unwrap_or("none".to_string()), })?; - bank + (bank, StartingSnapshotStorages::Archive) } else { let bank_snapshot = latest_bank_snapshot.ok_or_else(|| BankForksUtilsError::NoBankSnapshotDirectory { @@ -346,7 +378,8 @@ fn bank_forks_from_snapshot( // snapshot archive next time, which is safe. snapshot_utils::purge_all_bank_snapshots(&snapshot_config.bank_snapshots_dir); - bank + let storages = bank.get_snapshot_storages(None); + (bank, StartingSnapshotStorages::Fastboot(storages)) }; let full_snapshot_hash = FullSnapshotHash(( @@ -365,5 +398,9 @@ fn bank_forks_from_snapshot( incremental: incremental_snapshot_hash, }; - Ok((BankForks::new_rw_arc(bank), starting_snapshot_hashes)) + Ok(( + BankForks::new_rw_arc(bank), + starting_snapshot_hashes, + starting_snapshot_storages, + )) } From 6263537bf0afacc08a29a9f6261aa44970190094 Mon Sep 17 00:00:00 2001 From: Ashwin Sekar Date: Mon, 4 Mar 2024 17:16:31 -0800 Subject: [PATCH 09/84] blockstore_purge: fix inspect -> inspect_err (#66) --- ledger/src/blockstore/blockstore_purge.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ledger/src/blockstore/blockstore_purge.rs b/ledger/src/blockstore/blockstore_purge.rs index 15a5c4890e9f05..d8b4c7424cd8c1 100644 --- a/ledger/src/blockstore/blockstore_purge.rs +++ b/ledger/src/blockstore/blockstore_purge.rs @@ -213,7 +213,7 @@ impl Blockstore { delete_range_timer.stop(); let mut write_timer = Measure::start("write_batch"); - self.db.write(write_batch).inspect(|e| { + self.db.write(write_batch).inspect_err(|e| { error!( "Error: {:?} while submitting write batch for purge from_slot {} to_slot {}", e, from_slot, to_slot From 661de5bb76851c60edd97ec8bb1c82937127e38d Mon Sep 17 00:00:00 2001 From: Tyera Date: Mon, 4 Mar 2024 19:21:30 -0700 Subject: [PATCH 10/84] Rpc: deprecate `getStakeActivation` and make inactive_stake consistent (#69) * Make inactive_stake consistent * Add rpc_deprecated_v1_18 module * Move get_stake_activation to deprecated list * Fix typo --- rpc/src/rpc.rs | 74 +++++++++++++++++++++++------------------- rpc/src/rpc_service.rs | 6 ++-- 2 files changed, 45 insertions(+), 35 deletions(-) diff --git a/rpc/src/rpc.rs b/rpc/src/rpc.rs index 01f623dccdc108..41b26e5fa1e2c2 100644 --- a/rpc/src/rpc.rs +++ b/rpc/src/rpc.rs @@ -1786,16 +1786,10 @@ impl JsonRpcRequestProcessor { } else { StakeActivationState::Inactive }; - let inactive_stake = match stake_activation_state { - StakeActivationState::Activating => activating, - StakeActivationState::Active => 0, - StakeActivationState::Deactivating => stake_account - .lamports() - .saturating_sub(effective + rent_exempt_reserve), - StakeActivationState::Inactive => { - stake_account.lamports().saturating_sub(rent_exempt_reserve) - } - }; + let inactive_stake = stake_account + .lamports() + .saturating_sub(effective) + .saturating_sub(rent_exempt_reserve); Ok(RpcStakeActivation { state: stake_activation_state, active: effective, @@ -2991,14 +2985,6 @@ pub mod rpc_accounts { block: Slot, ) -> Result>; - #[rpc(meta, name = "getStakeActivation")] - fn get_stake_activation( - &self, - meta: Self::Metadata, - pubkey_str: String, - config: Option, - ) -> Result; - // SPL Token-specific RPC endpoints // See https://github.com/solana-labs/solana-program-library/releases/tag/token-v2.0.0 for // program details @@ -3071,20 +3057,6 @@ pub mod rpc_accounts { Ok(meta.get_block_commitment(block)) } - fn get_stake_activation( - &self, - meta: Self::Metadata, - pubkey_str: String, - config: Option, - ) -> Result { - debug!( - "get_stake_activation rpc request received: {:?}", - pubkey_str - ); - let pubkey = verify_pubkey(&pubkey_str)?; - meta.get_stake_activation(&pubkey, config) - } - fn get_token_account_balance( &self, meta: Self::Metadata, @@ -4091,7 +4063,43 @@ fn rpc_perf_sample_from_perf_sample(slot: u64, sample: PerfSample) -> RpcPerfSam } } -// RPC methods deprecated in v1.8 +pub mod rpc_deprecated_v1_18 { + use super::*; + #[rpc] + pub trait DeprecatedV1_18 { + type Metadata; + + // DEPRECATED + #[rpc(meta, name = "getStakeActivation")] + fn get_stake_activation( + &self, + meta: Self::Metadata, + pubkey_str: String, + config: Option, + ) -> Result; + } + + pub struct DeprecatedV1_18Impl; + impl DeprecatedV1_18 for DeprecatedV1_18Impl { + type Metadata = JsonRpcRequestProcessor; + + fn get_stake_activation( + &self, + meta: Self::Metadata, + pubkey_str: String, + config: Option, + ) -> Result { + debug!( + "get_stake_activation rpc request received: {:?}", + pubkey_str + ); + let pubkey = verify_pubkey(&pubkey_str)?; + meta.get_stake_activation(&pubkey, config) + } + } +} + +// RPC methods deprecated in v1.9 pub mod rpc_deprecated_v1_9 { #![allow(deprecated)] use super::*; diff --git a/rpc/src/rpc_service.rs b/rpc/src/rpc_service.rs index 8597394f102325..d8791ab6c3bf6b 100644 --- a/rpc/src/rpc_service.rs +++ b/rpc/src/rpc_service.rs @@ -6,8 +6,9 @@ use { max_slots::MaxSlots, optimistically_confirmed_bank_tracker::OptimisticallyConfirmedBank, rpc::{ - rpc_accounts::*, rpc_accounts_scan::*, rpc_bank::*, rpc_deprecated_v1_7::*, - rpc_deprecated_v1_9::*, rpc_full::*, rpc_minimal::*, rpc_obsolete_v1_7::*, *, + rpc_accounts::*, rpc_accounts_scan::*, rpc_bank::*, rpc_deprecated_v1_18::*, + rpc_deprecated_v1_7::*, rpc_deprecated_v1_9::*, rpc_full::*, rpc_minimal::*, + rpc_obsolete_v1_7::*, *, }, rpc_cache::LargestAccountsCache, rpc_health::*, @@ -510,6 +511,7 @@ impl JsonRpcService { io.extend_with(rpc_full::FullImpl.to_delegate()); io.extend_with(rpc_deprecated_v1_7::DeprecatedV1_7Impl.to_delegate()); io.extend_with(rpc_deprecated_v1_9::DeprecatedV1_9Impl.to_delegate()); + io.extend_with(rpc_deprecated_v1_18::DeprecatedV1_18Impl.to_delegate()); } if obsolete_v1_7_api { io.extend_with(rpc_obsolete_v1_7::ObsoleteV1_7Impl.to_delegate()); From b78c0703ff97d78634c937b04083fd5272595f0c Mon Sep 17 00:00:00 2001 From: Jon C Date: Tue, 5 Mar 2024 12:28:12 +0100 Subject: [PATCH 11/84] windows: Use vcpkg for openssl dep (#73) --- .github/workflows/release-artifacts.yml | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 45be181c3ce9e1..7aec77f0dac45f 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -43,17 +43,8 @@ jobs: id: build shell: bash run: | - choco install openssl --version=3.1.1 - if [[ -d "C:\Program Files\OpenSSL" ]]; then - echo "OPENSSL_DIR: C:\Program Files\OpenSSL" - export OPENSSL_DIR="C:\Program Files\OpenSSL" - elif [[ -d "C:\Program Files\OpenSSL-Win64" ]]; then - echo "OPENSSL_DIR: C:\Program Files\OpenSSL-Win64" - export OPENSSL_DIR="C:\Program Files\OpenSSL-Win64" - else - echo "can't determine OPENSSL_DIR" - exit 1 - fi + vcpkg install openssl:x64-windows-static-md + vcpkg integrate install choco install protoc export PROTOC="C:\ProgramData\chocolatey\lib\protoc\tools\bin\protoc.exe" source /tmp/env.sh From 1e133bc067555df9fa2d60835f256d962d3aed98 Mon Sep 17 00:00:00 2001 From: Brooks Date: Tue, 5 Mar 2024 12:02:47 -0500 Subject: [PATCH 12/84] Increases account hash's stack buffer to hold 200 bytes of data (#56) --- accounts-db/Cargo.toml | 2 +- accounts-db/src/accounts_db.rs | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/accounts-db/Cargo.toml b/accounts-db/Cargo.toml index 702f14f9f3b07d..0fc5a381fbda5e 100644 --- a/accounts-db/Cargo.toml +++ b/accounts-db/Cargo.toml @@ -42,7 +42,7 @@ regex = { workspace = true } seqlock = { workspace = true } serde = { workspace = true, features = ["rc"] } serde_derive = { workspace = true } -smallvec = { workspace = true } +smallvec = { workspace = true, features = ["const_generics"] } solana-bucket-map = { workspace = true } solana-config-program = { workspace = true } solana-frozen-abi = { workspace = true } diff --git a/accounts-db/src/accounts_db.rs b/accounts-db/src/accounts_db.rs index 1f3c36876f4531..cf4d17745b1b73 100644 --- a/accounts-db/src/accounts_db.rs +++ b/accounts-db/src/accounts_db.rs @@ -6119,17 +6119,18 @@ impl AccountsDb { } let mut hasher = blake3::Hasher::new(); - // allocate 128 bytes buffer on the stack - const BUFFER_SIZE: usize = 128; - const METADATA_SIZE: usize = 8 /* lamports */ + 8 /* rent_epoch */ + 1 /* executable */ + 32 /* owner */ + 32 /* pubkey */; - const REMAINING_SIZE: usize = BUFFER_SIZE - METADATA_SIZE; + // allocate a buffer on the stack that's big enough + // to hold a token account or a stake account + const META_SIZE: usize = 8 /* lamports */ + 8 /* rent_epoch */ + 1 /* executable */ + 32 /* owner */ + 32 /* pubkey */; + const DATA_SIZE: usize = 200; // stake acounts are 200 B and token accounts are 165-182ish B + const BUFFER_SIZE: usize = META_SIZE + DATA_SIZE; let mut buffer = SmallVec::<[u8; BUFFER_SIZE]>::new(); // collect lamports, rent_epoch into buffer to hash buffer.extend_from_slice(&lamports.to_le_bytes()); buffer.extend_from_slice(&rent_epoch.to_le_bytes()); - if data.len() > REMAINING_SIZE { + if data.len() > DATA_SIZE { // For larger accounts whose data can't fit into the buffer, update the hash now. hasher.update(&buffer); buffer.clear(); From ce34f3f01444e01fcc67630d2c77ba66aefa7539 Mon Sep 17 00:00:00 2001 From: steviez Date: Tue, 5 Mar 2024 12:09:17 -0600 Subject: [PATCH 13/84] Rename and uniquify QUIC thread names (#28) When viewing in various tools such as gdb and perf, it is not easy to distinguish which threads are serving which function (TPU or TPU FWD) --- client/src/connection_cache.rs | 1 + core/src/tpu.rs | 2 ++ quic-client/src/quic_client.rs | 2 +- quic-client/tests/quic_client.rs | 3 +++ streamer/src/quic.rs | 26 +++++++++++++++----------- 5 files changed, 22 insertions(+), 12 deletions(-) diff --git a/client/src/connection_cache.rs b/client/src/connection_cache.rs index b53b66b155e719..a94bc7cd3d8ca8 100644 --- a/client/src/connection_cache.rs +++ b/client/src/connection_cache.rs @@ -260,6 +260,7 @@ mod tests { thread: response_recv_thread, key_updater: _, } = solana_streamer::quic::spawn_server( + "solQuicTest", "quic_streamer_test", response_recv_socket, &keypair2, diff --git a/core/src/tpu.rs b/core/src/tpu.rs index 548b299148d935..0f5431da4eb43c 100644 --- a/core/src/tpu.rs +++ b/core/src/tpu.rs @@ -153,6 +153,7 @@ impl Tpu { thread: tpu_quic_t, key_updater, } = spawn_server( + "solQuicTpu", "quic_streamer_tpu", transactions_quic_sockets, keypair, @@ -172,6 +173,7 @@ impl Tpu { thread: tpu_forwards_quic_t, key_updater: forwards_key_updater, } = spawn_server( + "solQuicTpuFwd", "quic_streamer_tpu_forwards", transactions_forwards_quic_sockets, keypair, diff --git a/quic-client/src/quic_client.rs b/quic-client/src/quic_client.rs index f057980c79fe06..8c8e8e5338993f 100644 --- a/quic-client/src/quic_client.rs +++ b/quic-client/src/quic_client.rs @@ -69,7 +69,7 @@ lazy_static! { static ref ASYNC_TASK_SEMAPHORE: AsyncTaskSemaphore = AsyncTaskSemaphore::new(MAX_OUTSTANDING_TASK); static ref RUNTIME: Runtime = tokio::runtime::Builder::new_multi_thread() - .thread_name("quic-client") + .thread_name("solQuicClientRt") .enable_all() .build() .unwrap(); diff --git a/quic-client/tests/quic_client.rs b/quic-client/tests/quic_client.rs index 658ee6a57d672d..0237fc21d098dc 100644 --- a/quic-client/tests/quic_client.rs +++ b/quic-client/tests/quic_client.rs @@ -72,6 +72,7 @@ mod tests { thread: t, key_updater: _, } = solana_streamer::quic::spawn_server( + "solQuicTest", "quic_streamer_test", s.try_clone().unwrap(), &keypair, @@ -212,6 +213,7 @@ mod tests { thread: request_recv_thread, key_updater: _, } = solana_streamer::quic::spawn_server( + "solQuicTest", "quic_streamer_test", request_recv_socket.try_clone().unwrap(), &keypair, @@ -239,6 +241,7 @@ mod tests { thread: response_recv_thread, key_updater: _, } = solana_streamer::quic::spawn_server( + "solQuicTest", "quic_streamer_test", response_recv_socket, &keypair2, diff --git a/streamer/src/quic.rs b/streamer/src/quic.rs index 69a75532b8ca68..a7a08c73f4833b 100644 --- a/streamer/src/quic.rs +++ b/streamer/src/quic.rs @@ -100,9 +100,9 @@ pub(crate) fn configure_server( Ok((server_config, cert_chain_pem)) } -fn rt() -> Runtime { +fn rt(name: String) -> Runtime { tokio::runtime::Builder::new_multi_thread() - .thread_name("quic-server") + .thread_name(name) .enable_all() .build() .unwrap() @@ -431,7 +431,8 @@ impl StreamStats { #[allow(clippy::too_many_arguments)] pub fn spawn_server( - name: &'static str, + thread_name: &'static str, + metrics_name: &'static str, sock: UdpSocket, keypair: &Keypair, packet_sender: Sender, @@ -443,11 +444,11 @@ pub fn spawn_server( wait_for_chunk_timeout: Duration, coalesce: Duration, ) -> Result { - let runtime = rt(); + let runtime = rt(format!("{thread_name}Rt")); let (endpoint, _stats, task) = { let _guard = runtime.enter(); crate::nonblocking::quic::spawn_server( - name, + metrics_name, sock, keypair, packet_sender, @@ -461,7 +462,7 @@ pub fn spawn_server( ) }?; let handle = thread::Builder::new() - .name("solQuicServer".into()) + .name(thread_name.into()) .spawn(move || { if let Err(e) = runtime.block_on(task) { warn!("error from runtime.block_on: {:?}", e); @@ -505,6 +506,7 @@ mod test { thread: t, key_updater: _, } = spawn_server( + "solQuicTest", "quic_streamer_test", s, &keypair, @@ -532,7 +534,7 @@ mod test { fn test_quic_timeout() { solana_logger::setup(); let (t, exit, receiver, server_address) = setup_quic_server(); - let runtime = rt(); + let runtime = rt("solQuicTestRt".to_string()); runtime.block_on(check_timeout(receiver, server_address)); exit.store(true, Ordering::Relaxed); t.join().unwrap(); @@ -543,7 +545,7 @@ mod test { solana_logger::setup(); let (t, exit, _receiver, server_address) = setup_quic_server(); - let runtime = rt(); + let runtime = rt("solQuicTestRt".to_string()); runtime.block_on(check_block_multiple_connections(server_address)); exit.store(true, Ordering::Relaxed); t.join().unwrap(); @@ -563,6 +565,7 @@ mod test { thread: t, key_updater: _, } = spawn_server( + "solQuicTest", "quic_streamer_test", s, &keypair, @@ -577,7 +580,7 @@ mod test { ) .unwrap(); - let runtime = rt(); + let runtime = rt("solQuicTestRt".to_string()); runtime.block_on(check_multiple_streams(receiver, server_address)); exit.store(true, Ordering::Relaxed); t.join().unwrap(); @@ -588,7 +591,7 @@ mod test { solana_logger::setup(); let (t, exit, receiver, server_address) = setup_quic_server(); - let runtime = rt(); + let runtime = rt("solQuicTestRt".to_string()); runtime.block_on(check_multiple_writes(receiver, server_address, None)); exit.store(true, Ordering::Relaxed); t.join().unwrap(); @@ -608,6 +611,7 @@ mod test { thread: t, key_updater: _, } = spawn_server( + "solQuicTest", "quic_streamer_test", s, &keypair, @@ -622,7 +626,7 @@ mod test { ) .unwrap(); - let runtime = rt(); + let runtime = rt("solQuicTestRt".to_string()); runtime.block_on(check_unstaked_node_connect_failure(server_address)); exit.store(true, Ordering::Relaxed); t.join().unwrap(); From 1110fc93d75dbf294eb61534152fe2a891c98407 Mon Sep 17 00:00:00 2001 From: steviez Date: Tue, 5 Mar 2024 22:02:04 -0600 Subject: [PATCH 14/84] Give SigVerify and ShredFetch threads unique names (#98) - solTvuFetchPmod ==> solTvuPktMod + solTvuRepPktMod - solSigVerifier ==> solSigVerTpu + solSigVerTpuVot --- core/benches/sigverify_stage.rs | 2 +- core/src/shred_fetch_stage.rs | 5 ++++- core/src/sigverify_stage.rs | 23 +++++++++-------------- core/src/tpu.rs | 9 +++++++-- 4 files changed, 21 insertions(+), 18 deletions(-) diff --git a/core/benches/sigverify_stage.rs b/core/benches/sigverify_stage.rs index 70f33020dd3e70..7013f718e4ab2e 100644 --- a/core/benches/sigverify_stage.rs +++ b/core/benches/sigverify_stage.rs @@ -160,7 +160,7 @@ fn bench_sigverify_stage(bencher: &mut Bencher, use_same_tx: bool) { let (packet_s, packet_r) = unbounded(); let (verified_s, verified_r) = BankingTracer::channel_for_test(); let verifier = TransactionSigVerifier::new(verified_s); - let stage = SigVerifyStage::new(packet_r, verifier, "bench"); + let stage = SigVerifyStage::new(packet_r, verifier, "solSigVerBench", "bench"); bencher.iter(move || { let now = Instant::now(); diff --git a/core/src/shred_fetch_stage.rs b/core/src/shred_fetch_stage.rs index 39cc193adad96e..84f1520e649822 100644 --- a/core/src/shred_fetch_stage.rs +++ b/core/src/shred_fetch_stage.rs @@ -148,6 +148,7 @@ impl ShredFetchStage { #[allow(clippy::too_many_arguments)] fn packet_modifier( receiver_thread_name: &'static str, + modifier_thread_name: &'static str, sockets: Vec>, exit: Arc, sender: Sender, @@ -178,7 +179,7 @@ impl ShredFetchStage { }) .collect(); let modifier_hdl = Builder::new() - .name("solTvuFetchPMod".to_string()) + .name(modifier_thread_name.to_string()) .spawn(move || { let repair_context = repair_context .as_ref() @@ -215,6 +216,7 @@ impl ShredFetchStage { let (mut tvu_threads, tvu_filter) = Self::packet_modifier( "solRcvrShred", + "solTvuPktMod", sockets, exit.clone(), sender.clone(), @@ -229,6 +231,7 @@ impl ShredFetchStage { let (repair_receiver, repair_handler) = Self::packet_modifier( "solRcvrShredRep", + "solTvuRepPktMod", vec![repair_socket.clone()], exit.clone(), sender.clone(), diff --git a/core/src/sigverify_stage.rs b/core/src/sigverify_stage.rs index e5e06a3bc701c9..cde1735611c0d0 100644 --- a/core/src/sigverify_stage.rs +++ b/core/src/sigverify_stage.rs @@ -238,9 +238,11 @@ impl SigVerifyStage { pub fn new( packet_receiver: Receiver, verifier: T, - name: &'static str, + thread_name: &'static str, + metrics_name: &'static str, ) -> Self { - let thread_hdl = Self::verifier_services(packet_receiver, verifier, name); + let thread_hdl = + Self::verifier_service(packet_receiver, verifier, thread_name, metrics_name); Self { thread_hdl } } @@ -407,7 +409,8 @@ impl SigVerifyStage { fn verifier_service( packet_receiver: Receiver, mut verifier: T, - name: &'static str, + thread_name: &'static str, + metrics_name: &'static str, ) -> JoinHandle<()> { let mut stats = SigVerifierStats::default(); let mut last_print = Instant::now(); @@ -415,7 +418,7 @@ impl SigVerifyStage { const DEDUPER_FALSE_POSITIVE_RATE: f64 = 0.001; const DEDUPER_NUM_BITS: u64 = 63_999_979; Builder::new() - .name("solSigVerifier".to_string()) + .name(thread_name.to_string()) .spawn(move || { let mut rng = rand::thread_rng(); let mut deduper = Deduper::<2, [u8]>::new(&mut rng, DEDUPER_NUM_BITS); @@ -440,7 +443,7 @@ impl SigVerifyStage { } } if last_print.elapsed().as_secs() > 2 { - stats.report(name); + stats.report(metrics_name); stats = SigVerifierStats::default(); last_print = Instant::now(); } @@ -449,14 +452,6 @@ impl SigVerifyStage { .unwrap() } - fn verifier_services( - packet_receiver: Receiver, - verifier: T, - name: &'static str, - ) -> JoinHandle<()> { - Self::verifier_service(packet_receiver, verifier, name) - } - pub fn join(self) -> thread::Result<()> { self.thread_hdl.join() } @@ -552,7 +547,7 @@ mod tests { let (packet_s, packet_r) = unbounded(); let (verified_s, verified_r) = BankingTracer::channel_for_test(); let verifier = TransactionSigVerifier::new(verified_s); - let stage = SigVerifyStage::new(packet_r, verifier, "test"); + let stage = SigVerifyStage::new(packet_r, verifier, "solSigVerTest", "test"); let now = Instant::now(); let packets_per_batch = 128; diff --git a/core/src/tpu.rs b/core/src/tpu.rs index 0f5431da4eb43c..640caf64544d45 100644 --- a/core/src/tpu.rs +++ b/core/src/tpu.rs @@ -190,14 +190,19 @@ impl Tpu { let sigverify_stage = { let verifier = TransactionSigVerifier::new(non_vote_sender); - SigVerifyStage::new(packet_receiver, verifier, "tpu-verifier") + SigVerifyStage::new(packet_receiver, verifier, "solSigVerTpu", "tpu-verifier") }; let (tpu_vote_sender, tpu_vote_receiver) = banking_tracer.create_channel_tpu_vote(); let vote_sigverify_stage = { let verifier = TransactionSigVerifier::new_reject_non_vote(tpu_vote_sender); - SigVerifyStage::new(vote_packet_receiver, verifier, "tpu-vote-verifier") + SigVerifyStage::new( + vote_packet_receiver, + verifier, + "solSigVerTpuVot", + "tpu-vote-verifier", + ) }; let (gossip_vote_sender, gossip_vote_receiver) = From 3cf48347a984563595e2808dff7482af00888da8 Mon Sep 17 00:00:00 2001 From: Tyera Date: Wed, 6 Mar 2024 00:23:32 -0700 Subject: [PATCH 15/84] Cli: improve vote-account vote-authority display (#95) * Simplify vote-authority display * Add handling for new vote authority * Add proper None handling, because unit test (shouldn't happen IRL, though) * Unwrap->expect --- cli-output/src/cli_output.rs | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/cli-output/src/cli_output.rs b/cli-output/src/cli_output.rs index 0eca9cde5c1a52..62b66eddf27eb0 100644 --- a/cli-output/src/cli_output.rs +++ b/cli-output/src/cli_output.rs @@ -1648,7 +1648,23 @@ impl VerboseDisplay for CliAuthorizedVoters {} impl fmt::Display for CliAuthorizedVoters { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{:?}", self.authorized_voters) + if let Some((_epoch, current_authorized_voter)) = self.authorized_voters.first_key_value() { + write!(f, "{current_authorized_voter}")?; + } else { + write!(f, "None")?; + } + if self.authorized_voters.len() > 1 { + let (epoch, upcoming_authorized_voter) = self + .authorized_voters + .last_key_value() + .expect("CliAuthorizedVoters::authorized_voters.len() > 1"); + writeln!(f)?; + write!( + f, + " New Vote Authority as of Epoch {epoch}: {upcoming_authorized_voter}" + )?; + } + Ok(()) } } @@ -3379,12 +3395,12 @@ mod tests { ..CliVoteAccount::default() }; let s = format!("{c}"); - assert_eq!(s, "Account Balance: 0.00001 SOL\nValidator Identity: 11111111111111111111111111111111\nVote Authority: {}\nWithdraw Authority: \nCredits: 0\nCommission: 0%\nRoot Slot: ~\nRecent Timestamp: 1970-01-01T00:00:00Z from slot 0\nEpoch Rewards:\n Epoch Reward Slot Time Amount New Balance Percent Change APR Commission\n 1 100 1970-01-01 00:00:00 UTC ◎0.000000010 ◎0.000000100 11.000% 10.00% 1%\n 2 200 1970-01-12 13:46:40 UTC ◎0.000000012 ◎0.000000100 11.000% 13.00% 1%\n"); + assert_eq!(s, "Account Balance: 0.00001 SOL\nValidator Identity: 11111111111111111111111111111111\nVote Authority: None\nWithdraw Authority: \nCredits: 0\nCommission: 0%\nRoot Slot: ~\nRecent Timestamp: 1970-01-01T00:00:00Z from slot 0\nEpoch Rewards:\n Epoch Reward Slot Time Amount New Balance Percent Change APR Commission\n 1 100 1970-01-01 00:00:00 UTC ◎0.000000010 ◎0.000000100 11.000% 10.00% 1%\n 2 200 1970-01-12 13:46:40 UTC ◎0.000000012 ◎0.000000100 11.000% 13.00% 1%\n"); println!("{s}"); c.use_csv = true; let s = format!("{c}"); - assert_eq!(s, "Account Balance: 0.00001 SOL\nValidator Identity: 11111111111111111111111111111111\nVote Authority: {}\nWithdraw Authority: \nCredits: 0\nCommission: 0%\nRoot Slot: ~\nRecent Timestamp: 1970-01-01T00:00:00Z from slot 0\nEpoch Rewards:\nEpoch,Reward Slot,Time,Amount,New Balance,Percent Change,APR,Commission\n1,100,1970-01-01 00:00:00 UTC,0.00000001,0.0000001,11%,10.00%,1%\n2,200,1970-01-12 13:46:40 UTC,0.000000012,0.0000001,11%,13.00%,1%\n"); + assert_eq!(s, "Account Balance: 0.00001 SOL\nValidator Identity: 11111111111111111111111111111111\nVote Authority: None\nWithdraw Authority: \nCredits: 0\nCommission: 0%\nRoot Slot: ~\nRecent Timestamp: 1970-01-01T00:00:00Z from slot 0\nEpoch Rewards:\nEpoch,Reward Slot,Time,Amount,New Balance,Percent Change,APR,Commission\n1,100,1970-01-01 00:00:00 UTC,0.00000001,0.0000001,11%,10.00%,1%\n2,200,1970-01-12 13:46:40 UTC,0.000000012,0.0000001,11%,13.00%,1%\n"); println!("{s}"); } } From b6f6fdbc9a01ce3b37823fdc10212d807c8ed8fe Mon Sep 17 00:00:00 2001 From: Jon C Date: Wed, 6 Mar 2024 15:32:05 +0100 Subject: [PATCH 16/84] frozen-abi: Remove proc_macro_hygiene featurization (#109) --- frozen-abi/build.rs | 5 ----- frozen-abi/src/lib.rs | 1 - perf/build.rs | 5 ----- programs/address-lookup-table/src/lib.rs | 1 - sdk/program/src/lib.rs | 1 - sdk/src/lib.rs | 1 - 6 files changed, 14 deletions(-) diff --git a/frozen-abi/build.rs b/frozen-abi/build.rs index c9550c1c5c4f22..e17ca70cb4718b 100644 --- a/frozen-abi/build.rs +++ b/frozen-abi/build.rs @@ -17,11 +17,6 @@ fn main() { } Channel::Dev => { println!("cargo:rustc-cfg=RUSTC_WITH_SPECIALIZATION"); - // See https://github.com/solana-labs/solana/issues/11055 - // We may be running the custom `rust-bpf-builder` toolchain, - // which currently needs `#![feature(proc_macro_hygiene)]` to - // be applied. - println!("cargo:rustc-cfg=RUSTC_NEEDS_PROC_MACRO_HYGIENE"); } } } diff --git a/frozen-abi/src/lib.rs b/frozen-abi/src/lib.rs index 189535ccddaa74..4747cf64b9e50f 100644 --- a/frozen-abi/src/lib.rs +++ b/frozen-abi/src/lib.rs @@ -1,6 +1,5 @@ #![allow(incomplete_features)] #![cfg_attr(RUSTC_WITH_SPECIALIZATION, feature(specialization))] -#![cfg_attr(RUSTC_NEEDS_PROC_MACRO_HYGIENE, feature(proc_macro_hygiene))] // Allows macro expansion of `use ::solana_frozen_abi::*` to work within this crate extern crate self as solana_frozen_abi; diff --git a/perf/build.rs b/perf/build.rs index 4925ee898eb612..eef20dd887bc42 100644 --- a/perf/build.rs +++ b/perf/build.rs @@ -27,11 +27,6 @@ fn main() { } Channel::Dev => { println!("cargo:rustc-cfg=RUSTC_WITH_SPECIALIZATION"); - // See https://github.com/solana-labs/solana/issues/11055 - // We may be running the custom `rust-bpf-builder` toolchain, - // which currently needs `#![feature(proc_macro_hygiene)]` to - // be applied. - println!("cargo:rustc-cfg=RUSTC_NEEDS_PROC_MACRO_HYGIENE"); } } } diff --git a/programs/address-lookup-table/src/lib.rs b/programs/address-lookup-table/src/lib.rs index 737ec32c8f6782..737c35e4c4b2f4 100644 --- a/programs/address-lookup-table/src/lib.rs +++ b/programs/address-lookup-table/src/lib.rs @@ -1,6 +1,5 @@ #![allow(incomplete_features)] #![cfg_attr(RUSTC_WITH_SPECIALIZATION, feature(specialization))] -#![cfg_attr(RUSTC_NEEDS_PROC_MACRO_HYGIENE, feature(proc_macro_hygiene))] #[cfg(not(target_os = "solana"))] pub mod processor; diff --git a/sdk/program/src/lib.rs b/sdk/program/src/lib.rs index 54de9d817205a8..4d623524772ccb 100644 --- a/sdk/program/src/lib.rs +++ b/sdk/program/src/lib.rs @@ -465,7 +465,6 @@ #![allow(incomplete_features)] #![cfg_attr(RUSTC_WITH_SPECIALIZATION, feature(specialization))] -#![cfg_attr(RUSTC_NEEDS_PROC_MACRO_HYGIENE, feature(proc_macro_hygiene))] // Allows macro expansion of `use ::solana_program::*` to work within this crate extern crate self as solana_program; diff --git a/sdk/src/lib.rs b/sdk/src/lib.rs index 7c6b643884e449..ecc186f0494191 100644 --- a/sdk/src/lib.rs +++ b/sdk/src/lib.rs @@ -31,7 +31,6 @@ #![allow(incomplete_features)] #![cfg_attr(RUSTC_WITH_SPECIALIZATION, feature(specialization))] -#![cfg_attr(RUSTC_NEEDS_PROC_MACRO_HYGIENE, feature(proc_macro_hygiene))] // Allows macro expansion of `use ::solana_sdk::*` to work within this crate extern crate self as solana_sdk; From c1613517bf43b2f67c1feb4db3fb7536116fa9ff Mon Sep 17 00:00:00 2001 From: Tao Zhu <82401714+tao-stones@users.noreply.github.com> Date: Wed, 6 Mar 2024 11:08:49 -0600 Subject: [PATCH 17/84] assert simple vote tx const cost (#100) * assert simple vote tx const cost --- cost-model/src/transaction_cost.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/cost-model/src/transaction_cost.rs b/cost-model/src/transaction_cost.rs index e765eee3bc7038..76865fff30fd57 100644 --- a/cost-model/src/transaction_cost.rs +++ b/cost-model/src/transaction_cost.rs @@ -18,8 +18,19 @@ pub enum TransactionCost { impl TransactionCost { pub fn sum(&self) -> u64 { + #![allow(clippy::assertions_on_constants)] match self { - Self::SimpleVote { .. } => SIMPLE_VOTE_USAGE_COST, + Self::SimpleVote { .. } => { + const _: () = assert!( + SIMPLE_VOTE_USAGE_COST + == solana_vote_program::vote_processor::DEFAULT_COMPUTE_UNITS + + block_cost_limits::SIGNATURE_COST + + 2 * block_cost_limits::WRITE_LOCK_UNITS + + 8 + ); + + SIMPLE_VOTE_USAGE_COST + } Self::Transaction(usage_cost) => usage_cost.sum(), } } From b38ea4145ed60d882ae8fc348d28f9e91467d66c Mon Sep 17 00:00:00 2001 From: steviez Date: Wed, 6 Mar 2024 14:49:32 -0600 Subject: [PATCH 18/84] Use tokio directly instead of jsonrpc_server_utils's re-export (#116) --- Cargo.lock | 4 ++-- Cargo.toml | 1 - geyser-plugin-manager/Cargo.toml | 2 +- geyser-plugin-manager/src/geyser_plugin_manager.rs | 2 +- programs/sbf/Cargo.lock | 4 ++-- validator/Cargo.toml | 2 +- validator/src/admin_rpc_service.rs | 6 +++--- 7 files changed, 10 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 78ff40111ee0b0..db5431da6ef62e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -205,7 +205,6 @@ dependencies = [ "jsonrpc-core-client", "jsonrpc-derive", "jsonrpc-ipc-server", - "jsonrpc-server-utils", "lazy_static", "libc", "libloading", @@ -252,6 +251,7 @@ dependencies = [ "symlink", "thiserror", "tikv-jemallocator", + "tokio", ] [[package]] @@ -6232,7 +6232,6 @@ dependencies = [ "crossbeam-channel", "json5", "jsonrpc-core", - "jsonrpc-server-utils", "libloading", "log", "serde_json", @@ -6246,6 +6245,7 @@ dependencies = [ "solana-sdk", "solana-transaction-status", "thiserror", + "tokio", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 27376370297e26..4b8ae12dab0078 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -236,7 +236,6 @@ jsonrpc-derive = "18.0.0" jsonrpc-http-server = "18.0.0" jsonrpc-ipc-server = "18.0.0" jsonrpc-pubsub = "18.0.0" -jsonrpc-server-utils = "18.0.0" lazy_static = "1.4.0" libc = "0.2.153" libloading = "0.7.4" diff --git a/geyser-plugin-manager/Cargo.toml b/geyser-plugin-manager/Cargo.toml index ebef2f637f642d..a7b02f8d593a8d 100644 --- a/geyser-plugin-manager/Cargo.toml +++ b/geyser-plugin-manager/Cargo.toml @@ -15,7 +15,6 @@ bs58 = { workspace = true } crossbeam-channel = { workspace = true } json5 = { workspace = true } jsonrpc-core = { workspace = true } -jsonrpc-server-utils = { workspace = true } libloading = { workspace = true } log = { workspace = true } serde_json = { workspace = true } @@ -29,6 +28,7 @@ solana-runtime = { workspace = true } solana-sdk = { workspace = true } solana-transaction-status = { workspace = true } thiserror = { workspace = true } +tokio = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] diff --git a/geyser-plugin-manager/src/geyser_plugin_manager.rs b/geyser-plugin-manager/src/geyser_plugin_manager.rs index 3d0abe16899637..d88814d88e9470 100644 --- a/geyser-plugin-manager/src/geyser_plugin_manager.rs +++ b/geyser-plugin-manager/src/geyser_plugin_manager.rs @@ -1,13 +1,13 @@ use { agave_geyser_plugin_interface::geyser_plugin_interface::GeyserPlugin, jsonrpc_core::{ErrorCode, Result as JsonRpcResult}, - jsonrpc_server_utils::tokio::sync::oneshot::Sender as OneShotSender, libloading::Library, log::*, std::{ ops::{Deref, DerefMut}, path::Path, }, + tokio::sync::oneshot::Sender as OneShotSender, }; #[derive(Debug)] diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index cb0ad6f1ee448c..11a4bcab04d7c0 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -90,7 +90,6 @@ dependencies = [ "jsonrpc-core-client", "jsonrpc-derive", "jsonrpc-ipc-server", - "jsonrpc-server-utils", "lazy_static", "libc", "libloading", @@ -135,6 +134,7 @@ dependencies = [ "symlink", "thiserror", "tikv-jemallocator", + "tokio", ] [[package]] @@ -5127,7 +5127,6 @@ dependencies = [ "crossbeam-channel", "json5", "jsonrpc-core", - "jsonrpc-server-utils", "libloading", "log", "serde_json", @@ -5141,6 +5140,7 @@ dependencies = [ "solana-sdk", "solana-transaction-status", "thiserror", + "tokio", ] [[package]] diff --git a/validator/Cargo.toml b/validator/Cargo.toml index 362a07343b5e4a..844a2bca9aa97f 100644 --- a/validator/Cargo.toml +++ b/validator/Cargo.toml @@ -24,7 +24,6 @@ jsonrpc-core = { workspace = true } jsonrpc-core-client = { workspace = true, features = ["ipc"] } jsonrpc-derive = { workspace = true } jsonrpc-ipc-server = { workspace = true } -jsonrpc-server-utils = { workspace = true } lazy_static = { workspace = true } libloading = { workspace = true } log = { workspace = true } @@ -66,6 +65,7 @@ solana-version = { workspace = true } solana-vote-program = { workspace = true } symlink = { workspace = true } thiserror = { workspace = true } +tokio = { workspace = true } [dev-dependencies] solana-account-decoder = { workspace = true } diff --git a/validator/src/admin_rpc_service.rs b/validator/src/admin_rpc_service.rs index 57be4cf488865d..3881487882dc2e 100644 --- a/validator/src/admin_rpc_service.rs +++ b/validator/src/admin_rpc_service.rs @@ -6,7 +6,6 @@ use { jsonrpc_ipc_server::{ tokio::sync::oneshot::channel as oneshot_channel, RequestContext, ServerBuilder, }, - jsonrpc_server_utils::tokio, log::*, serde::{de::Deserializer, Deserialize, Serialize}, solana_accounts_db::accounts_index::AccountIndex, @@ -35,6 +34,7 @@ use { thread::{self, Builder}, time::{Duration, SystemTime}, }, + tokio::runtime::Runtime, }; #[derive(Clone)] @@ -815,8 +815,8 @@ pub async fn connect(ledger_path: &Path) -> std::result::Result jsonrpc_server_utils::tokio::runtime::Runtime { - jsonrpc_server_utils::tokio::runtime::Runtime::new().expect("new tokio runtime") +pub fn runtime() -> Runtime { + Runtime::new().expect("new tokio runtime") } #[derive(Default, Deserialize, Clone)] From 184b31c6b7a653cecb1941c295479bacc225cb3a Mon Sep 17 00:00:00 2001 From: Brennan Date: Wed, 6 Mar 2024 14:03:25 -0800 Subject: [PATCH 19/84] fix typo (#57) --- program-runtime/src/loaded_programs.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/program-runtime/src/loaded_programs.rs b/program-runtime/src/loaded_programs.rs index 926d1179837380..e8a691c537934f 100644 --- a/program-runtime/src/loaded_programs.rs +++ b/program-runtime/src/loaded_programs.rs @@ -67,7 +67,7 @@ pub enum LoadedProgramType { /// /// These can potentially come back alive if the environment changes. FailedVerification(ProgramRuntimeEnvironment), - /// Tombstone for programs which were explicitly undeployoed / closed. + /// Tombstone for programs which were explicitly undeployed / closed. #[default] Closed, /// Tombstone for programs which have recently been modified but the new version is not visible yet. From 8887cd19a1a86f61cfa44fb41a6e528e8857bc59 Mon Sep 17 00:00:00 2001 From: steviez Date: Wed, 6 Mar 2024 17:03:02 -0600 Subject: [PATCH 20/84] Name previously unnamed thread pool threads (#104) Several rayon and tokio threadpools did not have names; give them names to make tracking them in debug tools easier --- net-utils/src/ip_echo_server.rs | 6 +++++- runtime/src/bank.rs | 5 ++++- runtime/src/snapshot_utils/snapshot_storage_rebuilder.rs | 3 ++- validator/src/admin_rpc_service.rs | 6 +++++- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/net-utils/src/ip_echo_server.rs b/net-utils/src/ip_echo_server.rs index 7d4186ccb6a810..64fbedadc7acf9 100644 --- a/net-utils/src/ip_echo_server.rs +++ b/net-utils/src/ip_echo_server.rs @@ -173,7 +173,11 @@ pub fn ip_echo_server( ) -> IpEchoServer { tcp_listener.set_nonblocking(true).unwrap(); - let runtime = Runtime::new().expect("Failed to create Runtime"); + let runtime = tokio::runtime::Builder::new_multi_thread() + .thread_name("solIpEchoSrvrRt") + .enable_all() + .build() + .expect("new tokio runtime"); runtime.spawn(run_echo_server(tcp_listener, shred_version)); runtime } diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index 3ea316f857a2bc..1abf9403e3fef1 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -1483,7 +1483,10 @@ impl Bank { let epoch = self.epoch(); let slot = self.slot(); let (thread_pool, thread_pool_time) = measure!( - ThreadPoolBuilder::new().build().unwrap(), + ThreadPoolBuilder::new() + .thread_name(|i| format!("solBnkNewEpch{i:02}")) + .build() + .expect("new rayon threadpool"), "thread_pool_creation", ); diff --git a/runtime/src/snapshot_utils/snapshot_storage_rebuilder.rs b/runtime/src/snapshot_utils/snapshot_storage_rebuilder.rs index 0c6116274b1cb1..30cbbd4afd5970 100644 --- a/runtime/src/snapshot_utils/snapshot_storage_rebuilder.rs +++ b/runtime/src/snapshot_utils/snapshot_storage_rebuilder.rs @@ -418,9 +418,10 @@ impl SnapshotStorageRebuilder { /// Builds thread pool to rebuild with fn build_thread_pool(&self) -> ThreadPool { ThreadPoolBuilder::default() + .thread_name(|i| format!("solRbuildSnap{i:02}")) .num_threads(self.num_threads) .build() - .unwrap() + .expect("new rayon threadpool") } } diff --git a/validator/src/admin_rpc_service.rs b/validator/src/admin_rpc_service.rs index 3881487882dc2e..b6d65e3ec4a4df 100644 --- a/validator/src/admin_rpc_service.rs +++ b/validator/src/admin_rpc_service.rs @@ -816,7 +816,11 @@ pub async fn connect(ledger_path: &Path) -> std::result::Result Runtime { - Runtime::new().expect("new tokio runtime") + tokio::runtime::Builder::new_multi_thread() + .thread_name("solAdminRpcRt") + .enable_all() + .build() + .expect("new tokio runtime") } #[derive(Default, Deserialize, Clone)] From 9cc55349f7c1d1e7d046cbeb9b503f4de02c0888 Mon Sep 17 00:00:00 2001 From: Justin Starry Date: Thu, 7 Mar 2024 09:52:23 +0800 Subject: [PATCH 21/84] Refactor transaction account unlocking (#103) refactor: unlock accounts --- accounts-db/src/accounts.rs | 64 ++++++++++++++++++++++---------- runtime/src/bank.rs | 12 +++--- runtime/src/transaction_batch.rs | 9 ++++- 3 files changed, 57 insertions(+), 28 deletions(-) diff --git a/accounts-db/src/accounts.rs b/accounts-db/src/accounts.rs index 371db9eb08c095..33a57d56461c78 100644 --- a/accounts-db/src/accounts.rs +++ b/accounts-db/src/accounts.rs @@ -80,11 +80,20 @@ impl AccountLocks { if *count == 0 { occupied_entry.remove_entry(); } + } else { + debug_assert!( + false, + "Attempted to remove a read-lock for a key that wasn't read-locked" + ); } } fn unlock_write(&mut self, key: &Pubkey) { - self.write_locks.remove(key); + let removed = self.write_locks.remove(key); + debug_assert!( + removed, + "Attempted to remove a write-lock for a key that wasn't write-locked" + ); } } @@ -618,14 +627,16 @@ impl Accounts { #[allow(clippy::needless_collect)] pub fn unlock_accounts<'a>( &self, - txs: impl Iterator, - results: &[Result<()>], + txs_and_results: impl Iterator)>, ) { - let keys: Vec<_> = txs - .zip(results) + let keys: Vec<_> = txs_and_results .filter(|(_, res)| res.is_ok()) .map(|(tx, _)| tx.get_account_locks_unchecked()) .collect(); + if keys.is_empty() { + return; + } + let mut account_locks = self.account_locks.lock().unwrap(); debug!("bank unlock accounts"); keys.into_iter().for_each(|keys| { @@ -812,6 +823,7 @@ mod tests { }, std::{ borrow::Cow, + iter, sync::atomic::{AtomicBool, AtomicU64, Ordering}, thread, time, }, @@ -1099,8 +1111,8 @@ mod tests { let txs = vec![new_sanitized_tx(&[&keypair], message, Hash::default())]; let results = accounts.lock_accounts(txs.iter(), MAX_TX_ACCOUNT_LOCKS); - assert_eq!(results[0], Ok(())); - accounts.unlock_accounts(txs.iter(), &results); + assert_eq!(results, vec![Ok(())]); + accounts.unlock_accounts(txs.iter().zip(&results)); } // Disallow over MAX_TX_ACCOUNT_LOCKS @@ -1156,7 +1168,7 @@ mod tests { let tx = new_sanitized_tx(&[&keypair0], message, Hash::default()); let results0 = accounts.lock_accounts([tx.clone()].iter(), MAX_TX_ACCOUNT_LOCKS); - assert!(results0[0].is_ok()); + assert_eq!(results0, vec![Ok(())]); assert_eq!( *accounts .account_locks @@ -1190,9 +1202,13 @@ mod tests { let tx1 = new_sanitized_tx(&[&keypair1], message, Hash::default()); let txs = vec![tx0, tx1]; let results1 = accounts.lock_accounts(txs.iter(), MAX_TX_ACCOUNT_LOCKS); - - assert!(results1[0].is_ok()); // Read-only account (keypair1) can be referenced multiple times - assert!(results1[1].is_err()); // Read-only account (keypair1) cannot also be locked as writable + assert_eq!( + results1, + vec![ + Ok(()), // Read-only account (keypair1) can be referenced multiple times + Err(TransactionError::AccountInUse), // Read-only account (keypair1) cannot also be locked as writable + ], + ); assert_eq!( *accounts .account_locks @@ -1204,8 +1220,8 @@ mod tests { 2 ); - accounts.unlock_accounts([tx].iter(), &results0); - accounts.unlock_accounts(txs.iter(), &results1); + accounts.unlock_accounts(iter::once(&tx).zip(&results0)); + accounts.unlock_accounts(txs.iter().zip(&results1)); let instructions = vec![CompiledInstruction::new(2, &(), vec![0, 1])]; let message = Message::new_with_compiled_instructions( 1, @@ -1217,7 +1233,10 @@ mod tests { ); let tx = new_sanitized_tx(&[&keypair1], message, Hash::default()); let results2 = accounts.lock_accounts([tx].iter(), MAX_TX_ACCOUNT_LOCKS); - assert!(results2[0].is_ok()); // Now keypair1 account can be locked as writable + assert_eq!( + results2, + vec![Ok(())] // Now keypair1 account can be locked as writable + ); // Check that read-only lock with zero references is deleted assert!(accounts @@ -1285,7 +1304,7 @@ mod tests { counter_clone.clone().fetch_add(1, Ordering::SeqCst); } } - accounts_clone.unlock_accounts(txs.iter(), &results); + accounts_clone.unlock_accounts(txs.iter().zip(&results)); if exit_clone.clone().load(Ordering::Relaxed) { break; } @@ -1301,7 +1320,7 @@ mod tests { thread::sleep(time::Duration::from_millis(50)); assert_eq!(counter_value, counter_clone.clone().load(Ordering::SeqCst)); } - accounts_arc.unlock_accounts(txs.iter(), &results); + accounts_arc.unlock_accounts(txs.iter().zip(&results)); thread::sleep(time::Duration::from_millis(50)); } exit.store(true, Ordering::Relaxed); @@ -1442,9 +1461,14 @@ mod tests { MAX_TX_ACCOUNT_LOCKS, ); - assert!(results[0].is_ok()); // Read-only account (keypair0) can be referenced multiple times - assert!(results[1].is_err()); // is not locked due to !qos_results[1].is_ok() - assert!(results[2].is_ok()); // Read-only account (keypair0) can be referenced multiple times + assert_eq!( + results, + vec![ + Ok(()), // Read-only account (keypair0) can be referenced multiple times + Err(TransactionError::WouldExceedMaxBlockCostLimit), // is not locked due to !qos_results[1].is_ok() + Ok(()), // Read-only account (keypair0) can be referenced multiple times + ], + ); // verify that keypair0 read-only lock twice (for tx0 and tx2) assert_eq!( @@ -1466,7 +1490,7 @@ mod tests { .get(&keypair2.pubkey()) .is_none()); - accounts.unlock_accounts(txs.iter(), &results); + accounts.unlock_accounts(txs.iter().zip(&results)); // check all locks to be removed assert!(accounts diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index 1abf9403e3fef1..39df91c382feff 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -4379,13 +4379,11 @@ impl Bank { account_overrides } - pub fn unlock_accounts(&self, batch: &mut TransactionBatch) { - if batch.needs_unlock() { - batch.set_needs_unlock(false); - self.rc - .accounts - .unlock_accounts(batch.sanitized_transactions().iter(), batch.lock_results()) - } + pub fn unlock_accounts<'a>( + &self, + txs_and_results: impl Iterator)>, + ) { + self.rc.accounts.unlock_accounts(txs_and_results) } pub fn remove_unrooted_slots(&self, slots: &[(Slot, BankId)]) { diff --git a/runtime/src/transaction_batch.rs b/runtime/src/transaction_batch.rs index 66711fd5a1acd5..9d0ff5fb7ce007 100644 --- a/runtime/src/transaction_batch.rs +++ b/runtime/src/transaction_batch.rs @@ -51,7 +51,14 @@ impl<'a, 'b> TransactionBatch<'a, 'b> { // Unlock all locked accounts in destructor. impl<'a, 'b> Drop for TransactionBatch<'a, 'b> { fn drop(&mut self) { - self.bank.unlock_accounts(self) + if self.needs_unlock() { + self.set_needs_unlock(false); + self.bank.unlock_accounts( + self.sanitized_transactions() + .iter() + .zip(self.lock_results()), + ) + } } } From f968532d7f4ed27e16d1e7c4a4a9a1824285e016 Mon Sep 17 00:00:00 2001 From: Tyera Date: Wed, 6 Mar 2024 19:31:07 -0700 Subject: [PATCH 22/84] Prep Anchor downstream CI job for v2 bump (#123) * Add new script to patch spl in anchor downstream * Only specify major version for token-2022 * Add update for ahash * Patch spl in anchor * Remove dex and metadata features for now --- scripts/build-downstream-anchor-projects.sh | 15 +++++- scripts/patch-spl-crates-for-anchor.sh | 55 +++++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 scripts/patch-spl-crates-for-anchor.sh diff --git a/scripts/build-downstream-anchor-projects.sh b/scripts/build-downstream-anchor-projects.sh index cdfa0bae10addb..7d75ccc08ab0e2 100755 --- a/scripts/build-downstream-anchor-projects.sh +++ b/scripts/build-downstream-anchor-projects.sh @@ -8,6 +8,7 @@ cd "$(dirname "$0")"/.. source ci/_ source scripts/patch-crates.sh source scripts/read-cargo-variable.sh +source scripts/patch-spl-crates-for-anchor.sh anchor_version=$1 solana_ver=$(readCargoVariable version Cargo.toml) @@ -43,6 +44,14 @@ EOF # NOTE This isn't run in a subshell to get $anchor_dir and $anchor_ver anchor() { set -x + + rm -rf spl + git clone https://github.com/solana-labs/solana-program-library.git spl + cd spl || exit 1 + spl_dir=$PWD + get_spl_versions "$spl_dir" + cd .. + rm -rf anchor git clone https://github.com/coral-xyz/anchor.git cd anchor || exit 1 @@ -57,9 +66,13 @@ anchor() { update_solana_dependencies . "$solana_ver" patch_crates_io_solana Cargo.toml "$solana_dir" + patch_spl_crates . Cargo.toml "$spl_dir" $cargo test - (cd spl && $cargo_build_sbf --features dex metadata stake) + # serum_dex and mpl-token-metadata are using caret versions of solana and SPL dependencies + # rather pull and patch those as well, ignore for now + # (cd spl && $cargo_build_sbf --features dex metadata stake) + (cd spl && $cargo_build_sbf --features stake) (cd client && $cargo test --all-features) anchor_dir=$PWD diff --git a/scripts/patch-spl-crates-for-anchor.sh b/scripts/patch-spl-crates-for-anchor.sh new file mode 100644 index 00000000000000..93ea67b8fceb20 --- /dev/null +++ b/scripts/patch-spl-crates-for-anchor.sh @@ -0,0 +1,55 @@ +spl_memo_version= +spl_token_version= +spl_token_2022_version= +spl_tlv_account_resolution_verison= +spl_transfer_hook_interface_version= + +get_spl_versions() { + declare spl_dir="$1" + spl_memo_version=$(readCargoVariable version "$spl_dir/memo/program/Cargo.toml") + spl_token_version=$(readCargoVariable version "$spl_dir/token/program/Cargo.toml") + spl_token_2022_version=$(readCargoVariable version "$spl_dir/token/program-2022/Cargo.toml"| head -c1) # only use the major version for convenience + spl_tlv_account_resolution_verison=$(readCargoVariable version "$spl_dir/libraries/tlv-account-resolution/Cargo.toml") + spl_transfer_hook_interface_version=$(readCargoVariable version "$spl_dir/token/transfer-hook/interface/Cargo.toml") +} + +patch_spl_crates() { + declare project_root="$1" + declare Cargo_toml="$2" + declare spl_dir="$3" + update_spl_dependencies "$project_root" + patch_crates_io "$Cargo_toml" "$spl_dir" +} + +update_spl_dependencies() { + declare project_root="$1" + declare tomls=() + while IFS='' read -r line; do tomls+=("$line"); done < <(find "$project_root" -name Cargo.toml) + + sed -i -e "s#\(spl-memo = \"\)[^\"]*\(\"\)#\1$spl_memo_version\2#g" "${tomls[@]}" || return $? + sed -i -e "s#\(spl-memo = { version = \"\)[^\"]*\(\"\)#\1$spl_memo_version\2#g" "${tomls[@]}" || return $? + sed -i -e "s#\(spl-token = \"\)[^\"]*\(\"\)#\1$spl_token_version\2#g" "${tomls[@]}" || return $? + sed -i -e "s#\(spl-token = { version = \"\)[^\"]*\(\"\)#\1$spl_token_version\2#g" "${tomls[@]}" || return $? + sed -i -e "s#\(spl-token-2022 = \"\).*\(\"\)#\1$spl_token_2022_version\2#g" "${tomls[@]}" || return $? + sed -i -e "s#\(spl-token-2022 = { version = \"\)[^\"]*\(\"\)#\1$spl_token_2022_version\2#g" "${tomls[@]}" || return $? + sed -i -e "s#\(spl-tlv-account-resolution = \"\)[^\"]*\(\"\)#\1=$spl_tlv_account_resolution_verison\2#g" "${tomls[@]}" || return $? + sed -i -e "s#\(spl-tlv-account-resolution = { version = \"\)[^\"]*\(\"\)#\1=$spl_tlv_account_resolution_verison\2#g" "${tomls[@]}" || return $? + sed -i -e "s#\(spl-transfer-hook-interface = \"\)[^\"]*\(\"\)#\1=$spl_transfer_hook_interface_version\2#g" "${tomls[@]}" || return $? + sed -i -e "s#\(spl-transfer-hook-interface = { version = \"\)[^\"]*\(\"\)#\1=$spl_transfer_hook_interface_version\2#g" "${tomls[@]}" || return $? + + # patch ahash. This is super brittle; putting here for convenience, since we are already iterating through the tomls + ahash_minor_version="0.8" + sed -i -e "s#\(ahash = \"\)[^\"]*\(\"\)#\1$ahash_minor_version\2#g" "${tomls[@]}" || return $? +} + +patch_crates_io() { + declare Cargo_toml="$1" + declare spl_dir="$2" + cat >> "$Cargo_toml" < Date: Wed, 6 Mar 2024 18:51:50 -0800 Subject: [PATCH 23/84] [TieredStorage] Deprecate the use of account-hash in HotStorage (#93) #### Problem TieredStorage stores account hash as an optional field inside its HotStorage. However, the field isn't used and we have already decided to deprecate the account hash. #### Summary of Changes Remove account-hash from the tiered-storage. #### Test Plan Existing tiered-storage tests. Running validators w/ tiered-storage in mainnet-beta w/o storing account-hash. --- accounts-db/src/account_storage/meta.rs | 3 +- accounts-db/src/tiered_storage.rs | 8 +- accounts-db/src/tiered_storage/byte_block.rs | 25 +--- accounts-db/src/tiered_storage/hot.rs | 60 ++------- accounts-db/src/tiered_storage/meta.rs | 128 +++++-------------- accounts-db/src/tiered_storage/readable.rs | 6 - accounts-db/src/tiered_storage/test_utils.rs | 21 +-- 7 files changed, 58 insertions(+), 193 deletions(-) diff --git a/accounts-db/src/account_storage/meta.rs b/accounts-db/src/account_storage/meta.rs index 69c24d7be75f7d..b6c8d72042097a 100644 --- a/accounts-db/src/account_storage/meta.rs +++ b/accounts-db/src/account_storage/meta.rs @@ -128,7 +128,8 @@ impl<'storage> StoredAccountMeta<'storage> { pub fn hash(&self) -> &'storage AccountHash { match self { Self::AppendVec(av) => av.hash(), - Self::Hot(hot) => hot.hash().unwrap_or(&DEFAULT_ACCOUNT_HASH), + // tiered-storage has deprecated the use of AccountHash + Self::Hot(_) => &DEFAULT_ACCOUNT_HASH, } } diff --git a/accounts-db/src/tiered_storage.rs b/accounts-db/src/tiered_storage.rs index a6f4ea89428bf9..2f8ebac65e3b57 100644 --- a/accounts-db/src/tiered_storage.rs +++ b/accounts-db/src/tiered_storage.rs @@ -362,15 +362,15 @@ mod tests { let mut expected_accounts_map = HashMap::new(); for i in 0..num_accounts { - let (account, address, account_hash, _write_version) = storable_accounts.get(i); - expected_accounts_map.insert(address, (account, account_hash)); + let (account, address, _account_hash, _write_version) = storable_accounts.get(i); + expected_accounts_map.insert(address, account); } let mut index_offset = IndexOffset(0); let mut verified_accounts = HashSet::new(); while let Some((stored_meta, next)) = reader.get_account(index_offset).unwrap() { - if let Some((account, account_hash)) = expected_accounts_map.get(stored_meta.pubkey()) { - verify_test_account(&stored_meta, *account, stored_meta.pubkey(), account_hash); + if let Some(account) = expected_accounts_map.get(stored_meta.pubkey()) { + verify_test_account(&stored_meta, *account, stored_meta.pubkey()); verified_accounts.insert(stored_meta.pubkey()); } index_offset = next; diff --git a/accounts-db/src/tiered_storage/byte_block.rs b/accounts-db/src/tiered_storage/byte_block.rs index 1cd80add0c2307..6fc7dec611e9a9 100644 --- a/accounts-db/src/tiered_storage/byte_block.rs +++ b/accounts-db/src/tiered_storage/byte_block.rs @@ -95,9 +95,6 @@ impl ByteBlockWriter { if let Some(rent_epoch) = opt_fields.rent_epoch { size += self.write_pod(&rent_epoch)?; } - if let Some(hash) = opt_fields.account_hash { - size += self.write_pod(hash)?; - } debug_assert_eq!(size, opt_fields.size()); @@ -191,11 +188,7 @@ impl ByteBlockReader { #[cfg(test)] mod tests { - use { - super::*, - crate::accounts_hash::AccountHash, - solana_sdk::{hash::Hash, stake_history::Epoch}, - }; + use {super::*, solana_sdk::stake_history::Epoch}; fn read_type_unaligned(buffer: &[u8], offset: usize) -> (T, usize) { let size = std::mem::size_of::(); @@ -352,19 +345,13 @@ mod tests { let mut writer = ByteBlockWriter::new(format); let mut opt_fields_vec = vec![]; let mut some_count = 0; - let acc_hash = AccountHash(Hash::new_unique()); // prepare a vector of optional fields that contains all combinations // of Some and None. for rent_epoch in [None, Some(test_epoch)] { - for account_hash in [None, Some(&acc_hash)] { - some_count += rent_epoch.iter().count() + account_hash.iter().count(); + some_count += rent_epoch.iter().count(); - opt_fields_vec.push(AccountMetaOptionalFields { - rent_epoch, - account_hash, - }); - } + opt_fields_vec.push(AccountMetaOptionalFields { rent_epoch }); test_epoch += 1; } @@ -396,12 +383,6 @@ mod tests { verified_count += 1; offset += std::mem::size_of::(); } - if let Some(expected_hash) = opt_fields.account_hash { - let hash = read_pod::(&decoded_buffer, offset).unwrap(); - assert_eq!(hash, expected_hash); - verified_count += 1; - offset += std::mem::size_of::(); - } } // make sure the number of Some fields matches the number of fields we diff --git a/accounts-db/src/tiered_storage/hot.rs b/accounts-db/src/tiered_storage/hot.rs index f662c2e062ee11..34f7915186ba9b 100644 --- a/accounts-db/src/tiered_storage/hot.rs +++ b/accounts-db/src/tiered_storage/hot.rs @@ -242,19 +242,6 @@ impl TieredAccountMeta for HotAccountMeta { .flatten() } - /// Returns the account hash by parsing the specified account block. None - /// will be returned if this account does not persist this optional field. - fn account_hash<'a>(&self, account_block: &'a [u8]) -> Option<&'a AccountHash> { - self.flags() - .has_account_hash() - .then(|| { - let offset = self.optional_fields_offset(account_block) - + AccountMetaOptionalFields::account_hash_offset(self.flags()); - byte_block::read_pod::(account_block, offset) - }) - .flatten() - } - /// Returns the offset of the optional fields based on the specified account /// block. fn optional_fields_offset(&self, account_block: &[u8]) -> usize { @@ -488,9 +475,6 @@ fn write_optional_fields( if let Some(rent_epoch) = opt_fields.rent_epoch { size += file.write_pod(&rent_epoch)?; } - if let Some(hash) = opt_fields.account_hash { - size += file.write_pod(hash)?; - } debug_assert_eq!(size, opt_fields.size()); @@ -520,12 +504,8 @@ impl HotStorageWriter { account_data: &[u8], executable: bool, rent_epoch: Option, - account_hash: Option<&AccountHash>, ) -> TieredStorageResult { - let optional_fields = AccountMetaOptionalFields { - rent_epoch, - account_hash, - }; + let optional_fields = AccountMetaOptionalFields { rent_epoch }; let mut flags = AccountMetaFlags::new_from(&optional_fields); flags.set_executable(executable); @@ -574,7 +554,7 @@ impl HotStorageWriter { let total_input_accounts = len - skip; let mut stored_infos = Vec::with_capacity(total_input_accounts); for i in skip..len { - let (account, address, account_hash, _write_version) = accounts.get(i); + let (account, address, _account_hash, _write_version) = accounts.get(i); let index_entry = AccountIndexWriterEntry { address, offset: HotAccountOffset::new(cursor)?, @@ -582,7 +562,7 @@ impl HotStorageWriter { // Obtain necessary fields from the account, or default fields // for a zero-lamport account in the None case. - let (lamports, owner, data, executable, rent_epoch, account_hash) = account + let (lamports, owner, data, executable, rent_epoch) = account .map(|acc| { ( acc.lamports(), @@ -591,19 +571,12 @@ impl HotStorageWriter { acc.executable(), // only persist rent_epoch for those rent-paying accounts (acc.rent_epoch() != RENT_EXEMPT_RENT_EPOCH).then_some(acc.rent_epoch()), - Some(account_hash), ) }) - .unwrap_or((0, &OWNER_NO_OWNER, &[], false, None, None)); + .unwrap_or((0, &OWNER_NO_OWNER, &[], false, None)); let owner_offset = owners_table.insert(owner); - let stored_size = self.write_account( - lamports, - owner_offset, - data, - executable, - rent_epoch, - account_hash, - )?; + let stored_size = + self.write_account(lamports, owner_offset, data, executable, rent_epoch)?; cursor += stored_size; stored_infos.push(StoredAccountInfo { @@ -755,11 +728,9 @@ pub mod tests { const TEST_PADDING: u8 = 5; const TEST_OWNER_OFFSET: OwnerOffset = OwnerOffset(0x1fef_1234); const TEST_RENT_EPOCH: Epoch = 7; - let acc_hash = AccountHash(Hash::new_unique()); let optional_fields = AccountMetaOptionalFields { rent_epoch: Some(TEST_RENT_EPOCH), - account_hash: Some(&acc_hash), }; let flags = AccountMetaFlags::new_from(&optional_fields); @@ -779,7 +750,6 @@ pub mod tests { fn test_hot_account_meta_full() { let account_data = [11u8; 83]; let padding = [0u8; 5]; - let acc_hash = AccountHash(Hash::new_unique()); const TEST_LAMPORT: u64 = 2314232137; const OWNER_OFFSET: u32 = 0x1fef_1234; @@ -787,7 +757,6 @@ pub mod tests { let optional_fields = AccountMetaOptionalFields { rent_epoch: Some(TEST_RENT_EPOCH), - account_hash: Some(&acc_hash), }; let flags = AccountMetaFlags::new_from(&optional_fields); @@ -810,7 +779,6 @@ pub mod tests { let meta = byte_block::read_pod::(&buffer, 0).unwrap(); assert_eq!(expected_meta, *meta); assert!(meta.flags().has_rent_epoch()); - assert!(meta.flags().has_account_hash()); assert_eq!(meta.account_data_padding() as usize, padding.len()); let account_block = &buffer[std::mem::size_of::()..]; @@ -823,10 +791,6 @@ pub mod tests { assert_eq!(account_data.len(), meta.account_data_size(account_block)); assert_eq!(account_data, meta.account_data(account_block)); assert_eq!(meta.rent_epoch(account_block), optional_fields.rent_epoch); - assert_eq!( - (meta.account_hash(account_block).unwrap()), - optional_fields.account_hash.unwrap() - ); } #[test] @@ -1334,8 +1298,8 @@ pub mod tests { .unwrap() .unwrap(); - let (account, address, account_hash, _write_version) = storable_accounts.get(i); - verify_test_account(&stored_meta, account, address, account_hash); + let (account, address, _account_hash, _write_version) = storable_accounts.get(i); + verify_test_account(&stored_meta, account, address); assert_eq!(i + 1, next.0 as usize); } @@ -1352,9 +1316,9 @@ pub mod tests { .unwrap() .unwrap(); - let (account, address, account_hash, _write_version) = + let (account, address, _account_hash, _write_version) = storable_accounts.get(stored_info.offset); - verify_test_account(&stored_meta, account, address, account_hash); + verify_test_account(&stored_meta, account, address); } // verify get_accounts @@ -1362,8 +1326,8 @@ pub mod tests { // first, we verify everything for (i, stored_meta) in accounts.iter().enumerate() { - let (account, address, account_hash, _write_version) = storable_accounts.get(i); - verify_test_account(stored_meta, account, address, account_hash); + let (account, address, _account_hash, _write_version) = storable_accounts.get(i); + verify_test_account(stored_meta, account, address); } // second, we verify various initial position diff --git a/accounts-db/src/tiered_storage/meta.rs b/accounts-db/src/tiered_storage/meta.rs index 4e2bb0d95041ca..2aa53e5a4de1ed 100644 --- a/accounts-db/src/tiered_storage/meta.rs +++ b/accounts-db/src/tiered_storage/meta.rs @@ -1,7 +1,7 @@ //! The account meta and related structs for the tiered storage. use { - crate::{accounts_hash::AccountHash, tiered_storage::owners::OwnerOffset}, + crate::tiered_storage::owners::OwnerOffset, bytemuck::{Pod, Zeroable}, modular_bitfield::prelude::*, solana_sdk::stake_history::Epoch, @@ -14,12 +14,10 @@ use { pub struct AccountMetaFlags { /// whether the account meta has rent epoch pub has_rent_epoch: bool, - /// whether the account meta has account hash - pub has_account_hash: bool, /// whether the account is executable pub executable: bool, /// the reserved bits. - reserved: B29, + reserved: B30, } // Ensure there are no implicit padding bytes @@ -70,10 +68,6 @@ pub trait TieredAccountMeta: Sized { /// does not persist this optional field. fn rent_epoch(&self, _account_block: &[u8]) -> Option; - /// Returns the account hash by parsing the specified account block. None - /// will be returned if this account does not persist this optional field. - fn account_hash<'a>(&self, _account_block: &'a [u8]) -> Option<&'a AccountHash>; - /// Returns the offset of the optional fields based on the specified account /// block. fn optional_fields_offset(&self, _account_block: &[u8]) -> usize; @@ -91,7 +85,6 @@ impl AccountMetaFlags { pub fn new_from(optional_fields: &AccountMetaOptionalFields) -> Self { let mut flags = AccountMetaFlags::default(); flags.set_has_rent_epoch(optional_fields.rent_epoch.is_some()); - flags.set_has_account_hash(optional_fields.account_hash.is_some()); flags.set_executable(false); flags } @@ -102,20 +95,15 @@ impl AccountMetaFlags { /// Note that the storage representation of the optional fields might be /// different from its in-memory representation. #[derive(Debug, PartialEq, Eq, Clone)] -pub struct AccountMetaOptionalFields<'a> { +pub struct AccountMetaOptionalFields { /// the epoch at which its associated account will next owe rent pub rent_epoch: Option, - /// the hash of its associated account - pub account_hash: Option<&'a AccountHash>, } -impl<'a> AccountMetaOptionalFields<'a> { +impl AccountMetaOptionalFields { /// The size of the optional fields in bytes (excluding the boolean flags). pub fn size(&self) -> usize { self.rent_epoch.map_or(0, |_| std::mem::size_of::()) - + self - .account_hash - .map_or(0, |_| std::mem::size_of::()) } /// Given the specified AccountMetaFlags, returns the size of its @@ -125,9 +113,6 @@ impl<'a> AccountMetaOptionalFields<'a> { if flags.has_rent_epoch() { fields_size += std::mem::size_of::(); } - if flags.has_account_hash() { - fields_size += std::mem::size_of::(); - } fields_size } @@ -137,29 +122,17 @@ impl<'a> AccountMetaOptionalFields<'a> { pub fn rent_epoch_offset(_flags: &AccountMetaFlags) -> usize { 0 } - - /// Given the specified AccountMetaFlags, returns the relative offset - /// of its account_hash field to the offset of its optional fields entry. - pub fn account_hash_offset(flags: &AccountMetaFlags) -> usize { - let mut offset = Self::rent_epoch_offset(flags); - // rent_epoch is the previous field to account hash - if flags.has_rent_epoch() { - offset += std::mem::size_of::(); - } - offset - } } #[cfg(test)] pub mod tests { - use {super::*, solana_sdk::hash::Hash}; + use super::*; #[test] fn test_account_meta_flags_new() { let flags = AccountMetaFlags::new(); assert!(!flags.has_rent_epoch()); - assert!(!flags.has_account_hash()); assert_eq!(flags.reserved(), 0u32); assert_eq!( @@ -179,20 +152,11 @@ pub mod tests { flags.set_has_rent_epoch(true); assert!(flags.has_rent_epoch()); - assert!(!flags.has_account_hash()); - assert!(!flags.executable()); - verify_flags_serialization(&flags); - - flags.set_has_account_hash(true); - - assert!(flags.has_rent_epoch()); - assert!(flags.has_account_hash()); assert!(!flags.executable()); verify_flags_serialization(&flags); flags.set_executable(true); assert!(flags.has_rent_epoch()); - assert!(flags.has_account_hash()); assert!(flags.executable()); verify_flags_serialization(&flags); @@ -203,84 +167,58 @@ pub mod tests { fn update_and_verify_flags(opt_fields: &AccountMetaOptionalFields) { let flags: AccountMetaFlags = AccountMetaFlags::new_from(opt_fields); assert_eq!(flags.has_rent_epoch(), opt_fields.rent_epoch.is_some()); - assert_eq!(flags.has_account_hash(), opt_fields.account_hash.is_some()); assert_eq!(flags.reserved(), 0u32); } #[test] fn test_optional_fields_update_flags() { let test_epoch = 5432312; - let acc_hash = AccountHash(Hash::new_unique()); for rent_epoch in [None, Some(test_epoch)] { - for account_hash in [None, Some(&acc_hash)] { - update_and_verify_flags(&AccountMetaOptionalFields { - rent_epoch, - account_hash, - }); - } + update_and_verify_flags(&AccountMetaOptionalFields { rent_epoch }); } } #[test] fn test_optional_fields_size() { let test_epoch = 5432312; - let acc_hash = AccountHash(Hash::new_unique()); for rent_epoch in [None, Some(test_epoch)] { - for account_hash in [None, Some(&acc_hash)] { - let opt_fields = AccountMetaOptionalFields { - rent_epoch, - account_hash, - }; - assert_eq!( - opt_fields.size(), - rent_epoch.map_or(0, |_| std::mem::size_of::()) - + account_hash.map_or(0, |_| std::mem::size_of::()) - ); - assert_eq!( - opt_fields.size(), - AccountMetaOptionalFields::size_from_flags(&AccountMetaFlags::new_from( - &opt_fields - )) - ); - } + let opt_fields = AccountMetaOptionalFields { rent_epoch }; + assert_eq!( + opt_fields.size(), + rent_epoch.map_or(0, |_| std::mem::size_of::()), + ); + assert_eq!( + opt_fields.size(), + AccountMetaOptionalFields::size_from_flags(&AccountMetaFlags::new_from( + &opt_fields + )) + ); } } #[test] fn test_optional_fields_offset() { let test_epoch = 5432312; - let acc_hash = AccountHash(Hash::new_unique()); for rent_epoch in [None, Some(test_epoch)] { - for account_hash in [None, Some(&acc_hash)] { - let rent_epoch_offset = 0; - let account_hash_offset = - rent_epoch_offset + rent_epoch.as_ref().map(std::mem::size_of_val).unwrap_or(0); - let derived_size = account_hash_offset - + account_hash - .as_ref() - .map(|acc_hash| std::mem::size_of_val(*acc_hash)) - .unwrap_or(0); - let opt_fields = AccountMetaOptionalFields { - rent_epoch, - account_hash, - }; - let flags = AccountMetaFlags::new_from(&opt_fields); - assert_eq!( - AccountMetaOptionalFields::rent_epoch_offset(&flags), - rent_epoch_offset - ); - assert_eq!( - AccountMetaOptionalFields::account_hash_offset(&flags), - account_hash_offset - ); - assert_eq!( - AccountMetaOptionalFields::size_from_flags(&flags), - derived_size - ); - } + let rent_epoch_offset = 0; + let derived_size = if rent_epoch.is_some() { + std::mem::size_of::() + } else { + 0 + }; + let opt_fields = AccountMetaOptionalFields { rent_epoch }; + let flags = AccountMetaFlags::new_from(&opt_fields); + assert_eq!( + AccountMetaOptionalFields::rent_epoch_offset(&flags), + rent_epoch_offset + ); + assert_eq!( + AccountMetaOptionalFields::size_from_flags(&flags), + derived_size + ); } } } diff --git a/accounts-db/src/tiered_storage/readable.rs b/accounts-db/src/tiered_storage/readable.rs index 1801b04fcecd80..8f1d2007182a5b 100644 --- a/accounts-db/src/tiered_storage/readable.rs +++ b/accounts-db/src/tiered_storage/readable.rs @@ -2,7 +2,6 @@ use { crate::{ account_storage::meta::StoredAccountMeta, accounts_file::MatchAccountOwnerError, - accounts_hash::AccountHash, tiered_storage::{ footer::{AccountMetaFormat, TieredStorageFooter}, hot::HotStorageReader, @@ -40,11 +39,6 @@ impl<'accounts_file, M: TieredAccountMeta> TieredReadableAccount<'accounts_file, self.address } - /// Returns the hash of this account. - pub fn hash(&self) -> Option<&'accounts_file AccountHash> { - self.meta.account_hash(self.account_block) - } - /// Returns the index to this account in its AccountsFile. pub fn index(&self) -> IndexOffset { self.index diff --git a/accounts-db/src/tiered_storage/test_utils.rs b/accounts-db/src/tiered_storage/test_utils.rs index 2ed2399f30fbaa..f44f20f77cc5dd 100644 --- a/accounts-db/src/tiered_storage/test_utils.rs +++ b/accounts-db/src/tiered_storage/test_utils.rs @@ -48,20 +48,10 @@ pub(super) fn verify_test_account( stored_meta: &StoredAccountMeta<'_>, account: Option<&impl ReadableAccount>, address: &Pubkey, - account_hash: &AccountHash, ) { - let (lamports, owner, data, executable, account_hash) = account - .map(|acc| { - ( - acc.lamports(), - acc.owner(), - acc.data(), - acc.executable(), - // only persist rent_epoch for those rent-paying accounts - Some(*account_hash), - ) - }) - .unwrap_or((0, &OWNER_NO_OWNER, &[], false, None)); + let (lamports, owner, data, executable) = account + .map(|acc| (acc.lamports(), acc.owner(), acc.data(), acc.executable())) + .unwrap_or((0, &OWNER_NO_OWNER, &[], false)); assert_eq!(stored_meta.lamports(), lamports); assert_eq!(stored_meta.data().len(), data.len()); @@ -69,8 +59,5 @@ pub(super) fn verify_test_account( assert_eq!(stored_meta.executable(), executable); assert_eq!(stored_meta.owner(), owner); assert_eq!(stored_meta.pubkey(), address); - assert_eq!( - *stored_meta.hash(), - account_hash.unwrap_or(AccountHash(Hash::default())) - ); + assert_eq!(*stored_meta.hash(), AccountHash(Hash::default())); } From adefcbbb43231cf3516ec976df7fe03843aff3f7 Mon Sep 17 00:00:00 2001 From: Justin Starry Date: Thu, 7 Mar 2024 12:06:52 +0800 Subject: [PATCH 24/84] Add support for partial tx batch unlocking (#110) * Add support for partial tx batch unlocking * add assert * fix build * Add comments --- runtime/src/transaction_batch.rs | 96 ++++++++++++++++++++++++++------ 1 file changed, 78 insertions(+), 18 deletions(-) diff --git a/runtime/src/transaction_batch.rs b/runtime/src/transaction_batch.rs index 9d0ff5fb7ce007..ecec27e02e93aa 100644 --- a/runtime/src/transaction_batch.rs +++ b/runtime/src/transaction_batch.rs @@ -46,6 +46,39 @@ impl<'a, 'b> TransactionBatch<'a, 'b> { pub fn needs_unlock(&self) -> bool { self.needs_unlock } + + /// For every error result, if the corresponding transaction is + /// still locked, unlock the transaction and then record the new error. + pub fn unlock_failures(&mut self, transaction_results: Vec>) { + assert_eq!(self.lock_results.len(), transaction_results.len()); + // Shouldn't happen but if a batch was marked as not needing an unlock, + // don't unlock failures. + if !self.needs_unlock() { + return; + } + + let txs_and_results = transaction_results + .iter() + .enumerate() + .inspect(|(index, result)| { + // It's not valid to update a previously recorded lock error to + // become an "ok" result because this could lead to serious + // account lock violations where accounts are later unlocked + // when they were not currently locked. + assert!(!(result.is_ok() && self.lock_results[*index].is_err())) + }) + .filter(|(index, result)| result.is_err() && self.lock_results[*index].is_ok()) + .map(|(index, _)| (&self.sanitized_txs[index], &self.lock_results[index])); + + // Unlock the accounts for all transactions which will be updated to an + // lock error below. + self.bank.unlock_accounts(txs_and_results); + + // Record all new errors by overwriting lock results. Note that it's + // not valid to update from err -> ok and the assertion above enforces + // that validity constraint. + self.lock_results = transaction_results; + } } // Unlock all locked accounts in destructor. @@ -67,12 +100,12 @@ mod tests { use { super::*, crate::genesis_utils::{create_genesis_config_with_leader, GenesisConfigInfo}, - solana_sdk::{signature::Keypair, system_transaction}, + solana_sdk::{signature::Keypair, system_transaction, transaction::TransactionError}, }; #[test] fn test_transaction_batch() { - let (bank, txs) = setup(); + let (bank, txs) = setup(false); // Test getting locked accounts let batch = bank.prepare_sanitized_batch(&txs); @@ -94,7 +127,7 @@ mod tests { #[test] fn test_simulation_batch() { - let (bank, txs) = setup(); + let (bank, txs) = setup(false); // Prepare batch without locks let batch = bank.prepare_unlocked_batch_from_single_tx(&txs[0]); @@ -109,7 +142,37 @@ mod tests { assert!(batch3.lock_results().iter().all(|x| x.is_ok())); } - fn setup() -> (Bank, Vec) { + #[test] + fn test_unlock_failures() { + let (bank, txs) = setup(true); + + // Test getting locked accounts + let mut batch = bank.prepare_sanitized_batch(&txs); + assert_eq!( + batch.lock_results, + vec![Ok(()), Err(TransactionError::AccountInUse), Ok(())] + ); + + let qos_results = vec![ + Ok(()), + Err(TransactionError::AccountInUse), + Err(TransactionError::WouldExceedMaxBlockCostLimit), + ]; + batch.unlock_failures(qos_results.clone()); + assert_eq!(batch.lock_results, qos_results); + + // Dropping the batch should unlock remaining locked transactions + drop(batch); + + // The next batch should be able to lock all but the conflicting tx + let batch2 = bank.prepare_sanitized_batch(&txs); + assert_eq!( + batch2.lock_results, + vec![Ok(()), Err(TransactionError::AccountInUse), Ok(())] + ); + } + + fn setup(insert_conflicting_tx: bool) -> (Bank, Vec) { let dummy_leader_pubkey = solana_sdk::pubkey::new_rand(); let GenesisConfigInfo { genesis_config, @@ -122,20 +185,17 @@ mod tests { let keypair2 = Keypair::new(); let pubkey2 = solana_sdk::pubkey::new_rand(); - let txs = vec![ - SanitizedTransaction::from_transaction_for_tests(system_transaction::transfer( - &mint_keypair, - &pubkey, - 1, - genesis_config.hash(), - )), - SanitizedTransaction::from_transaction_for_tests(system_transaction::transfer( - &keypair2, - &pubkey2, - 1, - genesis_config.hash(), - )), - ]; + let mut txs = vec![SanitizedTransaction::from_transaction_for_tests( + system_transaction::transfer(&mint_keypair, &pubkey, 1, genesis_config.hash()), + )]; + if insert_conflicting_tx { + txs.push(SanitizedTransaction::from_transaction_for_tests( + system_transaction::transfer(&mint_keypair, &pubkey2, 1, genesis_config.hash()), + )); + } + txs.push(SanitizedTransaction::from_transaction_for_tests( + system_transaction::transfer(&keypair2, &pubkey2, 1, genesis_config.hash()), + )); (bank, txs) } From 8f3f06cc7f6eac156c18fd147f954af75fa403a7 Mon Sep 17 00:00:00 2001 From: Tao Zhu <82401714+tao-stones@users.noreply.github.com> Date: Thu, 7 Mar 2024 09:23:49 -0600 Subject: [PATCH 25/84] Combine builtin and BPF compute cost in cost model (#29) * Combine builtin and BPF execution cost into programs_execution_cost since VM has started to consume CUs uniformly * update tests * apply suggestions from code review --- core/src/banking_stage.rs | 3 +- core/src/banking_stage/consumer.rs | 15 ++-- core/src/banking_stage/qos_service.rs | 61 +++++---------- cost-model/src/cost_model.rs | 103 ++++++++++++++++---------- cost-model/src/cost_tracker.rs | 16 ++-- cost-model/src/transaction_cost.rs | 25 ++----- 6 files changed, 106 insertions(+), 117 deletions(-) diff --git a/core/src/banking_stage.rs b/core/src/banking_stage.rs index 652f2569f8fd43..603ff55f0003b4 100644 --- a/core/src/banking_stage.rs +++ b/core/src/banking_stage.rs @@ -285,8 +285,7 @@ pub struct BatchedTransactionCostDetails { pub batched_signature_cost: u64, pub batched_write_lock_cost: u64, pub batched_data_bytes_cost: u64, - pub batched_builtins_execute_cost: u64, - pub batched_bpf_execute_cost: u64, + pub batched_programs_execute_cost: u64, } #[derive(Debug, Default)] diff --git a/core/src/banking_stage/consumer.rs b/core/src/banking_stage/consumer.rs index f4ac6c6040eda8..957e190c873f64 100644 --- a/core/src/banking_stage/consumer.rs +++ b/core/src/banking_stage/consumer.rs @@ -1549,16 +1549,17 @@ mod tests { assert_eq!(retryable_transaction_indexes, vec![1]); let expected_block_cost = if !apply_cost_tracker_during_replay_enabled { - let actual_bpf_execution_cost = match commit_transactions_result.first().unwrap() { - CommitTransactionDetails::Committed { compute_units } => *compute_units, - CommitTransactionDetails::NotCommitted => { - unreachable!() - } - }; + let actual_programs_execution_cost = + match commit_transactions_result.first().unwrap() { + CommitTransactionDetails::Committed { compute_units } => *compute_units, + CommitTransactionDetails::NotCommitted => { + unreachable!() + } + }; let mut cost = CostModel::calculate_cost(&transactions[0], &bank.feature_set); if let TransactionCost::Transaction(ref mut usage_cost) = cost { - usage_cost.bpf_execution_cost = actual_bpf_execution_cost; + usage_cost.programs_execution_cost = actual_programs_execution_cost; } block_cost + cost.sum() diff --git a/core/src/banking_stage/qos_service.rs b/core/src/banking_stage/qos_service.rs index 77f05c73a3bc12..8c1507ae3fb91c 100644 --- a/core/src/banking_stage/qos_service.rs +++ b/core/src/banking_stage/qos_service.rs @@ -236,14 +236,10 @@ impl QosService { batched_transaction_details.costs.batched_data_bytes_cost, Ordering::Relaxed, ); - self.metrics.stats.estimated_builtins_execute_cu.fetch_add( + self.metrics.stats.estimated_programs_execute_cu.fetch_add( batched_transaction_details .costs - .batched_builtins_execute_cost, - Ordering::Relaxed, - ); - self.metrics.stats.estimated_bpf_execute_cu.fetch_add( - batched_transaction_details.costs.batched_bpf_execute_cost, + .batched_programs_execute_cost, Ordering::Relaxed, ); @@ -297,7 +293,7 @@ impl QosService { pub fn accumulate_actual_execute_cu(&self, units: u64) { self.metrics .stats - .actual_bpf_execute_cu + .actual_programs_execute_cu .fetch_add(units, Ordering::Relaxed); } @@ -331,12 +327,8 @@ impl QosService { saturating_add_assign!( batched_transaction_details .costs - .batched_builtins_execute_cost, - cost.builtins_execution_cost() - ); - saturating_add_assign!( - batched_transaction_details.costs.batched_bpf_execute_cost, - cost.bpf_execution_cost() + .batched_programs_execute_cost, + cost.programs_execution_cost() ); } Err(transaction_error) => match transaction_error { @@ -427,14 +419,11 @@ struct QosServiceMetricsStats { /// accumulated estimated instruction data Compute Units to be packed into block estimated_data_bytes_cu: AtomicU64, - /// accumulated estimated builtin programs Compute Units to be packed into block - estimated_builtins_execute_cu: AtomicU64, - - /// accumulated estimated SBF program Compute Units to be packed into block - estimated_bpf_execute_cu: AtomicU64, + /// accumulated estimated program Compute Units to be packed into block + estimated_programs_execute_cu: AtomicU64, /// accumulated actual program Compute Units that have been packed into block - actual_bpf_execute_cu: AtomicU64, + actual_programs_execute_cu: AtomicU64, /// accumulated actual program execute micro-sec that have been packed into block actual_execute_time_us: AtomicU64, @@ -515,24 +504,19 @@ impl QosServiceMetrics { i64 ), ( - "estimated_builtins_execute_cu", + "estimated_programs_execute_cu", self.stats - .estimated_builtins_execute_cu + .estimated_programs_execute_cu .swap(0, Ordering::Relaxed), i64 ), ( - "estimated_bpf_execute_cu", + "actual_programs_execute_cu", self.stats - .estimated_bpf_execute_cu + .actual_programs_execute_cu .swap(0, Ordering::Relaxed), i64 ), - ( - "actual_bpf_execute_cu", - self.stats.actual_bpf_execute_cu.swap(0, Ordering::Relaxed), - i64 - ), ( "actual_execute_time_us", self.stats.actual_execute_time_us.swap(0, Ordering::Relaxed), @@ -735,7 +719,7 @@ mod tests { let committed_status: Vec = qos_cost_results .iter() .map(|tx_cost| CommitTransactionDetails::Committed { - compute_units: tx_cost.as_ref().unwrap().bpf_execution_cost() + compute_units: tx_cost.as_ref().unwrap().programs_execution_cost() + execute_units_adjustment, }) .collect(); @@ -862,7 +846,7 @@ mod tests { CommitTransactionDetails::NotCommitted } else { CommitTransactionDetails::Committed { - compute_units: tx_cost.as_ref().unwrap().bpf_execution_cost() + compute_units: tx_cost.as_ref().unwrap().programs_execution_cost() + execute_units_adjustment, } } @@ -898,8 +882,7 @@ mod tests { let signature_cost = 1; let write_lock_cost = 2; let data_bytes_cost = 3; - let builtins_execution_cost = 4; - let bpf_execution_cost = 10; + let programs_execution_cost = 10; let num_txs = 4; let tx_cost_results: Vec<_> = (0..num_txs) @@ -909,8 +892,7 @@ mod tests { signature_cost, write_lock_cost, data_bytes_cost, - builtins_execution_cost, - bpf_execution_cost, + programs_execution_cost, ..UsageCostDetails::default() })) } else { @@ -922,8 +904,7 @@ mod tests { let expected_signatures = signature_cost * (num_txs / 2); let expected_write_locks = write_lock_cost * (num_txs / 2); let expected_data_bytes = data_bytes_cost * (num_txs / 2); - let expected_builtins_execution_costs = builtins_execution_cost * (num_txs / 2); - let expected_bpf_execution_costs = bpf_execution_cost * (num_txs / 2); + let expected_programs_execution_costs = programs_execution_cost * (num_txs / 2); let batched_transaction_details = QosService::accumulate_batched_transaction_costs(tx_cost_results.iter()); assert_eq!( @@ -939,14 +920,10 @@ mod tests { batched_transaction_details.costs.batched_data_bytes_cost ); assert_eq!( - expected_builtins_execution_costs, + expected_programs_execution_costs, batched_transaction_details .costs - .batched_builtins_execute_cost - ); - assert_eq!( - expected_bpf_execution_costs, - batched_transaction_details.costs.batched_bpf_execute_cost + .batched_programs_execute_cost ); } } diff --git a/cost-model/src/cost_model.rs b/cost-model/src/cost_model.rs index 1e15735426737f..b81ea24402d4df 100644 --- a/cost-model/src/cost_model.rs +++ b/cost-model/src/cost_model.rs @@ -93,21 +93,25 @@ impl CostModel { transaction: &SanitizedTransaction, feature_set: &FeatureSet, ) { - let mut builtin_costs = 0u64; - let mut bpf_costs = 0u64; + let mut programs_execution_costs = 0u64; let mut loaded_accounts_data_size_cost = 0u64; let mut data_bytes_len_total = 0u64; let mut compute_unit_limit_is_set = false; + let mut has_user_space_instructions = false; for (program_id, instruction) in transaction.message().program_instructions_iter() { - // to keep the same behavior, look for builtin first - if let Some(builtin_cost) = BUILT_IN_INSTRUCTION_COSTS.get(program_id) { - builtin_costs = builtin_costs.saturating_add(*builtin_cost); - } else { - bpf_costs = bpf_costs - .saturating_add(u64::from(DEFAULT_INSTRUCTION_COMPUTE_UNIT_LIMIT)) - .min(u64::from(MAX_COMPUTE_UNIT_LIMIT)); - } + let ix_execution_cost = + if let Some(builtin_cost) = BUILT_IN_INSTRUCTION_COSTS.get(program_id) { + *builtin_cost + } else { + has_user_space_instructions = true; + u64::from(DEFAULT_INSTRUCTION_COMPUTE_UNIT_LIMIT) + }; + + programs_execution_costs = programs_execution_costs + .saturating_add(ix_execution_cost) + .min(u64::from(MAX_COMPUTE_UNIT_LIMIT)); + data_bytes_len_total = data_bytes_len_total.saturating_add(instruction.data.len() as u64); @@ -120,8 +124,6 @@ impl CostModel { } } - // calculate bpf cost based on compute budget instructions - // if failed to process compute_budget instructions, the transaction will not be executed // by `bank`, therefore it should be considered as no execution cost by cost model. match process_compute_budget_instructions(transaction.message().program_instructions_iter()) @@ -132,8 +134,8 @@ impl CostModel { // 'compute_unit_limit_is_set' flag, because compute_budget does not distinguish // builtin and bpf instructions when calculating default compute-unit-limit. (see // compute_budget.rs test `test_process_mixed_instructions_without_compute_budget`) - if bpf_costs > 0 && compute_unit_limit_is_set { - bpf_costs = u64::from(compute_budget_limits.compute_unit_limit); + if has_user_space_instructions && compute_unit_limit_is_set { + programs_execution_costs = u64::from(compute_budget_limits.compute_unit_limit); } if feature_set @@ -146,13 +148,11 @@ impl CostModel { } } Err(_) => { - builtin_costs = 0; - bpf_costs = 0; + programs_execution_costs = 0; } } - tx_cost.builtins_execution_cost = builtin_costs; - tx_cost.bpf_execution_cost = bpf_costs; + tx_cost.programs_execution_cost = programs_execution_costs; tx_cost.loaded_accounts_data_size_cost = loaded_accounts_data_size_cost; tx_cost.data_bytes_cost = data_bytes_len_total / INSTRUCTION_DATA_BYTES_COST; } @@ -304,8 +304,7 @@ mod tests { &simple_transaction, &FeatureSet::all_enabled(), ); - assert_eq!(*expected_execution_cost, tx_cost.builtins_execution_cost); - assert_eq!(0, tx_cost.bpf_execution_cost); + assert_eq!(*expected_execution_cost, tx_cost.programs_execution_cost); assert_eq!(3, tx_cost.data_bytes_cost); } @@ -333,8 +332,10 @@ mod tests { &token_transaction, &FeatureSet::all_enabled(), ); - assert_eq!(0, tx_cost.builtins_execution_cost); - assert_eq!(200_000, tx_cost.bpf_execution_cost); + assert_eq!( + DEFAULT_INSTRUCTION_COMPUTE_UNIT_LIMIT as u64, + tx_cost.programs_execution_cost + ); assert_eq!(0, tx_cost.data_bytes_cost); } @@ -396,13 +397,8 @@ mod tests { &token_transaction, &FeatureSet::all_enabled(), ); - assert_eq!( - *BUILT_IN_INSTRUCTION_COSTS - .get(&compute_budget::id()) - .unwrap(), - tx_cost.builtins_execution_cost - ); - assert_eq!(12_345, tx_cost.bpf_execution_cost); + // If cu-limit is specified, that would the cost for all programs + assert_eq!(12_345, tx_cost.programs_execution_cost); assert_eq!(1, tx_cost.data_bytes_cost); } @@ -446,8 +442,7 @@ mod tests { &token_transaction, &FeatureSet::all_enabled(), ); - assert_eq!(0, tx_cost.builtins_execution_cost); - assert_eq!(0, tx_cost.bpf_execution_cost); + assert_eq!(0, tx_cost.programs_execution_cost); } #[test] @@ -474,8 +469,7 @@ mod tests { let mut tx_cost = UsageCostDetails::default(); CostModel::get_transaction_cost(&mut tx_cost, &tx, &FeatureSet::all_enabled()); - assert_eq!(expected_cost, tx_cost.builtins_execution_cost); - assert_eq!(0, tx_cost.bpf_execution_cost); + assert_eq!(expected_cost, tx_cost.programs_execution_cost); assert_eq!(6, tx_cost.data_bytes_cost); } @@ -506,8 +500,7 @@ mod tests { let expected_cost = DEFAULT_INSTRUCTION_COMPUTE_UNIT_LIMIT as u64 * 2; let mut tx_cost = UsageCostDetails::default(); CostModel::get_transaction_cost(&mut tx_cost, &tx, &FeatureSet::all_enabled()); - assert_eq!(0, tx_cost.builtins_execution_cost); - assert_eq!(expected_cost, tx_cost.bpf_execution_cost); + assert_eq!(expected_cost, tx_cost.programs_execution_cost); assert_eq!(0, tx_cost.data_bytes_cost); } @@ -567,7 +560,7 @@ mod tests { let tx_cost = CostModel::calculate_cost(&tx, &FeatureSet::all_enabled()); assert_eq!(expected_account_cost, tx_cost.write_lock_cost()); - assert_eq!(*expected_execution_cost, tx_cost.builtins_execution_cost()); + assert_eq!(*expected_execution_cost, tx_cost.programs_execution_cost()); assert_eq!(2, tx_cost.writable_accounts().len()); assert_eq!( expected_loaded_accounts_data_size_cost, @@ -596,7 +589,7 @@ mod tests { let tx_cost = CostModel::calculate_cost(&tx, &feature_set); assert_eq!(expected_account_cost, tx_cost.write_lock_cost()); - assert_eq!(*expected_execution_cost, tx_cost.builtins_execution_cost()); + assert_eq!(*expected_execution_cost, tx_cost.programs_execution_cost()); assert_eq!(2, tx_cost.writable_accounts().len()); assert_eq!( expected_loaded_accounts_data_size_cost, @@ -635,7 +628,7 @@ mod tests { let tx_cost = CostModel::calculate_cost(&tx, &feature_set); assert_eq!(expected_account_cost, tx_cost.write_lock_cost()); - assert_eq!(expected_execution_cost, tx_cost.builtins_execution_cost()); + assert_eq!(expected_execution_cost, tx_cost.programs_execution_cost()); assert_eq!(2, tx_cost.writable_accounts().len()); assert_eq!( expected_loaded_accounts_data_size_cost, @@ -666,7 +659,37 @@ mod tests { let mut tx_cost = UsageCostDetails::default(); CostModel::get_transaction_cost(&mut tx_cost, &transaction, &FeatureSet::all_enabled()); - assert_eq!(expected_builtin_cost, tx_cost.builtins_execution_cost); - assert_eq!(expected_bpf_cost as u64, tx_cost.bpf_execution_cost); + assert_eq!( + expected_builtin_cost + expected_bpf_cost as u64, + tx_cost.programs_execution_cost + ); + } + + #[test] + fn test_transaction_cost_with_mix_instruction_with_cu_limit() { + let (mint_keypair, start_hash) = test_setup(); + + let transaction = + SanitizedTransaction::from_transaction_for_tests(Transaction::new_signed_with_payer( + &[ + system_instruction::transfer(&mint_keypair.pubkey(), &Pubkey::new_unique(), 2), + ComputeBudgetInstruction::set_compute_unit_limit(12_345), + ], + Some(&mint_keypair.pubkey()), + &[&mint_keypair], + start_hash, + )); + // transaction has one builtin instruction, and one ComputeBudget::compute_unit_limit + let expected_cost = *BUILT_IN_INSTRUCTION_COSTS + .get(&solana_system_program::id()) + .unwrap() + + BUILT_IN_INSTRUCTION_COSTS + .get(&compute_budget::id()) + .unwrap(); + + let mut tx_cost = UsageCostDetails::default(); + CostModel::get_transaction_cost(&mut tx_cost, &transaction, &FeatureSet::all_enabled()); + + assert_eq!(expected_cost, tx_cost.programs_execution_cost); } } diff --git a/cost-model/src/cost_tracker.rs b/cost-model/src/cost_tracker.rs index 9d2b3b624afeb4..8fb092c36680a0 100644 --- a/cost-model/src/cost_tracker.rs +++ b/cost-model/src/cost_tracker.rs @@ -105,7 +105,7 @@ impl CostTracker { estimated_tx_cost: &TransactionCost, actual_execution_units: u64, ) { - let estimated_execution_units = estimated_tx_cost.bpf_execution_cost(); + let estimated_execution_units = estimated_tx_cost.programs_execution_cost(); match actual_execution_units.cmp(&estimated_execution_units) { Ordering::Equal => (), Ordering::Greater => { @@ -307,7 +307,7 @@ mod tests { system_transaction::transfer(mint_keypair, &keypair.pubkey(), 2, *start_hash), ); let mut tx_cost = UsageCostDetails::new_with_capacity(1); - tx_cost.bpf_execution_cost = 5; + tx_cost.programs_execution_cost = 5; tx_cost.writable_accounts.push(mint_keypair.pubkey()); (simple_transaction, TransactionCost::Transaction(tx_cost)) @@ -606,7 +606,7 @@ mod tests { { let tx_cost = TransactionCost::Transaction(UsageCostDetails { writable_accounts: vec![acct1, acct2, acct3], - bpf_execution_cost: cost, + programs_execution_cost: cost, ..UsageCostDetails::default() }); assert!(testee.try_add(&tx_cost).is_ok()); @@ -624,7 +624,7 @@ mod tests { { let tx_cost = TransactionCost::Transaction(UsageCostDetails { writable_accounts: vec![acct2], - bpf_execution_cost: cost, + programs_execution_cost: cost, ..UsageCostDetails::default() }); assert!(testee.try_add(&tx_cost).is_ok()); @@ -644,7 +644,7 @@ mod tests { { let tx_cost = TransactionCost::Transaction(UsageCostDetails { writable_accounts: vec![acct1, acct2], - bpf_execution_cost: cost, + programs_execution_cost: cost, ..UsageCostDetails::default() }); assert!(testee.try_add(&tx_cost).is_err()); @@ -668,7 +668,7 @@ mod tests { let mut testee = CostTracker::new(account_max, block_max, block_max); let tx_cost = TransactionCost::Transaction(UsageCostDetails { writable_accounts: vec![acct1, acct2, acct3], - bpf_execution_cost: cost, + programs_execution_cost: cost, ..UsageCostDetails::default() }); let mut expected_block_cost = tx_cost.sum(); @@ -755,7 +755,7 @@ mod tests { let tx_cost = TransactionCost::Transaction(UsageCostDetails { writable_accounts: vec![acct1, acct2, acct3], - bpf_execution_cost: cost, + programs_execution_cost: cost, ..UsageCostDetails::default() }); @@ -802,7 +802,7 @@ mod tests { let cost = 100u64; let tx_cost = TransactionCost::Transaction(UsageCostDetails { writable_accounts: vec![Pubkey::new_unique()], - bpf_execution_cost: cost, + programs_execution_cost: cost, ..UsageCostDetails::default() }); diff --git a/cost-model/src/transaction_cost.rs b/cost-model/src/transaction_cost.rs index 76865fff30fd57..c6e68bfe17b6f4 100644 --- a/cost-model/src/transaction_cost.rs +++ b/cost-model/src/transaction_cost.rs @@ -35,10 +35,10 @@ impl TransactionCost { } } - pub fn bpf_execution_cost(&self) -> u64 { + pub fn programs_execution_cost(&self) -> u64 { match self { - Self::SimpleVote { .. } => 0, - Self::Transaction(usage_cost) => usage_cost.bpf_execution_cost, + Self::SimpleVote { .. } => solana_vote_program::vote_processor::DEFAULT_COMPUTE_UNITS, + Self::Transaction(usage_cost) => usage_cost.programs_execution_cost, } } @@ -85,13 +85,6 @@ impl TransactionCost { } } - pub fn builtins_execution_cost(&self) -> u64 { - match self { - Self::SimpleVote { .. } => solana_vote_program::vote_processor::DEFAULT_COMPUTE_UNITS, - Self::Transaction(usage_cost) => usage_cost.builtins_execution_cost, - } - } - pub fn writable_accounts(&self) -> &[Pubkey] { match self { Self::SimpleVote { writable_accounts } => writable_accounts, @@ -109,8 +102,7 @@ pub struct UsageCostDetails { pub signature_cost: u64, pub write_lock_cost: u64, pub data_bytes_cost: u64, - pub builtins_execution_cost: u64, - pub bpf_execution_cost: u64, + pub programs_execution_cost: u64, pub loaded_accounts_data_size_cost: u64, pub account_data_size: u64, } @@ -122,8 +114,7 @@ impl Default for UsageCostDetails { signature_cost: 0u64, write_lock_cost: 0u64, data_bytes_cost: 0u64, - builtins_execution_cost: 0u64, - bpf_execution_cost: 0u64, + programs_execution_cost: 0u64, loaded_accounts_data_size_cost: 0u64, account_data_size: 0u64, } @@ -140,8 +131,7 @@ impl PartialEq for UsageCostDetails { self.signature_cost == other.signature_cost && self.write_lock_cost == other.write_lock_cost && self.data_bytes_cost == other.data_bytes_cost - && self.builtins_execution_cost == other.builtins_execution_cost - && self.bpf_execution_cost == other.bpf_execution_cost + && self.programs_execution_cost == other.programs_execution_cost && self.loaded_accounts_data_size_cost == other.loaded_accounts_data_size_cost && self.account_data_size == other.account_data_size && to_hash_set(&self.writable_accounts) == to_hash_set(&other.writable_accounts) @@ -168,8 +158,7 @@ impl UsageCostDetails { self.signature_cost .saturating_add(self.write_lock_cost) .saturating_add(self.data_bytes_cost) - .saturating_add(self.builtins_execution_cost) - .saturating_add(self.bpf_execution_cost) + .saturating_add(self.programs_execution_cost) .saturating_add(self.loaded_accounts_data_size_cost) } } From 85cfe23b46d745a225436a09b88d316ee470371b Mon Sep 17 00:00:00 2001 From: Lucas Steuernagel <38472950+LucasSte@users.noreply.github.com> Date: Thu, 7 Mar 2024 12:26:31 -0300 Subject: [PATCH 26/84] Add tests for `svm/transaction_processor.rs` (#112) --- Cargo.lock | 1 + programs/bpf_loader/src/lib.rs | 3 +- svm/Cargo.toml | 1 + svm/src/transaction_processor.rs | 529 ++++++++++++++++++++++++++++++- svm/tests/test_program.so | Bin 0 -> 170136 bytes 5 files changed, 531 insertions(+), 3 deletions(-) create mode 100755 svm/tests/test_program.so diff --git a/Cargo.lock b/Cargo.lock index db5431da6ef62e..afdb8b0a306578 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7288,6 +7288,7 @@ dependencies = [ name = "solana-svm" version = "1.19.0" dependencies = [ + "bincode", "itertools", "log", "percentage", diff --git a/programs/bpf_loader/src/lib.rs b/programs/bpf_loader/src/lib.rs index 5ba8b26e086c69..a9c34fbabfc6f6 100644 --- a/programs/bpf_loader/src/lib.rs +++ b/programs/bpf_loader/src/lib.rs @@ -39,7 +39,7 @@ use { }, instruction::{AccountMeta, InstructionError}, loader_upgradeable_instruction::UpgradeableLoaderInstruction, - native_loader, + loader_v4, native_loader, program_utils::limited_deserialize, pubkey::Pubkey, saturating_add_assign, @@ -193,6 +193,7 @@ pub fn check_loader_id(id: &Pubkey) -> bool { bpf_loader::check_id(id) || bpf_loader_deprecated::check_id(id) || bpf_loader_upgradeable::check_id(id) + || loader_v4::check_id(id) } /// Only used in macro, do not use directly! diff --git a/svm/Cargo.toml b/svm/Cargo.toml index ac672613c9c4fc..21da2f7105bd73 100644 --- a/svm/Cargo.toml +++ b/svm/Cargo.toml @@ -28,6 +28,7 @@ crate-type = ["lib"] name = "solana_svm" [dev-dependencies] +bincode = { workspace = true } solana-logger = { workspace = true } solana-sdk = { workspace = true, features = ["dev-context-only-utils"] } diff --git a/svm/src/transaction_processor.rs b/svm/src/transaction_processor.rs index 38c5c23affd4de..fa417850699372 100644 --- a/svm/src/transaction_processor.rs +++ b/svm/src/transaction_processor.rs @@ -87,6 +87,7 @@ pub trait TransactionProcessingCallback { } } +#[derive(Debug)] enum ProgramAccountLoadResult { AccountNotFound, InvalidAccountData(ProgramRuntimeEnvironment), @@ -921,8 +922,18 @@ impl TransactionBatchProcessor { mod tests { use { super::*, - solana_program_runtime::loaded_programs::BlockRelation, - solana_sdk::{sysvar::rent::Rent, transaction_context::TransactionContext}, + solana_program_runtime::{ + loaded_programs::BlockRelation, solana_rbpf::program::BuiltinProgram, + }, + solana_sdk::{ + account::WritableAccount, bpf_loader, sysvar::rent::Rent, + transaction_context::TransactionContext, + }, + std::{ + env, + fs::{self, File}, + io::Read, + }, }; struct TestForkGraph {} @@ -933,6 +944,43 @@ mod tests { } } + #[derive(Default)] + pub struct MockBankCallback { + rent_collector: RentCollector, + feature_set: Arc, + pub account_shared_data: HashMap, + } + + impl TransactionProcessingCallback for MockBankCallback { + fn account_matches_owners(&self, account: &Pubkey, owners: &[Pubkey]) -> Option { + if let Some(data) = self.account_shared_data.get(account) { + if data.lamports() == 0 { + None + } else { + owners.iter().position(|entry| data.owner() == entry) + } + } else { + None + } + } + + fn get_account_shared_data(&self, pubkey: &Pubkey) -> Option { + self.account_shared_data.get(pubkey).cloned() + } + + fn get_last_blockhash_and_lamports_per_signature(&self) -> (Hash, u64) { + todo!() + } + + fn get_rent_collector(&self) -> &RentCollector { + &self.rent_collector + } + + fn get_feature_set(&self) -> Arc { + self.feature_set.clone() + } + } + #[test] fn test_inner_instructions_list_from_instruction_trace() { let instruction_trace = [1, 2, 1, 1, 2, 3, 2]; @@ -980,4 +1028,481 @@ mod tests { ] ); } + + #[test] + fn test_load_program_accounts_account_not_found() { + let mut mock_bank = MockBankCallback::default(); + let key = Pubkey::new_unique(); + let environment = ProgramRuntimeEnvironments::default(); + let batch_processor = TransactionBatchProcessor::::default(); + + let result = batch_processor.load_program_accounts(&mock_bank, &key, &environment); + + assert!(matches!(result, ProgramAccountLoadResult::AccountNotFound)); + + let mut account_data = AccountSharedData::default(); + account_data.set_owner(bpf_loader_upgradeable::id()); + let state = UpgradeableLoaderState::Program { + programdata_address: Pubkey::new_unique(), + }; + account_data.set_data(bincode::serialize(&state).unwrap()); + mock_bank + .account_shared_data + .insert(key, account_data.clone()); + + let result = batch_processor.load_program_accounts(&mock_bank, &key, &environment); + assert!(matches!(result, ProgramAccountLoadResult::AccountNotFound)); + + account_data.set_data(Vec::new()); + mock_bank.account_shared_data.insert(key, account_data); + + let result = batch_processor.load_program_accounts(&mock_bank, &key, &environment); + + assert!(matches!( + result, + ProgramAccountLoadResult::InvalidAccountData(_) + )); + } + + #[test] + fn test_load_program_accounts_loader_v4() { + let key = Pubkey::new_unique(); + let mut mock_bank = MockBankCallback::default(); + let mut account_data = AccountSharedData::default(); + account_data.set_owner(loader_v4::id()); + let environment = ProgramRuntimeEnvironments::default(); + let batch_processor = TransactionBatchProcessor::::default(); + mock_bank + .account_shared_data + .insert(key, account_data.clone()); + + let result = batch_processor.load_program_accounts(&mock_bank, &key, &environment); + assert!(matches!( + result, + ProgramAccountLoadResult::InvalidAccountData(_) + )); + + account_data.set_data(vec![0; 64]); + mock_bank + .account_shared_data + .insert(key, account_data.clone()); + let result = batch_processor.load_program_accounts(&mock_bank, &key, &environment); + assert!(matches!( + result, + ProgramAccountLoadResult::InvalidAccountData(_) + )); + + let loader_data = LoaderV4State { + slot: 25, + authority_address: Pubkey::new_unique(), + status: LoaderV4Status::Deployed, + }; + let encoded = unsafe { + std::mem::transmute::<&LoaderV4State, &[u8; LoaderV4State::program_data_offset()]>( + &loader_data, + ) + }; + account_data.set_data(encoded.to_vec()); + mock_bank + .account_shared_data + .insert(key, account_data.clone()); + + let result = batch_processor.load_program_accounts(&mock_bank, &key, &environment); + + match result { + ProgramAccountLoadResult::ProgramOfLoaderV4(data, slot) => { + assert_eq!(data, account_data); + assert_eq!(slot, 25); + } + + _ => panic!("Invalid result"), + } + } + + #[test] + fn test_load_program_accounts_loader_v1_or_v2() { + let key = Pubkey::new_unique(); + let mut mock_bank = MockBankCallback::default(); + let mut account_data = AccountSharedData::default(); + account_data.set_owner(bpf_loader::id()); + let environment = ProgramRuntimeEnvironments::default(); + let batch_processor = TransactionBatchProcessor::::default(); + mock_bank + .account_shared_data + .insert(key, account_data.clone()); + + let result = batch_processor.load_program_accounts(&mock_bank, &key, &environment); + match result { + ProgramAccountLoadResult::ProgramOfLoaderV1orV2(data) => { + assert_eq!(data, account_data); + } + _ => panic!("Invalid result"), + } + } + + #[test] + fn test_load_program_accounts_success() { + let key1 = Pubkey::new_unique(); + let key2 = Pubkey::new_unique(); + let mut mock_bank = MockBankCallback::default(); + let environment = ProgramRuntimeEnvironments::default(); + let batch_processor = TransactionBatchProcessor::::default(); + + let mut account_data = AccountSharedData::default(); + account_data.set_owner(bpf_loader_upgradeable::id()); + + let state = UpgradeableLoaderState::Program { + programdata_address: key2, + }; + account_data.set_data(bincode::serialize(&state).unwrap()); + mock_bank + .account_shared_data + .insert(key1, account_data.clone()); + + let state = UpgradeableLoaderState::ProgramData { + slot: 25, + upgrade_authority_address: None, + }; + let mut account_data2 = AccountSharedData::default(); + account_data2.set_data(bincode::serialize(&state).unwrap()); + mock_bank + .account_shared_data + .insert(key2, account_data2.clone()); + + let result = batch_processor.load_program_accounts(&mock_bank, &key1, &environment); + + match result { + ProgramAccountLoadResult::ProgramOfLoaderV3(data1, data2, slot) => { + assert_eq!(data1, account_data); + assert_eq!(data2, account_data2); + assert_eq!(slot, 25); + } + + _ => panic!("Invalid result"), + } + } + + #[test] + fn test_load_program_from_bytes() { + let mut dir = env::current_dir().unwrap(); + dir.push("tests"); + dir.push("test_program.so"); + let mut file = File::open(dir.clone()).expect("file not found"); + let metadata = fs::metadata(dir).expect("Unable to read metadata"); + let mut buffer = vec![0; metadata.len() as usize]; + file.read_exact(&mut buffer).expect("Buffer overflow"); + + let mut metrics = LoadProgramMetrics::default(); + let loader = bpf_loader_upgradeable::id(); + let size = metadata.len() as usize; + let slot = 2; + let environment = ProgramRuntimeEnvironment::new(BuiltinProgram::new_mock()); + + let result = TransactionBatchProcessor::::load_program_from_bytes( + &mut metrics, + &buffer, + &loader, + size, + slot, + environment.clone(), + false, + ); + + assert!(result.is_ok()); + + let result = TransactionBatchProcessor::::load_program_from_bytes( + &mut metrics, + &buffer, + &loader, + size, + slot, + environment, + true, + ); + + assert!(result.is_ok()); + } + + #[test] + fn test_load_program_not_found() { + let mock_bank = MockBankCallback::default(); + let key = Pubkey::new_unique(); + let batch_processor = TransactionBatchProcessor::::default(); + + let result = batch_processor.load_program(&mock_bank, &key, false, 50); + + let loaded_program = LoadedProgram::new_tombstone(0, LoadedProgramType::Closed); + assert_eq!(result, Arc::new(loaded_program)); + } + + #[test] + fn test_load_program_invalid_account_data() { + let key = Pubkey::new_unique(); + let mut mock_bank = MockBankCallback::default(); + let mut account_data = AccountSharedData::default(); + account_data.set_owner(loader_v4::id()); + let batch_processor = TransactionBatchProcessor::::default(); + mock_bank + .account_shared_data + .insert(key, account_data.clone()); + + let result = batch_processor.load_program(&mock_bank, &key, false, 20); + + let loaded_program = LoadedProgram::new_tombstone( + 0, + LoadedProgramType::FailedVerification( + batch_processor + .loaded_programs_cache + .read() + .unwrap() + .get_environments_for_epoch(20) + .clone() + .program_runtime_v1, + ), + ); + assert_eq!(result, Arc::new(loaded_program)); + } + + #[test] + fn test_load_program_program_loader_v1_or_v2() { + let key = Pubkey::new_unique(); + let mut mock_bank = MockBankCallback::default(); + let mut account_data = AccountSharedData::default(); + account_data.set_owner(bpf_loader::id()); + let batch_processor = TransactionBatchProcessor::::default(); + mock_bank + .account_shared_data + .insert(key, account_data.clone()); + + // This should return an error + let result = batch_processor.load_program(&mock_bank, &key, false, 20); + let loaded_program = LoadedProgram::new_tombstone( + 0, + LoadedProgramType::FailedVerification( + batch_processor + .loaded_programs_cache + .read() + .unwrap() + .get_environments_for_epoch(20) + .clone() + .program_runtime_v1, + ), + ); + assert_eq!(result, Arc::new(loaded_program)); + + let mut dir = env::current_dir().unwrap(); + dir.push("tests"); + dir.push("test_program.so"); + let mut file = File::open(dir.clone()).expect("file not found"); + let metadata = fs::metadata(dir).expect("Unable to read metadata"); + let mut buffer = vec![0; metadata.len() as usize]; + file.read_exact(&mut buffer).expect("buffer overflow"); + account_data.set_data(buffer); + + mock_bank + .account_shared_data + .insert(key, account_data.clone()); + + let result = batch_processor.load_program(&mock_bank, &key, false, 20); + + let environments = ProgramRuntimeEnvironments::default(); + let expected = TransactionBatchProcessor::::load_program_from_bytes( + &mut LoadProgramMetrics::default(), + account_data.data(), + account_data.owner(), + account_data.data().len(), + 0, + environments.program_runtime_v1.clone(), + false, + ); + + assert_eq!(result, Arc::new(expected.unwrap())); + } + + #[test] + fn test_load_program_program_loader_v3() { + let key1 = Pubkey::new_unique(); + let key2 = Pubkey::new_unique(); + let mut mock_bank = MockBankCallback::default(); + let batch_processor = TransactionBatchProcessor::::default(); + + let mut account_data = AccountSharedData::default(); + account_data.set_owner(bpf_loader_upgradeable::id()); + + let state = UpgradeableLoaderState::Program { + programdata_address: key2, + }; + account_data.set_data(bincode::serialize(&state).unwrap()); + mock_bank + .account_shared_data + .insert(key1, account_data.clone()); + + let state = UpgradeableLoaderState::ProgramData { + slot: 0, + upgrade_authority_address: None, + }; + let mut account_data2 = AccountSharedData::default(); + account_data2.set_data(bincode::serialize(&state).unwrap()); + mock_bank + .account_shared_data + .insert(key2, account_data2.clone()); + + // This should return an error + let result = batch_processor.load_program(&mock_bank, &key1, false, 0); + let loaded_program = LoadedProgram::new_tombstone( + 0, + LoadedProgramType::FailedVerification( + batch_processor + .loaded_programs_cache + .read() + .unwrap() + .get_environments_for_epoch(0) + .clone() + .program_runtime_v1, + ), + ); + assert_eq!(result, Arc::new(loaded_program)); + + let mut dir = env::current_dir().unwrap(); + dir.push("tests"); + dir.push("test_program.so"); + let mut file = File::open(dir.clone()).expect("file not found"); + let metadata = fs::metadata(dir).expect("Unable to read metadata"); + let mut buffer = vec![0; metadata.len() as usize]; + file.read_exact(&mut buffer).expect("buffer overflow"); + let mut header = bincode::serialize(&state).unwrap(); + let mut complement = vec![ + 0; + std::cmp::max( + 0, + UpgradeableLoaderState::size_of_programdata_metadata() - header.len() + ) + ]; + header.append(&mut complement); + header.append(&mut buffer); + account_data.set_data(header); + + mock_bank + .account_shared_data + .insert(key2, account_data.clone()); + + let result = batch_processor.load_program(&mock_bank, &key1, false, 20); + + let data = account_data.data(); + account_data + .set_data(data[UpgradeableLoaderState::size_of_programdata_metadata()..].to_vec()); + + let environments = ProgramRuntimeEnvironments::default(); + let expected = TransactionBatchProcessor::::load_program_from_bytes( + &mut LoadProgramMetrics::default(), + account_data.data(), + account_data.owner(), + account_data.data().len(), + 0, + environments.program_runtime_v1.clone(), + false, + ); + assert_eq!(result, Arc::new(expected.unwrap())); + } + + #[test] + fn test_load_program_of_loader_v4() { + let key = Pubkey::new_unique(); + let mut mock_bank = MockBankCallback::default(); + let mut account_data = AccountSharedData::default(); + account_data.set_owner(loader_v4::id()); + let batch_processor = TransactionBatchProcessor::::default(); + + let loader_data = LoaderV4State { + slot: 0, + authority_address: Pubkey::new_unique(), + status: LoaderV4Status::Deployed, + }; + let encoded = unsafe { + std::mem::transmute::<&LoaderV4State, &[u8; LoaderV4State::program_data_offset()]>( + &loader_data, + ) + }; + account_data.set_data(encoded.to_vec()); + mock_bank + .account_shared_data + .insert(key, account_data.clone()); + + let result = batch_processor.load_program(&mock_bank, &key, false, 0); + let loaded_program = LoadedProgram::new_tombstone( + 0, + LoadedProgramType::FailedVerification( + batch_processor + .loaded_programs_cache + .read() + .unwrap() + .get_environments_for_epoch(0) + .clone() + .program_runtime_v1, + ), + ); + assert_eq!(result, Arc::new(loaded_program)); + + let mut header = account_data.data().to_vec(); + let mut complement = + vec![0; std::cmp::max(0, LoaderV4State::program_data_offset() - header.len())]; + header.append(&mut complement); + + let mut dir = env::current_dir().unwrap(); + dir.push("tests"); + dir.push("test_program.so"); + let mut file = File::open(dir.clone()).expect("file not found"); + let metadata = fs::metadata(dir).expect("Unable to read metadata"); + let mut buffer = vec![0; metadata.len() as usize]; + file.read_exact(&mut buffer).expect("buffer overflow"); + header.append(&mut buffer); + + account_data.set_data(header); + mock_bank + .account_shared_data + .insert(key, account_data.clone()); + + let result = batch_processor.load_program(&mock_bank, &key, false, 20); + + let data = account_data.data()[LoaderV4State::program_data_offset()..].to_vec(); + account_data.set_data(data); + mock_bank + .account_shared_data + .insert(key, account_data.clone()); + + let environments = ProgramRuntimeEnvironments::default(); + let expected = TransactionBatchProcessor::::load_program_from_bytes( + &mut LoadProgramMetrics::default(), + account_data.data(), + account_data.owner(), + account_data.data().len(), + 0, + environments.program_runtime_v1.clone(), + false, + ); + assert_eq!(result, Arc::new(expected.unwrap())); + } + + #[test] + fn test_load_program_effective_slot() { + let key = Pubkey::new_unique(); + let mut mock_bank = MockBankCallback::default(); + let mut account_data = AccountSharedData::default(); + account_data.set_owner(loader_v4::id()); + let batch_processor = TransactionBatchProcessor::::default(); + + batch_processor + .loaded_programs_cache + .write() + .unwrap() + .upcoming_environments = Some(ProgramRuntimeEnvironments::default()); + mock_bank + .account_shared_data + .insert(key, account_data.clone()); + + let result = batch_processor.load_program(&mock_bank, &key, false, 20); + + let slot = batch_processor.epoch_schedule.get_first_slot_in_epoch(20); + assert_eq!(result.effective_slot, slot); + } } diff --git a/svm/tests/test_program.so b/svm/tests/test_program.so new file mode 100755 index 0000000000000000000000000000000000000000..9336ced24df6b4eec7b37c9530c99003016c28d6 GIT binary patch literal 170136 zcmeFa51d_9buWHql1l>`XiP$gAVw2luKdwBxtWaNkCEI= z_;WbcOaKM_{rcpOAhotK6Clv)gV&~pzRvTemNsbH_ptU4eYQ2Vwm)8LYg((OR+!)S zyZ)TB@45Gep!jP0aYJU`z4zK{ueJ8tYp?z1oU>>BhBtS1bVMGeMBj~=TJtK+Sz`qc z+}CNpP!_F-rsD66=oEo1prm9d^6`@OTHtXsQ_(Q@B>da-W-ZUhM_#RP9*6a&qUgx5 z*2~A!7ic|?y&kD&drPcdK7K;0MIKMLbamcx`89E_W(0p{6K2_rZ zFQ0m{WDHD!jPT3r6{j@T%iuM?00}dE5RF~i6Gepx2rRv5F4GFf^2?CEPfDYxK2Lu* z9q}yX#M6X-YcYfWap3RmIsRtk_&YD)@5h1O$ArGDKkomj#{PH;Y3~o0du_J%%K=nl z&sC;c|0|k^`XTvKsHNz~K8d0^P~4xjF~XdKfTL!xAG?6F5WPYGr=c?IC*8*;nNfd8 zvZHALJ~lZy8RDz8z&8haedH=)&_Mj30RB85%>Ujiv;3FZADN#2@Z(}yPH%oE*AGX7 zesDfqK6Ak8(LfGK_fe5svUvJYN`U1%-7JS9uL$~oxA5D(>gJ!~#xRG(z|Ga-E|wO?D&c|{vKx4cYrhImNdRRNu$Jfj9Wn&mmO zp*)l4=*vWBh=+8p59q9)X8i-X(jq1%&-%VBpW`H1x#%c!DE~ttT`QNQ`P&-$Wloy^ zU<17JmvpN>9Ynu#q`JN^%@L4vm!y8(^dlUnJ|~hMj(ft-!FawhXrFxkJa}{cJ_!8- zXq5D!0D)=zkF?GMOj8AuK9*+n>Q`zyx%fe~A1gFnUugN=4@f@8#Rr4-iI1Xny{*?2 zp6Mq*aD2b=ALc)f{0S0)Q0cf;;`rk_j)?z(9R5XsNb&C#@F;%F=)6bM^%0A=X-xP- zuxT7`M?ZDX{X^ZKR=&oxoc8h=j?-~VgO5)m9opBy(?S2(Gg7Z;K2jmlhT^ejB|qtY zLgIM*h^7}mY4Ot%7dtz|UKgT78mdxQ2nWy}NQmeMkUpjv1}wjVdJKi={Ss5o==w|! z0Uu9*Zs8YkJm&N$9bL1eI`}vw^msa1tn^hnZx+9(c=B5XupZzQ`P{1YiT_<7!0E0A za_I(0d-~561TG(!TYRedCG!;jG$<`OPL^q!?VStVtE(7f9@G;4Ev;es&RluVRomXc z(ywm(lme{tDY=(GTCQ(o=}mOG}i)7cdSOLp>(h z9{VdE*YP}L`cZ%{&rluzBhiz1pXzt~km_~(@N=5~pw>$^%@O!y4G<$FOSC_SO4QOQ z8q(~{E-6`Jk62Fox+s_rNZ)Sw6;3DZz_k}zCBRevUu*Ra(7e_ux*$B(F;iT7K7 zjGKKj`x)<3c_k~={`-8H$Cu8JE$sXvwCCffYWQh)#V%RD9F*P($EW(u@tD%*evI2m z_p{i z>{Pt#n2xu3!~%H8lJIUHD^rkx5YN+bmzA4cwSJ*LD)ssVJgRhlQ}NomFe}%3h3N04 zoc->BKVuF)1dfAfl)u!^@O5lm&3sh9Ov`#t)7aO$<;#>$rWw3HT5YzYMQa*KA7ggwZAk^VoFx)}p2!pTFnDjL8S*6zpxob7~EqqqxXH@2d zgHT_u>AK~6KX4o_#d@{AMaxq;SpVel`g*mgpN+42(b~C0<6i5pw4X0gdYby#`L}g% z%D?yX0>#(NzX~!>XX1QX?EVM&J+}qFN%v0CYu7iQ4~X~Kz`iBjccpOTBr(4?>3&r5 z!|{HI6f}b_x>LMFXX@SGVEX7216X~U+p}TX=2y;{+Ez;dSi7fqWU9J zEOzoa%BQMn^7}{NFZqbt$NGMQgPO=bhr!i%X+G`P1Mkwdn8W&C!Md=%*Xp5&rEdSd zo#WO2a;`pA_@&hUyk8y<1hqNCJwj?bsJbnvHyWaXwbf8Z@){)#kz$-BdR-Iod5Edi77S}bG$u-agdI; z({=Wb&k!_GIDgO%5`H=Kx_FZMugNWD2geki?uFRPaZ7^^+Qn9Oa3$>s>?Wzq{LybW z%}?I$F9JTLGq4v+f^pVL56Dq_J;?Ukp4^C5zQBX!aX)opaFqp5)pmMXt%U&GSFkOI+VFuw8Dw zv7M(ubNr~{i666icL-S2^>K}TU&H+$)<4hc&&K{Y*F_8?XD}E1e?tA0`aP12=>N05 zc%SkSFV%ead$_L9{?l^GaV0xEo$2R zv1wRWJPkP<&}`s+jr>}*RT(HmpOLiF*|kmqYv*cQJPDH-g8R93lT&R(%eNaJUA8{) z^~!l@m;6jzsc`e5Buvk}SJNXZwrrdr8)Qxvgx8uMqu{#H$u+h59Hjry_PgjCxB<)ccFEFA?t3P~P;@zjvXw#fCUus39DmTLS-%>z2_VO?;2R5HKd5 zE?akHdi6GaruKz zQr)vaX)i=-U#a&fipo{f+p1KHZbCjG>nhHu>sHP8b-L42Tdw)FevL;iv-tc*c#dD< zV|S8&U&j`r?+U-ZFB8r)&j7A&?be4BALaM6;K}#HD;=uP@tCfck}3cYlA6w6$zHAJ z^F?*+ucW;pTNhQwzHaH?li2rNYqoz!Iy-^F?b-kmXwbGzpxom_iF z<6E>m`N%VpuG&7&P`B2vj%m9?-Txxxp}vv-i_fwFs>#6(cgu5uG&;JWswGKx;rTrjWl!NO&e=2|v z;g;m!P73IJsebCqxvsoW{82Kl@+e^K7_KXqAnp3sF82EoZjao53GLC7&u8oH6M_HL z06tlKO!d+Abfun$B^Rr|LOpvjkgxlf;eNnTj>m_D&Fnn?VNLt|&i)VaP(L0H`qTZd zWbsj>$HvRWN)PEc8t8F&-hJ#v=o5c4pik&+LEp0h9@2MrNZ$#_X-+Wiei%80cu2=+ z105z8+uuztw*Fgba=R*%*YrVJBn>Y`UPp5M*-l=e+)3wk4RpS^UafeUm0{ewurraEZFa;yPvg4>-m0UaXOG9#A|gPD|TRU z!}R%R5TVkk<;7_*J_zoA`+P~d=3x96BV8aBB6T>D8@0asyNRvy3PK&*6LgW!@Aci9 zCcd);pE`&I+`3~ZPSbtiR_#w0c-75(et zmL%psxjl8c(LW(RKZoJ@3xnII0c)?)p#qPqinnh5(oNM=Iw3CO4`a@^Ymf8;K-nw5RX;Vz(6-$+j2ttB{zq5HP|oNAK1TSWwu}PYy*7rehGWP_eawC z)#mZgzTOw$r+vk7u&l$zgM9MGCAr&)_;lmn#(Ow!e4P9l;9rh?t*^rWekuD}=fk8r zom}stdK8~!^v>6~GF8W^_b1m?Ea5%~^*gcoAz7;Mw1?7ql?T_gUfv zC`727#9GbvDY;hl{vgO8CCNv0o#Xw|wO6X8`W5d}yB0t6fcksO6>q%H{F~**-#%L} zEm!_{uE1Z;XV-0tH(6eiAIa=R5_@~;eIm0j&KLd32K!vi?AH2ld~8?z3t=~w;HUF< z5Wh)BviJ$1KR@4b-NSO~+2Zfh{wZ8VW($>{@UaU(26ti5uFEqyMfuOq2fn{W{boq6 zeN^ZuL^xbTsIQeMohRzbUgYwq+P#61S&Id*YWJ^3W}R*6rv=~0ER|<+X-V>vW!g{4 zJ=#CXR&s&%jgRWsze&BJS+9}&+SsJwIY-jG?<@B5V(WjuZ(;X&()|FSCFXcc`6@4E zE&2n$F8K0#Ra18ccIjCuPcD8&;$p`eBK|1P_WGX^JU)NsBUZ-)kuFUj|%I*b?CgPr5!A z(C6da^_F&?^tnCqeyaGr5VI$44-Np9@M$@IGx&a0ns4J^=#)d!FGDNUe$=Z9PrgZC zVtU1WVxgPJnSVx3erK?h`4Xk);az>2CcZ1A-0acEOH$DFfR&^AAR4?Jv7EbGJ)ak1 zT^_-{sJ?EVt?jkb)1j+WPbuFUpr6%;HJ!;(^)hdFU2p3qSKoJb`Hh^Gv%jb5cpJG4 z#3Lm4mLyK^d7Wnbj;Or+ydpVG=}(3PT=roo+QjT^A-YV*i;X+3Pwsj*AvBP$D}ciF zx^Ckso~7ln;g4rqyQdl*wqKfDto@#>d{Wx+d6Dz%FNNoseotk;;-w#S7wy5grJ=`< zGF{8)+K>5@c(&-2A_YnJo!ZVVqY|g%(C*O?el*}O>E4vWk?W-U3a#ILwZvh+kD~pg zdyD1=bZsJC)r_v+fMYoUJ!bIxs_8zR?K_A2bn7sFC$d)o(Z|d6ZQQh>jIQ{KX z!PmcBR#Wl|(N{|l&q4bjOK3&3ryyP*=qZ-89+^d;eURAn(ESJRZ_4Ybiw)S|(KV?) z#CJ>eG@bgdpoetOFM9?A6sNBcK;l~y^gG+9-|Fpof3>5lenVq>#CKngz7SvSKGn+? z!|Ux6--9`PG+|yI->r7R(}aIG2Y*TfypI3a)=}m3JVnQ|`>XCxxE~(dc#h9ie$(e7 z&>llO`Gj;mo};UXdR`vieN1#CqiaIa7ayDypI^(N0o11 z7rCDLxmlNu;~I%VXxBgCeiHTB^=tEu-w}H6)9P~1a8r)%BLUs{am;;+!dp$4Y#;jn z(}>B(EAZR*8EWHdNBp^l=KY52g@SnO=Y<~J16H|?-EHZ}(jT_;F}cGZkL|VeQAxL= zZ)qF)z6;v(^tIo&QXUJ@_8gsyj4r3Ml%sQwrJc^1l5R!k<~DSG?FXWBFh}Q3qr>S` zyBv?XJk=g&@>KgBk12mi_eEA;`AfRjSX%i@x>s0Q`AfQ&NZRFloy`x@?-1p6-8V}) z^_6yk`cDZ{Z+3nU^MKj?%1N0Y#&W{B9r-yF`FJ5-=8v4N@X~JQl&M3uU&(eZ$h9-X zc4&kVLVwTOAw2Pq2JNKsI78>>c+BKs{#86?@-Y7@9y58EpB0Z?A$%uioF(nXV=DLL z4E0muG3}S+jAd4?{g#};Q$E#K3hKHu&QH^%_qsDml0Jy}Lgb0eyj<6lU(6#6mFYSW z79xG#qh{l1#GhZ-r*O2VoOfui+MT~Y5uAhYJoK(Dq(}qh_ZyJ0_tVSS&ds@YKK&zV zXG^Y~PlfGNRo_N<5dooERlbK*J>dUN*Zwv?E*^_4eMsVXOzoKa?a3z|)pEP1lYHa> zNmGBJ`tYmnvif&wZ2nn1ruNV8HOJ$c?&)K#%+C39KHje1$Jjhe_Z6JJcwFh*JnNXW zBjXl$;&J6C-uJZRZ=R>;=e{q`b9-q|n{Dwt+fpfy$CUmG_74c^_Xt>@>4J<_DF88;VV0`B9FSvhCwLFxY&o70)@8huv@0j65H$FkUK|4ot z_j1H80RN*o{5YyOA^dl?pqu{0k>L4H(tlBI{MAkn|6M1De@PDiaDd^GE(g zdq^`3c>WaA!;$kP-|4CNdnOG30Aj^sdN{J~1mWq8>-@_&S^SL9AL_>yt>lgVdp4&Z zoq>FPeO>9;C-ar>N74>({whSDW^=F)4@i4eTQAp617d{j;J<`;qfVDmb%?pRhqcYh z5tWeMsBA_(^HhF*pV0f4e#HfV3)hqJze4}_B9r+jm-!U!5d+814J@xpam(@=%g3~w z`oncE_4~tZ{Je(&oFvAbUxy}U}hy(YXiyEQY`DYMfe%g@hZ~M!cTQN2+#Ik6VT!Pn3!Lfd^|^o`K!rx zVgVi7B}(N#()~Trm+S1g6yl?Y>gTW_pB(r8d}MOV5vdoJbAIx9EBVN?s)u`R-nH?} zex@E4qIU@)ex4fl8QfVV$***tF8Wsem?WdBt$TgnwP(Ewy4gOW!w?*w%%Z4i9vInv5emCwVH9r4~X?Lz%tO>^If zXYxSoo^$O+CkFn@-emUEDCf5%461^RcZ-!*C z=|do2pC_Kn_1^1&9#T6QQBTMKOK{HQ-92c)&OGutaPZs z=DK-BZlAA&a-I*Y$hC7m6yN#E+WA-I&;7pRw9^?pS491qXYE{_YvdC2bu93c5RkA4!mY0}T7DjhmM#kRi7$~C{z0R=$_ z{kTO)hxpe3f8F-iDHqxomQ$2$|BWd3{ZIEVhdWP}n|8yUT>?McIbYVL#m;k4OV0)5 z2mSyh)_YaK$1&~!O`z_(B{|$VBjQvy+*yin%?0qwIRTp(B8H!8^6Kok$#59{-uO(D{CbC}QmuG+exIOR1+L2q6s5#Z03A0a&NjTWK> zP5JD;r5tzyz!jX?(yHE&liUNuFNwJE4}`lo!>|F=i-QW7eL(3SI2E0gS;pS z<}+mT$$Uw>9d$ZGIqd^qem|)lz1jSDVssNf1N~BxM7rOJ_{8Z}MhaqG(7)q4Z+gF! zcWHUOkFB5`^b2!1pO+AOJJ-=@ArvB&gX9A+%U1+1hM)TiCBef{272~NgmukxGS9|) zRWbbiF7aN~&!pSdEg#nP$npAN7y2Q&Ul+iB?=(4`qYLf%`-x7cwPjk{O-=_h2;_@l zB;{8PUtSK!Ykw=+zfaMm`s-=;59xn&KKh-z3HJ6o$KJPjnbBwaRxE%0Re!aJGb`pLZp6)-(U87Xxv9x5%@dB4sDqEYlD31FDYZcGcZj%?e|Ei&yN7W+pS+D zU{SrKu|L0CoDxfh>q*`EscWgi`MNm1RL50(iP;BRKi5tj6~H0Wqx9aPiVb?rSp{?! z)Gn@CP?5Cvb5~W8(a0ql`*WbYuRv1B6Jz&>H_tTty8np!k5iUN#j4$d_Iq`OXrbiS zjGu|3B%^JpPECKxVqfR_du_I(!4k?3Tf7}`lD2rq<%*E?7_xP<&dV$&Vevz157Or+ z*9$o0aE_F(+6sgSzRyN|imHm6`Ru>tn1~qDZ!Dhlr^2tp*VT$!Y-X13#7$MLGIC8^-_k)P+bH1g9O z@KfHZWd-r(Ko8{29G^%0`RbH@G{bV9+lf4;=<|++=huo>X;m8@XeVBsm z!k}OLp76Z}2O#1dn#Wp*&5tTXYUg}ji5B+RrJ(vo`vMU~LwyRLRF%*BG%5PT z9M>zxgb&li7s4~0FVCkrFFzdGC(!XThWFc6pZ-Hw58Hof+V8p0&dteH@d&!`JVR$`7VFz zJN4ZCgmhkICc*$>{;YJ&lXTnXdKPMh@cAo_gCED|dN^bLDCEv5-N&ErLo>ZP$WPZF z=BE`q%|BYJcBI&Wmv10s&kb@F1UXuN9K>(_EKYkhvm5p=n9ugQFfQFsF3N5J$DsP3 z{yd-K^XCbfW_^F|FkVaS>h~}|ya)E`2K+2#J+A7oFBv?q$P&lz>qc)kmT-KjofPkS1L-(1$MPiJTs~9#B6< zfp4OQKM3X1v@O?*ted*Uly`h!*d}>y|Pw=PpM87(Y)1K1d`)W=f z=PlaZ3lNVY9_fBT`q#^MBcB$FDUcFnI885VyTaKl;yWPulj*BNh(B(@E3Zt z6#`^ze&FU=LU6k7pj>A`FETg*kNkOw5tFCS!^r~0SBT02==3)8qx3kv7l5uqs4;Sj z^1*o*DgZjwZ*GrA@2Gn@Hk(f4s}m-`+JPABnx1bpT5{W;+R ztY`@D_5Aq}e}0~L>Trk-m|f27lQQo1G@M7D4D82=%%fKXcFE__c&m+{YKde<`EzK} z^~wf&sdDBz7Nn+rkDvb%{t@so!9s+IQHfG{(Jo*5cVGM*bB}GmW>M0ofXx4!UEimQ zMi1Zp^;45R{>tPW_m!B$bG=6Ie4RH~?=t~k+#lk($Isy(^*WWe`FXU@&s~iI3+``( z^9Q;!y$=z-pY%!U??YOV`wV>U+5IBY@pRB1qytSw2O%!Tl*{K)$KgYHeFgw5XS?m} zzT1VmPT$dX;3@&{VT*_HTqEtM+w;thE8}j*eV%QGdpl}p{s!T&Ol*N~V-p?`J4e=H_@&51VEMbSFe*euL4io~r6a$S*znW;i{^ z3gOOb38#emdCo-OPmw~>O^EjEHFPZT5(u;&mwyo8=Wroi3t7L3&&5P5`O-f6i#KnY z;gsM3)5nxvzgNi02m^@u)6ZjF->8>2<6mZv?hyE}{2LH(R;~yOa+;6*nH!;?2yAy0 zakeks&+HSE3sk5O={oC#` z)_u_s4=uFAhxyd(G(Vt=`2%R2>-&ij{6W|a{iA)N9!mZVk{#`aeI)(?#FD?$@)42F z0sNEvn=K!}G=T$%nLh_cYDw@M9sTSu-F3Uxn*;cU{6&@zIvet>|GBT*kbjxMgU*J0 zq#5X6k)AF+W{3{p_s`((3=#Dwe)V{l^*?2?!(Rhu6-s^kThtT)WlN zx^AqU?`geX@(`3k<`3(C${+5|_hyBg| z@O;+mLYR-Z1A*=E+@AaGY?tdE&co$doAH-=f4Rgz(chowFBRcGj{fQZr#{njk@Krz zD7gjBkG}=m;{IE>-odugq`yyu>+l41M*MjLq~8S_c%9jsYp^&W%Ip078sln*6FNBv zw9{;VWC2GNV)LVLA4lrdb-Ih<1tlqu7pj4a?S=gHs$|Q1t^eq48U6HAX_x6Q)8F%X zoc4=Q6IcT=R62D0#&+*F+mE?R+Nb~fY~cSUi+4&s;m`!`N!~2!WbvI!@3^3qeX$22 zr}Q4B*(a{UY3Gy0hf@5D{!Z)guYdn4vF~;H_dr;W^H2JGJ<`X%8P
9FrUgzKH1 zpu^=<-=%ffuRFj%UM`i6a|OWf7Zf{4E!wr`<{b~^awI2jWVMxdr9=I?`lEs_;`tZl zoOPSpHkdtN(!IpL}}!x8i+Uv|P76qK)%d0zK9!eLsxv zS3CVLU0$4(z{iO3V{|({*X#SNU5-QQ^>y}M`~3Sf$;Wg%vt<=uCo{is{rs%(7s~Bf z$c^&e6+9o~_imDlpH}#Xln|~Tc~0o>h4B64CZH@YwzvJzn3(m;}H9A znYKfE{>tb=H0k%{viF`p*u;m7M|-ax`9Agy#l^q=UN_%=4cp~CbC*}RZhnI827Di3 zx~4eU--`lyQ?EHbxNc@l`oDr74j-_k_rY^&Y%S zaq_Q!Hz&D8^+fn;!25vI3-N6dd^#_Ld^|%w0zJ5k=|B%|&glVEU-W>n?<=$Ye+lH8 z-d{xicY^#--qa(v%W)qP2$ha6H_@%~%jD1~zb6IWJf4@y5DmHcMCc&-Va{r!X3hbMD< zvp+-oaCsZP&ky;2y|4|wm0tQ&49Q}p)9;D-eT!uAvqHb$xAW&E<5MAM*0c8u`?*a_TTK6zEb)g7G+wQ0KtXm=J-gOd1UFU1eam3*7 z&mF*T=?{BPBA;(#+VS~!01I;K3Hc=yB6Ml@MKzPB>uvF*Z>yXY4DIOmHXL8inf+4I zLD(0V@3d+shthM!;8%wKmr?f0#9otfUx$=NCTZF_^~uIQIgv@d4ADjg~} zmsfg!)b!EUEv~1elcDFLcPO7rHSOoNwb{DAmhv<8676TJSBN%BJ@@}77HIw0p5yg< zh@1~tCI7sAe5MXS|DF@=tiaiO#J$~#t%^6_uOl`-<9ka2;Qo%oQ(n|8f6gQRm{xGO zmn`QK?q9VRE*HNq=6YYV_m&>OrU3bMKHJSJm5vVxf3fLpb?YkmCi*{p4@V$>Vfqy<+J$hmHRs7Rg9fn*5lF;Qf(4|7`fYAH2VPUQOlH zg6?y2bh8Pk+x?kJhdN@eA0wZrOMk|)@yJKiUJjk2_SC;0!+G`Dz|VC1nq2j)w&&~al}}sy zha?Wy0qPH{pXd80@jjI|{qM)IF`mi6`e{talJy(w%X&B`M|KJ8?{BDYeA;^y(|w$A z!-q z7p4CI{2`9scJ{^X$pC-|SLslPF1|+FsorgNM%^?yM{2M$1=h;!jO%r3Usk$(nJsYS z`@DefBJRO4Fh8^>9GApgoUZtNoW^&T2p+{BTcmN`jwm|#WoqAiJXbp3pzuC^?~!CM zeqSdU-d;K$y}f&^y}*vF2>5XN(*9GjAa zW2UF~*?7EbiQujMyp6|CytMJyZoIwNIQ92a{a+s^=I@W#eO!N@A^DJqA^qN6I!;t9 z>HX8%km`BUIB9!+&!_vKf7d8}@7D=izj2>R#s~gT1yX;9nFzHZn}@x8>gkg2<1?Kn zOn-y+=vrw|5c)9j}t8v63;d<$I)A`Np)fzz*-==udd%R8Kgii%wyc_)~ zy~dyKdt}cuDI+H`p5BlfPps*5XLPZ54BlIgc zkFzL++oW)0k93x&C>`Zqk`IRiy@}y4-l1p?vBx`{f!SS}bFZgJ9K4P_V*d{S4u;qTm$+0a&Z3P_v7RFCda;#0Q&pw3D4S2H^(hZ zR5W7W?c#nDYxsM*eZMZ1&EZh1!ivRl+Ii&wA~>aBc2@+;GOB^@tSc@;aZ*YvoS z$9CU$JBXpF@b!0>f=BV$0v_>sP43%#ZLc56_=4|HS7Rw=0#-w@AIPocuZ+>AsvmK@Rp@da_v8 zXZ{}X0hE*Oit$m9w_L*R+4qKgKC5)ve%_gj1Y>-9N#c4{qA0yDzuwZjHKu(&grCj+ zn4WX`^XIXB53l?om8ZWKWdP%pn(Xs$(cVAEG=s|}+h*TnA^(1j z+4*Z^Pb}=pFda1(|yznB^ibD(_do@7NS0d z`vK(l0BSbN?nk$UgXQVxUPSN)jtNWgn ze7p}-;@WJjzYR)12S1;(n0&Ck=KJmcFj+_YI=M^OYTP$qA_X5_l+G-TRoz15h@*@)q2)nRhzFz z`cxiOI|p<*QeF&&=y9p%|K~zaaGa`yCwK_0;t+e#rd9ht;mI|K*t;-;mPtB69KdlJ652JJlaxK7*fo z`gkaxJ|gnV`n#t6{SJk9`%ypJ(j2@9X*e|^I2z&6C>;3 zydy`Sw?6=Kq#tcxvc65o!MI^=c26L&bC)##%?g)XtK-J~`1HB#y(IyNZ`FCj_hHuMh z+TLECC-6Ks<#;Q|>J0sQzhpPf`|cMcpU`!u+p+40>D{2+JFGwNEJ>Q>v=gacG@|)# z$93DQLe7^Si z+wW<)AK9~2J2rk1K54&%KMEb8?fi{0!g)L8Sxmvc$Y0nu#LP+O$=M|-iBGkDD4Lws zO1Ai)FbOHVzqiQ!xA;`8&uR0+_+Gi=_4|dXpP*#Qe4aF zMXe+NjwjWNTVA4G?ALbaPm5mY_CVy%`Lmz-z0eoc3#&)?Ul8~eM~OKeTBGr{i!>hI zB(cvAgum}6dIWv{5$Vx|+Q0SJTYQ3c#>Y##4`t&y)ibj{Zg;Zt5uL}v^ACCc-~4g+ zx7hDrL%+Ly$o!dh_F;)IKzqQD-m9Fa>C{eOP?FU4=T{4*g8Mo2p9o(Yvvtla!-q`4 z)8>A#-#3h_C8?0UA0jyax-Gb8@An{lytwR%ZdyzgJCtphwjFK9iCVn_T;>_-8)8NH;@q+N-48-}j!}@~FW1 zzQ2DTjdGIp3gQy#9kw{>en9cU?lbtlw!d#GvG1}Kr|z$F zUCw%rw;%9dMcUWv@wn1m?`Li7AEEP!D!EGcw`wKUR{w7`V2zfOPC?8E`I&o9e!JY9(QH&Z{^bZz|Wlca$5XB08>8Aum@Ryv-L^4gd_e^49O z@6OM){-Hh)!0{0e>ow!qFL=s#=snQlG+l4TV>*7yHSLd{GhT(-?ANzQ?BmSGm){Sk zUWoo$?Cozyx`djCEspo;c-t|LRJEe(jgnuq_q3<<>@7=v4=)-Z`1b;Qy!t*@dLP{O zxBNTb>3RE@+7X|Bd>*KD+$?zGF_pjT8TCZu3HsSG?d&LGAAi+-`u@2ge?HXCfqh*1 zJYVSq0@j0BA4Ob<^!{G$F?(Obetl2Fs*hhru=G7099C@Czt7Y^ZN5Br9qJYR)qH=X z_we^UBmM05)#;(1OS~EXSFl;orTUci^Ipl0UTQxNL+|ReQ)~ob05N}7I=&$EIR1EC z^{sCGIrgR@tBiUaC#%sHjDdD?s`banVwGElpH>BS-28CvpK=GB&*9I>J%^81 zQD@J&jY&1RSNl@r>GE^ARyxiRM85yfYCMhtqFF9F&X`ZGYvVb+Sg5jSuJ=l9Zr zxSyD3X&5i^UtVteoQekhtnbaPj_dn5eO~qVTTxFLhU~s&A^NKHo6|@AA4Pe2StDIM zw_wfy{xFUodr<0CI_5Oh=Rt3)`Wz`9qY`END|c%;9rtR!BhF)Dhb<;vQobF3N{HR>8OFYx zj86(1AL zSn6M%q2=j&D*DvH@^ipYehUFgIa6dlE`2?e?e}dGgjE|ac{?ieG8iTu19%~a>wC@4 zH@WXJ3JlC=$n;bNnc4S#DQ~v#t0dpw!_;iwXP%IKcfOt9W_zv#79w3Ixg7lcYA=WV z{674av|q=Bt+w_vPp|h+r9A3uBf6}2bU&xJ_s=ZV|PztVA1Rk?5+hWeT6w_ZoX zJrlK){+@+UFZDeMx}Qb&=S1JH@F3<%zxNnESN{a^!~6M1m^M4d_2JdQ`1R)i!sn+S z=Xb>oK0F8cM5qrxhJ4-mB7A+U4PSeH%=vosMfi%_@U<=A%b!Pgdpr$#_D|3YuHQHW z%1ikdc2B1*`92WSm4fE``FC;38zm!LUt;QxE=N5+=iRGjQQjlX`n02mke{EQ+z;XV z+C~uw%h=D^cu$NC`8|Zn$qS`jzsFRU8kh@n^~+NP4D-0c`?|bKvm<-GbTqM-En@wM z{D%1y?gBymzQcAfS3>%*#d$tismX(!ShYka(0qO0fB50~Q-yH;3gV?H512xu^3>3jDLD<3P;whUyhp+rpJ|2x%TM)j`X5Wa>xn$M2TLke`$dQ^$d6MvaKlbDK zIgz`Po%&x+cctSI!Q=h39rTotbJ${kZ~KnB6+yoLD%00W{X+DMibscEa%qMF20foCUv(VjQ-w~r^FTY~a( zoBm4iPS<;Gh3Gwk*Yzgf|E?FqxL5p9&TnXkVZhr1pX8u2eY4~fKZAe&3&URYZ`IN( zHGhFbQI{UOMzyM@o9wFLZ5p@MZn_>bdPqO@-07-Jf2$x)=`lFk6WXg)D=w7s6SgDm z=s`b_|CAo28AuP?@p-p$vL2-2{t&xO^65vq++81dZuTJaiIi}N`7`ae4(~VJH}U#S z{bu#pZ=X6YKJAcJa-uvlyW2-{vUzo{B-`8FA3$Exea{oQ@!}kk%2Q;7S z5vqQ9-G7mcZr@c2yGCi+L?`7Sq`J|65WaCc< zLj16{=kI+fc6LjJ`WmfwIU?c5_~>$fS&Jb*oiJ<0Z_~X?>(~6fx^#Yrzg?&P+qm?5 zQtqd7o?vi4y;As_N#;;r7z)u}OFwsQk!1S*r0l-F5_}V!C60H$r^fHV^1Z#R#n`_m zTqDB>P4_zqkNoYBV+pa(8^d^ylF*NG2KNi&MVhYPq47G2QTKE0nGynV{Qg!-Fj{iXDe zf3MR0toSk0a~&2D^_=}vUx7aey{5O$zd!%T_d!m|%&PI=g$jQY*{h>85=+gt)wSKK>#N zVh$np=aGCrFk4ru|LyhseU-j0%PI6mVo3h&RGa6HR+Lbi+t#9%ktxCVa^237r|u976v=QrtJh&_XR8M60DC|ztf z)UUe14bD>dn-Bp-6MmsrI5z#V_6aZgWqNg=qLKYr$|v1#e1{e!7q5_hNmee={y~4! zN|pQ7pQshJktREf>cp3~v%_v*6}`MSkbsp;DuHp-ujRCgAJ|06X((T+ zaLf;%7w7p7hids!g=Lx{x#}YA-|jUU-*SbBlYwgHOyZj`G|U=o{)OszU(OS z!+lxVr`2VWycgi<+&&-e8TVxw`#sU}Jmuf*XR>mqwlk)*(axwGl%HlfJS*@{e@M5= zp&h?ZHnp?!2jue#E}ksV-=m!)pNu*Fzlwd!R{PIi{FmT7!t6=cbk7zH7m^+o|FdC1lk+-H)pm6|U=Cjcc1U-iC-{;wg*W zpW`~_5b)*c+YY!A{vEb>2RjzQ{k{Rni#h53>^dpNeXA|nPxCdsYM#zxKL7JQmbjiR z-=|{z@^Ztg;A#)?+n*m!pKDyA02DK8tlGrc26DMXqR8h3p2Oc6^e^R9M1w9bzvtU^ zxz(GkaTn-f=-sVxUB#NtpFFn{IUz0q#P3V_eNxf|wkF9R;{jxl&I%nb1b?rizt^QW z)xNXA$qQi{V(A}?ef`S$w}kS;7Vo(2pGCgD{`B`Sx;-pTMaLkN&)54Op5MD(vZJ2! zNfpPlW)ur5*sir&?)NYz`Ze8T&%r-2w3>g&H|4~@do6^kOj#vq=VziM$*3QtU&T-7 zqgGV9s8_3z&vW5k#1r!rp7*iX;~Os1^gt?KtyhSaN`bdSI*ua8_v5`C@9ms# z^A+GXn#&4KIqjtjcZ2#OX;tA&|i5y$@=d%C)tAAdtWby_UKvM*Yxq_ z{l)W0_6+wAMiILo&3uNTmG2e2^qUDs`7rkP0)+MH;N|PH9#LiWbO1A-J)!O39*5l@ zN&WMSb-r?aPr7eb{>D%VA)CLBN`CYFbxiX8xy$DH>quJf$?b}he_PG_I)7#J%Hi&@O|oy=d{i9;}(H`LhG69vOf%FLGFR70)UP_3M(}lO){Yi9 z9)C`@yj$;0IzIn?Y@AxzvY zk?iBKV`>*^`hZ61^PD|%^!c-lE?r;c<>Y+y%u+e{y#(T8j^Fz$c9@*EYD1py@7Iz2 zb`Vq>((<;?0fSG5CjSHCEg}D~#eUy+$K^_w-}7}lG!1Yp$?oT}_t1Z~j`MlCr(X>- z<+%stoL4Bxn(6$8b*zQ{$0R`vb9Ea!m7}+te-5>NpHyp_7k6+%KybNd{iyUkgfc>G zUZDG=1C};)at_&QUZ9`zU@%T;e{cU*W{3Sep{~m(xu5uC4*s9;nBX_8V)a=ZZ)(xE~nSd-qbU$D6KcyW5-FWzGf)fbnvT{rLj*8$!QA$K{0iTT?;o zuYcqF@_YpGkUp@Itz*a(LgttJA-| z9nq0TI@9~&S-bkatgs&3w1#@GezH|PeP7m85!*M@bsF0{+VT8kd6xPa2gnpcA=3A@ z_B0UC;id-m z`2VK-&jUUGndN`!3CjPEaPQ6MPnW-a&Rf}y9u~}D*X4Dt z1ABa~MDjiw>aowqzVF90!vJEM$OyHQ&F_%<=Rmfb;kG{l^_w zXGf#pYRgBrCKp&ui3 z8ed7b`Y+r!DPqMpMy>Yj_d6?y(Hf1sZy--Br?jgOOoN^-p}XH z106F_?%!{rtR=l(>(7Os$uO}=;{inc5)Z|`jrs5o8RjEqdhW0!8~jJ%-~30yh&-p| z`X6bAYj$dU-AVK02iG%%zZ@|&f_wQ5dSi55V{~5kj((}fc^;1=P4-Xy+T`SsZx@&b zoA5t7q9MfoUT4=|_Pg+fm^4nT7c|j);AML5C7N!~W732226{}t*dEsf?eyT9isHM@ z@cR7)`Zd(k@@zf-P4y4?4ejXgi(a&y3#^^B8vA|?>w&z6{mO7WO@+{xpo)*Dc6|IO z^!b%7^x5BgEPfw4@+bQI6Md#e|3}d0U&Hy*eLDQ`9cKOE&wu#yr|!@2xj+o{?B3;k zgSYP-_4yS5No<=x8|c>fd?tCEXz7sQ+@ zik;I0f7hF}!1wX0L(~Gmp`OB_VFbVL>F0_(FD5;H4qja^V6v{jQ(Niz{eA#ZZr(q| zDa4x1Yn;cn*%<)y{hlD@%|N@$;P(OiyORB#uZsQzxP3p|^ZmPsGZtt)mwWkiN;i}H z0O~Tw-?PATex}Ro)c@*Qg9L*2=Y5(?{UiFXlq>zS=QufA;JEnI(QJL|{uJT+07Lj^ za_~PN!k?~){C>qvpbJ&fdxi_q)W1@m=D+oK%#nRH`gIH(XFUHXpUc@?(~Rp|_!sg8 zeGL8{o~|=A9p9vJvQy(}z(+iO4pScZF4Z>rS^GYSzkjo9w!*t!bZPfSwTkACP=E-; zM-q3Orsu6rPb$}{fQes8eJMoxJ_Emh?RM%WAQ5?3y9*&7k;iha-#^vlVS1CvgS*HW zxBUt?06CE%`dPdO$$fm%$H4fU_3|^-zk`~L{QH?*7ivawxyIBVkr$GzU+(yx%4@l< zpZ$4eKc^n)%jyw+$2#TLpNFG<5`ovFUQsT2xg7uu>vJ5j96uTF@%#0IW=ClUvv71z zc}NB5=l_EPT3X(w4*W)wC-E~Br>gw?J&l7lpA1haNV|k%`#buSU#3Y{PoFmI^JNM* zOFJloBQ)*0#>{7M{UHA=@wC(Z_-p}_aizcH@95xiO(k}&E%%yJ`E*4oAL|s(-$yn0 z7EAN61%da-iHi4UCoxyJKJq=}l8;#Ow+Xns|EB{NV5xVVh|B9UJ_+IPD-nBx^8Ts! zM1-dur~C+mXpi6f8U!8&-mlIA`tJtQtNEHcsLkNL6fQ>{|Ge*zh3U~oUS2`)c?I$o z2%cAE0dReqa()YP8yj?FK=^fZPmev%TM+F>d4rxE+tV}k+fsveF9@T9Hs3#P^!UDg z&(wbruwvn;##28p@d2yc$j{8EY-ewUNDhGaQ`Lp&} zj{h^hgM5m%6+Fj#t9+n}eE=Tv$={f@uzupG!Mue0HHeE-teJmY-<$dqBPBm*$@n^yf_c`MyTIDMjU3>Nk)dq07cw zyzhv#dyetJd$G$wPwyfjOYZATg64R?-lrSJI_m}SR*>FMyP}Z%dvs6yh_1H^vYU!} zhTqpysT|P`?Nr##3utE;-;pPNw!`4>ZzMkupU_JCHP|P`vK}M78(l8I%bNb3?SrWI z`>3~Lt3B_xRqL_8Snqlp7ldQIUMt^Y_?zfwM(>%lv#^#3427XY$tfzc%@2<|w$IL4T=)dj33>`{Dkba`AWJ_e|Kh z_4^#$KPSolK7Xbe4810zqpJrDt}te2&R37to8g>~UlP3ZOR{iu zbzRj_*3nnuAi(>^Ugd}-KYKJ-^kHWbf;rq-O+>Q(& z$sAmDlaZO;Q-mF2zUQ+T zk7@nNv~j`b^^A_Sj(4I@3Ed4Msc}QyIv)>!KGN;aEyeqd?~d<@JgZ|0$8p3kbd~no ziH|E{Xa9UGrH@N`$2-^dg-Ay)^4(s){|44VTFprG-dNMlkIR#3wy?QPe`fO0f#>6t z^~m5{+gC~PfluNy`?K2l_Ij0q?gu;nshu+UWN^-Rj}Y$fk)vOS6UajwZk6m*Zxl}} zeg8F;8_*FW?fm5AH&?rUxjwmGRyzJhTJU|O;m%*7xtMGmztL#?ppK8nxi~>6M2|>2 z^-)Q}AJg*Tj(siaWBeQa90z}f=cEeJFDad z9Nl~T`RM|l{v#po$5DTNANK$c2o~&b+@tg|zXJ!${@qx&3yuEvnsw~=KZVY_9nRzN z?~vStHcM>R^ugPixC@yG6DI$s5YsLV;2-_p^gP7)rMy1+z!dm{s+Z~WSwcVTiG4q5 z#&Ti>UW{*s3nYf$W_Z%)gv^g7-djHViuBwhk2kBQggRXp2=*hZdGDE$^Y(@=R(qV7 zo%HXI`FdtMfJ=b)b=Sr-1yQQcTz_$bOBSo$mHvDSThQR^bbrrUa_u4Q*Sodc{d4xu zH-RfTPvKa#^^awpvV&;$P-)^6Y=wR|pd z7UCJUe&GAFuybAV=MuwqX<)ov-xqTI@$&<=^I__n^AWWFhxlE${Xnl5kE>i*?|-t~ z+j8|Lf93kG zu)WXj_=?oyC$U@6`Q}``De1oDUrW93LpQuX()z(TcvDb6^lRV1{Jo?AG33TTx%@Nn zS--@On7*{=pVzX!Zl9-gH`>wZYl!bT;$fk^5OjYcX0zfJw24)t(lzuAdpk4zd=2o!RF-?^8Y(8fq&nBC*Pae z@b@<62jk%RDJ%b+)Si6!A4N}|hb?zLL%Dvt2t(?jBqXV;&IT+etO~} zYXo)TN7at|dm#OuPbl9<$tV6XgzX}Egn3Ti|3SNMXPuAG-hay4KNPf|w(I*OE$DuL z?PF{)RL6AwBA<6*B1-$GrsbiW?+)s{n0|)!<_7%`_V<}VzJDi-bezuosOWeE#&qD>*ef1 zSpPk&kADn)4$SvPm5>(y4Dp1GBl-)>KWy=^tUo`xhHwl6i21Xk$8YeXS4zIu*Y(KG z`Ud06@wZ=(C}RIUQu2{wI$!P;v@*ZEaO-P^nYys}@i(%cHiYu1DqN@!{~IV(u z`{49UfIo&-{4zh!Z~UmzmGS*%g2BG&`jvM0lQ%?nGVhIFKkoyi^Jjzp%$FCU*9zlY zFYrgYe_{JIyr<6m=Jmw?$0pK!8Gfh3(~PtA$ql?VoXO!Aiz=y}nH)&(V)*I! zF2?hO^Zum2ii@FpTyC-t@OyV1M<3>~dRkxfVfpo|^ZHO}yB}lc^;JQWo`aIUPm%7u zmLJf4_Ycvpm5x^mLhnc1YmzkAA81aVGvxZ%-yh3-hRlyA6xWkqkeK`84E~9Kn5DQ(>-qFe5U#9imHA`b3kM38o-?yNm+c)1w2*>*uIq#c&8?x`zaC;`{}_n>o3H<#jKT8_|G$Cv z)Q$&qy#Bq9@<=+WzN;kuz;_6@mpO<7{GTD8`FY@0g`=DvW4X<{KJSO~&)*YoZk`~W z&ph-wvHsVb_ntQq&)*U*;NzdJ`@`R1D{KG9Zwh^%WIK0jyCHpF&DHzqzwCdUP}jcW zRFUs}ZR$Ont9S4J_?-0Cdw=Uab<*=++SL0}uHKu{`v|`y?cLF)-WOTV=g~mEr-A2G zK3*R5`){+J&!fhFs7GK`{N$V2uJvyypATSvB#+;pb0?oP<=*erM|Iyctj~!tZQtqJ zN(rm}Jx=;swELbma`-mtwUWa>Gas}tH1c;;7qP&H`wJ@h%OMg&@Nyf zyk3azlKIZc?;jR; z-(SV8wC@x00OZQ{?Hq13(+srN%=h`TAg96^{KSxeHOZOZ(_?7H^PJcy_VbnBI`(%Z(CknWrO zHQRAJV)T-q1*q4k-$xqu@!ki1oPQr*tk=u-e12)n`nBNZx*Hb((c6#uA zJ{dm~3j}>SZYQW<2=x(-^L9_ZZwb1a$KzR2f%cLb&hk$9<*v7fkuD+M@540N>6c z^-!OI{uXm4wn{SnPHy_1GUyS@`=@n8D{!CF{m@2z+A@Xx``2jO?SbQ?zxf5UF<^Kq z(^U|y>VGAupMTHJFxuaVi(I@v&zSGuxCeco+8M*|{!CsT9Cy42)0+Mxdq{t{KO54| z^NEapm0$Qi6qOwRDxS{=e6W9>$3?3=J$e3i?6Q6VUxX)ryk{HI!Fxl*Lsy&h{yKj9 zd2s$7>f_6lD#8(S2uJ;8DWW|-zxusmzo$8PuGVkFJ9E^Z^Y{1l@*a&+PWYLl<&#vg z>T9&V!VEfkw?e&;C`+E)JvMxvwVr9@xDUMhx{3eQ_Gs3 zr|I&5BqP5sm!|u)JbX@q{mpgNB=r|!V$G)eBIt*#UvUrI`hRe%;WIiop9wz1#OL#( zKPR8wH%2>a08{L=d#wFPAo%@V*PD$4nqOb9G0UNrQLB8g&@!J>RV#Rs+$z5mNi@H{4^je9CDB>T~@p_l-@CQ<2Z1cRAwKh_AmCe-P-mkPm;~DANq-cpcWX z`$3gyssIO}7i&;dELcCjL*aODjre?AZQP>yzJ6f2^ka*B^4$tv`mqJP^ka+sQh;Ya zeoH^1-G+WNKGS~O4nTz7ofLq8fm<~Q`C z#uo48QZb4_jskYqn|-dlsXd^1x>%U|rz*t|*gQ~mmXq-5h(1>pBj*e^ccxnJk! z!j1EI)RF3Azt*2vqj9mYN`CNp6Sl*3AmfdDwcO9O{d>l}w`%@DjbMj9D8Cw?2ilW9 zx907XFI^&V{@hk-ulf|x#!EDII#YYqB46~pMZV~Hi+s`Z7WpMK>Fw?Z)1v3-=LS7z zWd!Og&pGmO`tc-9#qx9Uw{r-*fwV=M%V)IvSLc3xofROlD%6(eb`@!`JPjk5( z|2CRF(e0pVaCz|k4W!BW+_(e@gp6NR0AFtu9iEL74l!BqbB2PvMM|HSVSZN$e-PY$ zOzhP3?HbAb8|+BpGcqIlKKUMAwK6$Qx_ir(jXNq(9tW z!t-7NH)wM7@&cZ=B&7GJ?{lZ`1LEL9aCF*)-_jyjRvk+L$CR^pRzgp!}bglAd2Kf(xKI%)@4(VWj(;m%ThXjI`GoN;6 zH|mid_BYcrjLr#zXF2KeeAc6VnU9$Eg#6HkIXw6Ksae;XT-e_|fSJ1nKM-7RU4M3i zNXm6P;?;;Zk|~6=pMC#+tLAfjl2fc-M%yr7P=DC3B*pPMysw*Geg_P$y!Jaf z&P5-HU%&gl8~kJ09XUMZ&HZTBV@T<5+h68?%~yD>KRvCN?V zd#>V{SZerCD0apA|GgA8!tF*M<-qdyzfQ8FiK@mM`Yiosi_Pv)p4`{)_q)3sozK+n zktl@ZDm`zY-3!kH2T)Fs3ciHJ;Cy-eyHQBGHdwptH^RRd+yRk)N#<3-YxQU+81TJ4 ziRbQ;D4Jn%Nbzpb{N0G8ebaAme>d=x4yS)&iNaSp&=5kh((aur-cjE-qI?*Jy7j!j z5Y3bH#9E1>8Sej?AClkKt1nVGzn6FzI$lE7j@y(#f1lQj^$O?fydAe`!xLv%eI!}$ zu*L20y8iCiYxUO|UL={*KlM*189QHv=#M7}k;>8Zjq(yXp3hQ!j|S_xJX1N|AjQy! zM!(DNRp$4-{5c(e4#MxP9>#Dg0af}PP0L1oPUw>Hc(PRHot4Hs$@jLfYg-`X^ z`n_uR;b++RcYT|y>KRR3uJsDC>I7d#-{x73^tpT-zA{x2xO`UGbAe9R0ICzi#6rbi z>G%`T2Zv`n)YS6w-%RrfS%`i^()oKF?e6vX{nXf=%c5uh?z!-3EIg*pz)_eY~#@LYv`A2{8|g*?}wN^z>~|E<&d?0-ta*OeR1&iH;b%Q?AncsZP*u~lZW8@IDRhCYx69}1=k~lVeIt#zCpS#V(|W6 z%<#QC>)^;2(ITzxp)YN}fOIFTCgD1p8}lyM9`x@}*9&KfQmFYtOPJaFB&&zWi z`vT?>_cwh#QJHe8RCGMbUNL0Z&N&6|5AP|XBa?C`BbJVe#(vI>G{}fLSg)y{Pvig-DvrH zj9$0H?#KIcksG&2_1rzHXtdnNRymJx)a#xeb9`TPbDt9E`|Vw$gkrhYbAIpB>~y`8 z*t*8|bEzLCz|f8|u3v;d2<|u2zbGMH{}6NWo)42}vRL)e_hHVle(>|yn*dwFzdHJl z;T+RX9~WJBXujXua60N&Ykrqa-1>Yq`UCXg|ET;S{?#<&@`$BvT>5xEhsuEv+jD&9 z&{#`a*L8{Yx8H+ze}1Dq7wqSL=ai&c)ODA|7`F`my+!|SxXbwf%1BY8Klh*Sy=LT- zN3xBu_& znWS6F(gs@SP+ADIO=q7wT_9~~Nogs8E`&9+cS74_$U;*D9Li!4k$_ti@jD<(1i>gS zQ4pdm21SXu5JbNOM2U(LL@|oO?>YB;&ScI*B7xuk_xj(yntPu2dG2$bv!8qJotehH zsphf&x%t!ETlS^gYu38kTRT$QTAEXCI_g_ncDKg%xto%y=Juv^Q>t~v|#?JJ$#)$s%EsW!K$6LBn( zC#w!bu(hMDePvs$xvBlY%C^?T%DqkTn%1^hTU)BtN+KO=+M7zQa(inc5i^ty1>9@g z%^myP#~o*#*0_B&9-@iZUQ%;M&q}p*>}_AQs-t;-Yb;xRLex!e#+uzxDvPL_%AzCn zZrjJiv+2gYEwLo!cU2;jO6+NDN>*oOVNmddl{?9uwv~H35_EBUYGq9#*1Ef8Wov48 z6BUMZys0^v>a0n$#wfQnO)V=zX>T~`^Tp#aZ_wlMrdP(>g60ZOO`yhOWi-{CXi26h zukXY4^&RPSs&z+8%l3VgU#leFotqPz+8W!Mb~mS58?)`LmRs%?s-~8-J5E)ZY+L1S z&!pVc4BSm^?*H0OHnq0wD=1j?Xf+{Jp>--9dV0g3{JFZ!T&kqJd2K z8Qq&ox4Tbp@6V72ZF`#%P0hRAQF-Z@X!3N7pWGU2-kox%tOiE~>vp#`?as7E-Ksr! zygq**7z&$_SUizTrO{3|H>X<)`KQ4C)~5DYd~b@^;3+MMj(ya~+g8SQ$1X{&Y)kH0 zi3+!BCnlSHV-wknl9S2uel|Aacf(9$0hqa4_x|wGfrjFdLY}<)Z9+lUg6%{=9X<~ zg?mftzzX-)*uHE_YdgiLfk9NRXiU4?Gb`L@?5Ae;ev?GGQ+lUG>d)DbXxY-yR(D{_ zS*JHNuixqo@7mF@!GHR$jXMrpy#7?*mIGTGc3l!$*Hm}OfxxMCyX$wfoV%@M-ciP3-OV2+0qTM_8H?(f*NN-7Zp3}5t&#u!>IsL#{Tg}!Ddt#~ZzSxHCmjuEc z*=-%`Pv4aEpT4`L&6~c|x9*a2F5PwMIiVKwtjM;uot@j-_H6Q=o<93@Z`v0M9ti9= zL#Le+-?;zc4f}S)63MN5>axD}i?#$B4s6>Q3^XLSMJ{UD(-{bzdFj>z@%YwVr|v)V z%u9BpGl`D%4eKsCYoY_FCVXP5GnMGDny2Wg?^ZZQbY<0og$m02^i+E++YzTBneOLX zTbg%oZ@M(Kx!D@&dEx(mF;pc}7^bGg86o$<`$_ zsJF&woV2d6`d4dgf8!;ogf;wSV+k7l-7S}-TGM-5_Q%}mmezf-_I5EYP_Z<%yPNi9 z_ohU5m8rF++B;gCX}G6>-5QYIZ9lzFv|sW-jjm^8g`<38a7V?|hl$~YCRv1T8r*4I zk}fA&T4{8kiHR((+}DyMt2H(2TWBWTZ)GjEw>1?@9&qooD)Imgu_8*$SgX#<(zKS? zBm8W4uc~mzX%H48uN;zT>|ML|n6;EqnsM6n;jO|=Ju2P4%3UG32!O0|yFcV!PJb$v zS6uGCytJuiqQ+2E8F~*lT(t zK9A4q^ZER~fG_9^`NBTa7x8=iUcZmFu>$^}KjaVlO@Acd33vm(fIkoj1OuT!IA8`M zK~K;d^acIFKrk2#1;arz7zueo-jFZk4+TQOP$(1*nW0G76ZVFEVShLf4u(VFaM%n- zOpoa`eWu?Gm_ajShE3CqM99SmB_AQv5xOiwC>`5Nv#>QSw6(`tsa1#>*R9%|jG?Au z%_%7{-N{i2vedS&q=iho+o}$lUo1ha^#1=mtgiBZ^~e5S4+lG(;wVb*yqcPt3*xk# zX%Im>O$yOS?PfPGn-Oav4GWd9_>g%{EeMqZK1f=~U)X0kq|YMc>sz zY2AeG^scB5>NT`9qWPF+Au)N2o?L3PIc!dQNm-fQhU# z_uT)L?*>N>`N(x2|AKSY?74@W;0>(WxM}lg+s;U)&j04O7A!1tRm?bSekii4_e;ai zm4~};>Mg5SeNwvV#+&E1G(Pf|*UpXq?ceWg-|@LyYE~X!z4O-mZC||O?%w;p^U!0Z zGiDxs)T)y=o_W{ZPyIMwcEpj3m!5R;E3dxx&f`xw-N!CHuG(*|syl6K{q~)^8qPic z!e}Cu-qY53`G-Gx$30)^8yLF(E6ptrf8xT$A1ZM;S2)rR+sc~U!J{1BISZXj%8xEt zR&t7S_KCTBN|!j7IICTO8QV5o87`k!;i_7_G2%$L%02T+j&Up~vDKQ+EhQ_R6=mgR zweI7cGs{DcRV7E3IcJvDZw~us`O9it6;~d2)|O?i6XzXy+`_}=mv5s4PMLK?Sw(4` z>-h4H8S76vv2=AwMd_KPwvtLmN$&di(RHqh++7zg-Z-P8bk?D(N-ILko%3_wUX$ED zv#z{i)5ZmLuI;lnmsRB6+*EOt+2;6WU3E*kV@B?= zk8VxOy4q7Y@5Xx$o_gE24@SyPbe>;&T*an}>XJhb_MMm7;*6Bdtrbn;GvltSe|cj0 z?XO(vpJO|!bhguV<;Sjb?kSn&C@-tLIeKb&`YBU;hQc!Z02Dz&n`bA_mL}4 zbzHN4&f!=>HfN^O zq|8;j7H_+9$IPS1myl~Vsg{@JezdgW>QckzC@Co|wU?E;%F5?fESzz~%p+%2&Yn5P zS?M_BkVDJo+2%W|Y)3ebEL&h(XkRqX?O5hmF{8%jad_=M+uimr+3#`Q>-v}dt&;!P z-*&uHeqZN->upH{JZDuYTvj$Dere zhyVNX@7^(-R)Y?&TD^AjX%}4maf-Xb;C_xqDvls@`aawH#TeT#?8r8?&_~S^zb9Ucy{ddtFF2J zj=LUy~kDhdhkDvbIRtGSCxwfcTs6Y>5--9lr1aSRI%Ji zZNuT6vD|rN=?q8i%M`WJd%7cchii>vj$=)k=~`BD@SV9;u9b6FIF6Zf%$(fEoCiO1 z#EiqcK3TG|WHq(4s`A{!i`!@BetG1~lH5Bbxff>s^;So?{L1qW%k{f*KPsuHTJ5MP zHC>xrGfUfN9PK#Qc}{ums;Y$*^UAk6b000e_l}wKo!;A=SN`g_vY90%xw|T_{Ck{#HKGi%BL8UuBsxCNjw`dxY{y0^_*qc=@8v`=!C({0>T5;ZP3G;bU- z-@Rz2JG$t#<@seNdfdxf?iyKc?~SfF`mJb<@wPkkg?FN%|JYs(*(#0+FPZgX_}iO&F~yvX-iQB+U+*yv9_ZZoHt`t zc{%+;g{_?Cnv!LXHLeq@Z0;~!;B-+%l~ve}vaJ%AJ6)twVL#Glw?}AFa@wh3*p9Y4 zY%_#hLJGFS?DJ^KB5ktmvXwb1>_^*H)3r0{%4#xCRvjhOWXkL_EUUsLO4Kgoh4u*f zr&2k}R%dh4?WE1-vYlzOm(6s=ZT9jRWt;5_$ezs>o^7KPOJ>-Xl-tryTPZnZKf>;G zR61u-u+%n3d}FBNDEra$uhwoWbJ^@O%5BtKZ5{T-wo4pNd%3OD@jMj*d0Qqd+Fhj; zcAICB*Xbd-#8zEC)9$8B+8icD(uIyyF1x+QVVh+u6A3!(Pt+Q=M~^WaH`twrDLji zYvcF7#1Fm4^|e?^PGvtzeP0|?#rI)zaNj2R?zhyYT_BS2_{OLus2*h&^X?!o?rr=`ZN!b^Eial&`6qlp;o-%P1rz1AL)aF+)7u&dMPqd!$QGDEbs?E3nU@t|< zZ&WpmV!n^K@}Zb#h$}yf`A*_)*evE35|`f?Lw+xw@PcsR7(7k}i?91VAvZKe={i*o z>MWv0@QiG#co!;fs`zeHoQPL>Ew5&|~HD1B9r>MX7pLcKK7_hw>4 z)!0NF#c34t^N2&;Ft-+4q2F0Q&1S3>2I+`V6uD^~KO!!_MT_1KAgQhDv^wnDn%{bRSp#?{1UA}mh#l1$Nsqn~7C)nW}K7az}%Tuiv!hx~|ZiuL+3 z(naY58%N@gWl|1*{D9>x@`UNs*G|^f4-c9*XoPvdZTg-!$F;(`(yoUH< z*gXCWy}TkMt2m)j$@0(R&m(;`vM3LNbgFwyqgt^>Y*H7uWO6 zB&$9luD_LJOwmF#hRI&>Jn|^Tms&CM&wJT>lJtxH=lU;Dyej8%atzay@h{>_k&YY={2H2?pKhVw16A`{J#iH-=Ax@9UmhiWRd)g6stk&GoVb{Wj;8s?1+U6)Bqf1LTeVv?W(=ieyvV0GTIH#b2Z+-?32TQ? z?T^CYVO%NNkKnavNW<_`%;72zP_<6tx%y;Wl=);_tP3XNTc+S*Jw2Je7=kC`MuHyf zx9n@Qo`Dz>PZfDN!z^7b&fmdTw=&NF$C z$>U6RW8zl+<;3Jbhmr@G?81DY;@wQnFu9k>V@x)e>gm-pxrfQaOm-iqUms<1FO!Fu z+>gmtsOzo%e73dXL2`_dzswNzF_5b-A0#c_xoB z+1Q|8A3a5vvm2SbNtcH=>+&d*&C~SwIwlV?d4$PhO!kNc3LUC^>X_WX3V#W$(>9dWbz1;J==79bxf|_uE!5Exqp`)Kg8s|20cD^jxJ}<)8&39 zSK;1Qm2Y2E*B@cBE2hU+C3Lx)$-PYOXY%-N{rc*RE?YN*gp_}tCOy83$%eJcpU|&l zay64{ncTqSERzQ=*3%zi@+gzl4TJK(3O5i+_AoiyrrYv9*9N15#I(Dn079=b%2 zuivlB877Y~Iohe~4=}msfF56UsV>(uIm_fhCXX@M^&#C}1CzU%Jj&#T%k=BJm^{Md z&V#!C$W^**T%*fXOdeu#*R{HS-A8nJh{+9IdVKwLx}0Hh)kpRCVJ7FU*W>#>rpuWd zbh-NDx;)6_?2USSp2^kSdVB+ub4(ujgsz|aq%POqrOTa6?&{U!2bi3_M~@$6a_(L| z-uSXES2MZyYkK@RlcV3%UG9Eem-9@H{#B26zoEEnLNs5<9yx!DkgiFT*u@nlRKH5V{#9Z`+9j?q%`-lZTl+#$;Ec?!TMKCX?%#oMCc~$vsT&WAY%AhnPId zWb4Opipx8y`|n|L9h0L>?qqT|lY5ywz~n(Dk1%t=E_lT9YqF}Z=s876l!xr@m?OzvfJ zKa&TUJj~=VCXX}ONa*=@G1<*zlgaf=&M>)?$z4qDVRA2%`@Jy*9TYT6 zb$Nuz-81$0K_=JE(&KYX9$>P1aX|QEr9WH0e)MQvZa7AlyP4e2-6gz*6VUVldCr9@txSuSN7^o z)#HbloZqa+8>i`Vl*xG}ySC{1StbuKxqqv!UwyhRcQbjM$2g<_F8A!$`=?5FU#?%@)2+(`*e_H1l@I9go!CE9@%7k0 zQ*sphVM;b1)UU66NS6nm(&dpMUC#VemwT{ZrR+zsU!~-6Cg+~lukT~>2$Qof>iT0Z z>2d@1b5wc**uPP7C-!HQY`&pipJ8$xZiIxslm7QIxvzZUdJ*4us4nNix;(^Wx2eZx znVe^`Sv0xwWZz4}Hyud+XLy+}?1FrzT`YPx;42F;3(4ceuhz z0`$R{!e=}Bps?RM+YEYDPdlyv?_>NL@Gizb2A*g96W{}ke-3<*@!N+@ILvEpr2>@zkHZjz|-JExQ|nIn!&S7 z{}S*z+{Y>XE5N&%{pxUiL?NZd^MfBM@arlw2 z_?;}FulD(s{?X7s8T!khzXJN|If5H}4Y+!KpzLgco!PMCh5otFuf_8NrQbwccr=Rp zcBmT{gE!1kM4k_TcQJktyqoclf@c~31o$xHw}98HfdR>X5xkS}d%=4ce*k=h@dv@p zxq7}H1&=cRBk&yKKLa0Q{MX<^jK2(S9-^oFDtMIfzk%l%|2Oyu<4ziPM7vVYi&VA` zwYc3-66n&6$}i4p9MY&?gEd2k0~zA(R?HAjH4XJpnqFb zau*D!{VFf{GugUPUS=}})W8WlxumqCIFdKkO<_m;Zi48K|6={?5{o+w_1wS>`_~Y+ z#&e{r>^}uQ0^SGx!>s(!=a3Eqz-b|PmhmOvb&M|uk1`$r&oRCRyo>QuhztMKXI48= zFCQZ=+KGCO=Z2rRKwo_ZR*et81$U!?3_^dIH6PfFI@NH%{oq}UuLd7sd;@W79tT(D zc|G*2=EE>fp8)S<{1)(f#=i)jVfN z68;|wUig3+_yX`MrhhEB$@ntxTE_jvg`euP#`MuG>-Z3HQSa(`k{juM)Z%ty2YrT2 z9BSJ8EcDg$MYk18>q_WvfxcB-6!$IYtLIrt{|V?X#b( zx*uF9x%&Hk@H)nS2tIz4(v;_C!26i~ufVGo>-sM#`^?Utz(<&!H^A$e{=dPqcrKt) zDY4dz)W42VM4o4Z4>CLR!AF_C8+?q}KLNa&aUXa+F6hJxu>};-dYi_bJ@S z#`)mtJqpzh_rQ*8k)Gdn@GMLBa&SW##%0%nyP2IE!3UU~9`H`4e;ar|#b?*W zLDRH#Y@+qF;Nmmy;!yk`_&DP~1h+nX;}Gl0racbc)l0J9uuRTupXF09! zMZOxC{u<)KfAt+G9{6*!rEfQuVt%oj4aMCK{c+ewwDAk#B8ANHFo4rvsMQKFajp2CriJW!8GyW{flaN^lR;KZ3X@hg$HwO{SlQeun9{fcIlQRo)%| z&oljl;C+l=4?fKJC&4S3{V#ymGTsZ`!}!<0yBU|ik4p8<_!BDK6IG7n`Dt)-na-aB zuVeZzfj2PzC-82@{|4UA_}k!BEM1p%e?s?1j2{AC$M{0xqJE>`s<G9|-S`yrX}YnF!@+N7c9s$seyZ=r>4pB) z#D%4?yLsDFgve<=a~M#z{eSPSod|bZdU^pl9>f=uGG_=2R_QUy#J)b~t2lyYU6ITRhG-kUDg0d>(aHl{vqJCC+mD6ajQK?WjR}Fq_P}by|=6Q8gTV~ui{&+0Ige? z{axUrj7P!iYIT1u0*^A@4&KT572w8t-A%-cpc-z;Ju8$1fF61Rq!6h-voDW(ESl#Dkth)y~l2;kxC`Fdf#1f zH+UUuCmwM1KDg4aB`#8EVCkL)UdQ+j@Lt9*1Rr2L1MWFR&sQt>7}LKD+|Bf_1U-pzWt_bATnd>!0%ny&vaxUo^^gWwsa{|s?4&R@sInU|p7$Lzcc z-plN~30}KNPgi__ld%5{X1|iSi0NbcM}iMBz688-i*COLy#7?3%Y8|kG0gaS==ap= z`dh&}nf+bh8CEY*@DXO`B5;H0w}X!{{VTw0nZEqq78-9EzZv=sjDH?{kokWXanatY zqBbkTs=a*^++h3(aP_$jrTM*4j+@!}E_jCVLD+v0{qY#s zABO#UW`7iRMwy*Al>IYw|NjG?XS~$fzoz#P>UI4&;3JG50p4(yu751}5VOAmyp!1p zgIAuZ+gS%*&G;5@gW1^$Ud!|w!5bKFQudjhHt;OdzZ|@a@sEJ#nVp*yXZp8*4>0{Z z!TXv1SHOoD|2Ft2wd}@h#xpjPC?Db}A3$xeqv$GKT z>oD%A`_1LhA7}bO=!@?{7RLqFU&Gi4F22)P9P0L9C%E_yV{xpt{u)LKTzsdlI97qb z7NvaAdm8Hf-AeEiVv>)cJo~}Z;3FXk)O(u01g~5nc^3NT#HAe%5>($)w>=?w4*KIL zh);r7V%+O>NJDodsWj~q<}SuYnIR z{w8=Gv;Q`D_8i?#=^`Q1emmo{z%$IB!@$Rw{zC97mhKYpA!cVeaj{-fpKGmxTdS0P zroSFM%F;azJkR)9;GN8$bBT*|)#qzfx*6#AF*{lC%JcMclh0YHoiqD6%f8+E4ry_P z8XvZzomZm1)c1ehNcxkttNW(lPffu`r{E6vI$bEn{!Ou=s=F2-l|ISDn6 zeV4cxpYOzYTVu$>PvtAdjcSy$(!T}cNuKfQwA8NySM~VQX35p(p_HAH7Rjraoo|7Q z_u9naktgF1?#at-<}s7=6;trjh>LRQgZ*mQ{}6HO{wQX%h)VZPi`$K1ln>O6r(tIp zc3g;TvTza6}n>7NJQ&v*iSjPX6-Rm^@najU&G$^_N?ayRrlnSLL5mho?c_cH!{ z*dG8_@7>K=BFkZv>CXqRgg+<-Lw?UEJ@2?s5qYkLzK3xSc!qHkyqocL;A4!R20qI8 zc5pZI^L+4r#*^S?RQG=`_z2T)2k&J1mw{I?ehqjx;~yt3`b!=9i|W_k23Ox-pvHlh zVW&T?`}wM}AJh5Yz-yWQf569?e#ugCJw5ki`m?}?8J`DkGQJ3Wl<{itEaP7ATtfGM z6>-sjE93`@3h$+-EKZ*jLAkkcSvR=)9tTx!4-yysXjs{I$O~SDees(H;+QM3G4D7L zpmIn8;B+*&E2Z<}!Rwg*O7INR4}**Mf{Txn!Fw351Mg#Y&IGSZ7h5rmbHPofp8y|X z`g_5<8Sfx&l^b|3lKC0*&E0ys&qH7RW`rvL)yK2}qUv|A5Eq{2!C}Vu z3-rZrEr>(SQ_IK!VMqO@gyL<)MZ4v{-Eb@P2X2&hM&!wO+R~@@IoUYy7WDg(?l|hJf!hZ+ANcrD}4fDh1pO!4tNc!uf!7JQ89zY5;X^#2at%lO;i)yz)W zGT{Z?x6=Jn@gd&}p>+t|w-p~pK!2F=#o!*smw~&P{Q!6a(_agoWqcEO594Qmk1;O4 z=YjecOE(7nDpn2`f%h~0Ht;&8e;Ifu+Z0!-ga zT;$gkm!7*}?gH>?#;=2&`U6nJ>1J^CJ+dnH7Vs{ne+Rhwo>`@TKXEaBj>G>lMEnZ+ z!_3ah;I$u87UcOa#6|fJvhokn4VuVrC-PN`%i`9J8};kUbUS-3eY^2DR-Qf3uVU%m z2K_gg{*NtA>pi5a>iwVK>id=4NPg)`Sq_WoH*myJ0)D2&sr@iJmlGFP7*{|Mr)$CU zm+QP6y!tAge@<~`=ZoNDSL*urf!8zrZ-V#dbo~MF5vKn`@Tw2%`ac67VEVrSuV?%|<>@)r~ z#k+Jn4}lLd{l}I4YjypniHq^)M2zPSM85_7>T7g;r&sE)VDo}}uZ}+7#PnA|U;LJr zIMn!@CN3;>GXJyS4b0C=!Ly8C4L-p5$BB!4tz`N74!Do`^LyACVg9@Z-g~|9cjEXb zxaYbFCPH?f_)G7XGCq^I@F$FPWogj568fD?e;s%?<6FS<%%7d$#z#f^6GtO>J>yNp zg+C@MhfjfDh4EbZ`91Je%>O^bzUyNXX^POB;A0=xxxBHZ^%&DH_X|B*SKpxP9|E3Z z`U}9v7+(rL%FbLEx!4C_F0F_%e08U2{7xR8EyRX>{uD)+jt@k=$r}}2y&K2POpVawB zz$=-4H+b$QUH`M-b)VAt7s30P{+GdBpV0Nc0p7{deFVIo=|2hH$N0~{2N?eic;zSc zbYB6lXZ&^WDC2K|4>Mj86n1G}ot48J@Os9N0PkjeF?eN zz^gv3+n4Wc)B5}~IzI>cT}(egT(sNF=cWEIF1rW%W6aLi!3P+B7(CDH4}xcz{xixB z)BiR2Fyp@mH$JQ9YYcpx@qdB$d`{PQhJ;M}Y0RJ5;MJ^Ns=zxLUku*Q_;T3H#~JSdcipNi z$n$OBm5kp5p8X$P{{ir!yw1N1ZhS%KKLBrFT)t0F`zI{jUqQczrTa2?AG7mk@Nvff z0d6uowy^Mz`r{qCpEJP6nEv75BaANs@5$?SP5`fB+z*~-d@Xp6@l(P37(Wwykn!`t zt8O#2BLyC1yal|M@dMyvj9&%b#rO^2=IvUt#%I8@jNb;HXZ&999OK^rA7uOyaQ7GW zbe{z8W&CI0os9nm+{N0%D@y-P-ToMOKjZ%d@3~9YcbGz^=L<}KCU_Oop9dagTz;P) zeGdlXCqjSVZarN;c;O4Q5WNO`q*vG91ny@14Dcx9=YaPzo&X?q-m-KWmQTmL3 zSn1!Z>t6?M+@tfG!5bL=yt2diUEn>8_kj;E{vB|W`STcYvA+EoTmO$if0XI}6Wsk} z-G6&T$n@Nq@fqO#-_Z3B2d`s%5qRf)x_-5?&$v(7VSEkv*w=MCo4|*d{u$u)U(xl? z0XOc~c^o{$_#W^qv(o|I#q>W6-p}-}12_8gbU&f&d`;)Kg7-50yTNN2|0;M7;}3!l zG5$DsHB0wt@DWzd&w89eiVuKy44Jkz(W5_;6$Sh_R7N1308gLg8% zh`6|KQs1{;h2*zDe~{_#1g~O#Ho}hjo^WO7dgz->|5M=IjQi2GzZ<-P@r%Jb8UGOY0OQwy*D^nE1RrPm zpCvBF=LYN#*ijshTb!P2u=0N$JOl2A{!wdWeRVOu8N8SA4&vg9x`$u@rz^mtjDJMw zKcVa21YY@FoqrB|klDEdyo%YmAAE%A_k-t{{qKPfJgletBk*3ve*r$m?7RTp&-fp} zGtAB#;C+n$2Yj5_DO)T2qkVX$KbN?u_bc#xgaqqY3w_rkQdWOY1s`YpOmGk5=PCX@ zWkH@(;0;W_1-ysx1K{of-Og3uwT#~Y-u-=D|1*j+{oBAtnf|@tLrni0;8iSNkAQm~ z)zf_vyn*Te47`)^-zfdZbUUwr4>JAN!ShW2E$~W~uac95UAn(xd=7Zm<9fPBfM=Qh zvEZXD-4)=2Og{`>#mZ+LcrW8yz)faxbvsXi*E9Z0@F8aB1@JM(N5Ngp z&R@ZMf2gPXZ}1+bUwX38qt9hLsq4$%lcs(9r*wWK^!r)4EdlRi`ZeGq%pduC6=a{; zSr7eg#tr0^!-}QpBco( zdT$Wx1WS!H!&ZPkU&->j4t$KIyM?&0qkiAVLpH>*zgGOEeR7uWmEhxyUk^UY_@}^) zXUJU9@jtafpy!W2D}ogKQmv5bK7#2#0N%j(cfs?F|A4s2m--!|DzYk$H)_RS8qZm} z{{bIocFNXC{VJwE7rc-01>l)qi0dYfrQpMiuLM`Wm#VIafV+OKh&*oqH-4$})0G{j ze>QlI@tCsD?Cb&0GyM+Y*18w_63dW$9{T+(U-y7lGy7jx_8ET|Jj3`PaZ!&h{2o+2 z()}B8@myo{SpcUzpRit>X@BfFaGZSL`C*-}2Jd8iBlrO0+rX=seffJn^!;hi>-OW& z?`M23c!t@z1iX&%9C$6W^HK0VrvFLs%3taJ+zOs${BH0(<6i|IV*EjH^*g*OmB+!| z%+IHZi+WVQ1E%`pTj1*Vz!X1zgRn@?Re!B4%d-c(_cuCU1zykeH-P6D-wLjN*H>BF z1>VQ>qu_&#Uj*LI{AnjH`pfFGY?kTo%af4@Uv#Df4M_MY@crOwSvC&)d1l{!inPCX zr?j&Y_G`fZvt0t!U(N+TUH(9Zu?YIt5*L=nnf-3?>i>mjyTfy^;?*unr*6myfKE(KL@Gi!) z;I7~4b`F5&89xX<&iM7<^}pBcd=k8x@ms-b8NVC6`xV{JSHZg&e-M0t*?A1SlkuN` zk2C%(c*Cfk?u+1+EZskW8%+Q2;QdVh9dMKBmv0j3(faKVdb)>zyBJ>p-pBY7@F?Re z!0Y~`+Yf=)|54{BgEuh!I`C1(&s26^)$N=Mo?|=#UiBAUe=oTE&pPh_uV(sJfcG>0 z5oPB!-A*@nJ>#DxF6KM+JC}V=c*HDT8}Y1hk#cyI}5>wnf`I$1 z1NsZMNPlhxKMuSO{88{y@OJPg!IhtR@E5>UIXngaFYu+XBfc+5)YmQON7djsKQ}7v_M!qh!N;-g%7H%%-ua@`S3J}x?U;AKPkA!-fRFV`kU=?o72MS> zx%wWAs;IQ%I-v7&!MkC<3U+RfN&Tvaq<#kFBRxi~q<$arHG*<@ z3cUNMO{2zt6<)lBE zKW9(DpF@A|g8fR^IXH#>eefsq58eN-gO9(Z^M}Aknf{~T1C0L=yi!dFF!xjNVWvL} zKK72D?g+St*?9##%lK>HRa7^{$3MVDofRMNfcLyzOzHcfw+Wd(SI+Fu0k8d+u0J2V zit$C@1FSrc2k&R;t^^-rJPba{%I9QogYi?rO~%gv?`8aK@L|Rew#)X}gLb8U@3E2e zCu^S{oq}&fJFiChtM>VgDfCC+PY8HRu8Q%xqRjQ}kp*YjO96ZnXHQ>XH-vHk2((Qj5e2nRTLD_fc`genyjDH2Zmf8On zc$Dcs0^Y^=55Pwlm*4McGu*74pM$<LHX6Fs?VaDH5_Dgg-j(XY7 zqm0i0&m5)e9|}Ii^cR4SGJY(0J!=ok!1Ijz!Mn=zbXS89GW`wUBaClR`pnLD@Jhzd z12-8@fOj6P`?Cjpl{&$0gU4tO`y zcbzHQ`7q;i!N(at61=uT_vcvfLB^MZ_b?s=uRcV#a}s#Np*lYmJj-}Jcr6Le z{Wy3(<9om}tUa`Y*D-!Mcn_g3#(Ti4Sbg0FKF;*-0nf1fJ^-F${2}l@ z#vcdwF#mr7?q>X1rO(E_5%4I}{{wg@<9`JoVEKIue2D2g&ywwPl<`^MX-E-v-{p z?B4_KX7%y__z>d{D*KE-2HwT&KLy^0{L0)K#?O@fWAyU;6}XG>7s0C;A651j>vqP# zYnlEy_yFU^cG*sPnH?9n!T22TI>zUN=NW$+>wFXIXVWc9ViD<2w$2~wkoN^cu%q^! zPMbo%1L<}itCz!NiZ9XmRp32L|9bEN#y3W3-}Pz|04J}{b#`Qj6VlH#P|rfd#Rq^QSiRwbUp^&#rQb*Amhdk+1@&mo5nZ600XS^0Xx=c^E4!nW!dhlMx<@cV`_ZKl9 zh5i`h8F0^XJ>4vL9pjzg{fy_pdl~NnuVUrX4L;0x4|pBpdGILXz2N@VCY+ij&sw{t9bl<^Z4 zH+6jQ@hb2f<8E*d^RpU!l<9lG ztC+qC-pP0^xQq3pI`9FeUk^Uc_<;_rD=-e^U2+}LK>Cx7Cs$3u@1KG{HwAZ|J-Pjr zQ}A=A;MYvSH(-B0i}IP zuU)0vFKv+ixYp@>w&G0xaPWG@j{=V}z7)Kd@fF|(OE(BU%JkQQcdpU>+z4LJ`02_% zE6<(EKI0dFyHD2br@*Tj-v=ILyaRlk@yo&c8NUYHU8|>i19+bCPl4Alek=G8<9C9O zFuof7F^6{BjsAEa=}*=lhhb-w)z=H)j_AKkVdsc*CohLnrr<48@Xt-bAD@D+fuFVTUyYMvQ|P})^=R0g zh9{-xcfomvafm^mgE&OA7sD8qC+l}01h1Vf!7wh{5vAXBgii74_fzoO$j{02|4Hv> zOvW##c0L)uZVKK{`I=1MsA*4iwi`9AEy-AW%%~B6Yg$u#36OB0IkvAUVFlU_?33sA zR?;Q~iZp5xE&KMRn(2CqZjZ(3oM~%sl_7aH+FJI;nq!S^@pNNrM_YSiTdcXM{XmT` znTmPSX-_Pe428Wh?~26kjv9~Ac<#1{^@DOpy2nWA5&=$9$_Wr}{8qW=S@m<|Md@lZ14j|7uc zYC$WH0hwZ;F(4`;Kq=O2ZEWmlrWTh2h4QX zXXPa%^AeJ!7m}qHlBE}tr58f!S&n4eTYaHqYfHAVskxE-PozYHFC;|`IvMcAQi)j7 z8;Hk(iG*h&7m+OWpTwS~=G|U@tgS88+TNIsHSJ9$=_+3+Zl;3qP$))CDj7*{)XG#Z z;ZUJm{+q>cDijZT62VX~5(@{?l-=T5GBrsyNRh$+X3-2LyrFo^7c)a~f6zmpvnaOc z_2{Bk7k$F%kZcxmIH1v{rY6-Q2V~b2K$bxOg%gxr z*Bg}M5S?X71Q8!bd>HXz#D|rkFm-HEs3sImD4I|-$&lu_F8cK%@du5jNc@sLO&zJ$ zy~&GW0iT*bv#l+=TVwlb;vsJ!7z}tkF={j3hzIkgmc+XXQVfA!YcP$sw8&9!8k1_A zM$lj??u`UOiDV@3L8mx!l{$OG8>6AbpH9mu&WkaV&a%KUYLYCg65}SxA9SMg1d|?L zD4g_rgNa}`MPq1jA^GJL^ZwTwzCa`$NK0^SnU&tQ|gd)LsEX1mtI#O}%2lOry z(8n735?M`IA8P_(J=Xky6-28OU&I$Qy@8l7oleqhP2-3U;|QI-AJDGp^NN3Q)9(w? z0-NUS57t1_o;Y=WT9o>e-n8FjGj&KGxI%jQhV=3c>E#=eZ93i-G()N8L`yQ|^KG=w z>pPnECR43qIkZ`lra#n2ru2e$U*s)~8l6AsPxHUyfrAtyiZ?dHk;l@~VvN1z{_NK&&D`BQX{$wB= zN~dGK0L{Qu8vzg1f;?MO9m#UJ>XD-s$yPI_GtGQb(VBDxu*NY8_*IPGntDW(HKmGx zx>`Ur-mVW1;SUIe>lCa)N!g?zS2dtL*|8}h&z^x9QWocl!M6xVFESpH4SkU@u z#qRZZ-?d~H_o!CmtEr)uS5t##w`6O*!m{Po)NHp}Ff~W0Gk#Y6646YpCLhkEQb{wG z@OvZableQ2ViT)>;fOEzER>@p z8=B$ByBuUCwyCvc-^5B^IvWpoeKa43L$PFtvQGs@Z8#{F7|Qm4yCw0^92pG8JwabQ z7z;50h6?7fVHeeqbzpGx|YiEw}xFM5eZ zrdMKde;^r%`>5KIURnXM67zbj%^<2oF_4R?g6{LIIijZK{oLevf*#Whc>F0^E5|~9 zR=Hk}wG0eeZLBbZc+*YIO>LRR-K{PAsZ04hDSy}(p(aMnBo>cbtt)~%13Fvd2A!?( zgwECt0iETbE&|rzDgxG+Dgu)wJK4nL4XN3Rb|f0J9c>wM$WJpAm3`O~j?f(FrKJuv zcPcR4f=~c=ZxoVuW)!fz6iV`bMJUO8BN37}aw1gF_sY9G!Q_3USJ|fk)F>qNDIo1r zK-w2d@>WZP)a^sT z$t>%m#}o;$-9^)9#=NO?JWM?%6bsUgP;uw=()_N=`ufnz=54QEpE11wtD)=e7SASy zMa(p}HR7}Y44JfMONSzUYWuQv6mGaEpijzPwvwPFpPnI_mqq&uh!(Hql*TQJ@OgrA z2SNm_>JheAx#sXg`f|8(sEC~uw zH_$F(dquSG%i{F~mQU`x2-`BB6ku1-Hj8kWBBVbQV2)B7Bn`g@j*5WZYJJoyp<$JS z(6EX@1b7Z)#fX@~MY3`QlI3k(X4Rw>vI@fr6|D1Y2lQ6yV|Q#mx@CidvOp|r6WJ=H zAa|!ktZY+O$SM~rRPb1?#s!-w9J%JTnAMW3P{Cu_E-Y5CLbrpV4-Y6R#OmZ2Mc))7QAIQZM2o4e5M;Vcu)6gDpbf@ zsE{TbD}2F1dZB`?aN#;OlK3V(mSc+WGhFb~EaWF#$hTRrZx%eH-;#odasU!m%n2(K z*A~2v6f8#y_Gk^Jtgu?3($qR^P``VnMF5LuwHz#X5Sj3x;GbC7AYrekkQ1+mCsHU* zFO4#qKU9ka1BIl#ypvAUs8>$9qL{oitLwJ7T&O#*+!K~pu_1}Kf{FG)8xPVs+P{Er zrWz%%CF=K>jdV|GWx;GzO@j74P(&0$)zDrl3W!1lCKnTyl|l+?^dieBUAm%RkGd67 zQ+cA;f}WfLggw>C$V%Z_)#)h~9fLvz?^UNKhWer$6+A^36uZtcR&7lCDIk5KKp{%b zav~0+DTSmT0?CpPA?b$*$^26Yr7l>(uACKuw54q!5wNZl0c-6Q4qGcEJfu(?%r)7L z_@2~(nwt7pYg?+`65h{(+q%v3#eyN9FOg1)ZLQ)47nEfejEId+J;KVWh|rrPt-3VD za9E%H!Za`x8#1X=#rMYcr2I7zTJ^_Fx-+CFfnlFFh`n-TINjQEX{xy~7H_HvhXP4& zA{B}x;`BHumV}~p*OBr>%w!@Nk9qvzU_2Z^95pCfNLfA?9{;O_M{St*q)kDn4ZHW z)AS(D^oD$?n96XIyuN(%>X?UAQd0t_O|S< ziKoJJV{gVHrk@_CdDJugcuQ+rrY1#C=INnkgdPS5BB_YFPDaLJfq?0WMG_vLN&BLv z^2fR|<_(GGkn|YAOHbQVDv~zzlBpVciy$61<0-nCjfFymDsE5Kgv_9y9`Pk2^aRsS zuNlB5-5U7Y+LQja18shKDIrX)p^==Y+bY^KG5zrX-Qm&RM}nRuQW3|RTbgUA$9U=X zKAkjEv_FtYr~Py<8BBQmp-3_kOC)LnG(=m^WvMl_w6@pyg8^?Mod|g1^z0;=Qf(#^ zYs2)aPAvaQ~P%t&&c;PvA@a)#6_EcoQ z_}eJof$-6;dxAD==pH34mj4CseaT2D7Eh+bu~axt&n=R{q@Suc?)RsBNi!X*5&g-^ z=X+|(XC`VBX!@H;L0Tfyt!hMU(8m)YJTp;t0>1a!^TGX~-0bm((k6A*m_JSX{Nc2` zzrkJ)?I(%t0$~}iioBnpE07A(Ry=Ls1XGD{(qlbBm|nqHdoMwK7h$s7`S;$3riVh_ zFzrMq%n-d7kksv)AizIw_ zBuCd;Qy^e= zpJ_UgsO)1O1k{rLaVQM zkY02vDzyJ+Z@1AFyfyFp2_gVv-KKYxcxvj)z~CnN28t-qt=!7gcII0 z75wyTMs7)3ZTh`Snw}!6mJ<+tDZDS$o@q(8(Yp?!fG6Jbqd4mUTPW3*h>?@j@H%!k z((A9Hrm0;dg8{M2OA8gRTvS-SoCb9|%khQ+)(}N!dH+ZOYuKT)TBb3C~GzpQR*sr^DKypqF;!UuFLyi zijud%6bLHE*^}vah#5DXUWP^I>w2LET zby|`6={Ig#T+TNVDGvVpmr8mx}Z&z-e(ygm}heZVBCYlJzer|=ViWEpT1PaL& zGzH{1W0{caZ(_0~Q9zD!mI>Jy1S?#M`xIfs8e1$Qve*SHTp@=k5i1`YSRq-d#N-fa zrD2UnR>&F-txzE~xrHy79GAWHk~fl7J(39WpzfWBATxODjh>0Ct0;hEFyxCk%JH;o zRc=y?tKLTwlY87&QtxAQl)L4Y^T;8&IuSx)5}STQ^O^6X0h?$z*a!2{f zehN?S*Ne;1V#p#|426&(3L%pe5*uC99Bi$3F{Y>MwYK`Fr|Xjs6ufjxfR;zMb7)_* zDJ**#ZUTi8az;uM>JtpDonp}pDO9+5d+%XUJ{a-R({*W_0`H?j>9NIhOVdwvpjKaE z6+Owu6lIp)RG_^$+CB>hV>G&pnZp#T%IT&;CZ?_H#~5Tr4uS%hRgi_pXsg;=y#ibbtPRQ*suwrUE<3n?I- zrGPX_0c4Fr(ldc3-q^A3Vm7t3?u)gzr&_(fY%ED1FlkTGd*ge(K6$g5ZB3<{I>mxF zK-+-<`iw~+6b!`!^6gW4M+Ofvtk6UvvTP_8SrsAKL`A6JD28jn3c0`lFW5vrWYH1B zG$BH=m_$f6c?!u#E|xD7_6lCf6}O0$i(V^K(8CNTtiV&6VyvcTJz@}TkOFW?&3O?u zZBRh2w!IX>h(jT$Tl->`1=2@e)Epx8ksyVT6`F(!mttZeJymd&R>6q8R}iVn8vzkg zMMw-qO`(E4%zdPXxle=&v2tZEtf<1ITuj8GVu_(_C{*xXKFJc-$|qS?2*po1D(J~a zjot{gdRf5Y{R69bCR)1cgrtO_T!aexvTOM0Z6gbbfMr4iWFd->)kiF3m6sJN=qnSG zy@Vort$57?Cg^n(Tv+^K%7h8oF2x1Z!KOP*`D9C@*Kd#u3Rn$55Sbk>y_R5=IlcCw zUV5RBRjmTa=i*kVZ~?rc*a=jyVO59FFA&)v1(NMhgk-n1LIoRgF({b4i?l*;mzdR& z1e5K?3dvF@CLh{cOcp3H`6__L3ckp;B$#}yzzWHN7nY~Lnx%KyMT@eox7wW*n*Izz zpY$qRUnnx!4=hU)xhTZaLvEx z6Jf;`@`AyLu8^27 zQVJnMs>jlmaEan%-xbD?3F?Z4D^&$ir!Ux0RYeS0EOc#Q1sN-JXkv1e}vSLI?R;38NkNY8c7br?bzD*=TXze0Y$d7zc z&x$RS8~Ot2VaA~l`mbOU*A}jjFJ1{Nvq6rx(VjzXtLb zeO@BmK_6A8Pj^MK^zkkFQi#2&U?P)B>=CygF@MbKp^q}q>(8nG)7qKrNRkv`xC8PmEqyxi}_||zS!?$r@Lw#i7;sxCeh-f>Y})GM2VMGMVKD@U`997 zZ);!@v+;qSRhgeg`--wTxsSR;m=iWxDTG+f;)<78xr-l7c_<-+TpMxbBSlaKa2Sz% zGlFng`EV^Uf^n8-gM>_Sbv;Lc%6@zgvdUF#Xt(p-n3d8zW5*1*c}v)+Qs z*v7(jan0%3$>EY-B@LJ1@ZZGYvl54IEphlnIWrUjTic3{1tx7x$3#n3GyCyAIP?<^ zO$UdU7g%P7FszqqV4Kd$CksR8)lW8xFbptKacDOZFYQ|1 zvQs%;v9bF6@zZB7R!`r5|M`0_R?j~D_&NL(YoIbKI62p{bDkG!5+oVSdHUKFA79@- zZ(l9mnD?7WgLB7nfCObS zK5Tq*C;p6l+>)(AIn(%=nlW5dSNn>|3-WlUjW>mhrH(g#hR!!g!)8XmB1lDl{sazsdMJ#cD=^Glc4z#ZMJ zI-mk>FRr%dO;WXK3V=@mYEm^edYDN?y~#+EEhc?z9nl0S6Bl9C=R9GQ1Z z7jq@wyL5(0FE{FRn-)5-dG-NnOL*yzmb02I(LXI<GA zEKQwP^KO~y+6`4@-*slpyJewkPrO@BIuB;(TB&t4i_-Z%fxo4rYe!UiD5Ux8emsQE zg$hMbBP_mzis$5?R#2j=FMQI~V+j*Jsp^kUC;+EPELJEARutx=vj|Biw6%6^@v-tG zp@KN1WU+KvjWD6Eu!FEqx|+@>wYtNhlXV=}sQT9B+@m>kX-__d=F>S3KB}SP2%Sw% z-f8qup`zrN&-^K55h|M#i54a_7kl^Sqf2;XF?Tmz-VvV=ORe|t(Ulu<_*3j0ggs>U zIlG&$^7Tb`S#pYR-CfxOrLzE;3CoMdv8XUKm7+jMTA=`8#L0HD7D?Aw2@?iO68hO( zqA(b=8fNzBi#IbuUl16Y`RZWc-FE<6;PnCYIAAPEPK|Nj~k*Goij0keBaO zJpFiJf*42(-+d)5qz2NpKJYi*YbCJ~gmT+c`xRMRX3FY#x5$B!&kWp%2#j6_0}S56 z+a9hRPece=y0(>R%maTTM;nb+v2Z#7dACB^Hs(rO8-JtQwlZxajLM0Yh4B`+l`e0>w2egtb{0nAJP56F12fTf zL?}d8b3)gO9yn!e(1ePLOjQ_sbkKNyK+D9Y%3z5O`T#CtjKJ829c9opoLO5mc4(ta zd(*5#22M0{CL5$-aL>%tq6ZE%GZ|wZcC2L98UwSLnTjz3?c6XIOHXAwv>HdGGso6b zknr=^jgDvo&>kj7<8@OQRcu!AJPfoDO+=xG=@;vW#bokb1K1Ihlv6m!lPhyKU@`4n zU$e7%#1KXCcaCN7;pU6+=JEKY2edhvLn`pR-B9IenRIe+|F9Uv=^Q@2zJ7ZSRg2{i z_RS;EyMUCMRjL9KVfz&cw=?z*jKr~R8%(LH6dcb(I(>B?8l$|Wy31TlRpdmUtfqo# zUJWg1Y@HY~?3zjis~@#{jj!yz^s}z2XwR&~kO~O3V^&3{(?4F-d+CGlo^^GdWmQ`b zQ-NXi(JE3~sbgh0+9N%Ew@B&~EZf6pJRqUc7Jxn0p8H)67RY_v{kusRq>+dx>@$Tt8s3v|Fi5B;W$Q zozv~z^%`68i*Y?9JOtp3u2tuG%lcW;1IgWGm5wsZe-_Yk@o;L7UGIX5J(#cXsTb_c7j*~2sE=;{@xSgTi!18)~lS>}X8=aKcik^QOS zVs9^IT?}BLwY9wIQeKPz0dnXtOH2Ez$hEvDfx6ka-t^;k3oV)+_K;9CwOu~|f`AER z!$t>*7>K}1h)VK(M2%7bHA;oj$P{a!E&)oY3u+~s;GYWdco>rn^6Lq30*3re?RU=A zu+EJWU=?Xr^Z+=p*y?NZFtX1`ND*`cgbd)(TWC*!IIRZk@$(M+f?{{9Q^dO8Tzw9v z{I&rd<8pNeqPoNZaBeKd3OLCUV{RQ?hxq}I*Y}&dt6M5-+9b&%eUomIG|09U$OvE` ztS;|vbz=)0y8=VTZmO$cav0tTEvt#*-rije7sJF`JAfdPon3Ff;N|5lw#Gcy`n+G$ zm0_5GJR-*oR^AkmPIY&AEtafOe1IodMNW&Bf{pN{rRhX)-+`vU=G*PoH>5{f*y*3O zj)9Is4%<~Y$eVb&6 zd^Y@cK>eo{8~~R)0Peb?JW=^>(I1jz5U|^ecFolVlIKwNMLux@11o+woRvweVVvu>2COfrX$X)#uDB5H5LJ7i~7S@Cv!yVpnxivJCV0 zMn1h7Oi#tO=<;9Ua5YmeWiK@RtW%! z0ymq~4wHhVTQ6fkT?STCB;yF+a)rruhxh`3g67?YWtm3|ShifWF>+H0(g`$kOt%Zd{R#byfg&^46H; zEYYWgwP5fnAv7Pb?bwgHqZb?zl-VLE`<(Gux}@TgjsnhcDwzx`6B!1=!yB3jHF&5K zt07g2=(wl)-@R4>$QPg=SY6&0nC$Lo{l#{D(T@ewq4CQ!_b)4sLooF5Sgk>Fl6z1f zC99wU%dLU6A5dzK=z?EIAluU&~ZVsqu1&1J7 zLt^)^H!_$@D}*rIK>WbysjC9pem9j>r>&=BZ-%F>uwHL_E>T)-kktrG0jbTIjP>Op zcJ@V+wsG0@h<2$B@KSTT zpnh_TLfe9AR+pf_3VFlD=SEd|97)Y!$ll0-?~QeI_#mijmbb5 z(oWY+N3fegZOOW%ZC#>@=ZWOsE&&~i?Q^P?iR%a90Vf~k8l zFk~%*=BWIR%ivxa(h9J;g1x;KeVOT@FN59xA^;hunDIefv$Vrr8rOxjAPRTYjs(~n zGseZXw=Gvu-DAc;*{4joQefK}^txRZNcH&ZxK1xE9G#e7>@6*11sXL6-{ zd3*XtXXg9qPgyzi3`3b{)ALji?htjS_{B!ZRJXZ1DVkaHiY)X-DLdyK=qR{QmBmq`-6I8o5%?d};^6jW)W-nvm3T z3Oh+JwyXI}ws1}XNCySqH9c2x7|#)kRr1t+MG{S!pc4RjEVl4laLJ2Jpbf=aVQxfX zoS(nrXL}DpUrQJmj;@=58kzL9j&068FG~@AK+P~cr@HaI@h}!J?wUms_Rm38(~it} z+=GOJ1}N(r!=s_Sra@2SaU3ZMv>S|RLD=ISFGQ*8F8`l=4zQxw=$5+_=$Tv(=f(pS zXI3n0mG$GIS2hhIjA>+*aqC$xfNVj~?~CRG<6*iN7eNi^F_g_^z1&tbM?{q={KTJK z_svR|0EErRbbCOOxeCbJDH$Dzkvq}C@Oy7l2eXsi{%!y{YbSY8u$n^! zYvo5>SM(ET2`+O#E#V*W|W$a01cFxpY$2E1IB`)=1q z+uPaMcW_AvyQcT-6T9~}Jpc2 Date: Thu, 7 Mar 2024 17:56:40 +0000 Subject: [PATCH 27/84] adds api to obtain the parent node in the turbine retransmit tree (#115) Following commits will use this api to check retransmitter's signature on incoming shreds. --- Cargo.lock | 1 + turbine/Cargo.toml | 1 + turbine/src/cluster_nodes.rs | 199 +++++++++++++++++++++++++++++++---- 3 files changed, 179 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index afdb8b0a306578..0973d94da30c29 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7501,6 +7501,7 @@ dependencies = [ "solana-runtime", "solana-sdk", "solana-streamer", + "test-case", "thiserror", "tokio", ] diff --git a/turbine/Cargo.toml b/turbine/Cargo.toml index bedd870952af99..2d2b0a79574d27 100644 --- a/turbine/Cargo.toml +++ b/turbine/Cargo.toml @@ -43,6 +43,7 @@ tokio = { workspace = true } assert_matches = { workspace = true } solana-logger = { workspace = true } solana-runtime = { workspace = true, features = ["dev-context-only-utils"] } +test-case = { workspace = true } [[bench]] name = "cluster_info" diff --git a/turbine/src/cluster_nodes.rs b/turbine/src/cluster_nodes.rs index 0c55cb41e56472..6036907cd7dc5c 100644 --- a/turbine/src/cluster_nodes.rs +++ b/turbine/src/cluster_nodes.rs @@ -152,8 +152,7 @@ impl ClusterNodes { } pub(crate) fn get_broadcast_peer(&self, shred: &ShredId) -> Option<&ContactInfo> { - let shred_seed = shred.seed(&self.pubkey); - let mut rng = ChaChaRng::from_seed(shred_seed); + let mut rng = get_seeded_rng(/*leader:*/ &self.pubkey, shred); let index = self.weighted_shuffle.first(&mut rng)?; self.nodes[index].contact_info() } @@ -187,7 +186,6 @@ impl ClusterNodes { shred: &ShredId, fanout: usize, ) -> Result { - let shred_seed = shred.seed(slot_leader); let mut weighted_shuffle = self.weighted_shuffle.clone(); // Exclude slot leader from list of nodes. if slot_leader == &self.pubkey { @@ -200,7 +198,7 @@ impl ClusterNodes { weighted_shuffle.remove_index(*index); } let mut addrs = HashMap::::with_capacity(self.nodes.len()); - let mut rng = ChaChaRng::from_seed(shred_seed); + let mut rng = get_seeded_rng(slot_leader, shred); let protocol = get_broadcast_protocol(shred); let nodes: Vec<_> = weighted_shuffle .shuffle(&mut rng) @@ -233,6 +231,43 @@ impl ClusterNodes { addrs, }) } + + // Returns the parent node in the turbine broadcast tree. + // Returns None if the node is the root of the tree or if it is not staked. + #[allow(unused)] + fn get_retransmit_parent( + &self, + leader: &Pubkey, + shred: &ShredId, + fanout: usize, + ) -> Result, Error> { + // Exclude slot leader from list of nodes. + if leader == &self.pubkey { + return Err(Error::Loopback { + leader: *leader, + shred: *shred, + }); + } + // Unstaked nodes' position in the turbine tree is not deterministic + // and depends on gossip propagation of contact-infos. Therefore, if + // this node is not staked return None. + if self.nodes[self.index[&self.pubkey]].stake == 0 { + return Ok(None); + } + let mut weighted_shuffle = self.weighted_shuffle.clone(); + if let Some(index) = self.index.get(leader).copied() { + weighted_shuffle.remove_index(index); + } + let mut rng = get_seeded_rng(leader, shred); + // Only need shuffled nodes until this node itself. + let nodes: Vec<_> = weighted_shuffle + .shuffle(&mut rng) + .map(|index| &self.nodes[index]) + .take_while(|node| node.pubkey() != self.pubkey) + .collect(); + let parent = get_retransmit_parent(fanout, nodes.len(), &nodes); + Ok(parent.map(Node::pubkey)) + } } pub fn new_cluster_nodes( @@ -296,6 +331,11 @@ fn get_nodes(cluster_info: &ClusterInfo, stakes: &HashMap) -> Vec ChaChaRng { + let seed = shred.seed(leader); + ChaChaRng::from_seed(seed) +} + // root : [0] // 1st layer: [1, 2, ..., fanout] // 2nd layer: [[fanout + 1, ..., fanout * 2], @@ -327,6 +367,21 @@ fn get_retransmit_peers( .copied() } +// Returns the parent node in the turbine broadcast tree. +// Returns None if the node is the root of the tree. +fn get_retransmit_parent( + fanout: usize, + index: usize, // Local node's index within the nodes slice. + nodes: &[T], +) -> Option { + // Node's index within its neighborhood. + let offset = index.saturating_sub(1) % fanout; + let index = index.checked_sub(1)? / fanout; + let index = index - index.saturating_sub(1) % fanout; + let index = if index == 0 { index } else { index + offset }; + nodes.get(index).copied() +} + impl ClusterNodesCache { pub fn new( // Capacity of underlying LRU-cache in terms of number of epochs. @@ -516,7 +571,11 @@ pub fn check_feature_activation(feature: &Pubkey, shred_slot: Slot, root_bank: & #[cfg(test)] mod tests { - use super::*; + use { + super::*, + std::{fmt::Debug, hash::Hash}, + test_case::test_case, + }; #[test] fn test_cluster_nodes_retransmit() { @@ -589,10 +648,42 @@ mod tests { } } + // Checks (1) computed retransmit children against expected children and + // (2) computed parent of each child against the expected parent. + fn check_retransmit_nodes(fanout: usize, nodes: &[T], peers: Vec>) + where + T: Copy + Eq + PartialEq + Debug + Hash, + { + // Map node identities to their index within the shuffled tree. + let index: HashMap<_, _> = nodes + .iter() + .copied() + .enumerate() + .map(|(k, node)| (node, k)) + .collect(); + let offset = peers.len(); + // Root node's parent is None. + assert_eq!(get_retransmit_parent(fanout, /*index:*/ 0, nodes), None); + for (k, peers) in peers.into_iter().enumerate() { + assert_eq!( + get_retransmit_peers(fanout, k, nodes).collect::>(), + peers + ); + let parent = Some(nodes[k]); + for peer in peers { + assert_eq!(get_retransmit_parent(fanout, index[&peer], nodes), parent); + } + } + // Remaining nodes have no children. + for k in offset..=nodes.len() { + assert_eq!(get_retransmit_peers(fanout, k, nodes).next(), None); + } + } + #[test] - fn test_get_retransmit_peers() { + fn test_get_retransmit_nodes() { // fanout 2 - let index = vec![ + let nodes = [ 7, // root 6, 10, // 1st layer // 2nd layer @@ -620,16 +711,9 @@ mod tests { vec![16, 9], vec![8], ]; - for (k, peers) in peers.into_iter().enumerate() { - let retransmit_peers = get_retransmit_peers(/*fanout:*/ 2, k, &index); - assert_eq!(retransmit_peers.collect::>(), peers); - } - for k in 10..=index.len() { - let mut retransmit_peers = get_retransmit_peers(/*fanout:*/ 2, k, &index); - assert_eq!(retransmit_peers.next(), None); - } + check_retransmit_nodes(/*fanout:*/ 2, &nodes, peers); // fanout 3 - let index = vec![ + let nodes = [ 19, // root 14, 15, 28, // 1st layer // 2nd layer @@ -661,13 +745,84 @@ mod tests { vec![24, 32], vec![34], ]; - for (k, peers) in peers.into_iter().enumerate() { - let retransmit_peers = get_retransmit_peers(/*fanout:*/ 3, k, &index); - assert_eq!(retransmit_peers.collect::>(), peers); + check_retransmit_nodes(/*fanout:*/ 3, &nodes, peers); + let nodes = [ + 5, // root + 34, 52, 8, // 1st layer + // 2nd layar + 44, 18, 2, // 1st neigborhood + 42, 47, 46, // 2nd + 11, 26, 28, // 3rd + // 3rd layer + 53, 23, 37, // 1st neighborhood + 40, 13, 7, // 2nd + 50, 35, 22, // 3rd + 3, 27, 31, // 4th + 10, 48, 15, // 5th + 19, 6, 30, // 6th + 36, 45, 1, // 7th + 38, 12, 17, // 8th + 4, 32, 16, // 9th + // 4th layer + 41, 49, 24, // 1st neighborhood + 14, 9, 0, // 2nd + 29, 21, 39, // 3rd + 43, 51, 33, // 4th + 25, 20, // 5th + ]; + let peers = vec![ + vec![34, 52, 8], + vec![44, 42, 11], + vec![18, 47, 26], + vec![2, 46, 28], + vec![53, 40, 50], + vec![23, 13, 35], + vec![37, 7, 22], + vec![3, 10, 19], + vec![27, 48, 6], + vec![31, 15, 30], + vec![36, 38, 4], + vec![45, 12, 32], + vec![1, 17, 16], + vec![41, 14, 29], + vec![49, 9, 21], + vec![24, 0, 39], + vec![43, 25], + vec![51, 20], + vec![33], + ]; + check_retransmit_nodes(/*fanout:*/ 3, &nodes, peers); + } + + #[test_case(2, 1_347)] + #[test_case(3, 1_359)] + #[test_case(4, 4_296)] + #[test_case(5, 3_925)] + #[test_case(6, 8_778)] + #[test_case(7, 9_879)] + fn test_get_retransmit_nodes_round_trip(fanout: usize, size: usize) { + let mut rng = rand::thread_rng(); + let mut nodes: Vec<_> = (0..size).collect(); + nodes.shuffle(&mut rng); + // Map node identities to their index within the shuffled tree. + let index: HashMap<_, _> = nodes + .iter() + .copied() + .enumerate() + .map(|(k, node)| (node, k)) + .collect(); + // Root node's parent is None. + assert_eq!(get_retransmit_parent(fanout, /*index:*/ 0, &nodes), None); + for k in 1..size { + let parent = get_retransmit_parent(fanout, k, &nodes).unwrap(); + let mut peers = get_retransmit_peers(fanout, index[&parent], &nodes); + assert_eq!(peers.find(|&peer| peer == nodes[k]), Some(nodes[k])); } - for k in 13..=index.len() { - let mut retransmit_peers = get_retransmit_peers(/*fanout:*/ 3, k, &index); - assert_eq!(retransmit_peers.next(), None); + for k in 0..size { + let parent = Some(nodes[k]); + for peer in get_retransmit_peers(fanout, k, &nodes) { + assert_eq!(get_retransmit_parent(fanout, index[&peer], &nodes), parent); + } } } } From ba43f74dcf5d1413d8eaca3de094c9d3aee925fc Mon Sep 17 00:00:00 2001 From: Dmitri Makarov Date: Thu, 7 Mar 2024 13:16:16 -0500 Subject: [PATCH 28/84] [SVM] Move RuntimeConfig to program-runtime (#96) RuntimeConfig doesn't use anything SVM specific and logically belongs in program runtime rather than SVM. This change moves the definition of RuntimeConfig struct from the SVM crate to program-runtime and adjusts `use` statements accordingly. --- Cargo.lock | 4 +--- core/src/validator.rs | 2 +- core/tests/epoch_accounts_hash.rs | 2 +- core/tests/snapshots.rs | 2 +- ledger-tool/Cargo.toml | 1 - ledger-tool/src/args.rs | 2 +- ledger/src/blockstore_processor.rs | 12 ++++++------ program-runtime/src/lib.rs | 1 + program-runtime/src/runtime_config.rs | 17 +++++++++++++++++ program-test/src/lib.rs | 4 ++-- programs/sbf/Cargo.lock | 3 +-- runtime/src/bank.rs | 2 +- runtime/src/bank/serde_snapshot.rs | 2 +- runtime/src/serde_snapshot.rs | 2 +- runtime/src/snapshot_bank_utils.rs | 2 +- svm/src/lib.rs | 1 - svm/src/runtime_config.rs | 9 --------- svm/src/transaction_processor.rs | 2 +- test-validator/Cargo.toml | 1 - test-validator/src/lib.rs | 3 +-- validator/Cargo.toml | 2 +- validator/src/main.rs | 2 +- 22 files changed, 40 insertions(+), 38 deletions(-) create mode 100644 program-runtime/src/runtime_config.rs delete mode 100644 svm/src/runtime_config.rs diff --git a/Cargo.lock b/Cargo.lock index 0973d94da30c29..85641aff1b22d6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -177,7 +177,6 @@ dependencies = [ "solana-stake-program", "solana-storage-bigtable", "solana-streamer", - "solana-svm", "solana-transaction-status", "solana-unified-scheduler-pool", "solana-version", @@ -233,6 +232,7 @@ dependencies = [ "solana-net-utils", "solana-perf", "solana-poh", + "solana-program-runtime", "solana-rpc", "solana-rpc-client", "solana-rpc-client-api", @@ -241,7 +241,6 @@ dependencies = [ "solana-send-transaction-service", "solana-storage-bigtable", "solana-streamer", - "solana-svm", "solana-test-validator", "solana-tpu-client", "solana-unified-scheduler-pool", @@ -7345,7 +7344,6 @@ dependencies = [ "solana-runtime", "solana-sdk", "solana-streamer", - "solana-svm", "solana-tpu-client", "tokio", ] diff --git a/core/src/validator.rs b/core/src/validator.rs index 196dad5f25d17a..a1c8293f86cb3a 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -74,6 +74,7 @@ use { poh_recorder::PohRecorder, poh_service::{self, PohService}, }, + solana_program_runtime::runtime_config::RuntimeConfig, solana_rpc::{ max_slots::MaxSlots, optimistically_confirmed_bank_tracker::{ @@ -116,7 +117,6 @@ use { }, solana_send_transaction_service::send_transaction_service, solana_streamer::{socket::SocketAddrSpace, streamer::StakedNodes}, - solana_svm::runtime_config::RuntimeConfig, solana_turbine::{self, broadcast_stage::BroadcastStageType}, solana_unified_scheduler_pool::DefaultSchedulerPool, solana_vote_program::vote_state, diff --git a/core/tests/epoch_accounts_hash.rs b/core/tests/epoch_accounts_hash.rs index 62e31f0a88b766..25e97689923bb0 100755 --- a/core/tests/epoch_accounts_hash.rs +++ b/core/tests/epoch_accounts_hash.rs @@ -16,6 +16,7 @@ use { snapshot_packager_service::SnapshotPackagerService, }, solana_gossip::{cluster_info::ClusterInfo, contact_info::ContactInfo}, + solana_program_runtime::runtime_config::RuntimeConfig, solana_runtime::{ accounts_background_service::{ AbsRequestHandlers, AbsRequestSender, AccountsBackgroundService, DroppedSlotsReceiver, @@ -39,7 +40,6 @@ use { timing::timestamp, }, solana_streamer::socket::SocketAddrSpace, - solana_svm::runtime_config::RuntimeConfig, std::{ mem::ManuallyDrop, sync::{ diff --git a/core/tests/snapshots.rs b/core/tests/snapshots.rs index e67c942f07ab0b..1607ebd3fa2094 100644 --- a/core/tests/snapshots.rs +++ b/core/tests/snapshots.rs @@ -18,6 +18,7 @@ use { snapshot_packager_service::SnapshotPackagerService, }, solana_gossip::{cluster_info::ClusterInfo, contact_info::ContactInfo}, + solana_program_runtime::runtime_config::RuntimeConfig, solana_runtime::{ accounts_background_service::{ AbsRequestHandlers, AbsRequestSender, AccountsBackgroundService, @@ -50,7 +51,6 @@ use { timing::timestamp, }, solana_streamer::socket::SocketAddrSpace, - solana_svm::runtime_config::RuntimeConfig, std::{ collections::HashSet, fs, diff --git a/ledger-tool/Cargo.toml b/ledger-tool/Cargo.toml index cb87a0e16f4a36..88bb3d3ff83b72 100644 --- a/ledger-tool/Cargo.toml +++ b/ledger-tool/Cargo.toml @@ -44,7 +44,6 @@ solana-sdk = { workspace = true } solana-stake-program = { workspace = true } solana-storage-bigtable = { workspace = true } solana-streamer = { workspace = true } -solana-svm = { workspace = true } solana-transaction-status = { workspace = true } solana-unified-scheduler-pool = { workspace = true } solana-version = { workspace = true } diff --git a/ledger-tool/src/args.rs b/ledger-tool/src/args.rs index 80ea6f9715bf35..1f0c06966deffc 100644 --- a/ledger-tool/src/args.rs +++ b/ledger-tool/src/args.rs @@ -12,8 +12,8 @@ use { blockstore_processor::ProcessOptions, use_snapshot_archives_at_startup::{self, UseSnapshotArchivesAtStartup}, }, + solana_program_runtime::runtime_config::RuntimeConfig, solana_sdk::clock::Slot, - solana_svm::runtime_config::RuntimeConfig, std::{ collections::HashSet, path::{Path, PathBuf}, diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs index 2e172870d6e5f7..c999eab1a56fd4 100644 --- a/ledger/src/blockstore_processor.rs +++ b/ledger/src/blockstore_processor.rs @@ -27,7 +27,10 @@ use { }, solana_measure::{measure, measure::Measure}, solana_metrics::datapoint_error, - solana_program_runtime::timings::{ExecuteTimingType, ExecuteTimings, ThreadExecuteTimings}, + solana_program_runtime::{ + runtime_config::RuntimeConfig, + timings::{ExecuteTimingType, ExecuteTimings, ThreadExecuteTimings}, + }, solana_rayon_threadlimit::{get_max_thread_count, get_thread_count}, solana_runtime::{ accounts_background_service::{AbsRequestSender, SnapshotRequestKind}, @@ -54,11 +57,8 @@ use { VersionedTransaction, }, }, - solana_svm::{ - runtime_config::RuntimeConfig, - transaction_results::{ - TransactionExecutionDetails, TransactionExecutionResult, TransactionResults, - }, + solana_svm::transaction_results::{ + TransactionExecutionDetails, TransactionExecutionResult, TransactionResults, }, solana_transaction_status::token_balances::TransactionTokenBalancesSet, solana_vote::{vote_account::VoteAccountsHashMap, vote_sender_types::ReplayVoteSender}, diff --git a/program-runtime/src/lib.rs b/program-runtime/src/lib.rs index 5797626a00a756..079f214fa236f0 100644 --- a/program-runtime/src/lib.rs +++ b/program-runtime/src/lib.rs @@ -17,6 +17,7 @@ pub mod loaded_programs; pub mod log_collector; pub mod message_processor; pub mod prioritization_fee; +pub mod runtime_config; pub mod stable_log; pub mod sysvar_cache; pub mod timings; diff --git a/program-runtime/src/runtime_config.rs b/program-runtime/src/runtime_config.rs new file mode 100644 index 00000000000000..da6fc1dfba4db1 --- /dev/null +++ b/program-runtime/src/runtime_config.rs @@ -0,0 +1,17 @@ +use crate::compute_budget::ComputeBudget; + +#[cfg(RUSTC_WITH_SPECIALIZATION)] +impl ::solana_frozen_abi::abi_example::AbiExample for RuntimeConfig { + fn example() -> Self { + // RuntimeConfig is not Serialize so just rely on Default. + RuntimeConfig::default() + } +} + +/// Encapsulates flags that can be used to tweak the runtime behavior. +#[derive(Debug, Default, Clone)] +pub struct RuntimeConfig { + pub compute_budget: Option, + pub log_messages_bytes_limit: Option, + pub transaction_account_lock_limit: Option, +} diff --git a/program-test/src/lib.rs b/program-test/src/lib.rs index 20b9f5806e29c3..669cb15a595afb 100644 --- a/program-test/src/lib.rs +++ b/program-test/src/lib.rs @@ -17,7 +17,8 @@ use { solana_bpf_loader_program::serialization::serialize_parameters, solana_program_runtime::{ compute_budget::ComputeBudget, ic_msg, invoke_context::BuiltinFunctionWithContext, - loaded_programs::LoadedProgram, stable_log, timings::ExecuteTimings, + loaded_programs::LoadedProgram, runtime_config::RuntimeConfig, stable_log, + timings::ExecuteTimings, }, solana_runtime::{ accounts_background_service::{AbsRequestSender, SnapshotRequestKind}, @@ -45,7 +46,6 @@ use { stable_layout::stable_instruction::StableInstruction, sysvar::{Sysvar, SysvarId}, }, - solana_svm::runtime_config::RuntimeConfig, solana_vote_program::vote_state::{self, VoteState, VoteStateVersions}, std::{ cell::RefCell, diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index 11a4bcab04d7c0..b72b4110e336ad 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -117,6 +117,7 @@ dependencies = [ "solana-net-utils", "solana-perf", "solana-poh", + "solana-program-runtime", "solana-rpc", "solana-rpc-client", "solana-rpc-client-api", @@ -125,7 +126,6 @@ dependencies = [ "solana-send-transaction-service", "solana-storage-bigtable", "solana-streamer", - "solana-svm", "solana-test-validator", "solana-tpu-client", "solana-unified-scheduler-pool", @@ -6393,7 +6393,6 @@ dependencies = [ "solana-runtime", "solana-sdk", "solana-streamer", - "solana-svm", "solana-tpu-client", "tokio", ] diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index 39df91c382feff..ee04f20787cb9a 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -99,6 +99,7 @@ use { compute_budget_processor::process_compute_budget_instructions, invoke_context::BuiltinFunctionWithContext, loaded_programs::{LoadedProgram, LoadedProgramType, LoadedPrograms}, + runtime_config::RuntimeConfig, timings::{ExecuteTimingType, ExecuteTimings}, }, solana_sdk::{ @@ -163,7 +164,6 @@ use { solana_svm::{ account_loader::{TransactionCheckResult, TransactionLoadResult}, account_overrides::AccountOverrides, - runtime_config::RuntimeConfig, transaction_error_metrics::TransactionErrorMetrics, transaction_processor::{ TransactionBatchProcessor, TransactionLogMessages, TransactionProcessingCallback, diff --git a/runtime/src/bank/serde_snapshot.rs b/runtime/src/bank/serde_snapshot.rs index 8b78efbcf3e11a..f5b1653e8d6311 100644 --- a/runtime/src/bank/serde_snapshot.rs +++ b/runtime/src/bank/serde_snapshot.rs @@ -31,6 +31,7 @@ mod tests { epoch_accounts_hash::EpochAccountsHash, stake_rewards::StakeReward, }, + solana_program_runtime::runtime_config::RuntimeConfig, solana_sdk::{ epoch_schedule::EpochSchedule, genesis_config::create_genesis_config, @@ -38,7 +39,6 @@ mod tests { pubkey::Pubkey, signature::{Keypair, Signer}, }, - solana_svm::runtime_config::RuntimeConfig, std::{ io::{Cursor, Read, Write}, num::NonZeroUsize, diff --git a/runtime/src/serde_snapshot.rs b/runtime/src/serde_snapshot.rs index 4b066976d49048..8e678044e23670 100644 --- a/runtime/src/serde_snapshot.rs +++ b/runtime/src/serde_snapshot.rs @@ -27,6 +27,7 @@ use { epoch_accounts_hash::EpochAccountsHash, }, solana_measure::measure::Measure, + solana_program_runtime::runtime_config::RuntimeConfig, solana_sdk::{ clock::{Epoch, Slot, UnixTimestamp}, deserialize_utils::default_on_eof, @@ -39,7 +40,6 @@ use { pubkey::Pubkey, rent_collector::RentCollector, }, - solana_svm::runtime_config::RuntimeConfig, std::{ collections::{HashMap, HashSet}, io::{self, BufReader, BufWriter, Read, Write}, diff --git a/runtime/src/snapshot_bank_utils.rs b/runtime/src/snapshot_bank_utils.rs index 721021142f9258..ab3a76fc80945a 100644 --- a/runtime/src/snapshot_bank_utils.rs +++ b/runtime/src/snapshot_bank_utils.rs @@ -37,6 +37,7 @@ use { utils::delete_contents_of_path, }, solana_measure::{measure, measure::Measure}, + solana_program_runtime::runtime_config::RuntimeConfig, solana_sdk::{ clock::Slot, feature_set, @@ -45,7 +46,6 @@ use { pubkey::Pubkey, slot_history::{Check, SlotHistory}, }, - solana_svm::runtime_config::RuntimeConfig, std::{ collections::HashSet, fs, diff --git a/svm/src/lib.rs b/svm/src/lib.rs index 5505e34bea9d61..d0f679a15c448b 100644 --- a/svm/src/lib.rs +++ b/svm/src/lib.rs @@ -4,7 +4,6 @@ pub mod account_loader; pub mod account_overrides; pub mod account_rent_state; -pub mod runtime_config; pub mod transaction_account_state_info; pub mod transaction_error_metrics; pub mod transaction_processor; diff --git a/svm/src/runtime_config.rs b/svm/src/runtime_config.rs deleted file mode 100644 index 2439dd85c2e46f..00000000000000 --- a/svm/src/runtime_config.rs +++ /dev/null @@ -1,9 +0,0 @@ -use solana_program_runtime::compute_budget::ComputeBudget; - -/// Encapsulates flags that can be used to tweak the runtime behavior. -#[derive(AbiExample, Debug, Default, Clone)] -pub struct RuntimeConfig { - pub compute_budget: Option, - pub log_messages_bytes_limit: Option, - pub transaction_account_lock_limit: Option, -} diff --git a/svm/src/transaction_processor.rs b/svm/src/transaction_processor.rs index fa417850699372..e44b426df96b0d 100644 --- a/svm/src/transaction_processor.rs +++ b/svm/src/transaction_processor.rs @@ -4,7 +4,6 @@ use { load_accounts, LoadedTransaction, TransactionCheckResult, TransactionLoadResult, }, account_overrides::AccountOverrides, - runtime_config::RuntimeConfig, transaction_account_state_info::TransactionAccountStateInfo, transaction_error_metrics::TransactionErrorMetrics, transaction_results::{ @@ -23,6 +22,7 @@ use { }, log_collector::LogCollector, message_processor::MessageProcessor, + runtime_config::RuntimeConfig, sysvar_cache::SysvarCache, timings::{ExecuteDetailsTimings, ExecuteTimingType, ExecuteTimings}, }, diff --git a/test-validator/Cargo.toml b/test-validator/Cargo.toml index 2bc8deb5fc200e..60f299d01e58a0 100644 --- a/test-validator/Cargo.toml +++ b/test-validator/Cargo.toml @@ -32,7 +32,6 @@ solana-rpc-client = { workspace = true } solana-runtime = { workspace = true } solana-sdk = { workspace = true } solana-streamer = { workspace = true } -solana-svm = { workspace = true } solana-tpu-client = { workspace = true } tokio = { workspace = true, features = ["full"] } diff --git a/test-validator/src/lib.rs b/test-validator/src/lib.rs index c658b53305bf74..f551cb97820d06 100644 --- a/test-validator/src/lib.rs +++ b/test-validator/src/lib.rs @@ -29,7 +29,7 @@ use { create_new_tmp_ledger, }, solana_net_utils::PortRange, - solana_program_runtime::compute_budget::ComputeBudget, + solana_program_runtime::{compute_budget::ComputeBudget, runtime_config::RuntimeConfig}, solana_rpc::{rpc::JsonRpcConfig, rpc_pubsub_service::PubSubConfig}, solana_rpc_client::{nonblocking, rpc_client::RpcClient}, solana_runtime::{ @@ -54,7 +54,6 @@ use { signature::{read_keypair_file, write_keypair_file, Keypair, Signer}, }, solana_streamer::socket::SocketAddrSpace, - solana_svm::runtime_config::RuntimeConfig, solana_tpu_client::tpu_client::{ DEFAULT_TPU_CONNECTION_POOL_SIZE, DEFAULT_TPU_ENABLE_UDP, DEFAULT_TPU_USE_QUIC, }, diff --git a/validator/Cargo.toml b/validator/Cargo.toml index 844a2bca9aa97f..74742c90faa29d 100644 --- a/validator/Cargo.toml +++ b/validator/Cargo.toml @@ -49,6 +49,7 @@ solana-metrics = { workspace = true } solana-net-utils = { workspace = true } solana-perf = { workspace = true } solana-poh = { workspace = true } +solana-program-runtime = { workspace = true } solana-rpc = { workspace = true } solana-rpc-client = { workspace = true } solana-rpc-client-api = { workspace = true } @@ -57,7 +58,6 @@ solana-sdk = { workspace = true } solana-send-transaction-service = { workspace = true } solana-storage-bigtable = { workspace = true } solana-streamer = { workspace = true } -solana-svm = { workspace = true } solana-test-validator = { workspace = true } solana-tpu-client = { workspace = true } solana-unified-scheduler-pool = { workspace = true } diff --git a/validator/src/main.rs b/validator/src/main.rs index 9741a2aecd68a8..b00eabfef9a7b0 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -47,6 +47,7 @@ use { }, solana_perf::recycler::enable_recycler_warming, solana_poh::poh_service, + solana_program_runtime::runtime_config::RuntimeConfig, solana_rpc::{ rpc::{JsonRpcConfig, RpcBigtableConfig}, rpc_pubsub_service::PubSubConfig, @@ -67,7 +68,6 @@ use { }, solana_send_transaction_service::send_transaction_service, solana_streamer::socket::SocketAddrSpace, - solana_svm::runtime_config::RuntimeConfig, solana_tpu_client::tpu_client::DEFAULT_TPU_ENABLE_UDP, std::{ collections::{HashSet, VecDeque}, From 940bd30ac999dd93ecde92c7982369b12cbfa065 Mon Sep 17 00:00:00 2001 From: bji Date: Thu, 7 Mar 2024 11:52:22 -0800 Subject: [PATCH 29/84] Update maximum credits awarded per vote from 8 to 16 (#127) This reduces the maximum penalty for voting after the grace period by roughly 50%. This new value was derived from looking at the effects that TVC at max credits 8 would have for recent epochs (500+) and noting that the effect was a bit extreme, up to and exceeding 10% "bonus" for faster voters. This change reduces that maximum bonus by roughly half. In addition, the TVC feature key has been changed. --- programs/vote/src/vote_state/mod.rs | 103 +++++++++++++++++++--------- sdk/program/src/vote/state/mod.rs | 2 +- sdk/src/feature_set.rs | 2 +- 3 files changed, 74 insertions(+), 33 deletions(-) diff --git a/programs/vote/src/vote_state/mod.rs b/programs/vote/src/vote_state/mod.rs index e2a0cd449e8fbe..c3917085f4f691 100644 --- a/programs/vote/src/vote_state/mod.rs +++ b/programs/vote/src/vote_state/mod.rs @@ -2001,32 +2001,32 @@ mod tests { vec![32], 35, // root: 1 - // when slot 1 was voted on in slot 9, it earned 2 credits - 2, + // when slot 1 was voted on in slot 9, it earned 10 credits + 10, ), // Now another vote, should earn one credit ( vec![33], 36, // root: 2 - // when slot 2 was voted on in slot 9, it earned 3 credits - 2 + 3, // 5 + // when slot 2 was voted on in slot 9, it earned 11 credits + 10 + 11, // 21 ), // Two votes in sequence ( vec![34, 35], 37, // root: 4 - // when slots 3 and 4 were voted on in slot 9, they earned 4 and 5 credits - 5 + 4 + 5, // 14 + // when slots 3 and 4 were voted on in slot 9, they earned 12 and 13 credits + 21 + 12 + 13, // 46 ), // 3 votes in sequence ( vec![36, 37, 38], 39, // root: 7 - // slots 5, 6, and 7 earned 6, 7, and 8 credits when voted in slot 9 - 14 + 6 + 7 + 8, // 35 + // slots 5, 6, and 7 earned 14, 15, and 16 credits when voted in slot 9 + 46 + 14 + 15 + 16, // 91 ), ( // 30 votes in sequence @@ -2036,14 +2036,36 @@ mod tests { ], 69, // root: 37 - // slot 8 was voted in slot 9, earning 8 credits - // slots 9 - 25 earned 1 credit when voted in slot 34 - // slot 26, 27, 28, 29, 30, 31 earned 2, 3, 4, 5, 6, 7 credits when voted in slot 34 - // slot 32 earned 7 credits when voted in slot 35 - // slot 33 earned 7 credits when voted in slot 36 - // slot 34 and 35 earned 7 and 8 credits when voted in slot 37 - // slot 36 and 37 earned 7 and 8 credits when voted in slot 39 - 35 + 8 + ((25 - 9) + 1) + 2 + 3 + 4 + 5 + 6 + 7 + 7 + 7 + 7 + 8 + 7 + 8, // 131 + // slot 8 was voted in slot 9, earning 16 credits + // slots 9 - 25 earned 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 3, 4, 5, 6, 7, 8, and 9 credits when voted in + // slot 34 + // slot 26, 27, 28, 29, 30, 31 earned 10, 11, 12, 13, 14, 15 credits when voted in slot 34 + // slot 32 earned 15 credits when voted in slot 35 + // slot 33 earned 15 credits when voted in slot 36 + // slot 34 and 35 earned 15 and 16 credits when voted in slot 37 + // slot 36 and 37 earned 15 and 16 credits when voted in slot 39 + 91 + 16 + + 9 // * 1 + + 2 + + 3 + + 4 + + 5 + + 6 + + 7 + + 8 + + 9 + + 10 + + 11 + + 12 + + 13 + + 14 + + 15 + + 15 + + 15 + + 15 + + 16 + + 15 + + 16, // 327 ), // 31 votes in sequence ( @@ -2053,11 +2075,29 @@ mod tests { ], 100, // root: 68 - // slot 38 earned 8 credits when voted in slot 39 - // slot 39 - 60 earned 1 credit each when voted in slot 69 - // slot 61, 62, 63, 64, 65, 66, 67, 68 earned 2, 3, 4, 5, 6, 7, 8, and 8 credits when + // slot 38 earned 16 credits when voted in slot 39 + // slot 39 - 60 earned 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 3, 4, 5, 6, 7, 8, and 9 credits + // when voted in slot 69 + // slot 61, 62, 63, 64, 65, 66, 67, 68 earned 10, 11, 12, 13, 14, 15, 16, and 16 credits when // voted in slot 69 - 131 + 8 + ((60 - 39) + 1) + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 8, // 204 + 327 + 16 + + 14 // * 1 + + 2 + + 3 + + 4 + + 5 + + 6 + + 7 + + 8 + + 9 + + 10 + + 11 + + 12 + + 13 + + 14 + + 15 + + 16 + + 16, // 508 ), // Votes with expiry ( @@ -2066,7 +2106,7 @@ mod tests { // root: 74 // slots 96 - 114 expire // slots 69 - 74 earned 1 credit when voted in slot 100 - 204 + ((74 - 69) + 1), // 210 + 508 + ((74 - 69) + 1), // 514 ), // More votes with expiry of a large number of votes ( @@ -2074,7 +2114,7 @@ mod tests { 202, // root: 74 // slots 119 - 124 expire - 210, + 514, ), ( vec![ @@ -2083,18 +2123,19 @@ mod tests { ], 227, // root: 95 - // slot 75 - 91 earned 1 credit each when voted in slot 100 - // slot 92, 93, 94, 95 earned 2, 3, 4, 5, credits when voted in slot 100 - 210 + ((91 - 75) + 1) + 2 + 3 + 4 + 5, // 241 + // slot 75 - 91 earned 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 3, 4, 5, 6, 7, 8, and 9 credits when voted in + // slot 100 + // slot 92, 93, 94, 95 earned 10, 11, 12, 13, credits when voted in slot 100 + 514 + 9 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + 11 + 12 + 13, // 613 ), ( vec![227, 228, 229, 230, 231, 232, 233, 234, 235, 236], 237, // root: 205 - // slot 115 - 118 earned 1 credit when voted in slot 130 - // slot 200 and 201 earned 8 credits when voted in slot 202 + // slot 115 - 118 earned 3, 4, 5, and 6 credits when voted in slot 130 + // slot 200 and 201 earned 16 credits when voted in slot 202 // slots 202 - 205 earned 1 credit when voted in slot 227 - 241 + 1 + 1 + 1 + 1 + 8 + 8 + 1 + 1 + 1 + 1, // 265 + 613 + 3 + 4 + 5 + 6 + 16 + 16 + 1 + 1 + 1 + 1, // 667 ), ]; @@ -2224,9 +2265,9 @@ mod tests { 42, // root: 10 Some(10), - // when slots 1 - 6 were voted on in slot 12, they earned 1, 1, 1, 2, 3, and 4 credits - // when slots 7 - 10 were voted on in slot 11, they earned 6, 7, 8, and 8 credits - 1 + 1 + 1 + 2 + 3 + 4 + 6 + 7 + 8 + 8, + // when slots 1 - 6 were voted on in slot 12, they earned 7, 8, 9, 10, 11, and 12 credits + // when slots 7 - 10 were voted on in slot 11, they earned 14, 15, 16, and 16 credits + 7 + 8 + 9 + 10 + 11 + 12 + 14 + 15 + 16 + 16, ), ]; diff --git a/sdk/program/src/vote/state/mod.rs b/sdk/program/src/vote/state/mod.rs index a6e765472750c6..8853d5de6da143 100644 --- a/sdk/program/src/vote/state/mod.rs +++ b/sdk/program/src/vote/state/mod.rs @@ -45,7 +45,7 @@ const DEFAULT_PRIOR_VOTERS_OFFSET: usize = 114; pub const VOTE_CREDITS_GRACE_SLOTS: u8 = 2; // Maximum number of credits to award for a vote; this number of credits is awarded to votes on slots that land within the grace period. After that grace period, vote credits are reduced. -pub const VOTE_CREDITS_MAXIMUM_PER_SLOT: u8 = 8; +pub const VOTE_CREDITS_MAXIMUM_PER_SLOT: u8 = 16; #[frozen_abi(digest = "Ch2vVEwos2EjAVqSHCyJjnN2MNX1yrpapZTGhMSCjWUH")] #[derive(Serialize, Default, Deserialize, Debug, PartialEq, Eq, Clone, AbiExample)] diff --git a/sdk/src/feature_set.rs b/sdk/src/feature_set.rs index 98dc5a4037bd05..7d956bd13f405c 100644 --- a/sdk/src/feature_set.rs +++ b/sdk/src/feature_set.rs @@ -677,7 +677,7 @@ pub mod enable_poseidon_syscall { } pub mod timely_vote_credits { - solana_sdk::declare_id!("2oXpeh141pPZCTCFHBsvCwG2BtaHZZAtrVhwaxSy6brS"); + solana_sdk::declare_id!("tvcF6b1TRz353zKuhBjinZkKzjmihXmBAHJdjNYw1sQ"); } pub mod remaining_compute_units_syscall_enabled { From c6bd3883466e72771d14dc5f6d4aaa6edd386698 Mon Sep 17 00:00:00 2001 From: kirill lykov Date: Thu, 7 Mar 2024 12:51:44 -0800 Subject: [PATCH 30/84] Add get_blocks and get_slot methods to bench-tps-client (#94) * add get_block(s)/slot methods to BenchTpsClient * Update Cargo.lock * add commitment level for get_slot/blocks --- Cargo.lock | 1 + bench-tps/Cargo.toml | 1 + bench-tps/src/bench_tps_client.rs | 22 +++++++++++-- bench-tps/src/bench_tps_client/bank_client.rs | 24 ++++++++++++++ bench-tps/src/bench_tps_client/rpc_client.rs | 27 +++++++++++++++- bench-tps/src/bench_tps_client/thin_client.rs | 30 +++++++++++++++++ bench-tps/src/bench_tps_client/tpu_client.rs | 32 ++++++++++++++++++- 7 files changed, 132 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 85641aff1b22d6..b0b181a043c7c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5590,6 +5590,7 @@ dependencies = [ "solana-test-validator", "solana-thin-client", "solana-tpu-client", + "solana-transaction-status", "solana-version", "spl-instruction-padding", "tempfile", diff --git a/bench-tps/Cargo.toml b/bench-tps/Cargo.toml index cd40eb1c833c1c..2fc48c9e296d50 100644 --- a/bench-tps/Cargo.toml +++ b/bench-tps/Cargo.toml @@ -37,6 +37,7 @@ solana-sdk = { workspace = true } solana-streamer = { workspace = true } solana-thin-client = { workspace = true } solana-tpu-client = { workspace = true } +solana-transaction-status = { workspace = true } solana-version = { workspace = true } spl-instruction-padding = { workspace = true } thiserror = { workspace = true } diff --git a/bench-tps/src/bench_tps_client.rs b/bench-tps/src/bench_tps_client.rs index 3ab15bec11f7ee..0715d739879165 100644 --- a/bench-tps/src/bench_tps_client.rs +++ b/bench-tps/src/bench_tps_client.rs @@ -1,11 +1,12 @@ use { - solana_rpc_client_api::client_error::Error as ClientError, + solana_rpc_client_api::{client_error::Error as ClientError, config::RpcBlockConfig}, solana_sdk::{ account::Account, commitment_config::CommitmentConfig, epoch_info::EpochInfo, hash::Hash, - message::Message, pubkey::Pubkey, signature::Signature, transaction::Transaction, - transport::TransportError, + message::Message, pubkey::Pubkey, signature::Signature, slot_history::Slot, + transaction::Transaction, transport::TransportError, }, solana_tpu_client::tpu_client::TpuSenderError, + solana_transaction_status::UiConfirmedBlock, thiserror::Error, }; @@ -93,6 +94,21 @@ pub trait BenchTpsClient { ) -> Result; fn get_multiple_accounts(&self, pubkeys: &[Pubkey]) -> Result>>; + + fn get_slot_with_commitment(&self, commitment_config: CommitmentConfig) -> Result; + + fn get_blocks_with_commitment( + &self, + start_slot: Slot, + end_slot: Option, + commitment_config: CommitmentConfig, + ) -> Result>; + + fn get_block_with_config( + &self, + slot: Slot, + rpc_block_config: RpcBlockConfig, + ) -> Result; } mod bank_client; diff --git a/bench-tps/src/bench_tps_client/bank_client.rs b/bench-tps/src/bench_tps_client/bank_client.rs index 1aef7284c01ed6..3ea9080e51398a 100644 --- a/bench-tps/src/bench_tps_client/bank_client.rs +++ b/bench-tps/src/bench_tps_client/bank_client.rs @@ -1,5 +1,6 @@ use { crate::bench_tps_client::{BenchTpsClient, BenchTpsError, Result}, + solana_rpc_client_api::config::RpcBlockConfig, solana_runtime::bank_client::BankClient, solana_sdk::{ account::Account, @@ -10,8 +11,10 @@ use { message::Message, pubkey::Pubkey, signature::Signature, + slot_history::Slot, transaction::Transaction, }, + solana_transaction_status::UiConfirmedBlock, }; impl BenchTpsClient for BankClient { @@ -111,4 +114,25 @@ impl BenchTpsClient for BankClient { fn get_multiple_accounts(&self, _pubkeys: &[Pubkey]) -> Result>> { unimplemented!("BankClient doesn't support get_multiple_accounts"); } + + fn get_slot_with_commitment(&self, commitment_config: CommitmentConfig) -> Result { + SyncClient::get_slot_with_commitment(self, commitment_config).map_err(|err| err.into()) + } + + fn get_blocks_with_commitment( + &self, + _start_slot: Slot, + _end_slot: Option, + _commitment_config: CommitmentConfig, + ) -> Result> { + unimplemented!("BankClient doesn't support get_blocks"); + } + + fn get_block_with_config( + &self, + _slot: Slot, + _rpc_block_config: RpcBlockConfig, + ) -> Result { + unimplemented!("BankClient doesn't support get_block_with_config"); + } } diff --git a/bench-tps/src/bench_tps_client/rpc_client.rs b/bench-tps/src/bench_tps_client/rpc_client.rs index 2535099b464351..87ec1b8690c417 100644 --- a/bench-tps/src/bench_tps_client/rpc_client.rs +++ b/bench-tps/src/bench_tps_client/rpc_client.rs @@ -1,10 +1,13 @@ use { crate::bench_tps_client::{BenchTpsClient, BenchTpsError, Result}, solana_rpc_client::rpc_client::RpcClient, + solana_rpc_client_api::config::RpcBlockConfig, solana_sdk::{ account::Account, commitment_config::CommitmentConfig, epoch_info::EpochInfo, hash::Hash, - message::Message, pubkey::Pubkey, signature::Signature, transaction::Transaction, + message::Message, pubkey::Pubkey, signature::Signature, slot_history::Slot, + transaction::Transaction, }, + solana_transaction_status::UiConfirmedBlock, }; impl BenchTpsClient for RpcClient { @@ -104,4 +107,26 @@ impl BenchTpsClient for RpcClient { fn get_multiple_accounts(&self, pubkeys: &[Pubkey]) -> Result>> { RpcClient::get_multiple_accounts(self, pubkeys).map_err(|err| err.into()) } + + fn get_slot_with_commitment(&self, commitment_config: CommitmentConfig) -> Result { + RpcClient::get_slot_with_commitment(self, commitment_config).map_err(|err| err.into()) + } + + fn get_blocks_with_commitment( + &self, + start_slot: Slot, + end_slot: Option, + commitment_config: CommitmentConfig, + ) -> Result> { + RpcClient::get_blocks_with_commitment(self, start_slot, end_slot, commitment_config) + .map_err(|err| err.into()) + } + + fn get_block_with_config( + &self, + slot: Slot, + rpc_block_config: RpcBlockConfig, + ) -> Result { + RpcClient::get_block_with_config(self, slot, rpc_block_config).map_err(|err| err.into()) + } } diff --git a/bench-tps/src/bench_tps_client/thin_client.rs b/bench-tps/src/bench_tps_client/thin_client.rs index 6696774d679a8a..22945c4494f453 100644 --- a/bench-tps/src/bench_tps_client/thin_client.rs +++ b/bench-tps/src/bench_tps_client/thin_client.rs @@ -1,6 +1,7 @@ use { crate::bench_tps_client::{BenchTpsClient, BenchTpsError, Result}, solana_client::thin_client::ThinClient, + solana_rpc_client_api::config::RpcBlockConfig, solana_sdk::{ account::Account, client::{AsyncClient, Client, SyncClient}, @@ -10,8 +11,10 @@ use { message::Message, pubkey::Pubkey, signature::Signature, + slot_history::Slot, transaction::Transaction, }, + solana_transaction_status::UiConfirmedBlock, }; impl BenchTpsClient for ThinClient { @@ -110,4 +113,31 @@ impl BenchTpsClient for ThinClient { .get_multiple_accounts(pubkeys) .map_err(|err| err.into()) } + + fn get_slot_with_commitment(&self, commitment_config: CommitmentConfig) -> Result { + self.rpc_client() + .get_slot_with_commitment(commitment_config) + .map_err(|err| err.into()) + } + + fn get_blocks_with_commitment( + &self, + start_slot: Slot, + end_slot: Option, + commitment_config: CommitmentConfig, + ) -> Result> { + self.rpc_client() + .get_blocks_with_commitment(start_slot, end_slot, commitment_config) + .map_err(|err| err.into()) + } + + fn get_block_with_config( + &self, + slot: Slot, + rpc_block_config: RpcBlockConfig, + ) -> Result { + self.rpc_client() + .get_block_with_config(slot, rpc_block_config) + .map_err(|err| err.into()) + } } diff --git a/bench-tps/src/bench_tps_client/tpu_client.rs b/bench-tps/src/bench_tps_client/tpu_client.rs index c56da2ae6e880b..6c053271ad3eec 100644 --- a/bench-tps/src/bench_tps_client/tpu_client.rs +++ b/bench-tps/src/bench_tps_client/tpu_client.rs @@ -4,10 +4,13 @@ use { solana_connection_cache::connection_cache::{ ConnectionManager, ConnectionPool, NewConnectionConfig, }, + solana_rpc_client_api::config::RpcBlockConfig, solana_sdk::{ account::Account, commitment_config::CommitmentConfig, epoch_info::EpochInfo, hash::Hash, - message::Message, pubkey::Pubkey, signature::Signature, transaction::Transaction, + message::Message, pubkey::Pubkey, signature::Signature, slot_history::Slot, + transaction::Transaction, }, + solana_transaction_status::UiConfirmedBlock, }; impl BenchTpsClient for TpuClient @@ -130,4 +133,31 @@ where .get_multiple_accounts(pubkeys) .map_err(|err| err.into()) } + + fn get_slot_with_commitment(&self, commitment_config: CommitmentConfig) -> Result { + self.rpc_client() + .get_slot_with_commitment(commitment_config) + .map_err(|err| err.into()) + } + + fn get_blocks_with_commitment( + &self, + start_slot: Slot, + end_slot: Option, + commitment_config: CommitmentConfig, + ) -> Result> { + self.rpc_client() + .get_blocks_with_commitment(start_slot, end_slot, commitment_config) + .map_err(|err| err.into()) + } + + fn get_block_with_config( + &self, + slot: Slot, + rpc_block_config: RpcBlockConfig, + ) -> Result { + self.rpc_client() + .get_block_with_config(slot, rpc_block_config) + .map_err(|err| err.into()) + } } From 26692e666454d340a6691e2483194934e6a8ddfc Mon Sep 17 00:00:00 2001 From: steviez Date: Thu, 7 Mar 2024 16:06:31 -0600 Subject: [PATCH 31/84] blockstore: Remove unnecessary function and threadpool (#122) In a previous change, we removed the threadpool used to fetch entries in parallel in favor of combining all fetches into a single rocksdb multi_get() call. This change does the same thing, except for a threadpool that was used to fetch entries when we needed them to purge the transaction status and address signatures columns. --- ledger/src/blockstore.rs | 36 +---------------------- ledger/src/blockstore/blockstore_purge.rs | 3 +- 2 files changed, 3 insertions(+), 36 deletions(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index f8c8330843dfce..f15976abdb241b 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -31,10 +31,7 @@ use { itertools::Itertools, log::*, rand::Rng, - rayon::{ - iter::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator}, - ThreadPool, - }, + rayon::iter::{IntoParallelIterator, ParallelIterator}, rocksdb::{DBRawIterator, LiveFile}, solana_accounts_db::hardened_unpack::{ unpack_genesis_archive, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, @@ -94,16 +91,6 @@ pub use { rocksdb::properties as RocksProperties, }; -// get_max_thread_count to match number of threads in the old code. -// see: https://github.com/solana-labs/solana/pull/24853 -lazy_static! { - static ref PAR_THREAD_POOL_ALL_CPUS: ThreadPool = rayon::ThreadPoolBuilder::new() - .num_threads(num_cpus::get()) - .thread_name(|i| format!("solBstoreAll{i:02}")) - .build() - .unwrap(); -} - pub const MAX_REPLAY_WAKE_UP_SIGNALS: usize = 1; pub const MAX_COMPLETED_SLOTS_IN_CHANNEL: usize = 100_000; @@ -3283,27 +3270,6 @@ impl Blockstore { self.get_slot_entries_in_block(slot, vec![(start_index, end_index)], slot_meta) } - fn get_any_valid_slot_entries(&self, slot: Slot, start_index: u64) -> Vec { - let (completed_ranges, slot_meta) = self - .get_completed_ranges(slot, start_index) - .unwrap_or_default(); - if completed_ranges.is_empty() { - return vec![]; - } - let slot_meta = slot_meta.unwrap(); - - let entries: Vec> = PAR_THREAD_POOL_ALL_CPUS.install(|| { - completed_ranges - .par_iter() - .map(|(start_index, end_index)| { - self.get_entries_in_data_block(slot, *start_index, *end_index, Some(&slot_meta)) - .unwrap_or_default() - }) - .collect() - }); - entries.into_iter().flatten().collect() - } - /// Returns a mapping from each elements of `slots` to a list of the /// element's children slots. pub fn get_slots_since(&self, slots: &[Slot]) -> Result>> { diff --git a/ledger/src/blockstore/blockstore_purge.rs b/ledger/src/blockstore/blockstore_purge.rs index d8b4c7424cd8c1..d442732303fa2a 100644 --- a/ledger/src/blockstore/blockstore_purge.rs +++ b/ledger/src/blockstore/blockstore_purge.rs @@ -455,7 +455,8 @@ impl Blockstore { for slot in from_slot..=to_slot { let primary_indexes = slot_indexes(slot); - let slot_entries = self.get_any_valid_slot_entries(slot, 0); + let (slot_entries, _, _) = + self.get_slot_entries_with_shred_info(slot, 0, true /* allow_dead_slots */)?; let transactions = slot_entries .into_iter() .flat_map(|entry| entry.transactions); From 9770cd9083126b4dfe40fb207b0a3b8b21f33d21 Mon Sep 17 00:00:00 2001 From: Tao Zhu <82401714+tao-stones@users.noreply.github.com> Date: Thu, 7 Mar 2024 18:48:35 -0600 Subject: [PATCH 32/84] add precompile signature metrics to cost tracker (#133) --- cost-model/src/block_cost_limits.rs | 8 +- cost-model/src/cost_model.rs | 24 +++++- cost-model/src/cost_tracker.rs | 42 ++++++++++ cost-model/src/transaction_cost.rs | 31 +++++++ sdk/program/src/message/sanitized.rs | 116 ++++++++++++++++++++++++--- 5 files changed, 205 insertions(+), 16 deletions(-) diff --git a/cost-model/src/block_cost_limits.rs b/cost-model/src/block_cost_limits.rs index 328d89cd04198b..b04f289e0553af 100644 --- a/cost-model/src/block_cost_limits.rs +++ b/cost-model/src/block_cost_limits.rs @@ -24,6 +24,10 @@ pub const MAX_CONCURRENCY: u64 = 4; pub const COMPUTE_UNIT_TO_US_RATIO: u64 = 30; /// Number of compute units for one signature verification. pub const SIGNATURE_COST: u64 = COMPUTE_UNIT_TO_US_RATIO * 24; +/// Number of compute units for one secp256k1 signature verification. +pub const SECP256K1_VERIFY_COST: u64 = COMPUTE_UNIT_TO_US_RATIO * 223; +/// Number of compute units for one ed25519 signature verification. +pub const ED25519_VERIFY_COST: u64 = COMPUTE_UNIT_TO_US_RATIO * 76; /// Number of compute units for one write lock pub const WRITE_LOCK_UNITS: u64 = COMPUTE_UNIT_TO_US_RATIO * 10; /// Number of data bytes per compute units @@ -43,8 +47,8 @@ lazy_static! { (bpf_loader::id(), solana_bpf_loader_program::DEFAULT_LOADER_COMPUTE_UNITS), (loader_v4::id(), solana_loader_v4_program::DEFAULT_COMPUTE_UNITS), // Note: These are precompile, run directly in bank during sanitizing; - (secp256k1_program::id(), COMPUTE_UNIT_TO_US_RATIO * 24), - (ed25519_program::id(), COMPUTE_UNIT_TO_US_RATIO * 24), + (secp256k1_program::id(), 0), + (ed25519_program::id(), 0), ] .iter() .cloned() diff --git a/cost-model/src/cost_model.rs b/cost-model/src/cost_model.rs index b81ea24402d4df..fa12a7343bc7e0 100644 --- a/cost-model/src/cost_model.rs +++ b/cost-model/src/cost_model.rs @@ -43,7 +43,7 @@ impl CostModel { } else { let mut tx_cost = UsageCostDetails::new_with_default_capacity(); - tx_cost.signature_cost = Self::get_signature_cost(transaction); + Self::get_signature_cost(&mut tx_cost, transaction); Self::get_write_lock_cost(&mut tx_cost, transaction, feature_set); Self::get_transaction_cost(&mut tx_cost, transaction, feature_set); tx_cost.account_data_size = Self::calculate_account_data_size(transaction); @@ -53,8 +53,26 @@ impl CostModel { } } - fn get_signature_cost(transaction: &SanitizedTransaction) -> u64 { - transaction.signatures().len() as u64 * SIGNATURE_COST + fn get_signature_cost(tx_cost: &mut UsageCostDetails, transaction: &SanitizedTransaction) { + let signatures_count_detail = transaction.message().get_signature_details(); + tx_cost.num_transaction_signatures = signatures_count_detail.num_transaction_signatures(); + tx_cost.num_secp256k1_instruction_signatures = + signatures_count_detail.num_secp256k1_instruction_signatures(); + tx_cost.num_ed25519_instruction_signatures = + signatures_count_detail.num_ed25519_instruction_signatures(); + tx_cost.signature_cost = signatures_count_detail + .num_transaction_signatures() + .saturating_mul(SIGNATURE_COST) + .saturating_add( + signatures_count_detail + .num_secp256k1_instruction_signatures() + .saturating_mul(SECP256K1_VERIFY_COST), + ) + .saturating_add( + signatures_count_detail + .num_ed25519_instruction_signatures() + .saturating_mul(ED25519_VERIFY_COST), + ); } fn get_writable_accounts(transaction: &SanitizedTransaction) -> Vec { diff --git a/cost-model/src/cost_tracker.rs b/cost-model/src/cost_tracker.rs index 8fb092c36680a0..b5e3f9f4932a59 100644 --- a/cost-model/src/cost_tracker.rs +++ b/cost-model/src/cost_tracker.rs @@ -58,6 +58,9 @@ pub struct CostTracker { vote_cost: u64, transaction_count: u64, account_data_size: u64, + transaction_signature_count: u64, + secp256k1_instruction_signature_count: u64, + ed25519_instruction_signature_count: u64, } impl Default for CostTracker { @@ -77,6 +80,9 @@ impl Default for CostTracker { vote_cost: 0, transaction_count: 0, account_data_size: 0, + transaction_signature_count: 0, + secp256k1_instruction_signature_count: 0, + ed25519_instruction_signature_count: 0, } } } @@ -153,6 +159,21 @@ impl CostTracker { ("costliest_account", costliest_account.to_string(), String), ("costliest_account_cost", costliest_account_cost as i64, i64), ("account_data_size", self.account_data_size, i64), + ( + "transaction_signature_count", + self.transaction_signature_count, + i64 + ), + ( + "secp256k1_instruction_signature_count", + self.secp256k1_instruction_signature_count, + i64 + ), + ( + "ed25519_instruction_signature_count", + self.ed25519_instruction_signature_count, + i64 + ), ); } @@ -213,6 +234,18 @@ impl CostTracker { self.add_transaction_execution_cost(tx_cost, tx_cost.sum()); saturating_add_assign!(self.account_data_size, tx_cost.account_data_size()); saturating_add_assign!(self.transaction_count, 1); + saturating_add_assign!( + self.transaction_signature_count, + tx_cost.num_transaction_signatures() + ); + saturating_add_assign!( + self.secp256k1_instruction_signature_count, + tx_cost.num_secp256k1_instruction_signatures() + ); + saturating_add_assign!( + self.ed25519_instruction_signature_count, + tx_cost.num_ed25519_instruction_signatures() + ); } fn remove_transaction_cost(&mut self, tx_cost: &TransactionCost) { @@ -222,6 +255,15 @@ impl CostTracker { .account_data_size .saturating_sub(tx_cost.account_data_size()); self.transaction_count = self.transaction_count.saturating_sub(1); + self.transaction_signature_count = self + .transaction_signature_count + .saturating_sub(tx_cost.num_transaction_signatures()); + self.secp256k1_instruction_signature_count = self + .secp256k1_instruction_signature_count + .saturating_sub(tx_cost.num_secp256k1_instruction_signatures()); + self.ed25519_instruction_signature_count = self + .ed25519_instruction_signature_count + .saturating_sub(tx_cost.num_ed25519_instruction_signatures()); } /// Apply additional actual execution units to cost_tracker diff --git a/cost-model/src/transaction_cost.rs b/cost-model/src/transaction_cost.rs index c6e68bfe17b6f4..c92639676958ae 100644 --- a/cost-model/src/transaction_cost.rs +++ b/cost-model/src/transaction_cost.rs @@ -91,6 +91,27 @@ impl TransactionCost { Self::Transaction(usage_cost) => &usage_cost.writable_accounts, } } + + pub fn num_transaction_signatures(&self) -> u64 { + match self { + Self::SimpleVote { .. } => 1, + Self::Transaction(usage_cost) => usage_cost.num_transaction_signatures, + } + } + + pub fn num_secp256k1_instruction_signatures(&self) -> u64 { + match self { + Self::SimpleVote { .. } => 0, + Self::Transaction(usage_cost) => usage_cost.num_secp256k1_instruction_signatures, + } + } + + pub fn num_ed25519_instruction_signatures(&self) -> u64 { + match self { + Self::SimpleVote { .. } => 0, + Self::Transaction(usage_cost) => usage_cost.num_ed25519_instruction_signatures, + } + } } const MAX_WRITABLE_ACCOUNTS: usize = 256; @@ -105,6 +126,9 @@ pub struct UsageCostDetails { pub programs_execution_cost: u64, pub loaded_accounts_data_size_cost: u64, pub account_data_size: u64, + pub num_transaction_signatures: u64, + pub num_secp256k1_instruction_signatures: u64, + pub num_ed25519_instruction_signatures: u64, } impl Default for UsageCostDetails { @@ -117,6 +141,9 @@ impl Default for UsageCostDetails { programs_execution_cost: 0u64, loaded_accounts_data_size_cost: 0u64, account_data_size: 0u64, + num_transaction_signatures: 0u64, + num_secp256k1_instruction_signatures: 0u64, + num_ed25519_instruction_signatures: 0u64, } } } @@ -134,6 +161,10 @@ impl PartialEq for UsageCostDetails { && self.programs_execution_cost == other.programs_execution_cost && self.loaded_accounts_data_size_cost == other.loaded_accounts_data_size_cost && self.account_data_size == other.account_data_size + && self.num_transaction_signatures == other.num_transaction_signatures + && self.num_secp256k1_instruction_signatures + == other.num_secp256k1_instruction_signatures + && self.num_ed25519_instruction_signatures == other.num_ed25519_instruction_signatures && to_hash_set(&self.writable_accounts) == to_hash_set(&other.writable_accounts) } } diff --git a/sdk/program/src/message/sanitized.rs b/sdk/program/src/message/sanitized.rs index d4c7638e136a72..ce276a60ef69e7 100644 --- a/sdk/program/src/message/sanitized.rs +++ b/sdk/program/src/message/sanitized.rs @@ -345,17 +345,7 @@ impl SanitizedMessage { } pub fn num_signatures(&self) -> u64 { - let mut num_signatures = u64::from(self.header().num_required_signatures); - // This next part is really calculating the number of pre-processor - // operations being done and treating them like a signature - for (program_id, instruction) in self.program_instructions_iter() { - if secp256k1_program::check_id(program_id) || ed25519_program::check_id(program_id) { - if let Some(num_verifies) = instruction.data.first() { - num_signatures = num_signatures.saturating_add(u64::from(*num_verifies)); - } - } - } - num_signatures + self.get_signature_details().total_signatures() } /// Returns the number of requested write-locks in this message. @@ -365,6 +355,68 @@ impl SanitizedMessage { .len() .saturating_sub(self.num_readonly_accounts()) as u64 } + + /// return detailed signature counts + pub fn get_signature_details(&self) -> TransactionSignatureDetails { + let mut transaction_signature_details = TransactionSignatureDetails { + num_transaction_signatures: u64::from(self.header().num_required_signatures), + ..TransactionSignatureDetails::default() + }; + + // counting the number of pre-processor operations separately + for (program_id, instruction) in self.program_instructions_iter() { + if secp256k1_program::check_id(program_id) { + if let Some(num_verifies) = instruction.data.first() { + transaction_signature_details.num_secp256k1_instruction_signatures = + transaction_signature_details + .num_secp256k1_instruction_signatures + .saturating_add(u64::from(*num_verifies)); + } + } else if ed25519_program::check_id(program_id) { + if let Some(num_verifies) = instruction.data.first() { + transaction_signature_details.num_ed25519_instruction_signatures = + transaction_signature_details + .num_ed25519_instruction_signatures + .saturating_add(u64::from(*num_verifies)); + } + } + } + + transaction_signature_details + } +} + +#[derive(Default)] +/// Transaction signature details including the number of transaction signatures +/// and precompile signatures. +pub struct TransactionSignatureDetails { + num_transaction_signatures: u64, + num_secp256k1_instruction_signatures: u64, + num_ed25519_instruction_signatures: u64, +} + +impl TransactionSignatureDetails { + /// return total number of signature, treating pre-processor operations as signature + pub(crate) fn total_signatures(&self) -> u64 { + self.num_transaction_signatures + .saturating_add(self.num_secp256k1_instruction_signatures) + .saturating_add(self.num_ed25519_instruction_signatures) + } + + /// return the number of transaction signatures + pub fn num_transaction_signatures(&self) -> u64 { + self.num_transaction_signatures + } + + /// return the number of secp256k1 instruction signatures + pub fn num_secp256k1_instruction_signatures(&self) -> u64 { + self.num_secp256k1_instruction_signatures + } + + /// return the number of ed25519 instruction signatures + pub fn num_ed25519_instruction_signatures(&self) -> u64 { + self.num_ed25519_instruction_signatures + } } #[cfg(test)] @@ -563,4 +615,46 @@ mod tests { } } } + + #[test] + fn test_get_signature_details() { + let key0 = Pubkey::new_unique(); + let key1 = Pubkey::new_unique(); + let loader_key = Pubkey::new_unique(); + + let loader_instr = CompiledInstruction::new(2, &(), vec![0, 1]); + let mock_secp256k1_instr = CompiledInstruction::new(3, &[1u8; 10], vec![]); + let mock_ed25519_instr = CompiledInstruction::new(4, &[5u8; 10], vec![]); + + let message = SanitizedMessage::try_from_legacy_message( + legacy::Message::new_with_compiled_instructions( + 2, + 1, + 2, + vec![ + key0, + key1, + loader_key, + secp256k1_program::id(), + ed25519_program::id(), + ], + Hash::default(), + vec![ + loader_instr, + mock_secp256k1_instr.clone(), + mock_ed25519_instr, + mock_secp256k1_instr, + ], + ), + ) + .unwrap(); + + let signature_details = message.get_signature_details(); + // expect 2 required transaction signatures + assert_eq!(2, signature_details.num_transaction_signatures); + // expect 2 secp256k1 instruction signatures - 1 for each mock_secp2561k1_instr + assert_eq!(2, signature_details.num_secp256k1_instruction_signatures); + // expect 5 ed25519 instruction signatures from mock_ed25519_instr + assert_eq!(5, signature_details.num_ed25519_instruction_signatures); + } } From c0239c8eff109b8a278c3358a9dd92eccc821c5e Mon Sep 17 00:00:00 2001 From: Yihau Chen Date: Fri, 8 Mar 2024 10:48:39 +0800 Subject: [PATCH 33/84] ci: rename script (#125) --- ...nifest-keypair.sh => agave-install-update-manifest-keypair.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/{solana-install-update-manifest-keypair.sh => agave-install-update-manifest-keypair.sh} (100%) diff --git a/scripts/solana-install-update-manifest-keypair.sh b/scripts/agave-install-update-manifest-keypair.sh similarity index 100% rename from scripts/solana-install-update-manifest-keypair.sh rename to scripts/agave-install-update-manifest-keypair.sh From 377e1f911294395b666dfa799e30a25f8aefeb9b Mon Sep 17 00:00:00 2001 From: Jon C Date: Fri, 8 Mar 2024 11:02:33 +0100 Subject: [PATCH 34/84] runtime: Move `From` from sdk (#141) sdk: Move `From` into runtime --- runtime/src/bank/address_lookup_table.rs | 28 ++++++++++++++----- sdk/program/src/address_lookup_table/error.rs | 14 ---------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/runtime/src/bank/address_lookup_table.rs b/runtime/src/bank/address_lookup_table.rs index 483ec7cea00ea1..51eee794803e14 100644 --- a/runtime/src/bank/address_lookup_table.rs +++ b/runtime/src/bank/address_lookup_table.rs @@ -10,6 +10,17 @@ use { }, }; +fn into_address_loader_error(err: AddressLookupError) -> AddressLoaderError { + match err { + AddressLookupError::LookupTableAccountNotFound => { + AddressLoaderError::LookupTableAccountNotFound + } + AddressLookupError::InvalidAccountOwner => AddressLoaderError::InvalidAccountOwner, + AddressLookupError::InvalidAccountData => AddressLoaderError::InvalidAccountData, + AddressLookupError::InvalidLookupIndex => AddressLoaderError::InvalidLookupIndex, + } +} + impl AddressLoader for &Bank { fn load_addresses( self, @@ -23,15 +34,18 @@ impl AddressLoader for &Bank { .get_slot_hashes() .map_err(|_| AddressLoaderError::SlotHashesSysvarNotFound)?; - Ok(address_table_lookups + address_table_lookups .iter() .map(|address_table_lookup| { - self.rc.accounts.load_lookup_table_addresses( - &self.ancestors, - address_table_lookup, - &slot_hashes, - ) + self.rc + .accounts + .load_lookup_table_addresses( + &self.ancestors, + address_table_lookup, + &slot_hashes, + ) + .map_err(into_address_loader_error) }) - .collect::>()?) + .collect::>() } } diff --git a/sdk/program/src/address_lookup_table/error.rs b/sdk/program/src/address_lookup_table/error.rs index b427067afc386c..9925dee4dbbf4c 100644 --- a/sdk/program/src/address_lookup_table/error.rs +++ b/sdk/program/src/address_lookup_table/error.rs @@ -1,5 +1,3 @@ -#[cfg(not(target_os = "solana"))] -use solana_program::message::AddressLoaderError; use thiserror::Error; #[derive(Debug, Error, PartialEq, Eq, Clone)] @@ -20,15 +18,3 @@ pub enum AddressLookupError { #[error("Address lookup contains an invalid index")] InvalidLookupIndex, } - -#[cfg(not(target_os = "solana"))] -impl From for AddressLoaderError { - fn from(err: AddressLookupError) -> Self { - match err { - AddressLookupError::LookupTableAccountNotFound => Self::LookupTableAccountNotFound, - AddressLookupError::InvalidAccountOwner => Self::InvalidAccountOwner, - AddressLookupError::InvalidAccountData => Self::InvalidAccountData, - AddressLookupError::InvalidLookupIndex => Self::InvalidLookupIndex, - } - } -} From e027a8bd633f5ca280bbbc64a52bf250d0b6419f Mon Sep 17 00:00:00 2001 From: Lucas Steuernagel <38472950+LucasSte@users.noreply.github.com> Date: Fri, 8 Mar 2024 09:28:04 -0300 Subject: [PATCH 35/84] Gather recording booleans in a data structure (#134) --- core/src/banking_stage/consumer.rs | 5 ++-- ledger/src/blockstore_processor.rs | 16 +++++------ programs/sbf/tests/programs.rs | 13 +++++---- runtime/src/bank.rs | 37 +++++++++++------------- runtime/src/bank/tests.rs | 24 ++++++++-------- svm/src/transaction_processor.rs | 45 +++++++++++++++++++----------- 6 files changed, 74 insertions(+), 66 deletions(-) diff --git a/core/src/banking_stage/consumer.rs b/core/src/banking_stage/consumer.rs index 957e190c873f64..c5ed22a34278ce 100644 --- a/core/src/banking_stage/consumer.rs +++ b/core/src/banking_stage/consumer.rs @@ -34,6 +34,7 @@ use { solana_svm::{ account_loader::{validate_fee_payer, TransactionCheckResult}, transaction_error_metrics::TransactionErrorMetrics, + transaction_processor::ExecutionRecordingConfig, }, std::{ sync::{atomic::Ordering, Arc}, @@ -593,9 +594,7 @@ impl Consumer { .load_and_execute_transactions( batch, MAX_PROCESSING_AGE, - transaction_status_sender_enabled, - transaction_status_sender_enabled, - transaction_status_sender_enabled, + ExecutionRecordingConfig::new_single_setting(transaction_status_sender_enabled), &mut execute_and_commit_timings.execute_timings, None, // account_overrides self.log_messages_bytes_limit, diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs index c999eab1a56fd4..e4ae5f368b2afd 100644 --- a/ledger/src/blockstore_processor.rs +++ b/ledger/src/blockstore_processor.rs @@ -57,8 +57,11 @@ use { VersionedTransaction, }, }, - solana_svm::transaction_results::{ - TransactionExecutionDetails, TransactionExecutionResult, TransactionResults, + solana_svm::{ + transaction_processor::ExecutionRecordingConfig, + transaction_results::{ + TransactionExecutionDetails, TransactionExecutionResult, TransactionResults, + }, }, solana_transaction_status::token_balances::TransactionTokenBalancesSet, solana_vote::{vote_account::VoteAccountsHashMap, vote_sender_types::ReplayVoteSender}, @@ -163,9 +166,7 @@ pub fn execute_batch( batch, MAX_PROCESSING_AGE, transaction_status_sender.is_some(), - transaction_status_sender.is_some(), - transaction_status_sender.is_some(), - transaction_status_sender.is_some(), + ExecutionRecordingConfig::new_single_setting(transaction_status_sender.is_some()), timings, log_messages_bytes_limit, ); @@ -1972,6 +1973,7 @@ pub mod tests { system_transaction, transaction::{Transaction, TransactionError}, }, + solana_svm::transaction_processor::ExecutionRecordingConfig, solana_vote::vote_account::VoteAccount, solana_vote_program::{ self, @@ -3962,9 +3964,7 @@ pub mod tests { &batch, MAX_PROCESSING_AGE, false, - false, - false, - false, + ExecutionRecordingConfig::new_single_setting(false), &mut ExecuteTimings::default(), None, ); diff --git a/programs/sbf/tests/programs.rs b/programs/sbf/tests/programs.rs index dc4867ce7e40fd..22969bc482a28e 100644 --- a/programs/sbf/tests/programs.rs +++ b/programs/sbf/tests/programs.rs @@ -48,6 +48,7 @@ use { sysvar::{self, clock}, transaction::VersionedTransaction, }, + solana_svm::transaction_processor::ExecutionRecordingConfig, solana_svm::transaction_results::{ DurableNonceFee, InnerInstruction, TransactionExecutionDetails, TransactionExecutionResult, TransactionResults, @@ -104,9 +105,11 @@ fn process_transaction_and_record_inner( &tx_batch, MAX_PROCESSING_AGE, false, - true, - true, - false, + ExecutionRecordingConfig { + enable_cpi_recording: true, + enable_log_recording: true, + enable_return_data_recording: false, + }, &mut ExecuteTimings::default(), None, ) @@ -152,9 +155,7 @@ fn execute_transactions( &batch, std::usize::MAX, true, - true, - true, - true, + ExecutionRecordingConfig::new_single_setting(true), &mut timings, None, ); diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index ee04f20787cb9a..3e504d470de744 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -269,6 +269,7 @@ pub struct BankRc { #[cfg(RUSTC_WITH_SPECIALIZATION)] use solana_frozen_abi::abi_example::AbiExample; +use solana_svm::transaction_processor::ExecutionRecordingConfig; #[cfg(RUSTC_WITH_SPECIALIZATION)] impl AbiExample for BankRc { @@ -4297,9 +4298,11 @@ impl Bank { // for processing. During forwarding, the transaction could expire if the // delay is not accounted for. MAX_PROCESSING_AGE - MAX_TRANSACTION_FORWARDING_DELAY, - enable_cpi_recording, - true, - true, + ExecutionRecordingConfig { + enable_cpi_recording, + enable_log_recording: true, + enable_return_data_recording: true, + }, &mut timings, Some(&account_overrides), None, @@ -4548,9 +4551,7 @@ impl Bank { &self, batch: &TransactionBatch, max_age: usize, - enable_cpi_recording: bool, - enable_log_recording: bool, - enable_return_data_recording: bool, + recording_config: ExecutionRecordingConfig, timings: &mut ExecuteTimings, account_overrides: Option<&AccountOverrides>, log_messages_bytes_limit: Option, @@ -4614,9 +4615,7 @@ impl Bank { sanitized_txs, &mut check_results, &mut error_counters, - enable_cpi_recording, - enable_log_recording, - enable_return_data_recording, + recording_config, timings, account_overrides, self.builtin_programs.iter(), @@ -5642,9 +5641,7 @@ impl Bank { batch: &TransactionBatch, max_age: usize, collect_balances: bool, - enable_cpi_recording: bool, - enable_log_recording: bool, - enable_return_data_recording: bool, + recording_config: ExecutionRecordingConfig, timings: &mut ExecuteTimings, log_messages_bytes_limit: Option, ) -> (TransactionResults, TransactionBalancesSet) { @@ -5665,9 +5662,7 @@ impl Bank { } = self.load_and_execute_transactions( batch, max_age, - enable_cpi_recording, - enable_log_recording, - enable_return_data_recording, + recording_config, timings, None, log_messages_bytes_limit, @@ -5735,9 +5730,11 @@ impl Bank { &batch, MAX_PROCESSING_AGE, false, // collect_balances - false, // enable_cpi_recording - true, // enable_log_recording - true, // enable_return_data_recording + ExecutionRecordingConfig { + enable_cpi_recording: false, + enable_log_recording: true, + enable_return_data_recording: true, + }, &mut ExecuteTimings::default(), Some(1000 * 1000), ); @@ -5773,9 +5770,7 @@ impl Bank { batch, MAX_PROCESSING_AGE, false, - false, - false, - false, + ExecutionRecordingConfig::new_single_setting(false), &mut ExecuteTimings::default(), None, ) diff --git a/runtime/src/bank/tests.rs b/runtime/src/bank/tests.rs index 753116ff878e18..f9b846d85b1512 100644 --- a/runtime/src/bank/tests.rs +++ b/runtime/src/bank/tests.rs @@ -3122,9 +3122,7 @@ fn test_interleaving_locks() { &lock_result, MAX_PROCESSING_AGE, false, - false, - false, - false, + ExecutionRecordingConfig::new_single_setting(false), &mut ExecuteTimings::default(), None, ) @@ -5948,9 +5946,7 @@ fn test_pre_post_transaction_balances() { &lock_result, MAX_PROCESSING_AGE, true, - false, - false, - false, + ExecutionRecordingConfig::new_single_setting(false), &mut ExecuteTimings::default(), None, ); @@ -9230,9 +9226,11 @@ fn test_tx_log_order() { &batch, MAX_PROCESSING_AGE, false, - false, - true, - false, + ExecutionRecordingConfig { + enable_cpi_recording: false, + enable_log_recording: true, + enable_return_data_recording: false, + }, &mut ExecuteTimings::default(), None, ) @@ -9338,9 +9336,11 @@ fn test_tx_return_data() { &batch, MAX_PROCESSING_AGE, false, - false, - false, - true, + ExecutionRecordingConfig { + enable_cpi_recording: false, + enable_log_recording: false, + enable_return_data_recording: true, + }, &mut ExecuteTimings::default(), None, ) diff --git a/svm/src/transaction_processor.rs b/svm/src/transaction_processor.rs index e44b426df96b0d..d90afb0a428ea3 100644 --- a/svm/src/transaction_processor.rs +++ b/svm/src/transaction_processor.rs @@ -65,6 +65,24 @@ pub struct LoadAndExecuteSanitizedTransactionsOutput { pub execution_results: Vec, } +/// Configuration of the recording capabilities for transaction execution +#[derive(Copy, Clone)] +pub struct ExecutionRecordingConfig { + pub enable_cpi_recording: bool, + pub enable_log_recording: bool, + pub enable_return_data_recording: bool, +} + +impl ExecutionRecordingConfig { + pub fn new_single_setting(option: bool) -> Self { + ExecutionRecordingConfig { + enable_return_data_recording: option, + enable_log_recording: option, + enable_cpi_recording: option, + } + } +} + pub trait TransactionProcessingCallback { fn account_matches_owners(&self, account: &Pubkey, owners: &[Pubkey]) -> Option; @@ -184,9 +202,7 @@ impl TransactionBatchProcessor { sanitized_txs: &[SanitizedTransaction], check_results: &mut [TransactionCheckResult], error_counters: &mut TransactionErrorMetrics, - enable_cpi_recording: bool, - enable_log_recording: bool, - enable_return_data_recording: bool, + recording_config: ExecutionRecordingConfig, timings: &mut ExecuteTimings, account_overrides: Option<&AccountOverrides>, builtin_programs: impl Iterator, @@ -266,9 +282,7 @@ impl TransactionBatchProcessor { loaded_transaction, compute_budget, nonce.as_ref().map(DurableNonceFee::from), - enable_cpi_recording, - enable_log_recording, - enable_return_data_recording, + recording_config, timings, error_counters, log_messages_bytes_limit, @@ -466,9 +480,7 @@ impl TransactionBatchProcessor { loaded_transaction: &mut LoadedTransaction, compute_budget: ComputeBudget, durable_nonce_fee: Option, - enable_cpi_recording: bool, - enable_log_recording: bool, - enable_return_data_recording: bool, + recording_config: ExecutionRecordingConfig, timings: &mut ExecuteTimings, error_counters: &mut TransactionErrorMetrics, log_messages_bytes_limit: Option, @@ -506,7 +518,7 @@ impl TransactionBatchProcessor { tx.message(), ); - let log_collector = if enable_log_recording { + let log_collector = if recording_config.enable_log_recording { match log_messages_bytes_limit { None => Some(LogCollector::new_ref()), Some(log_messages_bytes_limit) => Some(LogCollector::new_ref_with_limit(Some( @@ -585,7 +597,7 @@ impl TransactionBatchProcessor { .ok() }); - let inner_instructions = if enable_cpi_recording { + let inner_instructions = if recording_config.enable_cpi_recording { Some(Self::inner_instructions_list_from_instruction_trace( &transaction_context, )) @@ -616,11 +628,12 @@ impl TransactionBatchProcessor { ); saturating_add_assign!(timings.details.changed_account_count, touched_account_count); - let return_data = if enable_return_data_recording && !return_data.data.is_empty() { - Some(return_data) - } else { - None - }; + let return_data = + if recording_config.enable_return_data_recording && !return_data.data.is_empty() { + Some(return_data) + } else { + None + }; TransactionExecutionResult::Executed { details: TransactionExecutionDetails { From 7a8e29d4d5edb1d0d467458e36f87871d8dfc0fe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 9 Mar 2024 00:59:43 +0800 Subject: [PATCH 36/84] build(deps): bump cc from 1.0.83 to 1.0.89 (#40) * build(deps): bump cc from 1.0.83 to 1.0.89 Bumps [cc](https://github.com/rust-lang/cc-rs) from 1.0.83 to 1.0.89. - [Release notes](https://github.com/rust-lang/cc-rs/releases) - [Commits](https://github.com/rust-lang/cc-rs/compare/1.0.83...1.0.89) --- updated-dependencies: - dependency-name: cc dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * [auto-commit] Update all Cargo lock files --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot-buildkite --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- programs/sbf/Cargo.lock | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b0b181a043c7c0..19b265863eba47 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1231,9 +1231,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.0.83" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +checksum = "a0ba8f7aaa012f30d5b2861462f6708eccd49c3c39863fe083a308035f63d723" dependencies = [ "jobserver", "libc", diff --git a/Cargo.toml b/Cargo.toml index 4b8ae12dab0078..16786e925c34b4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -170,7 +170,7 @@ bytes = "1.5" bzip2 = "0.4.4" caps = "0.5.5" cargo_metadata = "0.15.4" -cc = "1.0.83" +cc = "1.0.89" chrono = { version = "0.4.34", default-features = false } chrono-humanize = "0.2.3" clap = "2.33.1" diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index b72b4110e336ad..a3d350456afa9c 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -971,9 +971,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.83" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +checksum = "a0ba8f7aaa012f30d5b2861462f6708eccd49c3c39863fe083a308035f63d723" dependencies = [ "jobserver", "libc", From 68be105870d669b81999faafb4a1e6d217c26cbf Mon Sep 17 00:00:00 2001 From: Justin Starry Date: Sat, 9 Mar 2024 01:02:21 +0800 Subject: [PATCH 37/84] Use agave prefix in scripts for pre-installed binaries (#155) --- multinode-demo/common.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/multinode-demo/common.sh b/multinode-demo/common.sh index 1643208947b643..db43dd15cffaaf 100644 --- a/multinode-demo/common.sh +++ b/multinode-demo/common.sh @@ -30,7 +30,11 @@ if [[ -n $USE_INSTALL || ! -f "$SOLANA_ROOT"/Cargo.toml ]]; then if [[ -z $program ]]; then printf "solana" else - printf "solana-%s" "$program" + if [[ $program == "validator" || $program == "ledger-tool" || $program == "watchtower" || $program == "install" ]]; then + printf "agave-%s" "$program" + else + printf "solana-%s" "$program" + fi fi } else From 1ac523c121744376332693319c3113b946d39048 Mon Sep 17 00:00:00 2001 From: HaoranYi Date: Fri, 8 Mar 2024 12:14:40 -0600 Subject: [PATCH 38/84] Move delta hash test function to dev-context-utils (#151) move delta hash test function to dev-context-utils Co-authored-by: HaoranYi --- accounts-db/src/accounts_db.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/accounts-db/src/accounts_db.rs b/accounts-db/src/accounts_db.rs index cf4d17745b1b73..41ec05dce0e4a5 100644 --- a/accounts-db/src/accounts_db.rs +++ b/accounts-db/src/accounts_db.rs @@ -7882,6 +7882,7 @@ impl AccountsDb { /// /// As part of calculating the accounts delta hash, get a list of accounts modified this slot /// (aka dirty pubkeys) and add them to `self.uncleaned_pubkeys` for future cleaning. + #[cfg(feature = "dev-context-only-utils")] pub fn calculate_accounts_delta_hash(&self, slot: Slot) -> AccountsDeltaHash { self.calculate_accounts_delta_hash_internal(slot, None, HashMap::default()) } From bf0a3684eb77512ea6ea3b90a2c624c103bc9a9b Mon Sep 17 00:00:00 2001 From: steviez Date: Fri, 8 Mar 2024 12:52:35 -0600 Subject: [PATCH 39/84] Make ReplayStage create the parallel fork replay threadpool (#137) ReplayStage owning the pool allows for subsequent work to configure the size of the pool; configuring the size of the pool inside of the lazy_static would have been a little messy --- core/src/replay_stage.rs | 136 +++++++++++++++++++++------------------ 1 file changed, 73 insertions(+), 63 deletions(-) diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs index 46014e3f7912de..3683e257ed10a8 100644 --- a/core/src/replay_stage.rs +++ b/core/src/replay_stage.rs @@ -33,7 +33,6 @@ use { window_service::DuplicateSlotReceiver, }, crossbeam_channel::{Receiver, RecvTimeoutError, Sender}, - lazy_static::lazy_static, rayon::{prelude::*, ThreadPool}, solana_entry::entry::VerifyRecyclers, solana_geyser_plugin_manager::block_metadata_notifier_interface::BlockMetadataNotifierArc, @@ -102,14 +101,6 @@ const MAX_VOTE_REFRESH_INTERVAL_MILLIS: usize = 5000; const MAX_CONCURRENT_FORKS_TO_REPLAY: usize = 4; const MAX_REPAIR_RETRY_LOOP_ATTEMPTS: usize = 10; -lazy_static! { - static ref PAR_THREAD_POOL: ThreadPool = rayon::ThreadPoolBuilder::new() - .num_threads(MAX_CONCURRENT_FORKS_TO_REPLAY) - .thread_name(|i| format!("solReplay{i:02}")) - .build() - .unwrap(); -} - #[derive(PartialEq, Eq, Debug)] pub enum HeaviestForkFailures { LockedOut(u64), @@ -131,6 +122,11 @@ pub enum HeaviestForkFailures { ), } +enum ForkReplayMode { + Serial, + Parallel(ThreadPool), +} + #[derive(PartialEq, Eq, Debug)] enum ConfirmationType { SupermajorityVoted, @@ -656,6 +652,16 @@ impl ReplayStage { r_bank_forks.get_vote_only_mode_signal(), ) }; + let replay_mode = if replay_slots_concurrently { + ForkReplayMode::Serial + } else { + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(MAX_CONCURRENT_FORKS_TO_REPLAY) + .thread_name(|i| format!("solReplay{i:02}")) + .build() + .expect("new rayon threadpool"); + ForkReplayMode::Parallel(pool) + }; Self::reset_poh_recorder( &my_pubkey, @@ -717,7 +723,7 @@ impl ReplayStage { block_metadata_notifier.clone(), &mut replay_timing, log_messages_bytes_limit, - replay_slots_concurrently, + &replay_mode, &prioritization_fee_cache, &mut purge_repair_slot_counter, ); @@ -2706,6 +2712,7 @@ impl ReplayStage { fn replay_active_banks_concurrently( blockstore: &Blockstore, bank_forks: &RwLock, + thread_pool: &ThreadPool, my_pubkey: &Pubkey, vote_account: &Pubkey, progress: &mut ProgressMap, @@ -2723,7 +2730,7 @@ impl ReplayStage { let longest_replay_time_us = AtomicU64::new(0); // Allow for concurrent replaying of slots from different forks. - let replay_result_vec: Vec = PAR_THREAD_POOL.install(|| { + let replay_result_vec: Vec = thread_pool.install(|| { active_bank_slots .into_par_iter() .map(|bank_slot| { @@ -2737,7 +2744,7 @@ impl ReplayStage { trace!( "Replay active bank: slot {}, thread_idx {}", bank_slot, - PAR_THREAD_POOL.current_thread_index().unwrap_or_default() + thread_pool.current_thread_index().unwrap_or_default() ); let mut progress_lock = progress.write().unwrap(); if progress_lock @@ -3175,7 +3182,7 @@ impl ReplayStage { block_metadata_notifier: Option, replay_timing: &mut ReplayLoopTiming, log_messages_bytes_limit: Option, - replay_slots_concurrently: bool, + replay_mode: &ForkReplayMode, prioritization_fee_cache: &PrioritizationFeeCache, purge_repair_slot_counter: &mut PurgeRepairSlotCounter, ) -> bool /* completed a bank */ { @@ -3186,11 +3193,17 @@ impl ReplayStage { num_active_banks, active_bank_slots ); - if num_active_banks > 0 { - let replay_result_vec = if num_active_banks > 1 && replay_slots_concurrently { + if active_bank_slots.is_empty() { + return false; + } + + let replay_result_vec = match replay_mode { + // Skip the overhead of the threadpool if there is only one bank to play + ForkReplayMode::Parallel(thread_pool) if num_active_banks > 1 => { Self::replay_active_banks_concurrently( blockstore, bank_forks, + thread_pool, my_pubkey, vote_account, progress, @@ -3203,55 +3216,52 @@ impl ReplayStage { &active_bank_slots, prioritization_fee_cache, ) - } else { - active_bank_slots - .iter() - .map(|bank_slot| { - Self::replay_active_bank( - blockstore, - bank_forks, - my_pubkey, - vote_account, - progress, - transaction_status_sender, - entry_notification_sender, - verify_recyclers, - replay_vote_sender, - replay_timing, - log_messages_bytes_limit, - *bank_slot, - prioritization_fee_cache, - ) - }) - .collect() - }; + } + ForkReplayMode::Serial | ForkReplayMode::Parallel(_) => active_bank_slots + .iter() + .map(|bank_slot| { + Self::replay_active_bank( + blockstore, + bank_forks, + my_pubkey, + vote_account, + progress, + transaction_status_sender, + entry_notification_sender, + verify_recyclers, + replay_vote_sender, + replay_timing, + log_messages_bytes_limit, + *bank_slot, + prioritization_fee_cache, + ) + }) + .collect(), + }; - Self::process_replay_results( - blockstore, - bank_forks, - progress, - transaction_status_sender, - cache_block_meta_sender, - heaviest_subtree_fork_choice, - bank_notification_sender, - rewards_recorder_sender, - rpc_subscriptions, - duplicate_slots_tracker, - duplicate_confirmed_slots, - epoch_slots_frozen_slots, - unfrozen_gossip_verified_vote_hashes, - latest_validator_votes_for_frozen_banks, - cluster_slots_update_sender, - cost_update_sender, - duplicate_slots_to_repair, - ancestor_hashes_replay_update_sender, - block_metadata_notifier, - &replay_result_vec, - purge_repair_slot_counter, - ) - } else { - false - } + Self::process_replay_results( + blockstore, + bank_forks, + progress, + transaction_status_sender, + cache_block_meta_sender, + heaviest_subtree_fork_choice, + bank_notification_sender, + rewards_recorder_sender, + rpc_subscriptions, + duplicate_slots_tracker, + duplicate_confirmed_slots, + epoch_slots_frozen_slots, + unfrozen_gossip_verified_vote_hashes, + latest_validator_votes_for_frozen_banks, + cluster_slots_update_sender, + cost_update_sender, + duplicate_slots_to_repair, + ancestor_hashes_replay_update_sender, + block_metadata_notifier, + &replay_result_vec, + purge_repair_slot_counter, + ) } #[allow(clippy::too_many_arguments)] From d88050cda335f87e872eddbdf8506bc063f039d3 Mon Sep 17 00:00:00 2001 From: Dmitri Makarov Date: Fri, 8 Mar 2024 14:04:07 -0500 Subject: [PATCH 40/84] SVM: Add doc comments, restrict visibility of some xfaces to crate (#136) --- runtime/src/bank.rs | 2 +- svm/src/account_loader.rs | 385 ++++++++++++++++++---- svm/src/account_overrides.rs | 1 + svm/src/account_rent_state.rs | 33 +- svm/src/transaction_account_state_info.rs | 4 +- svm/src/transaction_processor.rs | 231 ++++++------- svm/tests/account_loader.rs | 214 ------------ svm/tests/rent_state.rs | 90 ----- 8 files changed, 463 insertions(+), 497 deletions(-) delete mode 100644 svm/tests/account_loader.rs delete mode 100644 svm/tests/rent_state.rs diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index 3e504d470de744..f0ba75defa0517 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -7499,7 +7499,7 @@ impl Bank { effective_epoch: Epoch, ) -> Arc { self.transaction_processor - .load_program(self, pubkey, reload, effective_epoch) + .load_program_with_pubkey(self, pubkey, reload, effective_epoch) } } diff --git a/svm/src/account_loader.rs b/svm/src/account_loader.rs index 1c02ded24665ff..bf9b5b9c40bfee 100644 --- a/svm/src/account_loader.rs +++ b/svm/src/account_loader.rs @@ -38,8 +38,11 @@ use { }; // for the load instructions -pub type TransactionRent = u64; -pub type TransactionProgramIndices = Vec>; +pub(crate) type TransactionRent = u64; +pub(crate) type TransactionProgramIndices = Vec>; +pub type TransactionCheckResult = (transaction::Result<()>, Option, Option); +pub type TransactionLoadResult = (Result, Option); + #[derive(PartialEq, Eq, Debug, Clone)] pub struct LoadedTransaction { pub accounts: Vec, @@ -48,10 +51,66 @@ pub struct LoadedTransaction { pub rent_debits: RentDebits, } -pub type TransactionLoadResult = (Result, Option); -pub type TransactionCheckResult = (transaction::Result<()>, Option, Option); +/// Check whether the payer_account is capable of paying the fee. The +/// side effect is to subtract the fee amount from the payer_account +/// balance of lamports. If the payer_acount is not able to pay the +/// fee, the error_counters is incremented, and a specific error is +/// returned. +pub fn validate_fee_payer( + payer_address: &Pubkey, + payer_account: &mut AccountSharedData, + payer_index: IndexOfAccount, + error_counters: &mut TransactionErrorMetrics, + rent_collector: &RentCollector, + fee: u64, +) -> Result<()> { + if payer_account.lamports() == 0 { + error_counters.account_not_found += 1; + return Err(TransactionError::AccountNotFound); + } + let system_account_kind = get_system_account_kind(payer_account).ok_or_else(|| { + error_counters.invalid_account_for_fee += 1; + TransactionError::InvalidAccountForFee + })?; + let min_balance = match system_account_kind { + SystemAccountKind::System => 0, + SystemAccountKind::Nonce => { + // Should we ever allow a fees charge to zero a nonce account's + // balance. The state MUST be set to uninitialized in that case + rent_collector.rent.minimum_balance(NonceState::size()) + } + }; + + payer_account + .lamports() + .checked_sub(min_balance) + .and_then(|v| v.checked_sub(fee)) + .ok_or_else(|| { + error_counters.insufficient_funds += 1; + TransactionError::InsufficientFundsForFee + })?; -pub fn load_accounts( + let payer_pre_rent_state = RentState::from_account(payer_account, &rent_collector.rent); + payer_account + .checked_sub_lamports(fee) + .map_err(|_| TransactionError::InsufficientFundsForFee)?; + + let payer_post_rent_state = RentState::from_account(payer_account, &rent_collector.rent); + RentState::check_rent_state_with_account( + &payer_pre_rent_state, + &payer_post_rent_state, + payer_address, + payer_account, + payer_index, + ) +} + +/// Collect information about accounts used in txs transactions and +/// return vector of tuples, one for each transaction in the +/// batch. Each tuple contains struct of information about accounts as +/// its first element and an optional transaction nonce info as its +/// second element. +pub(crate) fn load_accounts( callbacks: &CB, txs: &[SanitizedTransaction], lock_results: &[TransactionCheckResult], @@ -399,55 +458,6 @@ fn accumulate_and_check_loaded_account_data_size( } } -pub fn validate_fee_payer( - payer_address: &Pubkey, - payer_account: &mut AccountSharedData, - payer_index: IndexOfAccount, - error_counters: &mut TransactionErrorMetrics, - rent_collector: &RentCollector, - fee: u64, -) -> Result<()> { - if payer_account.lamports() == 0 { - error_counters.account_not_found += 1; - return Err(TransactionError::AccountNotFound); - } - let system_account_kind = get_system_account_kind(payer_account).ok_or_else(|| { - error_counters.invalid_account_for_fee += 1; - TransactionError::InvalidAccountForFee - })?; - let min_balance = match system_account_kind { - SystemAccountKind::System => 0, - SystemAccountKind::Nonce => { - // Should we ever allow a fees charge to zero a nonce account's - // balance. The state MUST be set to uninitialized in that case - rent_collector.rent.minimum_balance(NonceState::size()) - } - }; - - payer_account - .lamports() - .checked_sub(min_balance) - .and_then(|v| v.checked_sub(fee)) - .ok_or_else(|| { - error_counters.insufficient_funds += 1; - TransactionError::InsufficientFundsForFee - })?; - - let payer_pre_rent_state = RentState::from_account(payer_account, &rent_collector.rent); - payer_account - .checked_sub_lamports(fee) - .map_err(|_| TransactionError::InsufficientFundsForFee)?; - - let payer_post_rent_state = RentState::from_account(payer_account, &rent_collector.rent); - RentState::check_rent_state_with_account( - &payer_pre_rent_state, - &payer_post_rent_state, - payer_address, - payer_account, - payer_index, - ) -} - fn construct_instructions_account(message: &SanitizedMessage) -> AccountSharedData { AccountSharedData::from(Account { data: construct_instructions_data(&message.decompile_instructions()), @@ -460,11 +470,15 @@ fn construct_instructions_account(message: &SanitizedMessage) -> AccountSharedDa mod tests { use { super::*, - crate::transaction_processor::TransactionProcessingCallback, + crate::{ + transaction_account_state_info::TransactionAccountStateInfo, + transaction_processor::TransactionProcessingCallback, + }, nonce::state::Versions as NonceVersions, solana_program_runtime::{ + compute_budget::ComputeBudget, compute_budget_processor, - loaded_programs::LoadedProgram, + loaded_programs::{LoadedProgram, LoadedProgramsForTxBatch}, prioritization_fee::{PrioritizationFeeDetails, PrioritizationFeeType}, }, solana_sdk::{ @@ -473,22 +487,27 @@ mod tests { compute_budget::ComputeBudgetInstruction, epoch_schedule::EpochSchedule, feature_set::FeatureSet, + fee::FeeStructure, hash::Hash, instruction::CompiledInstruction, message::{ v0::{LoadedAddresses, LoadedMessage}, LegacyMessage, Message, MessageHeader, SanitizedMessage, }, + native_loader, + native_token::sol_to_lamports, nonce, + nonce_info::{NonceFull, NoncePartial}, pubkey::Pubkey, rent::Rent, - rent_collector::RentCollector, + rent_collector::{RentCollector, RENT_EXEMPT_RENT_EPOCH}, + rent_debits::RentDebits, signature::{Keypair, Signature, Signer}, - system_program, sysvar, - transaction::{Result, Transaction, TransactionError}, - transaction_context::TransactionAccount, + system_program, system_transaction, sysvar, + transaction::{Result, SanitizedTransaction, Transaction, TransactionError}, + transaction_context::{TransactionAccount, TransactionContext}, }, - std::{borrow::Cow, convert::TryFrom, sync::Arc}, + std::{borrow::Cow, collections::HashMap, convert::TryFrom, sync::Arc}, }; #[derive(Default)] @@ -2017,4 +2036,248 @@ mod tests { } ); } + + #[test] + fn test_rent_state_list_len() { + let mint_keypair = Keypair::new(); + let mut bank = TestCallbacks::default(); + let recipient = Pubkey::new_unique(); + let last_block_hash = Hash::new_unique(); + + let mut system_data = AccountSharedData::default(); + system_data.set_executable(true); + system_data.set_owner(native_loader::id()); + bank.accounts_map + .insert(Pubkey::new_from_array([0u8; 32]), system_data); + + let mut mint_data = AccountSharedData::default(); + mint_data.set_lamports(2); + bank.accounts_map.insert(mint_keypair.pubkey(), mint_data); + + bank.accounts_map + .insert(recipient, AccountSharedData::default()); + + let tx = system_transaction::transfer( + &mint_keypair, + &recipient, + sol_to_lamports(1.), + last_block_hash, + ); + let num_accounts = tx.message().account_keys.len(); + let sanitized_tx = SanitizedTransaction::from_transaction_for_tests(tx); + let mut error_counters = TransactionErrorMetrics::default(); + let loaded_txs = load_accounts( + &bank, + &[sanitized_tx.clone()], + &[(Ok(()), None, Some(0))], + &mut error_counters, + &FeeStructure::default(), + None, + &HashMap::new(), + &LoadedProgramsForTxBatch::default(), + ); + + let compute_budget = ComputeBudget::new(u64::from( + compute_budget_processor::DEFAULT_INSTRUCTION_COMPUTE_UNIT_LIMIT, + )); + let transaction_context = TransactionContext::new( + loaded_txs[0].0.as_ref().unwrap().accounts.clone(), + Rent::default(), + compute_budget.max_invoke_stack_height, + compute_budget.max_instruction_trace_length, + ); + + assert_eq!( + TransactionAccountStateInfo::new( + &Rent::default(), + &transaction_context, + sanitized_tx.message() + ) + .len(), + num_accounts, + ); + } + + #[test] + fn test_load_accounts_success() { + let key1 = Keypair::new(); + let key2 = Keypair::new(); + let key3 = Keypair::new(); + let key4 = Keypair::new(); + + let message = Message { + account_keys: vec![key2.pubkey(), key1.pubkey(), key4.pubkey()], + header: MessageHeader::default(), + instructions: vec![ + CompiledInstruction { + program_id_index: 1, + accounts: vec![0], + data: vec![], + }, + CompiledInstruction { + program_id_index: 1, + accounts: vec![2], + data: vec![], + }, + ], + recent_blockhash: Hash::default(), + }; + + let legacy = LegacyMessage::new(message); + let sanitized_message = SanitizedMessage::Legacy(legacy); + let mut mock_bank = TestCallbacks::default(); + let mut account_data = AccountSharedData::default(); + account_data.set_executable(true); + account_data.set_owner(key3.pubkey()); + mock_bank.accounts_map.insert(key1.pubkey(), account_data); + + let mut account_data = AccountSharedData::default(); + account_data.set_lamports(200); + mock_bank.accounts_map.insert(key2.pubkey(), account_data); + + let mut account_data = AccountSharedData::default(); + account_data.set_executable(true); + account_data.set_owner(native_loader::id()); + mock_bank.accounts_map.insert(key3.pubkey(), account_data); + + let mut error_counter = TransactionErrorMetrics::default(); + let loaded_programs = LoadedProgramsForTxBatch::default(); + + let sanitized_transaction = SanitizedTransaction::new_for_tests( + sanitized_message, + vec![Signature::new_unique()], + false, + ); + let lock_results = + (Ok(()), Some(NoncePartial::default()), Some(20u64)) as TransactionCheckResult; + + let results = load_accounts( + &mock_bank, + &[sanitized_transaction], + &[lock_results], + &mut error_counter, + &FeeStructure::default(), + None, + &HashMap::new(), + &loaded_programs, + ); + + let mut account_data = AccountSharedData::default(); + account_data.set_rent_epoch(RENT_EXEMPT_RENT_EPOCH); + + assert_eq!(results.len(), 1); + let (loaded_result, nonce) = results[0].clone(); + assert_eq!( + loaded_result.unwrap(), + LoadedTransaction { + accounts: vec![ + ( + key2.pubkey(), + mock_bank.accounts_map[&key2.pubkey()].clone() + ), + ( + key1.pubkey(), + mock_bank.accounts_map[&key1.pubkey()].clone() + ), + (key4.pubkey(), account_data), + ( + key3.pubkey(), + mock_bank.accounts_map[&key3.pubkey()].clone() + ), + ], + program_indices: vec![vec![3, 1], vec![3, 1]], + rent: 0, + rent_debits: RentDebits::default() + } + ); + + assert_eq!( + nonce.unwrap(), + NonceFull::new( + Pubkey::from([0; 32]), + AccountSharedData::default(), + Some(mock_bank.accounts_map[&key2.pubkey()].clone()) + ) + ); + } + + #[test] + fn test_load_accounts_error() { + let mock_bank = TestCallbacks::default(); + let message = Message { + account_keys: vec![Pubkey::new_from_array([0; 32])], + header: MessageHeader::default(), + instructions: vec![CompiledInstruction { + program_id_index: 0, + accounts: vec![], + data: vec![], + }], + recent_blockhash: Hash::default(), + }; + + let legacy = LegacyMessage::new(message); + let sanitized_message = SanitizedMessage::Legacy(legacy); + let sanitized_transaction = SanitizedTransaction::new_for_tests( + sanitized_message, + vec![Signature::new_unique()], + false, + ); + + let lock_results = (Ok(()), Some(NoncePartial::default()), None) as TransactionCheckResult; + let fee_structure = FeeStructure::default(); + + let result = load_accounts( + &mock_bank, + &[sanitized_transaction.clone()], + &[lock_results], + &mut TransactionErrorMetrics::default(), + &fee_structure, + None, + &HashMap::new(), + &LoadedProgramsForTxBatch::default(), + ); + + assert_eq!( + result, + vec![(Err(TransactionError::BlockhashNotFound), None)] + ); + + let lock_results = + (Ok(()), Some(NoncePartial::default()), Some(20u64)) as TransactionCheckResult; + + let result = load_accounts( + &mock_bank, + &[sanitized_transaction.clone()], + &[lock_results.clone()], + &mut TransactionErrorMetrics::default(), + &fee_structure, + None, + &HashMap::new(), + &LoadedProgramsForTxBatch::default(), + ); + + assert_eq!(result, vec![(Err(TransactionError::AccountNotFound), None)]); + + let lock_results = ( + Err(TransactionError::InvalidWritableAccount), + Some(NoncePartial::default()), + Some(20u64), + ) as TransactionCheckResult; + + let result = load_accounts( + &mock_bank, + &[sanitized_transaction.clone()], + &[lock_results], + &mut TransactionErrorMetrics::default(), + &fee_structure, + None, + &HashMap::new(), + &LoadedProgramsForTxBatch::default(), + ); + + assert_eq!( + result, + vec![(Err(TransactionError::InvalidWritableAccount), None)] + ); + } } diff --git a/svm/src/account_overrides.rs b/svm/src/account_overrides.rs index c88d77d54f30a9..8a205a798f66b1 100644 --- a/svm/src/account_overrides.rs +++ b/svm/src/account_overrides.rs @@ -10,6 +10,7 @@ pub struct AccountOverrides { } impl AccountOverrides { + /// Insert or remove an account with a given pubkey to/from the list of overrides. pub fn set_account(&mut self, pubkey: &Pubkey, account: Option) { match account { Some(account) => self.accounts.insert(*pubkey, account), diff --git a/svm/src/account_rent_state.rs b/svm/src/account_rent_state.rs index 6fae6e9033bd39..7e3501d0d6c649 100644 --- a/svm/src/account_rent_state.rs +++ b/svm/src/account_rent_state.rs @@ -23,6 +23,7 @@ pub enum RentState { } impl RentState { + /// Return a new RentState instance for a given account and rent. pub fn from_account(account: &AccountSharedData, rent: &Rent) -> Self { if account.lamports() == 0 { Self::Uninitialized @@ -36,6 +37,8 @@ impl RentState { } } + /// Check whether a transition from the pre_rent_state to this + /// state is valid. pub fn transition_allowed_from(&self, pre_rent_state: &RentState) -> bool { match self { Self::Uninitialized | Self::RentExempt => true, @@ -57,21 +60,6 @@ impl RentState { } } - fn submit_rent_state_metrics(pre_rent_state: &Self, post_rent_state: &Self) { - match (pre_rent_state, post_rent_state) { - (&RentState::Uninitialized, &RentState::RentPaying { .. }) => { - inc_new_counter_info!("rent_paying_err-new_account", 1); - } - (&RentState::RentPaying { .. }, &RentState::RentPaying { .. }) => { - inc_new_counter_info!("rent_paying_ok-legacy", 1); - } - (_, &RentState::RentPaying { .. }) => { - inc_new_counter_info!("rent_paying_err-other", 1); - } - _ => {} - } - } - pub(crate) fn check_rent_state( pre_rent_state: Option<&Self>, post_rent_state: Option<&Self>, @@ -118,6 +106,21 @@ impl RentState { Ok(()) } } + + fn submit_rent_state_metrics(pre_rent_state: &Self, post_rent_state: &Self) { + match (pre_rent_state, post_rent_state) { + (&RentState::Uninitialized, &RentState::RentPaying { .. }) => { + inc_new_counter_info!("rent_paying_err-new_account", 1); + } + (&RentState::RentPaying { .. }, &RentState::RentPaying { .. }) => { + inc_new_counter_info!("rent_paying_ok-legacy", 1); + } + (_, &RentState::RentPaying { .. }) => { + inc_new_counter_info!("rent_paying_err-other", 1); + } + _ => {} + } + } } #[cfg(test)] diff --git a/svm/src/transaction_account_state_info.rs b/svm/src/transaction_account_state_info.rs index ff5b93f6a6c459..0631050fe0e765 100644 --- a/svm/src/transaction_account_state_info.rs +++ b/svm/src/transaction_account_state_info.rs @@ -11,12 +11,12 @@ use { }; #[derive(PartialEq, Debug)] -pub struct TransactionAccountStateInfo { +pub(crate) struct TransactionAccountStateInfo { rent_state: Option, // None: readonly account } impl TransactionAccountStateInfo { - pub fn new( + pub(crate) fn new( rent: &Rent, transaction_context: &TransactionContext, message: &SanitizedMessage, diff --git a/svm/src/transaction_processor.rs b/svm/src/transaction_processor.rs index d90afb0a428ea3..fec908619f14f8 100644 --- a/svm/src/transaction_processor.rs +++ b/svm/src/transaction_processor.rs @@ -195,6 +195,7 @@ impl TransactionBatchProcessor { } } + /// Main entrypoint to the SVM. #[allow(clippy::too_many_arguments)] pub fn load_and_execute_sanitized_transactions<'a, CB: TransactionProcessingCallback>( &self, @@ -377,6 +378,112 @@ impl TransactionBatchProcessor { result } + /// Load program with a specific pubkey from loaded programs + /// cache, and update the program's access slot as a side-effect. + pub fn load_program_with_pubkey( + &self, + callbacks: &CB, + pubkey: &Pubkey, + reload: bool, + effective_epoch: Epoch, + ) -> Arc { + let loaded_programs_cache = self.loaded_programs_cache.read().unwrap(); + let environments = loaded_programs_cache.get_environments_for_epoch(effective_epoch); + let mut load_program_metrics = LoadProgramMetrics { + program_id: pubkey.to_string(), + ..LoadProgramMetrics::default() + }; + + let mut loaded_program = + match self.load_program_accounts(callbacks, pubkey, environments) { + ProgramAccountLoadResult::AccountNotFound => Ok(LoadedProgram::new_tombstone( + self.slot, + LoadedProgramType::Closed, + )), + + ProgramAccountLoadResult::InvalidAccountData(env) => Err((self.slot, env)), + + ProgramAccountLoadResult::ProgramOfLoaderV1orV2(program_account) => { + Self::load_program_from_bytes( + &mut load_program_metrics, + program_account.data(), + program_account.owner(), + program_account.data().len(), + 0, + environments.program_runtime_v1.clone(), + reload, + ) + .map_err(|_| (0, environments.program_runtime_v1.clone())) + } + + ProgramAccountLoadResult::ProgramOfLoaderV3( + program_account, + programdata_account, + slot, + ) => programdata_account + .data() + .get(UpgradeableLoaderState::size_of_programdata_metadata()..) + .ok_or(Box::new(InstructionError::InvalidAccountData).into()) + .and_then(|programdata| { + Self::load_program_from_bytes( + &mut load_program_metrics, + programdata, + program_account.owner(), + program_account + .data() + .len() + .saturating_add(programdata_account.data().len()), + slot, + environments.program_runtime_v1.clone(), + reload, + ) + }) + .map_err(|_| (slot, environments.program_runtime_v1.clone())), + + ProgramAccountLoadResult::ProgramOfLoaderV4(program_account, slot) => { + program_account + .data() + .get(LoaderV4State::program_data_offset()..) + .ok_or(Box::new(InstructionError::InvalidAccountData).into()) + .and_then(|elf_bytes| { + Self::load_program_from_bytes( + &mut load_program_metrics, + elf_bytes, + &loader_v4::id(), + program_account.data().len(), + slot, + environments.program_runtime_v2.clone(), + reload, + ) + }) + .map_err(|_| (slot, environments.program_runtime_v2.clone())) + } + } + .unwrap_or_else(|(slot, env)| { + LoadedProgram::new_tombstone(slot, LoadedProgramType::FailedVerification(env)) + }); + + let mut timings = ExecuteDetailsTimings::default(); + load_program_metrics.submit_datapoint(&mut timings); + if !Arc::ptr_eq( + &environments.program_runtime_v1, + &loaded_programs_cache.environments.program_runtime_v1, + ) || !Arc::ptr_eq( + &environments.program_runtime_v2, + &loaded_programs_cache.environments.program_runtime_v2, + ) { + // There can be two entries per program when the environment changes. + // One for the old environment before the epoch boundary and one for the new environment after the epoch boundary. + // These two entries have the same deployment slot, so they must differ in their effective slot instead. + // This is done by setting the effective slot of the entry for the new environment to the epoch boundary. + loaded_program.effective_slot = loaded_program + .effective_slot + .max(self.epoch_schedule.get_first_slot_in_epoch(effective_epoch)); + } + loaded_program.update_access_slot(self.slot); + Arc::new(loaded_program) + } + fn replenish_program_cache( &self, callback: &CB, @@ -454,7 +561,7 @@ impl TransactionBatchProcessor { if let Some((key, count)) = program_to_load { // Load, verify and compile one program. - let program = self.load_program(callback, &key, false, self.epoch); + let program = self.load_program_with_pubkey(callback, &key, false, self.epoch); program.tx_usage_counter.store(count, Ordering::Relaxed); program_to_store = Some((key, program)); } else if missing_programs.is_empty() { @@ -683,110 +790,6 @@ impl TransactionBatchProcessor { } } - pub fn load_program( - &self, - callbacks: &CB, - pubkey: &Pubkey, - reload: bool, - effective_epoch: Epoch, - ) -> Arc { - let loaded_programs_cache = self.loaded_programs_cache.read().unwrap(); - let environments = loaded_programs_cache.get_environments_for_epoch(effective_epoch); - let mut load_program_metrics = LoadProgramMetrics { - program_id: pubkey.to_string(), - ..LoadProgramMetrics::default() - }; - - let mut loaded_program = - match self.load_program_accounts(callbacks, pubkey, environments) { - ProgramAccountLoadResult::AccountNotFound => Ok(LoadedProgram::new_tombstone( - self.slot, - LoadedProgramType::Closed, - )), - - ProgramAccountLoadResult::InvalidAccountData(env) => Err((self.slot, env)), - - ProgramAccountLoadResult::ProgramOfLoaderV1orV2(program_account) => { - Self::load_program_from_bytes( - &mut load_program_metrics, - program_account.data(), - program_account.owner(), - program_account.data().len(), - 0, - environments.program_runtime_v1.clone(), - reload, - ) - .map_err(|_| (0, environments.program_runtime_v1.clone())) - } - - ProgramAccountLoadResult::ProgramOfLoaderV3( - program_account, - programdata_account, - slot, - ) => programdata_account - .data() - .get(UpgradeableLoaderState::size_of_programdata_metadata()..) - .ok_or(Box::new(InstructionError::InvalidAccountData).into()) - .and_then(|programdata| { - Self::load_program_from_bytes( - &mut load_program_metrics, - programdata, - program_account.owner(), - program_account - .data() - .len() - .saturating_add(programdata_account.data().len()), - slot, - environments.program_runtime_v1.clone(), - reload, - ) - }) - .map_err(|_| (slot, environments.program_runtime_v1.clone())), - - ProgramAccountLoadResult::ProgramOfLoaderV4(program_account, slot) => { - program_account - .data() - .get(LoaderV4State::program_data_offset()..) - .ok_or(Box::new(InstructionError::InvalidAccountData).into()) - .and_then(|elf_bytes| { - Self::load_program_from_bytes( - &mut load_program_metrics, - elf_bytes, - &loader_v4::id(), - program_account.data().len(), - slot, - environments.program_runtime_v2.clone(), - reload, - ) - }) - .map_err(|_| (slot, environments.program_runtime_v2.clone())) - } - } - .unwrap_or_else(|(slot, env)| { - LoadedProgram::new_tombstone(slot, LoadedProgramType::FailedVerification(env)) - }); - - let mut timings = ExecuteDetailsTimings::default(); - load_program_metrics.submit_datapoint(&mut timings); - if !Arc::ptr_eq( - &environments.program_runtime_v1, - &loaded_programs_cache.environments.program_runtime_v1, - ) || !Arc::ptr_eq( - &environments.program_runtime_v2, - &loaded_programs_cache.environments.program_runtime_v2, - ) { - // There can be two entries per program when the environment changes. - // One for the old environment before the epoch boundary and one for the new environment after the epoch boundary. - // These two entries have the same deployment slot, so they must differ in their effective slot instead. - // This is done by setting the effective slot of the entry for the new environment to the epoch boundary. - loaded_program.effective_slot = loaded_program - .effective_slot - .max(self.epoch_schedule.get_first_slot_in_epoch(effective_epoch)); - } - loaded_program.update_access_slot(self.slot); - Arc::new(loaded_program) - } - fn load_program_from_bytes( load_program_metrics: &mut LoadProgramMetrics, programdata: &[u8], @@ -1242,7 +1245,7 @@ mod tests { let key = Pubkey::new_unique(); let batch_processor = TransactionBatchProcessor::::default(); - let result = batch_processor.load_program(&mock_bank, &key, false, 50); + let result = batch_processor.load_program_with_pubkey(&mock_bank, &key, false, 50); let loaded_program = LoadedProgram::new_tombstone(0, LoadedProgramType::Closed); assert_eq!(result, Arc::new(loaded_program)); @@ -1259,7 +1262,7 @@ mod tests { .account_shared_data .insert(key, account_data.clone()); - let result = batch_processor.load_program(&mock_bank, &key, false, 20); + let result = batch_processor.load_program_with_pubkey(&mock_bank, &key, false, 20); let loaded_program = LoadedProgram::new_tombstone( 0, @@ -1288,7 +1291,7 @@ mod tests { .insert(key, account_data.clone()); // This should return an error - let result = batch_processor.load_program(&mock_bank, &key, false, 20); + let result = batch_processor.load_program_with_pubkey(&mock_bank, &key, false, 20); let loaded_program = LoadedProgram::new_tombstone( 0, LoadedProgramType::FailedVerification( @@ -1316,7 +1319,7 @@ mod tests { .account_shared_data .insert(key, account_data.clone()); - let result = batch_processor.load_program(&mock_bank, &key, false, 20); + let result = batch_processor.load_program_with_pubkey(&mock_bank, &key, false, 20); let environments = ProgramRuntimeEnvironments::default(); let expected = TransactionBatchProcessor::::load_program_from_bytes( @@ -1361,7 +1364,7 @@ mod tests { .insert(key2, account_data2.clone()); // This should return an error - let result = batch_processor.load_program(&mock_bank, &key1, false, 0); + let result = batch_processor.load_program_with_pubkey(&mock_bank, &key1, false, 0); let loaded_program = LoadedProgram::new_tombstone( 0, LoadedProgramType::FailedVerification( @@ -1399,7 +1402,7 @@ mod tests { .account_shared_data .insert(key2, account_data.clone()); - let result = batch_processor.load_program(&mock_bank, &key1, false, 20); + let result = batch_processor.load_program_with_pubkey(&mock_bank, &key1, false, 20); let data = account_data.data(); account_data @@ -1441,7 +1444,7 @@ mod tests { .account_shared_data .insert(key, account_data.clone()); - let result = batch_processor.load_program(&mock_bank, &key, false, 0); + let result = batch_processor.load_program_with_pubkey(&mock_bank, &key, false, 0); let loaded_program = LoadedProgram::new_tombstone( 0, LoadedProgramType::FailedVerification( @@ -1475,7 +1478,7 @@ mod tests { .account_shared_data .insert(key, account_data.clone()); - let result = batch_processor.load_program(&mock_bank, &key, false, 20); + let result = batch_processor.load_program_with_pubkey(&mock_bank, &key, false, 20); let data = account_data.data()[LoaderV4State::program_data_offset()..].to_vec(); account_data.set_data(data); @@ -1513,7 +1516,7 @@ mod tests { .account_shared_data .insert(key, account_data.clone()); - let result = batch_processor.load_program(&mock_bank, &key, false, 20); + let result = batch_processor.load_program_with_pubkey(&mock_bank, &key, false, 20); let slot = batch_processor.epoch_schedule.get_first_slot_in_epoch(20); assert_eq!(result.effective_slot, slot); diff --git a/svm/tests/account_loader.rs b/svm/tests/account_loader.rs deleted file mode 100644 index dd4cd046046399..00000000000000 --- a/svm/tests/account_loader.rs +++ /dev/null @@ -1,214 +0,0 @@ -use { - crate::mock_bank::MockBankCallback, - solana_program_runtime::loaded_programs::LoadedProgramsForTxBatch, - solana_sdk::{ - account::{AccountSharedData, WritableAccount}, - fee::FeeStructure, - hash::Hash, - instruction::CompiledInstruction, - message::{LegacyMessage, Message, MessageHeader, SanitizedMessage}, - native_loader, - nonce_info::{NonceFull, NoncePartial}, - pubkey::Pubkey, - rent_collector::RENT_EXEMPT_RENT_EPOCH, - rent_debits::RentDebits, - signature::{Keypair, Signature, Signer}, - transaction::{SanitizedTransaction, TransactionError}, - }, - solana_svm::{ - account_loader::{load_accounts, LoadedTransaction, TransactionCheckResult}, - transaction_error_metrics::TransactionErrorMetrics, - }, - std::collections::HashMap, -}; - -mod mock_bank; - -#[test] -fn test_load_accounts_success() { - let key1 = Keypair::new(); - let key2 = Keypair::new(); - let key3 = Keypair::new(); - let key4 = Keypair::new(); - - let message = Message { - account_keys: vec![key2.pubkey(), key1.pubkey(), key4.pubkey()], - header: MessageHeader::default(), - instructions: vec![ - CompiledInstruction { - program_id_index: 1, - accounts: vec![0], - data: vec![], - }, - CompiledInstruction { - program_id_index: 1, - accounts: vec![2], - data: vec![], - }, - ], - recent_blockhash: Hash::default(), - }; - - let legacy = LegacyMessage::new(message); - let sanitized_message = SanitizedMessage::Legacy(legacy); - let mut mock_bank = MockBankCallback::default(); - let mut account_data = AccountSharedData::default(); - account_data.set_executable(true); - account_data.set_owner(key3.pubkey()); - mock_bank - .account_shared_data - .insert(key1.pubkey(), account_data); - - let mut account_data = AccountSharedData::default(); - account_data.set_lamports(200); - mock_bank - .account_shared_data - .insert(key2.pubkey(), account_data); - - let mut account_data = AccountSharedData::default(); - account_data.set_executable(true); - account_data.set_owner(native_loader::id()); - mock_bank - .account_shared_data - .insert(key3.pubkey(), account_data); - - let mut error_counter = TransactionErrorMetrics::default(); - let loaded_programs = LoadedProgramsForTxBatch::default(); - - let sanitized_transaction = SanitizedTransaction::new_for_tests( - sanitized_message, - vec![Signature::new_unique()], - false, - ); - let lock_results = - (Ok(()), Some(NoncePartial::default()), Some(20u64)) as TransactionCheckResult; - - let results = load_accounts( - &mock_bank, - &[sanitized_transaction], - &[lock_results], - &mut error_counter, - &FeeStructure::default(), - None, - &HashMap::new(), - &loaded_programs, - ); - - let mut account_data = AccountSharedData::default(); - account_data.set_rent_epoch(RENT_EXEMPT_RENT_EPOCH); - - assert_eq!(results.len(), 1); - let (loaded_result, nonce) = results[0].clone(); - assert_eq!( - loaded_result.unwrap(), - LoadedTransaction { - accounts: vec![ - ( - key2.pubkey(), - mock_bank.account_shared_data[&key2.pubkey()].clone() - ), - ( - key1.pubkey(), - mock_bank.account_shared_data[&key1.pubkey()].clone() - ), - (key4.pubkey(), account_data), - ( - key3.pubkey(), - mock_bank.account_shared_data[&key3.pubkey()].clone() - ), - ], - program_indices: vec![vec![3, 1], vec![3, 1]], - rent: 0, - rent_debits: RentDebits::default() - } - ); - - assert_eq!( - nonce.unwrap(), - NonceFull::new( - Pubkey::from([0; 32]), - AccountSharedData::default(), - Some(mock_bank.account_shared_data[&key2.pubkey()].clone()) - ) - ); -} - -#[test] -fn test_load_accounts_error() { - let mock_bank = MockBankCallback::default(); - let message = Message { - account_keys: vec![Pubkey::new_from_array([0; 32])], - header: MessageHeader::default(), - instructions: vec![CompiledInstruction { - program_id_index: 0, - accounts: vec![], - data: vec![], - }], - recent_blockhash: Hash::default(), - }; - - let legacy = LegacyMessage::new(message); - let sanitized_message = SanitizedMessage::Legacy(legacy); - let sanitized_transaction = SanitizedTransaction::new_for_tests( - sanitized_message, - vec![Signature::new_unique()], - false, - ); - - let lock_results = (Ok(()), Some(NoncePartial::default()), None) as TransactionCheckResult; - let fee_structure = FeeStructure::default(); - - let result = load_accounts( - &mock_bank, - &[sanitized_transaction.clone()], - &[lock_results], - &mut TransactionErrorMetrics::default(), - &fee_structure, - None, - &HashMap::new(), - &LoadedProgramsForTxBatch::default(), - ); - - assert_eq!( - result, - vec![(Err(TransactionError::BlockhashNotFound), None)] - ); - - let lock_results = - (Ok(()), Some(NoncePartial::default()), Some(20u64)) as TransactionCheckResult; - - let result = load_accounts( - &mock_bank, - &[sanitized_transaction.clone()], - &[lock_results.clone()], - &mut TransactionErrorMetrics::default(), - &fee_structure, - None, - &HashMap::new(), - &LoadedProgramsForTxBatch::default(), - ); - - assert_eq!(result, vec![(Err(TransactionError::AccountNotFound), None)]); - - let lock_results = ( - Err(TransactionError::InvalidWritableAccount), - Some(NoncePartial::default()), - Some(20u64), - ) as TransactionCheckResult; - - let result = load_accounts( - &mock_bank, - &[sanitized_transaction.clone()], - &[lock_results], - &mut TransactionErrorMetrics::default(), - &fee_structure, - None, - &HashMap::new(), - &LoadedProgramsForTxBatch::default(), - ); - - assert_eq!( - result, - vec![(Err(TransactionError::InvalidWritableAccount), None)] - ); -} diff --git a/svm/tests/rent_state.rs b/svm/tests/rent_state.rs deleted file mode 100644 index f3ea728f6b874f..00000000000000 --- a/svm/tests/rent_state.rs +++ /dev/null @@ -1,90 +0,0 @@ -#![cfg(test)] - -use { - solana_program_runtime::{ - compute_budget::ComputeBudget, compute_budget_processor, - loaded_programs::LoadedProgramsForTxBatch, - }, - solana_sdk::{ - account::{AccountSharedData, WritableAccount}, - fee::FeeStructure, - hash::Hash, - native_loader, - native_token::sol_to_lamports, - pubkey::Pubkey, - rent::Rent, - signature::{Keypair, Signer}, - system_transaction, - transaction::SanitizedTransaction, - transaction_context::TransactionContext, - }, - solana_svm::{ - account_loader::load_accounts, transaction_account_state_info::TransactionAccountStateInfo, - transaction_error_metrics::TransactionErrorMetrics, - }, - std::collections::HashMap, -}; - -mod mock_bank; - -#[test] -fn test_rent_state_list_len() { - let mint_keypair = Keypair::new(); - let mut bank = mock_bank::MockBankCallback::default(); - let recipient = Pubkey::new_unique(); - let last_block_hash = Hash::new_unique(); - - let mut system_data = AccountSharedData::default(); - system_data.set_executable(true); - system_data.set_owner(native_loader::id()); - bank.account_shared_data - .insert(Pubkey::new_from_array([0u8; 32]), system_data); - - let mut mint_data = AccountSharedData::default(); - mint_data.set_lamports(2); - bank.account_shared_data - .insert(mint_keypair.pubkey(), mint_data); - - bank.account_shared_data - .insert(recipient, AccountSharedData::default()); - - let tx = system_transaction::transfer( - &mint_keypair, - &recipient, - sol_to_lamports(1.), - last_block_hash, - ); - let num_accounts = tx.message().account_keys.len(); - let sanitized_tx = SanitizedTransaction::from_transaction_for_tests(tx); - let mut error_counters = TransactionErrorMetrics::default(); - let loaded_txs = load_accounts( - &bank, - &[sanitized_tx.clone()], - &[(Ok(()), None, Some(0))], - &mut error_counters, - &FeeStructure::default(), - None, - &HashMap::new(), - &LoadedProgramsForTxBatch::default(), - ); - - let compute_budget = ComputeBudget::new(u64::from( - compute_budget_processor::DEFAULT_INSTRUCTION_COMPUTE_UNIT_LIMIT, - )); - let transaction_context = TransactionContext::new( - loaded_txs[0].0.as_ref().unwrap().accounts.clone(), - Rent::default(), - compute_budget.max_invoke_stack_height, - compute_budget.max_instruction_trace_length, - ); - - assert_eq!( - TransactionAccountStateInfo::new( - &Rent::default(), - &transaction_context, - sanitized_tx.message() - ) - .len(), - num_accounts, - ); -} From 3863bb1bdf0f7c9a0b35c2c19dc50943ca39657e Mon Sep 17 00:00:00 2001 From: Yihau Chen Date: Mon, 11 Mar 2024 12:29:24 +0800 Subject: [PATCH 41/84] ci: fix Windows gh release pipeline (#165) --- .github/workflows/release-artifacts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 7aec77f0dac45f..c840862a5e28f3 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -99,7 +99,7 @@ jobs: uses: actions/download-artifact@v3 with: name: windows-artifact - path: .windows-release/ + path: ./windows-release/ - name: Release uses: softprops/action-gh-release@v1 From 0e12172ddd44f0b44130581f82e595084a553fd0 Mon Sep 17 00:00:00 2001 From: Brooks Date: Mon, 11 Mar 2024 11:34:08 -0400 Subject: [PATCH 42/84] Moves accounts benches into accounts-db crate (#164) --- accounts-db/benches/accounts.rs | 343 ++++++++++++++++++++++++++++++++ runtime/benches/accounts.rs | 340 +------------------------------ 2 files changed, 349 insertions(+), 334 deletions(-) create mode 100644 accounts-db/benches/accounts.rs diff --git a/accounts-db/benches/accounts.rs b/accounts-db/benches/accounts.rs new file mode 100644 index 00000000000000..9b3b70600a60a2 --- /dev/null +++ b/accounts-db/benches/accounts.rs @@ -0,0 +1,343 @@ +#![feature(test)] +#![allow(clippy::arithmetic_side_effects)] + +extern crate test; + +use { + dashmap::DashMap, + rand::Rng, + rayon::iter::{IntoParallelRefIterator, ParallelIterator}, + solana_accounts_db::{ + accounts::{AccountAddressFilter, Accounts}, + accounts_db::{ + test_utils::create_test_accounts, AccountShrinkThreshold, AccountsDb, + VerifyAccountsHashAndLamportsConfig, ACCOUNTS_DB_CONFIG_FOR_BENCHMARKS, + }, + accounts_index::{AccountSecondaryIndexes, ScanConfig}, + ancestors::Ancestors, + }, + solana_sdk::{ + account::{Account, AccountSharedData, ReadableAccount}, + genesis_config::ClusterType, + hash::Hash, + pubkey::Pubkey, + rent_collector::RentCollector, + sysvar::epoch_schedule::EpochSchedule, + }, + std::{ + collections::{HashMap, HashSet}, + path::PathBuf, + sync::{Arc, RwLock}, + thread::Builder, + }, + test::Bencher, +}; + +fn new_accounts_db(account_paths: Vec) -> AccountsDb { + AccountsDb::new_with_config( + account_paths, + &ClusterType::Development, + AccountSecondaryIndexes::default(), + AccountShrinkThreshold::default(), + Some(ACCOUNTS_DB_CONFIG_FOR_BENCHMARKS), + None, + Arc::default(), + ) +} + +#[bench] +fn bench_accounts_hash_bank_hash(bencher: &mut Bencher) { + let accounts_db = new_accounts_db(vec![PathBuf::from("bench_accounts_hash_internal")]); + let accounts = Accounts::new(Arc::new(accounts_db)); + let mut pubkeys: Vec = vec![]; + let num_accounts = 60_000; + let slot = 0; + create_test_accounts(&accounts, &mut pubkeys, num_accounts, slot); + let ancestors = Ancestors::from(vec![0]); + let (_, total_lamports) = accounts + .accounts_db + .update_accounts_hash_for_tests(0, &ancestors, false, false); + accounts.add_root(slot); + accounts.accounts_db.flush_accounts_cache(true, Some(slot)); + bencher.iter(|| { + assert!(accounts.verify_accounts_hash_and_lamports( + 0, + total_lamports, + None, + VerifyAccountsHashAndLamportsConfig { + ancestors: &ancestors, + test_hash_calculation: false, + epoch_schedule: &EpochSchedule::default(), + rent_collector: &RentCollector::default(), + ignore_mismatch: false, + store_detailed_debug_info: false, + use_bg_thread_pool: false, + } + )) + }); +} + +#[bench] +fn bench_update_accounts_hash(bencher: &mut Bencher) { + solana_logger::setup(); + let accounts_db = new_accounts_db(vec![PathBuf::from("update_accounts_hash")]); + let accounts = Accounts::new(Arc::new(accounts_db)); + let mut pubkeys: Vec = vec![]; + create_test_accounts(&accounts, &mut pubkeys, 50_000, 0); + let ancestors = Ancestors::from(vec![0]); + bencher.iter(|| { + accounts + .accounts_db + .update_accounts_hash_for_tests(0, &ancestors, false, false); + }); +} + +#[bench] +fn bench_accounts_delta_hash(bencher: &mut Bencher) { + solana_logger::setup(); + let accounts_db = new_accounts_db(vec![PathBuf::from("accounts_delta_hash")]); + let accounts = Accounts::new(Arc::new(accounts_db)); + let mut pubkeys: Vec = vec![]; + create_test_accounts(&accounts, &mut pubkeys, 100_000, 0); + bencher.iter(|| { + accounts.accounts_db.calculate_accounts_delta_hash(0); + }); +} + +#[bench] +fn bench_delete_dependencies(bencher: &mut Bencher) { + solana_logger::setup(); + let accounts_db = new_accounts_db(vec![PathBuf::from("accounts_delete_deps")]); + let accounts = Accounts::new(Arc::new(accounts_db)); + let mut old_pubkey = Pubkey::default(); + let zero_account = AccountSharedData::new(0, 0, AccountSharedData::default().owner()); + for i in 0..1000 { + let pubkey = solana_sdk::pubkey::new_rand(); + let account = AccountSharedData::new(i + 1, 0, AccountSharedData::default().owner()); + accounts.store_slow_uncached(i, &pubkey, &account); + accounts.store_slow_uncached(i, &old_pubkey, &zero_account); + old_pubkey = pubkey; + accounts.add_root(i); + } + bencher.iter(|| { + accounts.accounts_db.clean_accounts_for_tests(); + }); +} + +fn store_accounts_with_possible_contention( + bench_name: &str, + bencher: &mut Bencher, + reader_f: F, +) where + F: Fn(&Accounts, &[Pubkey]) + Send + Copy, +{ + let num_readers = 5; + let accounts_db = new_accounts_db(vec![PathBuf::from( + std::env::var("FARF_DIR").unwrap_or_else(|_| "farf".to_string()), + ) + .join(bench_name)]); + let accounts = Arc::new(Accounts::new(Arc::new(accounts_db))); + let num_keys = 1000; + let slot = 0; + + let pubkeys: Vec<_> = std::iter::repeat_with(solana_sdk::pubkey::new_rand) + .take(num_keys) + .collect(); + let accounts_data: Vec<_> = std::iter::repeat(Account { + lamports: 1, + ..Default::default() + }) + .take(num_keys) + .collect(); + let storable_accounts: Vec<_> = pubkeys.iter().zip(accounts_data.iter()).collect(); + accounts.store_accounts_cached((slot, storable_accounts.as_slice())); + accounts.add_root(slot); + accounts + .accounts_db + .flush_accounts_cache_slot_for_tests(slot); + + let pubkeys = Arc::new(pubkeys); + for i in 0..num_readers { + let accounts = accounts.clone(); + let pubkeys = pubkeys.clone(); + Builder::new() + .name(format!("reader{i:02}")) + .spawn(move || { + reader_f(&accounts, &pubkeys); + }) + .unwrap(); + } + + let num_new_keys = 1000; + bencher.iter(|| { + let new_pubkeys: Vec<_> = std::iter::repeat_with(solana_sdk::pubkey::new_rand) + .take(num_new_keys) + .collect(); + let new_storable_accounts: Vec<_> = new_pubkeys.iter().zip(accounts_data.iter()).collect(); + // Write to a different slot than the one being read from. Because + // there's a new account pubkey being written to every time, will + // compete for the accounts index lock on every store + accounts.store_accounts_cached((slot + 1, new_storable_accounts.as_slice())); + }); +} + +#[bench] +fn bench_concurrent_read_write(bencher: &mut Bencher) { + store_accounts_with_possible_contention( + "concurrent_read_write", + bencher, + |accounts, pubkeys| { + let mut rng = rand::thread_rng(); + loop { + let i = rng.gen_range(0..pubkeys.len()); + test::black_box( + accounts + .load_without_fixed_root(&Ancestors::default(), &pubkeys[i]) + .unwrap(), + ); + } + }, + ) +} + +#[bench] +fn bench_concurrent_scan_write(bencher: &mut Bencher) { + store_accounts_with_possible_contention("concurrent_scan_write", bencher, |accounts, _| loop { + test::black_box( + accounts + .load_by_program( + &Ancestors::default(), + 0, + AccountSharedData::default().owner(), + &ScanConfig::default(), + ) + .unwrap(), + ); + }) +} + +#[bench] +#[ignore] +fn bench_dashmap_single_reader_with_n_writers(bencher: &mut Bencher) { + let num_readers = 5; + let num_keys = 10000; + let map = Arc::new(DashMap::new()); + for i in 0..num_keys { + map.insert(i, i); + } + for _ in 0..num_readers { + let map = map.clone(); + Builder::new() + .name("readers".to_string()) + .spawn(move || loop { + test::black_box(map.entry(5).or_insert(2)); + }) + .unwrap(); + } + bencher.iter(|| { + for _ in 0..num_keys { + test::black_box(map.get(&5).unwrap().value()); + } + }) +} + +#[bench] +#[ignore] +fn bench_rwlock_hashmap_single_reader_with_n_writers(bencher: &mut Bencher) { + let num_readers = 5; + let num_keys = 10000; + let map = Arc::new(RwLock::new(HashMap::new())); + for i in 0..num_keys { + map.write().unwrap().insert(i, i); + } + for _ in 0..num_readers { + let map = map.clone(); + Builder::new() + .name("readers".to_string()) + .spawn(move || loop { + test::black_box(map.write().unwrap().get(&5)); + }) + .unwrap(); + } + bencher.iter(|| { + for _ in 0..num_keys { + test::black_box(map.read().unwrap().get(&5)); + } + }) +} + +fn setup_bench_dashmap_iter() -> (Arc, DashMap) { + let accounts_db = new_accounts_db(vec![PathBuf::from( + std::env::var("FARF_DIR").unwrap_or_else(|_| "farf".to_string()), + ) + .join("bench_dashmap_par_iter")]); + let accounts = Arc::new(Accounts::new(Arc::new(accounts_db))); + + let dashmap = DashMap::new(); + let num_keys = std::env::var("NUM_BENCH_KEYS") + .map(|num_keys| num_keys.parse::().unwrap()) + .unwrap_or_else(|_| 10000); + for _ in 0..num_keys { + dashmap.insert( + Pubkey::new_unique(), + ( + AccountSharedData::new(1, 0, AccountSharedData::default().owner()), + Hash::new_unique(), + ), + ); + } + + (accounts, dashmap) +} + +#[bench] +fn bench_dashmap_par_iter(bencher: &mut Bencher) { + let (accounts, dashmap) = setup_bench_dashmap_iter(); + + bencher.iter(|| { + test::black_box(accounts.accounts_db.thread_pool.install(|| { + dashmap + .par_iter() + .map(|cached_account| (*cached_account.key(), cached_account.value().1)) + .collect::>() + })); + }); +} + +#[bench] +fn bench_dashmap_iter(bencher: &mut Bencher) { + let (_accounts, dashmap) = setup_bench_dashmap_iter(); + + bencher.iter(|| { + test::black_box( + dashmap + .iter() + .map(|cached_account| (*cached_account.key(), cached_account.value().1)) + .collect::>(), + ); + }); +} + +#[bench] +fn bench_load_largest_accounts(b: &mut Bencher) { + let accounts_db = new_accounts_db(Vec::new()); + let accounts = Accounts::new(Arc::new(accounts_db)); + let mut rng = rand::thread_rng(); + for _ in 0..10_000 { + let lamports = rng.gen(); + let pubkey = Pubkey::new_unique(); + let account = AccountSharedData::new(lamports, 0, &Pubkey::default()); + accounts.store_slow_uncached(0, &pubkey, &account); + } + let ancestors = Ancestors::from(vec![0]); + let bank_id = 0; + b.iter(|| { + accounts.load_largest_accounts( + &ancestors, + bank_id, + 20, + &HashSet::new(), + AccountAddressFilter::Exclude, + ) + }); +} diff --git a/runtime/benches/accounts.rs b/runtime/benches/accounts.rs index b99425b1507cab..9d09b5c3650c97 100644 --- a/runtime/benches/accounts.rs +++ b/runtime/benches/accounts.rs @@ -4,50 +4,19 @@ extern crate test; use { - dashmap::DashMap, - rand::Rng, - rayon::iter::{IntoParallelRefIterator, ParallelIterator}, - solana_accounts_db::{ - accounts::{AccountAddressFilter, Accounts}, - accounts_db::{ - test_utils::create_test_accounts, AccountShrinkThreshold, AccountsDb, - VerifyAccountsHashAndLamportsConfig, ACCOUNTS_DB_CONFIG_FOR_BENCHMARKS, - }, - accounts_index::{AccountSecondaryIndexes, ScanConfig}, - ancestors::Ancestors, - epoch_accounts_hash::EpochAccountsHash, - }, + solana_accounts_db::epoch_accounts_hash::EpochAccountsHash, solana_runtime::bank::*, solana_sdk::{ - account::{Account, AccountSharedData, ReadableAccount}, - genesis_config::{create_genesis_config, ClusterType}, + account::{AccountSharedData, ReadableAccount}, + genesis_config::create_genesis_config, hash::Hash, lamports::LamportsError, pubkey::Pubkey, - rent_collector::RentCollector, - sysvar::epoch_schedule::EpochSchedule, - }, - std::{ - collections::{HashMap, HashSet}, - path::PathBuf, - sync::{Arc, RwLock}, - thread::Builder, }, + std::{path::PathBuf, sync::Arc}, test::Bencher, }; -fn new_accounts_db(account_paths: Vec) -> AccountsDb { - AccountsDb::new_with_config( - account_paths, - &ClusterType::Development, - AccountSecondaryIndexes::default(), - AccountShrinkThreshold::default(), - Some(ACCOUNTS_DB_CONFIG_FOR_BENCHMARKS), - None, - Arc::default(), - ) -} - fn deposit_many(bank: &Bank, pubkeys: &mut Vec, num: usize) -> Result<(), LamportsError> { for t in 0..num { let pubkey = solana_sdk::pubkey::new_rand(); @@ -62,7 +31,7 @@ fn deposit_many(bank: &Bank, pubkeys: &mut Vec, num: usize) -> Result<() } #[bench] -fn test_accounts_create(bencher: &mut Bencher) { +fn bench_accounts_create(bencher: &mut Bencher) { let (genesis_config, _) = create_genesis_config(10_000); let bank0 = Bank::new_with_paths_for_benches(&genesis_config, vec![PathBuf::from("bench_a0")]); bencher.iter(|| { @@ -72,7 +41,7 @@ fn test_accounts_create(bencher: &mut Bencher) { } #[bench] -fn test_accounts_squash(bencher: &mut Bencher) { +fn bench_accounts_squash(bencher: &mut Bencher) { let (mut genesis_config, _) = create_genesis_config(100_000); genesis_config.rent.burn_percent = 100; // Avoid triggering an assert in Bank::distribute_rent_to_validators() let mut prev_bank = Arc::new(Bank::new_with_paths_for_benches( @@ -108,300 +77,3 @@ fn test_accounts_squash(bencher: &mut Bencher) { prev_bank = next_bank; }); } - -#[bench] -fn test_accounts_hash_bank_hash(bencher: &mut Bencher) { - let accounts_db = new_accounts_db(vec![PathBuf::from("bench_accounts_hash_internal")]); - let accounts = Accounts::new(Arc::new(accounts_db)); - let mut pubkeys: Vec = vec![]; - let num_accounts = 60_000; - let slot = 0; - create_test_accounts(&accounts, &mut pubkeys, num_accounts, slot); - let ancestors = Ancestors::from(vec![0]); - let (_, total_lamports) = accounts - .accounts_db - .update_accounts_hash_for_tests(0, &ancestors, false, false); - accounts.add_root(slot); - accounts.accounts_db.flush_accounts_cache(true, Some(slot)); - bencher.iter(|| { - assert!(accounts.verify_accounts_hash_and_lamports( - 0, - total_lamports, - None, - VerifyAccountsHashAndLamportsConfig { - ancestors: &ancestors, - test_hash_calculation: false, - epoch_schedule: &EpochSchedule::default(), - rent_collector: &RentCollector::default(), - ignore_mismatch: false, - store_detailed_debug_info: false, - use_bg_thread_pool: false, - } - )) - }); -} - -#[bench] -fn test_update_accounts_hash(bencher: &mut Bencher) { - solana_logger::setup(); - let accounts_db = new_accounts_db(vec![PathBuf::from("update_accounts_hash")]); - let accounts = Accounts::new(Arc::new(accounts_db)); - let mut pubkeys: Vec = vec![]; - create_test_accounts(&accounts, &mut pubkeys, 50_000, 0); - let ancestors = Ancestors::from(vec![0]); - bencher.iter(|| { - accounts - .accounts_db - .update_accounts_hash_for_tests(0, &ancestors, false, false); - }); -} - -#[bench] -fn test_accounts_delta_hash(bencher: &mut Bencher) { - solana_logger::setup(); - let accounts_db = new_accounts_db(vec![PathBuf::from("accounts_delta_hash")]); - let accounts = Accounts::new(Arc::new(accounts_db)); - let mut pubkeys: Vec = vec![]; - create_test_accounts(&accounts, &mut pubkeys, 100_000, 0); - bencher.iter(|| { - accounts.accounts_db.calculate_accounts_delta_hash(0); - }); -} - -#[bench] -fn bench_delete_dependencies(bencher: &mut Bencher) { - solana_logger::setup(); - let accounts_db = new_accounts_db(vec![PathBuf::from("accounts_delete_deps")]); - let accounts = Accounts::new(Arc::new(accounts_db)); - let mut old_pubkey = Pubkey::default(); - let zero_account = AccountSharedData::new(0, 0, AccountSharedData::default().owner()); - for i in 0..1000 { - let pubkey = solana_sdk::pubkey::new_rand(); - let account = AccountSharedData::new(i + 1, 0, AccountSharedData::default().owner()); - accounts.store_slow_uncached(i, &pubkey, &account); - accounts.store_slow_uncached(i, &old_pubkey, &zero_account); - old_pubkey = pubkey; - accounts.add_root(i); - } - bencher.iter(|| { - accounts.accounts_db.clean_accounts_for_tests(); - }); -} - -fn store_accounts_with_possible_contention( - bench_name: &str, - bencher: &mut Bencher, - reader_f: F, -) where - F: Fn(&Accounts, &[Pubkey]) + Send + Copy, -{ - let num_readers = 5; - let accounts_db = new_accounts_db(vec![PathBuf::from( - std::env::var("FARF_DIR").unwrap_or_else(|_| "farf".to_string()), - ) - .join(bench_name)]); - let accounts = Arc::new(Accounts::new(Arc::new(accounts_db))); - let num_keys = 1000; - let slot = 0; - - let pubkeys: Vec<_> = std::iter::repeat_with(solana_sdk::pubkey::new_rand) - .take(num_keys) - .collect(); - let accounts_data: Vec<_> = std::iter::repeat(Account { - lamports: 1, - ..Default::default() - }) - .take(num_keys) - .collect(); - let storable_accounts: Vec<_> = pubkeys.iter().zip(accounts_data.iter()).collect(); - accounts.store_accounts_cached((slot, storable_accounts.as_slice())); - accounts.add_root(slot); - accounts - .accounts_db - .flush_accounts_cache_slot_for_tests(slot); - - let pubkeys = Arc::new(pubkeys); - for i in 0..num_readers { - let accounts = accounts.clone(); - let pubkeys = pubkeys.clone(); - Builder::new() - .name(format!("reader{i:02}")) - .spawn(move || { - reader_f(&accounts, &pubkeys); - }) - .unwrap(); - } - - let num_new_keys = 1000; - bencher.iter(|| { - let new_pubkeys: Vec<_> = std::iter::repeat_with(solana_sdk::pubkey::new_rand) - .take(num_new_keys) - .collect(); - let new_storable_accounts: Vec<_> = new_pubkeys.iter().zip(accounts_data.iter()).collect(); - // Write to a different slot than the one being read from. Because - // there's a new account pubkey being written to every time, will - // compete for the accounts index lock on every store - accounts.store_accounts_cached((slot + 1, new_storable_accounts.as_slice())); - }); -} - -#[bench] -fn bench_concurrent_read_write(bencher: &mut Bencher) { - store_accounts_with_possible_contention( - "concurrent_read_write", - bencher, - |accounts, pubkeys| { - let mut rng = rand::thread_rng(); - loop { - let i = rng.gen_range(0..pubkeys.len()); - test::black_box( - accounts - .load_without_fixed_root(&Ancestors::default(), &pubkeys[i]) - .unwrap(), - ); - } - }, - ) -} - -#[bench] -fn bench_concurrent_scan_write(bencher: &mut Bencher) { - store_accounts_with_possible_contention("concurrent_scan_write", bencher, |accounts, _| loop { - test::black_box( - accounts - .load_by_program( - &Ancestors::default(), - 0, - AccountSharedData::default().owner(), - &ScanConfig::default(), - ) - .unwrap(), - ); - }) -} - -#[bench] -#[ignore] -fn bench_dashmap_single_reader_with_n_writers(bencher: &mut Bencher) { - let num_readers = 5; - let num_keys = 10000; - let map = Arc::new(DashMap::new()); - for i in 0..num_keys { - map.insert(i, i); - } - for _ in 0..num_readers { - let map = map.clone(); - Builder::new() - .name("readers".to_string()) - .spawn(move || loop { - test::black_box(map.entry(5).or_insert(2)); - }) - .unwrap(); - } - bencher.iter(|| { - for _ in 0..num_keys { - test::black_box(map.get(&5).unwrap().value()); - } - }) -} - -#[bench] -#[ignore] -fn bench_rwlock_hashmap_single_reader_with_n_writers(bencher: &mut Bencher) { - let num_readers = 5; - let num_keys = 10000; - let map = Arc::new(RwLock::new(HashMap::new())); - for i in 0..num_keys { - map.write().unwrap().insert(i, i); - } - for _ in 0..num_readers { - let map = map.clone(); - Builder::new() - .name("readers".to_string()) - .spawn(move || loop { - test::black_box(map.write().unwrap().get(&5)); - }) - .unwrap(); - } - bencher.iter(|| { - for _ in 0..num_keys { - test::black_box(map.read().unwrap().get(&5)); - } - }) -} - -fn setup_bench_dashmap_iter() -> (Arc, DashMap) { - let accounts_db = new_accounts_db(vec![PathBuf::from( - std::env::var("FARF_DIR").unwrap_or_else(|_| "farf".to_string()), - ) - .join("bench_dashmap_par_iter")]); - let accounts = Arc::new(Accounts::new(Arc::new(accounts_db))); - - let dashmap = DashMap::new(); - let num_keys = std::env::var("NUM_BENCH_KEYS") - .map(|num_keys| num_keys.parse::().unwrap()) - .unwrap_or_else(|_| 10000); - for _ in 0..num_keys { - dashmap.insert( - Pubkey::new_unique(), - ( - AccountSharedData::new(1, 0, AccountSharedData::default().owner()), - Hash::new_unique(), - ), - ); - } - - (accounts, dashmap) -} - -#[bench] -fn bench_dashmap_par_iter(bencher: &mut Bencher) { - let (accounts, dashmap) = setup_bench_dashmap_iter(); - - bencher.iter(|| { - test::black_box(accounts.accounts_db.thread_pool.install(|| { - dashmap - .par_iter() - .map(|cached_account| (*cached_account.key(), cached_account.value().1)) - .collect::>() - })); - }); -} - -#[bench] -fn bench_dashmap_iter(bencher: &mut Bencher) { - let (_accounts, dashmap) = setup_bench_dashmap_iter(); - - bencher.iter(|| { - test::black_box( - dashmap - .iter() - .map(|cached_account| (*cached_account.key(), cached_account.value().1)) - .collect::>(), - ); - }); -} - -#[bench] -fn bench_load_largest_accounts(b: &mut Bencher) { - let accounts_db = new_accounts_db(Vec::new()); - let accounts = Accounts::new(Arc::new(accounts_db)); - let mut rng = rand::thread_rng(); - for _ in 0..10_000 { - let lamports = rng.gen(); - let pubkey = Pubkey::new_unique(); - let account = AccountSharedData::new(lamports, 0, &Pubkey::default()); - accounts.store_slow_uncached(0, &pubkey, &account); - } - let ancestors = Ancestors::from(vec![0]); - let bank_id = 0; - b.iter(|| { - accounts.load_largest_accounts( - &ancestors, - bank_id, - 20, - &HashSet::new(), - AccountAddressFilter::Exclude, - ) - }); -} From 00c984fe4decfd8da1f08d6d756f3c98a3827acc Mon Sep 17 00:00:00 2001 From: Greg Cusack Date: Mon, 11 Mar 2024 13:13:56 -0400 Subject: [PATCH 43/84] deprecate `get_client` and `get_multi_client` (#177) deprecate get_client and get_multi_client --- dos/src/main.rs | 1 + gossip/src/gossip_service.rs | 2 ++ 2 files changed, 3 insertions(+) diff --git a/dos/src/main.rs b/dos/src/main.rs index 8e6c3c5b2b11b5..b9e0dceba40bf0 100644 --- a/dos/src/main.rs +++ b/dos/src/main.rs @@ -39,6 +39,7 @@ //! ``` //! #![allow(clippy::arithmetic_side_effects)] +#![allow(deprecated)] use { crossbeam_channel::{select, tick, unbounded, Receiver, Sender}, itertools::Itertools, diff --git a/gossip/src/gossip_service.rs b/gossip/src/gossip_service.rs index 806ee23a4fb0be..404a685aa75567 100644 --- a/gossip/src/gossip_service.rs +++ b/gossip/src/gossip_service.rs @@ -194,6 +194,7 @@ pub fn discover( } /// Creates a ThinClient by selecting a valid node at random +#[deprecated(since = "1.18.0", note = "Interface will change")] pub fn get_client( nodes: &[ContactInfo], socket_addr_space: &SocketAddrSpace, @@ -209,6 +210,7 @@ pub fn get_client( ThinClient::new(rpc, tpu, connection_cache) } +#[deprecated(since = "1.18.0", note = "Will be removed in favor of get_client")] pub fn get_multi_client( nodes: &[ContactInfo], socket_addr_space: &SocketAddrSpace, From 158c4e05d5e60c5f5b3dc8e47f5f60a90beca6e1 Mon Sep 17 00:00:00 2001 From: "Jeff Washington (jwash)" Date: Mon, 11 Mar 2024 12:21:51 -0500 Subject: [PATCH 44/84] remove dead code (#176) --- accounts-db/src/accounts_partition.rs | 30 --------------------------- runtime/src/bank/tests.rs | 28 ++++--------------------- 2 files changed, 4 insertions(+), 54 deletions(-) diff --git a/accounts-db/src/accounts_partition.rs b/accounts-db/src/accounts_partition.rs index 05d3993adcfb70..01d6929c3e07e5 100644 --- a/accounts-db/src/accounts_partition.rs +++ b/accounts-db/src/accounts_partition.rs @@ -98,36 +98,6 @@ pub fn get_partition_from_slot_indexes( (start_partition_index, end_partition_index, partition_count) } -/// used only by filler accounts in debug path -/// previous means slot - 1, not parent -// These functions/fields are only usable from a dev context (i.e. tests and benches) -#[cfg(feature = "dev-context-only-utils")] -pub fn variable_cycle_partition_from_previous_slot( - epoch_schedule: &EpochSchedule, - slot: Slot, -) -> Partition { - // similar code to Bank::variable_cycle_partitions - let (current_epoch, current_slot_index) = epoch_schedule.get_epoch_and_slot_index(slot); - let (parent_epoch, mut parent_slot_index) = - epoch_schedule.get_epoch_and_slot_index(slot.saturating_sub(1)); - let cycle_params = rent_single_epoch_collection_cycle_params( - current_epoch, - epoch_schedule.get_slots_in_epoch(current_epoch), - ); - - if parent_epoch < current_epoch { - parent_slot_index = 0; - } - - let generated_for_gapped_epochs = false; - get_partition_from_slot_indexes( - cycle_params, - parent_slot_index, - current_slot_index, - generated_for_gapped_epochs, - ) -} - /// return all end partition indexes for the given partition /// partition could be (0, 1, N). In this case we only return [1] /// the single 'end_index' that covers this partition. diff --git a/runtime/src/bank/tests.rs b/runtime/src/bank/tests.rs index f9b846d85b1512..29dbdc2e5aeacd 100644 --- a/runtime/src/bank/tests.rs +++ b/runtime/src/bank/tests.rs @@ -1279,26 +1279,6 @@ fn test_rent_complex() { assert_eq!(bank.collected_rent.load(Relaxed), rent_collected); } -fn test_rent_collection_partitions(bank: &Bank) -> Vec { - let partitions = bank.rent_collection_partitions(); - let slot = bank.slot(); - if slot.saturating_sub(1) == bank.parent_slot() { - let partition = accounts_partition::variable_cycle_partition_from_previous_slot( - bank.epoch_schedule(), - bank.slot(), - ); - assert_eq!( - partitions.last().unwrap(), - &partition, - "slot: {}, slots per epoch: {}, partitions: {:?}", - bank.slot(), - bank.epoch_schedule().slots_per_epoch, - partitions - ); - } - partitions -} - #[test] fn test_rent_eager_across_epoch_without_gap() { let mut bank = create_simple_test_arc_bank(1).0; @@ -1321,16 +1301,16 @@ fn test_rent_eager_across_epoch_without_gap_mnb() { genesis_config.cluster_type = ClusterType::MainnetBeta; let mut bank = Arc::new(Bank::new_for_tests(&genesis_config)); - assert_eq!(test_rent_collection_partitions(&bank), vec![(0, 0, 32)]); + assert_eq!(bank.rent_collection_partitions(), vec![(0, 0, 32)]); bank = Arc::new(new_from_parent(bank)); - assert_eq!(test_rent_collection_partitions(&bank), vec![(0, 1, 32)]); + assert_eq!(bank.rent_collection_partitions(), vec![(0, 1, 32)]); for _ in 2..32 { bank = Arc::new(new_from_parent(bank)); } - assert_eq!(test_rent_collection_partitions(&bank), vec![(30, 31, 32)]); + assert_eq!(bank.rent_collection_partitions(), vec![(30, 31, 32)]); bank = Arc::new(new_from_parent(bank)); - assert_eq!(test_rent_collection_partitions(&bank), vec![(0, 0, 64)]); + assert_eq!(bank.rent_collection_partitions(), vec![(0, 0, 64)]); } #[test] From 53e7b9ac474f046ef5ea4b1725f663209c2e5139 Mon Sep 17 00:00:00 2001 From: Tyera Date: Mon, 11 Mar 2024 12:11:22 -0600 Subject: [PATCH 45/84] Version bump v2.0.0 (#121) * Put solana-svm in alphabetical order * Update version to 2.0.0. --------- Co-authored-by: Will Hickey --- Cargo.lock | 228 ++++++++-------- Cargo.toml | 162 ++++++------ programs/sbf/Cargo.lock | 250 +++++++++--------- programs/sbf/Cargo.toml | 50 ++-- .../tests/crates/fail/Cargo.toml | 4 +- .../tests/crates/noop/Cargo.toml | 4 +- 6 files changed, 349 insertions(+), 349 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 19b265863eba47..88f0fa0925dcac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -64,7 +64,7 @@ dependencies = [ [[package]] name = "agave-cargo-registry" -version = "1.19.0" +version = "2.0.0" dependencies = [ "clap 2.33.3", "flate2", @@ -93,7 +93,7 @@ dependencies = [ [[package]] name = "agave-geyser-plugin-interface" -version = "1.19.0" +version = "2.0.0" dependencies = [ "log", "solana-sdk", @@ -103,7 +103,7 @@ dependencies = [ [[package]] name = "agave-install" -version = "1.19.0" +version = "2.0.0" dependencies = [ "atty", "bincode", @@ -138,7 +138,7 @@ dependencies = [ [[package]] name = "agave-ledger-tool" -version = "1.19.0" +version = "2.0.0" dependencies = [ "assert_cmd", "bs58", @@ -189,7 +189,7 @@ dependencies = [ [[package]] name = "agave-validator" -version = "1.19.0" +version = "2.0.0" dependencies = [ "agave-geyser-plugin-interface", "chrono", @@ -255,7 +255,7 @@ dependencies = [ [[package]] name = "agave-watchtower" -version = "1.19.0" +version = "2.0.0" dependencies = [ "clap 2.33.3", "humantime", @@ -2366,7 +2366,7 @@ dependencies = [ [[package]] name = "gen-headers" -version = "1.19.0" +version = "2.0.0" dependencies = [ "log", "regex", @@ -2374,7 +2374,7 @@ dependencies = [ [[package]] name = "gen-syscall-list" -version = "1.19.0" +version = "2.0.0" dependencies = [ "regex", ] @@ -4263,7 +4263,7 @@ dependencies = [ [[package]] name = "proto" -version = "1.19.0" +version = "2.0.0" dependencies = [ "protobuf-src", "tonic-build", @@ -4506,7 +4506,7 @@ dependencies = [ [[package]] name = "rbpf-cli" -version = "1.19.0" +version = "2.0.0" [[package]] name = "rdrand" @@ -5310,7 +5310,7 @@ dependencies = [ [[package]] name = "solana-account-decoder" -version = "1.19.0" +version = "2.0.0" dependencies = [ "Inflector", "assert_matches", @@ -5335,7 +5335,7 @@ dependencies = [ [[package]] name = "solana-accounts-bench" -version = "1.19.0" +version = "2.0.0" dependencies = [ "clap 2.33.3", "log", @@ -5349,7 +5349,7 @@ dependencies = [ [[package]] name = "solana-accounts-cluster-bench" -version = "1.19.0" +version = "2.0.0" dependencies = [ "clap 2.33.3", "log", @@ -5379,7 +5379,7 @@ dependencies = [ [[package]] name = "solana-accounts-db" -version = "1.19.0" +version = "2.0.0" dependencies = [ "arrayref", "assert_matches", @@ -5448,7 +5448,7 @@ dependencies = [ [[package]] name = "solana-address-lookup-table-program" -version = "1.19.0" +version = "2.0.0" dependencies = [ "bincode", "bytemuck", @@ -5467,7 +5467,7 @@ dependencies = [ [[package]] name = "solana-address-lookup-table-program-tests" -version = "1.19.0" +version = "2.0.0" dependencies = [ "assert_matches", "bincode", @@ -5478,7 +5478,7 @@ dependencies = [ [[package]] name = "solana-banking-bench" -version = "1.19.0" +version = "2.0.0" dependencies = [ "clap 3.2.23", "crossbeam-channel", @@ -5502,7 +5502,7 @@ dependencies = [ [[package]] name = "solana-banks-client" -version = "1.19.0" +version = "2.0.0" dependencies = [ "borsh 1.2.1", "futures 0.3.30", @@ -5519,7 +5519,7 @@ dependencies = [ [[package]] name = "solana-banks-interface" -version = "1.19.0" +version = "2.0.0" dependencies = [ "serde", "solana-sdk", @@ -5528,7 +5528,7 @@ dependencies = [ [[package]] name = "solana-banks-server" -version = "1.19.0" +version = "2.0.0" dependencies = [ "bincode", "crossbeam-channel", @@ -5546,7 +5546,7 @@ dependencies = [ [[package]] name = "solana-bench-streamer" -version = "1.19.0" +version = "2.0.0" dependencies = [ "clap 3.2.23", "crossbeam-channel", @@ -5557,7 +5557,7 @@ dependencies = [ [[package]] name = "solana-bench-tps" -version = "1.19.0" +version = "2.0.0" dependencies = [ "clap 2.33.3", "crossbeam-channel", @@ -5599,7 +5599,7 @@ dependencies = [ [[package]] name = "solana-bloom" -version = "1.19.0" +version = "2.0.0" dependencies = [ "bv", "fnv", @@ -5616,7 +5616,7 @@ dependencies = [ [[package]] name = "solana-bpf-loader-program" -version = "1.19.0" +version = "2.0.0" dependencies = [ "assert_matches", "bincode", @@ -5637,7 +5637,7 @@ dependencies = [ [[package]] name = "solana-bpf-loader-program-tests" -version = "1.19.0" +version = "2.0.0" dependencies = [ "assert_matches", "bincode", @@ -5648,7 +5648,7 @@ dependencies = [ [[package]] name = "solana-bucket-map" -version = "1.19.0" +version = "2.0.0" dependencies = [ "bv", "bytemuck", @@ -5667,7 +5667,7 @@ dependencies = [ [[package]] name = "solana-cargo-build-bpf" -version = "1.19.0" +version = "2.0.0" dependencies = [ "log", "solana-logger", @@ -5675,7 +5675,7 @@ dependencies = [ [[package]] name = "solana-cargo-build-sbf" -version = "1.19.0" +version = "2.0.0" dependencies = [ "assert_cmd", "bzip2", @@ -5696,11 +5696,11 @@ dependencies = [ [[package]] name = "solana-cargo-test-bpf" -version = "1.19.0" +version = "2.0.0" [[package]] name = "solana-cargo-test-sbf" -version = "1.19.0" +version = "2.0.0" dependencies = [ "cargo_metadata", "clap 3.2.23", @@ -5711,7 +5711,7 @@ dependencies = [ [[package]] name = "solana-clap-utils" -version = "1.19.0" +version = "2.0.0" dependencies = [ "assert_matches", "chrono", @@ -5728,7 +5728,7 @@ dependencies = [ [[package]] name = "solana-clap-v3-utils" -version = "1.19.0" +version = "2.0.0" dependencies = [ "assert_matches", "chrono", @@ -5746,7 +5746,7 @@ dependencies = [ [[package]] name = "solana-cli" -version = "1.19.0" +version = "2.0.0" dependencies = [ "assert_matches", "bincode", @@ -5800,7 +5800,7 @@ dependencies = [ [[package]] name = "solana-cli-config" -version = "1.19.0" +version = "2.0.0" dependencies = [ "anyhow", "dirs-next", @@ -5815,7 +5815,7 @@ dependencies = [ [[package]] name = "solana-cli-output" -version = "1.19.0" +version = "2.0.0" dependencies = [ "Inflector", "base64 0.21.7", @@ -5841,7 +5841,7 @@ dependencies = [ [[package]] name = "solana-client" -version = "1.19.0" +version = "2.0.0" dependencies = [ "async-trait", "bincode", @@ -5873,7 +5873,7 @@ dependencies = [ [[package]] name = "solana-client-test" -version = "1.19.0" +version = "2.0.0" dependencies = [ "futures-util", "rand 0.8.5", @@ -5903,7 +5903,7 @@ dependencies = [ [[package]] name = "solana-compute-budget-program" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program-runtime", "solana-sdk", @@ -5911,7 +5911,7 @@ dependencies = [ [[package]] name = "solana-config-program" -version = "1.19.0" +version = "2.0.0" dependencies = [ "bincode", "chrono", @@ -5924,7 +5924,7 @@ dependencies = [ [[package]] name = "solana-connection-cache" -version = "1.19.0" +version = "2.0.0" dependencies = [ "async-trait", "bincode", @@ -5947,7 +5947,7 @@ dependencies = [ [[package]] name = "solana-core" -version = "1.19.0" +version = "2.0.0" dependencies = [ "assert_matches", "base64 0.21.7", @@ -6033,7 +6033,7 @@ dependencies = [ [[package]] name = "solana-cost-model" -version = "1.19.0" +version = "2.0.0" dependencies = [ "lazy_static", "log", @@ -6058,7 +6058,7 @@ dependencies = [ [[package]] name = "solana-dos" -version = "1.19.0" +version = "2.0.0" dependencies = [ "bincode", "clap 3.2.23", @@ -6088,7 +6088,7 @@ dependencies = [ [[package]] name = "solana-download-utils" -version = "1.19.0" +version = "2.0.0" dependencies = [ "console", "indicatif", @@ -6100,7 +6100,7 @@ dependencies = [ [[package]] name = "solana-ed25519-program-tests" -version = "1.19.0" +version = "2.0.0" dependencies = [ "assert_matches", "ed25519-dalek", @@ -6111,7 +6111,7 @@ dependencies = [ [[package]] name = "solana-entry" -version = "1.19.0" +version = "2.0.0" dependencies = [ "assert_matches", "bincode", @@ -6133,7 +6133,7 @@ dependencies = [ [[package]] name = "solana-faucet" -version = "1.19.0" +version = "2.0.0" dependencies = [ "bincode", "byteorder", @@ -6155,7 +6155,7 @@ dependencies = [ [[package]] name = "solana-frozen-abi" -version = "1.19.0" +version = "2.0.0" dependencies = [ "bitflags 2.4.2", "block-buffer 0.10.4", @@ -6179,7 +6179,7 @@ dependencies = [ [[package]] name = "solana-frozen-abi-macro" -version = "1.19.0" +version = "2.0.0" dependencies = [ "proc-macro2", "quote", @@ -6189,7 +6189,7 @@ dependencies = [ [[package]] name = "solana-genesis" -version = "1.19.0" +version = "2.0.0" dependencies = [ "base64 0.21.7", "bincode", @@ -6214,7 +6214,7 @@ dependencies = [ [[package]] name = "solana-genesis-utils" -version = "1.19.0" +version = "2.0.0" dependencies = [ "log", "solana-accounts-db", @@ -6225,7 +6225,7 @@ dependencies = [ [[package]] name = "solana-geyser-plugin-manager" -version = "1.19.0" +version = "2.0.0" dependencies = [ "agave-geyser-plugin-interface", "bs58", @@ -6250,7 +6250,7 @@ dependencies = [ [[package]] name = "solana-gossip" -version = "1.19.0" +version = "2.0.0" dependencies = [ "assert_matches", "bincode", @@ -6301,7 +6301,7 @@ dependencies = [ [[package]] name = "solana-keygen" -version = "1.19.0" +version = "2.0.0" dependencies = [ "bs58", "clap 3.2.23", @@ -6318,7 +6318,7 @@ dependencies = [ [[package]] name = "solana-ledger" -version = "1.19.0" +version = "2.0.0" dependencies = [ "assert_matches", "bincode", @@ -6388,7 +6388,7 @@ dependencies = [ [[package]] name = "solana-loader-v4-program" -version = "1.19.0" +version = "2.0.0" dependencies = [ "bincode", "log", @@ -6400,7 +6400,7 @@ dependencies = [ [[package]] name = "solana-local-cluster" -version = "1.19.0" +version = "2.0.0" dependencies = [ "assert_matches", "crossbeam-channel", @@ -6439,7 +6439,7 @@ dependencies = [ [[package]] name = "solana-log-analyzer" -version = "1.19.0" +version = "2.0.0" dependencies = [ "byte-unit", "clap 3.2.23", @@ -6451,7 +6451,7 @@ dependencies = [ [[package]] name = "solana-logger" -version = "1.19.0" +version = "2.0.0" dependencies = [ "env_logger", "lazy_static", @@ -6460,7 +6460,7 @@ dependencies = [ [[package]] name = "solana-measure" -version = "1.19.0" +version = "2.0.0" dependencies = [ "log", "solana-sdk", @@ -6468,11 +6468,11 @@ dependencies = [ [[package]] name = "solana-memory-management" -version = "1.19.0" +version = "2.0.0" [[package]] name = "solana-merkle-root-bench" -version = "1.19.0" +version = "2.0.0" dependencies = [ "clap 2.33.3", "log", @@ -6485,7 +6485,7 @@ dependencies = [ [[package]] name = "solana-merkle-tree" -version = "1.19.0" +version = "2.0.0" dependencies = [ "fast-math", "hex", @@ -6494,7 +6494,7 @@ dependencies = [ [[package]] name = "solana-metrics" -version = "1.19.0" +version = "2.0.0" dependencies = [ "crossbeam-channel", "env_logger", @@ -6510,7 +6510,7 @@ dependencies = [ [[package]] name = "solana-net-shaper" -version = "1.19.0" +version = "2.0.0" dependencies = [ "clap 3.2.23", "rand 0.8.5", @@ -6521,7 +6521,7 @@ dependencies = [ [[package]] name = "solana-net-utils" -version = "1.19.0" +version = "2.0.0" dependencies = [ "bincode", "clap 3.2.23", @@ -6547,7 +6547,7 @@ checksum = "8b8a731ed60e89177c8a7ab05fe0f1511cedd3e70e773f288f9de33a9cfdc21e" [[package]] name = "solana-notifier" -version = "1.19.0" +version = "2.0.0" dependencies = [ "log", "reqwest", @@ -6557,7 +6557,7 @@ dependencies = [ [[package]] name = "solana-perf" -version = "1.19.0" +version = "2.0.0" dependencies = [ "ahash 0.8.10", "assert_matches", @@ -6588,7 +6588,7 @@ dependencies = [ [[package]] name = "solana-poh" -version = "1.19.0" +version = "2.0.0" dependencies = [ "assert_matches", "bincode", @@ -6610,7 +6610,7 @@ dependencies = [ [[package]] name = "solana-poh-bench" -version = "1.19.0" +version = "2.0.0" dependencies = [ "clap 3.2.23", "log", @@ -6625,7 +6625,7 @@ dependencies = [ [[package]] name = "solana-program" -version = "1.19.0" +version = "2.0.0" dependencies = [ "anyhow", "arbitrary", @@ -6684,7 +6684,7 @@ dependencies = [ [[package]] name = "solana-program-runtime" -version = "1.19.0" +version = "2.0.0" dependencies = [ "assert_matches", "base64 0.21.7", @@ -6714,7 +6714,7 @@ dependencies = [ [[package]] name = "solana-program-test" -version = "1.19.0" +version = "2.0.0" dependencies = [ "assert_matches", "async-trait", @@ -6744,7 +6744,7 @@ dependencies = [ [[package]] name = "solana-pubsub-client" -version = "1.19.0" +version = "2.0.0" dependencies = [ "anyhow", "crossbeam-channel", @@ -6768,7 +6768,7 @@ dependencies = [ [[package]] name = "solana-quic-client" -version = "1.19.0" +version = "2.0.0" dependencies = [ "async-mutex", "async-trait", @@ -6795,7 +6795,7 @@ dependencies = [ [[package]] name = "solana-rayon-threadlimit" -version = "1.19.0" +version = "2.0.0" dependencies = [ "lazy_static", "num_cpus", @@ -6803,7 +6803,7 @@ dependencies = [ [[package]] name = "solana-remote-wallet" -version = "1.19.0" +version = "2.0.0" dependencies = [ "assert_matches", "console", @@ -6822,7 +6822,7 @@ dependencies = [ [[package]] name = "solana-rpc" -version = "1.19.0" +version = "2.0.0" dependencies = [ "base64 0.21.7", "bincode", @@ -6882,7 +6882,7 @@ dependencies = [ [[package]] name = "solana-rpc-client" -version = "1.19.0" +version = "2.0.0" dependencies = [ "assert_matches", "async-trait", @@ -6911,7 +6911,7 @@ dependencies = [ [[package]] name = "solana-rpc-client-api" -version = "1.19.0" +version = "2.0.0" dependencies = [ "base64 0.21.7", "bs58", @@ -6931,7 +6931,7 @@ dependencies = [ [[package]] name = "solana-rpc-client-nonce-utils" -version = "1.19.0" +version = "2.0.0" dependencies = [ "anyhow", "clap 2.33.3", @@ -6948,7 +6948,7 @@ dependencies = [ [[package]] name = "solana-rpc-test" -version = "1.19.0" +version = "2.0.0" dependencies = [ "bincode", "bs58", @@ -6975,7 +6975,7 @@ dependencies = [ [[package]] name = "solana-runtime" -version = "1.19.0" +version = "2.0.0" dependencies = [ "aquamarine", "arrayref", @@ -7058,7 +7058,7 @@ dependencies = [ [[package]] name = "solana-runtime-transaction" -version = "1.19.0" +version = "2.0.0" dependencies = [ "bincode", "log", @@ -7072,7 +7072,7 @@ dependencies = [ [[package]] name = "solana-sdk" -version = "1.19.0" +version = "2.0.0" dependencies = [ "anyhow", "assert_matches", @@ -7131,7 +7131,7 @@ dependencies = [ [[package]] name = "solana-sdk-macro" -version = "1.19.0" +version = "2.0.0" dependencies = [ "bs58", "proc-macro2", @@ -7148,7 +7148,7 @@ checksum = "468aa43b7edb1f9b7b7b686d5c3aeb6630dc1708e86e31343499dd5c4d775183" [[package]] name = "solana-send-transaction-service" -version = "1.19.0" +version = "2.0.0" dependencies = [ "crossbeam-channel", "log", @@ -7163,7 +7163,7 @@ dependencies = [ [[package]] name = "solana-stake-accounts" -version = "1.19.0" +version = "2.0.0" dependencies = [ "clap 2.33.3", "solana-clap-utils", @@ -7179,7 +7179,7 @@ dependencies = [ [[package]] name = "solana-stake-program" -version = "1.19.0" +version = "2.0.0" dependencies = [ "assert_matches", "bincode", @@ -7196,7 +7196,7 @@ dependencies = [ [[package]] name = "solana-storage-bigtable" -version = "1.19.0" +version = "2.0.0" dependencies = [ "backoff", "bincode", @@ -7228,7 +7228,7 @@ dependencies = [ [[package]] name = "solana-storage-proto" -version = "1.19.0" +version = "2.0.0" dependencies = [ "bincode", "bs58", @@ -7244,7 +7244,7 @@ dependencies = [ [[package]] name = "solana-store-tool" -version = "1.19.0" +version = "2.0.0" dependencies = [ "clap 2.33.3", "log", @@ -7256,7 +7256,7 @@ dependencies = [ [[package]] name = "solana-streamer" -version = "1.19.0" +version = "2.0.0" dependencies = [ "assert_matches", "async-channel", @@ -7286,7 +7286,7 @@ dependencies = [ [[package]] name = "solana-svm" -version = "1.19.0" +version = "2.0.0" dependencies = [ "bincode", "itertools", @@ -7307,7 +7307,7 @@ dependencies = [ [[package]] name = "solana-system-program" -version = "1.19.0" +version = "2.0.0" dependencies = [ "assert_matches", "bincode", @@ -7321,7 +7321,7 @@ dependencies = [ [[package]] name = "solana-test-validator" -version = "1.19.0" +version = "2.0.0" dependencies = [ "base64 0.21.7", "bincode", @@ -7351,7 +7351,7 @@ dependencies = [ [[package]] name = "solana-thin-client" -version = "1.19.0" +version = "2.0.0" dependencies = [ "bincode", "log", @@ -7365,7 +7365,7 @@ dependencies = [ [[package]] name = "solana-tokens" -version = "1.19.0" +version = "2.0.0" dependencies = [ "assert_matches", "bincode", @@ -7398,7 +7398,7 @@ dependencies = [ [[package]] name = "solana-tpu-client" -version = "1.19.0" +version = "2.0.0" dependencies = [ "async-trait", "bincode", @@ -7420,7 +7420,7 @@ dependencies = [ [[package]] name = "solana-transaction-dos" -version = "1.19.0" +version = "2.0.0" dependencies = [ "bincode", "clap 2.33.3", @@ -7447,7 +7447,7 @@ dependencies = [ [[package]] name = "solana-transaction-status" -version = "1.19.0" +version = "2.0.0" dependencies = [ "Inflector", "base64 0.21.7", @@ -7470,7 +7470,7 @@ dependencies = [ [[package]] name = "solana-turbine" -version = "1.19.0" +version = "2.0.0" dependencies = [ "assert_matches", "bincode", @@ -7507,7 +7507,7 @@ dependencies = [ [[package]] name = "solana-udp-client" -version = "1.19.0" +version = "2.0.0" dependencies = [ "async-trait", "solana-connection-cache", @@ -7520,14 +7520,14 @@ dependencies = [ [[package]] name = "solana-unified-scheduler-logic" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-sdk", ] [[package]] name = "solana-unified-scheduler-pool" -version = "1.19.0" +version = "2.0.0" dependencies = [ "assert_matches", "crossbeam-channel", @@ -7544,7 +7544,7 @@ dependencies = [ [[package]] name = "solana-upload-perf" -version = "1.19.0" +version = "2.0.0" dependencies = [ "serde_json", "solana-metrics", @@ -7552,7 +7552,7 @@ dependencies = [ [[package]] name = "solana-version" -version = "1.19.0" +version = "2.0.0" dependencies = [ "log", "rustc_version 0.4.0", @@ -7566,7 +7566,7 @@ dependencies = [ [[package]] name = "solana-vote" -version = "1.19.0" +version = "2.0.0" dependencies = [ "bincode", "crossbeam-channel", @@ -7585,7 +7585,7 @@ dependencies = [ [[package]] name = "solana-vote-program" -version = "1.19.0" +version = "2.0.0" dependencies = [ "assert_matches", "bincode", @@ -7608,7 +7608,7 @@ dependencies = [ [[package]] name = "solana-wen-restart" -version = "1.19.0" +version = "2.0.0" dependencies = [ "anyhow", "assert_matches", @@ -7635,7 +7635,7 @@ dependencies = [ [[package]] name = "solana-zk-keygen" -version = "1.19.0" +version = "2.0.0" dependencies = [ "bs58", "clap 3.2.23", @@ -7654,7 +7654,7 @@ dependencies = [ [[package]] name = "solana-zk-token-proof-program" -version = "1.19.0" +version = "2.0.0" dependencies = [ "bytemuck", "criterion", @@ -7668,7 +7668,7 @@ dependencies = [ [[package]] name = "solana-zk-token-proof-program-tests" -version = "1.19.0" +version = "2.0.0" dependencies = [ "bytemuck", "curve25519-dalek", @@ -7680,7 +7680,7 @@ dependencies = [ [[package]] name = "solana-zk-token-sdk" -version = "1.19.0" +version = "2.0.0" dependencies = [ "aes-gcm-siv", "base64 0.21.7", diff --git a/Cargo.toml b/Cargo.toml index 16786e925c34b4..453408a53b956e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -127,7 +127,7 @@ exclude = ["programs/sbf"] resolver = "2" [workspace.package] -version = "1.19.0" +version = "2.0.0" authors = ["Solana Labs Maintainers "] repository = "https://github.com/solana-labs/solana" homepage = "https://solanalabs.com/" @@ -306,87 +306,87 @@ smallvec = "1.13.1" smpl_jwt = "0.7.1" socket2 = "0.5.6" soketto = "0.7" -solana-account-decoder = { path = "account-decoder", version = "=1.19.0" } -solana-accounts-db = { path = "accounts-db", version = "=1.19.0" } -solana-address-lookup-table-program = { path = "programs/address-lookup-table", version = "=1.19.0" } -solana-banks-client = { path = "banks-client", version = "=1.19.0" } -solana-banks-interface = { path = "banks-interface", version = "=1.19.0" } -solana-banks-server = { path = "banks-server", version = "=1.19.0" } -solana-bench-tps = { path = "bench-tps", version = "=1.19.0" } -solana-bloom = { path = "bloom", version = "=1.19.0" } -solana-bpf-loader-program = { path = "programs/bpf_loader", version = "=1.19.0" } -solana-bucket-map = { path = "bucket_map", version = "=1.19.0" } -agave-cargo-registry = { path = "cargo-registry", version = "=1.19.0" } -solana-clap-utils = { path = "clap-utils", version = "=1.19.0" } -solana-clap-v3-utils = { path = "clap-v3-utils", version = "=1.19.0" } -solana-cli = { path = "cli", version = "=1.19.0" } -solana-cli-config = { path = "cli-config", version = "=1.19.0" } -solana-cli-output = { path = "cli-output", version = "=1.19.0" } -solana-client = { path = "client", version = "=1.19.0" } -solana-compute-budget-program = { path = "programs/compute-budget", version = "=1.19.0" } -solana-config-program = { path = "programs/config", version = "=1.19.0" } -solana-connection-cache = { path = "connection-cache", version = "=1.19.0", default-features = false } -solana-core = { path = "core", version = "=1.19.0" } -solana-cost-model = { path = "cost-model", version = "=1.19.0" } -solana-download-utils = { path = "download-utils", version = "=1.19.0" } -solana-entry = { path = "entry", version = "=1.19.0" } -solana-faucet = { path = "faucet", version = "=1.19.0" } -solana-frozen-abi = { path = "frozen-abi", version = "=1.19.0" } -solana-frozen-abi-macro = { path = "frozen-abi/macro", version = "=1.19.0" } -solana-genesis = { path = "genesis", version = "=1.19.0" } -solana-genesis-utils = { path = "genesis-utils", version = "=1.19.0" } -agave-geyser-plugin-interface = { path = "geyser-plugin-interface", version = "=1.19.0" } -solana-geyser-plugin-manager = { path = "geyser-plugin-manager", version = "=1.19.0" } -solana-gossip = { path = "gossip", version = "=1.19.0" } -solana-ledger = { path = "ledger", version = "=1.19.0" } -solana-loader-v4-program = { path = "programs/loader-v4", version = "=1.19.0" } -solana-local-cluster = { path = "local-cluster", version = "=1.19.0" } -solana-logger = { path = "logger", version = "=1.19.0" } -solana-measure = { path = "measure", version = "=1.19.0" } -solana-merkle-tree = { path = "merkle-tree", version = "=1.19.0" } -solana-metrics = { path = "metrics", version = "=1.19.0" } -solana-net-utils = { path = "net-utils", version = "=1.19.0" } +solana-account-decoder = { path = "account-decoder", version = "=2.0.0" } +solana-accounts-db = { path = "accounts-db", version = "=2.0.0" } +solana-address-lookup-table-program = { path = "programs/address-lookup-table", version = "=2.0.0" } +solana-banks-client = { path = "banks-client", version = "=2.0.0" } +solana-banks-interface = { path = "banks-interface", version = "=2.0.0" } +solana-banks-server = { path = "banks-server", version = "=2.0.0" } +solana-bench-tps = { path = "bench-tps", version = "=2.0.0" } +solana-bloom = { path = "bloom", version = "=2.0.0" } +solana-bpf-loader-program = { path = "programs/bpf_loader", version = "=2.0.0" } +solana-bucket-map = { path = "bucket_map", version = "=2.0.0" } +agave-cargo-registry = { path = "cargo-registry", version = "=2.0.0" } +solana-clap-utils = { path = "clap-utils", version = "=2.0.0" } +solana-clap-v3-utils = { path = "clap-v3-utils", version = "=2.0.0" } +solana-cli = { path = "cli", version = "=2.0.0" } +solana-cli-config = { path = "cli-config", version = "=2.0.0" } +solana-cli-output = { path = "cli-output", version = "=2.0.0" } +solana-client = { path = "client", version = "=2.0.0" } +solana-compute-budget-program = { path = "programs/compute-budget", version = "=2.0.0" } +solana-config-program = { path = "programs/config", version = "=2.0.0" } +solana-connection-cache = { path = "connection-cache", version = "=2.0.0", default-features = false } +solana-core = { path = "core", version = "=2.0.0" } +solana-cost-model = { path = "cost-model", version = "=2.0.0" } +solana-download-utils = { path = "download-utils", version = "=2.0.0" } +solana-entry = { path = "entry", version = "=2.0.0" } +solana-faucet = { path = "faucet", version = "=2.0.0" } +solana-frozen-abi = { path = "frozen-abi", version = "=2.0.0" } +solana-frozen-abi-macro = { path = "frozen-abi/macro", version = "=2.0.0" } +solana-genesis = { path = "genesis", version = "=2.0.0" } +solana-genesis-utils = { path = "genesis-utils", version = "=2.0.0" } +agave-geyser-plugin-interface = { path = "geyser-plugin-interface", version = "=2.0.0" } +solana-geyser-plugin-manager = { path = "geyser-plugin-manager", version = "=2.0.0" } +solana-gossip = { path = "gossip", version = "=2.0.0" } +solana-ledger = { path = "ledger", version = "=2.0.0" } +solana-loader-v4-program = { path = "programs/loader-v4", version = "=2.0.0" } +solana-local-cluster = { path = "local-cluster", version = "=2.0.0" } +solana-logger = { path = "logger", version = "=2.0.0" } +solana-measure = { path = "measure", version = "=2.0.0" } +solana-merkle-tree = { path = "merkle-tree", version = "=2.0.0" } +solana-metrics = { path = "metrics", version = "=2.0.0" } +solana-net-utils = { path = "net-utils", version = "=2.0.0" } solana-nohash-hasher = "0.2.1" -solana-notifier = { path = "notifier", version = "=1.19.0" } -solana-perf = { path = "perf", version = "=1.19.0" } -solana-poh = { path = "poh", version = "=1.19.0" } -solana-program = { path = "sdk/program", version = "=1.19.0" } -solana-program-runtime = { path = "program-runtime", version = "=1.19.0" } -solana-program-test = { path = "program-test", version = "=1.19.0" } -solana-pubsub-client = { path = "pubsub-client", version = "=1.19.0" } -solana-quic-client = { path = "quic-client", version = "=1.19.0" } -solana-rayon-threadlimit = { path = "rayon-threadlimit", version = "=1.19.0" } -solana-remote-wallet = { path = "remote-wallet", version = "=1.19.0", default-features = false } -solana-unified-scheduler-logic = { path = "unified-scheduler-logic", version = "=1.19.0" } -solana-unified-scheduler-pool = { path = "unified-scheduler-pool", version = "=1.19.0" } -solana-rpc = { path = "rpc", version = "=1.19.0" } -solana-rpc-client = { path = "rpc-client", version = "=1.19.0", default-features = false } -solana-rpc-client-api = { path = "rpc-client-api", version = "=1.19.0" } -solana-rpc-client-nonce-utils = { path = "rpc-client-nonce-utils", version = "=1.19.0" } -solana-runtime = { path = "runtime", version = "=1.19.0" } -solana-runtime-transaction = { path = "runtime-transaction", version = "=1.19.0" } -solana-sdk = { path = "sdk", version = "=1.19.0" } -solana-sdk-macro = { path = "sdk/macro", version = "=1.19.0" } -solana-send-transaction-service = { path = "send-transaction-service", version = "=1.19.0" } -solana-stake-program = { path = "programs/stake", version = "=1.19.0" } -solana-storage-bigtable = { path = "storage-bigtable", version = "=1.19.0" } -solana-storage-proto = { path = "storage-proto", version = "=1.19.0" } -solana-streamer = { path = "streamer", version = "=1.19.0" } -solana-svm = { path = "svm", version = "=1.19.0" } -solana-system-program = { path = "programs/system", version = "=1.19.0" } -solana-test-validator = { path = "test-validator", version = "=1.19.0" } -solana-thin-client = { path = "thin-client", version = "=1.19.0" } -solana-tpu-client = { path = "tpu-client", version = "=1.19.0", default-features = false } -solana-transaction-status = { path = "transaction-status", version = "=1.19.0" } -solana-turbine = { path = "turbine", version = "=1.19.0" } -solana-udp-client = { path = "udp-client", version = "=1.19.0" } -solana-version = { path = "version", version = "=1.19.0" } -solana-vote = { path = "vote", version = "=1.19.0" } -solana-vote-program = { path = "programs/vote", version = "=1.19.0" } -solana-wen-restart = { path = "wen-restart", version = "=1.19.0" } -solana-zk-keygen = { path = "zk-keygen", version = "=1.19.0" } -solana-zk-token-proof-program = { path = "programs/zk-token-proof", version = "=1.19.0" } -solana-zk-token-sdk = { path = "zk-token-sdk", version = "=1.19.0" } +solana-notifier = { path = "notifier", version = "=2.0.0" } +solana-perf = { path = "perf", version = "=2.0.0" } +solana-poh = { path = "poh", version = "=2.0.0" } +solana-program = { path = "sdk/program", version = "=2.0.0" } +solana-program-runtime = { path = "program-runtime", version = "=2.0.0" } +solana-program-test = { path = "program-test", version = "=2.0.0" } +solana-pubsub-client = { path = "pubsub-client", version = "=2.0.0" } +solana-quic-client = { path = "quic-client", version = "=2.0.0" } +solana-rayon-threadlimit = { path = "rayon-threadlimit", version = "=2.0.0" } +solana-remote-wallet = { path = "remote-wallet", version = "=2.0.0", default-features = false } +solana-unified-scheduler-logic = { path = "unified-scheduler-logic", version = "=2.0.0" } +solana-unified-scheduler-pool = { path = "unified-scheduler-pool", version = "=2.0.0" } +solana-rpc = { path = "rpc", version = "=2.0.0" } +solana-rpc-client = { path = "rpc-client", version = "=2.0.0", default-features = false } +solana-rpc-client-api = { path = "rpc-client-api", version = "=2.0.0" } +solana-rpc-client-nonce-utils = { path = "rpc-client-nonce-utils", version = "=2.0.0" } +solana-runtime = { path = "runtime", version = "=2.0.0" } +solana-runtime-transaction = { path = "runtime-transaction", version = "=2.0.0" } +solana-sdk = { path = "sdk", version = "=2.0.0" } +solana-sdk-macro = { path = "sdk/macro", version = "=2.0.0" } +solana-send-transaction-service = { path = "send-transaction-service", version = "=2.0.0" } +solana-stake-program = { path = "programs/stake", version = "=2.0.0" } +solana-storage-bigtable = { path = "storage-bigtable", version = "=2.0.0" } +solana-storage-proto = { path = "storage-proto", version = "=2.0.0" } +solana-streamer = { path = "streamer", version = "=2.0.0" } +solana-svm = { path = "svm", version = "=2.0.0" } +solana-system-program = { path = "programs/system", version = "=2.0.0" } +solana-test-validator = { path = "test-validator", version = "=2.0.0" } +solana-thin-client = { path = "thin-client", version = "=2.0.0" } +solana-tpu-client = { path = "tpu-client", version = "=2.0.0", default-features = false } +solana-transaction-status = { path = "transaction-status", version = "=2.0.0" } +solana-turbine = { path = "turbine", version = "=2.0.0" } +solana-udp-client = { path = "udp-client", version = "=2.0.0" } +solana-version = { path = "version", version = "=2.0.0" } +solana-vote = { path = "vote", version = "=2.0.0" } +solana-vote-program = { path = "programs/vote", version = "=2.0.0" } +solana-wen-restart = { path = "wen-restart", version = "=2.0.0" } +solana-zk-keygen = { path = "zk-keygen", version = "=2.0.0" } +solana-zk-token-proof-program = { path = "programs/zk-token-proof", version = "=2.0.0" } +solana-zk-token-sdk = { path = "zk-token-sdk", version = "=2.0.0" } solana_rbpf = "=0.8.0" spl-associated-token-account = "=2.3.1" spl-instruction-padding = "0.1" diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index a3d350456afa9c..c31befdf34c092 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -65,7 +65,7 @@ dependencies = [ [[package]] name = "agave-geyser-plugin-interface" -version = "1.19.0" +version = "2.0.0" dependencies = [ "log", "solana-sdk", @@ -75,7 +75,7 @@ dependencies = [ [[package]] name = "agave-validator" -version = "1.19.0" +version = "2.0.0" dependencies = [ "agave-geyser-plugin-interface", "chrono", @@ -4598,7 +4598,7 @@ dependencies = [ [[package]] name = "solana-account-decoder" -version = "1.19.0" +version = "2.0.0" dependencies = [ "Inflector", "base64 0.21.7", @@ -4621,7 +4621,7 @@ dependencies = [ [[package]] name = "solana-accounts-db" -version = "1.19.0" +version = "2.0.0" dependencies = [ "arrayref", "bincode", @@ -4681,7 +4681,7 @@ dependencies = [ [[package]] name = "solana-address-lookup-table-program" -version = "1.19.0" +version = "2.0.0" dependencies = [ "bincode", "bytemuck", @@ -4700,7 +4700,7 @@ dependencies = [ [[package]] name = "solana-banks-client" -version = "1.19.0" +version = "2.0.0" dependencies = [ "borsh 1.2.1", "futures 0.3.30", @@ -4715,7 +4715,7 @@ dependencies = [ [[package]] name = "solana-banks-interface" -version = "1.19.0" +version = "2.0.0" dependencies = [ "serde", "solana-sdk", @@ -4724,7 +4724,7 @@ dependencies = [ [[package]] name = "solana-banks-server" -version = "1.19.0" +version = "2.0.0" dependencies = [ "bincode", "crossbeam-channel", @@ -4742,7 +4742,7 @@ dependencies = [ [[package]] name = "solana-bloom" -version = "1.19.0" +version = "2.0.0" dependencies = [ "bv", "fnv", @@ -4759,7 +4759,7 @@ dependencies = [ [[package]] name = "solana-bpf-loader-program" -version = "1.19.0" +version = "2.0.0" dependencies = [ "bincode", "byteorder 1.5.0", @@ -4776,7 +4776,7 @@ dependencies = [ [[package]] name = "solana-bpf-rust-big-mod-exp" -version = "1.19.0" +version = "2.0.0" dependencies = [ "array-bytes", "serde", @@ -4786,7 +4786,7 @@ dependencies = [ [[package]] name = "solana-bucket-map" -version = "1.19.0" +version = "2.0.0" dependencies = [ "bv", "bytemuck", @@ -4802,7 +4802,7 @@ dependencies = [ [[package]] name = "solana-clap-utils" -version = "1.19.0" +version = "2.0.0" dependencies = [ "chrono", "clap 2.33.3", @@ -4817,7 +4817,7 @@ dependencies = [ [[package]] name = "solana-cli-config" -version = "1.19.0" +version = "2.0.0" dependencies = [ "dirs-next", "lazy_static", @@ -4831,7 +4831,7 @@ dependencies = [ [[package]] name = "solana-cli-output" -version = "1.19.0" +version = "2.0.0" dependencies = [ "Inflector", "base64 0.21.7", @@ -4856,7 +4856,7 @@ dependencies = [ [[package]] name = "solana-client" -version = "1.19.0" +version = "2.0.0" dependencies = [ "async-trait", "bincode", @@ -4887,7 +4887,7 @@ dependencies = [ [[package]] name = "solana-compute-budget-program" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program-runtime", "solana-sdk", @@ -4895,7 +4895,7 @@ dependencies = [ [[package]] name = "solana-config-program" -version = "1.19.0" +version = "2.0.0" dependencies = [ "bincode", "chrono", @@ -4907,7 +4907,7 @@ dependencies = [ [[package]] name = "solana-connection-cache" -version = "1.19.0" +version = "2.0.0" dependencies = [ "async-trait", "bincode", @@ -4926,7 +4926,7 @@ dependencies = [ [[package]] name = "solana-core" -version = "1.19.0" +version = "2.0.0" dependencies = [ "base64 0.21.7", "bincode", @@ -5001,7 +5001,7 @@ dependencies = [ [[package]] name = "solana-cost-model" -version = "1.19.0" +version = "2.0.0" dependencies = [ "lazy_static", "log", @@ -5023,7 +5023,7 @@ dependencies = [ [[package]] name = "solana-download-utils" -version = "1.19.0" +version = "2.0.0" dependencies = [ "console", "indicatif", @@ -5035,7 +5035,7 @@ dependencies = [ [[package]] name = "solana-entry" -version = "1.19.0" +version = "2.0.0" dependencies = [ "bincode", "crossbeam-channel", @@ -5055,7 +5055,7 @@ dependencies = [ [[package]] name = "solana-faucet" -version = "1.19.0" +version = "2.0.0" dependencies = [ "bincode", "byteorder 1.5.0", @@ -5077,7 +5077,7 @@ dependencies = [ [[package]] name = "solana-frozen-abi" -version = "1.19.0" +version = "2.0.0" dependencies = [ "block-buffer 0.10.4", "bs58", @@ -5099,7 +5099,7 @@ dependencies = [ [[package]] name = "solana-frozen-abi-macro" -version = "1.19.0" +version = "2.0.0" dependencies = [ "proc-macro2", "quote", @@ -5109,7 +5109,7 @@ dependencies = [ [[package]] name = "solana-genesis-utils" -version = "1.19.0" +version = "2.0.0" dependencies = [ "log", "solana-accounts-db", @@ -5120,7 +5120,7 @@ dependencies = [ [[package]] name = "solana-geyser-plugin-manager" -version = "1.19.0" +version = "2.0.0" dependencies = [ "agave-geyser-plugin-interface", "bs58", @@ -5145,7 +5145,7 @@ dependencies = [ [[package]] name = "solana-gossip" -version = "1.19.0" +version = "2.0.0" dependencies = [ "assert_matches", "bincode", @@ -5193,7 +5193,7 @@ dependencies = [ [[package]] name = "solana-ledger" -version = "1.19.0" +version = "2.0.0" dependencies = [ "assert_matches", "bincode", @@ -5259,7 +5259,7 @@ dependencies = [ [[package]] name = "solana-loader-v4-program" -version = "1.19.0" +version = "2.0.0" dependencies = [ "log", "solana-measure", @@ -5270,7 +5270,7 @@ dependencies = [ [[package]] name = "solana-logger" -version = "1.19.0" +version = "2.0.0" dependencies = [ "env_logger", "lazy_static", @@ -5279,7 +5279,7 @@ dependencies = [ [[package]] name = "solana-measure" -version = "1.19.0" +version = "2.0.0" dependencies = [ "log", "solana-sdk", @@ -5287,7 +5287,7 @@ dependencies = [ [[package]] name = "solana-merkle-tree" -version = "1.19.0" +version = "2.0.0" dependencies = [ "fast-math", "solana-program", @@ -5295,7 +5295,7 @@ dependencies = [ [[package]] name = "solana-metrics" -version = "1.19.0" +version = "2.0.0" dependencies = [ "crossbeam-channel", "gethostname", @@ -5308,7 +5308,7 @@ dependencies = [ [[package]] name = "solana-net-utils" -version = "1.19.0" +version = "2.0.0" dependencies = [ "bincode", "clap 3.1.6", @@ -5334,7 +5334,7 @@ checksum = "8b8a731ed60e89177c8a7ab05fe0f1511cedd3e70e773f288f9de33a9cfdc21e" [[package]] name = "solana-perf" -version = "1.19.0" +version = "2.0.0" dependencies = [ "ahash 0.8.10", "bincode", @@ -5361,7 +5361,7 @@ dependencies = [ [[package]] name = "solana-poh" -version = "1.19.0" +version = "2.0.0" dependencies = [ "core_affinity", "crossbeam-channel", @@ -5377,7 +5377,7 @@ dependencies = [ [[package]] name = "solana-program" -version = "1.19.0" +version = "2.0.0" dependencies = [ "ark-bn254", "ark-ec", @@ -5430,7 +5430,7 @@ dependencies = [ [[package]] name = "solana-program-runtime" -version = "1.19.0" +version = "2.0.0" dependencies = [ "base64 0.21.7", "bincode", @@ -5456,7 +5456,7 @@ dependencies = [ [[package]] name = "solana-program-test" -version = "1.19.0" +version = "2.0.0" dependencies = [ "assert_matches", "async-trait", @@ -5485,7 +5485,7 @@ dependencies = [ [[package]] name = "solana-pubsub-client" -version = "1.19.0" +version = "2.0.0" dependencies = [ "crossbeam-channel", "futures-util", @@ -5508,7 +5508,7 @@ dependencies = [ [[package]] name = "solana-quic-client" -version = "1.19.0" +version = "2.0.0" dependencies = [ "async-mutex", "async-trait", @@ -5532,7 +5532,7 @@ dependencies = [ [[package]] name = "solana-rayon-threadlimit" -version = "1.19.0" +version = "2.0.0" dependencies = [ "lazy_static", "num_cpus", @@ -5540,7 +5540,7 @@ dependencies = [ [[package]] name = "solana-remote-wallet" -version = "1.19.0" +version = "2.0.0" dependencies = [ "console", "dialoguer", @@ -5557,7 +5557,7 @@ dependencies = [ [[package]] name = "solana-rpc" -version = "1.19.0" +version = "2.0.0" dependencies = [ "base64 0.21.7", "bincode", @@ -5613,7 +5613,7 @@ dependencies = [ [[package]] name = "solana-rpc-client" -version = "1.19.0" +version = "2.0.0" dependencies = [ "async-trait", "base64 0.21.7", @@ -5637,7 +5637,7 @@ dependencies = [ [[package]] name = "solana-rpc-client-api" -version = "1.19.0" +version = "2.0.0" dependencies = [ "base64 0.21.7", "bs58", @@ -5657,7 +5657,7 @@ dependencies = [ [[package]] name = "solana-rpc-client-nonce-utils" -version = "1.19.0" +version = "2.0.0" dependencies = [ "clap 2.33.3", "solana-clap-utils", @@ -5668,7 +5668,7 @@ dependencies = [ [[package]] name = "solana-runtime" -version = "1.19.0" +version = "2.0.0" dependencies = [ "aquamarine", "arrayref", @@ -5743,7 +5743,7 @@ dependencies = [ [[package]] name = "solana-sbf-programs" -version = "1.19.0" +version = "2.0.0" dependencies = [ "bincode", "byteorder 1.5.0", @@ -5773,7 +5773,7 @@ dependencies = [ [[package]] name = "solana-sbf-rust-128bit" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program", "solana-sbf-rust-128bit-dep", @@ -5781,21 +5781,21 @@ dependencies = [ [[package]] name = "solana-sbf-rust-128bit-dep" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-alloc" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-alt-bn128" -version = "1.19.0" +version = "2.0.0" dependencies = [ "array-bytes", "solana-program", @@ -5803,7 +5803,7 @@ dependencies = [ [[package]] name = "solana-sbf-rust-alt-bn128-compression" -version = "1.19.0" +version = "2.0.0" dependencies = [ "array-bytes", "solana-program", @@ -5811,21 +5811,21 @@ dependencies = [ [[package]] name = "solana-sbf-rust-call-depth" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-caller-access" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-curve25519" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program", "solana-zk-token-sdk", @@ -5833,14 +5833,14 @@ dependencies = [ [[package]] name = "solana-sbf-rust-custom-heap" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-dep-crate" -version = "1.19.0" +version = "2.0.0" dependencies = [ "byteorder 1.5.0", "solana-program", @@ -5848,21 +5848,21 @@ dependencies = [ [[package]] name = "solana-sbf-rust-deprecated-loader" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-dup-accounts" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-error-handling" -version = "1.19.0" +version = "2.0.0" dependencies = [ "num-derive 0.3.0", "num-traits", @@ -5872,42 +5872,42 @@ dependencies = [ [[package]] name = "solana-sbf-rust-external-spend" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-finalize" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-get-minimum-delegation" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-inner_instruction_alignment_check" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-instruction-introspection" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-invoke" -version = "1.19.0" +version = "2.0.0" dependencies = [ "rustversion", "solana-program", @@ -5917,49 +5917,49 @@ dependencies = [ [[package]] name = "solana-sbf-rust-invoke-and-error" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-invoke-and-ok" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-invoke-and-return" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-invoked" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-iter" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-log-data" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-many-args" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program", "solana-sbf-rust-many-args-dep", @@ -5967,14 +5967,14 @@ dependencies = [ [[package]] name = "solana-sbf-rust-many-args-dep" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-mem" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program", "solana-program-runtime", @@ -5984,7 +5984,7 @@ dependencies = [ [[package]] name = "solana-sbf-rust-membuiltins" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program", "solana-sbf-rust-mem", @@ -5992,21 +5992,21 @@ dependencies = [ [[package]] name = "solana-sbf-rust-noop" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-panic" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-param-passing" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program", "solana-sbf-rust-param-passing-dep", @@ -6014,14 +6014,14 @@ dependencies = [ [[package]] name = "solana-sbf-rust-param-passing-dep" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-poseidon" -version = "1.19.0" +version = "2.0.0" dependencies = [ "array-bytes", "solana-program", @@ -6029,7 +6029,7 @@ dependencies = [ [[package]] name = "solana-sbf-rust-rand" -version = "1.19.0" +version = "2.0.0" dependencies = [ "getrandom 0.2.10", "rand 0.8.5", @@ -6038,14 +6038,14 @@ dependencies = [ [[package]] name = "solana-sbf-rust-realloc" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-realloc-invoke" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program", "solana-sbf-rust-realloc", @@ -6053,7 +6053,7 @@ dependencies = [ [[package]] name = "solana-sbf-rust-remaining-compute-units" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program", "solana-program-runtime", @@ -6063,21 +6063,21 @@ dependencies = [ [[package]] name = "solana-sbf-rust-ro-account_modify" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-ro-modify" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-sanity" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program", "solana-program-runtime", @@ -6087,7 +6087,7 @@ dependencies = [ [[package]] name = "solana-sbf-rust-secp256k1-recover" -version = "1.19.0" +version = "2.0.0" dependencies = [ "libsecp256k1 0.7.0", "solana-program", @@ -6095,7 +6095,7 @@ dependencies = [ [[package]] name = "solana-sbf-rust-sha" -version = "1.19.0" +version = "2.0.0" dependencies = [ "blake3", "solana-program", @@ -6103,21 +6103,21 @@ dependencies = [ [[package]] name = "solana-sbf-rust-sibling-instructions" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-sibling_inner-instructions" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-simulation" -version = "1.19.0" +version = "2.0.0" dependencies = [ "agave-validator", "solana-logger", @@ -6128,21 +6128,21 @@ dependencies = [ [[package]] name = "solana-sbf-rust-spoof1" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-spoof1-system" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-sysvar" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program", "solana-program-runtime", @@ -6152,21 +6152,21 @@ dependencies = [ [[package]] name = "solana-sbf-rust-upgradeable" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program", ] [[package]] name = "solana-sbf-rust-upgraded" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-program", ] [[package]] name = "solana-sdk" -version = "1.19.0" +version = "2.0.0" dependencies = [ "assert_matches", "base64 0.21.7", @@ -6219,7 +6219,7 @@ dependencies = [ [[package]] name = "solana-sdk-macro" -version = "1.19.0" +version = "2.0.0" dependencies = [ "bs58", "proc-macro2", @@ -6236,7 +6236,7 @@ checksum = "468aa43b7edb1f9b7b7b686d5c3aeb6630dc1708e86e31343499dd5c4d775183" [[package]] name = "solana-send-transaction-service" -version = "1.19.0" +version = "2.0.0" dependencies = [ "crossbeam-channel", "log", @@ -6250,7 +6250,7 @@ dependencies = [ [[package]] name = "solana-stake-program" -version = "1.19.0" +version = "2.0.0" dependencies = [ "bincode", "log", @@ -6263,7 +6263,7 @@ dependencies = [ [[package]] name = "solana-storage-bigtable" -version = "1.19.0" +version = "2.0.0" dependencies = [ "backoff", "bincode", @@ -6295,7 +6295,7 @@ dependencies = [ [[package]] name = "solana-storage-proto" -version = "1.19.0" +version = "2.0.0" dependencies = [ "bincode", "bs58", @@ -6310,7 +6310,7 @@ dependencies = [ [[package]] name = "solana-streamer" -version = "1.19.0" +version = "2.0.0" dependencies = [ "async-channel", "bytes", @@ -6338,7 +6338,7 @@ dependencies = [ [[package]] name = "solana-svm" -version = "1.19.0" +version = "2.0.0" dependencies = [ "itertools", "log", @@ -6357,7 +6357,7 @@ dependencies = [ [[package]] name = "solana-system-program" -version = "1.19.0" +version = "2.0.0" dependencies = [ "bincode", "log", @@ -6369,7 +6369,7 @@ dependencies = [ [[package]] name = "solana-test-validator" -version = "1.19.0" +version = "2.0.0" dependencies = [ "base64 0.21.7", "bincode", @@ -6399,7 +6399,7 @@ dependencies = [ [[package]] name = "solana-thin-client" -version = "1.19.0" +version = "2.0.0" dependencies = [ "bincode", "log", @@ -6412,7 +6412,7 @@ dependencies = [ [[package]] name = "solana-tpu-client" -version = "1.19.0" +version = "2.0.0" dependencies = [ "async-trait", "bincode", @@ -6434,7 +6434,7 @@ dependencies = [ [[package]] name = "solana-transaction-status" -version = "1.19.0" +version = "2.0.0" dependencies = [ "Inflector", "base64 0.21.7", @@ -6457,7 +6457,7 @@ dependencies = [ [[package]] name = "solana-turbine" -version = "1.19.0" +version = "2.0.0" dependencies = [ "bincode", "bytes", @@ -6491,7 +6491,7 @@ dependencies = [ [[package]] name = "solana-udp-client" -version = "1.19.0" +version = "2.0.0" dependencies = [ "async-trait", "solana-connection-cache", @@ -6504,14 +6504,14 @@ dependencies = [ [[package]] name = "solana-unified-scheduler-logic" -version = "1.19.0" +version = "2.0.0" dependencies = [ "solana-sdk", ] [[package]] name = "solana-unified-scheduler-pool" -version = "1.19.0" +version = "2.0.0" dependencies = [ "assert_matches", "crossbeam-channel", @@ -6527,7 +6527,7 @@ dependencies = [ [[package]] name = "solana-version" -version = "1.19.0" +version = "2.0.0" dependencies = [ "log", "rustc_version", @@ -6541,7 +6541,7 @@ dependencies = [ [[package]] name = "solana-vote" -version = "1.19.0" +version = "2.0.0" dependencies = [ "crossbeam-channel", "itertools", @@ -6558,7 +6558,7 @@ dependencies = [ [[package]] name = "solana-vote-program" -version = "1.19.0" +version = "2.0.0" dependencies = [ "bincode", "log", @@ -6578,7 +6578,7 @@ dependencies = [ [[package]] name = "solana-wen-restart" -version = "1.19.0" +version = "2.0.0" dependencies = [ "anyhow", "log", @@ -6598,7 +6598,7 @@ dependencies = [ [[package]] name = "solana-zk-token-proof-program" -version = "1.19.0" +version = "2.0.0" dependencies = [ "bytemuck", "num-derive 0.4.2", @@ -6610,7 +6610,7 @@ dependencies = [ [[package]] name = "solana-zk-token-sdk" -version = "1.19.0" +version = "2.0.0" dependencies = [ "aes-gcm-siv", "base64 0.21.7", diff --git a/programs/sbf/Cargo.toml b/programs/sbf/Cargo.toml index dee6a947b1965d..c8f2b431f28e7a 100644 --- a/programs/sbf/Cargo.toml +++ b/programs/sbf/Cargo.toml @@ -1,5 +1,5 @@ [workspace.package] -version = "1.19.0" +version = "2.0.0" description = "Solana SBF test program written in Rust" authors = ["Solana Labs Maintainers "] repository = "https://github.com/solana-labs/solana" @@ -25,30 +25,30 @@ rand = "0.8" rustversion = "1.0.14" serde = "1.0.112" serde_json = "1.0.56" -solana-account-decoder = { path = "../../account-decoder", version = "=1.19.0" } -solana-accounts-db = { path = "../../accounts-db", version = "=1.19.0" } -solana-bpf-loader-program = { path = "../bpf_loader", version = "=1.19.0" } -solana-cli-output = { path = "../../cli-output", version = "=1.19.0" } -solana-ledger = { path = "../../ledger", version = "=1.19.0" } -solana-logger = { path = "../../logger", version = "=1.19.0" } -solana-measure = { path = "../../measure", version = "=1.19.0" } -solana-program = { path = "../../sdk/program", version = "=1.19.0" } -solana-program-runtime = { path = "../../program-runtime", version = "=1.19.0" } -solana-program-test = { path = "../../program-test", version = "=1.19.0" } -solana-runtime = { path = "../../runtime", version = "=1.19.0" } -solana-sbf-rust-128bit-dep = { path = "rust/128bit_dep", version = "=1.19.0" } -solana-sbf-rust-invoke = { path = "rust/invoke", version = "=1.19.0" } -solana-sbf-rust-invoked = { path = "rust/invoked", version = "=1.19.0", default-features = false } -solana-sbf-rust-many-args-dep = { path = "rust/many_args_dep", version = "=1.19.0" } -solana-sbf-rust-mem = { path = "rust/mem", version = "=1.19.0" } -solana-sbf-rust-param-passing-dep = { path = "rust/param_passing_dep", version = "=1.19.0" } -solana-sbf-rust-realloc = { path = "rust/realloc", version = "=1.19.0", default-features = false } -solana-sbf-rust-realloc-invoke = { path = "rust/realloc_invoke", version = "=1.19.0" } -solana-sdk = { path = "../../sdk", version = "=1.19.0" } -solana-transaction-status = { path = "../../transaction-status", version = "=1.19.0" } -agave-validator = { path = "../../validator", version = "=1.19.0" } -solana-zk-token-sdk = { path = "../../zk-token-sdk", version = "=1.19.0" } -solana-svm = { path = "../../svm", version = "=1.19.0" } +solana-account-decoder = { path = "../../account-decoder", version = "=2.0.0" } +solana-accounts-db = { path = "../../accounts-db", version = "=2.0.0" } +solana-bpf-loader-program = { path = "../bpf_loader", version = "=2.0.0" } +solana-cli-output = { path = "../../cli-output", version = "=2.0.0" } +solana-ledger = { path = "../../ledger", version = "=2.0.0" } +solana-logger = { path = "../../logger", version = "=2.0.0" } +solana-measure = { path = "../../measure", version = "=2.0.0" } +solana-program = { path = "../../sdk/program", version = "=2.0.0" } +solana-program-runtime = { path = "../../program-runtime", version = "=2.0.0" } +solana-program-test = { path = "../../program-test", version = "=2.0.0" } +solana-runtime = { path = "../../runtime", version = "=2.0.0" } +solana-sbf-rust-128bit-dep = { path = "rust/128bit_dep", version = "=2.0.0" } +solana-sbf-rust-invoke = { path = "rust/invoke", version = "=2.0.0" } +solana-sbf-rust-invoked = { path = "rust/invoked", version = "=2.0.0", default-features = false } +solana-sbf-rust-many-args-dep = { path = "rust/many_args_dep", version = "=2.0.0" } +solana-sbf-rust-mem = { path = "rust/mem", version = "=2.0.0" } +solana-sbf-rust-param-passing-dep = { path = "rust/param_passing_dep", version = "=2.0.0" } +solana-sbf-rust-realloc = { path = "rust/realloc", version = "=2.0.0", default-features = false } +solana-sbf-rust-realloc-invoke = { path = "rust/realloc_invoke", version = "=2.0.0" } +solana-sdk = { path = "../../sdk", version = "=2.0.0" } +solana-svm = { path = "../../svm", version = "=2.0.0" } +solana-transaction-status = { path = "../../transaction-status", version = "=2.0.0" } +agave-validator = { path = "../../validator", version = "=2.0.0" } +solana-zk-token-sdk = { path = "../../zk-token-sdk", version = "=2.0.0" } solana_rbpf = "=0.8.0" static_assertions = "1.1.0" thiserror = "1.0" diff --git a/sdk/cargo-build-sbf/tests/crates/fail/Cargo.toml b/sdk/cargo-build-sbf/tests/crates/fail/Cargo.toml index 8e1b7f77206707..7dc085d721af50 100644 --- a/sdk/cargo-build-sbf/tests/crates/fail/Cargo.toml +++ b/sdk/cargo-build-sbf/tests/crates/fail/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fail" -version = "1.19.0" +version = "2.0.0" description = "Solana SBF test program written in Rust" authors = ["Solana Labs Maintainers "] repository = "https://github.com/solana-labs/solana" @@ -10,7 +10,7 @@ edition = "2021" publish = false [dependencies] -solana-program = { path = "../../../../program", version = "=1.19.0" } +solana-program = { path = "../../../../program", version = "=2.0.0" } [lib] crate-type = ["cdylib"] diff --git a/sdk/cargo-build-sbf/tests/crates/noop/Cargo.toml b/sdk/cargo-build-sbf/tests/crates/noop/Cargo.toml index 2d48c1295424da..3d3946decdb6ab 100644 --- a/sdk/cargo-build-sbf/tests/crates/noop/Cargo.toml +++ b/sdk/cargo-build-sbf/tests/crates/noop/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "noop" -version = "1.19.0" +version = "2.0.0" description = "Solana SBF test program written in Rust" authors = ["Solana Labs Maintainers "] repository = "https://github.com/solana-labs/solana" @@ -10,7 +10,7 @@ edition = "2021" publish = false [dependencies] -solana-program = { path = "../../../../program", version = "=1.19.0" } +solana-program = { path = "../../../../program", version = "=2.0.0" } [lib] crate-type = ["cdylib"] From 8fa0e5c603e05ddfc98eb0e73c5684143ae355a1 Mon Sep 17 00:00:00 2001 From: steviez Date: Mon, 11 Mar 2024 13:11:43 -0500 Subject: [PATCH 46/84] Move AccountsDb replication arguments to deprecated list (#157) These arguments are not read by anything, and they appear to correspond to a proposed feature that is no longer in the codebase. --- validator/src/cli.rs | 66 +++++++++++++++++--------------------------- 1 file changed, 25 insertions(+), 41 deletions(-) diff --git a/validator/src/cli.rs b/validator/src/cli.rs index 8424d7973f0705..e3f46309724af7 100644 --- a/validator/src/cli.rs +++ b/validator/src/cli.rs @@ -1172,44 +1172,6 @@ pub fn app<'a>(version: &'a str, default_args: &'a DefaultArgs) -> App<'a, 'a> { .default_value(&default_args.rpc_max_request_body_size) .help("The maximum request body size accepted by rpc service"), ) - .arg( - Arg::with_name("enable_accountsdb_repl") - .long("enable-accountsdb-repl") - .takes_value(false) - .hidden(hidden_unless_forced()) - .help("Enable AccountsDb Replication"), - ) - .arg( - Arg::with_name("accountsdb_repl_bind_address") - .long("accountsdb-repl-bind-address") - .value_name("HOST") - .takes_value(true) - .validator(solana_net_utils::is_host) - .hidden(hidden_unless_forced()) - .help( - "IP address to bind the AccountsDb Replication port [default: use \ - --bind-address]", - ), - ) - .arg( - Arg::with_name("accountsdb_repl_port") - .long("accountsdb-repl-port") - .value_name("PORT") - .takes_value(true) - .validator(port_validator) - .hidden(hidden_unless_forced()) - .help("Enable AccountsDb Replication Service on this port"), - ) - .arg( - Arg::with_name("accountsdb_repl_threads") - .long("accountsdb-repl-threads") - .value_name("NUMBER") - .validator(is_parsable::) - .takes_value(true) - .default_value(&default_args.accountsdb_repl_threads) - .hidden(hidden_unless_forced()) - .help("Number of threads to use for servicing AccountsDb Replication requests"), - ) .arg( Arg::with_name("geyser_plugin_config") .long("geyser-plugin-config") @@ -1989,6 +1951,27 @@ fn deprecated_arguments() -> Vec { Ok(()) } })); + add_arg!(Arg::with_name("accountsdb_repl_bind_address") + .long("accountsdb-repl-bind-address") + .value_name("HOST") + .takes_value(true) + .validator(solana_net_utils::is_host) + .help( + "IP address to bind the AccountsDb Replication port [default: use \ + --bind-address]", + )); + add_arg!(Arg::with_name("accountsdb_repl_port") + .long("accountsdb-repl-port") + .value_name("PORT") + .takes_value(true) + .validator(port_validator) + .help("Enable AccountsDb Replication Service on this port")); + add_arg!(Arg::with_name("accountsdb_repl_threads") + .long("accountsdb-repl-threads") + .value_name("NUMBER") + .validator(is_parsable::) + .takes_value(true) + .help("Number of threads to use for servicing AccountsDb Replication requests")); add_arg!(Arg::with_name("disable_accounts_disk_index") .long("disable-accounts-disk-index") .help("Disable the disk-based accounts index if it is enabled by default.") @@ -1999,6 +1982,10 @@ fn deprecated_arguments() -> Vec { .takes_value(false), usage_warning: "The quic server cannot be disabled.", ); + add_arg!(Arg::with_name("enable_accountsdb_repl") + .long("enable-accountsdb-repl") + .takes_value(false) + .help("Enable AccountsDb Replication")); add_arg!( Arg::with_name("enable_cpi_and_log_storage") .long("enable-cpi-and-log-storage") @@ -2162,8 +2149,6 @@ pub struct DefaultArgs { pub contact_debug_interval: String, - pub accountsdb_repl_threads: String, - pub snapshot_version: SnapshotVersion, pub snapshot_archive_format: String, @@ -2239,7 +2224,6 @@ impl DefaultArgs { rpc_bigtable_max_message_size: solana_storage_bigtable::DEFAULT_MAX_MESSAGE_SIZE .to_string(), rpc_pubsub_worker_threads: "4".to_string(), - accountsdb_repl_threads: num_cpus::get().to_string(), maximum_full_snapshot_archives_to_retain: DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN .to_string(), maximum_incremental_snapshot_archives_to_retain: From f205d0e729e05f48ebba4e4bb47859714ff390f2 Mon Sep 17 00:00:00 2001 From: behzad nouri Date: Mon, 11 Mar 2024 18:49:35 +0000 Subject: [PATCH 47/84] expands weighted-shuffle benchmarks (#179) Adding separate benchmarks for WeightedShuffle::new and WeightedShuffle::shuffle. --- gossip/benches/weighted_shuffle.rs | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/gossip/benches/weighted_shuffle.rs b/gossip/benches/weighted_shuffle.rs index 58a7e37ce6a328..09615c57bbca15 100644 --- a/gossip/benches/weighted_shuffle.rs +++ b/gossip/benches/weighted_shuffle.rs @@ -11,18 +11,28 @@ use { }; fn make_weights(rng: &mut R) -> Vec { - repeat_with(|| rng.gen_range(1..100)).take(1000).collect() + repeat_with(|| rng.gen_range(1..100)).take(4000).collect() } #[bench] -fn bench_weighted_shuffle(bencher: &mut Bencher) { +fn bench_weighted_shuffle_new(bencher: &mut Bencher) { + let mut rng = rand::thread_rng(); + bencher.iter(|| { + let weights = make_weights(&mut rng); + std::hint::black_box(WeightedShuffle::new("", &weights)); + }); +} + +#[bench] +fn bench_weighted_shuffle_shuffle(bencher: &mut Bencher) { let mut seed = [0u8; 32]; let mut rng = rand::thread_rng(); let weights = make_weights(&mut rng); + let weighted_shuffle = WeightedShuffle::new("", &weights); bencher.iter(|| { rng.fill(&mut seed[..]); - WeightedShuffle::new("", &weights) - .shuffle(&mut ChaChaRng::from_seed(seed)) - .collect::>() + let mut rng = ChaChaRng::from_seed(seed); + let shuffle = weighted_shuffle.clone().shuffle(&mut rng); + std::hint::black_box(shuffle.collect::>()); }); } From 209924d220bb4bcaa9ca41ea2fc6738cf2917b01 Mon Sep 17 00:00:00 2001 From: Greg Cusack Date: Mon, 11 Mar 2024 15:33:19 -0400 Subject: [PATCH 48/84] bump deprecated version numbers for `get_client` and `get_multi_client` (#184) bump deprecated version numbers --- gossip/src/gossip_service.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gossip/src/gossip_service.rs b/gossip/src/gossip_service.rs index 404a685aa75567..9e1c56520993c5 100644 --- a/gossip/src/gossip_service.rs +++ b/gossip/src/gossip_service.rs @@ -194,7 +194,7 @@ pub fn discover( } /// Creates a ThinClient by selecting a valid node at random -#[deprecated(since = "1.18.0", note = "Interface will change")] +#[deprecated(since = "1.18.6", note = "Interface will change")] pub fn get_client( nodes: &[ContactInfo], socket_addr_space: &SocketAddrSpace, @@ -210,7 +210,7 @@ pub fn get_client( ThinClient::new(rpc, tpu, connection_cache) } -#[deprecated(since = "1.18.0", note = "Will be removed in favor of get_client")] +#[deprecated(since = "1.18.6", note = "Will be removed in favor of get_client")] pub fn get_multi_client( nodes: &[ContactInfo], socket_addr_space: &SocketAddrSpace, From 5c1df15e922e36051c3df23b1fee4658dab18854 Mon Sep 17 00:00:00 2001 From: Brooks Date: Mon, 11 Mar 2024 15:38:34 -0400 Subject: [PATCH 49/84] Removes the storage recycler (#118) --- accounts-db/src/accounts_db.rs | 474 ++------------------- accounts-db/src/ancient_append_vecs.rs | 5 +- runtime/src/accounts_background_service.rs | 21 - runtime/src/bank.rs | 4 - runtime/src/snapshot_minimizer.rs | 8 +- 5 files changed, 31 insertions(+), 481 deletions(-) diff --git a/accounts-db/src/accounts_db.rs b/accounts-db/src/accounts_db.rs index 41ec05dce0e4a5..18ffa2d02e37ca 100644 --- a/accounts-db/src/accounts_db.rs +++ b/accounts-db/src/accounts_db.rs @@ -106,7 +106,7 @@ use { path::{Path, PathBuf}, sync::{ atomic::{AtomicBool, AtomicU32, AtomicU64, AtomicUsize, Ordering}, - Arc, Condvar, Mutex, RwLock, + Arc, Condvar, Mutex, }, thread::{sleep, Builder}, time::{Duration, Instant}, @@ -115,7 +115,6 @@ use { }; const PAGE_SIZE: u64 = 4 * 1024; -pub(crate) const MAX_RECYCLE_STORES: usize = 1000; // when the accounts write cache exceeds this many bytes, we will flush it // this can be specified on the command line, too (--accounts-db-cache-limit-mb) const WRITE_CACHE_LIMIT_BYTES_DEFAULT: u64 = 15_000_000_000; @@ -1084,16 +1083,6 @@ impl AccountStorageEntry { *count_and_status = (count, status); } - pub fn recycle(&self, slot: Slot, id: AppendVecId) { - let mut count_and_status = self.count_and_status.lock_write(); - self.accounts.reset(); - *count_and_status = (0, AccountStorageStatus::Available); - self.slot.store(slot, Ordering::Release); - self.id.store(id, Ordering::Release); - self.approx_store_count.store(0, Ordering::Relaxed); - self.alive_bytes.store(0, Ordering::Release); - } - pub fn status(&self) -> AccountStorageStatus { self.count_and_status.read().1 } @@ -1267,76 +1256,6 @@ impl StoreAccountsTiming { } } -#[derive(Debug, Default)] -struct RecycleStores { - entries: Vec<(Instant, Arc)>, - total_bytes: u64, -} - -// 30 min should be enough to be certain there won't be any prospective recycle uses for given -// store entry -// That's because it already processed ~2500 slots and ~25 passes of AccountsBackgroundService -pub const EXPIRATION_TTL_SECONDS: u64 = 1800; - -impl RecycleStores { - fn add_entry(&mut self, new_entry: Arc) { - self.total_bytes += new_entry.capacity(); - self.entries.push((Instant::now(), new_entry)) - } - - fn iter(&self) -> std::slice::Iter<(Instant, Arc)> { - self.entries.iter() - } - - fn add_entries(&mut self, new_entries: Vec>) { - let now = Instant::now(); - for new_entry in new_entries { - self.total_bytes += new_entry.capacity(); - self.entries.push((now, new_entry)); - } - } - - fn expire_old_entries(&mut self) -> Vec> { - let mut expired = vec![]; - let now = Instant::now(); - let mut expired_bytes = 0; - self.entries.retain(|(recycled_time, entry)| { - if now.duration_since(*recycled_time).as_secs() > EXPIRATION_TTL_SECONDS { - if Arc::strong_count(entry) >= 2 { - warn!( - "Expiring still in-use recycled StorageEntry anyway...: id: {} slot: {}", - entry.append_vec_id(), - entry.slot(), - ); - } - expired_bytes += entry.capacity(); - expired.push(entry.clone()); - false - } else { - true - } - }); - - self.total_bytes -= expired_bytes; - - expired - } - - fn remove_entry(&mut self, index: usize) -> Arc { - let (_added_time, removed_entry) = self.entries.swap_remove(index); - self.total_bytes -= removed_entry.capacity(); - removed_entry - } - - fn entry_count(&self) -> usize { - self.entries.len() - } - - fn total_bytes(&self) -> u64 { - self.total_bytes - } -} - /// Removing unrooted slots in Accounts Background Service needs to be synchronized with flushing /// slots from the Accounts Cache. This keeps track of those slots and the Mutex + Condvar for /// synchronization. @@ -1377,8 +1296,6 @@ pub struct AccountsDb { sender_bg_hasher: Option>, read_only_accounts_cache: ReadOnlyAccountsCache, - recycle_stores: RwLock, - /// distribute the accounts across storage lists pub next_id: AtomicAppendVecId, @@ -1506,7 +1423,6 @@ pub struct AccountsStats { pub stakes_cache_check_and_store_us: AtomicU64, store_num_accounts: AtomicU64, store_total_data: AtomicU64, - recycle_store_count: AtomicU64, create_store_count: AtomicU64, store_get_slot_store: AtomicU64, store_find_existing: AtomicU64, @@ -1529,7 +1445,6 @@ pub struct PurgeStats { total_removed_storage_entries: AtomicUsize, total_removed_cached_bytes: AtomicU64, total_removed_stored_bytes: AtomicU64, - recycle_stores_write_elapsed: AtomicU64, scan_storages_elapsed: AtomicU64, purge_accounts_index_elapsed: AtomicU64, handle_reclaims_elapsed: AtomicU64, @@ -1591,11 +1506,6 @@ impl PurgeStats { self.total_removed_stored_bytes.swap(0, Ordering::Relaxed) as i64, i64 ), - ( - "recycle_stores_write_elapsed", - self.recycle_stores_write_elapsed.swap(0, Ordering::Relaxed) as i64, - i64 - ), ( "scan_storages_elapsed", self.scan_storages_elapsed.swap(0, Ordering::Relaxed) as i64, @@ -1972,7 +1882,6 @@ pub struct ShrinkStats { unpackable_slots_count: AtomicU64, newest_alive_packed_count: AtomicU64, drop_storage_entries_elapsed: AtomicU64, - recycle_stores_write_elapsed: AtomicU64, accounts_removed: AtomicUsize, bytes_removed: AtomicU64, bytes_written: AtomicU64, @@ -2038,11 +1947,6 @@ impl ShrinkStats { self.drop_storage_entries_elapsed.swap(0, Ordering::Relaxed) as i64, i64 ), - ( - "recycle_stores_write_time", - self.recycle_stores_write_elapsed.swap(0, Ordering::Relaxed) as i64, - i64 - ), ( "accounts_removed", self.accounts_removed.swap(0, Ordering::Relaxed) as i64, @@ -2169,13 +2073,6 @@ impl ShrinkAncientStats { .swap(0, Ordering::Relaxed) as i64, i64 ), - ( - "recycle_stores_write_time", - self.shrink_stats - .recycle_stores_write_elapsed - .swap(0, Ordering::Relaxed) as i64, - i64 - ), ( "accounts_removed", self.shrink_stats @@ -2425,7 +2322,6 @@ impl AccountsDb { MAX_READ_ONLY_CACHE_DATA_SIZE, READ_ONLY_CACHE_MS_TO_SKIP_LRU_UPDATE, ), - recycle_stores: RwLock::new(RecycleStores::default()), uncleaned_pubkeys: DashMap::new(), next_id: AtomicAppendVecId::new(0), shrink_candidate_slots: Mutex::new(ShrinkCandidates::default()), @@ -3949,6 +3845,7 @@ impl AccountsDb { shrink_in_progress, shrink_can_be_active, ); + let dead_storages_len = dead_storages.len(); if !shrink_collect.all_are_zero_lamports { self.add_uncleaned_pubkeys_after_shrink( @@ -3957,9 +3854,15 @@ impl AccountsDb { ); } - self.drop_or_recycle_stores(dead_storages, stats); + let (_, drop_storage_entries_elapsed) = measure_us!(drop(dead_storages)); time.stop(); + self.stats + .dropped_stores + .fetch_add(dead_storages_len as u64, Ordering::Relaxed); + stats + .drop_storage_entries_elapsed + .fetch_add(drop_storage_entries_elapsed, Ordering::Relaxed); stats .remove_old_stores_shrink_us .fetch_add(time.as_us(), Ordering::Relaxed); @@ -4148,42 +4051,10 @@ impl AccountsDb { dead_storages } - pub fn drop_or_recycle_stores( - &self, - dead_storages: Vec>, - stats: &ShrinkStats, - ) { - let mut recycle_stores_write_elapsed = Measure::start("recycle_stores_write_time"); - let mut recycle_stores = self.recycle_stores.write().unwrap(); - recycle_stores_write_elapsed.stop(); - - let mut drop_storage_entries_elapsed = Measure::start("drop_storage_entries_elapsed"); - if recycle_stores.entry_count() < MAX_RECYCLE_STORES { - recycle_stores.add_entries(dead_storages); - drop(recycle_stores); - } else { - self.stats - .dropped_stores - .fetch_add(dead_storages.len() as u64, Ordering::Relaxed); - drop(recycle_stores); - drop(dead_storages); - } - drop_storage_entries_elapsed.stop(); - stats - .drop_storage_entries_elapsed - .fetch_add(drop_storage_entries_elapsed.as_us(), Ordering::Relaxed); - stats - .recycle_stores_write_elapsed - .fetch_add(recycle_stores_write_elapsed.as_us(), Ordering::Relaxed); - } - /// return a store that can contain 'aligned_total' bytes pub fn get_store_for_shrink(&self, slot: Slot, aligned_total: u64) -> ShrinkInProgress<'_> { - let shrunken_store = self - .try_recycle_store(slot, aligned_total, aligned_total + 1024) - .unwrap_or_else(|| { - self.create_store(slot, aligned_total, "shrink", self.shrink_paths.as_slice()) - }); + let shrunken_store = + self.create_store(slot, aligned_total, "shrink", self.shrink_paths.as_slice()); self.storage.shrinking_in_progress(slot, shrunken_store) } @@ -5524,71 +5395,7 @@ impl AccountsDb { } } - fn try_recycle_and_insert_store( - &self, - slot: Slot, - min_size: u64, - max_size: u64, - ) -> Option> { - let store = self.try_recycle_store(slot, min_size, max_size)?; - self.insert_store(slot, store.clone()); - Some(store) - } - - fn try_recycle_store( - &self, - slot: Slot, - min_size: u64, - max_size: u64, - ) -> Option> { - let mut max = 0; - let mut min = std::u64::MAX; - let mut avail = 0; - let mut recycle_stores = self.recycle_stores.write().unwrap(); - for (i, (_recycled_time, store)) in recycle_stores.iter().enumerate() { - if Arc::strong_count(store) == 1 { - max = std::cmp::max(store.accounts.capacity(), max); - min = std::cmp::min(store.accounts.capacity(), min); - avail += 1; - - if store.accounts.is_recyclable() - && store.accounts.capacity() >= min_size - && store.accounts.capacity() < max_size - { - let ret = recycle_stores.remove_entry(i); - drop(recycle_stores); - let old_id = ret.append_vec_id(); - ret.recycle(slot, self.next_id()); - // This info shows the appendvec change history. It helps debugging - // the appendvec data corrupution issues related to recycling. - debug!( - "recycling store: old slot {}, old_id: {}, new slot {}, new id{}, path {:?} ", - slot, - old_id, - ret.slot(), - ret.append_vec_id(), - ret.get_path(), - ); - self.stats - .recycle_store_count - .fetch_add(1, Ordering::Relaxed); - return Some(ret); - } - } - } - debug!( - "no recycle stores max: {} min: {} len: {} looking: {}, {} avail: {}", - max, - min, - recycle_stores.entry_count(), - min_size, - max_size, - avail, - ); - None - } - - fn find_storage_candidate(&self, slot: Slot, size: usize) -> Arc { + fn find_storage_candidate(&self, slot: Slot) -> Arc { let mut get_slot_stores = Measure::start("get_slot_stores"); let store = self.storage.get_slot_storage_entry(slot); get_slot_stores.stop(); @@ -5612,11 +5419,7 @@ impl AccountsDb { .store_find_existing .fetch_add(find_existing.as_us(), Ordering::Relaxed); - let store = if let Some(store) = self.try_recycle_store(slot, size as u64, std::u64::MAX) { - store - } else { - self.create_store(slot, self.file_size, "store", &self.paths) - }; + let store = self.create_store(slot, self.file_size, "store", &self.paths); // try_available is like taking a lock on the store, // preventing other threads from using it. @@ -5730,28 +5533,6 @@ impl AccountsDb { self.purge_slots(std::iter::once(&slot)); } - fn recycle_slot_stores( - &self, - total_removed_storage_entries: usize, - slot_stores: &[Arc], - ) -> u64 { - let mut recycle_stores_write_elapsed = Measure::start("recycle_stores_write_elapsed"); - let mut recycle_stores = self.recycle_stores.write().unwrap(); - recycle_stores_write_elapsed.stop(); - - for (recycled_count, store) in slot_stores.iter().enumerate() { - if recycle_stores.entry_count() > MAX_RECYCLE_STORES { - let dropped_count = total_removed_storage_entries - recycled_count; - self.stats - .dropped_stores - .fetch_add(dropped_count as u64, Ordering::Relaxed); - return recycle_stores_write_elapsed.as_us(); - } - recycle_stores.add_entry(Arc::clone(store)); - } - recycle_stores_write_elapsed.as_us() - } - /// Purges every slot in `removed_slots` from both the cache and storage. This includes /// entries in the accounts index, cache entries, and any backing storage entries. pub fn purge_slots_from_cache_and_store<'a>( @@ -5831,7 +5612,6 @@ impl AccountsDb { .safety_checks_elapsed .fetch_add(safety_checks_elapsed.as_us(), Ordering::Relaxed); - let mut total_removed_storage_entries = 0; let mut total_removed_stored_bytes = 0; let mut all_removed_slot_storages = vec![]; @@ -5839,24 +5619,19 @@ impl AccountsDb { for remove_slot in removed_slots { // Remove the storage entries and collect some metrics if let Some(store) = self.storage.remove(remove_slot, false) { - { - total_removed_storage_entries += 1; - total_removed_stored_bytes += store.accounts.capacity(); - } + total_removed_stored_bytes += store.accounts.capacity(); all_removed_slot_storages.push(store); } } remove_storage_entries_elapsed.stop(); let num_stored_slots_removed = all_removed_slot_storages.len(); - let recycle_stores_write_elapsed = - self.recycle_slot_stores(total_removed_storage_entries, &all_removed_slot_storages); - - let mut drop_storage_entries_elapsed = Measure::start("drop_storage_entries_elapsed"); // Backing mmaps for removed storages entries explicitly dropped here outside // of any locks + let mut drop_storage_entries_elapsed = Measure::start("drop_storage_entries_elapsed"); drop(all_removed_slot_storages); drop_storage_entries_elapsed.stop(); + purge_stats .remove_storage_entries_elapsed .fetch_add(remove_storage_entries_elapsed.as_us(), Ordering::Relaxed); @@ -5868,13 +5643,13 @@ impl AccountsDb { .fetch_add(num_stored_slots_removed, Ordering::Relaxed); purge_stats .total_removed_storage_entries - .fetch_add(total_removed_storage_entries, Ordering::Relaxed); + .fetch_add(num_stored_slots_removed, Ordering::Relaxed); purge_stats .total_removed_stored_bytes .fetch_add(total_removed_stored_bytes, Ordering::Relaxed); - purge_stats - .recycle_stores_write_elapsed - .fetch_add(recycle_stores_write_elapsed, Ordering::Relaxed); + self.stats + .dropped_stores + .fetch_add(num_stored_slots_removed as u64, Ordering::Relaxed); } fn purge_slot_cache(&self, purged_slot: Slot, slot_cache: SlotCache) { @@ -6196,12 +5971,7 @@ impl AccountsDb { accounts_and_meta_to_store.len() ); let special_store_size = std::cmp::max(data_len * 2, self.file_size); - if self - .try_recycle_and_insert_store(slot, special_store_size, std::u64::MAX) - .is_none() - { - self.create_and_insert_store(slot, special_store_size, "large create"); - } + self.create_and_insert_store(slot, special_store_size, "large create"); } continue; } @@ -6237,25 +6007,6 @@ impl AccountsDb { self.accounts_cache.report_size(); } - pub fn expire_old_recycle_stores(&self) { - let mut recycle_stores_write_elapsed = Measure::start("recycle_stores_write_time"); - let recycle_stores = self.recycle_stores.write().unwrap().expire_old_entries(); - recycle_stores_write_elapsed.stop(); - - let mut drop_storage_entries_elapsed = Measure::start("drop_storage_entries_elapsed"); - drop(recycle_stores); - drop_storage_entries_elapsed.stop(); - - self.clean_accounts_stats - .purge_stats - .drop_storage_entries_elapsed - .fetch_add(drop_storage_entries_elapsed.as_us(), Ordering::Relaxed); - self.clean_accounts_stats - .purge_stats - .recycle_stores_write_elapsed - .fetch_add(recycle_stores_write_elapsed.as_us(), Ordering::Relaxed); - } - // These functions/fields are only usable from a dev context (i.e. tests and benches) #[cfg(feature = "dev-context-only-utils")] pub fn flush_accounts_cache_slot_for_tests(&self, slot: Slot) { @@ -6779,11 +6530,6 @@ impl AccountsDb { datapoint_info!( "accounts_db-stores", ("total_count", total_count, i64), - ( - "recycle_count", - self.recycle_stores.read().unwrap().entry_count() as u64, - i64 - ), ("total_bytes", total_bytes, i64), ("total_alive_bytes", total_alive_bytes, i64), ("total_alive_ratio", total_alive_ratio, f64), @@ -8410,7 +8156,7 @@ impl AccountsDb { /// Store the account update. /// only called by tests pub fn store_uncached(&self, slot: Slot, accounts: &[(&Pubkey, &AccountSharedData)]) { - let storage = self.find_storage_candidate(slot, 1); + let storage = self.find_storage_candidate(slot); self.store( (slot, accounts), &StoreTo::Storage(&storage), @@ -8568,24 +8314,8 @@ impl AccountsDb { ), ); - let recycle_stores = self.recycle_stores.read().unwrap(); datapoint_info!( "accounts_db_store_timings2", - ( - "recycle_store_count", - self.stats.recycle_store_count.swap(0, Ordering::Relaxed), - i64 - ), - ( - "current_recycle_store_count", - recycle_stores.entry_count(), - i64 - ), - ( - "current_recycle_store_bytes", - recycle_stores.total_bytes(), - i64 - ), ( "create_store_count", self.stats.create_store_count.swap(0, Ordering::Relaxed), @@ -9397,20 +9127,6 @@ impl AccountsDb { pub fn print_accounts_stats(&self, label: &str) { self.print_index(label); self.print_count_and_status(label); - info!("recycle_stores:"); - let recycle_stores = self.recycle_stores.read().unwrap(); - for (recycled_time, entry) in recycle_stores.iter() { - info!( - " slot: {} id: {} count_and_status: {:?} approx_store_count: {} len: {} capacity: {} (recycled: {:?})", - entry.slot(), - entry.append_vec_id(), - entry.count_and_status.read(), - entry.approx_store_count.load(Ordering::Relaxed), - entry.accounts.len(), - entry.accounts.capacity(), - recycled_time, - ); - } } fn print_index(&self, label: &str) { @@ -9766,7 +9482,7 @@ pub mod tests { std::{ iter::FromIterator, str::FromStr, - sync::atomic::AtomicBool, + sync::{atomic::AtomicBool, RwLock}, thread::{self, Builder, JoinHandle}, }, test_case::test_case, @@ -12522,7 +12238,7 @@ pub mod tests { db.store_accounts_unfrozen( (some_slot, &[(&key, &account)][..]), Some(vec![&AccountHash(Hash::default())]), - &StoreTo::Storage(&db.find_storage_candidate(some_slot, 1)), + &StoreTo::Storage(&db.find_storage_candidate(some_slot)), None, StoreReclaims::Default, UpdateIndexThreadSelection::PoolWithThreshold, @@ -12755,7 +12471,7 @@ pub mod tests { db.store_accounts_unfrozen( (some_slot, accounts), Some(vec![&some_hash]), - &StoreTo::Storage(&db.find_storage_candidate(some_slot, 1)), + &StoreTo::Storage(&db.find_storage_candidate(some_slot)), None, StoreReclaims::Default, UpdateIndexThreadSelection::PoolWithThreshold, @@ -13548,75 +13264,6 @@ pub mod tests { assert_eq!(accounts.accounts_index.ref_count_from_storage(&pubkey1), 0); } - #[test] - fn test_store_reuse() { - solana_logger::setup(); - let accounts = AccountsDb { - file_size: 4096, - ..AccountsDb::new_single_for_tests() - }; - - let size = 100; - let num_accounts: usize = 100; - let mut keys = Vec::new(); - for i in 0..num_accounts { - let account = AccountSharedData::new((i + 1) as u64, size, &Pubkey::default()); - let pubkey = solana_sdk::pubkey::new_rand(); - accounts.store_cached((0 as Slot, &[(&pubkey, &account)][..]), None); - keys.push(pubkey); - } - // get delta hash to feed these accounts to clean - accounts.calculate_accounts_delta_hash(0); - accounts.add_root(0); - // we have to flush just slot 0 - // if we slot 0 and 1 together, then they are cleaned and slot 0 doesn't contain the accounts - // this test wants to clean and then allow us to shrink - accounts.flush_accounts_cache(true, None); - - for (i, key) in keys[1..].iter().enumerate() { - let account = - AccountSharedData::new((1 + i + num_accounts) as u64, size, &Pubkey::default()); - accounts.store_cached((1 as Slot, &[(key, &account)][..]), None); - } - accounts.calculate_accounts_delta_hash(1); - accounts.add_root(1); - accounts.flush_accounts_cache(true, None); - accounts.clean_accounts_for_tests(); - accounts.shrink_all_slots(false, None, &EpochSchedule::default()); - - // Clean again to flush the dirty stores - // and allow them to be recycled in the next step - accounts.clean_accounts_for_tests(); - accounts.print_accounts_stats("post-shrink"); - let num_stores = accounts.recycle_stores.read().unwrap().entry_count(); - assert!(num_stores > 0); - - let mut account_refs = Vec::new(); - let num_to_store = 20; - for (i, key) in keys[..num_to_store].iter().enumerate() { - let account = AccountSharedData::new( - (1 + i + 2 * num_accounts) as u64, - i + 20, - &Pubkey::default(), - ); - accounts.store_uncached(2, &[(key, &account)]); - account_refs.push(account); - } - assert!(accounts.recycle_stores.read().unwrap().entry_count() < num_stores); - - accounts.print_accounts_stats("post-store"); - - let mut ancestors = Ancestors::default(); - ancestors.insert(1, 0); - ancestors.insert(2, 1); - for (key, account_ref) in keys[..num_to_store].iter().zip(account_refs) { - assert_eq!( - accounts.load_without_fixed_root(&ancestors, key).unwrap().0, - account_ref - ); - } - } - #[test] #[should_panic(expected = "We've run out of storage ids!")] fn test_wrapping_append_vec_id() { @@ -14870,77 +14517,6 @@ pub mod tests { assert!(!db.storage.is_empty_entry(1)); } - #[test] - fn test_recycle_stores_expiration() { - solana_logger::setup(); - - let common_store_path = Path::new(""); - let common_slot_id = 12; - let store_file_size = 1000; - - let store1_id = 22; - let entry1 = Arc::new(AccountStorageEntry::new( - common_store_path, - common_slot_id, - store1_id, - store_file_size, - )); - - let store2_id = 44; - let entry2 = Arc::new(AccountStorageEntry::new( - common_store_path, - common_slot_id, - store2_id, - store_file_size, - )); - - let mut recycle_stores = RecycleStores::default(); - recycle_stores.add_entry(entry1); - recycle_stores.add_entry(entry2); - assert_eq!(recycle_stores.entry_count(), 2); - - // no expiration for newly added entries - let expired = recycle_stores.expire_old_entries(); - assert_eq!( - expired - .iter() - .map(|e| e.append_vec_id()) - .collect::>(), - Vec::::new() - ); - assert_eq!( - recycle_stores - .iter() - .map(|(_, e)| e.append_vec_id()) - .collect::>(), - vec![store1_id, store2_id] - ); - assert_eq!(recycle_stores.entry_count(), 2); - assert_eq!(recycle_stores.total_bytes(), store_file_size * 2); - - // expiration for only too old entries - recycle_stores.entries[0].0 = Instant::now() - .checked_sub(Duration::from_secs(EXPIRATION_TTL_SECONDS + 1)) - .unwrap(); - let expired = recycle_stores.expire_old_entries(); - assert_eq!( - expired - .iter() - .map(|e| e.append_vec_id()) - .collect::>(), - vec![store1_id] - ); - assert_eq!( - recycle_stores - .iter() - .map(|(_, e)| e.append_vec_id()) - .collect::>(), - vec![store2_id] - ); - assert_eq!(recycle_stores.entry_count(), 1); - assert_eq!(recycle_stores.total_bytes(), store_file_size); - } - const RACY_SLEEP_MS: u64 = 10; const RACE_TIME: u64 = 5; diff --git a/accounts-db/src/ancient_append_vecs.rs b/accounts-db/src/ancient_append_vecs.rs index 1ebcc77763ae27..3925b21e69f586 100644 --- a/accounts-db/src/ancient_append_vecs.rs +++ b/accounts-db/src/ancient_append_vecs.rs @@ -985,7 +985,7 @@ pub mod tests { create_db_with_storages_and_index, create_storages_and_update_index, get_all_accounts, remove_account_for_tests, CAN_RANDOMLY_SHRINK_FALSE, }, - ShrinkCollectRefs, MAX_RECYCLE_STORES, + ShrinkCollectRefs, }, accounts_index::UpsertReclaim, append_vec::{aligned_stored_size, AppendVec, AppendVecStoredAccountMeta}, @@ -3023,6 +3023,9 @@ pub mod tests { #[test] fn test_shrink_packed_ancient() { + // NOTE: The recycler has been removed. Creating this many extra storages is no longer + // necessary, but also does no harm either. + const MAX_RECYCLE_STORES: usize = 1000; solana_logger::setup(); // When we pack ancient append vecs, the packed append vecs are recycled first if possible. This means they aren't dropped directly. diff --git a/runtime/src/accounts_background_service.rs b/runtime/src/accounts_background_service.rs index efc17176d7337b..8910b2f300c4a6 100644 --- a/runtime/src/accounts_background_service.rs +++ b/runtime/src/accounts_background_service.rs @@ -39,14 +39,6 @@ use { const INTERVAL_MS: u64 = 100; const CLEAN_INTERVAL_BLOCKS: u64 = 100; -// This value is chosen to spread the dropping cost over 3 expiration checks -// RecycleStores are fully populated almost all of its lifetime. So, otherwise -// this would drop MAX_RECYCLE_STORES mmaps at once in the worst case... -// (Anyway, the dropping part is outside the AccountsDb::recycle_stores lock -// and dropped in this AccountsBackgroundServe, so this shouldn't matter much) -const RECYCLE_STORE_EXPIRATION_INTERVAL_SECS: u64 = - solana_accounts_db::accounts_db::EXPIRATION_TTL_SECONDS / 3; - pub type SnapshotRequestSender = Sender; pub type SnapshotRequestReceiver = Receiver; pub type DroppedSlotsSender = Sender<(Slot, BankId)>; @@ -605,7 +597,6 @@ impl AccountsBackgroundService { let mut last_cleaned_block_height = 0; let mut removed_slots_count = 0; let mut total_remove_slots_time = 0; - let mut last_expiration_check_time = Instant::now(); let t_background = Builder::new() .name("solBgAccounts".to_string()) .spawn(move || { @@ -631,8 +622,6 @@ impl AccountsBackgroundService { &mut total_remove_slots_time, ); - Self::expire_old_recycle_stores(&bank, &mut last_expiration_check_time); - let non_snapshot_time = last_snapshot_end_time .map(|last_snapshot_end_time: Instant| { last_snapshot_end_time.elapsed().as_micros() @@ -759,16 +748,6 @@ impl AccountsBackgroundService { pub fn join(self) -> thread::Result<()> { self.t_background.join() } - - fn expire_old_recycle_stores(bank: &Bank, last_expiration_check_time: &mut Instant) { - let now = Instant::now(); - if now.duration_since(*last_expiration_check_time).as_secs() - > RECYCLE_STORE_EXPIRATION_INTERVAL_SECS - { - bank.expire_old_recycle_stores(); - *last_expiration_check_time = now; - } - } } /// Get the AccountsPackageKind from a given SnapshotRequest diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index f0ba75defa0517..d1a1805d0d3a20 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -5864,10 +5864,6 @@ impl Bank { .flush_accounts_cache(false, Some(self.slot())) } - pub fn expire_old_recycle_stores(&self) { - self.rc.accounts.accounts_db.expire_old_recycle_stores() - } - /// Technically this issues (or even burns!) new lamports, /// so be extra careful for its usage fn store_account_and_update_capitalization( diff --git a/runtime/src/snapshot_minimizer.rs b/runtime/src/snapshot_minimizer.rs index 556a854da0c41b..15fe706dc0e504 100644 --- a/runtime/src/snapshot_minimizer.rs +++ b/runtime/src/snapshot_minimizer.rs @@ -239,12 +239,8 @@ impl<'a> SnapshotMinimizer<'a> { measure!(self.purge_dead_slots(dead_slots), "purge dead slots"); info!("{purge_dead_slots_measure}"); - let (_, drop_or_recycle_stores_measure) = measure!( - self.accounts_db() - .drop_or_recycle_stores(dead_storages, &self.accounts_db().shrink_stats), - "drop or recycle stores" - ); - info!("{drop_or_recycle_stores_measure}"); + let (_, drop_storages_measure) = measure!(drop(dead_storages), "drop storages"); + info!("{drop_storages_measure}"); // Turn logging back on after minimization self.accounts_db() From ada06ca6ce22ea99ed04985d42a40d87800a982e Mon Sep 17 00:00:00 2001 From: Lucas Steuernagel <38472950+LucasSte@users.noreply.github.com> Date: Mon, 11 Mar 2024 17:30:04 -0300 Subject: [PATCH 50/84] Add tests for `svm/transaction_processor.rs` (#186) --- svm/src/transaction_processor.rs | 434 ++++++++++++++++++++++++++++++- 1 file changed, 431 insertions(+), 3 deletions(-) diff --git a/svm/src/transaction_processor.rs b/svm/src/transaction_processor.rs index fec908619f14f8..5801b3b8316fdc 100644 --- a/svm/src/transaction_processor.rs +++ b/svm/src/transaction_processor.rs @@ -538,6 +538,9 @@ impl TransactionBatchProcessor { .finish_cooperative_loading_task(self.slot, key, program) && limit_to_load_programs { + // This branch is taken when there is an error in assigning a program to a + // cache slot. It is not possible to mock this error for SVM unit + // tests purposes. let mut ret = LoadedProgramsForTxBatch::new( self.slot, loaded_programs_cache @@ -571,6 +574,10 @@ impl TransactionBatchProcessor { // Once a task completes we'll wake up and try to load the // missing programs inside the tx batch again. let _new_cookie = task_waiter.wait(task_cookie); + + // This branch is not tested in the SVM because it requires concurrent threads. + // In addition, one of them must be holding the mutex while the other must be + // trying to lock it. } } @@ -942,7 +949,12 @@ mod tests { loaded_programs::BlockRelation, solana_rbpf::program::BuiltinProgram, }, solana_sdk::{ - account::WritableAccount, bpf_loader, sysvar::rent::Rent, + account::WritableAccount, + bpf_loader, + message::{LegacyMessage, Message, MessageHeader}, + rent_debits::RentDebits, + signature::Signature, + sysvar::rent::Rent, transaction_context::TransactionContext, }, std::{ @@ -960,7 +972,7 @@ mod tests { } } - #[derive(Default)] + #[derive(Default, Clone)] pub struct MockBankCallback { rent_collector: RentCollector, feature_set: Arc, @@ -985,7 +997,7 @@ mod tests { } fn get_last_blockhash_and_lamports_per_signature(&self) -> (Hash, u64) { - todo!() + (Hash::new_unique(), 2) } fn get_rent_collector(&self) -> &RentCollector { @@ -1521,4 +1533,420 @@ mod tests { let slot = batch_processor.epoch_schedule.get_first_slot_in_epoch(20); assert_eq!(result.effective_slot, slot); } + + #[test] + fn test_program_modification_slot_account_not_found() { + let batch_processor = TransactionBatchProcessor::::default(); + let mut mock_bank = MockBankCallback::default(); + let key = Pubkey::new_unique(); + + let result = batch_processor.program_modification_slot(&mock_bank, &key); + assert_eq!(result.err(), Some(TransactionError::ProgramAccountNotFound)); + + let mut account_data = AccountSharedData::default(); + account_data.set_owner(bpf_loader_upgradeable::id()); + mock_bank + .account_shared_data + .insert(key, account_data.clone()); + + let result = batch_processor.program_modification_slot(&mock_bank, &key); + assert_eq!(result.err(), Some(TransactionError::ProgramAccountNotFound)); + + let state = UpgradeableLoaderState::Program { + programdata_address: Pubkey::new_unique(), + }; + account_data.set_data(bincode::serialize(&state).unwrap()); + mock_bank + .account_shared_data + .insert(key, account_data.clone()); + + let result = batch_processor.program_modification_slot(&mock_bank, &key); + assert_eq!(result.err(), Some(TransactionError::ProgramAccountNotFound)); + + account_data.set_owner(loader_v4::id()); + mock_bank + .account_shared_data + .insert(key, account_data.clone()); + + let result = batch_processor.program_modification_slot(&mock_bank, &key); + assert_eq!(result.err(), Some(TransactionError::ProgramAccountNotFound)); + } + + #[test] + fn test_program_modification_slot_success() { + let batch_processor = TransactionBatchProcessor::::default(); + let mut mock_bank = MockBankCallback::default(); + let key1 = Pubkey::new_unique(); + let key2 = Pubkey::new_unique(); + let mut account_data = AccountSharedData::default(); + account_data.set_owner(bpf_loader_upgradeable::id()); + + let state = UpgradeableLoaderState::Program { + programdata_address: key2, + }; + account_data.set_data(bincode::serialize(&state).unwrap()); + mock_bank.account_shared_data.insert(key1, account_data); + + let state = UpgradeableLoaderState::ProgramData { + slot: 77, + upgrade_authority_address: None, + }; + let mut account_data = AccountSharedData::default(); + account_data.set_data(bincode::serialize(&state).unwrap()); + mock_bank.account_shared_data.insert(key2, account_data); + + let result = batch_processor.program_modification_slot(&mock_bank, &key1); + assert_eq!(result.unwrap(), 77); + + let mut account_data = AccountSharedData::default(); + account_data.set_owner(loader_v4::id()); + let state = LoaderV4State { + slot: 58, + authority_address: Pubkey::new_unique(), + status: LoaderV4Status::Deployed, + }; + + let encoded = unsafe { + std::mem::transmute::<&LoaderV4State, &[u8; LoaderV4State::program_data_offset()]>( + &state, + ) + }; + account_data.set_data(encoded.to_vec()); + mock_bank + .account_shared_data + .insert(key1, account_data.clone()); + + let result = batch_processor.program_modification_slot(&mock_bank, &key1); + assert_eq!(result.unwrap(), 58); + + account_data.set_owner(Pubkey::new_unique()); + mock_bank.account_shared_data.insert(key2, account_data); + + let result = batch_processor.program_modification_slot(&mock_bank, &key2); + assert_eq!(result.unwrap(), 0); + } + + #[test] + fn test_execute_loaded_transaction_recordings() { + // Setting all the arguments correctly is too burdensome for testing + // execute_loaded_transaction separately.This function will be tested in an integration + // test with load_and_execute_sanitized_transactions + let message = Message { + account_keys: vec![Pubkey::new_from_array([0; 32])], + header: MessageHeader::default(), + instructions: vec![CompiledInstruction { + program_id_index: 0, + accounts: vec![], + data: vec![], + }], + recent_blockhash: Hash::default(), + }; + + let legacy = LegacyMessage::new(message); + let sanitized_message = SanitizedMessage::Legacy(legacy); + let loaded_programs = LoadedProgramsForTxBatch::default(); + let mock_bank = MockBankCallback::default(); + let batch_processor = TransactionBatchProcessor::::default(); + + let sanitized_transaction = SanitizedTransaction::new_for_tests( + sanitized_message, + vec![Signature::new_unique()], + false, + ); + + let mut loaded_transaction = LoadedTransaction { + accounts: vec![(Pubkey::new_unique(), AccountSharedData::default())], + program_indices: vec![vec![0]], + rent: 0, + rent_debits: RentDebits::default(), + }; + + let mut record_config = ExecutionRecordingConfig { + enable_cpi_recording: false, + enable_log_recording: true, + enable_return_data_recording: false, + }; + + let result = batch_processor.execute_loaded_transaction( + &mock_bank, + &sanitized_transaction, + &mut loaded_transaction, + ComputeBudget::default(), + None, + record_config, + &mut ExecuteTimings::default(), + &mut TransactionErrorMetrics::default(), + None, + &loaded_programs, + ); + + let TransactionExecutionResult::Executed { + details: TransactionExecutionDetails { log_messages, .. }, + .. + } = result + else { + panic!("Unexpected result") + }; + assert!(log_messages.is_some()); + + let result = batch_processor.execute_loaded_transaction( + &mock_bank, + &sanitized_transaction, + &mut loaded_transaction, + ComputeBudget::default(), + None, + record_config, + &mut ExecuteTimings::default(), + &mut TransactionErrorMetrics::default(), + Some(2), + &loaded_programs, + ); + + let TransactionExecutionResult::Executed { + details: + TransactionExecutionDetails { + log_messages, + inner_instructions, + .. + }, + .. + } = result + else { + panic!("Unexpected result") + }; + assert!(log_messages.is_some()); + assert!(inner_instructions.is_none()); + + record_config.enable_log_recording = false; + record_config.enable_cpi_recording = true; + + let result = batch_processor.execute_loaded_transaction( + &mock_bank, + &sanitized_transaction, + &mut loaded_transaction, + ComputeBudget::default(), + None, + record_config, + &mut ExecuteTimings::default(), + &mut TransactionErrorMetrics::default(), + None, + &loaded_programs, + ); + + let TransactionExecutionResult::Executed { + details: + TransactionExecutionDetails { + log_messages, + inner_instructions, + .. + }, + .. + } = result + else { + panic!("Unexpected result") + }; + assert!(log_messages.is_none()); + assert!(inner_instructions.is_some()); + } + + #[test] + fn test_execute_loaded_transaction_error_metrics() { + // Setting all the arguments correctly is too burdensome for testing + // execute_loaded_transaction separately.This function will be tested in an integration + // test with load_and_execute_sanitized_transactions + let key1 = Pubkey::new_unique(); + let key2 = Pubkey::new_unique(); + let message = Message { + account_keys: vec![key1, key2], + header: MessageHeader::default(), + instructions: vec![CompiledInstruction { + program_id_index: 0, + accounts: vec![2], + data: vec![], + }], + recent_blockhash: Hash::default(), + }; + + let legacy = LegacyMessage::new(message); + let sanitized_message = SanitizedMessage::Legacy(legacy); + let loaded_programs = LoadedProgramsForTxBatch::default(); + let mock_bank = MockBankCallback::default(); + let batch_processor = TransactionBatchProcessor::::default(); + + let sanitized_transaction = SanitizedTransaction::new_for_tests( + sanitized_message, + vec![Signature::new_unique()], + false, + ); + + let mut account_data = AccountSharedData::default(); + account_data.set_owner(bpf_loader::id()); + let mut loaded_transaction = LoadedTransaction { + accounts: vec![ + (key1, AccountSharedData::default()), + (key2, AccountSharedData::default()), + ], + program_indices: vec![vec![0]], + rent: 0, + rent_debits: RentDebits::default(), + }; + + let record_config = ExecutionRecordingConfig::new_single_setting(false); + let mut error_metrics = TransactionErrorMetrics::new(); + + let _ = batch_processor.execute_loaded_transaction( + &mock_bank, + &sanitized_transaction, + &mut loaded_transaction, + ComputeBudget::default(), + None, + record_config, + &mut ExecuteTimings::default(), + &mut error_metrics, + None, + &loaded_programs, + ); + + assert_eq!(error_metrics.instruction_error, 1); + } + + #[test] + fn test_replenish_program_cache() { + // Case 1 + let mut mock_bank = MockBankCallback::default(); + let mut batch_processor = TransactionBatchProcessor:: { + check_program_modification_slot: true, + ..TransactionBatchProcessor::default() + }; + batch_processor + .loaded_programs_cache + .write() + .unwrap() + .fork_graph = Some(Arc::new(RwLock::new(TestForkGraph {}))); + let key1 = Pubkey::new_unique(); + let key2 = Pubkey::new_unique(); + let owner = Pubkey::new_unique(); + + let mut account_data = AccountSharedData::default(); + account_data.set_owner(bpf_loader::id()); + mock_bank.account_shared_data.insert(key2, account_data); + + let mut account_maps: HashMap = HashMap::new(); + account_maps.insert(key1, (&owner, 2)); + + account_maps.insert(key2, (&owner, 4)); + let result = batch_processor.replenish_program_cache(&mock_bank, &account_maps, false); + + let program1 = result.find(&key1).unwrap(); + assert!(matches!(program1.program, LoadedProgramType::Closed)); + assert!(!result.hit_max_limit); + let program2 = result.find(&key2).unwrap(); + assert!(matches!( + program2.program, + LoadedProgramType::FailedVerification(_) + )); + + // Case 2 + batch_processor.check_program_modification_slot = false; + + let result = batch_processor.replenish_program_cache(&mock_bank, &account_maps, true); + + let program1 = result.find(&key1).unwrap(); + assert!(matches!(program1.program, LoadedProgramType::Closed)); + assert!(!result.hit_max_limit); + let program2 = result.find(&key2).unwrap(); + assert!(matches!( + program2.program, + LoadedProgramType::FailedVerification(_) + )); + } + + #[test] + fn test_filter_executable_program_accounts() { + let mut mock_bank = MockBankCallback::default(); + let key1 = Pubkey::new_unique(); + let owner1 = Pubkey::new_unique(); + + let mut data = AccountSharedData::default(); + data.set_owner(owner1); + data.set_lamports(93); + mock_bank.account_shared_data.insert(key1, data); + + let message = Message { + account_keys: vec![key1], + header: MessageHeader::default(), + instructions: vec![CompiledInstruction { + program_id_index: 0, + accounts: vec![], + data: vec![], + }], + recent_blockhash: Hash::default(), + }; + + let legacy = LegacyMessage::new(message); + let sanitized_message = SanitizedMessage::Legacy(legacy); + + let sanitized_transaction_1 = SanitizedTransaction::new_for_tests( + sanitized_message, + vec![Signature::new_unique()], + false, + ); + + let key2 = Pubkey::new_unique(); + let owner2 = Pubkey::new_unique(); + + let mut account_data = AccountSharedData::default(); + account_data.set_owner(owner2); + account_data.set_lamports(90); + mock_bank.account_shared_data.insert(key2, account_data); + + let message = Message { + account_keys: vec![key1, key2], + header: MessageHeader::default(), + instructions: vec![CompiledInstruction { + program_id_index: 0, + accounts: vec![], + data: vec![], + }], + recent_blockhash: Hash::default(), + }; + + let legacy = LegacyMessage::new(message); + let sanitized_message = SanitizedMessage::Legacy(legacy); + + let sanitized_transaction_2 = SanitizedTransaction::new_for_tests( + sanitized_message, + vec![Signature::new_unique()], + false, + ); + + let transactions = vec![ + sanitized_transaction_1.clone(), + sanitized_transaction_2.clone(), + sanitized_transaction_2, + sanitized_transaction_1, + ]; + let mut lock_results = vec![ + (Ok(()), None, Some(25)), + (Ok(()), None, Some(25)), + (Ok(()), None, None), + (Err(TransactionError::ProgramAccountNotFound), None, None), + ]; + let owners = vec![owner1, owner2]; + + let result = TransactionBatchProcessor::::filter_executable_program_accounts( + &mock_bank, + &transactions, + lock_results.as_mut_slice(), + &owners, + ); + + assert_eq!( + lock_results[2], + (Err(TransactionError::BlockhashNotFound), None, None) + ); + assert_eq!(result.len(), 2); + assert_eq!(result[&key1], (&owner1, 2)); + assert_eq!(result[&key2], (&owner2, 1)); + } } From 1a085c8d46ebb95d203a69c00773d30233e4a611 Mon Sep 17 00:00:00 2001 From: Brooks Date: Mon, 11 Mar 2024 17:09:08 -0400 Subject: [PATCH 51/84] Removes atomic-ness from AccountStorageEntry `id` and `slot` fields (#119) --- accounts-db/src/accounts_db.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/accounts-db/src/accounts_db.rs b/accounts-db/src/accounts_db.rs index 18ffa2d02e37ca..eab5ca33af417c 100644 --- a/accounts-db/src/accounts_db.rs +++ b/accounts-db/src/accounts_db.rs @@ -1005,9 +1005,9 @@ struct CleanKeyTimings { /// Persistent storage structure holding the accounts #[derive(Debug)] pub struct AccountStorageEntry { - pub(crate) id: AtomicAppendVecId, + pub(crate) id: AppendVecId, - pub(crate) slot: AtomicU64, + pub(crate) slot: Slot, /// storage holding the accounts pub accounts: AccountsFile, @@ -1037,8 +1037,8 @@ impl AccountStorageEntry { let accounts = AccountsFile::AppendVec(AppendVec::new(&path, true, file_size as usize)); Self { - id: AtomicAppendVecId::new(id), - slot: AtomicU64::new(slot), + id, + slot, accounts, count_and_status: SeqLock::new((0, AccountStorageStatus::Available)), approx_store_count: AtomicUsize::new(0), @@ -1053,8 +1053,8 @@ impl AccountStorageEntry { num_accounts: usize, ) -> Self { Self { - id: AtomicAppendVecId::new(id), - slot: AtomicU64::new(slot), + id, + slot, accounts, count_and_status: SeqLock::new((0, AccountStorageStatus::Available)), approx_store_count: AtomicUsize::new(num_accounts), @@ -1112,11 +1112,11 @@ impl AccountStorageEntry { } pub fn slot(&self) -> Slot { - self.slot.load(Ordering::Acquire) + self.slot } pub fn append_vec_id(&self) -> AppendVecId { - self.id.load(Ordering::Acquire) + self.id } pub fn flush(&self) -> Result<(), AccountsFileError> { From 88f6a7a45997fd5f56e5f9a0b4432f52f9f2a8b2 Mon Sep 17 00:00:00 2001 From: Brooks Date: Mon, 11 Mar 2024 17:09:26 -0400 Subject: [PATCH 52/84] Removes holding storages in AccountsHashVerifier for fastboot (#120) --- accounts-db/src/lib.rs | 1 - accounts-db/src/starting_snapshot_storages.rs | 19 ----- core/src/accounts_hash_verifier.rs | 44 ---------- core/src/validator.rs | 43 ++++------ core/tests/epoch_accounts_hash.rs | 2 - core/tests/snapshots.rs | 2 - ledger-tool/src/ledger_utils.rs | 32 +++----- ledger/src/bank_forks_utils.rs | 81 +++++-------------- 8 files changed, 52 insertions(+), 172 deletions(-) delete mode 100644 accounts-db/src/starting_snapshot_storages.rs diff --git a/accounts-db/src/lib.rs b/accounts-db/src/lib.rs index b7994fe4354118..7883f852d1e3f2 100644 --- a/accounts-db/src/lib.rs +++ b/accounts-db/src/lib.rs @@ -37,7 +37,6 @@ pub mod secondary_index; pub mod shared_buffer_reader; pub mod sorted_storages; pub mod stake_rewards; -pub mod starting_snapshot_storages; pub mod storable_accounts; pub mod tiered_storage; pub mod utils; diff --git a/accounts-db/src/starting_snapshot_storages.rs b/accounts-db/src/starting_snapshot_storages.rs deleted file mode 100644 index cc5e26c61872b7..00000000000000 --- a/accounts-db/src/starting_snapshot_storages.rs +++ /dev/null @@ -1,19 +0,0 @@ -use {crate::accounts_db::AccountStorageEntry, std::sync::Arc}; - -/// Snapshot storages that the node loaded from -/// -/// This is used to support fastboot. Since fastboot reuses existing storages, we must carefully -/// handle the storages used to load at startup. If we do not handle these storages properly, -/// restarting from the same local state (i.e. bank snapshot) may fail. -#[derive(Debug)] -pub enum StartingSnapshotStorages { - /// Starting from genesis has no storages yet - Genesis, - /// Starting from a snapshot archive always extracts the storages from the archive, so no - /// special handling is necessary to preserve them. - Archive, - /// Starting from local state must preserve the loaded storages. These storages must *not* be - /// recycled or removed prior to taking the next snapshot, otherwise restarting from the same - /// bank snapshot may fail. - Fastboot(Vec>), -} diff --git a/core/src/accounts_hash_verifier.rs b/core/src/accounts_hash_verifier.rs index f5572d94a3c7d1..20adba99835eeb 100644 --- a/core/src/accounts_hash_verifier.rs +++ b/core/src/accounts_hash_verifier.rs @@ -9,7 +9,6 @@ use { IncrementalAccountsHash, }, sorted_storages::SortedStorages, - starting_snapshot_storages::StartingSnapshotStorages, }, solana_measure::measure_us, solana_runtime::{ @@ -43,7 +42,6 @@ impl AccountsHashVerifier { accounts_package_sender: Sender, accounts_package_receiver: Receiver, snapshot_package_sender: Option>, - starting_snapshot_storages: StartingSnapshotStorages, exit: Arc, snapshot_config: SnapshotConfig, ) -> Self { @@ -53,14 +51,6 @@ impl AccountsHashVerifier { .name("solAcctHashVer".to_string()) .spawn(move || { info!("AccountsHashVerifier has started"); - // To support fastboot, we must ensure the storages used in the latest POST snapshot are - // not recycled nor removed early. Hold an Arc of their AppendVecs to prevent them from - // expiring. - let mut fastboot_storages = match starting_snapshot_storages { - StartingSnapshotStorages::Genesis => None, - StartingSnapshotStorages::Archive => None, - StartingSnapshotStorages::Fastboot(storages) => Some(storages), - }; loop { if exit.load(Ordering::Relaxed) { break; @@ -81,14 +71,6 @@ impl AccountsHashVerifier { info!("handling accounts package: {accounts_package:?}"); let enqueued_time = accounts_package.enqueued.elapsed(); - // If this accounts package is for a snapshot, then clone the storages to - // save for fastboot. - let snapshot_storages_for_fastboot = accounts_package - .snapshot_info - .is_some() - .then(|| accounts_package.snapshot_storages.clone()); - - let slot = accounts_package.slot; let (_, handling_time_us) = measure_us!(Self::process_accounts_package( accounts_package, snapshot_package_sender.as_ref(), @@ -96,25 +78,6 @@ impl AccountsHashVerifier { &exit, )); - if let Some(snapshot_storages_for_fastboot) = snapshot_storages_for_fastboot { - // Get the number of storages that are being kept alive for fastboot. - // Looking at the storage Arc's strong reference count, we know that one - // ref is for fastboot, and one ref is for snapshot packaging. If there - // are no others, then the storage will be kept alive because of fastboot. - let num_storages_kept_alive = snapshot_storages_for_fastboot - .iter() - .filter(|storage| Arc::strong_count(storage) == 2) - .count(); - let num_storages_total = snapshot_storages_for_fastboot.len(); - fastboot_storages = Some(snapshot_storages_for_fastboot); - datapoint_info!( - "fastboot", - ("slot", slot, i64), - ("num_storages_total", num_storages_total, i64), - ("num_storages_kept_alive", num_storages_kept_alive, i64), - ); - } - datapoint_info!( "accounts_hash_verifier", ( @@ -132,13 +95,6 @@ impl AccountsHashVerifier { ); } info!("AccountsHashVerifier has stopped"); - debug!( - "Number of storages kept alive for fastboot: {}", - fastboot_storages - .as_ref() - .map(|storages| storages.len()) - .unwrap_or(0) - ); }) .unwrap(); Self { diff --git a/core/src/validator.rs b/core/src/validator.rs index a1c8293f86cb3a..3d2a93daecba2f 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -35,7 +35,6 @@ use { accounts_index::AccountSecondaryIndexes, accounts_update_notifier_interface::AccountsUpdateNotifier, hardened_unpack::{open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE}, - starting_snapshot_storages::StartingSnapshotStorages, utils::{move_and_async_delete_path, move_and_async_delete_path_contents}, }, solana_client::connection_cache::{ConnectionCache, Protocol}, @@ -691,7 +690,6 @@ impl Validator { completed_slots_receiver, leader_schedule_cache, starting_snapshot_hashes, - starting_snapshot_storages, TransactionHistoryServices { transaction_status_sender, transaction_status_service, @@ -781,7 +779,6 @@ impl Validator { accounts_package_sender.clone(), accounts_package_receiver, snapshot_package_sender, - starting_snapshot_storages, exit.clone(), config.snapshot_config.clone(), ); @@ -1770,7 +1767,6 @@ fn load_blockstore( CompletedSlotsReceiver, LeaderScheduleCache, Option, - StartingSnapshotStorages, TransactionHistoryServices, blockstore_processor::ProcessOptions, BlockstoreRootScan, @@ -1860,27 +1856,23 @@ fn load_blockstore( let entry_notifier_service = entry_notifier .map(|entry_notifier| EntryNotifierService::new(entry_notifier, exit.clone())); - let ( - bank_forks, - mut leader_schedule_cache, - starting_snapshot_hashes, - starting_snapshot_storages, - ) = bank_forks_utils::load_bank_forks( - &genesis_config, - &blockstore, - config.account_paths.clone(), - Some(&config.snapshot_config), - &process_options, - transaction_history_services - .cache_block_meta_sender - .as_ref(), - entry_notifier_service - .as_ref() - .map(|service| service.sender()), - accounts_update_notifier, - exit, - ) - .map_err(|err| err.to_string())?; + let (bank_forks, mut leader_schedule_cache, starting_snapshot_hashes) = + bank_forks_utils::load_bank_forks( + &genesis_config, + &blockstore, + config.account_paths.clone(), + Some(&config.snapshot_config), + &process_options, + transaction_history_services + .cache_block_meta_sender + .as_ref(), + entry_notifier_service + .as_ref() + .map(|service| service.sender()), + accounts_update_notifier, + exit, + ) + .map_err(|err| err.to_string())?; // Before replay starts, set the callbacks in each of the banks in BankForks so that // all dropped banks come through the `pruned_banks_receiver` channel. This way all bank @@ -1906,7 +1898,6 @@ fn load_blockstore( completed_slots_receiver, leader_schedule_cache, starting_snapshot_hashes, - starting_snapshot_storages, transaction_history_services, process_options, blockstore_root_scan, diff --git a/core/tests/epoch_accounts_hash.rs b/core/tests/epoch_accounts_hash.rs index 25e97689923bb0..76b5e4c30dd018 100755 --- a/core/tests/epoch_accounts_hash.rs +++ b/core/tests/epoch_accounts_hash.rs @@ -9,7 +9,6 @@ use { accounts_hash::CalcAccountsHashConfig, accounts_index::AccountSecondaryIndexes, epoch_accounts_hash::EpochAccountsHash, - starting_snapshot_storages::StartingSnapshotStorages, }, solana_core::{ accounts_hash_verifier::AccountsHashVerifier, @@ -197,7 +196,6 @@ impl BackgroundServices { accounts_package_sender.clone(), accounts_package_receiver, Some(snapshot_package_sender), - StartingSnapshotStorages::Genesis, exit.clone(), snapshot_config.clone(), ); diff --git a/core/tests/snapshots.rs b/core/tests/snapshots.rs index 1607ebd3fa2094..730277e2c12a65 100644 --- a/core/tests/snapshots.rs +++ b/core/tests/snapshots.rs @@ -11,7 +11,6 @@ use { accounts_hash::AccountsHash, accounts_index::AccountSecondaryIndexes, epoch_accounts_hash::EpochAccountsHash, - starting_snapshot_storages::StartingSnapshotStorages, }, solana_core::{ accounts_hash_verifier::AccountsHashVerifier, @@ -1044,7 +1043,6 @@ fn test_snapshots_with_background_services( accounts_package_sender, accounts_package_receiver, Some(snapshot_package_sender), - StartingSnapshotStorages::Genesis, exit.clone(), snapshot_test_config.snapshot_config.clone(), ); diff --git a/ledger-tool/src/ledger_utils.rs b/ledger-tool/src/ledger_utils.rs index 8a8302d7e4e94b..c05cc6c2d64cd0 100644 --- a/ledger-tool/src/ledger_utils.rs +++ b/ledger-tool/src/ledger_utils.rs @@ -268,24 +268,19 @@ pub fn load_and_process_ledger( }; let exit = Arc::new(AtomicBool::new(false)); - let ( - bank_forks, - leader_schedule_cache, - starting_snapshot_hashes, - starting_snapshot_storages, - .., - ) = bank_forks_utils::load_bank_forks( - genesis_config, - blockstore.as_ref(), - account_paths, - snapshot_config.as_ref(), - &process_options, - None, - None, // Maybe support this later, though - accounts_update_notifier, - exit.clone(), - ) - .map_err(LoadAndProcessLedgerError::LoadBankForks)?; + let (bank_forks, leader_schedule_cache, starting_snapshot_hashes, ..) = + bank_forks_utils::load_bank_forks( + genesis_config, + blockstore.as_ref(), + account_paths, + snapshot_config.as_ref(), + &process_options, + None, + None, // Maybe support this later, though + accounts_update_notifier, + exit.clone(), + ) + .map_err(LoadAndProcessLedgerError::LoadBankForks)?; let block_verification_method = value_t!( arg_matches, "block_verification_method", @@ -330,7 +325,6 @@ pub fn load_and_process_ledger( accounts_package_sender.clone(), accounts_package_receiver, None, - starting_snapshot_storages, exit.clone(), SnapshotConfig::new_load_only(), ); diff --git a/ledger/src/bank_forks_utils.rs b/ledger/src/bank_forks_utils.rs index b30f90986bb9c2..17412c1801ac68 100644 --- a/ledger/src/bank_forks_utils.rs +++ b/ledger/src/bank_forks_utils.rs @@ -10,10 +10,7 @@ use { use_snapshot_archives_at_startup::{self, UseSnapshotArchivesAtStartup}, }, log::*, - solana_accounts_db::{ - accounts_update_notifier_interface::AccountsUpdateNotifier, - starting_snapshot_storages::StartingSnapshotStorages, - }, + solana_accounts_db::accounts_update_notifier_interface::AccountsUpdateNotifier, solana_runtime::{ accounts_background_service::AbsRequestSender, bank_forks::BankForks, @@ -70,7 +67,6 @@ pub type LoadResult = result::Result< Arc>, LeaderScheduleCache, Option, - StartingSnapshotStorages, ), BankForksUtilsError, >; @@ -92,13 +88,7 @@ pub fn load( accounts_update_notifier: Option, exit: Arc, ) -> LoadResult { - let ( - bank_forks, - leader_schedule_cache, - starting_snapshot_hashes, - starting_snapshot_storages, - .., - ) = load_bank_forks( + let (bank_forks, leader_schedule_cache, starting_snapshot_hashes, ..) = load_bank_forks( genesis_config, blockstore, account_paths, @@ -121,12 +111,7 @@ pub fn load( ) .map_err(BankForksUtilsError::ProcessBlockstoreFromRoot)?; - Ok(( - bank_forks, - leader_schedule_cache, - starting_snapshot_hashes, - starting_snapshot_storages, - )) + Ok((bank_forks, leader_schedule_cache, starting_snapshot_hashes)) } #[allow(clippy::too_many_arguments)] @@ -176,7 +161,7 @@ pub fn load_bank_forks( )) } - let (bank_forks, starting_snapshot_hashes, starting_snapshot_storages) = + let (bank_forks, starting_snapshot_hashes) = if let Some((full_snapshot_archive_info, incremental_snapshot_archive_info)) = get_snapshots_to_load(snapshot_config) { @@ -188,22 +173,17 @@ pub fn load_bank_forks( ); std::fs::create_dir_all(&snapshot_config.bank_snapshots_dir) .expect("create bank snapshots dir"); - let (bank_forks, starting_snapshot_hashes, starting_snapshot_storages) = - bank_forks_from_snapshot( - full_snapshot_archive_info, - incremental_snapshot_archive_info, - genesis_config, - account_paths, - snapshot_config, - process_options, - accounts_update_notifier, - exit, - )?; - ( - bank_forks, - Some(starting_snapshot_hashes), - starting_snapshot_storages, - ) + let (bank_forks, starting_snapshot_hashes) = bank_forks_from_snapshot( + full_snapshot_archive_info, + incremental_snapshot_archive_info, + genesis_config, + account_paths, + snapshot_config, + process_options, + accounts_update_notifier, + exit, + )?; + (bank_forks, Some(starting_snapshot_hashes)) } else { info!("Processing ledger from genesis"); let bank_forks = blockstore_processor::process_blockstore_for_bank_0( @@ -222,7 +202,7 @@ pub fn load_bank_forks( .root_bank() .set_startup_verification_complete(); - (bank_forks, None, StartingSnapshotStorages::Genesis) + (bank_forks, None) }; let mut leader_schedule_cache = @@ -238,12 +218,7 @@ pub fn load_bank_forks( .for_each(|hard_fork_slot| root_bank.register_hard_fork(*hard_fork_slot)); } - Ok(( - bank_forks, - leader_schedule_cache, - starting_snapshot_hashes, - starting_snapshot_storages, - )) + Ok((bank_forks, leader_schedule_cache, starting_snapshot_hashes)) } #[allow(clippy::too_many_arguments)] @@ -256,14 +231,7 @@ fn bank_forks_from_snapshot( process_options: &ProcessOptions, accounts_update_notifier: Option, exit: Arc, -) -> Result< - ( - Arc>, - StartingSnapshotHashes, - StartingSnapshotStorages, - ), - BankForksUtilsError, -> { +) -> Result<(Arc>, StartingSnapshotHashes), BankForksUtilsError> { // Fail hard here if snapshot fails to load, don't silently continue if account_paths.is_empty() { return Err(BankForksUtilsError::AccountPathsNotPresent); @@ -289,7 +257,7 @@ fn bank_forks_from_snapshot( .unwrap_or(true), }; - let (bank, starting_snapshot_storages) = if will_startup_from_snapshot_archives { + let bank = if will_startup_from_snapshot_archives { // Given that we are going to boot from an archive, the append vecs held in the snapshot dirs for fast-boot should // be released. They will be released by the account_background_service anyway. But in the case of the account_paths // using memory-mounted file system, they are not released early enough to give space for the new append-vecs from @@ -324,7 +292,7 @@ fn bank_forks_from_snapshot( .map(|archive| archive.path().display().to_string()) .unwrap_or("none".to_string()), })?; - (bank, StartingSnapshotStorages::Archive) + bank } else { let bank_snapshot = latest_bank_snapshot.ok_or_else(|| BankForksUtilsError::NoBankSnapshotDirectory { @@ -378,8 +346,7 @@ fn bank_forks_from_snapshot( // snapshot archive next time, which is safe. snapshot_utils::purge_all_bank_snapshots(&snapshot_config.bank_snapshots_dir); - let storages = bank.get_snapshot_storages(None); - (bank, StartingSnapshotStorages::Fastboot(storages)) + bank }; let full_snapshot_hash = FullSnapshotHash(( @@ -398,9 +365,5 @@ fn bank_forks_from_snapshot( incremental: incremental_snapshot_hash, }; - Ok(( - BankForks::new_rw_arc(bank), - starting_snapshot_hashes, - starting_snapshot_storages, - )) + Ok((BankForks::new_rw_arc(bank), starting_snapshot_hashes)) } From 218de23ce22a9112e53a38b37e184d3e0659ab3e Mon Sep 17 00:00:00 2001 From: Greg Cusack Date: Mon, 11 Mar 2024 18:19:48 -0400 Subject: [PATCH 53/84] Remove `ThinClient` from `dos/` (#117) * remove `ThinClient` from `dos/` and replace `ThinClient` with `TpuClient` * remove test for valid_client_facing_addr since it is no longer used --- Cargo.lock | 4 +- client/src/tpu_client.rs | 6 +++ dos/Cargo.toml | 2 +- dos/src/main.rs | 83 ++++++++++++++++++------------- gossip/Cargo.toml | 2 +- gossip/src/gossip_service.rs | 60 ++++++++++++---------- gossip/src/legacy_contact_info.rs | 33 ------------ programs/sbf/Cargo.lock | 2 +- 8 files changed, 92 insertions(+), 100 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 88f0fa0925dcac..46091cfbca5e82 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6077,11 +6077,11 @@ dependencies = [ "solana-measure", "solana-net-utils", "solana-perf", + "solana-quic-client", "solana-rpc", "solana-rpc-client", "solana-sdk", "solana-streamer", - "solana-thin-client", "solana-tpu-client", "solana-version", ] @@ -6276,6 +6276,7 @@ dependencies = [ "solana-bloom", "solana-clap-utils", "solana-client", + "solana-connection-cache", "solana-entry", "solana-frozen-abi", "solana-frozen-abi-macro", @@ -6289,7 +6290,6 @@ dependencies = [ "solana-runtime", "solana-sdk", "solana-streamer", - "solana-thin-client", "solana-tpu-client", "solana-version", "solana-vote", diff --git a/client/src/tpu_client.rs b/client/src/tpu_client.rs index 45394151340070..038dd86774ea98 100644 --- a/client/src/tpu_client.rs +++ b/client/src/tpu_client.rs @@ -13,6 +13,7 @@ use { transport::Result as TransportResult, }, solana_tpu_client::tpu_client::{Result, TpuClient as BackendTpuClient}, + solana_udp_client::{UdpConfig, UdpConnectionManager, UdpPool}, std::sync::Arc, }; pub use { @@ -20,6 +21,11 @@ pub use { solana_tpu_client::tpu_client::{TpuClientConfig, DEFAULT_FANOUT_SLOTS, MAX_FANOUT_SLOTS}, }; +pub enum TpuClientWrapper { + Quic(TpuClient), + Udp(TpuClient), +} + /// Client which sends transactions directly to the current leader's TPU port over UDP. /// The client uses RPC to determine the current leader and fetch node contact info /// This is just a thin wrapper over the "BackendTpuClient", use that directly for more efficiency. diff --git a/dos/Cargo.toml b/dos/Cargo.toml index 179fc40bf84820..0d7c76b007c4ea 100644 --- a/dos/Cargo.toml +++ b/dos/Cargo.toml @@ -26,6 +26,7 @@ solana-logger = { workspace = true } solana-measure = { workspace = true } solana-net-utils = { workspace = true } solana-perf = { workspace = true } +solana-quic-client = { workspace = true } solana-rpc = { workspace = true } solana-rpc-client = { workspace = true } solana-sdk = { workspace = true } @@ -38,4 +39,3 @@ targets = ["x86_64-unknown-linux-gnu"] [dev-dependencies] solana-local-cluster = { workspace = true } -solana-thin-client = { workspace = true } diff --git a/dos/src/main.rs b/dos/src/main.rs index b9e0dceba40bf0..055b1f4bb65d4c 100644 --- a/dos/src/main.rs +++ b/dos/src/main.rs @@ -46,12 +46,15 @@ use { log::*, rand::{thread_rng, Rng}, solana_bench_tps::{bench::generate_and_fund_keypairs, bench_tps_client::BenchTpsClient}, - solana_client::{connection_cache::ConnectionCache, tpu_connection::TpuConnection}, + solana_client::{ + connection_cache::ConnectionCache, tpu_client::TpuClientWrapper, + tpu_connection::TpuConnection, + }, solana_core::repair::serve_repair::{RepairProtocol, RepairRequestHeader, ServeRepair}, solana_dos::cli::*, solana_gossip::{ contact_info::Protocol, - gossip_service::{discover, get_multi_client}, + gossip_service::{discover, get_client}, legacy_contact_info::LegacyContactInfo as ContactInfo, }, solana_measure::measure::Measure, @@ -791,33 +794,30 @@ fn main() { DEFAULT_TPU_CONNECTION_POOL_SIZE, ), }; - let (client, num_clients) = get_multi_client( - &validators, - &SocketAddrSpace::Unspecified, - Arc::new(connection_cache), - ); - if validators.len() < num_clients { - eprintln!( - "Error: Insufficient nodes discovered. Expecting {} or more", - validators.len() - ); - exit(1); - } - (gossip_nodes, Some(Arc::new(client))) + let client = get_client(&validators, Arc::new(connection_cache)); + (gossip_nodes, Some(client)) } else { (vec![], None) }; info!("done found {} nodes", nodes.len()); - - run_dos(&nodes, 0, client, cmd_params); + if let Some(tpu_client) = client { + match tpu_client { + TpuClientWrapper::Quic(quic_client) => { + run_dos(&nodes, 0, Some(Arc::new(quic_client)), cmd_params); + } + TpuClientWrapper::Udp(udp_client) => { + run_dos(&nodes, 0, Some(Arc::new(udp_client)), cmd_params); + } + }; + } } #[cfg(test)] pub mod test { use { super::*, - solana_client::thin_client::ThinClient, + solana_client::tpu_client::TpuClient, solana_core::validator::ValidatorConfig, solana_faucet::faucet::run_local_faucet, solana_gossip::contact_info::LegacyContactInfo, @@ -826,8 +826,10 @@ pub mod test { local_cluster::{ClusterConfig, LocalCluster}, validator_configs::make_identical_validator_configs, }, + solana_quic_client::{QuicConfig, QuicConnectionManager, QuicPool}, solana_rpc::rpc::JsonRpcConfig, solana_sdk::timing::timestamp, + solana_tpu_client::tpu_client::TpuClientConfig, }; const TEST_SEND_BATCH_SIZE: usize = 1; @@ -835,7 +837,32 @@ pub mod test { // thin wrapper for the run_dos function // to avoid specifying everywhere generic parameters fn run_dos_no_client(nodes: &[ContactInfo], iterations: usize, params: DosClientParameters) { - run_dos::(nodes, iterations, None, params); + run_dos::>( + nodes, iterations, None, params, + ); + } + + fn build_tpu_quic_client( + cluster: &LocalCluster, + ) -> Arc> { + let rpc_pubsub_url = format!("ws://{}/", cluster.entry_point_info.rpc_pubsub().unwrap()); + let rpc_url = format!("http://{}", cluster.entry_point_info.rpc().unwrap()); + + let ConnectionCache::Quic(cache) = &*cluster.connection_cache else { + panic!("Expected a Quic ConnectionCache."); + }; + + Arc::new( + TpuClient::new_with_connection_cache( + Arc::new(RpcClient::new(rpc_url)), + rpc_pubsub_url.as_str(), + TpuClientConfig::default(), + cache.clone(), + ) + .unwrap_or_else(|err| { + panic!("Could not create TpuClient with Quic Cache {err:?}"); + }), + ) } #[test] @@ -975,14 +1002,7 @@ pub mod test { .unwrap(); let nodes_slice = [node]; - let client = Arc::new(ThinClient::new( - cluster.entry_point_info.rpc().unwrap(), - cluster - .entry_point_info - .tpu(cluster.connection_cache.protocol()) - .unwrap(), - cluster.connection_cache.clone(), - )); + let client = build_tpu_quic_client(&cluster); // creates one transaction with 8 valid signatures and sends it 10 times run_dos( @@ -1114,14 +1134,7 @@ pub mod test { .unwrap(); let nodes_slice = [node]; - let client = Arc::new(ThinClient::new( - cluster.entry_point_info.rpc().unwrap(), - cluster - .entry_point_info - .tpu(cluster.connection_cache.protocol()) - .unwrap(), - cluster.connection_cache.clone(), - )); + let client = build_tpu_quic_client(&cluster); // creates one transaction and sends it 10 times // this is done in single thread diff --git a/gossip/Cargo.toml b/gossip/Cargo.toml index f9870ac1ee380c..2e62bc66f6866c 100644 --- a/gossip/Cargo.toml +++ b/gossip/Cargo.toml @@ -31,6 +31,7 @@ serde_derive = { workspace = true } solana-bloom = { workspace = true } solana-clap-utils = { workspace = true } solana-client = { workspace = true } +solana-connection-cache = { workspace = true } solana-entry = { workspace = true } solana-frozen-abi = { workspace = true } solana-frozen-abi-macro = { workspace = true } @@ -44,7 +45,6 @@ solana-rayon-threadlimit = { workspace = true } solana-runtime = { workspace = true } solana-sdk = { workspace = true } solana-streamer = { workspace = true } -solana-thin-client = { workspace = true } solana-tpu-client = { workspace = true } solana-version = { workspace = true } solana-vote = { workspace = true } diff --git a/gossip/src/gossip_service.rs b/gossip/src/gossip_service.rs index 9e1c56520993c5..0bd4750e269a48 100644 --- a/gossip/src/gossip_service.rs +++ b/gossip/src/gossip_service.rs @@ -4,7 +4,11 @@ use { crate::{cluster_info::ClusterInfo, legacy_contact_info::LegacyContactInfo as ContactInfo}, crossbeam_channel::{unbounded, Sender}, rand::{thread_rng, Rng}, - solana_client::{connection_cache::ConnectionCache, thin_client::ThinClient}, + solana_client::{ + connection_cache::ConnectionCache, + rpc_client::RpcClient, + tpu_client::{TpuClient, TpuClientConfig, TpuClientWrapper}, + }, solana_perf::recycler::Recycler, solana_runtime::bank_forks::BankForks, solana_sdk::{ @@ -197,35 +201,37 @@ pub fn discover( #[deprecated(since = "1.18.6", note = "Interface will change")] pub fn get_client( nodes: &[ContactInfo], - socket_addr_space: &SocketAddrSpace, connection_cache: Arc, -) -> ThinClient { - let protocol = connection_cache.protocol(); - let nodes: Vec<_> = nodes - .iter() - .filter_map(|node| node.valid_client_facing_addr(protocol, socket_addr_space)) - .collect(); +) -> TpuClientWrapper { let select = thread_rng().gen_range(0..nodes.len()); - let (rpc, tpu) = nodes[select]; - ThinClient::new(rpc, tpu, connection_cache) -} -#[deprecated(since = "1.18.6", note = "Will be removed in favor of get_client")] -pub fn get_multi_client( - nodes: &[ContactInfo], - socket_addr_space: &SocketAddrSpace, - connection_cache: Arc, -) -> (ThinClient, usize) { - let protocol = connection_cache.protocol(); - let (rpc_addrs, tpu_addrs): (Vec<_>, Vec<_>) = nodes - .iter() - .filter_map(|node| node.valid_client_facing_addr(protocol, socket_addr_space)) - .unzip(); - let num_nodes = tpu_addrs.len(); - ( - ThinClient::new_from_addrs(rpc_addrs, tpu_addrs, connection_cache), - num_nodes, - ) + let rpc_pubsub_url = format!("ws://{}/", nodes[select].rpc_pubsub().unwrap()); + let rpc_url = format!("http://{}", nodes[select].rpc().unwrap()); + + match &*connection_cache { + ConnectionCache::Quic(cache) => TpuClientWrapper::Quic( + TpuClient::new_with_connection_cache( + Arc::new(RpcClient::new(rpc_url)), + rpc_pubsub_url.as_str(), + TpuClientConfig::default(), + cache.clone(), + ) + .unwrap_or_else(|err| { + panic!("Could not create TpuClient with Quic Cache {err:?}"); + }), + ), + ConnectionCache::Udp(cache) => TpuClientWrapper::Udp( + TpuClient::new_with_connection_cache( + Arc::new(RpcClient::new(rpc_url)), + rpc_pubsub_url.as_str(), + TpuClientConfig::default(), + cache.clone(), + ) + .unwrap_or_else(|err| { + panic!("Could not create TpuClient with Udp Cache {err:?}"); + }), + ), + } } fn spy( diff --git a/gossip/src/legacy_contact_info.rs b/gossip/src/legacy_contact_info.rs index d3dead1910d6ab..870f1c9aa49283 100644 --- a/gossip/src/legacy_contact_info.rs +++ b/gossip/src/legacy_contact_info.rs @@ -229,21 +229,6 @@ impl LegacyContactInfo { pub fn is_valid_address(addr: &SocketAddr, socket_addr_space: &SocketAddrSpace) -> bool { addr.port() != 0u16 && Self::is_valid_ip(addr.ip()) && socket_addr_space.check(addr) } - - pub(crate) fn valid_client_facing_addr( - &self, - protocol: Protocol, - socket_addr_space: &SocketAddrSpace, - ) -> Option<(SocketAddr, SocketAddr)> { - Some(( - self.rpc() - .ok() - .filter(|addr| socket_addr_space.check(addr))?, - self.tpu(protocol) - .ok() - .filter(|addr| socket_addr_space.check(addr))?, - )) - } } impl TryFrom<&ContactInfo> for LegacyContactInfo { @@ -342,24 +327,6 @@ mod tests { assert!(ci.serve_repair.ip().is_unspecified()); } - #[test] - fn test_valid_client_facing() { - let mut ci = LegacyContactInfo::default(); - assert_eq!( - ci.valid_client_facing_addr(Protocol::QUIC, &SocketAddrSpace::Unspecified), - None - ); - ci.tpu = socketaddr!(Ipv4Addr::LOCALHOST, 123); - assert_eq!( - ci.valid_client_facing_addr(Protocol::QUIC, &SocketAddrSpace::Unspecified), - None - ); - ci.rpc = socketaddr!(Ipv4Addr::LOCALHOST, 234); - assert!(ci - .valid_client_facing_addr(Protocol::QUIC, &SocketAddrSpace::Unspecified) - .is_some()); - } - #[test] fn test_sanitize() { let mut ci = LegacyContactInfo::default(); diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index c31befdf34c092..d148f0bea7b5d0 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -5169,6 +5169,7 @@ dependencies = [ "solana-bloom", "solana-clap-utils", "solana-client", + "solana-connection-cache", "solana-entry", "solana-frozen-abi", "solana-frozen-abi-macro", @@ -5182,7 +5183,6 @@ dependencies = [ "solana-runtime", "solana-sdk", "solana-streamer", - "solana-thin-client", "solana-tpu-client", "solana-version", "solana-vote", From 14454a4a000b1125d65f5cadbe2b1215752b2c79 Mon Sep 17 00:00:00 2001 From: Yihau Chen Date: Tue, 12 Mar 2024 11:42:21 +0800 Subject: [PATCH 54/84] ci: remove unused Github Actions (#124) --- .github/workflows/autolock_bot_PR.txt | 42 ---- .../workflows/autolock_bot_closed_issue.txt | 43 ---- .github/workflows/solana-action.yml.txt | 184 ------------------ 3 files changed, 269 deletions(-) delete mode 100644 .github/workflows/autolock_bot_PR.txt delete mode 100644 .github/workflows/autolock_bot_closed_issue.txt delete mode 100644 .github/workflows/solana-action.yml.txt diff --git a/.github/workflows/autolock_bot_PR.txt b/.github/workflows/autolock_bot_PR.txt deleted file mode 100644 index 4c6011fd4c8cfb..00000000000000 --- a/.github/workflows/autolock_bot_PR.txt +++ /dev/null @@ -1,42 +0,0 @@ -name: 'Autolock RitBot for for PR' - -on: - schedule: - - cron: '0 0 * * *' - workflow_dispatch: - -permissions: - issues: write - pull-requests: write - -concurrency: - group: lock - -jobs: - action: - # Forks do not need to run this, especially on cron schedule. - if: > - github.event_name != 'schedule' - || github.repository == 'solana-labs/solana' - - runs-on: ubuntu-latest - steps: - - uses: dessant/lock-threads@v3 - with: - - github-token: ${{ github.token }} - pr-inactive-days: '14' - exclude-pr-created-before: '' - exclude-pr-created-after: '' - exclude-pr-created-between: '' - exclude-pr-closed-before: '' - exclude-pr-closed-after: '' - exclude-pr-closed-between: '' - include-any-pr-labels: 'automerge' - include-all-pr-labels: '' - exclude-any-pr-labels: '' - add-pr-labels: 'locked PR' - remove-pr-labels: '' - pr-comment: 'This PR has been automatically locked since there has not been any activity in past 14 days after it was merged.' - pr-lock-reason: 'resolved' - log-output: true diff --git a/.github/workflows/autolock_bot_closed_issue.txt b/.github/workflows/autolock_bot_closed_issue.txt deleted file mode 100644 index dd8aa9ef835ba3..00000000000000 --- a/.github/workflows/autolock_bot_closed_issue.txt +++ /dev/null @@ -1,43 +0,0 @@ -name: 'Autolock NaviBot for closed issue' - -on: - schedule: - - cron: '0 0 * * *' - workflow_dispatch: - -permissions: - issues: write - pull-requests: write - -concurrency: - group: lock - -jobs: - action: - # Forks do not need to run this, especially on cron schedule. - if: > - github.event_name != 'schedule' - || github.repository == 'solana-labs/solana' - - runs-on: ubuntu-latest - steps: - - uses: dessant/lock-threads@v3 - with: - - github-token: ${{ github.token }} - issue-inactive-days: '7' - exclude-issue-created-before: '' - exclude-issue-created-after: '' - exclude-issue-created-between: '' - exclude-issue-closed-before: '' - exclude-issue-closed-after: '' - exclude-issue-closed-between: '' - include-any-issue-labels: '' - include-all-issue-labels: '' - exclude-any-issue-labels: '' - add-issue-labels: 'locked issue' - remove-issue-labels: '' - issue-comment: 'This issue has been automatically locked since there has not been any activity in past 7 days after it was closed. Please open a new issue for related bugs.' - issue-lock-reason: 'resolved' - process-only: 'issues' - log-output: true diff --git a/.github/workflows/solana-action.yml.txt b/.github/workflows/solana-action.yml.txt deleted file mode 100644 index 62c4c4864ffffd..00000000000000 --- a/.github/workflows/solana-action.yml.txt +++ /dev/null @@ -1,184 +0,0 @@ -name : minimal - -on: - push: - branches: [master] - pull_request: - branches: [master] - -jobs: - macos-artifacts: - needs: [Export_Github_Repositories] - strategy: - fail-fast: false - runs-on: macos-latest - if : ${{ github.event_name == 'api' && 'cron' || 'push' || startsWith(github.ref, 'refs/tags/v')}} - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - name: Setup | Rust - uses: ATiltedTree/setup-rust@v1 - with: - rust-version: stable - - name: release artifact - run: | - source ci/rust-version.sh - brew install coreutils - export PATH="/usr/local/opt/coreutils/libexec/gnubin:$PATH" - greadlink -f . - source ci/env.sh - rustup set profile default - ci/publish-tarball.sh - shell: bash - - - name: Cache modules - uses: actions/cache@master - id: yarn-cache - with: - path: node_modules - key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} - restore-keys: ${{ runner.os }}-yarn- - - -# - To stop from uploading on the production -# - uses: ochanje210/simple-s3-upload-action@master -# with: -# AWS_ACCESS_KEY_ID: ${{ secrets.AWS_KEY_ID }} -# AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY}} -# AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} -# SOURCE_DIR: 'travis-s3-upload1' -# DEST_DIR: 'giitsol' - -# - uses: ochanje210/simple-s3-upload-action@master -# with: -# AWS_ACCESS_KEY_ID: ${{ secrets.AWS_KEY_ID }} -# AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY}} -# AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} -# SOURCE_DIR: './docs/' -# DEST_DIR: 'giitsol' - - - windows-artifact: - needs: [Export_Github_Repositories] - strategy: - fail-fast: false - runs-on: windows-latest - if : ${{ github.event_name == 'api' && 'cron' || 'push' || startsWith(github.ref, 'refs/tags/v')}} - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - name: Setup | Rust - uses: ATiltedTree/setup-rust@v1 - with: - rust-version: stable - release-artifact: - needs: windows-artifact - runs-on: windows-latest - if : ${{ github.event_name == 'api' && 'cron' || github.ref == 'refs/heads/master'}} - steps: - - name: release artifact - run: | - git clone git://git.openssl.org/openssl.git - cd openssl - make - make test - make install - openssl version -# choco install openssl -# vcpkg integrate install -# refreshenv - - - name: Checkout repository - uses: actions/checkout@v2 - - uses: actions/checkout@v2 - - run: choco install msys2 - - uses: actions/checkout@v2 - - run: | - openssl version - bash ci/rust-version.sh - readlink -f . - bash ci/env.sh - rustup set profile default - bash ci/publish-tarball.sh - shell: bash - - - name: Cache modules - uses: actions/cache@v1 - id: yarn-cache - with: - path: node_modules - key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} - restore-keys: ${{ runner.os }}-yarn- - -# - To stop from uploading on the production -# - name: Config. aws cred -# uses: aws-actions/configure-aws-credentials@v1 -# with: -# aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} -# aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} -# aws-region: us-east-2 -# - name: Deploy -# uses: shallwefootball/s3-upload-action@master -# with: -# folder: build -# aws_bucket: ${{ secrets.AWS_S3_BUCKET }} -# aws_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} -# aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} -# destination_dir: / -# bucket-region: us-east-2 -# delete-removed: true -# no-cache: true -# private: true - -# Docs: -# needs: [windows-artifact,release-artifact] -# runs-on: ubuntu-latest -# env: -# GITHUB_TOKEN: ${{secrets.PAT_NEW}} -# GITHUB_EVENT_BEFORE: ${{ github.event.before }} -# GITHUB_EVENT_AFTER: ${{ github.event.after }} -# COMMIT_RANGE: ${{ github.event.before}}...${{ github.event.after}} -# steps: -# - name: Checkout repo -# uses: actions/checkout@v2 -# with: -# fetch-depth: 2 -# - name: docs -# if: ${{github.event_name == 'pull_request' || startsWith(github.ref, 'refs/tags/v')}} -# run: | -# touch .env -# echo "COMMIT_RANGE=($COMMIT_RANGE)" > .env -# source ci/env.sh -# .travis/channel_restriction.sh edge beta || exit 0 -# .travis/affects.sh docs/ .travis || exit 0 -# cd docs/ -# source .travis/before_install.sh -# source .travis/script.sh -# - name: setup-node -# uses: actions/checkout@v2 -# - name: setup-node -# uses: actions/setup-node@v2 -# with: -# node-version: 'lts/*' -# - name: Cache -# uses: actions/cache@v1 -# with: -# path: ~/.npm -# key: ${{ runner.OS }}-npm-cache-${{ hashFiles('**/package-lock.json') }} -# restore-keys: | -# ${{ runner.OS }}-npm-cache-2 - -# auto_bump: -# needs: [windows-artifact,release-artifact,Docs] -# runs-on: ubuntu-latest -# steps: -# - name : checkout repo -# uses: actions/checkout@v2 -# with: -# fetch-depth: '0' -# - name: Bump version and push tag -# uses: anothrNick/github-tag-action@1.26.0 -# env: -# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -# WITH_V: true -# DEFAULT_BUMP: patch From 6b3d35e995f9910643d5df26c54521694d2ac683 Mon Sep 17 00:00:00 2001 From: Yihau Chen Date: Tue, 12 Mar 2024 11:42:42 +0800 Subject: [PATCH 55/84] Revert "build(deps): bump cc from 1.0.83 to 1.0.89 (#40)" (#174) This reverts commit 7a8e29d4d5edb1d0d467458e36f87871d8dfc0fe. --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- programs/sbf/Cargo.lock | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 46091cfbca5e82..ce428b22a3283b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1231,9 +1231,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.0.89" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0ba8f7aaa012f30d5b2861462f6708eccd49c3c39863fe083a308035f63d723" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" dependencies = [ "jobserver", "libc", diff --git a/Cargo.toml b/Cargo.toml index 453408a53b956e..6cbd56762fbfe8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -170,7 +170,7 @@ bytes = "1.5" bzip2 = "0.4.4" caps = "0.5.5" cargo_metadata = "0.15.4" -cc = "1.0.89" +cc = "1.0.83" chrono = { version = "0.4.34", default-features = false } chrono-humanize = "0.2.3" clap = "2.33.1" diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index d148f0bea7b5d0..1043b74c67c619 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -971,9 +971,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.89" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0ba8f7aaa012f30d5b2861462f6708eccd49c3c39863fe083a308035f63d723" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" dependencies = [ "jobserver", "libc", From 9e9aa05b332acf427b2e71c05fd933431e465593 Mon Sep 17 00:00:00 2001 From: Tyera Date: Mon, 11 Mar 2024 23:59:18 -0600 Subject: [PATCH 56/84] Rpc: add support for minimum context slot to `getBlocks(WithLimit)` endpoints (#191) * Support min_context_slot field in getBlocksWithLimit input * Use min_context_slot in get_blocks_with_limit * Support min_context_slot field in getBlocks input * Use min_context_slot in get_blocks --- rpc-client-api/src/config.rs | 6 +- rpc/src/rpc.rs | 132 ++++++++++++++++++++++------------- 2 files changed, 87 insertions(+), 51 deletions(-) diff --git a/rpc-client-api/src/config.rs b/rpc-client-api/src/config.rs index cecc0b64bdf7b2..be8bb1742457c4 100644 --- a/rpc-client-api/src/config.rs +++ b/rpc-client-api/src/config.rs @@ -321,14 +321,14 @@ impl EncodingConfig for RpcTransactionConfig { #[serde(untagged)] pub enum RpcBlocksConfigWrapper { EndSlotOnly(Option), - CommitmentOnly(Option), + ConfigOnly(Option), } impl RpcBlocksConfigWrapper { - pub fn unzip(&self) -> (Option, Option) { + pub fn unzip(&self) -> (Option, Option) { match &self { RpcBlocksConfigWrapper::EndSlotOnly(end_slot) => (*end_slot, None), - RpcBlocksConfigWrapper::CommitmentOnly(commitment) => (None, *commitment), + RpcBlocksConfigWrapper::ConfigOnly(config) => (None, *config), } } } diff --git a/rpc/src/rpc.rs b/rpc/src/rpc.rs index 41b26e5fa1e2c2..9c979ab1f5a6b2 100644 --- a/rpc/src/rpc.rs +++ b/rpc/src/rpc.rs @@ -527,13 +527,14 @@ impl JsonRpcRequestProcessor { let config = config.unwrap_or_default(); let epoch_schedule = self.get_epoch_schedule(); let first_available_block = self.get_first_available_block().await; + let context_config = RpcContextConfig { + commitment: config.commitment, + min_context_slot: config.min_context_slot, + }; let epoch = match config.epoch { Some(epoch) => epoch, None => epoch_schedule - .get_epoch(self.get_slot(RpcContextConfig { - commitment: config.commitment, - min_context_slot: config.min_context_slot, - })?) + .get_epoch(self.get_slot(context_config)?) .saturating_sub(1), }; @@ -555,7 +556,7 @@ impl JsonRpcRequestProcessor { } let first_confirmed_block_in_epoch = *self - .get_blocks_with_limit(first_slot_in_epoch, 1, config.commitment) + .get_blocks_with_limit(first_slot_in_epoch, 1, Some(context_config)) .await? .first() .ok_or(RpcCustomError::BlockNotAvailable { @@ -1170,9 +1171,10 @@ impl JsonRpcRequestProcessor { &self, start_slot: Slot, end_slot: Option, - commitment: Option, + config: Option, ) -> Result> { - let commitment = commitment.unwrap_or_default(); + let config = config.unwrap_or_default(); + let commitment = config.commitment.unwrap_or_default(); check_is_at_least_confirmed(commitment)?; let highest_super_majority_root = self @@ -1181,12 +1183,20 @@ impl JsonRpcRequestProcessor { .unwrap() .highest_super_majority_root(); + let min_context_slot = config.min_context_slot.unwrap_or_default(); + if commitment.is_finalized() && highest_super_majority_root < min_context_slot { + return Err(RpcCustomError::MinContextSlotNotReached { + context_slot: highest_super_majority_root, + } + .into()); + } + let end_slot = min( end_slot.unwrap_or_else(|| start_slot.saturating_add(MAX_GET_CONFIRMED_BLOCKS_RANGE)), if commitment.is_finalized() { highest_super_majority_root } else { - self.bank(Some(CommitmentConfig::confirmed())).slot() + self.get_bank_with_config(config)?.slot() }, ); if end_slot < start_slot { @@ -1236,14 +1246,16 @@ impl JsonRpcRequestProcessor { .unwrap_or_else(|| start_slot.saturating_sub(1)); // Maybe add confirmed blocks - if commitment.is_confirmed() && last_element < end_slot { - let confirmed_bank = self.bank(Some(CommitmentConfig::confirmed())); - let mut confirmed_blocks = confirmed_bank - .status_cache_ancestors() - .into_iter() - .filter(|&slot| slot <= end_slot && slot > last_element) - .collect(); - blocks.append(&mut confirmed_blocks); + if commitment.is_confirmed() { + let confirmed_bank = self.get_bank_with_config(config)?; + if last_element < end_slot { + let mut confirmed_blocks = confirmed_bank + .status_cache_ancestors() + .into_iter() + .filter(|&slot| slot <= end_slot && slot > last_element) + .collect(); + blocks.append(&mut confirmed_blocks); + } } Ok(blocks) @@ -1253,9 +1265,10 @@ impl JsonRpcRequestProcessor { &self, start_slot: Slot, limit: usize, - commitment: Option, + config: Option, ) -> Result> { - let commitment = commitment.unwrap_or_default(); + let config = config.unwrap_or_default(); + let commitment = config.commitment.unwrap_or_default(); check_is_at_least_confirmed(commitment)?; if limit > MAX_GET_CONFIRMED_BLOCKS_RANGE as usize { @@ -1287,6 +1300,16 @@ impl JsonRpcRequestProcessor { .unwrap() .highest_super_majority_root(); + if commitment.is_finalized() { + let min_context_slot = config.min_context_slot.unwrap_or_default(); + if highest_super_majority_root < min_context_slot { + return Err(RpcCustomError::MinContextSlotNotReached { + context_slot: highest_super_majority_root, + } + .into()); + } + } + // Finalized blocks let mut blocks: Vec<_> = self .blockstore @@ -1297,19 +1320,21 @@ impl JsonRpcRequestProcessor { .collect(); // Maybe add confirmed blocks - if commitment.is_confirmed() && blocks.len() < limit { - let last_element = blocks - .last() - .cloned() - .unwrap_or_else(|| start_slot.saturating_sub(1)); - let confirmed_bank = self.bank(Some(CommitmentConfig::confirmed())); - let mut confirmed_blocks = confirmed_bank - .status_cache_ancestors() - .into_iter() - .filter(|&slot| slot > last_element) - .collect(); - blocks.append(&mut confirmed_blocks); - blocks.truncate(limit); + if commitment.is_confirmed() { + let confirmed_bank = self.get_bank_with_config(config)?; + if blocks.len() < limit { + let last_element = blocks + .last() + .cloned() + .unwrap_or_else(|| start_slot.saturating_sub(1)); + let mut confirmed_blocks = confirmed_bank + .status_cache_ancestors() + .into_iter() + .filter(|&slot| slot > last_element) + .collect(); + blocks.append(&mut confirmed_blocks); + blocks.truncate(limit); + } } Ok(blocks) @@ -3339,8 +3364,8 @@ pub mod rpc_full { &self, meta: Self::Metadata, start_slot: Slot, - config: Option, - commitment: Option, + wrapper: Option, + config: Option, ) -> BoxFuture>>; #[rpc(meta, name = "getBlocksWithLimit")] @@ -3349,7 +3374,7 @@ pub mod rpc_full { meta: Self::Metadata, start_slot: Slot, limit: usize, - commitment: Option, + config: Option, ) -> BoxFuture>>; #[rpc(meta, name = "getTransaction")] @@ -3841,17 +3866,17 @@ pub mod rpc_full { &self, meta: Self::Metadata, start_slot: Slot, - config: Option, - commitment: Option, + wrapper: Option, + config: Option, ) -> BoxFuture>> { - let (end_slot, maybe_commitment) = - config.map(|config| config.unzip()).unwrap_or_default(); + let (end_slot, maybe_config) = + wrapper.map(|wrapper| wrapper.unzip()).unwrap_or_default(); debug!( "get_blocks rpc request received: {}-{:?}", start_slot, end_slot ); Box::pin(async move { - meta.get_blocks(start_slot, end_slot, commitment.or(maybe_commitment)) + meta.get_blocks(start_slot, end_slot, config.or(maybe_config)) .await }) } @@ -3861,16 +3886,13 @@ pub mod rpc_full { meta: Self::Metadata, start_slot: Slot, limit: usize, - commitment: Option, + config: Option, ) -> BoxFuture>> { debug!( "get_blocks_with_limit rpc request received: {}-{}", start_slot, limit, ); - Box::pin(async move { - meta.get_blocks_with_limit(start_slot, limit, commitment) - .await - }) + Box::pin(async move { meta.get_blocks_with_limit(start_slot, limit, config).await }) } fn get_block_time( @@ -4282,8 +4304,15 @@ pub mod rpc_deprecated_v1_7 { start_slot, end_slot ); Box::pin(async move { - meta.get_blocks(start_slot, end_slot, commitment.or(maybe_commitment)) - .await + meta.get_blocks( + start_slot, + end_slot, + Some(RpcContextConfig { + commitment: commitment.or(maybe_commitment), + min_context_slot: None, + }), + ) + .await }) } @@ -4299,8 +4328,15 @@ pub mod rpc_deprecated_v1_7 { start_slot, limit, ); Box::pin(async move { - meta.get_blocks_with_limit(start_slot, limit, commitment) - .await + meta.get_blocks_with_limit( + start_slot, + limit, + Some(RpcContextConfig { + commitment, + min_context_slot: None, + }), + ) + .await }) } From aaf3a91a95e6c50a7eaa194313f574cb958a5714 Mon Sep 17 00:00:00 2001 From: Brooks Date: Tue, 12 Mar 2024 02:25:59 -0400 Subject: [PATCH 57/84] Removes redundant imports (#193) --- frozen-abi/src/abi_digester.rs | 5 +---- programs/bpf_loader/gen-syscall-list/build.rs | 2 +- sdk/macro/src/lib.rs | 1 - 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/frozen-abi/src/abi_digester.rs b/frozen-abi/src/abi_digester.rs index b014efd2ba1570..fb1fc78fa6735e 100644 --- a/frozen-abi/src/abi_digester.rs +++ b/frozen-abi/src/abi_digester.rs @@ -4,10 +4,7 @@ use { hash::{Hash, Hasher}, }, log::*, - serde::{ - ser::{Error as SerdeError, *}, - Serialize, Serializer, - }, + serde::ser::{Error as SerdeError, *}, std::{any::type_name, io::Write}, thiserror::Error, }; diff --git a/programs/bpf_loader/gen-syscall-list/build.rs b/programs/bpf_loader/gen-syscall-list/build.rs index 96af426ec4763b..f06039ae84d696 100644 --- a/programs/bpf_loader/gen-syscall-list/build.rs +++ b/programs/bpf_loader/gen-syscall-list/build.rs @@ -2,7 +2,7 @@ use { regex::Regex, std::{ fs::File, - io::{prelude::*, BufWriter, Read}, + io::{prelude::*, BufWriter}, path::PathBuf, str, }, diff --git a/sdk/macro/src/lib.rs b/sdk/macro/src/lib.rs index 157592dc37bcaa..3c2a3bdb86a9ee 100644 --- a/sdk/macro/src/lib.rs +++ b/sdk/macro/src/lib.rs @@ -8,7 +8,6 @@ use { proc_macro::TokenStream, proc_macro2::{Delimiter, Span, TokenTree}, quote::{quote, ToTokens}, - std::convert::TryFrom, syn::{ bracketed, parse::{Parse, ParseStream, Result}, From 2ddb50d2f38a33dd110755c3bd2113ef4bad9438 Mon Sep 17 00:00:00 2001 From: steviez Date: Tue, 12 Mar 2024 01:27:31 -0500 Subject: [PATCH 58/84] Make --wait-for-supermajority require --expected-shred-version (#192) In cluster restart scenarios, an important step is scanning the Blockstore for blocks that occur after the chosen restart slot with an incorrect shred version. This check ensures that any blocks that occurred pre-cluster restart and after the chosen restart slot get deleted. If a node skips this step, the node can encounter problems when that block is created again, after the cluster has restarted. This check only occurs if --wait-for-supermajority AND --expected-shred-version are set; however, --expected-... is currently optional when using --wait-... Our restart instructions typically mention that one should specify --expected-... as well, but we should just enforce it at the CLI level to prevent mistakes / wasted time debuggging. --- validator/src/cli.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/validator/src/cli.rs b/validator/src/cli.rs index e3f46309724af7..d1ad63b760f031 100644 --- a/validator/src/cli.rs +++ b/validator/src/cli.rs @@ -745,6 +745,7 @@ pub fn app<'a>(version: &'a str, default_args: &'a DefaultArgs) -> App<'a, 'a> { Arg::with_name("wait_for_supermajority") .long("wait-for-supermajority") .requires("expected_bank_hash") + .requires("expected_shred_version") .value_name("SLOT") .validator(is_slot) .help( From 076329381ae860d5832c723ab44fb193c27e1991 Mon Sep 17 00:00:00 2001 From: Brooks Date: Tue, 12 Mar 2024 07:46:23 -0400 Subject: [PATCH 59/84] Moves a clippy attribute (#194) --- memory-management/src/aligned_memory.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/memory-management/src/aligned_memory.rs b/memory-management/src/aligned_memory.rs index 689daaaed5e6aa..e5c203064414dc 100644 --- a/memory-management/src/aligned_memory.rs +++ b/memory-management/src/aligned_memory.rs @@ -207,8 +207,8 @@ impl> From for AlignedMemory { } #[cfg(test)] +#[allow(clippy::arithmetic_side_effects)] mod tests { - #![allow(clippy::arithmetic_side_effects)] use {super::*, std::io::Write}; fn do_test() { From 0f1ca20d38a111bb47621aa3108a2f9fc107f998 Mon Sep 17 00:00:00 2001 From: Brooks Date: Tue, 12 Mar 2024 07:46:41 -0400 Subject: [PATCH 60/84] [anza migration] Sets client id to Agave (#163) --- version/src/lib.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/version/src/lib.rs b/version/src/lib.rs index edeca08c960243..7a59406cf0647d 100644 --- a/version/src/lib.rs +++ b/version/src/lib.rs @@ -17,6 +17,7 @@ enum ClientId { SolanaLabs, JitoLabs, Firedancer, + Agave, // If new variants are added, update From and TryFrom. Unknown(u16), } @@ -63,7 +64,7 @@ impl Default for Version { commit: compute_commit(option_env!("CI_COMMIT")).unwrap_or_default(), feature_set, // Other client implementations need to modify this line. - client: u16::try_from(ClientId::SolanaLabs).unwrap(), + client: u16::try_from(ClientId::Agave).unwrap(), } } } @@ -97,6 +98,7 @@ impl From for ClientId { 0u16 => Self::SolanaLabs, 1u16 => Self::JitoLabs, 2u16 => Self::Firedancer, + 3u16 => Self::Agave, _ => Self::Unknown(client), } } @@ -110,7 +112,8 @@ impl TryFrom for u16 { ClientId::SolanaLabs => Ok(0u16), ClientId::JitoLabs => Ok(1u16), ClientId::Firedancer => Ok(2u16), - ClientId::Unknown(client @ 0u16..=2u16) => Err(format!("Invalid client: {client}")), + ClientId::Agave => Ok(3u16), + ClientId::Unknown(client @ 0u16..=3u16) => Err(format!("Invalid client: {client}")), ClientId::Unknown(client) => Ok(client), } } @@ -147,19 +150,21 @@ mod test { assert_eq!(ClientId::from(0u16), ClientId::SolanaLabs); assert_eq!(ClientId::from(1u16), ClientId::JitoLabs); assert_eq!(ClientId::from(2u16), ClientId::Firedancer); - for client in 3u16..=u16::MAX { + assert_eq!(ClientId::from(3u16), ClientId::Agave); + for client in 4u16..=u16::MAX { assert_eq!(ClientId::from(client), ClientId::Unknown(client)); } assert_eq!(u16::try_from(ClientId::SolanaLabs), Ok(0u16)); assert_eq!(u16::try_from(ClientId::JitoLabs), Ok(1u16)); assert_eq!(u16::try_from(ClientId::Firedancer), Ok(2u16)); - for client in 0..=2u16 { + assert_eq!(u16::try_from(ClientId::Agave), Ok(3u16)); + for client in 0..=3u16 { assert_eq!( u16::try_from(ClientId::Unknown(client)), Err(format!("Invalid client: {client}")) ); } - for client in 3u16..=u16::MAX { + for client in 4u16..=u16::MAX { assert_eq!(u16::try_from(ClientId::Unknown(client)), Ok(client)); } } From 5f6616518641240dfd38fe6d24e356eea95345cf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Mar 2024 23:08:28 +0800 Subject: [PATCH 61/84] build(deps): bump proc-macro2 from 1.0.78 to 1.0.79 (#203) * build(deps): bump proc-macro2 from 1.0.78 to 1.0.79 Bumps [proc-macro2](https://github.com/dtolnay/proc-macro2) from 1.0.78 to 1.0.79. - [Release notes](https://github.com/dtolnay/proc-macro2/releases) - [Commits](https://github.com/dtolnay/proc-macro2/compare/1.0.78...1.0.79) --- updated-dependencies: - dependency-name: proc-macro2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * [auto-commit] Update all Cargo lock files --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot-buildkite --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- programs/sbf/Cargo.lock | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ce428b22a3283b..503b2280d86ec9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4180,9 +4180,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.78" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" +checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e" dependencies = [ "unicode-ident", ] diff --git a/Cargo.toml b/Cargo.toml index 6cbd56762fbfe8..f1ac84f6875b8f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -265,7 +265,7 @@ pickledb = { version = "0.5.1", default-features = false } predicates = "2.1" pretty-hex = "0.3.0" prio-graph = "0.2.1" -proc-macro2 = "1.0.78" +proc-macro2 = "1.0.79" proptest = "1.4" prost = "0.11.9" prost-build = "0.11.9" diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index 1043b74c67c619..4d606fc4e9ed51 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -3646,9 +3646,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.78" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" +checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e" dependencies = [ "unicode-ident", ] From 0705a07b52af10f789e271618d9dbb51a5dc0aec Mon Sep 17 00:00:00 2001 From: Andrew Fitzgerald Date: Tue, 12 Mar 2024 11:48:05 -0500 Subject: [PATCH 62/84] Remove unused account_deps (#188) --- svm/src/account_loader.rs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/svm/src/account_loader.rs b/svm/src/account_loader.rs index bf9b5b9c40bfee..ee06dd5fbf2198 100644 --- a/svm/src/account_loader.rs +++ b/svm/src/account_loader.rs @@ -196,7 +196,6 @@ fn load_transaction_accounts( let mut tx_rent: TransactionRent = 0; let account_keys = message.account_keys(); let mut accounts_found = Vec::with_capacity(account_keys.len()); - let mut account_deps = Vec::with_capacity(account_keys.len()); let mut rent_debits = RentDebits::default(); let rent_collector = callbacks.get_rent_collector(); @@ -316,13 +315,6 @@ fn load_transaction_accounts( return Err(TransactionError::AccountNotFound); } - // Appends the account_deps at the end of the accounts, - // this way they can be accessed in a uniform way. - // At places where only the accounts are needed, - // the account_deps are truncated using e.g: - // accounts.iter().take(message.account_keys.len()) - accounts.append(&mut account_deps); - let builtins_start_index = accounts.len(); let program_indices = message .instructions() From 2078153aa1c857ad0e66913c6acce27572ff1d27 Mon Sep 17 00:00:00 2001 From: Yihau Chen Date: Wed, 13 Mar 2024 02:15:59 +0800 Subject: [PATCH 63/84] [anza migration]: fix download path for cluster test (#204) --- net/net.sh | 2 +- scripts/agave-install-deploy.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/net/net.sh b/net/net.sh index 36bc48efdb7861..c1e93d095be7eb 100755 --- a/net/net.sh +++ b/net/net.sh @@ -563,7 +563,7 @@ prepareDeploy() { if [[ -n $releaseChannel ]]; then echo "Downloading release from channel: $releaseChannel" rm -f "$SOLANA_ROOT"/solana-release.tar.bz2 - declare updateDownloadUrl=https://release.solana.com/"$releaseChannel"/solana-release-x86_64-unknown-linux-gnu.tar.bz2 + declare updateDownloadUrl=https://release.agave.xyz/"$releaseChannel"/solana-release-x86_64-unknown-linux-gnu.tar.bz2 ( set -x curl -L -I "$updateDownloadUrl" diff --git a/scripts/agave-install-deploy.sh b/scripts/agave-install-deploy.sh index a8f8eeb65b3857..01366a1cfbc5af 100755 --- a/scripts/agave-install-deploy.sh +++ b/scripts/agave-install-deploy.sh @@ -57,10 +57,10 @@ esac case $TAG in edge|beta) - DOWNLOAD_URL=https://release.solana.com/"$TAG"/solana-release-$TARGET.tar.bz2 + DOWNLOAD_URL=https://release.agave.xyz/"$TAG"/solana-release-$TARGET.tar.bz2 ;; *) - DOWNLOAD_URL=https://github.com/solana-labs/solana/releases/download/"$TAG"/solana-release-$TARGET.tar.bz2 + DOWNLOAD_URL=https://github.com/anza-xyz/agave/releases/download/"$TAG"/solana-release-$TARGET.tar.bz2 ;; esac From 7a144e2b9faabf18077cc6a3df118b2bbe2d26dd Mon Sep 17 00:00:00 2001 From: steviez Date: Tue, 12 Mar 2024 13:21:11 -0500 Subject: [PATCH 64/84] Make ReplayStage own the threadpool for tx replay (#190) The threadpool used to replay multiple transactions in parallel is currently global state via a lazy_static definition. Making this pool owned by ReplayStage will enable subsequent work to make the pool size configurable on the CLI. This makes `ReplayStage` create and hold the threadpool which is passed down to blockstore_processor::confirm_slot(). blockstore_processor::process_blockstore_from_root() now creates its' own threadpool as well; however, this pool is only alive while for the scope of that function and does not persist the lifetime of the process. --- core/src/replay_stage.rs | 36 +++++++++++++--- ledger/src/blockstore_processor.rs | 69 ++++++++++++++++++++++++------ ledger/src/lib.rs | 3 -- 3 files changed, 86 insertions(+), 22 deletions(-) diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs index 3683e257ed10a8..015ec5360448f9 100644 --- a/core/src/replay_stage.rs +++ b/core/src/replay_stage.rs @@ -51,6 +51,7 @@ use { solana_measure::measure::Measure, solana_poh::poh_recorder::{PohLeaderStatus, PohRecorder, GRACE_TICKS_FACTOR, MAX_GRACE_SLOTS}, solana_program_runtime::timings::ExecuteTimings, + solana_rayon_threadlimit::get_max_thread_count, solana_rpc::{ optimistically_confirmed_bank_tracker::{BankNotification, BankNotificationSenderConfig}, rpc_subscriptions::RpcSubscriptions, @@ -652,16 +653,23 @@ impl ReplayStage { r_bank_forks.get_vote_only_mode_signal(), ) }; + // Thread pool to (maybe) replay multiple threads in parallel let replay_mode = if replay_slots_concurrently { ForkReplayMode::Serial } else { let pool = rayon::ThreadPoolBuilder::new() .num_threads(MAX_CONCURRENT_FORKS_TO_REPLAY) - .thread_name(|i| format!("solReplay{i:02}")) + .thread_name(|i| format!("solReplayFork{i:02}")) .build() .expect("new rayon threadpool"); ForkReplayMode::Parallel(pool) }; + // Thread pool to replay multiple transactions within one block in parallel + let replay_tx_thread_pool = rayon::ThreadPoolBuilder::new() + .num_threads(get_max_thread_count()) + .thread_name(|i| format!("solReplayTx{i:02}")) + .build() + .expect("new rayon threadpool"); Self::reset_poh_recorder( &my_pubkey, @@ -724,6 +732,7 @@ impl ReplayStage { &mut replay_timing, log_messages_bytes_limit, &replay_mode, + &replay_tx_thread_pool, &prioritization_fee_cache, &mut purge_repair_slot_counter, ); @@ -2136,6 +2145,7 @@ impl ReplayStage { fn replay_blockstore_into_bank( bank: &BankWithScheduler, blockstore: &Blockstore, + replay_tx_thread_pool: &ThreadPool, replay_stats: &RwLock, replay_progress: &RwLock, transaction_status_sender: Option<&TransactionStatusSender>, @@ -2154,6 +2164,7 @@ impl ReplayStage { blockstore_processor::confirm_slot( blockstore, bank, + replay_tx_thread_pool, &mut w_replay_stats, &mut w_replay_progress, false, @@ -2712,7 +2723,8 @@ impl ReplayStage { fn replay_active_banks_concurrently( blockstore: &Blockstore, bank_forks: &RwLock, - thread_pool: &ThreadPool, + fork_thread_pool: &ThreadPool, + replay_tx_thread_pool: &ThreadPool, my_pubkey: &Pubkey, vote_account: &Pubkey, progress: &mut ProgressMap, @@ -2730,7 +2742,7 @@ impl ReplayStage { let longest_replay_time_us = AtomicU64::new(0); // Allow for concurrent replaying of slots from different forks. - let replay_result_vec: Vec = thread_pool.install(|| { + let replay_result_vec: Vec = fork_thread_pool.install(|| { active_bank_slots .into_par_iter() .map(|bank_slot| { @@ -2744,7 +2756,7 @@ impl ReplayStage { trace!( "Replay active bank: slot {}, thread_idx {}", bank_slot, - thread_pool.current_thread_index().unwrap_or_default() + fork_thread_pool.current_thread_index().unwrap_or_default() ); let mut progress_lock = progress.write().unwrap(); if progress_lock @@ -2797,6 +2809,7 @@ impl ReplayStage { let blockstore_result = Self::replay_blockstore_into_bank( &bank, blockstore, + replay_tx_thread_pool, &replay_stats, &replay_progress, transaction_status_sender, @@ -2826,6 +2839,7 @@ impl ReplayStage { fn replay_active_bank( blockstore: &Blockstore, bank_forks: &RwLock, + replay_tx_thread_pool: &ThreadPool, my_pubkey: &Pubkey, vote_account: &Pubkey, progress: &mut ProgressMap, @@ -2884,6 +2898,7 @@ impl ReplayStage { let blockstore_result = Self::replay_blockstore_into_bank( &bank, blockstore, + replay_tx_thread_pool, &bank_progress.replay_stats, &bank_progress.replay_progress, transaction_status_sender, @@ -3183,6 +3198,7 @@ impl ReplayStage { replay_timing: &mut ReplayLoopTiming, log_messages_bytes_limit: Option, replay_mode: &ForkReplayMode, + replay_tx_thread_pool: &ThreadPool, prioritization_fee_cache: &PrioritizationFeeCache, purge_repair_slot_counter: &mut PurgeRepairSlotCounter, ) -> bool /* completed a bank */ { @@ -3199,11 +3215,12 @@ impl ReplayStage { let replay_result_vec = match replay_mode { // Skip the overhead of the threadpool if there is only one bank to play - ForkReplayMode::Parallel(thread_pool) if num_active_banks > 1 => { + ForkReplayMode::Parallel(fork_thread_pool) if num_active_banks > 1 => { Self::replay_active_banks_concurrently( blockstore, bank_forks, - thread_pool, + fork_thread_pool, + replay_tx_thread_pool, my_pubkey, vote_account, progress, @@ -3223,6 +3240,7 @@ impl ReplayStage { Self::replay_active_bank( blockstore, bank_forks, + replay_tx_thread_pool, my_pubkey, vote_account, progress, @@ -5034,9 +5052,15 @@ pub(crate) mod tests { blockstore.insert_shreds(shreds, None, false).unwrap(); let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::default())); let exit = Arc::new(AtomicBool::new(false)); + let replay_tx_thread_pool = rayon::ThreadPoolBuilder::new() + .num_threads(1) + .thread_name(|i| format!("solReplayTest{i:02}")) + .build() + .expect("new rayon threadpool"); let res = ReplayStage::replay_blockstore_into_bank( &bank1, &blockstore, + &replay_tx_thread_pool, &bank1_progress.replay_stats, &bank1_progress.replay_progress, None, diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs index e4ae5f368b2afd..a76387f7cb2054 100644 --- a/ledger/src/blockstore_processor.rs +++ b/ledger/src/blockstore_processor.rs @@ -89,16 +89,6 @@ struct ReplayEntry { starting_index: usize, } -// get_max_thread_count to match number of threads in the old code. -// see: https://github.com/solana-labs/solana/pull/24853 -lazy_static! { - static ref PAR_THREAD_POOL: ThreadPool = rayon::ThreadPoolBuilder::new() - .num_threads(get_max_thread_count()) - .thread_name(|i| format!("solBstoreProc{i:02}")) - .build() - .unwrap(); -} - fn first_err(results: &[Result<()>]) -> Result<()> { for r in results { if r.is_err() { @@ -139,6 +129,14 @@ fn get_first_error( first_err } +fn create_thread_pool(num_threads: usize) -> ThreadPool { + rayon::ThreadPoolBuilder::new() + .num_threads(num_threads) + .thread_name(|i| format!("solReplayTx{i:02}")) + .build() + .expect("new rayon threadpool") +} + pub fn execute_batch( batch: &TransactionBatchWithIndexes, bank: &Arc, @@ -242,6 +240,7 @@ impl ExecuteBatchesInternalMetrics { fn execute_batches_internal( bank: &Arc, + replay_tx_thread_pool: &ThreadPool, batches: &[TransactionBatchWithIndexes], transaction_status_sender: Option<&TransactionStatusSender>, replay_vote_sender: Option<&ReplayVoteSender>, @@ -253,7 +252,7 @@ fn execute_batches_internal( Mutex::new(HashMap::new()); let mut execute_batches_elapsed = Measure::start("execute_batches_elapsed"); - let results: Vec> = PAR_THREAD_POOL.install(|| { + let results: Vec> = replay_tx_thread_pool.install(|| { batches .into_par_iter() .map(|transaction_batch| { @@ -275,7 +274,7 @@ fn execute_batches_internal( "execute_batch", ); - let thread_index = PAR_THREAD_POOL.current_thread_index().unwrap(); + let thread_index = replay_tx_thread_pool.current_thread_index().unwrap(); execution_timings_per_thread .lock() .unwrap() @@ -324,6 +323,7 @@ fn execute_batches_internal( // invocation). fn process_batches( bank: &BankWithScheduler, + replay_tx_thread_pool: &ThreadPool, batches: &[TransactionBatchWithIndexes], transaction_status_sender: Option<&TransactionStatusSender>, replay_vote_sender: Option<&ReplayVoteSender>, @@ -348,6 +348,7 @@ fn process_batches( ); rebatch_and_execute_batches( bank, + replay_tx_thread_pool, batches, transaction_status_sender, replay_vote_sender, @@ -398,6 +399,7 @@ fn rebatch_transactions<'a>( fn rebatch_and_execute_batches( bank: &Arc, + replay_tx_thread_pool: &ThreadPool, batches: &[TransactionBatchWithIndexes], transaction_status_sender: Option<&TransactionStatusSender>, replay_vote_sender: Option<&ReplayVoteSender>, @@ -481,6 +483,7 @@ fn rebatch_and_execute_batches( let execute_batches_internal_metrics = execute_batches_internal( bank, + replay_tx_thread_pool, rebatched_txs, transaction_status_sender, replay_vote_sender, @@ -506,6 +509,7 @@ pub fn process_entries_for_tests( transaction_status_sender: Option<&TransactionStatusSender>, replay_vote_sender: Option<&ReplayVoteSender>, ) -> Result<()> { + let replay_tx_thread_pool = create_thread_pool(1); let verify_transaction = { let bank = bank.clone_with_scheduler(); move |versioned_tx: VersionedTransaction| -> Result { @@ -533,6 +537,7 @@ pub fn process_entries_for_tests( let ignored_prioritization_fee_cache = PrioritizationFeeCache::new(0u64); let result = process_entries( bank, + &replay_tx_thread_pool, &mut replay_entries, transaction_status_sender, replay_vote_sender, @@ -547,6 +552,7 @@ pub fn process_entries_for_tests( fn process_entries( bank: &BankWithScheduler, + replay_tx_thread_pool: &ThreadPool, entries: &mut [ReplayEntry], transaction_status_sender: Option<&TransactionStatusSender>, replay_vote_sender: Option<&ReplayVoteSender>, @@ -572,6 +578,7 @@ fn process_entries( // execute the group and register the tick process_batches( bank, + replay_tx_thread_pool, &batches, transaction_status_sender, replay_vote_sender, @@ -625,6 +632,7 @@ fn process_entries( // execute the current queue and try to process this entry again process_batches( bank, + replay_tx_thread_pool, &batches, transaction_status_sender, replay_vote_sender, @@ -640,6 +648,7 @@ fn process_entries( } process_batches( bank, + replay_tx_thread_pool, &batches, transaction_status_sender, replay_vote_sender, @@ -805,6 +814,7 @@ pub(crate) fn process_blockstore_for_bank_0( let bank_forks = BankForks::new_rw_arc(bank0); info!("Processing ledger for slot 0..."); + let replay_tx_thread_pool = create_thread_pool(get_max_thread_count()); process_bank_0( &bank_forks .read() @@ -812,6 +822,7 @@ pub(crate) fn process_blockstore_for_bank_0( .get_with_scheduler(bank0_slot) .unwrap(), blockstore, + &replay_tx_thread_pool, opts, &VerifyRecyclers::default(), cache_block_meta_sender, @@ -871,10 +882,12 @@ pub fn process_blockstore_from_root( .meta(start_slot) .unwrap_or_else(|_| panic!("Failed to get meta for slot {start_slot}")) { + let replay_tx_thread_pool = create_thread_pool(get_max_thread_count()); load_frozen_forks( bank_forks, &start_slot_meta, blockstore, + &replay_tx_thread_pool, leader_schedule_cache, opts, transaction_status_sender, @@ -978,9 +991,11 @@ fn verify_ticks( Ok(()) } +#[allow(clippy::too_many_arguments)] fn confirm_full_slot( blockstore: &Blockstore, bank: &BankWithScheduler, + replay_tx_thread_pool: &ThreadPool, opts: &ProcessOptions, recyclers: &VerifyRecyclers, progress: &mut ConfirmationProgress, @@ -996,6 +1011,7 @@ fn confirm_full_slot( confirm_slot( blockstore, bank, + replay_tx_thread_pool, &mut confirmation_timing, progress, skip_verification, @@ -1142,6 +1158,7 @@ impl ConfirmationProgress { pub fn confirm_slot( blockstore: &Blockstore, bank: &BankWithScheduler, + replay_tx_thread_pool: &ThreadPool, timing: &mut ConfirmationTiming, progress: &mut ConfirmationProgress, skip_verification: bool, @@ -1171,6 +1188,7 @@ pub fn confirm_slot( confirm_slot_entries( bank, + replay_tx_thread_pool, slot_entries_load_result, timing, progress, @@ -1187,6 +1205,7 @@ pub fn confirm_slot( #[allow(clippy::too_many_arguments)] fn confirm_slot_entries( bank: &BankWithScheduler, + replay_tx_thread_pool: &ThreadPool, slot_entries_load_result: (Vec, u64, bool), timing: &mut ConfirmationTiming, progress: &mut ConfirmationProgress, @@ -1328,6 +1347,7 @@ fn confirm_slot_entries( .collect(); let process_result = process_entries( bank, + replay_tx_thread_pool, &mut replay_entries, transaction_status_sender, replay_vote_sender, @@ -1385,6 +1405,7 @@ fn confirm_slot_entries( fn process_bank_0( bank0: &BankWithScheduler, blockstore: &Blockstore, + replay_tx_thread_pool: &ThreadPool, opts: &ProcessOptions, recyclers: &VerifyRecyclers, cache_block_meta_sender: Option<&CacheBlockMetaSender>, @@ -1395,6 +1416,7 @@ fn process_bank_0( confirm_full_slot( blockstore, bank0, + replay_tx_thread_pool, opts, recyclers, &mut progress, @@ -1479,6 +1501,7 @@ fn load_frozen_forks( bank_forks: &RwLock, start_slot_meta: &SlotMeta, blockstore: &Blockstore, + replay_tx_thread_pool: &ThreadPool, leader_schedule_cache: &LeaderScheduleCache, opts: &ProcessOptions, transaction_status_sender: Option<&TransactionStatusSender>, @@ -1566,6 +1589,7 @@ fn load_frozen_forks( if process_single_slot( blockstore, &bank, + replay_tx_thread_pool, opts, &recyclers, &mut progress, @@ -1771,6 +1795,7 @@ fn supermajority_root_from_vote_accounts( fn process_single_slot( blockstore: &Blockstore, bank: &BankWithScheduler, + replay_tx_thread_pool: &ThreadPool, opts: &ProcessOptions, recyclers: &VerifyRecyclers, progress: &mut ConfirmationProgress, @@ -1785,6 +1810,7 @@ fn process_single_slot( confirm_full_slot( blockstore, bank, + replay_tx_thread_pool, opts, recyclers, progress, @@ -3692,7 +3718,16 @@ pub mod tests { ..ProcessOptions::default() }; let recyclers = VerifyRecyclers::default(); - process_bank_0(&bank0, &blockstore, &opts, &recyclers, None, None); + let replay_tx_thread_pool = create_thread_pool(1); + process_bank_0( + &bank0, + &blockstore, + &replay_tx_thread_pool, + &opts, + &recyclers, + None, + None, + ); let bank0_last_blockhash = bank0.last_blockhash(); let bank1 = bank_forks.write().unwrap().insert(Bank::new_from_parent( bank0.clone_without_scheduler(), @@ -3702,6 +3737,7 @@ pub mod tests { confirm_full_slot( &blockstore, &bank1, + &replay_tx_thread_pool, &opts, &recyclers, &mut ConfirmationProgress::new(bank0_last_blockhash), @@ -4342,8 +4378,10 @@ pub mod tests { slot_full: bool, prev_entry_hash: Hash, ) -> result::Result<(), BlockstoreProcessorError> { + let replay_tx_thread_pool = create_thread_pool(1); confirm_slot_entries( &BankWithScheduler::new_without_scheduler(bank.clone()), + &replay_tx_thread_pool, (slot_entries, 0, slot_full), &mut ConfirmationTiming::default(), &mut ConfirmationProgress::new(prev_entry_hash), @@ -4400,6 +4438,7 @@ pub mod tests { let bank = BankWithScheduler::new_without_scheduler( Bank::new_with_bank_forks_for_tests(&genesis_config).0, ); + let replay_tx_thread_pool = create_thread_pool(1); let mut timing = ConfirmationTiming::default(); let mut progress = ConfirmationProgress::new(genesis_hash); let amount = genesis_config.rent.minimum_balance(0); @@ -4436,6 +4475,7 @@ pub mod tests { confirm_slot_entries( &bank, + &replay_tx_thread_pool, (vec![entry], 0, false), &mut timing, &mut progress, @@ -4480,6 +4520,7 @@ pub mod tests { confirm_slot_entries( &bank, + &replay_tx_thread_pool, (vec![entry], 0, false), &mut timing, &mut progress, @@ -4592,10 +4633,12 @@ pub mod tests { transaction_indexes: (0..txs.len()).collect(), }; + let replay_tx_thread_pool = create_thread_pool(1); let mut batch_execution_timing = BatchExecutionTiming::default(); let ignored_prioritization_fee_cache = PrioritizationFeeCache::new(0u64); assert!(process_batches( &bank, + &replay_tx_thread_pool, &[batch_with_indexes], None, None, diff --git a/ledger/src/lib.rs b/ledger/src/lib.rs index 10dd5182717841..5f577e3c938aaf 100644 --- a/ledger/src/lib.rs +++ b/ledger/src/lib.rs @@ -39,8 +39,5 @@ extern crate solana_metrics; #[macro_use] extern crate log; -#[macro_use] -extern crate lazy_static; - #[macro_use] extern crate solana_frozen_abi_macro; From 7020864d6c23c94ac744faddb95528cc2375cc35 Mon Sep 17 00:00:00 2001 From: Brooks Date: Tue, 12 Mar 2024 14:25:47 -0400 Subject: [PATCH 65/84] Adds a new bench for accounts delta hash (#210) --- accounts-db/benches/bench_hashing.rs | 50 +++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 4 deletions(-) diff --git a/accounts-db/benches/bench_hashing.rs b/accounts-db/benches/bench_hashing.rs index 3158f78c7a938f..78df86a97f5168 100644 --- a/accounts-db/benches/bench_hashing.rs +++ b/accounts-db/benches/bench_hashing.rs @@ -1,7 +1,11 @@ use { - criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}, - solana_accounts_db::accounts_db::AccountsDb, - solana_sdk::{account::AccountSharedData, pubkey::Pubkey}, + criterion::{criterion_group, criterion_main, BatchSize, BenchmarkId, Criterion, Throughput}, + rand::seq::SliceRandom, + solana_accounts_db::{ + accounts_db::AccountsDb, + accounts_hash::{AccountHash, AccountsHasher}, + }, + solana_sdk::{account::AccountSharedData, hash::Hash, pubkey::Pubkey}, }; const KB: usize = 1024; @@ -39,5 +43,43 @@ fn bench_hash_account(c: &mut Criterion) { } } -criterion_group!(benches, bench_hash_account,); +fn bench_accounts_delta_hash(c: &mut Criterion) { + const ACCOUNTS_COUNTS: [usize; 4] = [ + 1, // the smallest count; will bench overhead + 100, // number of accounts written per slot on mnb (with *no* rent rewrites) + 1_000, // number of accounts written slot on mnb (with rent rewrites) + 10_000, // reasonable largest number of accounts written per slot + ]; + + fn create_account_hashes(accounts_count: usize) -> Vec<(Pubkey, AccountHash)> { + let mut account_hashes: Vec<_> = std::iter::repeat_with(|| { + let address = Pubkey::new_unique(); + let hash = AccountHash(Hash::new_unique()); + (address, hash) + }) + .take(accounts_count) + .collect(); + + // since the accounts delta hash needs to sort the accounts first, ensure we're not + // creating a pre-sorted vec. + let mut rng = rand::thread_rng(); + account_hashes.shuffle(&mut rng); + account_hashes + } + + let mut group = c.benchmark_group("accounts_delta_hash"); + for accounts_count in ACCOUNTS_COUNTS { + group.throughput(Throughput::Elements(accounts_count as u64)); + let account_hashes = create_account_hashes(accounts_count); + group.bench_function(BenchmarkId::new("accounts_count", accounts_count), |b| { + b.iter_batched( + || account_hashes.clone(), + AccountsHasher::accumulate_account_hashes, + BatchSize::SmallInput, + ); + }); + } +} + +criterion_group!(benches, bench_hash_account, bench_accounts_delta_hash); criterion_main!(benches); From 8c446f26cdadf26c442e1a2ce9a46b4ae41f484d Mon Sep 17 00:00:00 2001 From: Greg Cusack Date: Tue, 12 Mar 2024 13:41:54 -0500 Subject: [PATCH 66/84] Fully remove `ThinClient` from `bench-tps` (#132) remove ThinClient from bench-tps --- Cargo.lock | 1 - bench-tps/Cargo.toml | 1 - bench-tps/src/bench_tps_client.rs | 1 - bench-tps/src/bench_tps_client/thin_client.rs | 143 ------------------ 4 files changed, 146 deletions(-) delete mode 100644 bench-tps/src/bench_tps_client/thin_client.rs diff --git a/Cargo.lock b/Cargo.lock index 503b2280d86ec9..df8f1e1586134c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5588,7 +5588,6 @@ dependencies = [ "solana-sdk", "solana-streamer", "solana-test-validator", - "solana-thin-client", "solana-tpu-client", "solana-transaction-status", "solana-version", diff --git a/bench-tps/Cargo.toml b/bench-tps/Cargo.toml index 2fc48c9e296d50..2c7060175f0a8c 100644 --- a/bench-tps/Cargo.toml +++ b/bench-tps/Cargo.toml @@ -35,7 +35,6 @@ solana-rpc-client-nonce-utils = { workspace = true } solana-runtime = { workspace = true } solana-sdk = { workspace = true } solana-streamer = { workspace = true } -solana-thin-client = { workspace = true } solana-tpu-client = { workspace = true } solana-transaction-status = { workspace = true } solana-version = { workspace = true } diff --git a/bench-tps/src/bench_tps_client.rs b/bench-tps/src/bench_tps_client.rs index 0715d739879165..173cdd7cc3e2a5 100644 --- a/bench-tps/src/bench_tps_client.rs +++ b/bench-tps/src/bench_tps_client.rs @@ -113,5 +113,4 @@ pub trait BenchTpsClient { mod bank_client; mod rpc_client; -mod thin_client; mod tpu_client; diff --git a/bench-tps/src/bench_tps_client/thin_client.rs b/bench-tps/src/bench_tps_client/thin_client.rs deleted file mode 100644 index 22945c4494f453..00000000000000 --- a/bench-tps/src/bench_tps_client/thin_client.rs +++ /dev/null @@ -1,143 +0,0 @@ -use { - crate::bench_tps_client::{BenchTpsClient, BenchTpsError, Result}, - solana_client::thin_client::ThinClient, - solana_rpc_client_api::config::RpcBlockConfig, - solana_sdk::{ - account::Account, - client::{AsyncClient, Client, SyncClient}, - commitment_config::CommitmentConfig, - epoch_info::EpochInfo, - hash::Hash, - message::Message, - pubkey::Pubkey, - signature::Signature, - slot_history::Slot, - transaction::Transaction, - }, - solana_transaction_status::UiConfirmedBlock, -}; - -impl BenchTpsClient for ThinClient { - fn send_transaction(&self, transaction: Transaction) -> Result { - AsyncClient::async_send_transaction(self, transaction).map_err(|err| err.into()) - } - fn send_batch(&self, transactions: Vec) -> Result<()> { - AsyncClient::async_send_batch(self, transactions).map_err(|err| err.into()) - } - fn get_latest_blockhash(&self) -> Result { - SyncClient::get_latest_blockhash(self).map_err(|err| err.into()) - } - - fn get_latest_blockhash_with_commitment( - &self, - commitment_config: CommitmentConfig, - ) -> Result<(Hash, u64)> { - SyncClient::get_latest_blockhash_with_commitment(self, commitment_config) - .map_err(|err| err.into()) - } - - fn get_transaction_count(&self) -> Result { - SyncClient::get_transaction_count(self).map_err(|err| err.into()) - } - - fn get_transaction_count_with_commitment( - &self, - commitment_config: CommitmentConfig, - ) -> Result { - SyncClient::get_transaction_count_with_commitment(self, commitment_config) - .map_err(|err| err.into()) - } - - fn get_epoch_info(&self) -> Result { - SyncClient::get_epoch_info(self).map_err(|err| err.into()) - } - - fn get_balance(&self, pubkey: &Pubkey) -> Result { - SyncClient::get_balance(self, pubkey).map_err(|err| err.into()) - } - - fn get_balance_with_commitment( - &self, - pubkey: &Pubkey, - commitment_config: CommitmentConfig, - ) -> Result { - SyncClient::get_balance_with_commitment(self, pubkey, commitment_config) - .map_err(|err| err.into()) - } - - fn get_fee_for_message(&self, message: &Message) -> Result { - SyncClient::get_fee_for_message(self, message).map_err(|err| err.into()) - } - - fn get_minimum_balance_for_rent_exemption(&self, data_len: usize) -> Result { - SyncClient::get_minimum_balance_for_rent_exemption(self, data_len).map_err(|err| err.into()) - } - - fn addr(&self) -> String { - Client::tpu_addr(self) - } - - fn request_airdrop_with_blockhash( - &self, - pubkey: &Pubkey, - lamports: u64, - recent_blockhash: &Hash, - ) -> Result { - self.rpc_client() - .request_airdrop_with_blockhash(pubkey, lamports, recent_blockhash) - .map_err(|err| err.into()) - } - - fn get_account(&self, pubkey: &Pubkey) -> Result { - self.rpc_client() - .get_account(pubkey) - .map_err(|err| err.into()) - } - - fn get_account_with_commitment( - &self, - pubkey: &Pubkey, - commitment_config: CommitmentConfig, - ) -> Result { - SyncClient::get_account_with_commitment(self, pubkey, commitment_config) - .map_err(|err| err.into()) - .and_then(|account| { - account.ok_or_else(|| { - BenchTpsError::Custom(format!("AccountNotFound: pubkey={pubkey}")) - }) - }) - } - - fn get_multiple_accounts(&self, pubkeys: &[Pubkey]) -> Result>> { - self.rpc_client() - .get_multiple_accounts(pubkeys) - .map_err(|err| err.into()) - } - - fn get_slot_with_commitment(&self, commitment_config: CommitmentConfig) -> Result { - self.rpc_client() - .get_slot_with_commitment(commitment_config) - .map_err(|err| err.into()) - } - - fn get_blocks_with_commitment( - &self, - start_slot: Slot, - end_slot: Option, - commitment_config: CommitmentConfig, - ) -> Result> { - self.rpc_client() - .get_blocks_with_commitment(start_slot, end_slot, commitment_config) - .map_err(|err| err.into()) - } - - fn get_block_with_config( - &self, - slot: Slot, - rpc_block_config: RpcBlockConfig, - ) -> Result { - self.rpc_client() - .get_block_with_config(slot, rpc_block_config) - .map_err(|err| err.into()) - } -} From f8bb98b5f4e7710a5ea75a44c0c0f6eddfb09dd7 Mon Sep 17 00:00:00 2001 From: steviez Date: Tue, 12 Mar 2024 16:11:44 -0500 Subject: [PATCH 67/84] Move default value for --rpc-pubsub-notification-threads to CLI (#158) The default value was previously being determined down where the thread pool is being created. Providing a default value at the CLI level is consistent with other args, and gives an operator better visibility into what the default will actually be --- Cargo.lock | 1 + programs/sbf/Cargo.lock | 1 + rpc/src/rpc_pubsub_service.rs | 8 ++-- rpc/src/rpc_subscriptions.rs | 71 +++++++++++++++-------------------- validator/Cargo.toml | 1 + validator/src/cli.rs | 8 ++++ validator/src/main.rs | 8 ++-- 7 files changed, 50 insertions(+), 48 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index df8f1e1586134c..54d1cbc4e0b4b8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -233,6 +233,7 @@ dependencies = [ "solana-perf", "solana-poh", "solana-program-runtime", + "solana-rayon-threadlimit", "solana-rpc", "solana-rpc-client", "solana-rpc-client-api", diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index 4d606fc4e9ed51..9c78461b1b0a81 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -118,6 +118,7 @@ dependencies = [ "solana-perf", "solana-poh", "solana-program-runtime", + "solana-rayon-threadlimit", "solana-rpc", "solana-rpc-client", "solana-rpc-client-api", diff --git a/rpc/src/rpc_pubsub_service.rs b/rpc/src/rpc_pubsub_service.rs index 3e32503691d78e..99155e678675f5 100644 --- a/rpc/src/rpc_pubsub_service.rs +++ b/rpc/src/rpc_pubsub_service.rs @@ -12,10 +12,12 @@ use { jsonrpc_core::IoHandler, soketto::handshake::{server, Server}, solana_metrics::TokenCounter, + solana_rayon_threadlimit::get_thread_count, solana_sdk::timing::AtomicInterval, std::{ io, net::SocketAddr, + num::NonZeroUsize, str, sync::{ atomic::{AtomicU64, AtomicUsize, Ordering}, @@ -43,7 +45,7 @@ pub struct PubSubConfig { pub queue_capacity_items: usize, pub queue_capacity_bytes: usize, pub worker_threads: usize, - pub notification_threads: Option, + pub notification_threads: Option, } impl Default for PubSubConfig { @@ -55,7 +57,7 @@ impl Default for PubSubConfig { queue_capacity_items: DEFAULT_QUEUE_CAPACITY_ITEMS, queue_capacity_bytes: DEFAULT_QUEUE_CAPACITY_BYTES, worker_threads: DEFAULT_WORKER_THREADS, - notification_threads: None, + notification_threads: NonZeroUsize::new(get_thread_count()), } } } @@ -69,7 +71,7 @@ impl PubSubConfig { queue_capacity_items: DEFAULT_TEST_QUEUE_CAPACITY_ITEMS, queue_capacity_bytes: DEFAULT_QUEUE_CAPACITY_BYTES, worker_threads: DEFAULT_WORKER_THREADS, - notification_threads: Some(2), + notification_threads: NonZeroUsize::new(2), } } } diff --git a/rpc/src/rpc_subscriptions.rs b/rpc/src/rpc_subscriptions.rs index 7ecfd6a31a42cc..39d746c48049de 100644 --- a/rpc/src/rpc_subscriptions.rs +++ b/rpc/src/rpc_subscriptions.rs @@ -19,7 +19,6 @@ use { solana_account_decoder::{parse_token::is_known_spl_token_id, UiAccount, UiAccountEncoding}, solana_ledger::{blockstore::Blockstore, get_tmp_ledger_path}, solana_measure::measure::Measure, - solana_rayon_threadlimit::get_thread_count, solana_rpc_client_api::response::{ ProcessedSignatureResult, ReceivedSignatureResult, Response as RpcResponse, RpcBlockUpdate, RpcBlockUpdateError, RpcKeyedAccount, RpcLogsResponse, RpcResponseContext, @@ -631,41 +630,37 @@ impl RpcSubscriptions { config.queue_capacity_bytes, )), }; - let notification_threads = config.notification_threads.unwrap_or_else(get_thread_count); - let t_cleanup = if notification_threads == 0 { - None - } else { + + let t_cleanup = config.notification_threads.map(|notification_threads| { let exit = exit.clone(); - Some( - Builder::new() - .name("solRpcNotifier".to_string()) - .spawn(move || { - let pool = rayon::ThreadPoolBuilder::new() - .num_threads(notification_threads) - .thread_name(|i| format!("solRpcNotify{i:02}")) - .build() - .unwrap(); - pool.install(|| { - if let Some(rpc_notifier_ready) = rpc_notifier_ready { - rpc_notifier_ready.fetch_or(true, Ordering::Relaxed); - } - Self::process_notifications( - exit, - max_complete_transaction_status_slot, - max_complete_rewards_slot, - blockstore, - notifier, - notification_receiver, - subscriptions, - bank_forks, - block_commitment_cache, - optimistically_confirmed_bank, - ) - }); - }) - .unwrap(), - ) - }; + Builder::new() + .name("solRpcNotifier".to_string()) + .spawn(move || { + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(notification_threads.get()) + .thread_name(|i| format!("solRpcNotify{i:02}")) + .build() + .unwrap(); + pool.install(|| { + if let Some(rpc_notifier_ready) = rpc_notifier_ready { + rpc_notifier_ready.fetch_or(true, Ordering::Relaxed); + } + Self::process_notifications( + exit, + max_complete_transaction_status_slot, + max_complete_rewards_slot, + blockstore, + notifier, + notification_receiver, + subscriptions, + bank_forks, + block_commitment_cache, + optimistically_confirmed_bank, + ) + }); + }) + .unwrap() + }); let control = SubscriptionControl::new( config.max_active_subscriptions, @@ -674,11 +669,7 @@ impl RpcSubscriptions { ); Self { - notification_sender: if notification_threads == 0 { - None - } else { - Some(notification_sender) - }, + notification_sender: config.notification_threads.map(|_| notification_sender), t_cleanup, exit, control, diff --git a/validator/Cargo.toml b/validator/Cargo.toml index 74742c90faa29d..0a6324f454e2b2 100644 --- a/validator/Cargo.toml +++ b/validator/Cargo.toml @@ -50,6 +50,7 @@ solana-net-utils = { workspace = true } solana-perf = { workspace = true } solana-poh = { workspace = true } solana-program-runtime = { workspace = true } +solana-rayon-threadlimit = { workspace = true } solana-rpc = { workspace = true } solana-rpc-client = { workspace = true } solana-rpc-client-api = { workspace = true } diff --git a/validator/src/cli.rs b/validator/src/cli.rs index d1ad63b760f031..e9298d9c02928e 100644 --- a/validator/src/cli.rs +++ b/validator/src/cli.rs @@ -26,6 +26,7 @@ use { solana_faucet::faucet::{self, FAUCET_PORT}, solana_ledger::use_snapshot_archives_at_startup, solana_net_utils::{MINIMUM_VALIDATOR_PORT_RANGE_WIDTH, VALIDATOR_PORT_RANGE}, + solana_rayon_threadlimit::get_thread_count, solana_rpc::{rpc::MAX_REQUEST_BODY_SIZE, rpc_pubsub_service::PubSubConfig}, solana_rpc_client_api::request::MAX_MULTIPLE_ACCOUNTS, solana_runtime::{ @@ -1079,6 +1080,11 @@ pub fn app<'a>(version: &'a str, default_args: &'a DefaultArgs) -> App<'a, 'a> { .takes_value(true) .value_name("NUM_THREADS") .validator(is_parsable::) + .default_value_if( + "full_rpc_api", + None, + &default_args.rpc_pubsub_notification_threads, + ) .help( "The maximum number of threads that RPC PubSub will use for generating \ notifications. 0 will disable RPC PubSub notifications", @@ -2138,6 +2144,7 @@ pub struct DefaultArgs { pub rpc_bigtable_max_message_size: String, pub rpc_max_request_body_size: String, pub rpc_pubsub_worker_threads: String, + pub rpc_pubsub_notification_threads: String, pub maximum_local_snapshot_age: String, pub maximum_full_snapshot_archives_to_retain: String, @@ -2225,6 +2232,7 @@ impl DefaultArgs { rpc_bigtable_max_message_size: solana_storage_bigtable::DEFAULT_MAX_MESSAGE_SIZE .to_string(), rpc_pubsub_worker_threads: "4".to_string(), + rpc_pubsub_notification_threads: get_thread_count().to_string(), maximum_full_snapshot_archives_to_retain: DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN .to_string(), maximum_incremental_snapshot_archives_to_retain: diff --git a/validator/src/main.rs b/validator/src/main.rs index b00eabfef9a7b0..7f3de66b457c74 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -1382,11 +1382,9 @@ pub fn main() { usize ), worker_threads: value_t_or_exit!(matches, "rpc_pubsub_worker_threads", usize), - notification_threads: if full_api { - value_of(&matches, "rpc_pubsub_notification_threads") - } else { - Some(0) - }, + notification_threads: value_t!(matches, "rpc_pubsub_notification_threads", usize) + .ok() + .and_then(NonZeroUsize::new), }, voting_disabled: matches.is_present("no_voting") || restricted_repair_only_mode, wait_for_supermajority: value_t!(matches, "wait_for_supermajority", Slot).ok(), From e682fec28e6c40741c7e6c5a566553d0c3f916f4 Mon Sep 17 00:00:00 2001 From: Yihau Chen Date: Wed, 13 Mar 2024 12:17:08 +0800 Subject: [PATCH 68/84] [anza migration]: fix download link for net scripts (#219) --- net/net.sh | 2 +- scripts/agave-install-deploy.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/net/net.sh b/net/net.sh index c1e93d095be7eb..b82872f5046d8e 100755 --- a/net/net.sh +++ b/net/net.sh @@ -563,7 +563,7 @@ prepareDeploy() { if [[ -n $releaseChannel ]]; then echo "Downloading release from channel: $releaseChannel" rm -f "$SOLANA_ROOT"/solana-release.tar.bz2 - declare updateDownloadUrl=https://release.agave.xyz/"$releaseChannel"/solana-release-x86_64-unknown-linux-gnu.tar.bz2 + declare updateDownloadUrl=https://release.anza.xyz/"$releaseChannel"/solana-release-x86_64-unknown-linux-gnu.tar.bz2 ( set -x curl -L -I "$updateDownloadUrl" diff --git a/scripts/agave-install-deploy.sh b/scripts/agave-install-deploy.sh index 01366a1cfbc5af..dcdec14ffb635d 100755 --- a/scripts/agave-install-deploy.sh +++ b/scripts/agave-install-deploy.sh @@ -57,7 +57,7 @@ esac case $TAG in edge|beta) - DOWNLOAD_URL=https://release.agave.xyz/"$TAG"/solana-release-$TARGET.tar.bz2 + DOWNLOAD_URL=https://release.anza.xyz/"$TAG"/solana-release-$TARGET.tar.bz2 ;; *) DOWNLOAD_URL=https://github.com/anza-xyz/agave/releases/download/"$TAG"/solana-release-$TARGET.tar.bz2 From 91bff85ffcbe85ba7d7154ac87372f5a73b06d43 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 13 Mar 2024 17:37:40 +0800 Subject: [PATCH 69/84] build(deps): bump http from 0.2.11 to 0.2.12 (#85) * build(deps): bump http from 0.2.11 to 0.2.12 Bumps [http](https://github.com/hyperium/http) from 0.2.11 to 0.2.12. - [Release notes](https://github.com/hyperium/http/releases) - [Changelog](https://github.com/hyperium/http/blob/v0.2.12/CHANGELOG.md) - [Commits](https://github.com/hyperium/http/compare/v0.2.11...v0.2.12) --- updated-dependencies: - dependency-name: http dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * [auto-commit] Update all Cargo lock files --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot-buildkite --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- programs/sbf/Cargo.lock | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 54d1cbc4e0b4b8..ac5ce30231bd0c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2661,9 +2661,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" dependencies = [ "bytes", "fnv", diff --git a/Cargo.toml b/Cargo.toml index f1ac84f6875b8f..430ce25c4bf102 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -216,7 +216,7 @@ hex = "0.4.3" hidapi = { version = "2.6.0", default-features = false } histogram = "0.6.9" hmac = "0.12.1" -http = "0.2.11" +http = "0.2.12" humantime = "2.0.1" hyper = "0.14.28" hyper-proxy = "0.9.1" diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index 9c78461b1b0a81..fadf7b3e28a5cc 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -2138,9 +2138,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" dependencies = [ "bytes", "fnv", From 9b16df25d33a0f00caef85d328a4d4d4e39572c9 Mon Sep 17 00:00:00 2001 From: ripatel-fd Date: Wed, 13 Mar 2024 13:24:39 +0100 Subject: [PATCH 70/84] sdk: support Pubkey::to_bytes as constexpr (#220) --- sdk/program/src/pubkey.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/program/src/pubkey.rs b/sdk/program/src/pubkey.rs index 728a5cd252d89f..2f1ccbdcfbfd7e 100644 --- a/sdk/program/src/pubkey.rs +++ b/sdk/program/src/pubkey.rs @@ -631,7 +631,7 @@ impl Pubkey { } } - pub fn to_bytes(self) -> [u8; 32] { + pub const fn to_bytes(self) -> [u8; 32] { self.0 } From 7f27644e6556a9f3b765df0ee52dd05098364cda Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 13 Mar 2024 21:59:25 +0800 Subject: [PATCH 71/84] build(deps): bump raptorq from 1.8.0 to 1.8.1 (#222) Bumps [raptorq](https://github.com/cberner/raptorq) from 1.8.0 to 1.8.1. - [Release notes](https://github.com/cberner/raptorq/releases) - [Commits](https://github.com/cberner/raptorq/compare/v1.8.0...v1.8.1) --- updated-dependencies: - dependency-name: raptorq dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ac5ce30231bd0c..8555c174ad1cbf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4481,9 +4481,9 @@ dependencies = [ [[package]] name = "raptorq" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c9cf9270cc5903afdef387f06ef1cd89fb77f45c357c2a425bae78b839fd866" +checksum = "7cc8cd0bcb2d520fff368264b5a6295e064c60955349517d09b14473afae4856" [[package]] name = "rayon" diff --git a/Cargo.toml b/Cargo.toml index 430ce25c4bf102..89a373d295135d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -278,7 +278,7 @@ quinn-proto = "0.10.6" quote = "1.0" rand = "0.8.5" rand_chacha = "0.3.1" -raptorq = "1.8.0" +raptorq = "1.8.1" rayon = "1.9.0" reed-solomon-erasure = "6.0.0" regex = "1.10.3" From 81075e60b36a68dbf718d46798769617571aaa6e Mon Sep 17 00:00:00 2001 From: Brennan Date: Wed, 13 Mar 2024 08:21:54 -0700 Subject: [PATCH 72/84] Loaded program test robustness (#59) * tests robust to prog entry count --- program-runtime/src/loaded_programs.rs | 320 +++++++++++-------------- 1 file changed, 135 insertions(+), 185 deletions(-) diff --git a/program-runtime/src/loaded_programs.rs b/program-runtime/src/loaded_programs.rs index e8a691c537934f..f6163d63cd738c 100644 --- a/program-runtime/src/loaded_programs.rs +++ b/program-runtime/src/loaded_programs.rs @@ -1301,100 +1301,93 @@ mod tests { assert_eq!(program.decayed_usage_counter(100), 0); } - #[test] - fn test_random_eviction() { - let mut programs = vec![]; - - let mut cache = new_mock_cache::(); - - // This test adds different kind of entries to the cache. - // Tombstones and unloaded entries are expected to not be evicted. - // It also adds multiple entries for three programs as it tries to create a typical cache instance. - let program1 = Pubkey::new_unique(); - let program1_deployment_slots = [0, 10, 20]; - let program1_usage_counters = [4, 5, 25]; - program1_deployment_slots + fn program_deploy_test_helper( + cache: &mut LoadedPrograms, + program: Pubkey, + deployment_slots: Vec, + usage_counters: Vec, + programs: &mut Vec<(Pubkey, Slot, u64)>, + ) { + // Add multiple entries for program + deployment_slots .iter() .enumerate() .for_each(|(i, deployment_slot)| { - let usage_counter = *program1_usage_counters.get(i).unwrap_or(&0); + let usage_counter = *usage_counters.get(i).unwrap_or(&0); cache.assign_program( - program1, + program, new_test_loaded_program_with_usage( *deployment_slot, - (*deployment_slot) + 2, + (*deployment_slot).saturating_add(2), AtomicU64::new(usage_counter), ), ); - programs.push((program1, *deployment_slot, usage_counter)); + programs.push((program, *deployment_slot, usage_counter)); }); + // Add tombstones entries for program let env = Arc::new(BuiltinProgram::new_mock()); for slot in 21..31 { set_tombstone( - &mut cache, - program1, + cache, + program, slot, LoadedProgramType::FailedVerification(env.clone()), ); } + // Add unloaded entries for program for slot in 31..41 { - insert_unloaded_program(&mut cache, program1, slot); + insert_unloaded_program(cache, program, slot); } + } - let program2 = Pubkey::new_unique(); - let program2_deployment_slots = [5, 11]; - let program2_usage_counters = [0, 2]; - program2_deployment_slots - .iter() - .enumerate() - .for_each(|(i, deployment_slot)| { - let usage_counter = *program2_usage_counters.get(i).unwrap_or(&0); - cache.assign_program( - program2, - new_test_loaded_program_with_usage( - *deployment_slot, - (*deployment_slot) + 2, - AtomicU64::new(usage_counter), - ), - ); - programs.push((program2, *deployment_slot, usage_counter)); - }); + #[test] + fn test_random_eviction() { + let mut programs = vec![]; - for slot in 31..41 { - insert_unloaded_program(&mut cache, program2, slot); - } + let mut cache = new_mock_cache::(); - let program3 = Pubkey::new_unique(); - let program3_deployment_slots = [0, 5, 15]; - let program3_usage_counters = [100, 3, 20]; - program3_deployment_slots - .iter() - .enumerate() - .for_each(|(i, deployment_slot)| { - let usage_counter = *program3_usage_counters.get(i).unwrap_or(&0); - cache.assign_program( - program3, - new_test_loaded_program_with_usage( - *deployment_slot, - (*deployment_slot) + 2, - AtomicU64::new(usage_counter), - ), - ); - programs.push((program3, *deployment_slot, usage_counter)); - }); + // This test adds different kind of entries to the cache. + // Tombstones and unloaded entries are expected to not be evicted. + // It also adds multiple entries for three programs as it tries to create a typical cache instance. - for slot in 21..31 { - set_tombstone(&mut cache, program3, slot, LoadedProgramType::Closed); - } + // Program 1 + program_deploy_test_helper( + &mut cache, + Pubkey::new_unique(), + vec![0, 10, 20], + vec![4, 5, 25], + &mut programs, + ); - for slot in 31..41 { - insert_unloaded_program(&mut cache, program3, slot); - } + // Program 2 + program_deploy_test_helper( + &mut cache, + Pubkey::new_unique(), + vec![5, 11], + vec![0, 2], + &mut programs, + ); - programs.sort_by_key(|(_id, _slot, usage_count)| *usage_count); + // Program 3 + program_deploy_test_helper( + &mut cache, + Pubkey::new_unique(), + vec![0, 5, 15], + vec![100, 3, 20], + &mut programs, + ); + // 1 for each deployment slot + let num_loaded_expected = 8; + // 10 for each program + let num_unloaded_expected = 30; + // 10 for each program + let num_tombstones_expected = 30; + + // Count the number of loaded, unloaded and tombstone entries. + programs.sort_by_key(|(_id, _slot, usage_count)| *usage_count); let num_loaded = num_matching_entries(&cache, |program_type| { matches!(program_type, LoadedProgramType::TestLoaded(_)) }); @@ -1411,16 +1404,19 @@ mod tests { }); // Test that the cache is constructed with the expected number of entries. - assert_eq!(num_loaded, 8); - assert_eq!(num_unloaded, 30); - assert_eq!(num_tombstones, 20); + assert_eq!(num_loaded, num_loaded_expected); + assert_eq!(num_unloaded, num_unloaded_expected); + assert_eq!(num_tombstones, num_tombstones_expected); - // Evicting to 2% should update cache with - // * 5 active entries - // * 33 unloaded entries (3 active programs will get unloaded) - // * 20 tombstones (tombstones are not evicted) - cache.evict_using_2s_random_selection(Percentage::from(2), 21); + // Evict entries from the cache + let eviction_pct = 2; + let num_loaded_expected = + Percentage::from(eviction_pct).apply_to(crate::loaded_programs::MAX_LOADED_ENTRY_COUNT); + let num_unloaded_expected = num_unloaded_expected + num_loaded - num_loaded_expected; + cache.evict_using_2s_random_selection(Percentage::from(eviction_pct), 21); + + // Count the number of loaded, unloaded and tombstone entries. let num_loaded = num_matching_entries(&cache, |program_type| { matches!(program_type, LoadedProgramType::TestLoaded(_)) }); @@ -1428,111 +1424,58 @@ mod tests { matches!(program_type, LoadedProgramType::Unloaded(_)) }); let num_tombstones = num_matching_entries(&cache, |program_type| { - matches!( - program_type, - LoadedProgramType::DelayVisibility - | LoadedProgramType::FailedVerification(_) - | LoadedProgramType::Closed - ) + matches!(program_type, LoadedProgramType::FailedVerification(_)) }); - // Test that expected number of loaded entries get evicted/unloaded. - assert_eq!(num_loaded, 5); - assert_eq!(num_unloaded, 33); - assert_eq!(num_tombstones, 20); + // However many entries are left after the shrink + assert_eq!(num_loaded, num_loaded_expected); + // The original unloaded entries + the evicted loaded entries + assert_eq!(num_unloaded, num_unloaded_expected); + // The original tombstones are not evicted + assert_eq!(num_tombstones, num_tombstones_expected); } #[test] fn test_eviction() { let mut programs = vec![]; - let mut cache = new_mock_cache::(); - let program1 = Pubkey::new_unique(); - let program1_deployment_slots = [0, 10, 20]; - let program1_usage_counters = [4, 5, 25]; - program1_deployment_slots - .iter() - .enumerate() - .for_each(|(i, deployment_slot)| { - let usage_counter = *program1_usage_counters.get(i).unwrap_or(&0); - cache.assign_program( - program1, - new_test_loaded_program_with_usage( - *deployment_slot, - (*deployment_slot) + 2, - AtomicU64::new(usage_counter), - ), - ); - programs.push((program1, *deployment_slot, usage_counter)); - }); - - let env = Arc::new(BuiltinProgram::new_mock()); - for slot in 21..31 { - set_tombstone( - &mut cache, - program1, - slot, - LoadedProgramType::FailedVerification(env.clone()), - ); - } - - for slot in 31..41 { - insert_unloaded_program(&mut cache, program1, slot); - } - - let program2 = Pubkey::new_unique(); - let program2_deployment_slots = [5, 11]; - let program2_usage_counters = [0, 2]; - program2_deployment_slots - .iter() - .enumerate() - .for_each(|(i, deployment_slot)| { - let usage_counter = *program2_usage_counters.get(i).unwrap_or(&0); - cache.assign_program( - program2, - new_test_loaded_program_with_usage( - *deployment_slot, - (*deployment_slot) + 2, - AtomicU64::new(usage_counter), - ), - ); - programs.push((program2, *deployment_slot, usage_counter)); - }); - - for slot in 31..41 { - insert_unloaded_program(&mut cache, program2, slot); - } + // Program 1 + program_deploy_test_helper( + &mut cache, + Pubkey::new_unique(), + vec![0, 10, 20], + vec![4, 5, 25], + &mut programs, + ); - let program3 = Pubkey::new_unique(); - let program3_deployment_slots = [0, 5, 15]; - let program3_usage_counters = [100, 3, 20]; - program3_deployment_slots - .iter() - .enumerate() - .for_each(|(i, deployment_slot)| { - let usage_counter = *program3_usage_counters.get(i).unwrap_or(&0); - cache.assign_program( - program3, - new_test_loaded_program_with_usage( - *deployment_slot, - (*deployment_slot) + 2, - AtomicU64::new(usage_counter), - ), - ); - programs.push((program3, *deployment_slot, usage_counter)); - }); + // Program 2 + program_deploy_test_helper( + &mut cache, + Pubkey::new_unique(), + vec![5, 11], + vec![0, 2], + &mut programs, + ); - for slot in 21..31 { - set_tombstone(&mut cache, program3, slot, LoadedProgramType::Closed); - } + // Program 3 + program_deploy_test_helper( + &mut cache, + Pubkey::new_unique(), + vec![0, 5, 15], + vec![100, 3, 20], + &mut programs, + ); - for slot in 31..41 { - insert_unloaded_program(&mut cache, program3, slot); - } + // 1 for each deployment slot + let num_loaded_expected = 8; + // 10 for each program + let num_unloaded_expected = 30; + // 10 for each program + let num_tombstones_expected = 30; + // Count the number of loaded, unloaded and tombstone entries. programs.sort_by_key(|(_id, _slot, usage_count)| *usage_count); - let num_loaded = num_matching_entries(&cache, |program_type| { matches!(program_type, LoadedProgramType::TestLoaded(_)) }); @@ -1540,23 +1483,23 @@ mod tests { matches!(program_type, LoadedProgramType::Unloaded(_)) }); let num_tombstones = num_matching_entries(&cache, |program_type| { - matches!( - program_type, - LoadedProgramType::DelayVisibility - | LoadedProgramType::FailedVerification(_) - | LoadedProgramType::Closed - ) + matches!(program_type, LoadedProgramType::FailedVerification(_)) }); - assert_eq!(num_loaded, 8); - assert_eq!(num_unloaded, 30); - assert_eq!(num_tombstones, 20); + // Test that the cache is constructed with the expected number of entries. + assert_eq!(num_loaded, num_loaded_expected); + assert_eq!(num_unloaded, num_unloaded_expected); + assert_eq!(num_tombstones, num_tombstones_expected); + + // Evict entries from the cache + let eviction_pct = 2; + + let num_loaded_expected = + Percentage::from(eviction_pct).apply_to(crate::loaded_programs::MAX_LOADED_ENTRY_COUNT); + let num_unloaded_expected = num_unloaded_expected + num_loaded - num_loaded_expected; + + cache.sort_and_unload(Percentage::from(eviction_pct)); - // Evicting to 2% should update cache with - // * 5 active entries - // * 33 unloaded entries (3 active programs will get unloaded) - // * 20 tombstones (tombstones are not evicted) - cache.sort_and_unload(Percentage::from(2)); // Check that every program is still in the cache. programs.iter().for_each(|entry| { assert!(cache.entries.get(&entry.0).is_some()); @@ -1578,6 +1521,7 @@ mod tests { assert!(unloaded.contains(&(expected.0, expected.2))); } + // Count the number of loaded, unloaded and tombstone entries. let num_loaded = num_matching_entries(&cache, |program_type| { matches!(program_type, LoadedProgramType::TestLoaded(_)) }); @@ -1593,9 +1537,12 @@ mod tests { ) }); - assert_eq!(num_loaded, 5); - assert_eq!(num_unloaded, 33); - assert_eq!(num_tombstones, 20); + // However many entries are left after the shrink + assert_eq!(num_loaded, num_loaded_expected); + // The original unloaded entries + the evicted loaded entries + assert_eq!(num_unloaded, num_unloaded_expected); + // The original tombstones are not evicted + assert_eq!(num_tombstones, num_tombstones_expected); } #[test] @@ -1603,7 +1550,11 @@ mod tests { let mut cache = new_mock_cache::(); let program = Pubkey::new_unique(); - let num_total_programs = 6; + let evict_to_pct = 2; + let cache_capacity_after_shrink = + Percentage::from(evict_to_pct).apply_to(crate::loaded_programs::MAX_LOADED_ENTRY_COUNT); + // Add enough programs to the cache to trigger 1 eviction after shrinking. + let num_total_programs = (cache_capacity_after_shrink + 1) as u64; (0..num_total_programs).for_each(|i| { cache.assign_program( program, @@ -1611,8 +1562,7 @@ mod tests { ); }); - // This will unload the program deployed at slot 0, with usage count = 10 - cache.sort_and_unload(Percentage::from(2)); + cache.sort_and_unload(Percentage::from(evict_to_pct)); let num_unloaded = num_matching_entries(&cache, |program_type| { matches!(program_type, LoadedProgramType::Unloaded(_)) From 33f941d473092dd2dbd576f171d6c1c945aedcb4 Mon Sep 17 00:00:00 2001 From: Dmitri Makarov Date: Wed, 13 Mar 2024 12:15:24 -0400 Subject: [PATCH 73/84] SVM: Eliminate filter_executable_program_accounts from pub interface (#214) --- svm/src/transaction_processor.rs | 188 +++++++++++++++++++++++++- svm/tests/transaction_processor.rs | 206 ----------------------------- 2 files changed, 186 insertions(+), 208 deletions(-) delete mode 100644 svm/tests/transaction_processor.rs diff --git a/svm/src/transaction_processor.rs b/svm/src/transaction_processor.rs index 5801b3b8316fdc..c42566fc9876f9 100644 --- a/svm/src/transaction_processor.rs +++ b/svm/src/transaction_processor.rs @@ -339,7 +339,7 @@ impl TransactionBatchProcessor { /// Returns a hash map of executable program accounts (program accounts that are not writable /// in the given transactions), and their owners, for the transactions with a valid /// blockhash or nonce. - pub fn filter_executable_program_accounts<'a, CB: TransactionProcessingCallback>( + fn filter_executable_program_accounts<'a, CB: TransactionProcessingCallback>( callbacks: &CB, txs: &[SanitizedTransaction], lock_results: &mut [TransactionCheckResult], @@ -953,8 +953,9 @@ mod tests { bpf_loader, message::{LegacyMessage, Message, MessageHeader}, rent_debits::RentDebits, - signature::Signature, + signature::{Keypair, Signature}, sysvar::rent::Rent, + transaction::{SanitizedTransaction, Transaction, TransactionError}, transaction_context::TransactionContext, }, std::{ @@ -1949,4 +1950,187 @@ mod tests { assert_eq!(result[&key1], (&owner1, 2)); assert_eq!(result[&key2], (&owner2, 1)); } + + #[test] + fn test_filter_executable_program_accounts_no_errors() { + let keypair1 = Keypair::new(); + let keypair2 = Keypair::new(); + + let non_program_pubkey1 = Pubkey::new_unique(); + let non_program_pubkey2 = Pubkey::new_unique(); + let program1_pubkey = Pubkey::new_unique(); + let program2_pubkey = Pubkey::new_unique(); + let account1_pubkey = Pubkey::new_unique(); + let account2_pubkey = Pubkey::new_unique(); + let account3_pubkey = Pubkey::new_unique(); + let account4_pubkey = Pubkey::new_unique(); + + let account5_pubkey = Pubkey::new_unique(); + + let mut bank = MockBankCallback::default(); + bank.account_shared_data.insert( + non_program_pubkey1, + AccountSharedData::new(1, 10, &account5_pubkey), + ); + bank.account_shared_data.insert( + non_program_pubkey2, + AccountSharedData::new(1, 10, &account5_pubkey), + ); + bank.account_shared_data.insert( + program1_pubkey, + AccountSharedData::new(40, 1, &account5_pubkey), + ); + bank.account_shared_data.insert( + program2_pubkey, + AccountSharedData::new(40, 1, &account5_pubkey), + ); + bank.account_shared_data.insert( + account1_pubkey, + AccountSharedData::new(1, 10, &non_program_pubkey1), + ); + bank.account_shared_data.insert( + account2_pubkey, + AccountSharedData::new(1, 10, &non_program_pubkey2), + ); + bank.account_shared_data.insert( + account3_pubkey, + AccountSharedData::new(40, 1, &program1_pubkey), + ); + bank.account_shared_data.insert( + account4_pubkey, + AccountSharedData::new(40, 1, &program2_pubkey), + ); + + let tx1 = Transaction::new_with_compiled_instructions( + &[&keypair1], + &[non_program_pubkey1], + Hash::new_unique(), + vec![account1_pubkey, account2_pubkey, account3_pubkey], + vec![CompiledInstruction::new(1, &(), vec![0])], + ); + let sanitized_tx1 = SanitizedTransaction::from_transaction_for_tests(tx1); + + let tx2 = Transaction::new_with_compiled_instructions( + &[&keypair2], + &[non_program_pubkey2], + Hash::new_unique(), + vec![account4_pubkey, account3_pubkey, account2_pubkey], + vec![CompiledInstruction::new(1, &(), vec![0])], + ); + let sanitized_tx2 = SanitizedTransaction::from_transaction_for_tests(tx2); + + let owners = &[program1_pubkey, program2_pubkey]; + let programs = + TransactionBatchProcessor::::filter_executable_program_accounts( + &bank, + &[sanitized_tx1, sanitized_tx2], + &mut [(Ok(()), None, Some(0)), (Ok(()), None, Some(0))], + owners, + ); + + // The result should contain only account3_pubkey, and account4_pubkey as the program accounts + assert_eq!(programs.len(), 2); + assert_eq!( + programs + .get(&account3_pubkey) + .expect("failed to find the program account"), + &(&program1_pubkey, 2) + ); + assert_eq!( + programs + .get(&account4_pubkey) + .expect("failed to find the program account"), + &(&program2_pubkey, 1) + ); + } + + #[test] + fn test_filter_executable_program_accounts_invalid_blockhash() { + let keypair1 = Keypair::new(); + let keypair2 = Keypair::new(); + + let non_program_pubkey1 = Pubkey::new_unique(); + let non_program_pubkey2 = Pubkey::new_unique(); + let program1_pubkey = Pubkey::new_unique(); + let program2_pubkey = Pubkey::new_unique(); + let account1_pubkey = Pubkey::new_unique(); + let account2_pubkey = Pubkey::new_unique(); + let account3_pubkey = Pubkey::new_unique(); + let account4_pubkey = Pubkey::new_unique(); + + let account5_pubkey = Pubkey::new_unique(); + + let mut bank = MockBankCallback::default(); + bank.account_shared_data.insert( + non_program_pubkey1, + AccountSharedData::new(1, 10, &account5_pubkey), + ); + bank.account_shared_data.insert( + non_program_pubkey2, + AccountSharedData::new(1, 10, &account5_pubkey), + ); + bank.account_shared_data.insert( + program1_pubkey, + AccountSharedData::new(40, 1, &account5_pubkey), + ); + bank.account_shared_data.insert( + program2_pubkey, + AccountSharedData::new(40, 1, &account5_pubkey), + ); + bank.account_shared_data.insert( + account1_pubkey, + AccountSharedData::new(1, 10, &non_program_pubkey1), + ); + bank.account_shared_data.insert( + account2_pubkey, + AccountSharedData::new(1, 10, &non_program_pubkey2), + ); + bank.account_shared_data.insert( + account3_pubkey, + AccountSharedData::new(40, 1, &program1_pubkey), + ); + bank.account_shared_data.insert( + account4_pubkey, + AccountSharedData::new(40, 1, &program2_pubkey), + ); + + let tx1 = Transaction::new_with_compiled_instructions( + &[&keypair1], + &[non_program_pubkey1], + Hash::new_unique(), + vec![account1_pubkey, account2_pubkey, account3_pubkey], + vec![CompiledInstruction::new(1, &(), vec![0])], + ); + let sanitized_tx1 = SanitizedTransaction::from_transaction_for_tests(tx1); + + let tx2 = Transaction::new_with_compiled_instructions( + &[&keypair2], + &[non_program_pubkey2], + Hash::new_unique(), + vec![account4_pubkey, account3_pubkey, account2_pubkey], + vec![CompiledInstruction::new(1, &(), vec![0])], + ); + // Let's not register blockhash from tx2. This should cause the tx2 to fail + let sanitized_tx2 = SanitizedTransaction::from_transaction_for_tests(tx2); + + let owners = &[program1_pubkey, program2_pubkey]; + let mut lock_results = vec![(Ok(()), None, Some(0)), (Ok(()), None, None)]; + let programs = + TransactionBatchProcessor::::filter_executable_program_accounts( + &bank, + &[sanitized_tx1, sanitized_tx2], + &mut lock_results, + owners, + ); + + // The result should contain only account3_pubkey as the program accounts + assert_eq!(programs.len(), 1); + assert_eq!( + programs + .get(&account3_pubkey) + .expect("failed to find the program account"), + &(&program1_pubkey, 1) + ); + assert_eq!(lock_results[1].0, Err(TransactionError::BlockhashNotFound)); + } } diff --git a/svm/tests/transaction_processor.rs b/svm/tests/transaction_processor.rs deleted file mode 100644 index 1704054246748d..00000000000000 --- a/svm/tests/transaction_processor.rs +++ /dev/null @@ -1,206 +0,0 @@ -#![cfg(test)] - -use { - solana_program_runtime::loaded_programs::{BlockRelation, ForkGraph}, - solana_sdk::{ - account::AccountSharedData, - clock::Slot, - hash::Hash, - instruction::CompiledInstruction, - pubkey::Pubkey, - signature::Keypair, - transaction::{SanitizedTransaction, Transaction, TransactionError}, - }, - solana_svm::transaction_processor::TransactionBatchProcessor, -}; - -mod mock_bank; - -struct MockForkGraph {} - -impl ForkGraph for MockForkGraph { - fn relationship(&self, _a: Slot, _b: Slot) -> BlockRelation { - todo!() - } -} - -#[test] -fn test_filter_executable_program_accounts() { - let keypair1 = Keypair::new(); - let keypair2 = Keypair::new(); - - let non_program_pubkey1 = Pubkey::new_unique(); - let non_program_pubkey2 = Pubkey::new_unique(); - let program1_pubkey = Pubkey::new_unique(); - let program2_pubkey = Pubkey::new_unique(); - let account1_pubkey = Pubkey::new_unique(); - let account2_pubkey = Pubkey::new_unique(); - let account3_pubkey = Pubkey::new_unique(); - let account4_pubkey = Pubkey::new_unique(); - - let account5_pubkey = Pubkey::new_unique(); - - let mut bank = mock_bank::MockBankCallback::default(); - bank.account_shared_data.insert( - non_program_pubkey1, - AccountSharedData::new(1, 10, &account5_pubkey), - ); - bank.account_shared_data.insert( - non_program_pubkey2, - AccountSharedData::new(1, 10, &account5_pubkey), - ); - bank.account_shared_data.insert( - program1_pubkey, - AccountSharedData::new(40, 1, &account5_pubkey), - ); - bank.account_shared_data.insert( - program2_pubkey, - AccountSharedData::new(40, 1, &account5_pubkey), - ); - bank.account_shared_data.insert( - account1_pubkey, - AccountSharedData::new(1, 10, &non_program_pubkey1), - ); - bank.account_shared_data.insert( - account2_pubkey, - AccountSharedData::new(1, 10, &non_program_pubkey2), - ); - bank.account_shared_data.insert( - account3_pubkey, - AccountSharedData::new(40, 1, &program1_pubkey), - ); - bank.account_shared_data.insert( - account4_pubkey, - AccountSharedData::new(40, 1, &program2_pubkey), - ); - - let tx1 = Transaction::new_with_compiled_instructions( - &[&keypair1], - &[non_program_pubkey1], - Hash::new_unique(), - vec![account1_pubkey, account2_pubkey, account3_pubkey], - vec![CompiledInstruction::new(1, &(), vec![0])], - ); - let sanitized_tx1 = SanitizedTransaction::from_transaction_for_tests(tx1); - - let tx2 = Transaction::new_with_compiled_instructions( - &[&keypair2], - &[non_program_pubkey2], - Hash::new_unique(), - vec![account4_pubkey, account3_pubkey, account2_pubkey], - vec![CompiledInstruction::new(1, &(), vec![0])], - ); - let sanitized_tx2 = SanitizedTransaction::from_transaction_for_tests(tx2); - - let owners = &[program1_pubkey, program2_pubkey]; - let programs = TransactionBatchProcessor::::filter_executable_program_accounts( - &bank, - &[sanitized_tx1, sanitized_tx2], - &mut [(Ok(()), None, Some(0)), (Ok(()), None, Some(0))], - owners, - ); - - // The result should contain only account3_pubkey, and account4_pubkey as the program accounts - assert_eq!(programs.len(), 2); - assert_eq!( - programs - .get(&account3_pubkey) - .expect("failed to find the program account"), - &(&program1_pubkey, 2) - ); - assert_eq!( - programs - .get(&account4_pubkey) - .expect("failed to find the program account"), - &(&program2_pubkey, 1) - ); -} - -#[test] -fn test_filter_executable_program_accounts_invalid_blockhash() { - let keypair1 = Keypair::new(); - let keypair2 = Keypair::new(); - - let non_program_pubkey1 = Pubkey::new_unique(); - let non_program_pubkey2 = Pubkey::new_unique(); - let program1_pubkey = Pubkey::new_unique(); - let program2_pubkey = Pubkey::new_unique(); - let account1_pubkey = Pubkey::new_unique(); - let account2_pubkey = Pubkey::new_unique(); - let account3_pubkey = Pubkey::new_unique(); - let account4_pubkey = Pubkey::new_unique(); - - let account5_pubkey = Pubkey::new_unique(); - - let mut bank = mock_bank::MockBankCallback::default(); - bank.account_shared_data.insert( - non_program_pubkey1, - AccountSharedData::new(1, 10, &account5_pubkey), - ); - bank.account_shared_data.insert( - non_program_pubkey2, - AccountSharedData::new(1, 10, &account5_pubkey), - ); - bank.account_shared_data.insert( - program1_pubkey, - AccountSharedData::new(40, 1, &account5_pubkey), - ); - bank.account_shared_data.insert( - program2_pubkey, - AccountSharedData::new(40, 1, &account5_pubkey), - ); - bank.account_shared_data.insert( - account1_pubkey, - AccountSharedData::new(1, 10, &non_program_pubkey1), - ); - bank.account_shared_data.insert( - account2_pubkey, - AccountSharedData::new(1, 10, &non_program_pubkey2), - ); - bank.account_shared_data.insert( - account3_pubkey, - AccountSharedData::new(40, 1, &program1_pubkey), - ); - bank.account_shared_data.insert( - account4_pubkey, - AccountSharedData::new(40, 1, &program2_pubkey), - ); - - let tx1 = Transaction::new_with_compiled_instructions( - &[&keypair1], - &[non_program_pubkey1], - Hash::new_unique(), - vec![account1_pubkey, account2_pubkey, account3_pubkey], - vec![CompiledInstruction::new(1, &(), vec![0])], - ); - let sanitized_tx1 = SanitizedTransaction::from_transaction_for_tests(tx1); - - let tx2 = Transaction::new_with_compiled_instructions( - &[&keypair2], - &[non_program_pubkey2], - Hash::new_unique(), - vec![account4_pubkey, account3_pubkey, account2_pubkey], - vec![CompiledInstruction::new(1, &(), vec![0])], - ); - // Let's not register blockhash from tx2. This should cause the tx2 to fail - let sanitized_tx2 = SanitizedTransaction::from_transaction_for_tests(tx2); - - let owners = &[program1_pubkey, program2_pubkey]; - let mut lock_results = vec![(Ok(()), None, Some(0)), (Ok(()), None, None)]; - let programs = TransactionBatchProcessor::::filter_executable_program_accounts( - &bank, - &[sanitized_tx1, sanitized_tx2], - &mut lock_results, - owners, - ); - - // The result should contain only account3_pubkey as the program accounts - assert_eq!(programs.len(), 1); - assert_eq!( - programs - .get(&account3_pubkey) - .expect("failed to find the program account"), - &(&program1_pubkey, 1) - ); - assert_eq!(lock_results[1].0, Err(TransactionError::BlockhashNotFound)); -} From 5ed30beb2a771820879e3eb5ffec93361a1ed8b4 Mon Sep 17 00:00:00 2001 From: steviez Date: Wed, 13 Mar 2024 11:29:05 -0500 Subject: [PATCH 74/84] ledger-tool: Allow compute-slot-cost to operate on dead slots (#213) Make this command accept the --allow-dead-slots arg as well --- ledger-tool/src/main.rs | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index 8445782f840931..94298623e953cd 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -445,14 +445,14 @@ fn graph_forks(bank_forks: &BankForks, config: &GraphConfig) -> String { dot.join("\n") } -fn compute_slot_cost(blockstore: &Blockstore, slot: Slot) -> Result<(), String> { - if blockstore.is_dead(slot) { - return Err("Dead slot".to_string()); - } - +fn compute_slot_cost( + blockstore: &Blockstore, + slot: Slot, + allow_dead_slots: bool, +) -> Result<(), String> { let (entries, _num_shreds, _is_full) = blockstore - .get_slot_entries_with_shred_info(slot, 0, false) - .map_err(|err| format!(" Slot: {slot}, Failed to load entries, err {err:?}"))?; + .get_slot_entries_with_shred_info(slot, 0, allow_dead_slots) + .map_err(|err| format!("Slot: {slot}, Failed to load entries, err {err:?}"))?; let num_entries = entries.len(); let mut num_transactions = 0; @@ -1482,7 +1482,8 @@ fn main() { "Slots that their blocks are computed for cost, default to all slots \ in ledger", ), - ), + ) + .arg(&allow_dead_slots_arg), ) .program_subcommand() .get_matches(); @@ -2947,9 +2948,10 @@ fn main() { } else { slots = values_t_or_exit!(arg_matches, "slots", Slot); } + let allow_dead_slots = arg_matches.is_present("allow_dead_slots"); for slot in slots { - if let Err(err) = compute_slot_cost(&blockstore, slot) { + if let Err(err) = compute_slot_cost(&blockstore, slot, allow_dead_slots) { eprintln!("{err}"); } } From e13fbeb198fdb00c4381cff0c7ca627bd4cec365 Mon Sep 17 00:00:00 2001 From: Yueh-Hsuan Chiang <93241502+yhchiang-sol@users.noreply.github.com> Date: Wed, 13 Mar 2024 10:07:11 -0700 Subject: [PATCH 75/84] [TieredStorage] Repurpose TieredReadableAccount to HotAccount (#218) #### Problem As we further optimize the HotStorageMeta in #146, there is a need for a HotAccount struct that contains all the hot account information. Meanwhile, we currently don't have plans to develop a cold account format at this moment. As a result, this makes it desirable to repurpose TieredReadableAccount to HotAccount. #### Summary of Changes Repurpose TieredReadableAccount to HotAccount. #### Test Plan Existing tiered-storage tests. --- accounts-db/src/account_storage/meta.rs | 4 +- accounts-db/src/tiered_storage/hot.rs | 78 +++++++++++++++++++- accounts-db/src/tiered_storage/readable.rs | 83 +--------------------- 3 files changed, 79 insertions(+), 86 deletions(-) diff --git a/accounts-db/src/account_storage/meta.rs b/accounts-db/src/account_storage/meta.rs index b6c8d72042097a..cc01ba164b077f 100644 --- a/accounts-db/src/account_storage/meta.rs +++ b/accounts-db/src/account_storage/meta.rs @@ -3,7 +3,7 @@ use { accounts_hash::AccountHash, append_vec::AppendVecStoredAccountMeta, storable_accounts::StorableAccounts, - tiered_storage::{hot::HotAccountMeta, readable::TieredReadableAccount}, + tiered_storage::hot::{HotAccount, HotAccountMeta}, }, solana_sdk::{account::ReadableAccount, hash::Hash, pubkey::Pubkey, stake_history::Epoch}, std::{borrow::Borrow, marker::PhantomData}, @@ -114,7 +114,7 @@ impl< #[derive(PartialEq, Eq, Debug)] pub enum StoredAccountMeta<'storage> { AppendVec(AppendVecStoredAccountMeta<'storage>), - Hot(TieredReadableAccount<'storage, HotAccountMeta>), + Hot(HotAccount<'storage, HotAccountMeta>), } impl<'storage> StoredAccountMeta<'storage> { diff --git a/accounts-db/src/tiered_storage/hot.rs b/accounts-db/src/tiered_storage/hot.rs index 34f7915186ba9b..5448c9b0f8a5ee 100644 --- a/accounts-db/src/tiered_storage/hot.rs +++ b/accounts-db/src/tiered_storage/hot.rs @@ -13,7 +13,6 @@ use { meta::{AccountMetaFlags, AccountMetaOptionalFields, TieredAccountMeta}, mmap_utils::{get_pod, get_slice}, owners::{OwnerOffset, OwnersBlockFormat, OwnersTable, OWNER_NO_OWNER}, - readable::TieredReadableAccount, StorableAccounts, StorableAccountsWithHashesAndWriteVersions, TieredStorageError, TieredStorageFormat, TieredStorageResult, }, @@ -264,6 +263,81 @@ impl TieredAccountMeta for HotAccountMeta { } } +/// The struct that offers read APIs for accessing a hot account. +#[derive(PartialEq, Eq, Debug)] +pub struct HotAccount<'accounts_file, M: TieredAccountMeta> { + /// TieredAccountMeta + pub meta: &'accounts_file M, + /// The address of the account + pub address: &'accounts_file Pubkey, + /// The address of the account owner + pub owner: &'accounts_file Pubkey, + /// The index for accessing the account inside its belonging AccountsFile + pub index: IndexOffset, + /// The account block that contains this account. Note that this account + /// block may be shared with other accounts. + pub account_block: &'accounts_file [u8], +} + +impl<'accounts_file, M: TieredAccountMeta> HotAccount<'accounts_file, M> { + /// Returns the address of this account. + pub fn address(&self) -> &'accounts_file Pubkey { + self.address + } + + /// Returns the index to this account in its AccountsFile. + pub fn index(&self) -> IndexOffset { + self.index + } + + /// Returns the data associated to this account. + pub fn data(&self) -> &'accounts_file [u8] { + self.meta.account_data(self.account_block) + } +} + +impl<'accounts_file, M: TieredAccountMeta> ReadableAccount for HotAccount<'accounts_file, M> { + /// Returns the balance of the lamports of this account. + fn lamports(&self) -> u64 { + self.meta.lamports() + } + + /// Returns the address of the owner of this account. + fn owner(&self) -> &'accounts_file Pubkey { + self.owner + } + + /// Returns true if the data associated to this account is executable. + fn executable(&self) -> bool { + self.meta.flags().executable() + } + + /// Returns the epoch that this account will next owe rent by parsing + /// the specified account block. RENT_EXEMPT_RENT_EPOCH will be returned + /// if the account is rent-exempt. + /// + /// For a zero-lamport account, Epoch::default() will be returned to + /// default states of an AccountSharedData. + fn rent_epoch(&self) -> Epoch { + self.meta + .rent_epoch(self.account_block) + .unwrap_or(if self.lamports() != 0 { + RENT_EXEMPT_RENT_EPOCH + } else { + // While there is no valid-values for any fields of a zero + // lamport account, here we return Epoch::default() to + // match the default states of AccountSharedData. Otherwise, + // a hash mismatch will occur. + Epoch::default() + }) + } + + /// Returns the data associated to this account. + fn data(&self) -> &'accounts_file [u8] { + self.data() + } +} + /// The reader to a hot accounts file. #[derive(Debug)] pub struct HotStorageReader { @@ -437,7 +511,7 @@ impl HotStorageReader { let account_block = self.get_account_block(account_offset, index_offset)?; Ok(Some(( - StoredAccountMeta::Hot(TieredReadableAccount { + StoredAccountMeta::Hot(HotAccount { meta, address, owner, diff --git a/accounts-db/src/tiered_storage/readable.rs b/accounts-db/src/tiered_storage/readable.rs index 8f1d2007182a5b..e3d169d4f6d99e 100644 --- a/accounts-db/src/tiered_storage/readable.rs +++ b/accounts-db/src/tiered_storage/readable.rs @@ -6,94 +6,13 @@ use { footer::{AccountMetaFormat, TieredStorageFooter}, hot::HotStorageReader, index::IndexOffset, - meta::TieredAccountMeta, TieredStorageResult, }, }, - solana_sdk::{ - account::ReadableAccount, pubkey::Pubkey, rent_collector::RENT_EXEMPT_RENT_EPOCH, - stake_history::Epoch, - }, + solana_sdk::pubkey::Pubkey, std::path::Path, }; -/// The struct that offers read APIs for accessing a TieredAccount. -#[derive(PartialEq, Eq, Debug)] -pub struct TieredReadableAccount<'accounts_file, M: TieredAccountMeta> { - /// TieredAccountMeta - pub meta: &'accounts_file M, - /// The address of the account - pub address: &'accounts_file Pubkey, - /// The address of the account owner - pub owner: &'accounts_file Pubkey, - /// The index for accessing the account inside its belonging AccountsFile - pub index: IndexOffset, - /// The account block that contains this account. Note that this account - /// block may be shared with other accounts. - pub account_block: &'accounts_file [u8], -} - -impl<'accounts_file, M: TieredAccountMeta> TieredReadableAccount<'accounts_file, M> { - /// Returns the address of this account. - pub fn address(&self) -> &'accounts_file Pubkey { - self.address - } - - /// Returns the index to this account in its AccountsFile. - pub fn index(&self) -> IndexOffset { - self.index - } - - /// Returns the data associated to this account. - pub fn data(&self) -> &'accounts_file [u8] { - self.meta.account_data(self.account_block) - } -} - -impl<'accounts_file, M: TieredAccountMeta> ReadableAccount - for TieredReadableAccount<'accounts_file, M> -{ - /// Returns the balance of the lamports of this account. - fn lamports(&self) -> u64 { - self.meta.lamports() - } - - /// Returns the address of the owner of this account. - fn owner(&self) -> &'accounts_file Pubkey { - self.owner - } - - /// Returns true if the data associated to this account is executable. - fn executable(&self) -> bool { - self.meta.flags().executable() - } - - /// Returns the epoch that this account will next owe rent by parsing - /// the specified account block. RENT_EXEMPT_RENT_EPOCH will be returned - /// if the account is rent-exempt. - /// - /// For a zero-lamport account, Epoch::default() will be returned to - /// default states of an AccountSharedData. - fn rent_epoch(&self) -> Epoch { - self.meta - .rent_epoch(self.account_block) - .unwrap_or(if self.lamports() != 0 { - RENT_EXEMPT_RENT_EPOCH - } else { - // While there is no valid-values for any fields of a zero - // lamport account, here we return Epoch::default() to - // match the default states of AccountSharedData. Otherwise, - // a hash mismatch will occur. - Epoch::default() - }) - } - - /// Returns the data associated to this account. - fn data(&self) -> &'accounts_file [u8] { - self.data() - } -} - /// The reader of a tiered storage instance. #[derive(Debug)] pub enum TieredStorageReader { From 69b6d5a376d2bf4ccfe5e6a58bf629a19ea25478 Mon Sep 17 00:00:00 2001 From: Yueh-Hsuan Chiang <93241502+yhchiang-sol@users.noreply.github.com> Date: Wed, 13 Mar 2024 10:26:37 -0700 Subject: [PATCH 76/84] [TieredStorage] Remove the general-purposed TieredStorageWriter (#196) #### Problem tiered_storage/writer.rs was added when we planned to support multiple tiers in the tiered-storage (i.e., at least hot and cold). However, as we changed our plan to handle cold accounts as state-compressed accounts, we don't need a general purposed tiered-storage writer at this moment. #### Summary of Changes Remove tiered_storage/writer.rs as we currently don't have plans to develop cold storage. #### Test Plan Existing tiered-storage tests. --- accounts-db/src/tiered_storage.rs | 1 - accounts-db/src/tiered_storage/writer.rs | 63 ------------------------ 2 files changed, 64 deletions(-) delete mode 100644 accounts-db/src/tiered_storage/writer.rs diff --git a/accounts-db/src/tiered_storage.rs b/accounts-db/src/tiered_storage.rs index 2f8ebac65e3b57..e15adb388605c2 100644 --- a/accounts-db/src/tiered_storage.rs +++ b/accounts-db/src/tiered_storage.rs @@ -11,7 +11,6 @@ pub mod mmap_utils; pub mod owners; pub mod readable; mod test_utils; -pub mod writer; use { crate::{ diff --git a/accounts-db/src/tiered_storage/writer.rs b/accounts-db/src/tiered_storage/writer.rs deleted file mode 100644 index 113d331e4a15c4..00000000000000 --- a/accounts-db/src/tiered_storage/writer.rs +++ /dev/null @@ -1,63 +0,0 @@ -//! docs/src/proposals/append-vec-storage.md - -use { - crate::{ - account_storage::meta::{StorableAccountsWithHashesAndWriteVersions, StoredAccountInfo}, - accounts_hash::AccountHash, - storable_accounts::StorableAccounts, - tiered_storage::{ - error::TieredStorageError, file::TieredStorageFile, footer::TieredStorageFooter, - TieredStorageFormat, TieredStorageResult, - }, - }, - solana_sdk::account::ReadableAccount, - std::{borrow::Borrow, path::Path}, -}; - -#[derive(Debug)] -pub struct TieredStorageWriter<'format> { - storage: TieredStorageFile, - format: &'format TieredStorageFormat, -} - -impl<'format> TieredStorageWriter<'format> { - pub fn new( - file_path: impl AsRef, - format: &'format TieredStorageFormat, - ) -> TieredStorageResult { - Ok(Self { - storage: TieredStorageFile::new_writable(file_path)?, - format, - }) - } - - pub fn write_accounts< - 'a, - 'b, - T: ReadableAccount + Sync, - U: StorableAccounts<'a, T>, - V: Borrow, - >( - &self, - accounts: &StorableAccountsWithHashesAndWriteVersions<'a, 'b, T, U, V>, - skip: usize, - ) -> TieredStorageResult> { - let footer = TieredStorageFooter { - account_meta_format: self.format.account_meta_format, - owners_block_format: self.format.owners_block_format, - account_block_format: self.format.account_block_format, - index_block_format: self.format.index_block_format, - account_entry_count: accounts - .accounts - .len() - .saturating_sub(skip) - .try_into() - .expect("num accounts <= u32::MAX"), - ..TieredStorageFooter::default() - }; - - footer.write_footer_block(&self.storage)?; - - Err(TieredStorageError::Unsupported()) - } -} From 41658f2f589fb09fbabead6888c2ac6c06d39b6a Mon Sep 17 00:00:00 2001 From: Tao Zhu Date: Thu, 29 Feb 2024 00:58:19 +0000 Subject: [PATCH 77/84] Add functions to collect executed transactions fee in details; --- runtime/src/bank.rs | 90 ++++++++++++++++++++++++++++++++++++++++++++- sdk/src/fee.rs | 17 +++++++++ 2 files changed, 106 insertions(+), 1 deletion(-) diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index d1a1805d0d3a20..bb1bf3f5da8a0a 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -122,7 +122,7 @@ use { self, include_loaded_accounts_data_size_in_fee_calculation, remove_rounding_in_fee_calculation, FeatureSet, }, - fee::FeeStructure, + fee::{FeeDetails, FeeStructure}, fee_calculator::{FeeCalculator, FeeRateGovernor}, genesis_config::{ClusterType, GenesisConfig}, hard_forks::HardForks, @@ -253,6 +253,24 @@ impl AddAssign for SquashTiming { } } +#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)] +#[serde(rename_all = "camelCase")] +pub(crate) struct CollectorFeeDetails { + pub transaction_fee: u64, + pub priority_fee: u64, +} + +impl CollectorFeeDetails { + pub(crate) fn add(&mut self, fee_details: &FeeDetails) { + self.transaction_fee = self + .transaction_fee + .saturating_add(fee_details.transaction_fee()); + self.priority_fee = self + .priority_fee + .saturating_add(fee_details.prioritization_fee()); + } +} + #[derive(Debug)] pub struct BankRc { /// where all the Accounts are stored @@ -548,6 +566,7 @@ impl PartialEq for Bank { loaded_programs_cache: _, epoch_reward_status: _, transaction_processor: _, + collector_fee_details, // Ignore new fields explicitly if they do not impact PartialEq. // Adding ".." will remove compile-time checks that if a new field // is added to the struct, this PartialEq is accordingly updated. @@ -581,6 +600,8 @@ impl PartialEq for Bank { && *stakes_cache.stakes() == *other.stakes_cache.stakes() && epoch_stakes == &other.epoch_stakes && is_delta.load(Relaxed) == other.is_delta.load(Relaxed) + && *collector_fee_details.read().unwrap() + == *other.collector_fee_details.read().unwrap() } } @@ -808,6 +829,9 @@ pub struct Bank { epoch_reward_status: EpochRewardStatus, transaction_processor: TransactionBatchProcessor, + + /// Collected fee details + collector_fee_details: RwLock, } struct VoteWithStakeDelegations { @@ -994,6 +1018,7 @@ impl Bank { ))), epoch_reward_status: EpochRewardStatus::default(), transaction_processor: TransactionBatchProcessor::default(), + collector_fee_details: RwLock::new(CollectorFeeDetails::default()), }; bank.transaction_processor = TransactionBatchProcessor::new( @@ -1312,6 +1337,7 @@ impl Bank { loaded_programs_cache: parent.loaded_programs_cache.clone(), epoch_reward_status: parent.epoch_reward_status.clone(), transaction_processor: TransactionBatchProcessor::default(), + collector_fee_details: RwLock::new(CollectorFeeDetails::default()), }; new.transaction_processor = TransactionBatchProcessor::new( @@ -1832,6 +1858,8 @@ impl Bank { ))), epoch_reward_status: fields.epoch_reward_status, transaction_processor: TransactionBatchProcessor::default(), + // collector_fee_details is not serialized to snapshot + collector_fee_details: RwLock::new(CollectorFeeDetails::default()), }; bank.transaction_processor = TransactionBatchProcessor::new( @@ -4871,6 +4899,66 @@ impl Bank { results } + // Note: this function is not yet used; next PR will call it behind a feature gate + #[allow(dead_code)] + fn filter_program_errors_and_collect_fee_details( + &self, + txs: &[SanitizedTransaction], + execution_results: &[TransactionExecutionResult], + ) -> Vec> { + let mut accumulated_fee_details = FeeDetails::default(); + + let results = txs + .iter() + .zip(execution_results) + .map(|(tx, execution_result)| { + let (execution_status, durable_nonce_fee) = match &execution_result { + TransactionExecutionResult::Executed { details, .. } => { + Ok((&details.status, details.durable_nonce_fee.as_ref())) + } + TransactionExecutionResult::NotExecuted(err) => Err(err.clone()), + }?; + let is_nonce = durable_nonce_fee.is_some(); + + let message = tx.message(); + let fee_details = self.fee_structure.calculate_fee_details( + message, + &process_compute_budget_instructions(message.program_instructions_iter()) + .unwrap_or_default() + .into(), + self.feature_set + .is_active(&include_loaded_accounts_data_size_in_fee_calculation::id()), + ); + + // In case of instruction error, even though no accounts + // were stored we still need to charge the payer the + // fee. + // + //...except nonce accounts, which already have their + // post-load, fee deducted, pre-execute account state + // stored + if execution_status.is_err() && !is_nonce { + self.withdraw( + tx.message().fee_payer(), + fee_details.total_fee( + self.feature_set + .is_active(&remove_rounding_in_fee_calculation::id()), + ), + )?; + } + + accumulated_fee_details.accumulate(&fee_details); + Ok(()) + }) + .collect(); + + self.collector_fee_details + .write() + .unwrap() + .add(&accumulated_fee_details); + results + } + /// `committed_transactions_count` is the number of transactions out of `sanitized_txs` /// that was executed. Of those, `committed_transactions_count`, /// `committed_with_failure_result_count` is the number of executed transactions that returned diff --git a/sdk/src/fee.rs b/sdk/src/fee.rs index b325a23ac08d9d..25c4b62acdf61b 100644 --- a/sdk/src/fee.rs +++ b/sdk/src/fee.rs @@ -47,6 +47,23 @@ impl FeeDetails { (total_fee as f64).round() as u64 } } + + pub fn accumulate(&mut self, fee_details: &FeeDetails) { + self.transaction_fee = self + .transaction_fee + .saturating_add(fee_details.transaction_fee); + self.prioritization_fee = self + .prioritization_fee + .saturating_add(fee_details.prioritization_fee) + } + + pub fn transaction_fee(&self) -> u64 { + self.transaction_fee + } + + pub fn prioritization_fee(&self) -> u64 { + self.prioritization_fee + } } pub const ACCOUNT_DATA_COST_PAGE_SIZE: u64 = 32_u64.saturating_mul(1024); From 0b1c908a134ec71975c52b4ea02e0ec830e9049b Mon Sep 17 00:00:00 2001 From: Tao Zhu Date: Mon, 11 Mar 2024 18:15:48 +0000 Subject: [PATCH 78/84] remove unnecessary derive attributes --- runtime/src/bank.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index bb1bf3f5da8a0a..9886f9b2dddd21 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -253,8 +253,7 @@ impl AddAssign for SquashTiming { } } -#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)] -#[serde(rename_all = "camelCase")] +#[derive(Debug, Default, PartialEq)] pub(crate) struct CollectorFeeDetails { pub transaction_fee: u64, pub priority_fee: u64, From 4cf1b49d6dc67f6affd08d6c97c2566e0b8b49f2 Mon Sep 17 00:00:00 2001 From: Tao Zhu Date: Tue, 12 Mar 2024 03:00:28 +0000 Subject: [PATCH 79/84] change function name from add to accumulate; remove collector_fee_details from PartialEq --- runtime/src/bank.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index 9886f9b2dddd21..6fff47afce5fb3 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -260,7 +260,7 @@ pub(crate) struct CollectorFeeDetails { } impl CollectorFeeDetails { - pub(crate) fn add(&mut self, fee_details: &FeeDetails) { + pub(crate) fn accumulate(&mut self, fee_details: &FeeDetails) { self.transaction_fee = self .transaction_fee .saturating_add(fee_details.transaction_fee()); @@ -565,7 +565,7 @@ impl PartialEq for Bank { loaded_programs_cache: _, epoch_reward_status: _, transaction_processor: _, - collector_fee_details, + collector_fee_details: _, // Ignore new fields explicitly if they do not impact PartialEq. // Adding ".." will remove compile-time checks that if a new field // is added to the struct, this PartialEq is accordingly updated. @@ -599,8 +599,6 @@ impl PartialEq for Bank { && *stakes_cache.stakes() == *other.stakes_cache.stakes() && epoch_stakes == &other.epoch_stakes && is_delta.load(Relaxed) == other.is_delta.load(Relaxed) - && *collector_fee_details.read().unwrap() - == *other.collector_fee_details.read().unwrap() } } @@ -4954,7 +4952,7 @@ impl Bank { self.collector_fee_details .write() .unwrap() - .add(&accumulated_fee_details); + .accumulate(&accumulated_fee_details); results } From e032eeb1f7ede3d21471714210ccd10c84caa72b Mon Sep 17 00:00:00 2001 From: Tao Zhu Date: Tue, 12 Mar 2024 03:11:48 +0000 Subject: [PATCH 80/84] add AbiExample --- runtime/src/bank.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index 6fff47afce5fb3..0b9bb1ab0b799e 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -253,7 +253,7 @@ impl AddAssign for SquashTiming { } } -#[derive(Debug, Default, PartialEq)] +#[derive(AbiExample, Debug, Default, PartialEq)] pub(crate) struct CollectorFeeDetails { pub transaction_fee: u64, pub priority_fee: u64, From ca0ae44681bd4ba0e6c9f2af8b3756ddaf199cc9 Mon Sep 17 00:00:00 2001 From: Tao Zhu Date: Tue, 12 Mar 2024 21:00:25 +0000 Subject: [PATCH 81/84] add test --- runtime/src/bank/tests.rs | 70 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/runtime/src/bank/tests.rs b/runtime/src/bank/tests.rs index 29dbdc2e5aeacd..56ebd51916ebf7 100644 --- a/runtime/src/bank/tests.rs +++ b/runtime/src/bank/tests.rs @@ -13794,3 +13794,73 @@ fn test_failed_simulation_compute_units() { let simulation = bank.simulate_transaction(&sanitized, false); assert_eq!(expected_consumed_units, simulation.units_consumed); } + +#[test] +fn test_filter_program_errors_and_collect_fee_details() { + // TX | EXECUTION RESULT | COLLECT | ADDITIONAL | COLLECT + // | (TX_FEE, PRIO_FEE) | WITHDRAW FROM PAYER | RESULT + // -------------------------------------------------------------------------------------------------- + // tx1 | executed and no error | (5_000, 1_000) | 0 | Ok + // tx2 | executed has error | (5_000, 1_000) | 6_000 | Ok + // tx3 | not executed | (0 , 0) | 0 | Original Err + // tx4 | executed error, payer insufficient fund | (0 , 0) | 0 | InsufficientFundsForFee + // + let initial_payer_balance = 7_000; + let additional_payer_withdraw = 6_000; + let expected_collected_fee_details = CollectorFeeDetails { + transaction_fee: 10_000, + priority_fee: 2_000, + }; + + let GenesisConfigInfo { + mut genesis_config, + mint_keypair, + .. + } = create_genesis_config_with_leader(initial_payer_balance, &Pubkey::new_unique(), 3); + genesis_config.fee_rate_governor = FeeRateGovernor::new(5000, 0); + let bank = Bank::new_for_tests(&genesis_config); + + let tx = SanitizedTransaction::from_transaction_for_tests(Transaction::new_signed_with_payer( + &[ + system_instruction::transfer(&mint_keypair.pubkey(), &Pubkey::new_unique(), 2), + ComputeBudgetInstruction::set_compute_unit_limit(1_000), + ComputeBudgetInstruction::set_compute_unit_price(1_000_000), + ], + Some(&mint_keypair.pubkey()), + &[&mint_keypair], + genesis_config.hash(), + )); + let txs = vec![tx.clone(), tx.clone(), tx.clone(), tx]; + + let results = vec![ + new_execution_result(Ok(()), None), + new_execution_result( + Err(TransactionError::InstructionError( + 1, + SystemError::ResultWithNegativeLamports.into(), + )), + None, + ), + TransactionExecutionResult::NotExecuted(TransactionError::AccountNotFound), + new_execution_result(Err(TransactionError::AccountNotFound), None), + ]; + + let expected_collect_results = vec![ + Ok(()), + Ok(()), + Err(TransactionError::AccountNotFound), + Err(TransactionError::InsufficientFundsForFee), + ]; + + let results = bank.filter_program_errors_and_collect_fee_details(&txs, &results); + + assert_eq!( + expected_collected_fee_details, + *bank.collector_fee_details.read().unwrap() + ); + assert_eq!( + initial_payer_balance - additional_payer_withdraw, + bank.get_balance(&mint_keypair.pubkey()) + ); + assert_eq!(expected_collect_results, results); +} From 1af263316cda8276079b10b39b010b3b14245400 Mon Sep 17 00:00:00 2001 From: Tao Zhu Date: Wed, 13 Mar 2024 20:49:10 +0000 Subject: [PATCH 82/84] use target function for benching --- runtime/src/bank.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index 0b9bb1ab0b799e..69b153fadf7c6a 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -3710,6 +3710,14 @@ impl Bank { self.update_slot_history(); self.run_incinerator(); + // to bench target function + { + let CollectorFeeDetails { + transaction_fee: _, + priority_fee: _, + } = *self.collector_fee_details.read().unwrap(); + } + // freeze is a one-way trip, idempotent self.freeze_started.store(true, Relaxed); *hash = self.hash_internal_state(); @@ -5069,6 +5077,8 @@ impl Bank { self.update_transaction_statuses(sanitized_txs, &execution_results); let fee_collection_results = self.filter_program_errors_and_collect_fee(sanitized_txs, &execution_results); + // to bench target function + let _ = self.filter_program_errors_and_collect_fee_details(sanitized_txs, &execution_results); update_transaction_statuses_time.stop(); timings.saturating_add_in_place( ExecuteTimingType::UpdateTransactionStatuses, From 91ece909f37b7ec4fe2c2b1b7f4539638f010260 Mon Sep 17 00:00:00 2001 From: Tao Zhu Date: Wed, 13 Mar 2024 21:17:07 +0000 Subject: [PATCH 83/84] Use Mutex --- runtime/src/bank.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index 69b153fadf7c6a..2a24bb18e5bc7c 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -828,7 +828,7 @@ pub struct Bank { transaction_processor: TransactionBatchProcessor, /// Collected fee details - collector_fee_details: RwLock, + collector_fee_details: Mutex, } struct VoteWithStakeDelegations { @@ -1015,7 +1015,7 @@ impl Bank { ))), epoch_reward_status: EpochRewardStatus::default(), transaction_processor: TransactionBatchProcessor::default(), - collector_fee_details: RwLock::new(CollectorFeeDetails::default()), + collector_fee_details: Mutex::new(CollectorFeeDetails::default()), }; bank.transaction_processor = TransactionBatchProcessor::new( @@ -1334,7 +1334,7 @@ impl Bank { loaded_programs_cache: parent.loaded_programs_cache.clone(), epoch_reward_status: parent.epoch_reward_status.clone(), transaction_processor: TransactionBatchProcessor::default(), - collector_fee_details: RwLock::new(CollectorFeeDetails::default()), + collector_fee_details: Mutex::new(CollectorFeeDetails::default()), }; new.transaction_processor = TransactionBatchProcessor::new( @@ -1856,7 +1856,7 @@ impl Bank { epoch_reward_status: fields.epoch_reward_status, transaction_processor: TransactionBatchProcessor::default(), // collector_fee_details is not serialized to snapshot - collector_fee_details: RwLock::new(CollectorFeeDetails::default()), + collector_fee_details: Mutex::new(CollectorFeeDetails::default()), }; bank.transaction_processor = TransactionBatchProcessor::new( @@ -3715,7 +3715,7 @@ impl Bank { let CollectorFeeDetails { transaction_fee: _, priority_fee: _, - } = *self.collector_fee_details.read().unwrap(); + } = *self.collector_fee_details.lock().unwrap(); } // freeze is a one-way trip, idempotent @@ -4958,7 +4958,7 @@ impl Bank { .collect(); self.collector_fee_details - .write() + .lock() .unwrap() .accumulate(&accumulated_fee_details); results From 0b77e2ba88ca42dd0e33e0ab910d9743d268889a Mon Sep 17 00:00:00 2001 From: Tao Zhu Date: Wed, 13 Mar 2024 21:30:32 +0000 Subject: [PATCH 84/84] use atomic --- runtime/src/bank.rs | 37 +++++++++++-------------------------- 1 file changed, 11 insertions(+), 26 deletions(-) diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index 2a24bb18e5bc7c..15816f746cab45 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -253,21 +253,10 @@ impl AddAssign for SquashTiming { } } -#[derive(AbiExample, Debug, Default, PartialEq)] +#[derive(AbiExample, Debug, Default)] pub(crate) struct CollectorFeeDetails { - pub transaction_fee: u64, - pub priority_fee: u64, -} - -impl CollectorFeeDetails { - pub(crate) fn accumulate(&mut self, fee_details: &FeeDetails) { - self.transaction_fee = self - .transaction_fee - .saturating_add(fee_details.transaction_fee()); - self.priority_fee = self - .priority_fee - .saturating_add(fee_details.prioritization_fee()); - } + pub transaction_fee: AtomicU64, + pub priority_fee: AtomicU64, } #[derive(Debug)] @@ -828,7 +817,7 @@ pub struct Bank { transaction_processor: TransactionBatchProcessor, /// Collected fee details - collector_fee_details: Mutex, + collector_fee_details: Arc, } struct VoteWithStakeDelegations { @@ -1015,7 +1004,7 @@ impl Bank { ))), epoch_reward_status: EpochRewardStatus::default(), transaction_processor: TransactionBatchProcessor::default(), - collector_fee_details: Mutex::new(CollectorFeeDetails::default()), + collector_fee_details: Arc::new(CollectorFeeDetails::default()), }; bank.transaction_processor = TransactionBatchProcessor::new( @@ -1334,7 +1323,7 @@ impl Bank { loaded_programs_cache: parent.loaded_programs_cache.clone(), epoch_reward_status: parent.epoch_reward_status.clone(), transaction_processor: TransactionBatchProcessor::default(), - collector_fee_details: Mutex::new(CollectorFeeDetails::default()), + collector_fee_details: Arc::new(CollectorFeeDetails::default()), }; new.transaction_processor = TransactionBatchProcessor::new( @@ -1856,7 +1845,7 @@ impl Bank { epoch_reward_status: fields.epoch_reward_status, transaction_processor: TransactionBatchProcessor::default(), // collector_fee_details is not serialized to snapshot - collector_fee_details: Mutex::new(CollectorFeeDetails::default()), + collector_fee_details: Arc::new(CollectorFeeDetails::default()), }; bank.transaction_processor = TransactionBatchProcessor::new( @@ -3712,10 +3701,8 @@ impl Bank { // to bench target function { - let CollectorFeeDetails { - transaction_fee: _, - priority_fee: _, - } = *self.collector_fee_details.lock().unwrap(); + let _transaction_fee = self.collector_fee_details.transaction_fee.load(Relaxed); + let _priority_fee = self.collector_fee_details.priority_fee.load(Relaxed); } // freeze is a one-way trip, idempotent @@ -4957,10 +4944,8 @@ impl Bank { }) .collect(); - self.collector_fee_details - .lock() - .unwrap() - .accumulate(&accumulated_fee_details); + self.collector_fee_details.transaction_fee.fetch_add(accumulated_fee_details.transaction_fee(), Relaxed); + self.collector_fee_details.priority_fee.fetch_add(accumulated_fee_details.prioritization_fee(), Relaxed); results }