From 3b940a47467aebe390e43eb493123578a6c8b0a2 Mon Sep 17 00:00:00 2001 From: Will Hickey Date: Sat, 2 Mar 2024 22:58:37 -0600 Subject: [PATCH 01/71] Add GHA to close new issues with a comment (#35151) * Add GHA to close new issues with a comment * Remove trailing whitespace --- .github/workflows/close-new-issues.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .github/workflows/close-new-issues.yml diff --git a/.github/workflows/close-new-issues.yml b/.github/workflows/close-new-issues.yml new file mode 100644 index 00000000000000..2d10a66fd5c2b9 --- /dev/null +++ b/.github/workflows/close-new-issues.yml @@ -0,0 +1,20 @@ +name: Close new issues + +on: + issues: + types: [opened, reopened] + +jobs: + comment-and-close: + runs-on: ubuntu-latest + steps: + - shell: bash + env: + ISSUE_NUMBER: ${{ github.event.issue.number }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + COMMENT: > + This repository is no longer in use. Please re-open this + issue in the agave repo: https://github.com/anza-xyz/agave + run: > + gh issue close "$ISSUE_NUMBER" --comment "$COMMENT" From a780ffb94181a6c01319d9903368a7d03dcb91dc Mon Sep 17 00:00:00 2001 From: Will Hickey Date: Sat, 2 Mar 2024 23:19:37 -0600 Subject: [PATCH 02/71] Update SECURITY.md to direct people to agave (#35399) --- SECURITY.md | 190 ++-------------------------------------------------- 1 file changed, 4 insertions(+), 186 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index 99f4ac8c113bad..a69294baa95e6f 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,187 +1,5 @@ -# Security Policy +## This repo will be archived soon +Active development in this repo is ending 2024-03-02. Anza is continuing development +in the [agave fork](https://github.com/anza-xyz/agave). Please refer to +[the security policy in that repo](https://github.com/anza-xyz/agave/security) -1. [Reporting security problems](#reporting) -4. [Security Bug Bounties](#bounty) -2. [Incident Response Process](#process) - - -## Reporting security problems in the Solana Labs Validator Client - -**DO NOT CREATE A GITHUB ISSUE** to report a security problem. - -Instead please use this [Report a Vulnerability](https://github.com/solana-labs/solana/security/advisories/new) link. -Provide a helpful title, detailed description of the vulnerability and an exploit -proof-of-concept. Speculative submissions without proof-of-concept will be closed -with no further consideration. - -Please refer to the -[Solana Program Library (SPL) security policy](https://github.com/solana-labs/solana-program-library/security/policy) -for vulnerabilities regarding SPL programs such as SPL Token. - -If you haven't done so already, please **enable two-factor auth** in your GitHub account. - -Expect a response as fast as possible in the advisory, typically within 72 hours. - --- - -If you do not receive a response in the advisory, send an email to -security@solanalabs.com with the full URL of the advisory you have created. DO NOT -include attachments or provide detail sufficient for exploitation regarding the -security issue in this email. **Only provide such details in the advisory**. - -If you do not receive a response from security@solanalabs.com please followup with -the team directly. You can do this in the `#core-technology` channel of the -[Solana Tech discord server](https://solana.com/discord), by pinging the `Solana Labs` -role in the channel and referencing the fact that you submitted a security problem. - - -## Incident Response Process - -In case an incident is discovered or reported, the following process will be -followed to contain, respond and remediate: - -### 1. Accept the new report -In response a newly reported security problem, a member of the -`solana-labs/admins` group will accept the report to turn it into a draft -advisory. The `solana-labs/security-incident-response` group should be added to -the draft security advisory, and create a private fork of the repository (grey -button towards the bottom of the page) if necessary. - -If the advisory is the result of an audit finding, follow the same process as above but add the auditor's github user(s) and begin the title with "[Audit]". - -If the report is out of scope, a member of the `solana-labs/admins` group will -comment as such and then close the report. - -### 2. Triage -Within the draft security advisory, discuss and determine the severity of the issue. If necessary, members of the solana-labs/security-incident-response group may add other github users to the advisory to assist. -If it is determined that this is not a critical network issue then the advisory should be closed and if more follow-up is required a normal Solana public github issue should be created. - -### 3. Prepare Fixes -For the affected branches, typically all three (edge, beta and stable), prepare a fix for the issue and push them to the corresponding branch in the private repository associated with the draft security advisory. -There is no CI available in the private repository so you must build from source and manually verify fixes. -Code review from the reporter is ideal, as well as from multiple members of the core development team. - -### 4. Notify Security Group Validators -Once an ETA is available for the fix, a member of the solana-labs/security-incident-response group should notify the validators so they can prepare for an update using the "Solana Red Alert" notification system. -The teams are all over the world and it's critical to provide actionable information at the right time. Don't be the person that wakes everybody up at 2am when a fix won't be available for hours. - -### 5. Ship the patch -Once the fix is accepted, a member of the solana-labs/security-incident-response group should prepare a single patch file for each affected branch. The commit title for the patch should only contain the advisory id, and not disclose any further details about the incident. -Copy the patches to https://release.solana.com/ under a subdirectory named after the advisory id (example: https://release.solana.com/GHSA-hx59-f5g4-jghh/v1.4.patch). Contact a member of the solana-labs/admins group if you require access to release.solana.com -Using the "Solana Red Alert" channel: - a) Notify validators that there's an issue and a patch will be provided in X minutes - b) If X minutes expires and there's no patch, notify of the delay and provide a new ETA - c) Provide links to patches of https://release.solana.com/ for each affected branch -Validators can be expected to build the patch from source against the latest release for the affected branch. -Since the software version will not change after the patch is applied, request that each validator notify in the existing channel once they've updated. Manually monitor the roll out until a sufficient amount of stake has updated - typically at least 33.3% or 66.6% depending on the issue. - -### 6. Public Disclosure and Release -Once the fix has been deployed to the security group validators, the patches from the security advisory may be merged into the main source repository. A new official release for each affected branch should be shipped and all validators requested to upgrade as quickly as possible. - -### 7. Security Advisory Bounty Accounting and Cleanup -If this issue is [eligible](#eligibility) for a bounty, prefix the title of the -security advisory with one of the following, depending on the severity: -- [Bounty Category: Critical: Loss of Funds] -- [Bounty Category: Critical: Consensus / Safety Violations] -- [Bounty Category: Critical: Liveness / Loss of Availability] -- [Bounty Category: Critical: DoS Attacks] -- [Bounty Category: Supply Chain Attacks] -- [Bounty Category: RPC] - -Confirm with the reporter that they agree with the severity assessment, and discuss as required to reach a conclusion. - -We currently do not use the Github workflow to publish security advisories. Once the issue and fix have been disclosed, and a bounty category is assessed if appropriate, the GitHub security advisory is no longer needed and can be closed. - - -## Security Bug Bounties -At its sole discretion, the Solana Foundation may offer a bounty for -[valid reports](#reporting) of critical Solana vulnerabilities. Please see below -for more details. The submitter is not required to provide a -mitigation to qualify. - -#### IMPORTANT | PLEASE NOTE -_Beginning February 1st 2024, the Security bounty program payouts will be updated in the following ways:_ -- _Bug Bounty rewards will be denominated in SOL tokens, rather than USD value._ -_This change is to better reflect for changing value of the Solana network._ -- _Categories will now have a discretionary range to distinguish the varying severity_ -_and impact of bugs reported within each broader category._ - -_Note: Payments will continue to be paid out in 12-month locked SOL._ - - -#### Loss of Funds: -_**As of 2/1/24:** Max: 25,000 SOL tokens. Min: 6,250 SOL tokens_ - -* Theft of funds without users signature from any account -* Theft of funds without users interaction in system, stake, vote programs -* Theft of funds that requires users signature - creating a vote program that drains the delegated stakes. - -#### Consensus/Safety Violations: -_**As of 2/1/24:** Max: 12,500 SOL tokens. Min: 3,125 SOL tokens_ - -* Consensus safety violation -* Tricking a validator to accept an optimistic confirmation or rooted slot without a double vote, etc. - -#### Liveness / Loss of Availability: -_**As of 2/1/24:** Max: 5,000 SOL tokens. Min: 1,250 SOL tokens_ - -* Whereby consensus halts and requires human intervention -* Eclipse attacks, -* Remote attacks that partition the network, - -#### DoS Attacks: -_**As of 2/1/24:** Max: 1,250 SOL tokens. Min: 315 SOL tokens_ - -* Remote resource exhaustion via Non-RPC protocols - -#### Supply Chain Attacks: -_**As of 2/1/24:** Max: 1,250 SOL tokens. Min: 315 SOL tokens_ - -* Non-social attacks against source code change management, automated testing, release build, release publication and release hosting infrastructure of the monorepo. - -#### RPC DoS/Crashes: -_**As of 2/1/24:** Max: 65 SOL tokens. Min: 20 SOL tokens_ - -* RPC attacks - -### Out of Scope: -The following components are out of scope for the bounty program -* Metrics: `/metrics` in the monorepo as well as https://metrics.solana.com -* Any encrypted credentials, auth tokens, etc. checked into the repo -* Bugs in dependencies. Please take them upstream! -* Attacks that require social engineering -* Any undeveloped automated tooling (scanners, etc) results. (OK with developed PoC) -* Any asset whose source code does not exist in this repository (including, but not limited -to, any and all web properties not explicitly listed on this page) -* Programs in the Solana Program Library, such as SPL Token. Please refer to the -[SPL security policy](https://github.com/solana-labs/solana-program-library/security/policy). - -### Eligibility: -* Submissions _MUST_ include an exploit proof-of-concept to be considered eligible -* The participant submitting the bug report shall follow the process outlined within this document -* Valid exploits can be eligible even if they are not successfully executed on a public cluster -* Multiple submissions for the same class of exploit are still eligible for compensation, though may be compensated at a lower rate, however these will be assessed on a case-by-case basis -* Participants must complete KYC and sign the participation agreement here when the registrations are open https://solana.foundation/kyc. Security exploits will still be assessed and open for submission at all times. This needs only be done prior to distribution of tokens. - -### Duplicate Reports -Compensation for duplicative reports will be split among reporters with first to report taking priority using the following equation -``` -R: total reports -ri: report priority -bi: bounty share - -bi = 2 ^ (R - ri) / ((2^R) - 1) -``` -#### Bounty Split Examples -| total reports | priority | share | | total reports | priority | share | | total reports | priority | share | -| ------------- | -------- | -----: | - | ------------- | -------- | -----: | - | ------------- | -------- | -----: | -| 1 | 1 | 100% | | 2 | 1 | 66.67% | | 5 | 1 | 51.61% | -| | | | | 2 | 2 | 33.33% | | 5 | 2 | 25.81% | -| 4 | 1 | 53.33% | | | | | | 5 | 3 | 12.90% | -| 4 | 2 | 26.67% | | 3 | 1 | 57.14% | | 5 | 4 | 6.45% | -| 4 | 3 | 13.33% | | 3 | 2 | 28.57% | | 5 | 5 | 3.23% | -| 4 | 4 | 6.67% | | 3 | 3 | 14.29% | | | | | - -### Payment of Bug Bounties: -* Bounties are currently awarded on a rolling/weekly basis and paid out within 30 days upon receipt of an invoice. -* Bug bounties that are paid out in SOL are paid to stake accounts with a lockup expiring 12 months from the date of delivery of SOL. From 4b25ab207ce41e7006ae6a43f84259a15908b40d Mon Sep 17 00:00:00 2001 From: Will Hickey Date: Mon, 4 Mar 2024 11:17:29 -0600 Subject: [PATCH 03/71] Add github action to close new pull requests (#35400) Add github action to close new pull requests with a comment directing them to agave --- .github/workflows/close-new-pull-requests.yml | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .github/workflows/close-new-pull-requests.yml diff --git a/.github/workflows/close-new-pull-requests.yml b/.github/workflows/close-new-pull-requests.yml new file mode 100644 index 00000000000000..1da0ad1d33ba14 --- /dev/null +++ b/.github/workflows/close-new-pull-requests.yml @@ -0,0 +1,20 @@ +name: Close new pull requests + +on: + pull_request: + types: [opened, reopened] + +jobs: + comment-and-close: + runs-on: ubuntu-latest + steps: + - shell: bash + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + COMMENT: > + This repository is no longer in use. Please re-open this + pull request in the agave repo: https://github.com/anza-xyz/agave + run: > + gh pr close "$PR_NUMBER" --comment "$COMMENT" From b84d81b7a51c98c3bec28a86e18ec871724b3fe7 Mon Sep 17 00:00:00 2001 From: Yihau Chen Date: Mon, 12 Feb 2024 15:17:51 +0800 Subject: [PATCH 04/71] [anza migration] replace binaries backend (#6) * ci: add upload-gcs-artifact * ci: publish release binaries to GCS * ci: redirect github repo to anza-xyz * ci: publish windows binaries to GCS * replace release.solana.com with release.anza.xyz * use a explicit name for credential --- .github/workflows/release-artifacts-auto.yml | 6 ++-- .../workflows/release-artifacts-manually.yml | 4 +-- .github/workflows/release-artifacts.yml | 33 +++++++++++-------- ci/publish-installer.sh | 12 +++---- ci/publish-tarball.sh | 6 ++-- ci/upload-ci-artifact.sh | 10 ++++++ ci/upload-github-release-asset.sh | 2 +- install/solana-install-init.sh | 4 +-- install/src/command.rs | 8 ++--- 9 files changed, 48 insertions(+), 37 deletions(-) diff --git a/.github/workflows/release-artifacts-auto.yml b/.github/workflows/release-artifacts-auto.yml index a8309cdffc8a72..0cdd176e04396c 100644 --- a/.github/workflows/release-artifacts-auto.yml +++ b/.github/workflows/release-artifacts-auto.yml @@ -14,14 +14,12 @@ concurrency: jobs: release-artifacts: - if: github.repository == 'solana-labs/solana' + if: github.repository == 'anza-xyz/agave' uses: ./.github/workflows/release-artifacts.yml with: commit: ${{ github.sha }} secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} + GCS_RELEASE_BUCKET_WRITER_CREDIENTIAL: ${{ secrets.GCS_RELEASE_BUCKET_WRITER_CREDIENTIAL }} error_reporting: needs: diff --git a/.github/workflows/release-artifacts-manually.yml b/.github/workflows/release-artifacts-manually.yml index 35de72922c32c8..fe5c1b03b638b3 100644 --- a/.github/workflows/release-artifacts-manually.yml +++ b/.github/workflows/release-artifacts-manually.yml @@ -14,6 +14,4 @@ jobs: with: commit: ${{ github.event.inputs.commit }} secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} + GCS_RELEASE_BUCKET_WRITER_CREDIENTIAL: ${{ secrets.GCS_RELEASE_BUCKET_WRITER_CREDIENTIAL }} diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 98dc697920262c..14760837ba0dbd 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -7,11 +7,7 @@ on: required: false type: string secrets: - AWS_ACCESS_KEY_ID: - required: true - AWS_SECRET_ACCESS_KEY: - required: true - AWS_S3_BUCKET: + GCS_RELEASE_BUCKET_WRITER_CREDIENTIAL: required: true jobs: @@ -71,17 +67,17 @@ jobs: shell: bash run: | FOLDER_NAME=${{ steps.build.outputs.tag || steps.build.outputs.channel }} - mkdir -p "github-action-s3-upload/$FOLDER_NAME" - cp -v "solana-release-x86_64-pc-windows-msvc.tar.bz2" "github-action-s3-upload/$FOLDER_NAME/" - cp -v "solana-release-x86_64-pc-windows-msvc.yml" "github-action-s3-upload/$FOLDER_NAME/" - cp -v "solana-install-init-x86_64-pc-windows-msvc"* "github-action-s3-upload/$FOLDER_NAME" + mkdir -p "windows-release/$FOLDER_NAME" + cp -v "solana-release-x86_64-pc-windows-msvc.tar.bz2" "windows-release/$FOLDER_NAME/" + cp -v "solana-release-x86_64-pc-windows-msvc.yml" "windows-release/$FOLDER_NAME/" + cp -v "solana-install-init-x86_64-pc-windows-msvc"* "windows-release/$FOLDER_NAME" - name: Upload Artifacts if: ${{ steps.build.outputs.channel != '' || steps.build.outputs.tag != '' }} uses: actions/upload-artifact@v3 with: name: windows-artifact - path: github-action-s3-upload/ + path: windows-release/ windows-s3-upload: if: ${{ needs.windows-build.outputs.channel != '' || needs.windows-build.outputs.tag != '' }} @@ -92,7 +88,16 @@ jobs: uses: actions/download-artifact@v3 with: name: windows-artifact - path: ./github-action-s3-upload + path: .windows-release/ + + - name: Setup crediential + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_RELEASE_BUCKET_WRITER_CREDIENTIAL }}" + + - name: Upload files to GCS + run: | + gcloud storage cp --recursive windows-release/* gs://anza-release/ - name: Upload uses: jakejarvis/s3-sync-action@master @@ -103,7 +108,7 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} AWS_REGION: "us-west-1" - SOURCE_DIR: "github-action-s3-upload" + SOURCE_DIR: "windows-release" windows-gh-release: if: ${{ needs.windows-build.outputs.tag != '' }} @@ -114,7 +119,7 @@ jobs: uses: actions/download-artifact@v3 with: name: windows-artifact - path: ./github-action-s3-upload + path: .windows-release/ - name: Release uses: softprops/action-gh-release@v1 @@ -122,4 +127,4 @@ jobs: tag_name: ${{ needs.windows-build.outputs.tag }} draft: true files: | - github-action-s3-upload/${{ needs.windows-build.outputs.tag }}/* + windows-release/${{ needs.windows-build.outputs.tag }}/* diff --git a/ci/publish-installer.sh b/ci/publish-installer.sh index 4b5345ae0d26fe..e58fd939dd1a40 100755 --- a/ci/publish-installer.sh +++ b/ci/publish-installer.sh @@ -26,14 +26,14 @@ fi # upload install script source ci/upload-ci-artifact.sh -cat >release.solana.com-install <release.anza.xyz-install <>release.solana.com-install +cat install/solana-install-init.sh >>release.anza.xyz-install -echo --- AWS S3 Store: "install" -upload-s3-artifact "/solana/release.solana.com-install" "s3://release.solana.com/$CHANNEL_OR_TAG/install" +echo --- GCS: "install" +upload-gcs-artifact "/solana/release.anza.xyz-install" "gs://anza-release/$CHANNEL_OR_TAG/install" echo Published to: -ci/format-url.sh https://release.solana.com/"$CHANNEL_OR_TAG"/install +ci/format-url.sh https://release.anza.xyz/"$CHANNEL_OR_TAG"/install diff --git a/ci/publish-tarball.sh b/ci/publish-tarball.sh index ff72bb7da2d066..5c64f09564fe9f 100755 --- a/ci/publish-tarball.sh +++ b/ci/publish-tarball.sh @@ -118,11 +118,11 @@ for file in "${TARBALL_BASENAME}"-$TARGET.tar.bz2 "${TARBALL_BASENAME}"-$TARGET. fi if [[ -n $BUILDKITE ]]; then - echo --- AWS S3 Store: "$file" - upload-s3-artifact "/solana/$file" s3://release.solana.com/"$CHANNEL_OR_TAG"/"$file" + echo --- GCS Store: "$file" + upload-gcs-artifact "/solana/$file" gs://anza-release/"$CHANNEL_OR_TAG"/"$file" echo Published to: - $DRYRUN ci/format-url.sh https://release.solana.com/"$CHANNEL_OR_TAG"/"$file" + $DRYRUN ci/format-url.sh https://release.anza.xyz/"$CHANNEL_OR_TAG"/"$file" if [[ -n $TAG ]]; then ci/upload-github-release-asset.sh "$file" diff --git a/ci/upload-ci-artifact.sh b/ci/upload-ci-artifact.sh index 1236da9f27114a..e7cc34ab2b2d8c 100644 --- a/ci/upload-ci-artifact.sh +++ b/ci/upload-ci-artifact.sh @@ -40,3 +40,13 @@ upload-s3-artifact() { docker run "${args[@]}" ) } + +upload-gcs-artifact() { + echo "--- artifact: $1 to $2" + docker run --rm \ + -v "$GCS_RELEASE_BUCKET_WRITER_CREDIENTIAL:/application_default_credentials.json" \ + -v "$PWD:/solana" \ + -e CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=/application_default_credentials.json \ + gcr.io/google.com/cloudsdktool/google-cloud-cli:latest \ + gcloud storage cp "$1" "$2" +} diff --git a/ci/upload-github-release-asset.sh b/ci/upload-github-release-asset.sh index ca2ae2a8f60443..229fb8993edafd 100755 --- a/ci/upload-github-release-asset.sh +++ b/ci/upload-github-release-asset.sh @@ -26,7 +26,7 @@ fi # Force CI_REPO_SLUG since sometimes # BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG is not set correctly, causing the # artifact upload to fail -CI_REPO_SLUG=solana-labs/solana +CI_REPO_SLUG=anza-xyz/agave #if [[ -z $CI_REPO_SLUG ]]; then # echo Error: CI_REPO_SLUG not defined # exit 1 diff --git a/install/solana-install-init.sh b/install/solana-install-init.sh index db36dc61e2ff30..4f28e300be52ab 100755 --- a/install/solana-install-init.sh +++ b/install/solana-install-init.sh @@ -16,9 +16,9 @@ { # this ensures the entire script is downloaded # if [ -z "$SOLANA_DOWNLOAD_ROOT" ]; then - SOLANA_DOWNLOAD_ROOT="https://github.com/solana-labs/solana/releases/download/" + SOLANA_DOWNLOAD_ROOT="https://github.com/anza-xyz/agave/releases/download/" fi -GH_LATEST_RELEASE="https://api.github.com/repos/solana-labs/solana/releases/latest" +GH_LATEST_RELEASE="https://api.github.com/repos/anza-xyz/agave/releases/latest" set -e diff --git a/install/src/command.rs b/install/src/command.rs index d7b92c17690bda..218e815467e9a9 100644 --- a/install/src/command.rs +++ b/install/src/command.rs @@ -572,7 +572,7 @@ pub fn init( fn github_release_download_url(release_semver: &str) -> String { format!( - "https://github.com/solana-labs/solana/releases/download/v{}/solana-release-{}.tar.bz2", + "https://github.com/anza-xyz/agave/releases/download/v{}/solana-release-{}.tar.bz2", release_semver, crate::build_env::TARGET ) @@ -580,7 +580,7 @@ fn github_release_download_url(release_semver: &str) -> String { fn release_channel_download_url(release_channel: &str) -> String { format!( - "https://release.solana.com/{}/solana-release-{}.tar.bz2", + "https://release.anza.xyz/{}/solana-release-{}.tar.bz2", release_channel, crate::build_env::TARGET ) @@ -588,7 +588,7 @@ fn release_channel_download_url(release_channel: &str) -> String { fn release_channel_version_url(release_channel: &str) -> String { format!( - "https://release.solana.com/{}/solana-release-{}.yml", + "https://release.anza.xyz/{}/solana-release-{}.yml", release_channel, crate::build_env::TARGET ) @@ -905,7 +905,7 @@ fn check_for_newer_github_release( while page == 1 || releases.len() == PER_PAGE { let url = reqwest::Url::parse_with_params( - "https://api.github.com/repos/solana-labs/solana/releases", + "https://api.github.com/repos/anza-xyz/agave/releases", &[ ("per_page", &format!("{PER_PAGE}")), ("page", &format!("{page}")), From f01ec82284c4325dfe0f6d3961cecab4223ce307 Mon Sep 17 00:00:00 2001 From: Yihau Chen Date: Mon, 12 Feb 2024 15:18:11 +0800 Subject: [PATCH 05/71] [anza migration] ci (#5) * Update README.md * ci: update CodeCov report link * ci: update github pr link * ci: rename secondary pipeline * replace org name in .mergify * update channel info link * update dependabot pr link * use anza docker image * delete travis --------- Co-authored-by: Will Hickey --- .mergify.yml | 4 +- .travis.yml | 94 -------------------------------- README.md | 34 +----------- ci/buildkite-pipeline-in-disk.sh | 4 +- ci/buildkite-pipeline.sh | 8 +-- ci/buildkite-solana-private.sh | 2 +- ci/channel-info.sh | 2 +- ci/dependabot-pr.sh | 2 +- ci/rust-version.sh | 2 +- ci/test-coverage.sh | 2 +- 10 files changed, 14 insertions(+), 140 deletions(-) delete mode 100644 .travis.yml diff --git a/.mergify.yml b/.mergify.yml index ef576943d5d635..166f59a5f365d1 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -33,7 +33,7 @@ pull_request_rules: actions: request_reviews: teams: - - "@solana-labs/community-pr-subscribers" + - "@anza-xyz/community-pr-subscribers" - name: label changes from monorepo-triage conditions: - author≠@core-contributors @@ -102,7 +102,7 @@ pull_request_rules: actions: backport: assignees: &BackportAssignee - - "{{ merged_by|replace('mergify[bot]', label|select('equalto', 'community')|first|default(author)|replace('community', '@solana-labs/community-pr-subscribers')) }}" + - "{{ merged_by|replace('mergify[bot]', label|select('equalto', 'community')|first|default(author)|replace('community', '@anza-xyz/community-pr-subscribers')) }}" title: "{{ destination_branch }}: {{ title }} (backport of #{{ number }})" ignore_conflicts: true labels: diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index c2dd13e32551ba..00000000000000 --- a/.travis.yml +++ /dev/null @@ -1,94 +0,0 @@ -branches: - only: - - master - - /^v\d+\.\d+/ - -notifications: - email: false - slack: - on_success: change - if: NOT type = pull_request - secure: F4IjOE05MyaMOdPRL+r8qhs7jBvv4yDM3RmFKE1zNXnfUOqV4X38oQM1EI+YVsgpMQLj/pxnEB7wcTE4Bf86N6moLssEULCpvAuMVoXj4QbWdomLX+01WbFa6fLVeNQIg45NHrz2XzVBhoKOrMNnl+QI5mbR2AlS5oqsudHsXDnyLzZtd4Y5SDMdYG1zVWM01+oNNjgNfjcCGmOE/K0CnOMl6GPi3X9C34tJ19P2XT7MTDsz1/IfEF7fro2Q8DHEYL9dchJMoisXSkem5z7IDQkGzXsWdWT4NnndUvmd1MlTCE9qgoXDqRf95Qh8sB1Dz08HtvgfaosP2XjtNTfDI9BBYS15Ibw9y7PchAJE1luteNjF35EOy6OgmCLw/YpnweqfuNViBZz+yOPWXVC0kxnPIXKZ1wyH9ibeH6E4hr7a8o9SV/6SiWIlbYF+IR9jPXyTCLP/cc3sYljPWxDnhWFwFdRVIi3PbVAhVu7uWtVUO17Oc9gtGPgs/GrhOMkJfwQPXaudRJDpVZowxTX4x9kefNotlMAMRgq+Drbmgt4eEBiCNp0ITWgh17BiE1U09WS3myuduhoct85+FoVeaUkp1sxzHVtGsNQH0hcz7WcpZyOM+AwistJA/qzeEDQao5zi1eKWPbO2xAhi2rV1bDH6bPf/4lDBwLRqSiwvlWU= - -os: linux -dist: bionic -language: minimal - -jobs: - include: - - &release-artifacts - if: type IN (api, cron) OR tag IS present - name: "macOS release artifacts" - os: osx - osx_image: xcode12 - language: rust - rust: - - stable - install: - - source ci/rust-version.sh - - PATH="/usr/local/opt/coreutils/libexec/gnubin:$PATH" - - readlink -f . - - brew install gnu-tar - - PATH="/usr/local/opt/gnu-tar/libexec/gnubin:$PATH" - - tar --version - script: - - source ci/env.sh - - rustup set profile default - - ci/publish-tarball.sh - deploy: - - provider: s3 - access_key_id: $AWS_ACCESS_KEY_ID - secret_access_key: $AWS_SECRET_ACCESS_KEY - bucket: release.solana.com - region: us-west-1 - skip_cleanup: true - acl: public_read - local_dir: travis-s3-upload - on: - all_branches: true - - provider: releases - token: $GITHUB_TOKEN - skip_cleanup: true - file_glob: true - file: travis-release-upload/* - on: - tags: true - - <<: *release-artifacts - name: "Windows release artifacts" - os: windows - install: - - choco install openssl - - export OPENSSL_DIR="C:\Program Files\OpenSSL-Win64" - - source ci/rust-version.sh - - PATH="/usr/local/opt/coreutils/libexec/gnubin:$PATH" - - readlink -f . - # Linux release artifacts are still built by ci/buildkite-secondary.yml - #- <<: *release-artifacts - # name: "Linux release artifacts" - # os: linux - # before_install: - # - sudo apt-get install libssl-dev libudev-dev - - # docs pull request - - name: "docs" - if: type IN (push, pull_request) OR tag IS present - language: node_js - node_js: - - "lts/*" - - services: - - docker - - cache: - directories: - - ~/.npm - - before_install: - - source ci/env.sh - - .travis/channel_restriction.sh edge beta || travis_terminate 0 - - .travis/affects.sh docs/ .travis || travis_terminate 0 - - cd docs/ - - source .travis/before_install.sh - - script: - - source .travis/script.sh diff --git a/README.md b/README.md index c6183f6ab6183e..bbaeb3d019a658 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@

- Solana + Solana

@@ -113,35 +113,3 @@ problem is solved by this code?" On the other hand, if a test does fail and you better way to solve the same problem, a Pull Request with your solution would most certainly be welcome! Likewise, if rewriting a test can better communicate what code it's protecting, please send us that patch! - -# Disclaimer - -All claims, content, designs, algorithms, estimates, roadmaps, -specifications, and performance measurements described in this project -are done with the Solana Labs, Inc. (“SL”) good faith efforts. It is up to -the reader to check and validate their accuracy and truthfulness. -Furthermore, nothing in this project constitutes a solicitation for -investment. - -Any content produced by SL or developer resources that SL provides are -for educational and inspirational purposes only. SL does not encourage, -induce or sanction the deployment, integration or use of any such -applications (including the code comprising the Solana blockchain -protocol) in violation of applicable laws or regulations and hereby -prohibits any such deployment, integration or use. This includes the use of -any such applications by the reader (a) in violation of export control -or sanctions laws of the United States or any other applicable -jurisdiction, (b) if the reader is located in or ordinarily resident in -a country or territory subject to comprehensive sanctions administered -by the U.S. Office of Foreign Assets Control (OFAC), or (c) if the -reader is or is working on behalf of a Specially Designated National -(SDN) or a person subject to similar blocking or denied party -prohibitions. - -The reader should be aware that U.S. export control and sanctions laws prohibit -U.S. persons (and other persons that are subject to such laws) from transacting -with persons in certain countries and territories or that are on the SDN list. -Accordingly, there is a risk to individuals that other persons using any of the -code contained in this repo, or a derivation thereof, may be sanctioned persons -and that transactions with such persons would be a violation of U.S. export -controls and sanctions law. diff --git a/ci/buildkite-pipeline-in-disk.sh b/ci/buildkite-pipeline-in-disk.sh index ad12e1fc000a89..2ce8af0432106b 100755 --- a/ci/buildkite-pipeline-in-disk.sh +++ b/ci/buildkite-pipeline-in-disk.sh @@ -289,7 +289,7 @@ if [[ -n $BUILDKITE_TAG ]]; then start_pipeline "Tag pipeline for $BUILDKITE_TAG" annotate --style info --context release-tag \ - "https://github.com/solana-labs/solana/releases/$BUILDKITE_TAG" + "https://github.com/anza-xyz/agave/releases/$BUILDKITE_TAG" # Jump directly to the secondary build to publish release artifacts quickly trigger_secondary_step @@ -307,7 +307,7 @@ if [[ $BUILDKITE_BRANCH =~ ^pull ]]; then # Add helpful link back to the corresponding Github Pull Request annotate --style info --context pr-backlink \ - "Github Pull Request: https://github.com/solana-labs/solana/$BUILDKITE_BRANCH" + "Github Pull Request: https://github.com/anza-xyz/agave/$BUILDKITE_BRANCH" if [[ $GITHUB_USER = "dependabot[bot]" ]]; then command_step dependabot "ci/dependabot-pr.sh" 5 diff --git a/ci/buildkite-pipeline.sh b/ci/buildkite-pipeline.sh index d40273863cc7a3..4ae00c9feab586 100755 --- a/ci/buildkite-pipeline.sh +++ b/ci/buildkite-pipeline.sh @@ -121,8 +121,8 @@ EOF trigger_secondary_step() { cat >> "$output_file" <<"EOF" - - name: "Trigger Build on solana-secondary" - trigger: "solana-secondary" + - name: "Trigger Build on agave-secondary" + trigger: "agave-secondary" branches: "!pull/*" async: true soft_fail: true @@ -315,7 +315,7 @@ if [[ -n $BUILDKITE_TAG ]]; then start_pipeline "Tag pipeline for $BUILDKITE_TAG" annotate --style info --context release-tag \ - "https://github.com/solana-labs/solana/releases/$BUILDKITE_TAG" + "https://github.com/anza-xyz/agave/releases/$BUILDKITE_TAG" # Jump directly to the secondary build to publish release artifacts quickly trigger_secondary_step @@ -333,7 +333,7 @@ if [[ $BUILDKITE_BRANCH =~ ^pull ]]; then # Add helpful link back to the corresponding Github Pull Request annotate --style info --context pr-backlink \ - "Github Pull Request: https://github.com/solana-labs/solana/$BUILDKITE_BRANCH" + "Github Pull Request: https://github.com/anza-xyz/agave/$BUILDKITE_BRANCH" if [[ $GITHUB_USER = "dependabot[bot]" ]]; then command_step dependabot "ci/dependabot-pr.sh" 5 diff --git a/ci/buildkite-solana-private.sh b/ci/buildkite-solana-private.sh index 70d8e4bfe4f59f..e5886a314eb27c 100755 --- a/ci/buildkite-solana-private.sh +++ b/ci/buildkite-solana-private.sh @@ -287,7 +287,7 @@ if [[ $BUILDKITE_BRANCH =~ ^pull ]]; then # Add helpful link back to the corresponding Github Pull Request annotate --style info --context pr-backlink \ - "Github Pull Request: https://github.com/solana-labs/solana/$BUILDKITE_BRANCH" + "Github Pull Request: https://github.com/anza-xyz/agave/$BUILDKITE_BRANCH" if [[ $GITHUB_USER = "dependabot[bot]" ]]; then command_step dependabot "ci/dependabot-pr.sh" 5 diff --git a/ci/channel-info.sh b/ci/channel-info.sh index c82806454d012c..2bb808365653c6 100755 --- a/ci/channel-info.sh +++ b/ci/channel-info.sh @@ -11,7 +11,7 @@ here="$(dirname "$0")" # shellcheck source=ci/semver_bash/semver.sh source "$here"/semver_bash/semver.sh -remote=https://github.com/solana-labs/solana.git +remote=https://github.com/anza-xyz/agave.git # Fetch all vX.Y.Z tags # diff --git a/ci/dependabot-pr.sh b/ci/dependabot-pr.sh index 91ecd5948c9a43..bb019001a0bcfa 100755 --- a/ci/dependabot-pr.sh +++ b/ci/dependabot-pr.sh @@ -21,7 +21,7 @@ fi echo --- "(FAILING) Backpropagating dependabot-triggered Cargo.lock updates" name="dependabot-buildkite" -api_base="https://api.github.com/repos/solana-labs/solana/pulls" +api_base="https://api.github.com/repos/anza-xyz/agave/pulls" pr_num=$(echo "$BUILDKITE_BRANCH" | grep -Eo '[0-9]+') branch=$(curl -s "$api_base/$pr_num" | python3 -c 'import json,sys;print(json.load(sys.stdin)["head"]["ref"])') diff --git a/ci/rust-version.sh b/ci/rust-version.sh index 3321f1d5ecb6a1..97ebb1c7935006 100644 --- a/ci/rust-version.sh +++ b/ci/rust-version.sh @@ -37,7 +37,7 @@ export rust_stable="$stable_version" export rust_nightly=nightly-"$nightly_version" -export ci_docker_image="solanalabs/ci:rust_${rust_stable}_${rust_nightly}" +export ci_docker_image="anzaxyz/ci:rust_${rust_stable}_${rust_nightly}" [[ -z $1 ]] || ( diff --git a/ci/test-coverage.sh b/ci/test-coverage.sh index 44231cd338a13e..ffd362acd287b8 100755 --- a/ci/test-coverage.sh +++ b/ci/test-coverage.sh @@ -32,5 +32,5 @@ else codecov -t "${CODECOV_TOKEN}" annotate --style success --context codecov.io \ - "CodeCov report: https://codecov.io/github/solana-labs/solana/commit/${CI_COMMIT:0:9}" + "CodeCov report: https://codecov.io/github/anza-xyz/agave/commit/${CI_COMMIT:0:9}" fi From d2a4f0f5e6c08f988a3985fed0bc1df1632ce746 Mon Sep 17 00:00:00 2001 From: Yihau Chen Date: Tue, 13 Feb 2024 00:00:58 +0800 Subject: [PATCH 06/71] [anza migration] ci: fix path (#8) * ci: fix windows build * ci: publish sdk docker image with the new name * update automerge status --- .github/workflows/release-artifacts.yml | 4 ++-- .mergify.yml | 2 +- sdk/docker-solana/build.sh | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 14760837ba0dbd..a77fd672d8b4b6 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -79,7 +79,7 @@ jobs: name: windows-artifact path: windows-release/ - windows-s3-upload: + windows-gcs-upload: if: ${{ needs.windows-build.outputs.channel != '' || needs.windows-build.outputs.tag != '' }} needs: [windows-build] runs-on: ubuntu-20.04 @@ -88,7 +88,7 @@ jobs: uses: actions/download-artifact@v3 with: name: windows-artifact - path: .windows-release/ + path: ./windows-release - name: Setup crediential uses: "google-github-actions/auth@v2" diff --git a/.mergify.yml b/.mergify.yml index 166f59a5f365d1..19f9b8f116a78a 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -50,7 +50,7 @@ pull_request_rules: - name: automatic merge (squash) on CI success conditions: - and: - - status-success=buildkite/solana + - status-success=buildkite/agave - status-success=ci-gate - label=automerge - label!=no-automerge diff --git a/sdk/docker-solana/build.sh b/sdk/docker-solana/build.sh index f1c8ee265d6d56..70e3d0d23e44de 100755 --- a/sdk/docker-solana/build.sh +++ b/sdk/docker-solana/build.sh @@ -29,7 +29,7 @@ cp -f ../../fetch-spl.sh usr/bin/ ./fetch-spl.sh ) -docker build -t solanalabs/solana:"$CHANNEL_OR_TAG" . +docker build -t anzaxyz/agave:"$CHANNEL_OR_TAG" . maybeEcho= if [[ -z $CI ]]; then @@ -43,4 +43,4 @@ else fi ) fi -$maybeEcho docker push solanalabs/solana:"$CHANNEL_OR_TAG" +$maybeEcho docker push anzaxyz/agave:"$CHANNEL_OR_TAG" From ff1d892277ea9c009ba663a3e177f32d5d63e034 Mon Sep 17 00:00:00 2001 From: Yihau Chen Date: Thu, 15 Feb 2024 22:06:51 +0800 Subject: [PATCH 07/71] [anza migration] ci: removed unused s3 upload in Windows build (#9) ci: removed unused s3 upload in Windows build --- .github/workflows/release-artifacts.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index a77fd672d8b4b6..d8e22c42ce5da8 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -99,17 +99,6 @@ jobs: run: | gcloud storage cp --recursive windows-release/* gs://anza-release/ - - name: Upload - uses: jakejarvis/s3-sync-action@master - with: - args: --acl public-read --follow-symlinks - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} - AWS_REGION: "us-west-1" - SOURCE_DIR: "windows-release" - windows-gh-release: if: ${{ needs.windows-build.outputs.tag != '' }} needs: [windows-build] From 14cb9cff9272a3947c5004b43b0ad5a60f3b10eb Mon Sep 17 00:00:00 2001 From: Yihau Chen Date: Thu, 22 Feb 2024 11:44:01 +0800 Subject: [PATCH 08/71] [anza migration] rename crates (#10) * rename geyser-plugin-interface * rename cargo registry * rename watchtower * rename ledger tool * rename validator * rename install * rename geyser plugin interface when patch --- .../scripts/downstream-project-spl-common.sh | 3 + .github/workflows/release-artifacts.yml | 2 +- Cargo.lock | 424 +++++++++--------- Cargo.toml | 4 +- cargo-registry/Cargo.toml | 4 +- ci/localnet-sanity.sh | 4 +- ci/publish-installer.sh | 2 +- ci/publish-tarball.sh | 4 +- ci/run-sanity.sh | 2 +- docs/src/backwards-compatibility.md | 2 +- docs/src/cli/install.md | 12 +- docs/src/clusters/available.md | 12 +- docs/src/clusters/benchmark.md | 2 +- docs/src/implemented-proposals/installer.md | 58 +-- .../rpc-transaction-history.md | 2 +- docs/src/operations/best-practices/general.md | 28 +- .../operations/best-practices/monitoring.md | 28 +- docs/src/operations/guides/restart-cluster.md | 14 +- .../operations/guides/validator-failover.md | 12 +- docs/src/operations/guides/validator-start.md | 32 +- docs/src/operations/guides/vote-accounts.md | 8 +- docs/src/operations/setup-a-validator.md | 24 +- docs/src/operations/setup-an-rpc-node.md | 4 +- docs/src/validator/geyser.md | 8 +- geyser-plugin-interface/Cargo.toml | 4 +- .../src/geyser_plugin_interface.rs | 2 +- geyser-plugin-manager/Cargo.toml | 2 +- .../src/accounts_update_notifier.rs | 6 +- .../src/block_metadata_notifier.rs | 4 +- geyser-plugin-manager/src/entry_notifier.rs | 6 +- .../src/geyser_plugin_manager.rs | 4 +- .../src/slot_status_notifier.rs | 2 +- .../src/transaction_notifier.rs | 4 +- install/Cargo.toml | 4 +- ...-install-init.sh => agave-install-init.sh} | 16 +- install/install-help.sh | 6 +- ...-install-init.rs => agave-install-init.rs} | 2 +- install/src/command.rs | 4 +- install/src/lib.rs | 2 +- install/src/main.rs | 2 +- ledger-tool/Cargo.toml | 4 +- ledger-tool/src/blockstore.rs | 2 +- ledger-tool/src/ledger_utils.rs | 8 +- ledger/src/blockstore_db.rs | 2 +- local-cluster/tests/local_cluster.rs | 4 +- multinode-demo/bootstrap-validator.sh | 4 +- multinode-demo/common.sh | 6 +- multinode-demo/validator.sh | 6 +- net/net.sh | 8 +- net/remote/remote-deploy-update.sh | 2 +- net/remote/remote-node.sh | 8 +- net/remote/remote-sanity.sh | 4 +- notifier/src/lib.rs | 4 +- programs/sbf/Cargo.lock | 152 +++---- programs/sbf/Cargo.toml | 2 +- programs/sbf/rust/simulation/Cargo.toml | 2 +- .../sbf/rust/simulation/tests/validator.rs | 2 +- .../src/nonblocking/pubsub_client.rs | 6 +- pubsub-client/src/pubsub_client.rs | 6 +- rbpf-cli/src/main.rs | 4 +- rpc/src/rpc.rs | 8 +- ...tall-deploy.sh => agave-install-deploy.sh} | 4 +- scripts/cargo-install-all.sh | 14 +- scripts/check-dev-context-only-utils.sh | 2 +- scripts/run.sh | 6 +- .../abi-testcases/mixed-validator-test.sh | 6 +- .../stability-testcases/gossip-dos-test.sh | 6 +- validator/Cargo.toml | 8 +- validator/src/bin/solana-test-validator.rs | 8 +- validator/src/bootstrap.rs | 2 +- validator/src/main.rs | 20 +- watchtower/Cargo.toml | 4 +- watchtower/README.md | 2 +- watchtower/src/main.rs | 10 +- 74 files changed, 554 insertions(+), 547 deletions(-) rename install/{solana-install-init.sh => agave-install-init.sh} (89%) rename install/src/bin/{solana-install-init.rs => agave-install-init.rs} (92%) rename scripts/{solana-install-deploy.sh => agave-install-deploy.sh} (90%) diff --git a/.github/scripts/downstream-project-spl-common.sh b/.github/scripts/downstream-project-spl-common.sh index c6dcfaca007867..861be12c7d1a45 100644 --- a/.github/scripts/downstream-project-spl-common.sh +++ b/.github/scripts/downstream-project-spl-common.sh @@ -22,3 +22,6 @@ if semverGT "$project_used_solana_version" "$SOLANA_VER"; then fi ./patch.crates-io.sh "$SOLANA_DIR" + +# anza migration stopgap. can be removed when agave is fully recommended for public usage. +sed -i 's/solana-geyser-plugin-interface/agave-geyser-plugin-interface/g' ./Cargo.toml diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index d8e22c42ce5da8..45be181c3ce9e1 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -70,7 +70,7 @@ jobs: mkdir -p "windows-release/$FOLDER_NAME" cp -v "solana-release-x86_64-pc-windows-msvc.tar.bz2" "windows-release/$FOLDER_NAME/" cp -v "solana-release-x86_64-pc-windows-msvc.yml" "windows-release/$FOLDER_NAME/" - cp -v "solana-install-init-x86_64-pc-windows-msvc"* "windows-release/$FOLDER_NAME" + cp -v "agave-install-init-x86_64-pc-windows-msvc"* "windows-release/$FOLDER_NAME" - name: Upload Artifacts if: ${{ steps.build.outputs.channel != '' || steps.build.outputs.tag != '' }} diff --git a/Cargo.lock b/Cargo.lock index 650b369d205c5c..c72b90930d7cf9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -62,6 +62,217 @@ dependencies = [ "zeroize", ] +[[package]] +name = "agave-cargo-registry" +version = "1.19.0" +dependencies = [ + "clap 2.33.3", + "flate2", + "hex", + "hyper", + "log", + "rustc_version 0.4.0", + "serde", + "serde_json", + "sha2 0.10.8", + "solana-clap-utils", + "solana-cli", + "solana-cli-config", + "solana-cli-output", + "solana-logger", + "solana-remote-wallet", + "solana-rpc-client", + "solana-rpc-client-api", + "solana-sdk", + "solana-version", + "tar", + "tempfile", + "tokio", + "toml 0.8.10", +] + +[[package]] +name = "agave-geyser-plugin-interface" +version = "1.19.0" +dependencies = [ + "log", + "solana-sdk", + "solana-transaction-status", + "thiserror", +] + +[[package]] +name = "agave-install" +version = "1.19.0" +dependencies = [ + "atty", + "bincode", + "bzip2", + "chrono", + "clap 2.33.3", + "console", + "crossbeam-channel", + "ctrlc", + "dirs-next", + "indicatif", + "lazy_static", + "nix 0.26.4", + "reqwest", + "scopeguard", + "semver 1.0.22", + "serde", + "serde_yaml 0.8.26", + "serde_yaml 0.9.32", + "solana-clap-utils", + "solana-config-program", + "solana-logger", + "solana-rpc-client", + "solana-sdk", + "solana-version", + "tar", + "tempfile", + "url 2.5.0", + "winapi 0.3.9", + "winreg", +] + +[[package]] +name = "agave-ledger-tool" +version = "1.19.0" +dependencies = [ + "assert_cmd", + "bs58", + "bytecount", + "chrono", + "clap 2.33.3", + "crossbeam-channel", + "csv", + "dashmap", + "futures 0.3.30", + "histogram", + "itertools", + "log", + "num_cpus", + "regex", + "serde", + "serde_json", + "signal-hook", + "solana-account-decoder", + "solana-accounts-db", + "solana-bpf-loader-program", + "solana-clap-utils", + "solana-cli-output", + "solana-core", + "solana-cost-model", + "solana-entry", + "solana-geyser-plugin-manager", + "solana-gossip", + "solana-ledger", + "solana-logger", + "solana-measure", + "solana-program-runtime", + "solana-rpc", + "solana-runtime", + "solana-sdk", + "solana-stake-program", + "solana-storage-bigtable", + "solana-streamer", + "solana-svm", + "solana-transaction-status", + "solana-unified-scheduler-pool", + "solana-version", + "solana-vote-program", + "solana_rbpf", + "thiserror", + "tikv-jemallocator", + "tokio", +] + +[[package]] +name = "agave-validator" +version = "1.19.0" +dependencies = [ + "agave-geyser-plugin-interface", + "chrono", + "clap 2.33.3", + "console", + "core_affinity", + "crossbeam-channel", + "fd-lock", + "indicatif", + "itertools", + "jsonrpc-core", + "jsonrpc-core-client", + "jsonrpc-derive", + "jsonrpc-ipc-server", + "jsonrpc-server-utils", + "lazy_static", + "libc", + "libloading", + "log", + "num_cpus", + "rand 0.8.5", + "rayon", + "serde", + "serde_json", + "serde_yaml 0.9.32", + "signal-hook", + "solana-account-decoder", + "solana-accounts-db", + "solana-clap-utils", + "solana-cli-config", + "solana-core", + "solana-download-utils", + "solana-entry", + "solana-faucet", + "solana-genesis-utils", + "solana-geyser-plugin-manager", + "solana-gossip", + "solana-ledger", + "solana-logger", + "solana-metrics", + "solana-net-utils", + "solana-perf", + "solana-poh", + "solana-rpc", + "solana-rpc-client", + "solana-rpc-client-api", + "solana-runtime", + "solana-sdk", + "solana-send-transaction-service", + "solana-storage-bigtable", + "solana-streamer", + "solana-svm", + "solana-test-validator", + "solana-tpu-client", + "solana-unified-scheduler-pool", + "solana-version", + "solana-vote-program", + "spl-token-2022", + "symlink", + "thiserror", + "tikv-jemallocator", +] + +[[package]] +name = "agave-watchtower" +version = "1.19.0" +dependencies = [ + "clap 2.33.3", + "humantime", + "log", + "solana-clap-utils", + "solana-cli-config", + "solana-cli-output", + "solana-logger", + "solana-metrics", + "solana-notifier", + "solana-rpc-client", + "solana-rpc-client-api", + "solana-sdk", + "solana-version", +] + [[package]] name = "ahash" version = "0.7.6" @@ -5482,35 +5693,6 @@ dependencies = [ "tar", ] -[[package]] -name = "solana-cargo-registry" -version = "1.19.0" -dependencies = [ - "clap 2.33.3", - "flate2", - "hex", - "hyper", - "log", - "rustc_version 0.4.0", - "serde", - "serde_json", - "sha2 0.10.8", - "solana-clap-utils", - "solana-cli", - "solana-cli-config", - "solana-cli-output", - "solana-logger", - "solana-remote-wallet", - "solana-rpc-client", - "solana-rpc-client-api", - "solana-sdk", - "solana-version", - "tar", - "tempfile", - "tokio", - "toml 0.8.10", -] - [[package]] name = "solana-cargo-test-bpf" version = "1.19.0" @@ -6040,20 +6222,11 @@ dependencies = [ "solana-sdk", ] -[[package]] -name = "solana-geyser-plugin-interface" -version = "1.19.0" -dependencies = [ - "log", - "solana-sdk", - "solana-transaction-status", - "thiserror", -] - [[package]] name = "solana-geyser-plugin-manager" version = "1.19.0" dependencies = [ + "agave-geyser-plugin-interface", "bs58", "crossbeam-channel", "json5", @@ -6064,7 +6237,6 @@ dependencies = [ "serde_json", "solana-accounts-db", "solana-entry", - "solana-geyser-plugin-interface", "solana-ledger", "solana-measure", "solana-metrics", @@ -6126,41 +6298,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "solana-install" -version = "1.19.0" -dependencies = [ - "atty", - "bincode", - "bzip2", - "chrono", - "clap 2.33.3", - "console", - "crossbeam-channel", - "ctrlc", - "dirs-next", - "indicatif", - "lazy_static", - "nix 0.26.4", - "reqwest", - "scopeguard", - "semver 1.0.22", - "serde", - "serde_yaml 0.8.26", - "serde_yaml 0.9.32", - "solana-clap-utils", - "solana-config-program", - "solana-logger", - "solana-rpc-client", - "solana-sdk", - "solana-version", - "tar", - "tempfile", - "url 2.5.0", - "winapi 0.3.9", - "winreg", -] - [[package]] name = "solana-keygen" version = "1.19.0" @@ -6248,58 +6385,6 @@ dependencies = [ "trees", ] -[[package]] -name = "solana-ledger-tool" -version = "1.19.0" -dependencies = [ - "assert_cmd", - "bs58", - "bytecount", - "chrono", - "clap 2.33.3", - "crossbeam-channel", - "csv", - "dashmap", - "futures 0.3.30", - "histogram", - "itertools", - "log", - "num_cpus", - "regex", - "serde", - "serde_json", - "signal-hook", - "solana-account-decoder", - "solana-accounts-db", - "solana-bpf-loader-program", - "solana-clap-utils", - "solana-cli-output", - "solana-core", - "solana-cost-model", - "solana-entry", - "solana-geyser-plugin-manager", - "solana-gossip", - "solana-ledger", - "solana-logger", - "solana-measure", - "solana-program-runtime", - "solana-rpc", - "solana-runtime", - "solana-sdk", - "solana-stake-program", - "solana-storage-bigtable", - "solana-streamer", - "solana-svm", - "solana-transaction-status", - "solana-unified-scheduler-pool", - "solana-version", - "solana-vote-program", - "solana_rbpf", - "thiserror", - "tikv-jemallocator", - "tokio", -] - [[package]] name = "solana-loader-v4-program" version = "1.19.0" @@ -7463,72 +7548,6 @@ dependencies = [ "solana-metrics", ] -[[package]] -name = "solana-validator" -version = "1.19.0" -dependencies = [ - "chrono", - "clap 2.33.3", - "console", - "core_affinity", - "crossbeam-channel", - "fd-lock", - "indicatif", - "itertools", - "jsonrpc-core", - "jsonrpc-core-client", - "jsonrpc-derive", - "jsonrpc-ipc-server", - "jsonrpc-server-utils", - "lazy_static", - "libc", - "libloading", - "log", - "num_cpus", - "rand 0.8.5", - "rayon", - "serde", - "serde_json", - "serde_yaml 0.9.32", - "signal-hook", - "solana-account-decoder", - "solana-accounts-db", - "solana-clap-utils", - "solana-cli-config", - "solana-core", - "solana-download-utils", - "solana-entry", - "solana-faucet", - "solana-genesis-utils", - "solana-geyser-plugin-interface", - "solana-geyser-plugin-manager", - "solana-gossip", - "solana-ledger", - "solana-logger", - "solana-metrics", - "solana-net-utils", - "solana-perf", - "solana-poh", - "solana-rpc", - "solana-rpc-client", - "solana-rpc-client-api", - "solana-runtime", - "solana-sdk", - "solana-send-transaction-service", - "solana-storage-bigtable", - "solana-streamer", - "solana-svm", - "solana-test-validator", - "solana-tpu-client", - "solana-unified-scheduler-pool", - "solana-version", - "solana-vote-program", - "spl-token-2022", - "symlink", - "thiserror", - "tikv-jemallocator", -] - [[package]] name = "solana-version" version = "1.19.0" @@ -7585,25 +7604,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "solana-watchtower" -version = "1.19.0" -dependencies = [ - "clap 2.33.3", - "humantime", - "log", - "solana-clap-utils", - "solana-cli-config", - "solana-cli-output", - "solana-logger", - "solana-metrics", - "solana-notifier", - "solana-rpc-client", - "solana-rpc-client-api", - "solana-sdk", - "solana-version", -] - [[package]] name = "solana-wen-restart" version = "1.19.0" diff --git a/Cargo.toml b/Cargo.toml index 66436c9cfb3fd8..27376370297e26 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -317,7 +317,7 @@ solana-bench-tps = { path = "bench-tps", version = "=1.19.0" } solana-bloom = { path = "bloom", version = "=1.19.0" } solana-bpf-loader-program = { path = "programs/bpf_loader", version = "=1.19.0" } solana-bucket-map = { path = "bucket_map", version = "=1.19.0" } -solana-cargo-registry = { path = "cargo-registry", version = "=1.19.0" } +agave-cargo-registry = { path = "cargo-registry", version = "=1.19.0" } solana-clap-utils = { path = "clap-utils", version = "=1.19.0" } solana-clap-v3-utils = { path = "clap-v3-utils", version = "=1.19.0" } solana-cli = { path = "cli", version = "=1.19.0" } @@ -336,7 +336,7 @@ solana-frozen-abi = { path = "frozen-abi", version = "=1.19.0" } solana-frozen-abi-macro = { path = "frozen-abi/macro", version = "=1.19.0" } solana-genesis = { path = "genesis", version = "=1.19.0" } solana-genesis-utils = { path = "genesis-utils", version = "=1.19.0" } -solana-geyser-plugin-interface = { path = "geyser-plugin-interface", version = "=1.19.0" } +agave-geyser-plugin-interface = { path = "geyser-plugin-interface", version = "=1.19.0" } solana-geyser-plugin-manager = { path = "geyser-plugin-manager", version = "=1.19.0" } solana-gossip = { path = "gossip", version = "=1.19.0" } solana-ledger = { path = "ledger", version = "=1.19.0" } diff --git a/cargo-registry/Cargo.toml b/cargo-registry/Cargo.toml index 4e13f477ee5e2d..395493a8e85f00 100644 --- a/cargo-registry/Cargo.toml +++ b/cargo-registry/Cargo.toml @@ -1,7 +1,7 @@ [package] -name = "solana-cargo-registry" +name = "agave-cargo-registry" description = "Solana cargo registry" -documentation = "https://docs.rs/solana-cargo-registry" +documentation = "https://docs.rs/agave-cargo-registry" version = { workspace = true } authors = { workspace = true } repository = { workspace = true } diff --git a/ci/localnet-sanity.sh b/ci/localnet-sanity.sh index e6734e180aa2da..b01eca31d50d81 100755 --- a/ci/localnet-sanity.sh +++ b/ci/localnet-sanity.sh @@ -202,8 +202,8 @@ killNodes() { # Try to use the RPC exit API to cleanly exit the first two nodes # (dynamic nodes, -x, are just killed) echo "--- RPC exit" - $solana_validator --ledger "$SOLANA_CONFIG_DIR"/bootstrap-validator exit --force || true - $solana_validator --ledger "$SOLANA_CONFIG_DIR"/validator exit --force || true + $agave_validator --ledger "$SOLANA_CONFIG_DIR"/bootstrap-validator exit --force || true + $agave_validator --ledger "$SOLANA_CONFIG_DIR"/validator exit --force || true # Give the nodes a splash of time to cleanly exit before killing them sleep 2 diff --git a/ci/publish-installer.sh b/ci/publish-installer.sh index e58fd939dd1a40..f7d98ffd5ddcf9 100755 --- a/ci/publish-installer.sh +++ b/ci/publish-installer.sh @@ -31,7 +31,7 @@ SOLANA_RELEASE=$CHANNEL_OR_TAG SOLANA_INSTALL_INIT_ARGS=$CHANNEL_OR_TAG SOLANA_DOWNLOAD_ROOT=https://release.anza.xyz EOF -cat install/solana-install-init.sh >>release.anza.xyz-install +cat install/agave-install-init.sh >>release.anza.xyz-install echo --- GCS: "install" upload-gcs-artifact "/solana/release.anza.xyz-install" "gs://anza-release/$CHANNEL_OR_TAG/install" diff --git a/ci/publish-tarball.sh b/ci/publish-tarball.sh index 5c64f09564fe9f..da5862fb3de1d2 100755 --- a/ci/publish-tarball.sh +++ b/ci/publish-tarball.sh @@ -93,7 +93,7 @@ echo --- Creating release tarball tar cvf "${TARBALL_BASENAME}"-$TARGET.tar "${RELEASE_BASENAME}" bzip2 "${TARBALL_BASENAME}"-$TARGET.tar - cp "${RELEASE_BASENAME}"/bin/solana-install-init solana-install-init-$TARGET + cp "${RELEASE_BASENAME}"/bin/agave-install-init agave-install-init-$TARGET cp "${RELEASE_BASENAME}"/version.yml "${TARBALL_BASENAME}"-$TARGET.yml ) @@ -110,7 +110,7 @@ fi source ci/upload-ci-artifact.sh -for file in "${TARBALL_BASENAME}"-$TARGET.tar.bz2 "${TARBALL_BASENAME}"-$TARGET.yml solana-install-init-"$TARGET"* $MAYBE_TARBALLS; do +for file in "${TARBALL_BASENAME}"-$TARGET.tar.bz2 "${TARBALL_BASENAME}"-$TARGET.yml agave-install-init-"$TARGET"* $MAYBE_TARBALLS; do if [[ -n $DO_NOT_PUBLISH_TAR ]]; then upload-ci-artifact "$file" echo "Skipped $file due to DO_NOT_PUBLISH_TAR" diff --git a/ci/run-sanity.sh b/ci/run-sanity.sh index 8108d13a061fd5..88a6f40b1adf28 100755 --- a/ci/run-sanity.sh +++ b/ci/run-sanity.sh @@ -31,7 +31,7 @@ while [[ $latest_slot -le $((snapshot_slot + 1)) ]]; do latest_slot=$($solana_cli --url http://localhost:8899 slot --commitment processed) done -$solana_validator --ledger config/ledger exit --force || true +$agave_validator --ledger config/ledger exit --force || true wait $pid diff --git a/docs/src/backwards-compatibility.md b/docs/src/backwards-compatibility.md index 4a3c60b8e129bd..0fdc388ea2dbae 100644 --- a/docs/src/backwards-compatibility.md +++ b/docs/src/backwards-compatibility.md @@ -76,7 +76,7 @@ Major releases: - [`solana-program`](https://docs.rs/solana-program/) - Rust SDK for writing programs - [`solana-client`](https://docs.rs/solana-client/) - Rust client for connecting to RPC API - [`solana-cli-config`](https://docs.rs/solana-cli-config/) - Rust client for managing Solana CLI config files -- [`solana-geyser-plugin-interface`](https://docs.rs/solana-geyser-plugin-interface/) - Rust interface for developing Solana Geyser plugins. +- [`agave-geyser-plugin-interface`](https://docs.rs/agave-geyser-plugin-interface/) - Rust interface for developing Solana Geyser plugins. Patch releases: diff --git a/docs/src/cli/install.md b/docs/src/cli/install.md index 3667c733e3f4d4..20f6516314fb02 100644 --- a/docs/src/cli/install.md +++ b/docs/src/cli/install.md @@ -56,7 +56,7 @@ Please update your PATH environment variable to include the solana programs: solana --version ``` -- After a successful install, `solana-install update` may be used to easily +- After a successful install, `agave-install update` may be used to easily update the Solana software to a newer version at any time. --- @@ -74,7 +74,7 @@ solana --version installer into a temporary directory: ```bash -cmd /c "curl https://release.solana.com/LATEST_SOLANA_RELEASE_VERSION/solana-install-init-x86_64-pc-windows-msvc.exe --output C:\solana-install-tmp\solana-install-init.exe --create-dirs" +cmd /c "curl https://release.solana.com/LATEST_SOLANA_RELEASE_VERSION/agave-install-init-x86_64-pc-windows-msvc.exe --output C:\agave-install-tmp\agave-install-init.exe --create-dirs" ``` - Copy and paste the following command, then press Enter to install the latest @@ -82,7 +82,7 @@ cmd /c "curl https://release.solana.com/LATEST_SOLANA_RELEASE_VERSION/solana-ins to allow the program to run. ```bash -C:\solana-install-tmp\solana-install-init.exe LATEST_SOLANA_RELEASE_VERSION +C:\agave-install-tmp\agave-install-init.exe LATEST_SOLANA_RELEASE_VERSION ``` - When the installer is finished, press Enter. @@ -97,12 +97,12 @@ C:\solana-install-tmp\solana-install-init.exe LATEST_SOLANA_RELEASE_VERSION solana --version ``` -- After a successful install, `solana-install update` may be used to easily +- After a successful install, `agave-install update` may be used to easily update the Solana software to a newer version at any time. ## Download Prebuilt Binaries -If you would rather not use `solana-install` to manage the install, you can +If you would rather not use `agave-install` to manage the install, you can manually download and install the binaries. ### Linux @@ -255,7 +255,7 @@ You can then run the following command to obtain the same result as with prebuilt binaries: ```bash -solana-install init +agave-install init ``` ## Use Homebrew diff --git a/docs/src/clusters/available.md b/docs/src/clusters/available.md index dfbca41672b499..52a7d469ad0cc5 100644 --- a/docs/src/clusters/available.md +++ b/docs/src/clusters/available.md @@ -41,10 +41,10 @@ export SOLANA_METRICS_CONFIG="host=https://metrics.solana.com:8086,db=devnet,u=s solana config set --url https://api.devnet.solana.com ``` -##### Example `solana-validator` command-line +##### Example `agave-validator` command-line ```bash -$ solana-validator \ +$ agave-validator \ --identity validator-keypair.json \ --vote-account vote-account-keypair.json \ --known-validator dv1ZAGvdsz5hHLwWXsVnM94hWf1pjbKVau1QVkaMJ92 \ @@ -93,10 +93,10 @@ export SOLANA_METRICS_CONFIG="host=https://metrics.solana.com:8086,db=tds,u=test solana config set --url https://api.testnet.solana.com ``` -##### Example `solana-validator` command-line +##### Example `agave-validator` command-line ```bash -$ solana-validator \ +$ agave-validator \ --identity validator-keypair.json \ --vote-account vote-account-keypair.json \ --known-validator 5D1fNXzvv5NjV1ysLjirC4WY92RNsVH18vjmcszZd8on \ @@ -145,10 +145,10 @@ export SOLANA_METRICS_CONFIG="host=https://metrics.solana.com:8086,db=mainnet-be solana config set --url https://api.mainnet-beta.solana.com ``` -##### Example `solana-validator` command-line +##### Example `agave-validator` command-line ```bash -$ solana-validator \ +$ agave-validator \ --identity ~/validator-keypair.json \ --vote-account ~/vote-account-keypair.json \ --known-validator 7Np41oeYqPefeNQEHSv1UDhYrehxin3NStELsSKCT4K2 \ diff --git a/docs/src/clusters/benchmark.md b/docs/src/clusters/benchmark.md index d913f9e5f16392..35978cdd0967dd 100644 --- a/docs/src/clusters/benchmark.md +++ b/docs/src/clusters/benchmark.md @@ -108,7 +108,7 @@ For example Generally we are using `debug` for infrequent debug messages, `trace` for potentially frequent messages and `info` for performance-related logging. -You can also attach to a running process with GDB. The leader's process is named _solana-validator_: +You can also attach to a running process with GDB. The leader's process is named _agave-validator_: ```bash sudo gdb diff --git a/docs/src/implemented-proposals/installer.md b/docs/src/implemented-proposals/installer.md index a3ad797171c5b8..c052aa7b4e54e5 100644 --- a/docs/src/implemented-proposals/installer.md +++ b/docs/src/implemented-proposals/installer.md @@ -13,16 +13,16 @@ This document proposes an easy to use software install and updater that can be u The easiest install method for supported platforms: ```bash -$ curl -sSf https://raw.githubusercontent.com/solana-labs/solana/v1.0.0/install/solana-install-init.sh | sh +$ curl -sSf https://raw.githubusercontent.com/solana-labs/solana/v1.0.0/install/agave-install-init.sh | sh ``` -This script will check github for the latest tagged release and download and run the `solana-install-init` binary from there. +This script will check github for the latest tagged release and download and run the `agave-install-init` binary from there. If additional arguments need to be specified during the installation, the following shell syntax is used: ```bash -$ init_args=.... # arguments for `solana-install-init ...` -$ curl -sSf https://raw.githubusercontent.com/solana-labs/solana/v1.0.0/install/solana-install-init.sh | sh -s - ${init_args} +$ init_args=.... # arguments for `agave-install-init ...` +$ curl -sSf https://raw.githubusercontent.com/solana-labs/solana/v1.0.0/install/agave-install-init.sh | sh -s - ${init_args} ``` ### Fetch and run a pre-built installer from a Github release @@ -30,9 +30,9 @@ $ curl -sSf https://raw.githubusercontent.com/solana-labs/solana/v1.0.0/install/ With a well-known release URL, a pre-built binary can be obtained for supported platforms: ```bash -$ curl -o solana-install-init https://github.com/solana-labs/solana/releases/download/v1.0.0/solana-install-init-x86_64-apple-darwin -$ chmod +x ./solana-install-init -$ ./solana-install-init --help +$ curl -o agave-install-init https://github.com/solana-labs/solana/releases/download/v1.0.0/agave-install-init-x86_64-apple-darwin +$ chmod +x ./agave-install-init +$ ./agave-install-init --help ``` ### Build and run the installer from source @@ -51,16 +51,16 @@ Given a solana release tarball \(as created by `ci/publish-tarball.sh`\) that ha ```bash $ solana-keygen new -o update-manifest.json # <-- only generated once, the public key is shared with users -$ solana-install deploy http://example.com/path/to/solana-release.tar.bz2 update-manifest.json +$ agave-install deploy http://example.com/path/to/solana-release.tar.bz2 update-manifest.json ``` ### Run a validator node that auto updates itself ```bash -$ solana-install init --pubkey 92DMonmBYXwEMHJ99c9ceRSpAmk9v6i3RdvDdXaVcrfj # <-- pubkey is obtained from whoever is deploying the updates -$ export PATH=~/.local/share/solana-install/bin:$PATH +$ agave-install init --pubkey 92DMonmBYXwEMHJ99c9ceRSpAmk9v6i3RdvDdXaVcrfj # <-- pubkey is obtained from whoever is deploying the updates +$ export PATH=~/.local/share/agave-install/bin:$PATH $ solana-keygen ... # <-- runs the latest solana-keygen -$ solana-install run solana-validator ... # <-- runs a validator, restarting it as necessary when an update is applied +$ agave-install run agave-validator ... # <-- runs a validator, restarting it as necessary when an update is applied ``` ## On-chain Update Manifest @@ -87,9 +87,9 @@ pub struct SignedUpdateManifest { } ``` -Note that the `manifest` field itself contains a corresponding signature \(`manifest_signature`\) to guard against man-in-the-middle attacks between the `solana-install` tool and the solana cluster RPC API. +Note that the `manifest` field itself contains a corresponding signature \(`manifest_signature`\) to guard against man-in-the-middle attacks between the `agave-install` tool and the solana cluster RPC API. -To guard against rollback attacks, `solana-install` will refuse to install an update with an older `timestamp_secs` than what is currently installed. +To guard against rollback attacks, `agave-install` will refuse to install an update with an older `timestamp_secs` than what is currently installed. ## Release Archive Contents @@ -101,17 +101,17 @@ A release archive is expected to be a tar file compressed with bzip2 with the fo - `/bin/` -- directory containing available programs in the release. - `solana-install` will symlink this directory to + `agave-install` will symlink this directory to - `~/.local/share/solana-install/bin` for use by the `PATH` environment + `~/.local/share/agave-install/bin` for use by the `PATH` environment variable. - `...` -- any additional files and directories are permitted -## solana-install Tool +## agave-install Tool -The `solana-install` tool is used by the user to install and update their cluster software. +The `agave-install` tool is used by the user to install and update their cluster software. It manages the following files and directories in the user's home directory: @@ -122,11 +122,11 @@ It manages the following files and directories in the user's home directory: ### Command-line Interface ```text -solana-install 0.16.0 +agave-install 0.16.0 The solana cluster software installer USAGE: - solana-install [OPTIONS] + agave-install [OPTIONS] FLAGS: -h, --help Prints help information @@ -145,11 +145,11 @@ SUBCOMMANDS: ``` ```text -solana-install-init +agave-install-init initializes a new installation USAGE: - solana-install init [OPTIONS] + agave-install init [OPTIONS] FLAGS: -h, --help Prints help information @@ -161,11 +161,11 @@ OPTIONS: ``` ```text -solana-install info +agave-install info displays information about the current installation USAGE: - solana-install info [FLAGS] + agave-install info [FLAGS] FLAGS: -h, --help Prints help information @@ -173,11 +173,11 @@ FLAGS: ``` ```text -solana-install deploy +agave-install deploy deploys a new update USAGE: - solana-install deploy + agave-install deploy FLAGS: -h, --help Prints help information @@ -188,22 +188,22 @@ ARGS: ``` ```text -solana-install update +agave-install update checks for an update, and if available downloads and applies it USAGE: - solana-install update + agave-install update FLAGS: -h, --help Prints help information ``` ```text -solana-install run +agave-install run Runs a program while periodically checking and applying software updates USAGE: - solana-install run [program_arguments]... + agave-install run [program_arguments]... FLAGS: -h, --help Prints help information diff --git a/docs/src/implemented-proposals/rpc-transaction-history.md b/docs/src/implemented-proposals/rpc-transaction-history.md index 54288ad9659bd7..607a79ce658b98 100644 --- a/docs/src/implemented-proposals/rpc-transaction-history.md +++ b/docs/src/implemented-proposals/rpc-transaction-history.md @@ -68,7 +68,7 @@ the results of BigTable queries more complicated but is not a significant issue. ## Data Population The ongoing population of instance data will occur on an epoch cadence through -the use of a new `solana-ledger-tool` command that will convert rocksdb data for +the use of a new `agave-ledger-tool` command that will convert rocksdb data for a given slot range into the instance schema. The same process will be run once, manually, to backfill the existing ledger diff --git a/docs/src/operations/best-practices/general.md b/docs/src/operations/best-practices/general.md index 29ef42c81b7f5f..3e531b0160c571 100644 --- a/docs/src/operations/best-practices/general.md +++ b/docs/src/operations/best-practices/general.md @@ -23,12 +23,12 @@ watch past workshops through the ## Help with the validator command line -From within the Solana CLI, you can execute the `solana-validator` command with +From within the Solana CLI, you can execute the `agave-validator` command with the `--help` flag to get a better understanding of the flags and sub commands available. ``` -solana-validator --help +agave-validator --help ``` ## Restarting your validator @@ -49,14 +49,14 @@ solana leader-schedule Based on the current slot and the leader schedule, you can calculate open time windows where your validator is not expected to produce blocks. -Assuming you are ready to restart, you may use the `solana-validator exit` +Assuming you are ready to restart, you may use the `agave-validator exit` command. The command exits your validator process when an appropriate idle time window is reached. Assuming that you have systemd implemented for your validator process, the validator should restart automatically after the exit. See the below help command for details: ``` -solana-validator exit --help +agave-validator exit --help ``` ## Upgrading @@ -75,28 +75,28 @@ process. It is a best practice to always build your Solana binaries from source. If you build from source, you are certain that the code you are building has not been tampered with before the binary was created. You may also be able to optimize -your `solana-validator` binary to your specific hardware. +your `agave-validator` binary to your specific hardware. If you build from source on the validator machine (or a machine with the same CPU), you can target your specific architecture using the `-march` flag. Refer to the following doc for [instructions on building from source](../../cli/install.md#build-from-source). -### solana-install +### agave-install If you are not comfortable building from source, or you need to quickly install a new version to test something out, you could instead try using the -`solana-install` command. +`agave-install` command. Assuming you want to install Solana version `1.14.17`, you would execute the following: ``` -solana-install init 1.14.17 +agave-install init 1.14.17 ``` This command downloads the executable for `1.14.17` and installs it into a -`.local` directory. You can also look at `solana-install --help` for more +`.local` directory. You can also look at `agave-install --help` for more options. > **Note** this command only works if you already have the solana cli installed. @@ -106,7 +106,7 @@ options. ### Restart For all install methods, the validator process will need to be restarted before -the newly installed version is in use. Use `solana-validator exit` to restart +the newly installed version is in use. Use `agave-validator exit` to restart your validator process. ### Verifying version @@ -132,13 +132,13 @@ have state locally. In other cases such as restarts for upgrades, a snapshot download should be avoided. To avoid downloading a snapshot on restart, add the following flag to the -`solana-validator` command: +`agave-validator` command: ``` --no-snapshot-fetch ``` -If you use this flag with the `solana-validator` command, make sure that you run +If you use this flag with the `agave-validator` command, make sure that you run `solana catchup ` after your validator starts to make sure that the validator is catching up in a reasonable time. After some time (potentially a few hours), if it appears that your validator continues to fall behind, then you @@ -199,7 +199,7 @@ It is important that you do not accidentally run out of funds in your identity account, as your node will stop voting. It is also important to note that this account keypair is the most vulnerable of the three keypairs in a vote account because the keypair for the identity account is stored on your validator when -running the `solana-validator` software. How much SOL you should store there is +running the `agave-validator` software. How much SOL you should store there is up to you. As a best practice, make sure to check the account regularly and refill or deduct from it as needed. To check the account balance do: @@ -207,7 +207,7 @@ refill or deduct from it as needed. To check the account balance do: solana balance validator-keypair.json ``` -> **Note** `solana-watchtower` can monitor for a minimum validator identity +> **Note** `agave-watchtower` can monitor for a minimum validator identity > balance. See [monitoring best practices](./monitoring.md) for details. ## Withdrawing From The Vote Account diff --git a/docs/src/operations/best-practices/monitoring.md b/docs/src/operations/best-practices/monitoring.md index 6d04fc38487be7..a0f2ef9df9fa22 100644 --- a/docs/src/operations/best-practices/monitoring.md +++ b/docs/src/operations/best-practices/monitoring.md @@ -4,34 +4,34 @@ sidebar_label: Monitoring pagination_label: "Best Practices: Validator Monitoring" --- -It is essential that you have monitoring in place on your validator. In the event that your validator is delinquent (behind the rest of the network) you want to respond immediately to fix the issue. One very useful tool to monitor your validator is [`solana-watchtower`](#solana-watchtower). +It is essential that you have monitoring in place on your validator. In the event that your validator is delinquent (behind the rest of the network) you want to respond immediately to fix the issue. One very useful tool to monitor your validator is [`agave-watchtower`](#agave-watchtower). ## Solana Watchtower -Solana Watchtower is an extremely useful monitoring tool that will regularly monitor the health of your validator. It can monitor your validator for delinquency then notify you on your application of choice: Slack, Discord, Telegram or Twilio. Additionally, `solana-watchtower` has the ability to monitor the health of the entire cluster so that you can be aware of any cluster wide problems. +Solana Watchtower is an extremely useful monitoring tool that will regularly monitor the health of your validator. It can monitor your validator for delinquency then notify you on your application of choice: Slack, Discord, Telegram or Twilio. Additionally, `agave-watchtower` has the ability to monitor the health of the entire cluster so that you can be aware of any cluster wide problems. ### Getting Started -To get started with Solana Watchtower, run `solana-watchtower --help`. From the help menu, you can see the optional flags and an explanation of the command. +To get started with Solana Watchtower, run `agave-watchtower --help`. From the help menu, you can see the optional flags and an explanation of the command. Here is a sample command that will monitor a validator node with an identity public key of `2uTk98rqqwENevkPH2AHHzGHXgeGc1h6ku8hQUqWeXZp`: ``` -solana-watchtower --monitor-active-stake --validator-identity \ +agave-watchtower --monitor-active-stake --validator-identity \ 2uTk98rqqwENevkPH2AHHzGHXgeGc1h6ku8hQUqWeXZp ``` -The command will monitor your validator, but you will not get notifications unless you added the environment variables mentioned in `solana-watchtower --help`. Since getting each of these services setup for notifications is not straight forward, the next section will walk through [setting up watchtower notifications on Telegram](#setup-telegram-notifications). +The command will monitor your validator, but you will not get notifications unless you added the environment variables mentioned in `agave-watchtower --help`. Since getting each of these services setup for notifications is not straight forward, the next section will walk through [setting up watchtower notifications on Telegram](#setup-telegram-notifications). ### Best Practices -It is a best practice to run the `solana-watchtower` command on a separate server from your validator. +It is a best practice to run the `agave-watchtower` command on a separate server from your validator. -In the case that you run `solana-watchtower` on the same computer as your `solana-validator` process, then during catastrophic events like a power outage, you will not be aware of the issue, because your `solana-watchtower` process will stop at the same time as your `solana-validator` process. +In the case that you run `agave-watchtower` on the same computer as your `agave-validator` process, then during catastrophic events like a power outage, you will not be aware of the issue, because your `agave-watchtower` process will stop at the same time as your `agave-validator` process. -Additionally, while running the `solana-watchtower` process manually with environment variables set in the terminal is a good way to test out the command, it is not operationally sound because the process will not be restarted when the terminal closes or during a system restart. +Additionally, while running the `agave-watchtower` process manually with environment variables set in the terminal is a good way to test out the command, it is not operationally sound because the process will not be restarted when the terminal closes or during a system restart. -Instead, you could run your `solana-watchtower` command as a system process similar to `solana-validator`. In the system process file, you can specify the environment variables for your bot. +Instead, you could run your `agave-watchtower` command as a system process similar to `agave-validator`. In the system process file, you can specify the environment variables for your bot. ### Setup Telegram Notifications @@ -41,7 +41,7 @@ To send validator health notifications to your Telegram account, we are going to 2. Send a message to the bot 3. Create a Telegram group that will get the watchtower notifications 4. Add the environment variables to your command line environment -5. Restart the `solana-watchtower` command +5. Restart the `agave-watchtower` command #### Create a Bot Using BotFather @@ -61,7 +61,7 @@ In Telegram, click on the new message icon and then select new group. Find your Now that you have a bot setup, you will need to set the environment variables for the bot so that watchtower can send notifications. -First, recall the chat message that you got from _@BotFather_. In the message, there was an HTTP API token for your bot. The token will have this format: `389178471:MMTKMrnZB4ErUzJmuFIXTKE6DupLSgoa7h4o`. You will use that token to set the `TELEGRAM_BOT_TOKEN` environment variable. In the terminal where you plan to run `solana-watchtower`, run the following: +First, recall the chat message that you got from _@BotFather_. In the message, there was an HTTP API token for your bot. The token will have this format: `389178471:MMTKMrnZB4ErUzJmuFIXTKE6DupLSgoa7h4o`. You will use that token to set the `TELEGRAM_BOT_TOKEN` environment variable. In the terminal where you plan to run `agave-watchtower`, run the following: ``` export TELEGRAM_BOT_TOKEN= @@ -73,14 +73,14 @@ Next, in your browser, go to `https://api.telegram.org/bot/getUp The response should be in JSON. Search for the string `"chat":` in the JSON. The `id` value of that chat is your `TELEGRAM_CHAT_ID`. It will be a negative number like: `-781559558`. Remember to include the negative sign! If you cannot find `"chat":` in the JSON, then you may have to remove the bot from your chat group and add it again. -With your Telegram chat id in hand, export the environment variable where you plan to run `solana-watchtower`: +With your Telegram chat id in hand, export the environment variable where you plan to run `agave-watchtower`: ``` export TELEGRAM_CHAT_ID= ``` -#### Restart solana-watchtower +#### Restart agave-watchtower -Once your environment variables are set, restart `solana-watchtower`. You should see output about your validator. +Once your environment variables are set, restart `agave-watchtower`. You should see output about your validator. To test that your Telegram configuration is working properly, you could stop your validator briefly until it is labeled as delinquent. Up to a minute after the validator is delinquent, you should receive a message in the Telegram group from your bot. Start the validator again and verify that you get another message in your Telegram group from the bot. The message should say `all clear`. \ No newline at end of file diff --git a/docs/src/operations/guides/restart-cluster.md b/docs/src/operations/guides/restart-cluster.md index 85d4731d604c65..cda3f30a5a016d 100644 --- a/docs/src/operations/guides/restart-cluster.md +++ b/docs/src/operations/guides/restart-cluster.md @@ -11,7 +11,7 @@ pagination_label: "Validator Guides: Restart a Cluster" In Solana 1.14 or greater, run the following command to output the latest optimistically confirmed slot your validator observed: ```bash -solana-ledger-tool -l ledger latest-optimistic-slots +agave-ledger-tool -l ledger latest-optimistic-slots ``` In Solana 1.13 or less, the latest optimistically confirmed can be found by looking for the more recent occurrence of @@ -34,11 +34,11 @@ instead. ### Step 4. Create a new snapshot for slot `SLOT_X` with a hard fork at slot `SLOT_X` ```bash -$ solana-ledger-tool -l --snapshot-archive-path --incremental-snapshot-archive-path create-snapshot SLOT_X --hard-fork SLOT_X +$ agave-ledger-tool -l --snapshot-archive-path --incremental-snapshot-archive-path create-snapshot SLOT_X --hard-fork SLOT_X ``` The snapshots directory should now contain the new snapshot. -`solana-ledger-tool create-snapshot` will also output the new shred version, and bank hash value, +`agave-ledger-tool create-snapshot` will also output the new shred version, and bank hash value, call this NEW_SHRED_VERSION and NEW_BANK_HASH respectively. Adjust your validator's arguments: @@ -68,7 +68,7 @@ Post something like the following to #announcements (adjusting the text as appro > 2. a. Preferred method, start from your local ledger with: > > ```bash -> solana-validator +> agave-validator > --wait-for-supermajority SLOT_X # <-- NEW! IMPORTANT! REMOVE AFTER THIS RESTART > --expected-bank-hash NEW_BANK_HASH # <-- NEW! IMPORTANT! REMOVE AFTER THIS RESTART > --hard-fork SLOT_X # <-- NEW! IMPORTANT! REMOVE AFTER THIS RESTART @@ -84,7 +84,7 @@ Post something like the following to #announcements (adjusting the text as appro > b. If your validator doesn't have ledger up to slot SLOT_X or if you have deleted your ledger, have it instead download a snapshot with: > > ```bash -> solana-validator +> agave-validator > --wait-for-supermajority SLOT_X # <-- NEW! IMPORTANT! REMOVE AFTER THIS RESTART > --expected-bank-hash NEW_BANK_HASH # <-- NEW! IMPORTANT! REMOVE AFTER THIS RESTART > --entrypoint entrypoint.testnet.solana.com:8001 @@ -95,7 +95,7 @@ Post something like the following to #announcements (adjusting the text as appro > ... # <-- your other --identity/--vote-account/etc arguments > ``` > -> You can check for which slots your ledger has with: `solana-ledger-tool -l path/to/ledger bounds` +> You can check for which slots your ledger has with: `agave-ledger-tool -l path/to/ledger bounds` > > 3. Wait until 80% of the stake comes online > @@ -122,7 +122,7 @@ and create a new snapshot with additional `--destake-vote-account ` arguments for each of the non-responsive validator's vote account address ```bash -$ solana-ledger-tool -l ledger create-snapshot SLOT_X ledger --hard-fork SLOT_X \ +$ agave-ledger-tool -l ledger create-snapshot SLOT_X ledger --hard-fork SLOT_X \ --destake-vote-account \ --destake-vote-account \ . diff --git a/docs/src/operations/guides/validator-failover.md b/docs/src/operations/guides/validator-failover.md index 168a1a4312cec0..b7b3fea568194b 100644 --- a/docs/src/operations/guides/validator-failover.md +++ b/docs/src/operations/guides/validator-failover.md @@ -85,11 +85,11 @@ For more information on etcd TLS setup, please refer to https://etcd.io/docs/v3.5/op-guide/security/#example-2-client-to-server-authentication-with-https-client-certificates ### Primary Validator -The following additional `solana-validator` parameters are required to enable +The following additional `agave-validator` parameters are required to enable tower storage into etcd: ``` -solana-validator ... \ +agave-validator ... \ --tower-storage etcd \ --etcd-cacert-file certs/etcd-ca.pem \ --etcd-cert-file certs/validator.pem \ @@ -103,7 +103,7 @@ that your etcd endpoint remain accessible at all times. ### Secondary Validator Configure the secondary validator like the primary with the exception of the -following `solana-validator` command-line argument changes: +following `agave-validator` command-line argument changes: * Generate and use a secondary validator identity: `--identity secondary-validator-keypair.json` * Add `--no-check-vote-account` * Add `--authorized-voter validator-keypair.json` (where @@ -114,8 +114,8 @@ When both validators are running normally and caught up to the cluster, a failover from primary to secondary can be triggered by running the following command on the secondary validator: ```bash -$ solana-validator wait-for-restart-window --identity validator-keypair.json \ - && solana-validator set-identity validator-keypair.json +$ agave-validator wait-for-restart-window --identity validator-keypair.json \ + && agave-validator set-identity validator-keypair.json ``` The secondary validator will acquire a lock on the tower in etcd to ensure @@ -131,7 +131,7 @@ exit. However if/when the secondary validator restarts, it will do so using the secondary validator identity and thus the restart cycle is broken. ## Triggering a failover via monitoring -Monitoring of your choosing can invoke the `solana-validator set-identity +Monitoring of your choosing can invoke the `agave-validator set-identity validator-keypair.json` command mentioned in the previous section. It is not necessary to guarantee the primary validator has halted before failing diff --git a/docs/src/operations/guides/validator-start.md b/docs/src/operations/guides/validator-start.md index 378783798b3ce8..d86c714be4e6a6 100644 --- a/docs/src/operations/guides/validator-start.md +++ b/docs/src/operations/guides/validator-start.md @@ -32,7 +32,7 @@ detail on cluster activity. ## Enabling CUDA If your machine has a GPU with CUDA installed \(Linux-only currently\), include -the `--cuda` argument to `solana-validator`. +the `--cuda` argument to `agave-validator`. When your validator is started look for the following log message to indicate that CUDA is enabled: `"[ solana::validator] CUDA is enabled"` @@ -47,7 +47,7 @@ the following commands. #### **Optimize sysctl knobs** ```bash -sudo bash -c "cat >/etc/sysctl.d/21-solana-validator.conf </etc/sysctl.d/21-agave-validator.conf <` -argument to `solana-validator`. You can specify multiple ones by repeating the argument `--known-validator --known-validator `. +argument to `agave-validator`. You can specify multiple ones by repeating the argument `--known-validator --known-validator `. This has two effects, one is when the validator is booting with `--only-known-rpc`, it will only ask that set of known nodes for downloading genesis and snapshot data. Another is that in combination with the `--halt-on-known-validators-accounts-hash-mismatch` option, it will monitor the merkle root hash of the entire accounts state of other known nodes on gossip and if the hashes produce any mismatch, @@ -277,13 +277,13 @@ account state divergence. Connect to the cluster by running: ```bash -solana-validator \ +agave-validator \ --identity ~/validator-keypair.json \ --vote-account ~/vote-account-keypair.json \ --rpc-port 8899 \ --entrypoint entrypoint.devnet.solana.com:8001 \ --limit-ledger-size \ - --log ~/solana-validator.log + --log ~/agave-validator.log ``` To force validator logging to the console add a `--log -` argument, otherwise @@ -296,7 +296,7 @@ The ledger will be placed in the `ledger/` directory by default, use the > [paper wallet seed phrase](../../cli/wallets/paper.md) > for your `--identity` and/or > `--authorized-voter` keypairs. To use these, pass the respective argument as -> `solana-validator --identity ASK ... --authorized-voter ASK ...` +> `agave-validator --identity ASK ... --authorized-voter ASK ...` > and you will be prompted to enter your seed phrases and optional passphrase. Confirm your validator is connected to the network by opening a new terminal and @@ -312,7 +312,7 @@ If your validator is connected, its public key and IP address will appear in the By default the validator will dynamically select available network ports in the 8000-10000 range, and may be overridden with `--dynamic-port-range`. For -example, `solana-validator --dynamic-port-range 11000-11020 ...` will restrict +example, `agave-validator --dynamic-port-range 11000-11020 ...` will restrict the validator to ports 11000-11020. ### Limiting ledger size to conserve disk space @@ -366,8 +366,8 @@ WantedBy=multi-user.target ``` Now create `/home/sol/bin/validator.sh` to include the desired -`solana-validator` command-line. Ensure that the 'exec' command is used to -start the validator process (i.e. "exec solana-validator ..."). This is +`agave-validator` command-line. Ensure that the 'exec' command is used to +start the validator process (i.e. "exec agave-validator ..."). This is important because without it, logrotate will end up killing the validator every time the logs are rotated. @@ -394,14 +394,14 @@ to be reverted and the issue reproduced before help can be provided. #### Log rotation -The validator log file, as specified by `--log ~/solana-validator.log`, can get +The validator log file, as specified by `--log ~/agave-validator.log`, can get very large over time and it's recommended that log rotation be configured. The validator will re-open its log file when it receives the `USR1` signal, which is the basic primitive that enables log rotation. If the validator is being started by a wrapper shell script, it is important to -launch the process with `exec` (`exec solana-validator ...`) when using logrotate. +launch the process with `exec` (`exec agave-validator ...`) when using logrotate. This will prevent the `USR1` signal from being sent to the script's process instead of the validator's, which will kill them both. @@ -409,13 +409,13 @@ instead of the validator's, which will kill them both. An example setup for the `logrotate`, which assumes that the validator is running as a systemd service called `sol.service` and writes a log file at -/home/sol/solana-validator.log: +/home/sol/agave-validator.log: ```bash # Setup log rotation cat > logrotate.sol </etc/sysctl.d/21-solana-validator.conf </etc/sysctl.d/21-agave-validator.conf < For more explanation on the flags used in the command, refer to the `solana-validator --help` command +> For more explanation on the flags used in the command, refer to the `agave-validator --help` command ``` #!/bin/bash -exec solana-validator \ +exec agave-validator \ --identity /home/sol/validator-keypair.json \ --known-validator 5D1fNXzvv5NjV1ysLjirC4WY92RNsVH18vjmcszZd8on \ --known-validator dDzy5SR3AXdYWVqbDEkVFdvSPCtS9ihF5kJkHCtXoFs \ diff --git a/docs/src/validator/geyser.md b/docs/src/validator/geyser.md index 769856303767d6..efea2e18e30269 100644 --- a/docs/src/validator/geyser.md +++ b/docs/src/validator/geyser.md @@ -24,20 +24,20 @@ implementation for the PostgreSQL database. ### Important Crates: -- [`solana-geyser-plugin-interface`] — This crate defines the plugin +- [`agave-geyser-plugin-interface`] — This crate defines the plugin interfaces. - [`solana-accountsdb-plugin-postgres`] — The crate for the referential plugin implementation for the PostgreSQL database. -[`solana-geyser-plugin-interface`]: https://docs.rs/solana-geyser-plugin-interface +[`agave-geyser-plugin-interface`]: https://docs.rs/agave-geyser-plugin-interface [`solana-accountsdb-plugin-postgres`]: https://docs.rs/solana-accountsdb-plugin-postgres [`solana-sdk`]: https://docs.rs/solana-sdk [`solana-transaction-status`]: https://docs.rs/solana-transaction-status ## The Plugin Interface -The Plugin interface is declared in [`solana-geyser-plugin-interface`]. It +The Plugin interface is declared in [`agave-geyser-plugin-interface`]. It is defined by the trait `GeyserPlugin`. The plugin should implement the trait and expose a "C" function `_create_plugin` to return the pointer to this trait. For example, in the referential implementation, the following code @@ -166,7 +166,7 @@ please refer to [`solana-sdk`] and [`solana-transaction-status`] The `slot` points to the slot the transaction is executed at. For more details, please refer to the Rust documentation in -[`solana-geyser-plugin-interface`]. +[`agave-geyser-plugin-interface`]. ## Example PostgreSQL Plugin diff --git a/geyser-plugin-interface/Cargo.toml b/geyser-plugin-interface/Cargo.toml index af99758b47d630..56f42fd4612cec 100644 --- a/geyser-plugin-interface/Cargo.toml +++ b/geyser-plugin-interface/Cargo.toml @@ -1,7 +1,7 @@ [package] -name = "solana-geyser-plugin-interface" +name = "agave-geyser-plugin-interface" description = "The Solana Geyser plugin interface." -documentation = "https://docs.rs/solana-geyser-plugin-interface" +documentation = "https://docs.rs/agave-geyser-plugin-interface" version = { workspace = true } authors = { workspace = true } repository = { workspace = true } diff --git a/geyser-plugin-interface/src/geyser_plugin_interface.rs b/geyser-plugin-interface/src/geyser_plugin_interface.rs index 037aedf8b87e89..d9a3b00f8dc4c8 100644 --- a/geyser-plugin-interface/src/geyser_plugin_interface.rs +++ b/geyser-plugin-interface/src/geyser_plugin_interface.rs @@ -327,7 +327,7 @@ pub trait GeyserPlugin: Any + Send + Sync + std::fmt::Debug { /// # Examples /// /// ``` - /// use solana_geyser_plugin_interface::geyser_plugin_interface::{GeyserPlugin, + /// use agave_geyser_plugin_interface::geyser_plugin_interface::{GeyserPlugin, /// GeyserPluginError, Result}; /// /// #[derive(Debug)] diff --git a/geyser-plugin-manager/Cargo.toml b/geyser-plugin-manager/Cargo.toml index d905248150b717..ebef2f637f642d 100644 --- a/geyser-plugin-manager/Cargo.toml +++ b/geyser-plugin-manager/Cargo.toml @@ -10,6 +10,7 @@ license = { workspace = true } edition = { workspace = true } [dependencies] +agave-geyser-plugin-interface = { workspace = true } bs58 = { workspace = true } crossbeam-channel = { workspace = true } json5 = { workspace = true } @@ -20,7 +21,6 @@ log = { workspace = true } serde_json = { workspace = true } solana-accounts-db = { workspace = true } solana-entry = { workspace = true } -solana-geyser-plugin-interface = { workspace = true } solana-ledger = { workspace = true } solana-measure = { workspace = true } solana-metrics = { workspace = true } diff --git a/geyser-plugin-manager/src/accounts_update_notifier.rs b/geyser-plugin-manager/src/accounts_update_notifier.rs index 7c7e3370fc00eb..90ab0b7998a35c 100644 --- a/geyser-plugin-manager/src/accounts_update_notifier.rs +++ b/geyser-plugin-manager/src/accounts_update_notifier.rs @@ -1,14 +1,14 @@ /// Module responsible for notifying plugins of account updates use { crate::geyser_plugin_manager::GeyserPluginManager, + agave_geyser_plugin_interface::geyser_plugin_interface::{ + ReplicaAccountInfoV3, ReplicaAccountInfoVersions, + }, log::*, solana_accounts_db::{ account_storage::meta::StoredAccountMeta, accounts_update_notifier_interface::AccountsUpdateNotifierInterface, }, - solana_geyser_plugin_interface::geyser_plugin_interface::{ - ReplicaAccountInfoV3, ReplicaAccountInfoVersions, - }, solana_measure::measure::Measure, solana_metrics::*, solana_sdk::{ diff --git a/geyser-plugin-manager/src/block_metadata_notifier.rs b/geyser-plugin-manager/src/block_metadata_notifier.rs index 76d203c5e0ed44..87f15f41fc0ae0 100644 --- a/geyser-plugin-manager/src/block_metadata_notifier.rs +++ b/geyser-plugin-manager/src/block_metadata_notifier.rs @@ -3,10 +3,10 @@ use { block_metadata_notifier_interface::BlockMetadataNotifier, geyser_plugin_manager::GeyserPluginManager, }, - log::*, - solana_geyser_plugin_interface::geyser_plugin_interface::{ + agave_geyser_plugin_interface::geyser_plugin_interface::{ ReplicaBlockInfoV3, ReplicaBlockInfoVersions, }, + log::*, solana_measure::measure::Measure, solana_metrics::*, solana_sdk::{clock::UnixTimestamp, pubkey::Pubkey, reward_info::RewardInfo}, diff --git a/geyser-plugin-manager/src/entry_notifier.rs b/geyser-plugin-manager/src/entry_notifier.rs index ea14592b615db8..da9a9698ed1540 100644 --- a/geyser-plugin-manager/src/entry_notifier.rs +++ b/geyser-plugin-manager/src/entry_notifier.rs @@ -1,11 +1,11 @@ /// Module responsible for notifying plugins about entries use { crate::geyser_plugin_manager::GeyserPluginManager, - log::*, - solana_entry::entry::EntrySummary, - solana_geyser_plugin_interface::geyser_plugin_interface::{ + agave_geyser_plugin_interface::geyser_plugin_interface::{ ReplicaEntryInfoV2, ReplicaEntryInfoVersions, }, + log::*, + solana_entry::entry::EntrySummary, solana_ledger::entry_notifier_interface::EntryNotifier, solana_measure::measure::Measure, solana_metrics::*, diff --git a/geyser-plugin-manager/src/geyser_plugin_manager.rs b/geyser-plugin-manager/src/geyser_plugin_manager.rs index a15f9e1318075d..3d0abe16899637 100644 --- a/geyser-plugin-manager/src/geyser_plugin_manager.rs +++ b/geyser-plugin-manager/src/geyser_plugin_manager.rs @@ -1,9 +1,9 @@ use { + agave_geyser_plugin_interface::geyser_plugin_interface::GeyserPlugin, jsonrpc_core::{ErrorCode, Result as JsonRpcResult}, jsonrpc_server_utils::tokio::sync::oneshot::Sender as OneShotSender, libloading::Library, log::*, - solana_geyser_plugin_interface::geyser_plugin_interface::GeyserPlugin, std::{ ops::{Deref, DerefMut}, path::Path, @@ -442,8 +442,8 @@ mod tests { crate::geyser_plugin_manager::{ GeyserPluginManager, LoadedGeyserPlugin, TESTPLUGIN2_CONFIG, TESTPLUGIN_CONFIG, }, + agave_geyser_plugin_interface::geyser_plugin_interface::GeyserPlugin, libloading::Library, - solana_geyser_plugin_interface::geyser_plugin_interface::GeyserPlugin, std::sync::{Arc, RwLock}, }; diff --git a/geyser-plugin-manager/src/slot_status_notifier.rs b/geyser-plugin-manager/src/slot_status_notifier.rs index 587abe2f79d4de..1557bb2d4d8c36 100644 --- a/geyser-plugin-manager/src/slot_status_notifier.rs +++ b/geyser-plugin-manager/src/slot_status_notifier.rs @@ -1,7 +1,7 @@ use { crate::geyser_plugin_manager::GeyserPluginManager, + agave_geyser_plugin_interface::geyser_plugin_interface::SlotStatus, log::*, - solana_geyser_plugin_interface::geyser_plugin_interface::SlotStatus, solana_measure::measure::Measure, solana_metrics::*, solana_sdk::clock::Slot, diff --git a/geyser-plugin-manager/src/transaction_notifier.rs b/geyser-plugin-manager/src/transaction_notifier.rs index ab821e811047d2..b757c1202b377d 100644 --- a/geyser-plugin-manager/src/transaction_notifier.rs +++ b/geyser-plugin-manager/src/transaction_notifier.rs @@ -1,10 +1,10 @@ /// Module responsible for notifying plugins of transactions use { crate::geyser_plugin_manager::GeyserPluginManager, - log::*, - solana_geyser_plugin_interface::geyser_plugin_interface::{ + agave_geyser_plugin_interface::geyser_plugin_interface::{ ReplicaTransactionInfoV2, ReplicaTransactionInfoVersions, }, + log::*, solana_measure::measure::Measure, solana_metrics::*, solana_rpc::transaction_notifier_interface::TransactionNotifier, diff --git a/install/Cargo.toml b/install/Cargo.toml index 588d4315df5f35..c40a0ee6e9eee3 100644 --- a/install/Cargo.toml +++ b/install/Cargo.toml @@ -1,7 +1,7 @@ [package] -name = "solana-install" +name = "agave-install" description = "The solana cluster software installer" -documentation = "https://docs.rs/solana-install" +documentation = "https://docs.rs/agave-install" version = { workspace = true } authors = { workspace = true } repository = { workspace = true } diff --git a/install/solana-install-init.sh b/install/agave-install-init.sh similarity index 89% rename from install/solana-install-init.sh rename to install/agave-install-init.sh index 4f28e300be52ab..cf2d1babf3c306 100755 --- a/install/solana-install-init.sh +++ b/install/agave-install-init.sh @@ -10,7 +10,7 @@ # except according to those terms. # This is just a little script that can be downloaded from the internet to -# install solana-install. It just does platform detection, downloads the installer +# install agave-install. It just does platform detection, downloads the installer # and runs it. { # this ensures the entire script is downloaded # @@ -24,11 +24,11 @@ set -e usage() { cat 1>&2 < --pubkey + agave-install-init [FLAGS] [OPTIONS] --data_dir --pubkey FLAGS: -h, --help Prints help information @@ -81,7 +81,7 @@ main() { esac TARGET="${_cputype}-${_ostype}" - temp_dir="$(mktemp -d 2>/dev/null || ensure mktemp -d -t solana-install-init)" + temp_dir="$(mktemp -d 2>/dev/null || ensure mktemp -d -t agave-install-init)" ensure mkdir -p "$temp_dir" # Check for SOLANA_RELEASE environment variable override. Otherwise fetch @@ -101,8 +101,8 @@ main() { fi fi - download_url="$SOLANA_DOWNLOAD_ROOT/$release/solana-install-init-$TARGET" - solana_install_init="$temp_dir/solana-install-init" + download_url="$SOLANA_DOWNLOAD_ROOT/$release/agave-install-init-$TARGET" + solana_install_init="$temp_dir/agave-install-init" printf 'downloading %s installer\n' "$release" 1>&2 @@ -111,7 +111,7 @@ main() { ensure chmod u+x "$solana_install_init" if [ ! -x "$solana_install_init" ]; then printf '%s\n' "Cannot execute $solana_install_init (likely because of mounting /tmp as noexec)." 1>&2 - printf '%s\n' "Please copy the file to a location where you can execute binaries and run ./solana-install-init." 1>&2 + printf '%s\n' "Please copy the file to a location where you can execute binaries and run ./agave-install-init." 1>&2 exit 1 fi @@ -130,7 +130,7 @@ main() { } err() { - printf 'solana-install-init: %s\n' "$1" >&2 + printf 'agave-install-init: %s\n' "$1" >&2 exit 1 } diff --git a/install/install-help.sh b/install/install-help.sh index 9fb08afa6d14c9..7604777e378677 100755 --- a/install/install-help.sh +++ b/install/install-help.sh @@ -4,11 +4,11 @@ set -e cd "$(dirname "$0")"/.. cargo="$(readlink -f "./cargo")" -"$cargo" build --package solana-install +"$cargo" build --package agave-install export PATH=$PWD/target/debug:$PATH echo "\`\`\`manpage" -solana-install --help +agave-install --help echo "\`\`\`" echo "" @@ -16,7 +16,7 @@ commands=(init info deploy update run) for x in "${commands[@]}"; do echo "\`\`\`manpage" - solana-install "${x}" --help + agave-install "${x}" --help echo "\`\`\`" echo "" done diff --git a/install/src/bin/solana-install-init.rs b/install/src/bin/agave-install-init.rs similarity index 92% rename from install/src/bin/solana-install-init.rs rename to install/src/bin/agave-install-init.rs index ec888d8f452090..84c154ac12b35e 100644 --- a/install/src/bin/solana-install-init.rs +++ b/install/src/bin/agave-install-init.rs @@ -16,7 +16,7 @@ fn press_enter() { } fn main() { - solana_install::main_init().unwrap_or_else(|err| { + agave_install::main_init().unwrap_or_else(|err| { println!("Error: {err}"); press_enter(); exit(1); diff --git a/install/src/command.rs b/install/src/command.rs index 218e815467e9a9..4ae9e7ee38cedd 100644 --- a/install/src/command.rs +++ b/install/src/command.rs @@ -540,7 +540,7 @@ pub fn init( explicit_release: Option, ) -> Result<(), String> { let config = { - // Write new config file only if different, so that running |solana-install init| + // Write new config file only if different, so that running |agave-install init| // repeatedly doesn't unnecessarily re-download let mut current_config = Config::load(config_file).unwrap_or_default(); current_config.current_update_manifest = None; @@ -870,7 +870,7 @@ fn check_for_newer_github_release( prerelease_allowed: bool, ) -> Result, String> { let client = reqwest::blocking::Client::builder() - .user_agent("solana-install") + .user_agent("agave-install") .build() .map_err(|err| err.to_string())?; diff --git a/install/src/lib.rs b/install/src/lib.rs index 159317edd2e5a8..a28b963d65f825 100644 --- a/install/src/lib.rs +++ b/install/src/lib.rs @@ -281,7 +281,7 @@ pub fn main() -> Result<(), String> { pub fn main_init() -> Result<(), String> { solana_logger::setup(); - let matches = App::new("solana-install-init") + let matches = App::new("agave-install-init") .about("Initializes a new installation") .version(solana_version::version!()) .arg({ diff --git a/install/src/main.rs b/install/src/main.rs index c7b15aa6a67206..245f09825ddc6a 100644 --- a/install/src/main.rs +++ b/install/src/main.rs @@ -1,3 +1,3 @@ fn main() -> Result<(), String> { - solana_install::main() + agave_install::main() } diff --git a/ledger-tool/Cargo.toml b/ledger-tool/Cargo.toml index 6da42940a4ba7f..cb87a0e16f4a36 100644 --- a/ledger-tool/Cargo.toml +++ b/ledger-tool/Cargo.toml @@ -1,7 +1,7 @@ [package] -name = "solana-ledger-tool" +name = "agave-ledger-tool" description = "Blockchain, Rebuilt for Scale" -documentation = "https://docs.rs/solana-ledger-tool" +documentation = "https://docs.rs/agave-ledger-tool" version = { workspace = true } authors = { workspace = true } repository = { workspace = true } diff --git a/ledger-tool/src/blockstore.rs b/ledger-tool/src/blockstore.rs index 453a801702f864..fed6abde2f2d08 100644 --- a/ledger-tool/src/blockstore.rs +++ b/ledger-tool/src/blockstore.rs @@ -359,7 +359,7 @@ pub fn blockstore_subcommands<'a, 'b>(hidden: bool) -> Vec> { and timestamps.", ) // This command is important in cluster restart scenarios, so do not hide it ever - // such that the subcommand will be visible as the top level of solana-ledger-tool + // such that the subcommand will be visible as the top level of agave-ledger-tool .arg( Arg::with_name("num_slots") .long("num-slots") diff --git a/ledger-tool/src/ledger_utils.rs b/ledger-tool/src/ledger_utils.rs index 116b21527ae4d8..c05cc6c2d64cd0 100644 --- a/ledger-tool/src/ledger_utils.rs +++ b/ledger-tool/src/ledger_utils.rs @@ -187,14 +187,14 @@ pub fn load_and_process_ledger( } let account_paths = if let Some(account_paths) = arg_matches.value_of("account_paths") { - // If this blockstore access is Primary, no other process (solana-validator) can hold + // If this blockstore access is Primary, no other process (agave-validator) can hold // Primary access. So, allow a custom accounts path without worry of wiping the accounts - // of solana-validator. + // of agave-validator. if !blockstore.is_primary_access() { // Attempt to open the Blockstore in Primary access; if successful, no other process // was holding Primary so allow things to proceed with custom accounts path. Release - // the Primary access instead of holding it to give priority to solana-validator over - // solana-ledger-tool should solana-validator start before we've finished. + // the Primary access instead of holding it to give priority to agave-validator over + // agave-ledger-tool should agave-validator start before we've finished. info!( "Checking if another process currently holding Primary access to {:?}", blockstore.ledger_path() diff --git a/ledger/src/blockstore_db.rs b/ledger/src/blockstore_db.rs index 18ba491ea34bd1..8b6b44edae61f6 100644 --- a/ledger/src/blockstore_db.rs +++ b/ledger/src/blockstore_db.rs @@ -431,7 +431,7 @@ impl Rocks { info!( "Opening Rocks with secondary (read only) access at: {secondary_path:?}. \ This secondary access could temporarily degrade other accesses, such as \ - by solana-validator" + by agave-validator" ); DB::open_cf_descriptors_as_secondary( &db_options, diff --git a/local-cluster/tests/local_cluster.rs b/local-cluster/tests/local_cluster.rs index 3b18ba44bf2d03..20eef0bb0e3e2d 100644 --- a/local-cluster/tests/local_cluster.rs +++ b/local-cluster/tests/local_cluster.rs @@ -2321,13 +2321,13 @@ fn test_hard_fork_with_gap_in_roots() { ); // create hard-forked snapshot only for validator a, emulating the manual cluster restart - // procedure with `solana-ledger-tool create-snapshot` + // procedure with `agave-ledger-tool create-snapshot` let genesis_slot = 0; { let blockstore_a = Blockstore::open(&val_a_ledger_path).unwrap(); create_snapshot_to_hard_fork(&blockstore_a, hard_fork_slot, vec![hard_fork_slot]); - // Intentionally make solana-validator unbootable by replaying blocks from the genesis to + // Intentionally make agave-validator unbootable by replaying blocks from the genesis to // ensure the hard-forked snapshot is used always. Otherwise, we couldn't create a gap // in the ledger roots column family reliably. // There was a bug which caused the hard-forked snapshot at an unrooted slot to forget diff --git a/multinode-demo/bootstrap-validator.sh b/multinode-demo/bootstrap-validator.sh index 5afc543b2f0032..2872af5cc426af 100755 --- a/multinode-demo/bootstrap-validator.sh +++ b/multinode-demo/bootstrap-validator.sh @@ -14,9 +14,9 @@ if [[ "$SOLANA_GPU_MISSING" -eq 1 ]]; then fi if [[ -n $SOLANA_CUDA ]]; then - program=$solana_validator_cuda + program=$agave_validator_cuda else - program=$solana_validator + program=$agave_validator fi no_restart=0 diff --git a/multinode-demo/common.sh b/multinode-demo/common.sh index 9ae9331cb7a11d..1643208947b643 100644 --- a/multinode-demo/common.sh +++ b/multinode-demo/common.sh @@ -40,6 +40,8 @@ else if [[ -z $program ]]; then crate="cli" program="solana" + elif [[ $program == "validator" || $program == "ledger-tool" || $program == "watchtower" || $program == "install" ]]; then + program="agave-$program" else program="solana-$program" fi @@ -63,8 +65,8 @@ fi solana_bench_tps=$(solana_program bench-tps) solana_faucet=$(solana_program faucet) -solana_validator=$(solana_program validator) -solana_validator_cuda="$solana_validator --cuda" +agave_validator=$(solana_program validator) +agave_validator_cuda="$agave_validator --cuda" solana_genesis=$(solana_program genesis) solana_gossip=$(solana_program gossip) solana_keygen=$(solana_program keygen) diff --git a/multinode-demo/validator.sh b/multinode-demo/validator.sh index 487154101ac979..efb7a6afd56ea0 100755 --- a/multinode-demo/validator.sh +++ b/multinode-demo/validator.sh @@ -64,7 +64,7 @@ while [[ -n $1 ]]; do elif [[ $1 = --no-airdrop ]]; then airdrops_enabled=0 shift - # solana-validator options + # agave-validator options elif [[ $1 = --expected-genesis-hash ]]; then args+=("$1" "$2") shift 2 @@ -270,9 +270,9 @@ if [[ $maybeRequireTower = true ]]; then fi if [[ -n $SOLANA_CUDA ]]; then - program=$solana_validator_cuda + program=$agave_validator_cuda else - program=$solana_validator + program=$agave_validator fi set -e diff --git a/net/net.sh b/net/net.sh index a2d16cef20f417..36bc48efdb7861 100755 --- a/net/net.sh +++ b/net/net.sh @@ -122,7 +122,7 @@ Operate a configured testnet sanity/start-specific options: -F - Discard validator nodes that didn't bootup successfully - -o noInstallCheck - Skip solana-install sanity + -o noInstallCheck - Skip agave-install sanity -o rejectExtraNodes - Require the exact number of nodes stop-specific options: @@ -138,7 +138,7 @@ Operate a configured testnet --netem-cmd - Optional command argument to netem. Default is "add". Use "cleanup" to remove rules. update-specific options: - --platform linux|osx|windows - Deploy the tarball using 'solana-install deploy ...' for the + --platform linux|osx|windows - Deploy the tarball using 'agave-install deploy ...' for the given platform (multiple platforms may be specified) (-t option must be supplied as well) @@ -514,11 +514,11 @@ deployUpdate() { declare bootstrapLeader=${validatorIpList[0]} for updatePlatform in $updatePlatforms; do - echo "--- Deploying solana-install update: $updatePlatform" + echo "--- Deploying agave-install update: $updatePlatform" ( set -x - scripts/solana-install-update-manifest-keypair.sh "$updatePlatform" + scripts/agave-install-update-manifest-keypair.sh "$updatePlatform" timeout 30s scp "${sshOptions[@]}" \ update_manifest_keypair.json "$bootstrapLeader:solana/update_manifest_keypair.json" diff --git a/net/remote/remote-deploy-update.sh b/net/remote/remote-deploy-update.sh index dd772927c0e119..3a71cf5725123e 100755 --- a/net/remote/remote-deploy-update.sh +++ b/net/remote/remote-deploy-update.sh @@ -35,6 +35,6 @@ loadConfigFile PATH="$HOME"/.cargo/bin:"$PATH" set -x -scripts/solana-install-deploy.sh \ +scripts/agave-install-deploy.sh \ --keypair config/faucet.json \ localhost "$releaseChannel" "$updatePlatform" diff --git a/net/remote/remote-node.sh b/net/remote/remote-node.sh index aeb920bd50bab0..b7d224088da9f9 100755 --- a/net/remote/remote-node.sh +++ b/net/remote/remote-node.sh @@ -121,7 +121,7 @@ cat >> ~/solana/on-reboot < system-stats.pid if ${GPU_CUDA_OK} && [[ -e /dev/nvidia0 ]]; then - echo Selecting solana-validator-cuda + echo Selecting agave-validator-cuda export SOLANA_CUDA=1 elif ${GPU_FAIL_IF_NONE} ; then echo "Expected GPU, found none!" @@ -257,13 +257,13 @@ EOF if [[ -n "$maybeWarpSlot" ]]; then # shellcheck disable=SC2086 # Do not want to quote $maybeWarSlot - solana-ledger-tool -l config/bootstrap-validator create-snapshot 0 config/bootstrap-validator $maybeWarpSlot + agave-ledger-tool -l config/bootstrap-validator create-snapshot 0 config/bootstrap-validator $maybeWarpSlot fi - solana-ledger-tool -l config/bootstrap-validator shred-version --max-genesis-archive-unpacked-size 1073741824 | tee config/shred-version + agave-ledger-tool -l config/bootstrap-validator shred-version --max-genesis-archive-unpacked-size 1073741824 | tee config/shred-version if [[ -n "$maybeWaitForSupermajority" ]]; then - bankHash=$(solana-ledger-tool -l config/bootstrap-validator bank-hash --halt-at-slot 0) + bankHash=$(agave-ledger-tool -l config/bootstrap-validator bank-hash --halt-at-slot 0) extraNodeArgs="$extraNodeArgs --expected-bank-hash $bankHash" echo "$bankHash" > config/bank-hash fi diff --git a/net/remote/remote-sanity.sh b/net/remote/remote-sanity.sh index 8c36e99ffdf936..91dae4b57336fa 100755 --- a/net/remote/remote-sanity.sh +++ b/net/remote/remote-sanity.sh @@ -65,7 +65,7 @@ local|tar|skip) export USE_INSTALL=1 solana_cli=solana solana_gossip=solana-gossip - solana_install=solana-install + solana_install=agave-install ;; *) echo "Unknown deployment method: $deployMethod" @@ -122,7 +122,7 @@ else fi if $installCheck && [[ -r update_manifest_keypair.json ]]; then - echo "--- $sanityTargetIp: solana-install test" + echo "--- $sanityTargetIp: agave-install test" ( set -x diff --git a/notifier/src/lib.rs b/notifier/src/lib.rs index a369225772492c..75406d2fbdae33 100644 --- a/notifier/src/lib.rs +++ b/notifier/src/lib.rs @@ -19,7 +19,7 @@ /// /// To receive a Twilio SMS notification on failure, having a Twilio account, /// and a sending number owned by that account, -/// define environment variable before running `solana-watchtower`: +/// define environment variable before running `agave-watchtower`: /// ```bash /// export TWILIO_CONFIG='ACCOUNT=,TOKEN=,TO=,FROM=' /// ``` @@ -208,7 +208,7 @@ impl Notifier { NotificationType::Resolve { ref incident } => incident.clone().to_string(), }; - let data = json!({"payload":{"summary":msg,"source":"solana-watchtower","severity":"critical"},"routing_key":routing_key,"event_action":event_action,"dedup_key":dedup_key}); + let data = json!({"payload":{"summary":msg,"source":"agave-watchtower","severity":"critical"},"routing_key":routing_key,"event_action":event_action,"dedup_key":dedup_key}); let url = "https://events.pagerduty.com/v2/enqueue"; if let Err(err) = self.client.post(url).json(&data).send() { diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index 2829cf27b6da6f..7546c56bd2b26a 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -63,6 +63,80 @@ dependencies = [ "zeroize", ] +[[package]] +name = "agave-geyser-plugin-interface" +version = "1.19.0" +dependencies = [ + "log", + "solana-sdk", + "solana-transaction-status", + "thiserror", +] + +[[package]] +name = "agave-validator" +version = "1.19.0" +dependencies = [ + "agave-geyser-plugin-interface", + "chrono", + "clap 2.33.3", + "console", + "core_affinity", + "crossbeam-channel", + "fd-lock", + "indicatif", + "itertools", + "jsonrpc-core", + "jsonrpc-core-client", + "jsonrpc-derive", + "jsonrpc-ipc-server", + "jsonrpc-server-utils", + "lazy_static", + "libc", + "libloading", + "log", + "num_cpus", + "rand 0.8.5", + "rayon", + "serde", + "serde_json", + "serde_yaml", + "signal-hook", + "solana-accounts-db", + "solana-clap-utils", + "solana-cli-config", + "solana-core", + "solana-download-utils", + "solana-entry", + "solana-faucet", + "solana-genesis-utils", + "solana-geyser-plugin-manager", + "solana-gossip", + "solana-ledger", + "solana-logger", + "solana-metrics", + "solana-net-utils", + "solana-perf", + "solana-poh", + "solana-rpc", + "solana-rpc-client", + "solana-rpc-client-api", + "solana-runtime", + "solana-sdk", + "solana-send-transaction-service", + "solana-storage-bigtable", + "solana-streamer", + "solana-svm", + "solana-test-validator", + "solana-tpu-client", + "solana-unified-scheduler-pool", + "solana-version", + "solana-vote-program", + "symlink", + "thiserror", + "tikv-jemallocator", +] + [[package]] name = "ahash" version = "0.7.6" @@ -5044,20 +5118,11 @@ dependencies = [ "solana-sdk", ] -[[package]] -name = "solana-geyser-plugin-interface" -version = "1.19.0" -dependencies = [ - "log", - "solana-sdk", - "solana-transaction-status", - "thiserror", -] - [[package]] name = "solana-geyser-plugin-manager" version = "1.19.0" dependencies = [ + "agave-geyser-plugin-interface", "bs58", "crossbeam-channel", "json5", @@ -5068,7 +5133,6 @@ dependencies = [ "serde_json", "solana-accounts-db", "solana-entry", - "solana-geyser-plugin-interface", "solana-ledger", "solana-measure", "solana-metrics", @@ -6055,11 +6119,11 @@ dependencies = [ name = "solana-sbf-rust-simulation" version = "1.19.0" dependencies = [ + "agave-validator", "solana-logger", "solana-program", "solana-program-test", "solana-sdk", - "solana-validator", ] [[package]] @@ -6462,70 +6526,6 @@ dependencies = [ "solana-vote", ] -[[package]] -name = "solana-validator" -version = "1.19.0" -dependencies = [ - "chrono", - "clap 2.33.3", - "console", - "core_affinity", - "crossbeam-channel", - "fd-lock", - "indicatif", - "itertools", - "jsonrpc-core", - "jsonrpc-core-client", - "jsonrpc-derive", - "jsonrpc-ipc-server", - "jsonrpc-server-utils", - "lazy_static", - "libc", - "libloading", - "log", - "num_cpus", - "rand 0.8.5", - "rayon", - "serde", - "serde_json", - "serde_yaml", - "signal-hook", - "solana-accounts-db", - "solana-clap-utils", - "solana-cli-config", - "solana-core", - "solana-download-utils", - "solana-entry", - "solana-faucet", - "solana-genesis-utils", - "solana-geyser-plugin-interface", - "solana-geyser-plugin-manager", - "solana-gossip", - "solana-ledger", - "solana-logger", - "solana-metrics", - "solana-net-utils", - "solana-perf", - "solana-poh", - "solana-rpc", - "solana-rpc-client", - "solana-rpc-client-api", - "solana-runtime", - "solana-sdk", - "solana-send-transaction-service", - "solana-storage-bigtable", - "solana-streamer", - "solana-svm", - "solana-test-validator", - "solana-tpu-client", - "solana-unified-scheduler-pool", - "solana-version", - "solana-vote-program", - "symlink", - "thiserror", - "tikv-jemallocator", -] - [[package]] name = "solana-version" version = "1.19.0" diff --git a/programs/sbf/Cargo.toml b/programs/sbf/Cargo.toml index 8a99a0f005471a..dee6a947b1965d 100644 --- a/programs/sbf/Cargo.toml +++ b/programs/sbf/Cargo.toml @@ -46,7 +46,7 @@ solana-sbf-rust-realloc = { path = "rust/realloc", version = "=1.19.0", default- solana-sbf-rust-realloc-invoke = { path = "rust/realloc_invoke", version = "=1.19.0" } solana-sdk = { path = "../../sdk", version = "=1.19.0" } solana-transaction-status = { path = "../../transaction-status", version = "=1.19.0" } -solana-validator = { path = "../../validator", version = "=1.19.0" } +agave-validator = { path = "../../validator", version = "=1.19.0" } solana-zk-token-sdk = { path = "../../zk-token-sdk", version = "=1.19.0" } solana-svm = { path = "../../svm", version = "=1.19.0" } solana_rbpf = "=0.8.0" diff --git a/programs/sbf/rust/simulation/Cargo.toml b/programs/sbf/rust/simulation/Cargo.toml index 7091ef9d5ade0c..e9728e5916b801 100644 --- a/programs/sbf/rust/simulation/Cargo.toml +++ b/programs/sbf/rust/simulation/Cargo.toml @@ -16,10 +16,10 @@ test-bpf = [] solana-program = { workspace = true } [dev-dependencies] +agave-validator = { workspace = true } solana-logger = { workspace = true } solana-program-test = { workspace = true } solana-sdk = { workspace = true } -solana-validator = { workspace = true } [lib] crate-type = ["cdylib", "lib"] diff --git a/programs/sbf/rust/simulation/tests/validator.rs b/programs/sbf/rust/simulation/tests/validator.rs index 3044ad9a642629..17de51e665e3ec 100644 --- a/programs/sbf/rust/simulation/tests/validator.rs +++ b/programs/sbf/rust/simulation/tests/validator.rs @@ -1,13 +1,13 @@ #![cfg(feature = "test-bpf")] use { + agave_validator::test_validator::*, solana_program::{ instruction::{AccountMeta, Instruction}, pubkey::Pubkey, sysvar, }, solana_sdk::{signature::Signer, transaction::Transaction}, - solana_validator::test_validator::*, }; #[test] diff --git a/pubsub-client/src/nonblocking/pubsub_client.rs b/pubsub-client/src/nonblocking/pubsub_client.rs index 408df60454e4e1..b79e91f681b97f 100644 --- a/pubsub-client/src/nonblocking/pubsub_client.rs +++ b/pubsub-client/src/nonblocking/pubsub_client.rs @@ -33,7 +33,7 @@ //! By default the [`block_subscribe`] and [`vote_subscribe`] events are //! disabled on RPC nodes. They can be enabled by passing //! `--rpc-pubsub-enable-block-subscription` and -//! `--rpc-pubsub-enable-vote-subscription` to `solana-validator`. When these +//! `--rpc-pubsub-enable-vote-subscription` to `agave-validator`. When these //! methods are disabled, the RPC server will return a "Method not found" error //! message. //! @@ -381,7 +381,7 @@ impl PubsubClient { /// Receives messages of type [`RpcBlockUpdate`] when a block is confirmed or finalized. /// /// This method is disabled by default. It can be enabled by passing - /// `--rpc-pubsub-enable-block-subscription` to `solana-validator`. + /// `--rpc-pubsub-enable-block-subscription` to `agave-validator`. /// /// # RPC Reference /// @@ -452,7 +452,7 @@ impl PubsubClient { /// votes are observed prior to confirmation and may never be confirmed. /// /// This method is disabled by default. It can be enabled by passing - /// `--rpc-pubsub-enable-vote-subscription` to `solana-validator`. + /// `--rpc-pubsub-enable-vote-subscription` to `agave-validator`. /// /// # RPC Reference /// diff --git a/pubsub-client/src/pubsub_client.rs b/pubsub-client/src/pubsub_client.rs index e1a2dd34546528..70769619db1f4d 100644 --- a/pubsub-client/src/pubsub_client.rs +++ b/pubsub-client/src/pubsub_client.rs @@ -32,7 +32,7 @@ //! By default the [`block_subscribe`] and [`vote_subscribe`] events are //! disabled on RPC nodes. They can be enabled by passing //! `--rpc-pubsub-enable-block-subscription` and -//! `--rpc-pubsub-enable-vote-subscription` to `solana-validator`. When these +//! `--rpc-pubsub-enable-vote-subscription` to `agave-validator`. When these //! methods are disabled, the RPC server will return a "Method not found" error //! message. //! @@ -416,7 +416,7 @@ impl PubsubClient { /// Receives messages of type [`RpcBlockUpdate`] when a block is confirmed or finalized. /// /// This method is disabled by default. It can be enabled by passing - /// `--rpc-pubsub-enable-block-subscription` to `solana-validator`. + /// `--rpc-pubsub-enable-block-subscription` to `agave-validator`. /// /// # RPC Reference /// @@ -578,7 +578,7 @@ impl PubsubClient { /// votes are observed prior to confirmation and may never be confirmed. /// /// This method is disabled by default. It can be enabled by passing - /// `--rpc-pubsub-enable-vote-subscription` to `solana-validator`. + /// `--rpc-pubsub-enable-vote-subscription` to `agave-validator`. /// /// # RPC Reference /// diff --git a/rbpf-cli/src/main.rs b/rbpf-cli/src/main.rs index e7db982026f82a..9e243f0836aa0f 100644 --- a/rbpf-cli/src/main.rs +++ b/rbpf-cli/src/main.rs @@ -1,6 +1,6 @@ fn main() { println!( - r##"rbpf-cli is replaced by solana-ledger-tool program run subcommand. -Please, use 'solana-ledger-tool program run --help' for more information."## + r##"rbpf-cli is replaced by agave-ledger-tool program run subcommand. +Please, use 'agave-ledger-tool program run --help' for more information."## ); } diff --git a/rpc/src/rpc.rs b/rpc/src/rpc.rs index caeb0953109fbb..01f623dccdc108 100644 --- a/rpc/src/rpc.rs +++ b/rpc/src/rpc.rs @@ -2561,7 +2561,7 @@ pub mod rpc_minimal { #[rpc(meta, name = "getVersion")] fn get_version(&self, meta: Self::Metadata) -> Result; - // TODO: Refactor `solana-validator wait-for-restart-window` to not require this method, so + // TODO: Refactor `agave-validator wait-for-restart-window` to not require this method, so // it can be removed from rpc_minimal #[rpc(meta, name = "getVoteAccounts")] fn get_vote_accounts( @@ -2570,7 +2570,7 @@ pub mod rpc_minimal { config: Option, ) -> Result; - // TODO: Refactor `solana-validator wait-for-restart-window` to not require this method, so + // TODO: Refactor `agave-validator wait-for-restart-window` to not require this method, so // it can be removed from rpc_minimal #[rpc(meta, name = "getLeaderSchedule")] fn get_leader_schedule( @@ -2696,7 +2696,7 @@ pub mod rpc_minimal { }) } - // TODO: Refactor `solana-validator wait-for-restart-window` to not require this method, so + // TODO: Refactor `agave-validator wait-for-restart-window` to not require this method, so // it can be removed from rpc_minimal fn get_vote_accounts( &self, @@ -2707,7 +2707,7 @@ pub mod rpc_minimal { meta.get_vote_accounts(config) } - // TODO: Refactor `solana-validator wait-for-restart-window` to not require this method, so + // TODO: Refactor `agave-validator wait-for-restart-window` to not require this method, so // it can be removed from rpc_minimal fn get_leader_schedule( &self, diff --git a/scripts/solana-install-deploy.sh b/scripts/agave-install-deploy.sh similarity index 90% rename from scripts/solana-install-deploy.sh rename to scripts/agave-install-deploy.sh index ea77ca34bc9ea3..a8f8eeb65b3857 100755 --- a/scripts/solana-install-deploy.sh +++ b/scripts/agave-install-deploy.sh @@ -26,7 +26,7 @@ if [[ -z $URL || -z $TAG ]]; then fi if [[ ! -f update_manifest_keypair.json ]]; then - "$SOLANA_ROOT"/scripts/solana-install-update-manifest-keypair.sh "$OS" + "$SOLANA_ROOT"/scripts/agave-install-update-manifest-keypair.sh "$OS" fi case "$OS" in @@ -76,4 +76,4 @@ if [[ $balance = "0 lamports" ]]; then fi # shellcheck disable=SC2086 # Don't want to double quote $maybeKeypair -solana-install deploy $maybeKeypair --url "$URL" "$DOWNLOAD_URL" update_manifest_keypair.json +agave-install deploy $maybeKeypair --url "$URL" "$DOWNLOAD_URL" update_manifest_keypair.json diff --git a/scripts/cargo-install-all.sh b/scripts/cargo-install-all.sh index 549aa15550b0eb..029b1fbf27943d 100755 --- a/scripts/cargo-install-all.sh +++ b/scripts/cargo-install-all.sh @@ -91,8 +91,8 @@ if [[ $CI_OS_NAME = windows ]]; then cargo-test-bpf cargo-test-sbf solana - solana-install - solana-install-init + agave-install + agave-install-init solana-keygen solana-stake-accounts solana-test-validator @@ -106,12 +106,12 @@ else solana-bench-tps solana-faucet solana-gossip - solana-install + agave-install solana-keygen - solana-ledger-tool + agave-ledger-tool solana-log-analyzer solana-net-shaper - solana-validator + agave-validator rbpf-cli ) @@ -123,11 +123,11 @@ else cargo-test-bpf cargo-test-sbf solana-dos - solana-install-init + agave-install-init solana-stake-accounts solana-test-validator solana-tokens - solana-watchtower + agave-watchtower ) fi diff --git a/scripts/check-dev-context-only-utils.sh b/scripts/check-dev-context-only-utils.sh index 8719af96a212e4..6a4f798c633e26 100755 --- a/scripts/check-dev-context-only-utils.sh +++ b/scripts/check-dev-context-only-utils.sh @@ -31,7 +31,7 @@ source ci/rust-version.sh nightly declare tainted_packages=( solana-accounts-bench solana-banking-bench - solana-ledger-tool + agave-ledger-tool ) # convert to comma separeted (ref: https://stackoverflow.com/a/53839433) diff --git a/scripts/run.sh b/scripts/run.sh index 699bfce3e253e3..2d8e1ec88ac450 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -23,9 +23,11 @@ fi PATH=$PWD/target/$profile:$PATH ok=true -for program in solana-{faucet,genesis,keygen,validator}; do +for program in solana-{faucet,genesis,keygen}; do $program -V || ok=false done +agave-validator -V || ok=false + $ok || { echo echo "Unable to locate required programs. Try building them first with:" @@ -115,7 +117,7 @@ args=( --no-os-network-limits-test ) # shellcheck disable=SC2086 -solana-validator "${args[@]}" $SOLANA_RUN_SH_VALIDATOR_ARGS & +agave-validator "${args[@]}" $SOLANA_RUN_SH_VALIDATOR_ARGS & validator=$! wait "$validator" diff --git a/system-test/abi-testcases/mixed-validator-test.sh b/system-test/abi-testcases/mixed-validator-test.sh index 8ab673b26a3d21..c0400560dc519e 100755 --- a/system-test/abi-testcases/mixed-validator-test.sh +++ b/system-test/abi-testcases/mixed-validator-test.sh @@ -30,14 +30,14 @@ solanaInstallGlobalOpts=( bootstrapInstall() { declare v=$1 if [[ ! -h $solanaInstallDataDir/active_release ]]; then - sh "$SOLANA_ROOT"/install/solana-install-init.sh "$v" "${solanaInstallGlobalOpts[@]}" + sh "$SOLANA_ROOT"/install/agave-install-init.sh "$v" "${solanaInstallGlobalOpts[@]}" fi export PATH="$solanaInstallDataDir/active_release/bin/:$PATH" } bootstrapInstall "$baselineVersion" for v in "${otherVersions[@]}"; do - solana-install-init "${solanaInstallGlobalOpts[@]}" "$v" + agave-install-init "${solanaInstallGlobalOpts[@]}" "$v" solana -V done @@ -113,7 +113,7 @@ for v in "${otherVersions[@]}"; do ( set -x tmux new-window -t abi -n "$v" " \ - $SOLANA_BIN/solana-validator \ + $SOLANA_BIN/agave-validator \ --ledger $ledger \ --no-snapshot-fetch \ --entrypoint 127.0.0.1:8001 \ diff --git a/system-test/stability-testcases/gossip-dos-test.sh b/system-test/stability-testcases/gossip-dos-test.sh index f8afade75dc847..68c3c540d5948c 100755 --- a/system-test/stability-testcases/gossip-dos-test.sh +++ b/system-test/stability-testcases/gossip-dos-test.sh @@ -19,14 +19,14 @@ solanaInstallGlobalOpts=( bootstrapInstall() { declare v=$1 if [[ ! -h $solanaInstallDataDir/active_release ]]; then - sh "$SOLANA_ROOT"/install/solana-install-init.sh "$v" "${solanaInstallGlobalOpts[@]}" + sh "$SOLANA_ROOT"/install/agave-install-init.sh "$v" "${solanaInstallGlobalOpts[@]}" fi export PATH="$solanaInstallDataDir/active_release/bin/:$PATH" } bootstrapInstall "edge" -solana-install-init --version -solana-install-init edge +agave-install-init --version +agave-install-init edge solana-gossip --version solana-dos --version diff --git a/validator/Cargo.toml b/validator/Cargo.toml index 5cc76a810116b3..362a07343b5e4a 100644 --- a/validator/Cargo.toml +++ b/validator/Cargo.toml @@ -1,8 +1,8 @@ [package] -name = "solana-validator" +name = "agave-validator" description = "Blockchain, Rebuilt for Scale" -documentation = "https://docs.rs/solana-validator" -default-run = "solana-validator" +documentation = "https://docs.rs/agave-validator" +default-run = "agave-validator" version = { workspace = true } authors = { workspace = true } repository = { workspace = true } @@ -11,6 +11,7 @@ license = { workspace = true } edition = { workspace = true } [dependencies] +agave-geyser-plugin-interface = { workspace = true } chrono = { workspace = true, features = ["default", "serde"] } clap = { workspace = true } console = { workspace = true } @@ -41,7 +42,6 @@ solana-download-utils = { workspace = true } solana-entry = { workspace = true } solana-faucet = { workspace = true } solana-genesis-utils = { workspace = true } -solana-geyser-plugin-interface = { workspace = true } solana-geyser-plugin-manager = { workspace = true } solana-gossip = { workspace = true } solana-ledger = { workspace = true } diff --git a/validator/src/bin/solana-test-validator.rs b/validator/src/bin/solana-test-validator.rs index 42f5a0634c0cfa..68e6bcca4fd96f 100644 --- a/validator/src/bin/solana-test-validator.rs +++ b/validator/src/bin/solana-test-validator.rs @@ -1,4 +1,8 @@ use { + agave_validator::{ + admin_rpc_service, cli, dashboard::Dashboard, ledger_lockfile, lock_ledger, + println_name_value, redirect_stderr_to_file, + }, clap::{crate_name, value_t, value_t_or_exit, values_t_or_exit}, crossbeam_channel::unbounded, itertools::Itertools, @@ -28,10 +32,6 @@ use { }, solana_streamer::socket::SocketAddrSpace, solana_test_validator::*, - solana_validator::{ - admin_rpc_service, cli, dashboard::Dashboard, ledger_lockfile, lock_ledger, - println_name_value, redirect_stderr_to_file, - }, std::{ collections::HashSet, fs, io, diff --git a/validator/src/bootstrap.rs b/validator/src/bootstrap.rs index 8d5457744a23b8..12bbd0b21001c9 100644 --- a/validator/src/bootstrap.rs +++ b/validator/src/bootstrap.rs @@ -447,7 +447,7 @@ pub fn attempt_download_genesis_and_snapshot( ) .unwrap_or_else(|err| { // Consider failures here to be more likely due to user error (eg, - // incorrect `solana-validator` command-line arguments) rather than the + // incorrect `agave-validator` command-line arguments) rather than the // RPC node failing. // // Power users can always use the `--no-check-vote-account` option to diff --git a/validator/src/main.rs b/validator/src/main.rs index ec70796130e7d2..9741a2aecd68a8 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -2,6 +2,15 @@ #[cfg(not(target_env = "msvc"))] use jemallocator::Jemalloc; use { + agave_validator::{ + admin_rpc_service, + admin_rpc_service::{load_staked_nodes_overrides, StakedNodesOverrides}, + bootstrap, + cli::{app, warn_for_deprecated_arguments, DefaultArgs}, + dashboard::Dashboard, + ledger_lockfile, lock_ledger, new_spinner_progress_bar, println_name_value, + redirect_stderr_to_file, + }, clap::{crate_name, value_t, value_t_or_exit, values_t, values_t_or_exit, ArgMatches}, console::style, crossbeam_channel::unbounded, @@ -60,15 +69,6 @@ use { solana_streamer::socket::SocketAddrSpace, solana_svm::runtime_config::RuntimeConfig, solana_tpu_client::tpu_client::DEFAULT_TPU_ENABLE_UDP, - solana_validator::{ - admin_rpc_service, - admin_rpc_service::{load_staked_nodes_overrides, StakedNodesOverrides}, - bootstrap, - cli::{app, warn_for_deprecated_arguments, DefaultArgs}, - dashboard::Dashboard, - ledger_lockfile, lock_ledger, new_spinner_progress_bar, println_name_value, - redirect_stderr_to_file, - }, std::{ collections::{HashSet, VecDeque}, env, @@ -917,7 +917,7 @@ pub fn main() { let logfile = matches .value_of("logfile") .map(|s| s.into()) - .unwrap_or_else(|| format!("solana-validator-{}.log", identity_keypair.pubkey())); + .unwrap_or_else(|| format!("agave-validator-{}.log", identity_keypair.pubkey())); if logfile == "-" { None diff --git a/watchtower/Cargo.toml b/watchtower/Cargo.toml index d8bad3cf4d18f0..4088ee7d9b51ab 100644 --- a/watchtower/Cargo.toml +++ b/watchtower/Cargo.toml @@ -1,7 +1,7 @@ [package] -name = "solana-watchtower" +name = "agave-watchtower" description = "Blockchain, Rebuilt for Scale" -documentation = "https://docs.rs/solana-watchtower" +documentation = "https://docs.rs/agave-watchtower" version = { workspace = true } authors = { workspace = true } repository = { workspace = true } diff --git a/watchtower/README.md b/watchtower/README.md index 33a13939cd260c..ab219be67575eb 100644 --- a/watchtower/README.md +++ b/watchtower/README.md @@ -1,4 +1,4 @@ -The `solana-watchtower` program is used to monitor the health of a cluster. It +The `agave-watchtower` program is used to monitor the health of a cluster. It periodically polls the cluster over an RPC API to confirm that the transaction count is advancing, new blockhashes are available, and no validators are delinquent. Results are reported as InfluxDB metrics, with an optional push diff --git a/watchtower/src/main.rs b/watchtower/src/main.rs index f42acdaadaabc6..341b7903c0a33e 100644 --- a/watchtower/src/main.rs +++ b/watchtower/src/main.rs @@ -47,7 +47,7 @@ fn get_config() -> Config { .version(solana_version::version!()) .after_help("ADDITIONAL HELP: To receive a Slack, Discord, PagerDuty and/or Telegram notification on sanity failure, - define environment variables before running `solana-watchtower`: + define environment variables before running `agave-watchtower`: export SLACK_WEBHOOK=... export DISCORD_WEBHOOK=... @@ -63,7 +63,7 @@ fn get_config() -> Config { To receive a Twilio SMS notification on failure, having a Twilio account, and a sending number owned by that account, - define environment variable before running `solana-watchtower`: + define environment variable before running `agave-watchtower`: export TWILIO_CONFIG='ACCOUNT=,TOKEN=,TO=,FROM='") .arg({ @@ -166,7 +166,7 @@ fn get_config() -> Config { .value_name("SUFFIX") .takes_value(true) .default_value("") - .help("Add this string into all notification messages after \"solana-watchtower\"") + .help("Add this string into all notification messages after \"agave-watchtower\"") ) .get_matches(); @@ -381,7 +381,7 @@ fn main() -> Result<(), Box> { if let Some((failure_test_name, failure_error_message)) = &failure { let notification_msg = format!( - "solana-watchtower{}: Error: {}: {}", + "agave-watchtower{}: Error: {}: {}", config.name_suffix, failure_test_name, failure_error_message ); num_consecutive_failures += 1; @@ -415,7 +415,7 @@ fn main() -> Result<(), Box> { ); info!("{}", all_clear_msg); notifier.send( - &format!("solana-watchtower{}: {}", config.name_suffix, all_clear_msg), + &format!("agave-watchtower{}: {}", config.name_suffix, all_clear_msg), &NotificationType::Resolve { incident }, ); } From 932bcd902a63216adfe84d60fcae9ef0d59621f2 Mon Sep 17 00:00:00 2001 From: Brooks Date: Mon, 4 Mar 2024 13:59:35 -0500 Subject: [PATCH 09/71] Adds a bench for hash_account() (#47) --- Cargo.lock | 1 + accounts-db/Cargo.toml | 5 ++++ accounts-db/benches/bench_hashing.rs | 43 ++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+) create mode 100644 accounts-db/benches/bench_hashing.rs diff --git a/Cargo.lock b/Cargo.lock index c72b90930d7cf9..a67b7aba0d6b64 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5390,6 +5390,7 @@ dependencies = [ "bytemuck", "byteorder", "bzip2", + "criterion", "crossbeam-channel", "dashmap", "ed25519-dalek", diff --git a/accounts-db/Cargo.toml b/accounts-db/Cargo.toml index b986c17de0636b..702f14f9f3b07d 100644 --- a/accounts-db/Cargo.toml +++ b/accounts-db/Cargo.toml @@ -70,6 +70,7 @@ name = "solana_accounts_db" [dev-dependencies] assert_matches = { workspace = true } +criterion = { workspace = true } ed25519-dalek = { workspace = true } libsecp256k1 = { workspace = true } memoffset = { workspace = true } @@ -89,3 +90,7 @@ rustc_version = { workspace = true } [features] dev-context-only-utils = [] + +[[bench]] +name = "bench_hashing" +harness = false diff --git a/accounts-db/benches/bench_hashing.rs b/accounts-db/benches/bench_hashing.rs new file mode 100644 index 00000000000000..3158f78c7a938f --- /dev/null +++ b/accounts-db/benches/bench_hashing.rs @@ -0,0 +1,43 @@ +use { + criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}, + solana_accounts_db::accounts_db::AccountsDb, + solana_sdk::{account::AccountSharedData, pubkey::Pubkey}, +}; + +const KB: usize = 1024; +const MB: usize = KB * KB; + +const DATA_SIZES: [usize; 6] = [ + 0, // the smallest account + 165, // the size of an spl token account + 200, // the size of a stake account + KB, // a medium sized account + MB, // a large sized account + 10 * MB, // the largest account +]; + +/// The number of bytes of *non account data* that are also hashed as +/// part of computing an account's hash. +/// +/// Ensure this constant stays in sync with the value of `META_SIZE` in +/// AccountsDb::hash_account_data(). +const META_SIZE: usize = 81; + +fn bench_hash_account(c: &mut Criterion) { + let lamports = 123_456_789; + let owner = Pubkey::default(); + let address = Pubkey::default(); + + let mut group = c.benchmark_group("hash_account"); + for data_size in DATA_SIZES { + let num_bytes = META_SIZE.checked_add(data_size).unwrap(); + group.throughput(Throughput::Bytes(num_bytes as u64)); + let account = AccountSharedData::new(lamports, data_size, &owner); + group.bench_function(BenchmarkId::new("data_size", data_size), |b| { + b.iter(|| AccountsDb::hash_account(&account, &address)); + }); + } +} + +criterion_group!(benches, bench_hash_account,); +criterion_main!(benches); From b63a9af2ebf173a57f4b33df845bd36f3e7afc37 Mon Sep 17 00:00:00 2001 From: Brennan Date: Mon, 4 Mar 2024 13:17:34 -0800 Subject: [PATCH 10/71] update mio to 0.8.11 (#60) --- Cargo.lock | 4 ++-- programs/sbf/Cargo.lock | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a67b7aba0d6b64..78ff40111ee0b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3394,9 +3394,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.8" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", "wasi 0.11.0+wasi-snapshot-preview1", diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index 7546c56bd2b26a..cb0ad6f1ee448c 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -2892,9 +2892,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.8" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", "wasi 0.11.0+wasi-snapshot-preview1", From f94752d5142861f51da04e1354c802c5e548bd17 Mon Sep 17 00:00:00 2001 From: Brooks Date: Mon, 4 Mar 2024 16:32:51 -0500 Subject: [PATCH 11/71] Adds StartingSnapshotStorages to AccountsHashVerifier (#58) --- accounts-db/src/lib.rs | 1 + accounts-db/src/starting_snapshot_storages.rs | 19 +++++ core/src/accounts_hash_verifier.rs | 8 +- core/src/validator.rs | 43 ++++++---- core/tests/epoch_accounts_hash.rs | 2 + core/tests/snapshots.rs | 2 + ledger-tool/src/ledger_utils.rs | 32 +++++--- ledger/src/bank_forks_utils.rs | 81 ++++++++++++++----- 8 files changed, 135 insertions(+), 53 deletions(-) create mode 100644 accounts-db/src/starting_snapshot_storages.rs diff --git a/accounts-db/src/lib.rs b/accounts-db/src/lib.rs index 7883f852d1e3f2..b7994fe4354118 100644 --- a/accounts-db/src/lib.rs +++ b/accounts-db/src/lib.rs @@ -37,6 +37,7 @@ pub mod secondary_index; pub mod shared_buffer_reader; pub mod sorted_storages; pub mod stake_rewards; +pub mod starting_snapshot_storages; pub mod storable_accounts; pub mod tiered_storage; pub mod utils; diff --git a/accounts-db/src/starting_snapshot_storages.rs b/accounts-db/src/starting_snapshot_storages.rs new file mode 100644 index 00000000000000..cc5e26c61872b7 --- /dev/null +++ b/accounts-db/src/starting_snapshot_storages.rs @@ -0,0 +1,19 @@ +use {crate::accounts_db::AccountStorageEntry, std::sync::Arc}; + +/// Snapshot storages that the node loaded from +/// +/// This is used to support fastboot. Since fastboot reuses existing storages, we must carefully +/// handle the storages used to load at startup. If we do not handle these storages properly, +/// restarting from the same local state (i.e. bank snapshot) may fail. +#[derive(Debug)] +pub enum StartingSnapshotStorages { + /// Starting from genesis has no storages yet + Genesis, + /// Starting from a snapshot archive always extracts the storages from the archive, so no + /// special handling is necessary to preserve them. + Archive, + /// Starting from local state must preserve the loaded storages. These storages must *not* be + /// recycled or removed prior to taking the next snapshot, otherwise restarting from the same + /// bank snapshot may fail. + Fastboot(Vec>), +} diff --git a/core/src/accounts_hash_verifier.rs b/core/src/accounts_hash_verifier.rs index 0e427d0675a2b1..f5572d94a3c7d1 100644 --- a/core/src/accounts_hash_verifier.rs +++ b/core/src/accounts_hash_verifier.rs @@ -9,6 +9,7 @@ use { IncrementalAccountsHash, }, sorted_storages::SortedStorages, + starting_snapshot_storages::StartingSnapshotStorages, }, solana_measure::measure_us, solana_runtime::{ @@ -42,6 +43,7 @@ impl AccountsHashVerifier { accounts_package_sender: Sender, accounts_package_receiver: Receiver, snapshot_package_sender: Option>, + starting_snapshot_storages: StartingSnapshotStorages, exit: Arc, snapshot_config: SnapshotConfig, ) -> Self { @@ -54,7 +56,11 @@ impl AccountsHashVerifier { // To support fastboot, we must ensure the storages used in the latest POST snapshot are // not recycled nor removed early. Hold an Arc of their AppendVecs to prevent them from // expiring. - let mut fastboot_storages = None; + let mut fastboot_storages = match starting_snapshot_storages { + StartingSnapshotStorages::Genesis => None, + StartingSnapshotStorages::Archive => None, + StartingSnapshotStorages::Fastboot(storages) => Some(storages), + }; loop { if exit.load(Ordering::Relaxed) { break; diff --git a/core/src/validator.rs b/core/src/validator.rs index a6d5921bcef5c9..196dad5f25d17a 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -35,6 +35,7 @@ use { accounts_index::AccountSecondaryIndexes, accounts_update_notifier_interface::AccountsUpdateNotifier, hardened_unpack::{open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE}, + starting_snapshot_storages::StartingSnapshotStorages, utils::{move_and_async_delete_path, move_and_async_delete_path_contents}, }, solana_client::connection_cache::{ConnectionCache, Protocol}, @@ -690,6 +691,7 @@ impl Validator { completed_slots_receiver, leader_schedule_cache, starting_snapshot_hashes, + starting_snapshot_storages, TransactionHistoryServices { transaction_status_sender, transaction_status_service, @@ -779,6 +781,7 @@ impl Validator { accounts_package_sender.clone(), accounts_package_receiver, snapshot_package_sender, + starting_snapshot_storages, exit.clone(), config.snapshot_config.clone(), ); @@ -1767,6 +1770,7 @@ fn load_blockstore( CompletedSlotsReceiver, LeaderScheduleCache, Option, + StartingSnapshotStorages, TransactionHistoryServices, blockstore_processor::ProcessOptions, BlockstoreRootScan, @@ -1856,23 +1860,27 @@ fn load_blockstore( let entry_notifier_service = entry_notifier .map(|entry_notifier| EntryNotifierService::new(entry_notifier, exit.clone())); - let (bank_forks, mut leader_schedule_cache, starting_snapshot_hashes) = - bank_forks_utils::load_bank_forks( - &genesis_config, - &blockstore, - config.account_paths.clone(), - Some(&config.snapshot_config), - &process_options, - transaction_history_services - .cache_block_meta_sender - .as_ref(), - entry_notifier_service - .as_ref() - .map(|service| service.sender()), - accounts_update_notifier, - exit, - ) - .map_err(|err| err.to_string())?; + let ( + bank_forks, + mut leader_schedule_cache, + starting_snapshot_hashes, + starting_snapshot_storages, + ) = bank_forks_utils::load_bank_forks( + &genesis_config, + &blockstore, + config.account_paths.clone(), + Some(&config.snapshot_config), + &process_options, + transaction_history_services + .cache_block_meta_sender + .as_ref(), + entry_notifier_service + .as_ref() + .map(|service| service.sender()), + accounts_update_notifier, + exit, + ) + .map_err(|err| err.to_string())?; // Before replay starts, set the callbacks in each of the banks in BankForks so that // all dropped banks come through the `pruned_banks_receiver` channel. This way all bank @@ -1898,6 +1906,7 @@ fn load_blockstore( completed_slots_receiver, leader_schedule_cache, starting_snapshot_hashes, + starting_snapshot_storages, transaction_history_services, process_options, blockstore_root_scan, diff --git a/core/tests/epoch_accounts_hash.rs b/core/tests/epoch_accounts_hash.rs index b0dd111676af79..62e31f0a88b766 100755 --- a/core/tests/epoch_accounts_hash.rs +++ b/core/tests/epoch_accounts_hash.rs @@ -9,6 +9,7 @@ use { accounts_hash::CalcAccountsHashConfig, accounts_index::AccountSecondaryIndexes, epoch_accounts_hash::EpochAccountsHash, + starting_snapshot_storages::StartingSnapshotStorages, }, solana_core::{ accounts_hash_verifier::AccountsHashVerifier, @@ -196,6 +197,7 @@ impl BackgroundServices { accounts_package_sender.clone(), accounts_package_receiver, Some(snapshot_package_sender), + StartingSnapshotStorages::Genesis, exit.clone(), snapshot_config.clone(), ); diff --git a/core/tests/snapshots.rs b/core/tests/snapshots.rs index 2694f7294a7217..e67c942f07ab0b 100644 --- a/core/tests/snapshots.rs +++ b/core/tests/snapshots.rs @@ -11,6 +11,7 @@ use { accounts_hash::AccountsHash, accounts_index::AccountSecondaryIndexes, epoch_accounts_hash::EpochAccountsHash, + starting_snapshot_storages::StartingSnapshotStorages, }, solana_core::{ accounts_hash_verifier::AccountsHashVerifier, @@ -1043,6 +1044,7 @@ fn test_snapshots_with_background_services( accounts_package_sender, accounts_package_receiver, Some(snapshot_package_sender), + StartingSnapshotStorages::Genesis, exit.clone(), snapshot_test_config.snapshot_config.clone(), ); diff --git a/ledger-tool/src/ledger_utils.rs b/ledger-tool/src/ledger_utils.rs index c05cc6c2d64cd0..8a8302d7e4e94b 100644 --- a/ledger-tool/src/ledger_utils.rs +++ b/ledger-tool/src/ledger_utils.rs @@ -268,19 +268,24 @@ pub fn load_and_process_ledger( }; let exit = Arc::new(AtomicBool::new(false)); - let (bank_forks, leader_schedule_cache, starting_snapshot_hashes, ..) = - bank_forks_utils::load_bank_forks( - genesis_config, - blockstore.as_ref(), - account_paths, - snapshot_config.as_ref(), - &process_options, - None, - None, // Maybe support this later, though - accounts_update_notifier, - exit.clone(), - ) - .map_err(LoadAndProcessLedgerError::LoadBankForks)?; + let ( + bank_forks, + leader_schedule_cache, + starting_snapshot_hashes, + starting_snapshot_storages, + .., + ) = bank_forks_utils::load_bank_forks( + genesis_config, + blockstore.as_ref(), + account_paths, + snapshot_config.as_ref(), + &process_options, + None, + None, // Maybe support this later, though + accounts_update_notifier, + exit.clone(), + ) + .map_err(LoadAndProcessLedgerError::LoadBankForks)?; let block_verification_method = value_t!( arg_matches, "block_verification_method", @@ -325,6 +330,7 @@ pub fn load_and_process_ledger( accounts_package_sender.clone(), accounts_package_receiver, None, + starting_snapshot_storages, exit.clone(), SnapshotConfig::new_load_only(), ); diff --git a/ledger/src/bank_forks_utils.rs b/ledger/src/bank_forks_utils.rs index 17412c1801ac68..b30f90986bb9c2 100644 --- a/ledger/src/bank_forks_utils.rs +++ b/ledger/src/bank_forks_utils.rs @@ -10,7 +10,10 @@ use { use_snapshot_archives_at_startup::{self, UseSnapshotArchivesAtStartup}, }, log::*, - solana_accounts_db::accounts_update_notifier_interface::AccountsUpdateNotifier, + solana_accounts_db::{ + accounts_update_notifier_interface::AccountsUpdateNotifier, + starting_snapshot_storages::StartingSnapshotStorages, + }, solana_runtime::{ accounts_background_service::AbsRequestSender, bank_forks::BankForks, @@ -67,6 +70,7 @@ pub type LoadResult = result::Result< Arc>, LeaderScheduleCache, Option, + StartingSnapshotStorages, ), BankForksUtilsError, >; @@ -88,7 +92,13 @@ pub fn load( accounts_update_notifier: Option, exit: Arc, ) -> LoadResult { - let (bank_forks, leader_schedule_cache, starting_snapshot_hashes, ..) = load_bank_forks( + let ( + bank_forks, + leader_schedule_cache, + starting_snapshot_hashes, + starting_snapshot_storages, + .., + ) = load_bank_forks( genesis_config, blockstore, account_paths, @@ -111,7 +121,12 @@ pub fn load( ) .map_err(BankForksUtilsError::ProcessBlockstoreFromRoot)?; - Ok((bank_forks, leader_schedule_cache, starting_snapshot_hashes)) + Ok(( + bank_forks, + leader_schedule_cache, + starting_snapshot_hashes, + starting_snapshot_storages, + )) } #[allow(clippy::too_many_arguments)] @@ -161,7 +176,7 @@ pub fn load_bank_forks( )) } - let (bank_forks, starting_snapshot_hashes) = + let (bank_forks, starting_snapshot_hashes, starting_snapshot_storages) = if let Some((full_snapshot_archive_info, incremental_snapshot_archive_info)) = get_snapshots_to_load(snapshot_config) { @@ -173,17 +188,22 @@ pub fn load_bank_forks( ); std::fs::create_dir_all(&snapshot_config.bank_snapshots_dir) .expect("create bank snapshots dir"); - let (bank_forks, starting_snapshot_hashes) = bank_forks_from_snapshot( - full_snapshot_archive_info, - incremental_snapshot_archive_info, - genesis_config, - account_paths, - snapshot_config, - process_options, - accounts_update_notifier, - exit, - )?; - (bank_forks, Some(starting_snapshot_hashes)) + let (bank_forks, starting_snapshot_hashes, starting_snapshot_storages) = + bank_forks_from_snapshot( + full_snapshot_archive_info, + incremental_snapshot_archive_info, + genesis_config, + account_paths, + snapshot_config, + process_options, + accounts_update_notifier, + exit, + )?; + ( + bank_forks, + Some(starting_snapshot_hashes), + starting_snapshot_storages, + ) } else { info!("Processing ledger from genesis"); let bank_forks = blockstore_processor::process_blockstore_for_bank_0( @@ -202,7 +222,7 @@ pub fn load_bank_forks( .root_bank() .set_startup_verification_complete(); - (bank_forks, None) + (bank_forks, None, StartingSnapshotStorages::Genesis) }; let mut leader_schedule_cache = @@ -218,7 +238,12 @@ pub fn load_bank_forks( .for_each(|hard_fork_slot| root_bank.register_hard_fork(*hard_fork_slot)); } - Ok((bank_forks, leader_schedule_cache, starting_snapshot_hashes)) + Ok(( + bank_forks, + leader_schedule_cache, + starting_snapshot_hashes, + starting_snapshot_storages, + )) } #[allow(clippy::too_many_arguments)] @@ -231,7 +256,14 @@ fn bank_forks_from_snapshot( process_options: &ProcessOptions, accounts_update_notifier: Option, exit: Arc, -) -> Result<(Arc>, StartingSnapshotHashes), BankForksUtilsError> { +) -> Result< + ( + Arc>, + StartingSnapshotHashes, + StartingSnapshotStorages, + ), + BankForksUtilsError, +> { // Fail hard here if snapshot fails to load, don't silently continue if account_paths.is_empty() { return Err(BankForksUtilsError::AccountPathsNotPresent); @@ -257,7 +289,7 @@ fn bank_forks_from_snapshot( .unwrap_or(true), }; - let bank = if will_startup_from_snapshot_archives { + let (bank, starting_snapshot_storages) = if will_startup_from_snapshot_archives { // Given that we are going to boot from an archive, the append vecs held in the snapshot dirs for fast-boot should // be released. They will be released by the account_background_service anyway. But in the case of the account_paths // using memory-mounted file system, they are not released early enough to give space for the new append-vecs from @@ -292,7 +324,7 @@ fn bank_forks_from_snapshot( .map(|archive| archive.path().display().to_string()) .unwrap_or("none".to_string()), })?; - bank + (bank, StartingSnapshotStorages::Archive) } else { let bank_snapshot = latest_bank_snapshot.ok_or_else(|| BankForksUtilsError::NoBankSnapshotDirectory { @@ -346,7 +378,8 @@ fn bank_forks_from_snapshot( // snapshot archive next time, which is safe. snapshot_utils::purge_all_bank_snapshots(&snapshot_config.bank_snapshots_dir); - bank + let storages = bank.get_snapshot_storages(None); + (bank, StartingSnapshotStorages::Fastboot(storages)) }; let full_snapshot_hash = FullSnapshotHash(( @@ -365,5 +398,9 @@ fn bank_forks_from_snapshot( incremental: incremental_snapshot_hash, }; - Ok((BankForks::new_rw_arc(bank), starting_snapshot_hashes)) + Ok(( + BankForks::new_rw_arc(bank), + starting_snapshot_hashes, + starting_snapshot_storages, + )) } From 94ed7a31e9a0acd6d2fbf2885aeb6194e8f603ab Mon Sep 17 00:00:00 2001 From: Ashwin Sekar Date: Mon, 4 Mar 2024 17:16:31 -0800 Subject: [PATCH 12/71] blockstore_purge: fix inspect -> inspect_err (#66) --- ledger/src/blockstore/blockstore_purge.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ledger/src/blockstore/blockstore_purge.rs b/ledger/src/blockstore/blockstore_purge.rs index 15a5c4890e9f05..d8b4c7424cd8c1 100644 --- a/ledger/src/blockstore/blockstore_purge.rs +++ b/ledger/src/blockstore/blockstore_purge.rs @@ -213,7 +213,7 @@ impl Blockstore { delete_range_timer.stop(); let mut write_timer = Measure::start("write_batch"); - self.db.write(write_batch).inspect(|e| { + self.db.write(write_batch).inspect_err(|e| { error!( "Error: {:?} while submitting write batch for purge from_slot {} to_slot {}", e, from_slot, to_slot From 5efc4eb7abc6ff1473e333cca8555b9bc0aefaeb Mon Sep 17 00:00:00 2001 From: Tyera Date: Mon, 4 Mar 2024 19:21:30 -0700 Subject: [PATCH 13/71] Rpc: deprecate `getStakeActivation` and make inactive_stake consistent (#69) * Make inactive_stake consistent * Add rpc_deprecated_v1_18 module * Move get_stake_activation to deprecated list * Fix typo --- rpc/src/rpc.rs | 74 +++++++++++++++++++++++------------------- rpc/src/rpc_service.rs | 6 ++-- 2 files changed, 45 insertions(+), 35 deletions(-) diff --git a/rpc/src/rpc.rs b/rpc/src/rpc.rs index 01f623dccdc108..41b26e5fa1e2c2 100644 --- a/rpc/src/rpc.rs +++ b/rpc/src/rpc.rs @@ -1786,16 +1786,10 @@ impl JsonRpcRequestProcessor { } else { StakeActivationState::Inactive }; - let inactive_stake = match stake_activation_state { - StakeActivationState::Activating => activating, - StakeActivationState::Active => 0, - StakeActivationState::Deactivating => stake_account - .lamports() - .saturating_sub(effective + rent_exempt_reserve), - StakeActivationState::Inactive => { - stake_account.lamports().saturating_sub(rent_exempt_reserve) - } - }; + let inactive_stake = stake_account + .lamports() + .saturating_sub(effective) + .saturating_sub(rent_exempt_reserve); Ok(RpcStakeActivation { state: stake_activation_state, active: effective, @@ -2991,14 +2985,6 @@ pub mod rpc_accounts { block: Slot, ) -> Result>; - #[rpc(meta, name = "getStakeActivation")] - fn get_stake_activation( - &self, - meta: Self::Metadata, - pubkey_str: String, - config: Option, - ) -> Result; - // SPL Token-specific RPC endpoints // See https://github.com/solana-labs/solana-program-library/releases/tag/token-v2.0.0 for // program details @@ -3071,20 +3057,6 @@ pub mod rpc_accounts { Ok(meta.get_block_commitment(block)) } - fn get_stake_activation( - &self, - meta: Self::Metadata, - pubkey_str: String, - config: Option, - ) -> Result { - debug!( - "get_stake_activation rpc request received: {:?}", - pubkey_str - ); - let pubkey = verify_pubkey(&pubkey_str)?; - meta.get_stake_activation(&pubkey, config) - } - fn get_token_account_balance( &self, meta: Self::Metadata, @@ -4091,7 +4063,43 @@ fn rpc_perf_sample_from_perf_sample(slot: u64, sample: PerfSample) -> RpcPerfSam } } -// RPC methods deprecated in v1.8 +pub mod rpc_deprecated_v1_18 { + use super::*; + #[rpc] + pub trait DeprecatedV1_18 { + type Metadata; + + // DEPRECATED + #[rpc(meta, name = "getStakeActivation")] + fn get_stake_activation( + &self, + meta: Self::Metadata, + pubkey_str: String, + config: Option, + ) -> Result; + } + + pub struct DeprecatedV1_18Impl; + impl DeprecatedV1_18 for DeprecatedV1_18Impl { + type Metadata = JsonRpcRequestProcessor; + + fn get_stake_activation( + &self, + meta: Self::Metadata, + pubkey_str: String, + config: Option, + ) -> Result { + debug!( + "get_stake_activation rpc request received: {:?}", + pubkey_str + ); + let pubkey = verify_pubkey(&pubkey_str)?; + meta.get_stake_activation(&pubkey, config) + } + } +} + +// RPC methods deprecated in v1.9 pub mod rpc_deprecated_v1_9 { #![allow(deprecated)] use super::*; diff --git a/rpc/src/rpc_service.rs b/rpc/src/rpc_service.rs index 8597394f102325..d8791ab6c3bf6b 100644 --- a/rpc/src/rpc_service.rs +++ b/rpc/src/rpc_service.rs @@ -6,8 +6,9 @@ use { max_slots::MaxSlots, optimistically_confirmed_bank_tracker::OptimisticallyConfirmedBank, rpc::{ - rpc_accounts::*, rpc_accounts_scan::*, rpc_bank::*, rpc_deprecated_v1_7::*, - rpc_deprecated_v1_9::*, rpc_full::*, rpc_minimal::*, rpc_obsolete_v1_7::*, *, + rpc_accounts::*, rpc_accounts_scan::*, rpc_bank::*, rpc_deprecated_v1_18::*, + rpc_deprecated_v1_7::*, rpc_deprecated_v1_9::*, rpc_full::*, rpc_minimal::*, + rpc_obsolete_v1_7::*, *, }, rpc_cache::LargestAccountsCache, rpc_health::*, @@ -510,6 +511,7 @@ impl JsonRpcService { io.extend_with(rpc_full::FullImpl.to_delegate()); io.extend_with(rpc_deprecated_v1_7::DeprecatedV1_7Impl.to_delegate()); io.extend_with(rpc_deprecated_v1_9::DeprecatedV1_9Impl.to_delegate()); + io.extend_with(rpc_deprecated_v1_18::DeprecatedV1_18Impl.to_delegate()); } if obsolete_v1_7_api { io.extend_with(rpc_obsolete_v1_7::ObsoleteV1_7Impl.to_delegate()); From 0eb0e7cfcc822739bc9f66de56a37fa8b0be1222 Mon Sep 17 00:00:00 2001 From: Jon C Date: Tue, 5 Mar 2024 12:28:12 +0100 Subject: [PATCH 14/71] windows: Use vcpkg for openssl dep (#73) --- .github/workflows/release-artifacts.yml | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 45be181c3ce9e1..7aec77f0dac45f 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -43,17 +43,8 @@ jobs: id: build shell: bash run: | - choco install openssl --version=3.1.1 - if [[ -d "C:\Program Files\OpenSSL" ]]; then - echo "OPENSSL_DIR: C:\Program Files\OpenSSL" - export OPENSSL_DIR="C:\Program Files\OpenSSL" - elif [[ -d "C:\Program Files\OpenSSL-Win64" ]]; then - echo "OPENSSL_DIR: C:\Program Files\OpenSSL-Win64" - export OPENSSL_DIR="C:\Program Files\OpenSSL-Win64" - else - echo "can't determine OPENSSL_DIR" - exit 1 - fi + vcpkg install openssl:x64-windows-static-md + vcpkg integrate install choco install protoc export PROTOC="C:\ProgramData\chocolatey\lib\protoc\tools\bin\protoc.exe" source /tmp/env.sh From df57657985b458b10c61eeeffd6ecff86f929819 Mon Sep 17 00:00:00 2001 From: Will Hickey Date: Tue, 5 Mar 2024 10:18:50 -0600 Subject: [PATCH 15/71] Revert "[anza migration] rename crates (#10)" This reverts commit 3f9a7a52eac0da3a8feb48fd83c939b222c66bc4. --- .../scripts/downstream-project-spl-common.sh | 3 - .github/workflows/release-artifacts.yml | 2 +- Cargo.lock | 424 +++++++++--------- Cargo.toml | 4 +- cargo-registry/Cargo.toml | 4 +- ci/localnet-sanity.sh | 4 +- ci/publish-installer.sh | 2 +- ci/publish-tarball.sh | 4 +- ci/run-sanity.sh | 2 +- docs/src/backwards-compatibility.md | 2 +- docs/src/cli/install.md | 12 +- docs/src/clusters/available.md | 12 +- docs/src/clusters/benchmark.md | 2 +- docs/src/implemented-proposals/installer.md | 58 +-- .../rpc-transaction-history.md | 2 +- docs/src/operations/best-practices/general.md | 28 +- .../operations/best-practices/monitoring.md | 28 +- docs/src/operations/guides/restart-cluster.md | 14 +- .../operations/guides/validator-failover.md | 12 +- docs/src/operations/guides/validator-start.md | 32 +- docs/src/operations/guides/vote-accounts.md | 8 +- docs/src/operations/setup-a-validator.md | 24 +- docs/src/operations/setup-an-rpc-node.md | 4 +- docs/src/validator/geyser.md | 8 +- geyser-plugin-interface/Cargo.toml | 4 +- .../src/geyser_plugin_interface.rs | 2 +- geyser-plugin-manager/Cargo.toml | 2 +- .../src/accounts_update_notifier.rs | 6 +- .../src/block_metadata_notifier.rs | 4 +- geyser-plugin-manager/src/entry_notifier.rs | 6 +- .../src/geyser_plugin_manager.rs | 4 +- .../src/slot_status_notifier.rs | 2 +- .../src/transaction_notifier.rs | 4 +- install/Cargo.toml | 4 +- install/install-help.sh | 6 +- ...install-init.sh => solana-install-init.sh} | 16 +- ...install-init.rs => solana-install-init.rs} | 2 +- install/src/command.rs | 4 +- install/src/lib.rs | 2 +- install/src/main.rs | 2 +- ledger-tool/Cargo.toml | 4 +- ledger-tool/src/blockstore.rs | 2 +- ledger-tool/src/ledger_utils.rs | 8 +- ledger/src/blockstore_db.rs | 2 +- local-cluster/tests/local_cluster.rs | 4 +- multinode-demo/bootstrap-validator.sh | 4 +- multinode-demo/common.sh | 6 +- multinode-demo/validator.sh | 6 +- net/net.sh | 8 +- net/remote/remote-deploy-update.sh | 2 +- net/remote/remote-node.sh | 8 +- net/remote/remote-sanity.sh | 4 +- notifier/src/lib.rs | 4 +- programs/sbf/Cargo.lock | 152 +++---- programs/sbf/Cargo.toml | 2 +- programs/sbf/rust/simulation/Cargo.toml | 2 +- .../sbf/rust/simulation/tests/validator.rs | 2 +- .../src/nonblocking/pubsub_client.rs | 6 +- pubsub-client/src/pubsub_client.rs | 6 +- rbpf-cli/src/main.rs | 4 +- rpc/src/rpc.rs | 8 +- scripts/cargo-install-all.sh | 14 +- scripts/check-dev-context-only-utils.sh | 2 +- scripts/run.sh | 6 +- ...all-deploy.sh => solana-install-deploy.sh} | 4 +- .../abi-testcases/mixed-validator-test.sh | 6 +- .../stability-testcases/gossip-dos-test.sh | 6 +- validator/Cargo.toml | 8 +- validator/src/bin/solana-test-validator.rs | 8 +- validator/src/bootstrap.rs | 2 +- validator/src/main.rs | 20 +- watchtower/Cargo.toml | 4 +- watchtower/README.md | 2 +- watchtower/src/main.rs | 10 +- 74 files changed, 547 insertions(+), 554 deletions(-) rename install/{agave-install-init.sh => solana-install-init.sh} (89%) rename install/src/bin/{agave-install-init.rs => solana-install-init.rs} (92%) rename scripts/{agave-install-deploy.sh => solana-install-deploy.sh} (90%) diff --git a/.github/scripts/downstream-project-spl-common.sh b/.github/scripts/downstream-project-spl-common.sh index 861be12c7d1a45..c6dcfaca007867 100644 --- a/.github/scripts/downstream-project-spl-common.sh +++ b/.github/scripts/downstream-project-spl-common.sh @@ -22,6 +22,3 @@ if semverGT "$project_used_solana_version" "$SOLANA_VER"; then fi ./patch.crates-io.sh "$SOLANA_DIR" - -# anza migration stopgap. can be removed when agave is fully recommended for public usage. -sed -i 's/solana-geyser-plugin-interface/agave-geyser-plugin-interface/g' ./Cargo.toml diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 7aec77f0dac45f..f32bd22eed8c82 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -61,7 +61,7 @@ jobs: mkdir -p "windows-release/$FOLDER_NAME" cp -v "solana-release-x86_64-pc-windows-msvc.tar.bz2" "windows-release/$FOLDER_NAME/" cp -v "solana-release-x86_64-pc-windows-msvc.yml" "windows-release/$FOLDER_NAME/" - cp -v "agave-install-init-x86_64-pc-windows-msvc"* "windows-release/$FOLDER_NAME" + cp -v "solana-install-init-x86_64-pc-windows-msvc"* "windows-release/$FOLDER_NAME" - name: Upload Artifacts if: ${{ steps.build.outputs.channel != '' || steps.build.outputs.tag != '' }} diff --git a/Cargo.lock b/Cargo.lock index 78ff40111ee0b0..528c7d00f33d04 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -62,217 +62,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "agave-cargo-registry" -version = "1.19.0" -dependencies = [ - "clap 2.33.3", - "flate2", - "hex", - "hyper", - "log", - "rustc_version 0.4.0", - "serde", - "serde_json", - "sha2 0.10.8", - "solana-clap-utils", - "solana-cli", - "solana-cli-config", - "solana-cli-output", - "solana-logger", - "solana-remote-wallet", - "solana-rpc-client", - "solana-rpc-client-api", - "solana-sdk", - "solana-version", - "tar", - "tempfile", - "tokio", - "toml 0.8.10", -] - -[[package]] -name = "agave-geyser-plugin-interface" -version = "1.19.0" -dependencies = [ - "log", - "solana-sdk", - "solana-transaction-status", - "thiserror", -] - -[[package]] -name = "agave-install" -version = "1.19.0" -dependencies = [ - "atty", - "bincode", - "bzip2", - "chrono", - "clap 2.33.3", - "console", - "crossbeam-channel", - "ctrlc", - "dirs-next", - "indicatif", - "lazy_static", - "nix 0.26.4", - "reqwest", - "scopeguard", - "semver 1.0.22", - "serde", - "serde_yaml 0.8.26", - "serde_yaml 0.9.32", - "solana-clap-utils", - "solana-config-program", - "solana-logger", - "solana-rpc-client", - "solana-sdk", - "solana-version", - "tar", - "tempfile", - "url 2.5.0", - "winapi 0.3.9", - "winreg", -] - -[[package]] -name = "agave-ledger-tool" -version = "1.19.0" -dependencies = [ - "assert_cmd", - "bs58", - "bytecount", - "chrono", - "clap 2.33.3", - "crossbeam-channel", - "csv", - "dashmap", - "futures 0.3.30", - "histogram", - "itertools", - "log", - "num_cpus", - "regex", - "serde", - "serde_json", - "signal-hook", - "solana-account-decoder", - "solana-accounts-db", - "solana-bpf-loader-program", - "solana-clap-utils", - "solana-cli-output", - "solana-core", - "solana-cost-model", - "solana-entry", - "solana-geyser-plugin-manager", - "solana-gossip", - "solana-ledger", - "solana-logger", - "solana-measure", - "solana-program-runtime", - "solana-rpc", - "solana-runtime", - "solana-sdk", - "solana-stake-program", - "solana-storage-bigtable", - "solana-streamer", - "solana-svm", - "solana-transaction-status", - "solana-unified-scheduler-pool", - "solana-version", - "solana-vote-program", - "solana_rbpf", - "thiserror", - "tikv-jemallocator", - "tokio", -] - -[[package]] -name = "agave-validator" -version = "1.19.0" -dependencies = [ - "agave-geyser-plugin-interface", - "chrono", - "clap 2.33.3", - "console", - "core_affinity", - "crossbeam-channel", - "fd-lock", - "indicatif", - "itertools", - "jsonrpc-core", - "jsonrpc-core-client", - "jsonrpc-derive", - "jsonrpc-ipc-server", - "jsonrpc-server-utils", - "lazy_static", - "libc", - "libloading", - "log", - "num_cpus", - "rand 0.8.5", - "rayon", - "serde", - "serde_json", - "serde_yaml 0.9.32", - "signal-hook", - "solana-account-decoder", - "solana-accounts-db", - "solana-clap-utils", - "solana-cli-config", - "solana-core", - "solana-download-utils", - "solana-entry", - "solana-faucet", - "solana-genesis-utils", - "solana-geyser-plugin-manager", - "solana-gossip", - "solana-ledger", - "solana-logger", - "solana-metrics", - "solana-net-utils", - "solana-perf", - "solana-poh", - "solana-rpc", - "solana-rpc-client", - "solana-rpc-client-api", - "solana-runtime", - "solana-sdk", - "solana-send-transaction-service", - "solana-storage-bigtable", - "solana-streamer", - "solana-svm", - "solana-test-validator", - "solana-tpu-client", - "solana-unified-scheduler-pool", - "solana-version", - "solana-vote-program", - "spl-token-2022", - "symlink", - "thiserror", - "tikv-jemallocator", -] - -[[package]] -name = "agave-watchtower" -version = "1.19.0" -dependencies = [ - "clap 2.33.3", - "humantime", - "log", - "solana-clap-utils", - "solana-cli-config", - "solana-cli-output", - "solana-logger", - "solana-metrics", - "solana-notifier", - "solana-rpc-client", - "solana-rpc-client-api", - "solana-sdk", - "solana-version", -] - [[package]] name = "ahash" version = "0.7.6" @@ -5694,6 +5483,35 @@ dependencies = [ "tar", ] +[[package]] +name = "solana-cargo-registry" +version = "1.19.0" +dependencies = [ + "clap 2.33.3", + "flate2", + "hex", + "hyper", + "log", + "rustc_version 0.4.0", + "serde", + "serde_json", + "sha2 0.10.8", + "solana-clap-utils", + "solana-cli", + "solana-cli-config", + "solana-cli-output", + "solana-logger", + "solana-remote-wallet", + "solana-rpc-client", + "solana-rpc-client-api", + "solana-sdk", + "solana-version", + "tar", + "tempfile", + "tokio", + "toml 0.8.10", +] + [[package]] name = "solana-cargo-test-bpf" version = "1.19.0" @@ -6223,11 +6041,20 @@ dependencies = [ "solana-sdk", ] +[[package]] +name = "solana-geyser-plugin-interface" +version = "1.19.0" +dependencies = [ + "log", + "solana-sdk", + "solana-transaction-status", + "thiserror", +] + [[package]] name = "solana-geyser-plugin-manager" version = "1.19.0" dependencies = [ - "agave-geyser-plugin-interface", "bs58", "crossbeam-channel", "json5", @@ -6238,6 +6065,7 @@ dependencies = [ "serde_json", "solana-accounts-db", "solana-entry", + "solana-geyser-plugin-interface", "solana-ledger", "solana-measure", "solana-metrics", @@ -6299,6 +6127,41 @@ dependencies = [ "thiserror", ] +[[package]] +name = "solana-install" +version = "1.19.0" +dependencies = [ + "atty", + "bincode", + "bzip2", + "chrono", + "clap 2.33.3", + "console", + "crossbeam-channel", + "ctrlc", + "dirs-next", + "indicatif", + "lazy_static", + "nix 0.26.4", + "reqwest", + "scopeguard", + "semver 1.0.22", + "serde", + "serde_yaml 0.8.26", + "serde_yaml 0.9.32", + "solana-clap-utils", + "solana-config-program", + "solana-logger", + "solana-rpc-client", + "solana-sdk", + "solana-version", + "tar", + "tempfile", + "url 2.5.0", + "winapi 0.3.9", + "winreg", +] + [[package]] name = "solana-keygen" version = "1.19.0" @@ -6386,6 +6249,58 @@ dependencies = [ "trees", ] +[[package]] +name = "solana-ledger-tool" +version = "1.19.0" +dependencies = [ + "assert_cmd", + "bs58", + "bytecount", + "chrono", + "clap 2.33.3", + "crossbeam-channel", + "csv", + "dashmap", + "futures 0.3.30", + "histogram", + "itertools", + "log", + "num_cpus", + "regex", + "serde", + "serde_json", + "signal-hook", + "solana-account-decoder", + "solana-accounts-db", + "solana-bpf-loader-program", + "solana-clap-utils", + "solana-cli-output", + "solana-core", + "solana-cost-model", + "solana-entry", + "solana-geyser-plugin-manager", + "solana-gossip", + "solana-ledger", + "solana-logger", + "solana-measure", + "solana-program-runtime", + "solana-rpc", + "solana-runtime", + "solana-sdk", + "solana-stake-program", + "solana-storage-bigtable", + "solana-streamer", + "solana-svm", + "solana-transaction-status", + "solana-unified-scheduler-pool", + "solana-version", + "solana-vote-program", + "solana_rbpf", + "thiserror", + "tikv-jemallocator", + "tokio", +] + [[package]] name = "solana-loader-v4-program" version = "1.19.0" @@ -7549,6 +7464,72 @@ dependencies = [ "solana-metrics", ] +[[package]] +name = "solana-validator" +version = "1.19.0" +dependencies = [ + "chrono", + "clap 2.33.3", + "console", + "core_affinity", + "crossbeam-channel", + "fd-lock", + "indicatif", + "itertools", + "jsonrpc-core", + "jsonrpc-core-client", + "jsonrpc-derive", + "jsonrpc-ipc-server", + "jsonrpc-server-utils", + "lazy_static", + "libc", + "libloading", + "log", + "num_cpus", + "rand 0.8.5", + "rayon", + "serde", + "serde_json", + "serde_yaml 0.9.32", + "signal-hook", + "solana-account-decoder", + "solana-accounts-db", + "solana-clap-utils", + "solana-cli-config", + "solana-core", + "solana-download-utils", + "solana-entry", + "solana-faucet", + "solana-genesis-utils", + "solana-geyser-plugin-interface", + "solana-geyser-plugin-manager", + "solana-gossip", + "solana-ledger", + "solana-logger", + "solana-metrics", + "solana-net-utils", + "solana-perf", + "solana-poh", + "solana-rpc", + "solana-rpc-client", + "solana-rpc-client-api", + "solana-runtime", + "solana-sdk", + "solana-send-transaction-service", + "solana-storage-bigtable", + "solana-streamer", + "solana-svm", + "solana-test-validator", + "solana-tpu-client", + "solana-unified-scheduler-pool", + "solana-version", + "solana-vote-program", + "spl-token-2022", + "symlink", + "thiserror", + "tikv-jemallocator", +] + [[package]] name = "solana-version" version = "1.19.0" @@ -7605,6 +7586,25 @@ dependencies = [ "thiserror", ] +[[package]] +name = "solana-watchtower" +version = "1.19.0" +dependencies = [ + "clap 2.33.3", + "humantime", + "log", + "solana-clap-utils", + "solana-cli-config", + "solana-cli-output", + "solana-logger", + "solana-metrics", + "solana-notifier", + "solana-rpc-client", + "solana-rpc-client-api", + "solana-sdk", + "solana-version", +] + [[package]] name = "solana-wen-restart" version = "1.19.0" diff --git a/Cargo.toml b/Cargo.toml index 27376370297e26..66436c9cfb3fd8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -317,7 +317,7 @@ solana-bench-tps = { path = "bench-tps", version = "=1.19.0" } solana-bloom = { path = "bloom", version = "=1.19.0" } solana-bpf-loader-program = { path = "programs/bpf_loader", version = "=1.19.0" } solana-bucket-map = { path = "bucket_map", version = "=1.19.0" } -agave-cargo-registry = { path = "cargo-registry", version = "=1.19.0" } +solana-cargo-registry = { path = "cargo-registry", version = "=1.19.0" } solana-clap-utils = { path = "clap-utils", version = "=1.19.0" } solana-clap-v3-utils = { path = "clap-v3-utils", version = "=1.19.0" } solana-cli = { path = "cli", version = "=1.19.0" } @@ -336,7 +336,7 @@ solana-frozen-abi = { path = "frozen-abi", version = "=1.19.0" } solana-frozen-abi-macro = { path = "frozen-abi/macro", version = "=1.19.0" } solana-genesis = { path = "genesis", version = "=1.19.0" } solana-genesis-utils = { path = "genesis-utils", version = "=1.19.0" } -agave-geyser-plugin-interface = { path = "geyser-plugin-interface", version = "=1.19.0" } +solana-geyser-plugin-interface = { path = "geyser-plugin-interface", version = "=1.19.0" } solana-geyser-plugin-manager = { path = "geyser-plugin-manager", version = "=1.19.0" } solana-gossip = { path = "gossip", version = "=1.19.0" } solana-ledger = { path = "ledger", version = "=1.19.0" } diff --git a/cargo-registry/Cargo.toml b/cargo-registry/Cargo.toml index 395493a8e85f00..4e13f477ee5e2d 100644 --- a/cargo-registry/Cargo.toml +++ b/cargo-registry/Cargo.toml @@ -1,7 +1,7 @@ [package] -name = "agave-cargo-registry" +name = "solana-cargo-registry" description = "Solana cargo registry" -documentation = "https://docs.rs/agave-cargo-registry" +documentation = "https://docs.rs/solana-cargo-registry" version = { workspace = true } authors = { workspace = true } repository = { workspace = true } diff --git a/ci/localnet-sanity.sh b/ci/localnet-sanity.sh index b01eca31d50d81..e6734e180aa2da 100755 --- a/ci/localnet-sanity.sh +++ b/ci/localnet-sanity.sh @@ -202,8 +202,8 @@ killNodes() { # Try to use the RPC exit API to cleanly exit the first two nodes # (dynamic nodes, -x, are just killed) echo "--- RPC exit" - $agave_validator --ledger "$SOLANA_CONFIG_DIR"/bootstrap-validator exit --force || true - $agave_validator --ledger "$SOLANA_CONFIG_DIR"/validator exit --force || true + $solana_validator --ledger "$SOLANA_CONFIG_DIR"/bootstrap-validator exit --force || true + $solana_validator --ledger "$SOLANA_CONFIG_DIR"/validator exit --force || true # Give the nodes a splash of time to cleanly exit before killing them sleep 2 diff --git a/ci/publish-installer.sh b/ci/publish-installer.sh index f7d98ffd5ddcf9..e58fd939dd1a40 100755 --- a/ci/publish-installer.sh +++ b/ci/publish-installer.sh @@ -31,7 +31,7 @@ SOLANA_RELEASE=$CHANNEL_OR_TAG SOLANA_INSTALL_INIT_ARGS=$CHANNEL_OR_TAG SOLANA_DOWNLOAD_ROOT=https://release.anza.xyz EOF -cat install/agave-install-init.sh >>release.anza.xyz-install +cat install/solana-install-init.sh >>release.anza.xyz-install echo --- GCS: "install" upload-gcs-artifact "/solana/release.anza.xyz-install" "gs://anza-release/$CHANNEL_OR_TAG/install" diff --git a/ci/publish-tarball.sh b/ci/publish-tarball.sh index da5862fb3de1d2..5c64f09564fe9f 100755 --- a/ci/publish-tarball.sh +++ b/ci/publish-tarball.sh @@ -93,7 +93,7 @@ echo --- Creating release tarball tar cvf "${TARBALL_BASENAME}"-$TARGET.tar "${RELEASE_BASENAME}" bzip2 "${TARBALL_BASENAME}"-$TARGET.tar - cp "${RELEASE_BASENAME}"/bin/agave-install-init agave-install-init-$TARGET + cp "${RELEASE_BASENAME}"/bin/solana-install-init solana-install-init-$TARGET cp "${RELEASE_BASENAME}"/version.yml "${TARBALL_BASENAME}"-$TARGET.yml ) @@ -110,7 +110,7 @@ fi source ci/upload-ci-artifact.sh -for file in "${TARBALL_BASENAME}"-$TARGET.tar.bz2 "${TARBALL_BASENAME}"-$TARGET.yml agave-install-init-"$TARGET"* $MAYBE_TARBALLS; do +for file in "${TARBALL_BASENAME}"-$TARGET.tar.bz2 "${TARBALL_BASENAME}"-$TARGET.yml solana-install-init-"$TARGET"* $MAYBE_TARBALLS; do if [[ -n $DO_NOT_PUBLISH_TAR ]]; then upload-ci-artifact "$file" echo "Skipped $file due to DO_NOT_PUBLISH_TAR" diff --git a/ci/run-sanity.sh b/ci/run-sanity.sh index 88a6f40b1adf28..8108d13a061fd5 100755 --- a/ci/run-sanity.sh +++ b/ci/run-sanity.sh @@ -31,7 +31,7 @@ while [[ $latest_slot -le $((snapshot_slot + 1)) ]]; do latest_slot=$($solana_cli --url http://localhost:8899 slot --commitment processed) done -$agave_validator --ledger config/ledger exit --force || true +$solana_validator --ledger config/ledger exit --force || true wait $pid diff --git a/docs/src/backwards-compatibility.md b/docs/src/backwards-compatibility.md index 0fdc388ea2dbae..4a3c60b8e129bd 100644 --- a/docs/src/backwards-compatibility.md +++ b/docs/src/backwards-compatibility.md @@ -76,7 +76,7 @@ Major releases: - [`solana-program`](https://docs.rs/solana-program/) - Rust SDK for writing programs - [`solana-client`](https://docs.rs/solana-client/) - Rust client for connecting to RPC API - [`solana-cli-config`](https://docs.rs/solana-cli-config/) - Rust client for managing Solana CLI config files -- [`agave-geyser-plugin-interface`](https://docs.rs/agave-geyser-plugin-interface/) - Rust interface for developing Solana Geyser plugins. +- [`solana-geyser-plugin-interface`](https://docs.rs/solana-geyser-plugin-interface/) - Rust interface for developing Solana Geyser plugins. Patch releases: diff --git a/docs/src/cli/install.md b/docs/src/cli/install.md index 20f6516314fb02..3667c733e3f4d4 100644 --- a/docs/src/cli/install.md +++ b/docs/src/cli/install.md @@ -56,7 +56,7 @@ Please update your PATH environment variable to include the solana programs: solana --version ``` -- After a successful install, `agave-install update` may be used to easily +- After a successful install, `solana-install update` may be used to easily update the Solana software to a newer version at any time. --- @@ -74,7 +74,7 @@ solana --version installer into a temporary directory: ```bash -cmd /c "curl https://release.solana.com/LATEST_SOLANA_RELEASE_VERSION/agave-install-init-x86_64-pc-windows-msvc.exe --output C:\agave-install-tmp\agave-install-init.exe --create-dirs" +cmd /c "curl https://release.solana.com/LATEST_SOLANA_RELEASE_VERSION/solana-install-init-x86_64-pc-windows-msvc.exe --output C:\solana-install-tmp\solana-install-init.exe --create-dirs" ``` - Copy and paste the following command, then press Enter to install the latest @@ -82,7 +82,7 @@ cmd /c "curl https://release.solana.com/LATEST_SOLANA_RELEASE_VERSION/agave-inst to allow the program to run. ```bash -C:\agave-install-tmp\agave-install-init.exe LATEST_SOLANA_RELEASE_VERSION +C:\solana-install-tmp\solana-install-init.exe LATEST_SOLANA_RELEASE_VERSION ``` - When the installer is finished, press Enter. @@ -97,12 +97,12 @@ C:\agave-install-tmp\agave-install-init.exe LATEST_SOLANA_RELEASE_VERSION solana --version ``` -- After a successful install, `agave-install update` may be used to easily +- After a successful install, `solana-install update` may be used to easily update the Solana software to a newer version at any time. ## Download Prebuilt Binaries -If you would rather not use `agave-install` to manage the install, you can +If you would rather not use `solana-install` to manage the install, you can manually download and install the binaries. ### Linux @@ -255,7 +255,7 @@ You can then run the following command to obtain the same result as with prebuilt binaries: ```bash -agave-install init +solana-install init ``` ## Use Homebrew diff --git a/docs/src/clusters/available.md b/docs/src/clusters/available.md index 52a7d469ad0cc5..dfbca41672b499 100644 --- a/docs/src/clusters/available.md +++ b/docs/src/clusters/available.md @@ -41,10 +41,10 @@ export SOLANA_METRICS_CONFIG="host=https://metrics.solana.com:8086,db=devnet,u=s solana config set --url https://api.devnet.solana.com ``` -##### Example `agave-validator` command-line +##### Example `solana-validator` command-line ```bash -$ agave-validator \ +$ solana-validator \ --identity validator-keypair.json \ --vote-account vote-account-keypair.json \ --known-validator dv1ZAGvdsz5hHLwWXsVnM94hWf1pjbKVau1QVkaMJ92 \ @@ -93,10 +93,10 @@ export SOLANA_METRICS_CONFIG="host=https://metrics.solana.com:8086,db=tds,u=test solana config set --url https://api.testnet.solana.com ``` -##### Example `agave-validator` command-line +##### Example `solana-validator` command-line ```bash -$ agave-validator \ +$ solana-validator \ --identity validator-keypair.json \ --vote-account vote-account-keypair.json \ --known-validator 5D1fNXzvv5NjV1ysLjirC4WY92RNsVH18vjmcszZd8on \ @@ -145,10 +145,10 @@ export SOLANA_METRICS_CONFIG="host=https://metrics.solana.com:8086,db=mainnet-be solana config set --url https://api.mainnet-beta.solana.com ``` -##### Example `agave-validator` command-line +##### Example `solana-validator` command-line ```bash -$ agave-validator \ +$ solana-validator \ --identity ~/validator-keypair.json \ --vote-account ~/vote-account-keypair.json \ --known-validator 7Np41oeYqPefeNQEHSv1UDhYrehxin3NStELsSKCT4K2 \ diff --git a/docs/src/clusters/benchmark.md b/docs/src/clusters/benchmark.md index 35978cdd0967dd..d913f9e5f16392 100644 --- a/docs/src/clusters/benchmark.md +++ b/docs/src/clusters/benchmark.md @@ -108,7 +108,7 @@ For example Generally we are using `debug` for infrequent debug messages, `trace` for potentially frequent messages and `info` for performance-related logging. -You can also attach to a running process with GDB. The leader's process is named _agave-validator_: +You can also attach to a running process with GDB. The leader's process is named _solana-validator_: ```bash sudo gdb diff --git a/docs/src/implemented-proposals/installer.md b/docs/src/implemented-proposals/installer.md index c052aa7b4e54e5..a3ad797171c5b8 100644 --- a/docs/src/implemented-proposals/installer.md +++ b/docs/src/implemented-proposals/installer.md @@ -13,16 +13,16 @@ This document proposes an easy to use software install and updater that can be u The easiest install method for supported platforms: ```bash -$ curl -sSf https://raw.githubusercontent.com/solana-labs/solana/v1.0.0/install/agave-install-init.sh | sh +$ curl -sSf https://raw.githubusercontent.com/solana-labs/solana/v1.0.0/install/solana-install-init.sh | sh ``` -This script will check github for the latest tagged release and download and run the `agave-install-init` binary from there. +This script will check github for the latest tagged release and download and run the `solana-install-init` binary from there. If additional arguments need to be specified during the installation, the following shell syntax is used: ```bash -$ init_args=.... # arguments for `agave-install-init ...` -$ curl -sSf https://raw.githubusercontent.com/solana-labs/solana/v1.0.0/install/agave-install-init.sh | sh -s - ${init_args} +$ init_args=.... # arguments for `solana-install-init ...` +$ curl -sSf https://raw.githubusercontent.com/solana-labs/solana/v1.0.0/install/solana-install-init.sh | sh -s - ${init_args} ``` ### Fetch and run a pre-built installer from a Github release @@ -30,9 +30,9 @@ $ curl -sSf https://raw.githubusercontent.com/solana-labs/solana/v1.0.0/install/ With a well-known release URL, a pre-built binary can be obtained for supported platforms: ```bash -$ curl -o agave-install-init https://github.com/solana-labs/solana/releases/download/v1.0.0/agave-install-init-x86_64-apple-darwin -$ chmod +x ./agave-install-init -$ ./agave-install-init --help +$ curl -o solana-install-init https://github.com/solana-labs/solana/releases/download/v1.0.0/solana-install-init-x86_64-apple-darwin +$ chmod +x ./solana-install-init +$ ./solana-install-init --help ``` ### Build and run the installer from source @@ -51,16 +51,16 @@ Given a solana release tarball \(as created by `ci/publish-tarball.sh`\) that ha ```bash $ solana-keygen new -o update-manifest.json # <-- only generated once, the public key is shared with users -$ agave-install deploy http://example.com/path/to/solana-release.tar.bz2 update-manifest.json +$ solana-install deploy http://example.com/path/to/solana-release.tar.bz2 update-manifest.json ``` ### Run a validator node that auto updates itself ```bash -$ agave-install init --pubkey 92DMonmBYXwEMHJ99c9ceRSpAmk9v6i3RdvDdXaVcrfj # <-- pubkey is obtained from whoever is deploying the updates -$ export PATH=~/.local/share/agave-install/bin:$PATH +$ solana-install init --pubkey 92DMonmBYXwEMHJ99c9ceRSpAmk9v6i3RdvDdXaVcrfj # <-- pubkey is obtained from whoever is deploying the updates +$ export PATH=~/.local/share/solana-install/bin:$PATH $ solana-keygen ... # <-- runs the latest solana-keygen -$ agave-install run agave-validator ... # <-- runs a validator, restarting it as necessary when an update is applied +$ solana-install run solana-validator ... # <-- runs a validator, restarting it as necessary when an update is applied ``` ## On-chain Update Manifest @@ -87,9 +87,9 @@ pub struct SignedUpdateManifest { } ``` -Note that the `manifest` field itself contains a corresponding signature \(`manifest_signature`\) to guard against man-in-the-middle attacks between the `agave-install` tool and the solana cluster RPC API. +Note that the `manifest` field itself contains a corresponding signature \(`manifest_signature`\) to guard against man-in-the-middle attacks between the `solana-install` tool and the solana cluster RPC API. -To guard against rollback attacks, `agave-install` will refuse to install an update with an older `timestamp_secs` than what is currently installed. +To guard against rollback attacks, `solana-install` will refuse to install an update with an older `timestamp_secs` than what is currently installed. ## Release Archive Contents @@ -101,17 +101,17 @@ A release archive is expected to be a tar file compressed with bzip2 with the fo - `/bin/` -- directory containing available programs in the release. - `agave-install` will symlink this directory to + `solana-install` will symlink this directory to - `~/.local/share/agave-install/bin` for use by the `PATH` environment + `~/.local/share/solana-install/bin` for use by the `PATH` environment variable. - `...` -- any additional files and directories are permitted -## agave-install Tool +## solana-install Tool -The `agave-install` tool is used by the user to install and update their cluster software. +The `solana-install` tool is used by the user to install and update their cluster software. It manages the following files and directories in the user's home directory: @@ -122,11 +122,11 @@ It manages the following files and directories in the user's home directory: ### Command-line Interface ```text -agave-install 0.16.0 +solana-install 0.16.0 The solana cluster software installer USAGE: - agave-install [OPTIONS] + solana-install [OPTIONS] FLAGS: -h, --help Prints help information @@ -145,11 +145,11 @@ SUBCOMMANDS: ``` ```text -agave-install-init +solana-install-init initializes a new installation USAGE: - agave-install init [OPTIONS] + solana-install init [OPTIONS] FLAGS: -h, --help Prints help information @@ -161,11 +161,11 @@ OPTIONS: ``` ```text -agave-install info +solana-install info displays information about the current installation USAGE: - agave-install info [FLAGS] + solana-install info [FLAGS] FLAGS: -h, --help Prints help information @@ -173,11 +173,11 @@ FLAGS: ``` ```text -agave-install deploy +solana-install deploy deploys a new update USAGE: - agave-install deploy + solana-install deploy FLAGS: -h, --help Prints help information @@ -188,22 +188,22 @@ ARGS: ``` ```text -agave-install update +solana-install update checks for an update, and if available downloads and applies it USAGE: - agave-install update + solana-install update FLAGS: -h, --help Prints help information ``` ```text -agave-install run +solana-install run Runs a program while periodically checking and applying software updates USAGE: - agave-install run [program_arguments]... + solana-install run [program_arguments]... FLAGS: -h, --help Prints help information diff --git a/docs/src/implemented-proposals/rpc-transaction-history.md b/docs/src/implemented-proposals/rpc-transaction-history.md index 607a79ce658b98..54288ad9659bd7 100644 --- a/docs/src/implemented-proposals/rpc-transaction-history.md +++ b/docs/src/implemented-proposals/rpc-transaction-history.md @@ -68,7 +68,7 @@ the results of BigTable queries more complicated but is not a significant issue. ## Data Population The ongoing population of instance data will occur on an epoch cadence through -the use of a new `agave-ledger-tool` command that will convert rocksdb data for +the use of a new `solana-ledger-tool` command that will convert rocksdb data for a given slot range into the instance schema. The same process will be run once, manually, to backfill the existing ledger diff --git a/docs/src/operations/best-practices/general.md b/docs/src/operations/best-practices/general.md index 3e531b0160c571..29ef42c81b7f5f 100644 --- a/docs/src/operations/best-practices/general.md +++ b/docs/src/operations/best-practices/general.md @@ -23,12 +23,12 @@ watch past workshops through the ## Help with the validator command line -From within the Solana CLI, you can execute the `agave-validator` command with +From within the Solana CLI, you can execute the `solana-validator` command with the `--help` flag to get a better understanding of the flags and sub commands available. ``` -agave-validator --help +solana-validator --help ``` ## Restarting your validator @@ -49,14 +49,14 @@ solana leader-schedule Based on the current slot and the leader schedule, you can calculate open time windows where your validator is not expected to produce blocks. -Assuming you are ready to restart, you may use the `agave-validator exit` +Assuming you are ready to restart, you may use the `solana-validator exit` command. The command exits your validator process when an appropriate idle time window is reached. Assuming that you have systemd implemented for your validator process, the validator should restart automatically after the exit. See the below help command for details: ``` -agave-validator exit --help +solana-validator exit --help ``` ## Upgrading @@ -75,28 +75,28 @@ process. It is a best practice to always build your Solana binaries from source. If you build from source, you are certain that the code you are building has not been tampered with before the binary was created. You may also be able to optimize -your `agave-validator` binary to your specific hardware. +your `solana-validator` binary to your specific hardware. If you build from source on the validator machine (or a machine with the same CPU), you can target your specific architecture using the `-march` flag. Refer to the following doc for [instructions on building from source](../../cli/install.md#build-from-source). -### agave-install +### solana-install If you are not comfortable building from source, or you need to quickly install a new version to test something out, you could instead try using the -`agave-install` command. +`solana-install` command. Assuming you want to install Solana version `1.14.17`, you would execute the following: ``` -agave-install init 1.14.17 +solana-install init 1.14.17 ``` This command downloads the executable for `1.14.17` and installs it into a -`.local` directory. You can also look at `agave-install --help` for more +`.local` directory. You can also look at `solana-install --help` for more options. > **Note** this command only works if you already have the solana cli installed. @@ -106,7 +106,7 @@ options. ### Restart For all install methods, the validator process will need to be restarted before -the newly installed version is in use. Use `agave-validator exit` to restart +the newly installed version is in use. Use `solana-validator exit` to restart your validator process. ### Verifying version @@ -132,13 +132,13 @@ have state locally. In other cases such as restarts for upgrades, a snapshot download should be avoided. To avoid downloading a snapshot on restart, add the following flag to the -`agave-validator` command: +`solana-validator` command: ``` --no-snapshot-fetch ``` -If you use this flag with the `agave-validator` command, make sure that you run +If you use this flag with the `solana-validator` command, make sure that you run `solana catchup ` after your validator starts to make sure that the validator is catching up in a reasonable time. After some time (potentially a few hours), if it appears that your validator continues to fall behind, then you @@ -199,7 +199,7 @@ It is important that you do not accidentally run out of funds in your identity account, as your node will stop voting. It is also important to note that this account keypair is the most vulnerable of the three keypairs in a vote account because the keypair for the identity account is stored on your validator when -running the `agave-validator` software. How much SOL you should store there is +running the `solana-validator` software. How much SOL you should store there is up to you. As a best practice, make sure to check the account regularly and refill or deduct from it as needed. To check the account balance do: @@ -207,7 +207,7 @@ refill or deduct from it as needed. To check the account balance do: solana balance validator-keypair.json ``` -> **Note** `agave-watchtower` can monitor for a minimum validator identity +> **Note** `solana-watchtower` can monitor for a minimum validator identity > balance. See [monitoring best practices](./monitoring.md) for details. ## Withdrawing From The Vote Account diff --git a/docs/src/operations/best-practices/monitoring.md b/docs/src/operations/best-practices/monitoring.md index a0f2ef9df9fa22..6d04fc38487be7 100644 --- a/docs/src/operations/best-practices/monitoring.md +++ b/docs/src/operations/best-practices/monitoring.md @@ -4,34 +4,34 @@ sidebar_label: Monitoring pagination_label: "Best Practices: Validator Monitoring" --- -It is essential that you have monitoring in place on your validator. In the event that your validator is delinquent (behind the rest of the network) you want to respond immediately to fix the issue. One very useful tool to monitor your validator is [`agave-watchtower`](#agave-watchtower). +It is essential that you have monitoring in place on your validator. In the event that your validator is delinquent (behind the rest of the network) you want to respond immediately to fix the issue. One very useful tool to monitor your validator is [`solana-watchtower`](#solana-watchtower). ## Solana Watchtower -Solana Watchtower is an extremely useful monitoring tool that will regularly monitor the health of your validator. It can monitor your validator for delinquency then notify you on your application of choice: Slack, Discord, Telegram or Twilio. Additionally, `agave-watchtower` has the ability to monitor the health of the entire cluster so that you can be aware of any cluster wide problems. +Solana Watchtower is an extremely useful monitoring tool that will regularly monitor the health of your validator. It can monitor your validator for delinquency then notify you on your application of choice: Slack, Discord, Telegram or Twilio. Additionally, `solana-watchtower` has the ability to monitor the health of the entire cluster so that you can be aware of any cluster wide problems. ### Getting Started -To get started with Solana Watchtower, run `agave-watchtower --help`. From the help menu, you can see the optional flags and an explanation of the command. +To get started with Solana Watchtower, run `solana-watchtower --help`. From the help menu, you can see the optional flags and an explanation of the command. Here is a sample command that will monitor a validator node with an identity public key of `2uTk98rqqwENevkPH2AHHzGHXgeGc1h6ku8hQUqWeXZp`: ``` -agave-watchtower --monitor-active-stake --validator-identity \ +solana-watchtower --monitor-active-stake --validator-identity \ 2uTk98rqqwENevkPH2AHHzGHXgeGc1h6ku8hQUqWeXZp ``` -The command will monitor your validator, but you will not get notifications unless you added the environment variables mentioned in `agave-watchtower --help`. Since getting each of these services setup for notifications is not straight forward, the next section will walk through [setting up watchtower notifications on Telegram](#setup-telegram-notifications). +The command will monitor your validator, but you will not get notifications unless you added the environment variables mentioned in `solana-watchtower --help`. Since getting each of these services setup for notifications is not straight forward, the next section will walk through [setting up watchtower notifications on Telegram](#setup-telegram-notifications). ### Best Practices -It is a best practice to run the `agave-watchtower` command on a separate server from your validator. +It is a best practice to run the `solana-watchtower` command on a separate server from your validator. -In the case that you run `agave-watchtower` on the same computer as your `agave-validator` process, then during catastrophic events like a power outage, you will not be aware of the issue, because your `agave-watchtower` process will stop at the same time as your `agave-validator` process. +In the case that you run `solana-watchtower` on the same computer as your `solana-validator` process, then during catastrophic events like a power outage, you will not be aware of the issue, because your `solana-watchtower` process will stop at the same time as your `solana-validator` process. -Additionally, while running the `agave-watchtower` process manually with environment variables set in the terminal is a good way to test out the command, it is not operationally sound because the process will not be restarted when the terminal closes or during a system restart. +Additionally, while running the `solana-watchtower` process manually with environment variables set in the terminal is a good way to test out the command, it is not operationally sound because the process will not be restarted when the terminal closes or during a system restart. -Instead, you could run your `agave-watchtower` command as a system process similar to `agave-validator`. In the system process file, you can specify the environment variables for your bot. +Instead, you could run your `solana-watchtower` command as a system process similar to `solana-validator`. In the system process file, you can specify the environment variables for your bot. ### Setup Telegram Notifications @@ -41,7 +41,7 @@ To send validator health notifications to your Telegram account, we are going to 2. Send a message to the bot 3. Create a Telegram group that will get the watchtower notifications 4. Add the environment variables to your command line environment -5. Restart the `agave-watchtower` command +5. Restart the `solana-watchtower` command #### Create a Bot Using BotFather @@ -61,7 +61,7 @@ In Telegram, click on the new message icon and then select new group. Find your Now that you have a bot setup, you will need to set the environment variables for the bot so that watchtower can send notifications. -First, recall the chat message that you got from _@BotFather_. In the message, there was an HTTP API token for your bot. The token will have this format: `389178471:MMTKMrnZB4ErUzJmuFIXTKE6DupLSgoa7h4o`. You will use that token to set the `TELEGRAM_BOT_TOKEN` environment variable. In the terminal where you plan to run `agave-watchtower`, run the following: +First, recall the chat message that you got from _@BotFather_. In the message, there was an HTTP API token for your bot. The token will have this format: `389178471:MMTKMrnZB4ErUzJmuFIXTKE6DupLSgoa7h4o`. You will use that token to set the `TELEGRAM_BOT_TOKEN` environment variable. In the terminal where you plan to run `solana-watchtower`, run the following: ``` export TELEGRAM_BOT_TOKEN= @@ -73,14 +73,14 @@ Next, in your browser, go to `https://api.telegram.org/bot/getUp The response should be in JSON. Search for the string `"chat":` in the JSON. The `id` value of that chat is your `TELEGRAM_CHAT_ID`. It will be a negative number like: `-781559558`. Remember to include the negative sign! If you cannot find `"chat":` in the JSON, then you may have to remove the bot from your chat group and add it again. -With your Telegram chat id in hand, export the environment variable where you plan to run `agave-watchtower`: +With your Telegram chat id in hand, export the environment variable where you plan to run `solana-watchtower`: ``` export TELEGRAM_CHAT_ID= ``` -#### Restart agave-watchtower +#### Restart solana-watchtower -Once your environment variables are set, restart `agave-watchtower`. You should see output about your validator. +Once your environment variables are set, restart `solana-watchtower`. You should see output about your validator. To test that your Telegram configuration is working properly, you could stop your validator briefly until it is labeled as delinquent. Up to a minute after the validator is delinquent, you should receive a message in the Telegram group from your bot. Start the validator again and verify that you get another message in your Telegram group from the bot. The message should say `all clear`. \ No newline at end of file diff --git a/docs/src/operations/guides/restart-cluster.md b/docs/src/operations/guides/restart-cluster.md index cda3f30a5a016d..85d4731d604c65 100644 --- a/docs/src/operations/guides/restart-cluster.md +++ b/docs/src/operations/guides/restart-cluster.md @@ -11,7 +11,7 @@ pagination_label: "Validator Guides: Restart a Cluster" In Solana 1.14 or greater, run the following command to output the latest optimistically confirmed slot your validator observed: ```bash -agave-ledger-tool -l ledger latest-optimistic-slots +solana-ledger-tool -l ledger latest-optimistic-slots ``` In Solana 1.13 or less, the latest optimistically confirmed can be found by looking for the more recent occurrence of @@ -34,11 +34,11 @@ instead. ### Step 4. Create a new snapshot for slot `SLOT_X` with a hard fork at slot `SLOT_X` ```bash -$ agave-ledger-tool -l --snapshot-archive-path --incremental-snapshot-archive-path create-snapshot SLOT_X --hard-fork SLOT_X +$ solana-ledger-tool -l --snapshot-archive-path --incremental-snapshot-archive-path create-snapshot SLOT_X --hard-fork SLOT_X ``` The snapshots directory should now contain the new snapshot. -`agave-ledger-tool create-snapshot` will also output the new shred version, and bank hash value, +`solana-ledger-tool create-snapshot` will also output the new shred version, and bank hash value, call this NEW_SHRED_VERSION and NEW_BANK_HASH respectively. Adjust your validator's arguments: @@ -68,7 +68,7 @@ Post something like the following to #announcements (adjusting the text as appro > 2. a. Preferred method, start from your local ledger with: > > ```bash -> agave-validator +> solana-validator > --wait-for-supermajority SLOT_X # <-- NEW! IMPORTANT! REMOVE AFTER THIS RESTART > --expected-bank-hash NEW_BANK_HASH # <-- NEW! IMPORTANT! REMOVE AFTER THIS RESTART > --hard-fork SLOT_X # <-- NEW! IMPORTANT! REMOVE AFTER THIS RESTART @@ -84,7 +84,7 @@ Post something like the following to #announcements (adjusting the text as appro > b. If your validator doesn't have ledger up to slot SLOT_X or if you have deleted your ledger, have it instead download a snapshot with: > > ```bash -> agave-validator +> solana-validator > --wait-for-supermajority SLOT_X # <-- NEW! IMPORTANT! REMOVE AFTER THIS RESTART > --expected-bank-hash NEW_BANK_HASH # <-- NEW! IMPORTANT! REMOVE AFTER THIS RESTART > --entrypoint entrypoint.testnet.solana.com:8001 @@ -95,7 +95,7 @@ Post something like the following to #announcements (adjusting the text as appro > ... # <-- your other --identity/--vote-account/etc arguments > ``` > -> You can check for which slots your ledger has with: `agave-ledger-tool -l path/to/ledger bounds` +> You can check for which slots your ledger has with: `solana-ledger-tool -l path/to/ledger bounds` > > 3. Wait until 80% of the stake comes online > @@ -122,7 +122,7 @@ and create a new snapshot with additional `--destake-vote-account ` arguments for each of the non-responsive validator's vote account address ```bash -$ agave-ledger-tool -l ledger create-snapshot SLOT_X ledger --hard-fork SLOT_X \ +$ solana-ledger-tool -l ledger create-snapshot SLOT_X ledger --hard-fork SLOT_X \ --destake-vote-account \ --destake-vote-account \ . diff --git a/docs/src/operations/guides/validator-failover.md b/docs/src/operations/guides/validator-failover.md index b7b3fea568194b..168a1a4312cec0 100644 --- a/docs/src/operations/guides/validator-failover.md +++ b/docs/src/operations/guides/validator-failover.md @@ -85,11 +85,11 @@ For more information on etcd TLS setup, please refer to https://etcd.io/docs/v3.5/op-guide/security/#example-2-client-to-server-authentication-with-https-client-certificates ### Primary Validator -The following additional `agave-validator` parameters are required to enable +The following additional `solana-validator` parameters are required to enable tower storage into etcd: ``` -agave-validator ... \ +solana-validator ... \ --tower-storage etcd \ --etcd-cacert-file certs/etcd-ca.pem \ --etcd-cert-file certs/validator.pem \ @@ -103,7 +103,7 @@ that your etcd endpoint remain accessible at all times. ### Secondary Validator Configure the secondary validator like the primary with the exception of the -following `agave-validator` command-line argument changes: +following `solana-validator` command-line argument changes: * Generate and use a secondary validator identity: `--identity secondary-validator-keypair.json` * Add `--no-check-vote-account` * Add `--authorized-voter validator-keypair.json` (where @@ -114,8 +114,8 @@ When both validators are running normally and caught up to the cluster, a failover from primary to secondary can be triggered by running the following command on the secondary validator: ```bash -$ agave-validator wait-for-restart-window --identity validator-keypair.json \ - && agave-validator set-identity validator-keypair.json +$ solana-validator wait-for-restart-window --identity validator-keypair.json \ + && solana-validator set-identity validator-keypair.json ``` The secondary validator will acquire a lock on the tower in etcd to ensure @@ -131,7 +131,7 @@ exit. However if/when the secondary validator restarts, it will do so using the secondary validator identity and thus the restart cycle is broken. ## Triggering a failover via monitoring -Monitoring of your choosing can invoke the `agave-validator set-identity +Monitoring of your choosing can invoke the `solana-validator set-identity validator-keypair.json` command mentioned in the previous section. It is not necessary to guarantee the primary validator has halted before failing diff --git a/docs/src/operations/guides/validator-start.md b/docs/src/operations/guides/validator-start.md index d86c714be4e6a6..378783798b3ce8 100644 --- a/docs/src/operations/guides/validator-start.md +++ b/docs/src/operations/guides/validator-start.md @@ -32,7 +32,7 @@ detail on cluster activity. ## Enabling CUDA If your machine has a GPU with CUDA installed \(Linux-only currently\), include -the `--cuda` argument to `agave-validator`. +the `--cuda` argument to `solana-validator`. When your validator is started look for the following log message to indicate that CUDA is enabled: `"[ solana::validator] CUDA is enabled"` @@ -47,7 +47,7 @@ the following commands. #### **Optimize sysctl knobs** ```bash -sudo bash -c "cat >/etc/sysctl.d/21-agave-validator.conf </etc/sysctl.d/21-solana-validator.conf <` -argument to `agave-validator`. You can specify multiple ones by repeating the argument `--known-validator --known-validator `. +argument to `solana-validator`. You can specify multiple ones by repeating the argument `--known-validator --known-validator `. This has two effects, one is when the validator is booting with `--only-known-rpc`, it will only ask that set of known nodes for downloading genesis and snapshot data. Another is that in combination with the `--halt-on-known-validators-accounts-hash-mismatch` option, it will monitor the merkle root hash of the entire accounts state of other known nodes on gossip and if the hashes produce any mismatch, @@ -277,13 +277,13 @@ account state divergence. Connect to the cluster by running: ```bash -agave-validator \ +solana-validator \ --identity ~/validator-keypair.json \ --vote-account ~/vote-account-keypair.json \ --rpc-port 8899 \ --entrypoint entrypoint.devnet.solana.com:8001 \ --limit-ledger-size \ - --log ~/agave-validator.log + --log ~/solana-validator.log ``` To force validator logging to the console add a `--log -` argument, otherwise @@ -296,7 +296,7 @@ The ledger will be placed in the `ledger/` directory by default, use the > [paper wallet seed phrase](../../cli/wallets/paper.md) > for your `--identity` and/or > `--authorized-voter` keypairs. To use these, pass the respective argument as -> `agave-validator --identity ASK ... --authorized-voter ASK ...` +> `solana-validator --identity ASK ... --authorized-voter ASK ...` > and you will be prompted to enter your seed phrases and optional passphrase. Confirm your validator is connected to the network by opening a new terminal and @@ -312,7 +312,7 @@ If your validator is connected, its public key and IP address will appear in the By default the validator will dynamically select available network ports in the 8000-10000 range, and may be overridden with `--dynamic-port-range`. For -example, `agave-validator --dynamic-port-range 11000-11020 ...` will restrict +example, `solana-validator --dynamic-port-range 11000-11020 ...` will restrict the validator to ports 11000-11020. ### Limiting ledger size to conserve disk space @@ -366,8 +366,8 @@ WantedBy=multi-user.target ``` Now create `/home/sol/bin/validator.sh` to include the desired -`agave-validator` command-line. Ensure that the 'exec' command is used to -start the validator process (i.e. "exec agave-validator ..."). This is +`solana-validator` command-line. Ensure that the 'exec' command is used to +start the validator process (i.e. "exec solana-validator ..."). This is important because without it, logrotate will end up killing the validator every time the logs are rotated. @@ -394,14 +394,14 @@ to be reverted and the issue reproduced before help can be provided. #### Log rotation -The validator log file, as specified by `--log ~/agave-validator.log`, can get +The validator log file, as specified by `--log ~/solana-validator.log`, can get very large over time and it's recommended that log rotation be configured. The validator will re-open its log file when it receives the `USR1` signal, which is the basic primitive that enables log rotation. If the validator is being started by a wrapper shell script, it is important to -launch the process with `exec` (`exec agave-validator ...`) when using logrotate. +launch the process with `exec` (`exec solana-validator ...`) when using logrotate. This will prevent the `USR1` signal from being sent to the script's process instead of the validator's, which will kill them both. @@ -409,13 +409,13 @@ instead of the validator's, which will kill them both. An example setup for the `logrotate`, which assumes that the validator is running as a systemd service called `sol.service` and writes a log file at -/home/sol/agave-validator.log: +/home/sol/solana-validator.log: ```bash # Setup log rotation cat > logrotate.sol </etc/sysctl.d/21-agave-validator.conf </etc/sysctl.d/21-solana-validator.conf < For more explanation on the flags used in the command, refer to the `agave-validator --help` command +> For more explanation on the flags used in the command, refer to the `solana-validator --help` command ``` #!/bin/bash -exec agave-validator \ +exec solana-validator \ --identity /home/sol/validator-keypair.json \ --known-validator 5D1fNXzvv5NjV1ysLjirC4WY92RNsVH18vjmcszZd8on \ --known-validator dDzy5SR3AXdYWVqbDEkVFdvSPCtS9ihF5kJkHCtXoFs \ diff --git a/docs/src/validator/geyser.md b/docs/src/validator/geyser.md index efea2e18e30269..769856303767d6 100644 --- a/docs/src/validator/geyser.md +++ b/docs/src/validator/geyser.md @@ -24,20 +24,20 @@ implementation for the PostgreSQL database. ### Important Crates: -- [`agave-geyser-plugin-interface`] — This crate defines the plugin +- [`solana-geyser-plugin-interface`] — This crate defines the plugin interfaces. - [`solana-accountsdb-plugin-postgres`] — The crate for the referential plugin implementation for the PostgreSQL database. -[`agave-geyser-plugin-interface`]: https://docs.rs/agave-geyser-plugin-interface +[`solana-geyser-plugin-interface`]: https://docs.rs/solana-geyser-plugin-interface [`solana-accountsdb-plugin-postgres`]: https://docs.rs/solana-accountsdb-plugin-postgres [`solana-sdk`]: https://docs.rs/solana-sdk [`solana-transaction-status`]: https://docs.rs/solana-transaction-status ## The Plugin Interface -The Plugin interface is declared in [`agave-geyser-plugin-interface`]. It +The Plugin interface is declared in [`solana-geyser-plugin-interface`]. It is defined by the trait `GeyserPlugin`. The plugin should implement the trait and expose a "C" function `_create_plugin` to return the pointer to this trait. For example, in the referential implementation, the following code @@ -166,7 +166,7 @@ please refer to [`solana-sdk`] and [`solana-transaction-status`] The `slot` points to the slot the transaction is executed at. For more details, please refer to the Rust documentation in -[`agave-geyser-plugin-interface`]. +[`solana-geyser-plugin-interface`]. ## Example PostgreSQL Plugin diff --git a/geyser-plugin-interface/Cargo.toml b/geyser-plugin-interface/Cargo.toml index 56f42fd4612cec..af99758b47d630 100644 --- a/geyser-plugin-interface/Cargo.toml +++ b/geyser-plugin-interface/Cargo.toml @@ -1,7 +1,7 @@ [package] -name = "agave-geyser-plugin-interface" +name = "solana-geyser-plugin-interface" description = "The Solana Geyser plugin interface." -documentation = "https://docs.rs/agave-geyser-plugin-interface" +documentation = "https://docs.rs/solana-geyser-plugin-interface" version = { workspace = true } authors = { workspace = true } repository = { workspace = true } diff --git a/geyser-plugin-interface/src/geyser_plugin_interface.rs b/geyser-plugin-interface/src/geyser_plugin_interface.rs index d9a3b00f8dc4c8..037aedf8b87e89 100644 --- a/geyser-plugin-interface/src/geyser_plugin_interface.rs +++ b/geyser-plugin-interface/src/geyser_plugin_interface.rs @@ -327,7 +327,7 @@ pub trait GeyserPlugin: Any + Send + Sync + std::fmt::Debug { /// # Examples /// /// ``` - /// use agave_geyser_plugin_interface::geyser_plugin_interface::{GeyserPlugin, + /// use solana_geyser_plugin_interface::geyser_plugin_interface::{GeyserPlugin, /// GeyserPluginError, Result}; /// /// #[derive(Debug)] diff --git a/geyser-plugin-manager/Cargo.toml b/geyser-plugin-manager/Cargo.toml index ebef2f637f642d..d905248150b717 100644 --- a/geyser-plugin-manager/Cargo.toml +++ b/geyser-plugin-manager/Cargo.toml @@ -10,7 +10,6 @@ license = { workspace = true } edition = { workspace = true } [dependencies] -agave-geyser-plugin-interface = { workspace = true } bs58 = { workspace = true } crossbeam-channel = { workspace = true } json5 = { workspace = true } @@ -21,6 +20,7 @@ log = { workspace = true } serde_json = { workspace = true } solana-accounts-db = { workspace = true } solana-entry = { workspace = true } +solana-geyser-plugin-interface = { workspace = true } solana-ledger = { workspace = true } solana-measure = { workspace = true } solana-metrics = { workspace = true } diff --git a/geyser-plugin-manager/src/accounts_update_notifier.rs b/geyser-plugin-manager/src/accounts_update_notifier.rs index 90ab0b7998a35c..7c7e3370fc00eb 100644 --- a/geyser-plugin-manager/src/accounts_update_notifier.rs +++ b/geyser-plugin-manager/src/accounts_update_notifier.rs @@ -1,14 +1,14 @@ /// Module responsible for notifying plugins of account updates use { crate::geyser_plugin_manager::GeyserPluginManager, - agave_geyser_plugin_interface::geyser_plugin_interface::{ - ReplicaAccountInfoV3, ReplicaAccountInfoVersions, - }, log::*, solana_accounts_db::{ account_storage::meta::StoredAccountMeta, accounts_update_notifier_interface::AccountsUpdateNotifierInterface, }, + solana_geyser_plugin_interface::geyser_plugin_interface::{ + ReplicaAccountInfoV3, ReplicaAccountInfoVersions, + }, solana_measure::measure::Measure, solana_metrics::*, solana_sdk::{ diff --git a/geyser-plugin-manager/src/block_metadata_notifier.rs b/geyser-plugin-manager/src/block_metadata_notifier.rs index 87f15f41fc0ae0..76d203c5e0ed44 100644 --- a/geyser-plugin-manager/src/block_metadata_notifier.rs +++ b/geyser-plugin-manager/src/block_metadata_notifier.rs @@ -3,10 +3,10 @@ use { block_metadata_notifier_interface::BlockMetadataNotifier, geyser_plugin_manager::GeyserPluginManager, }, - agave_geyser_plugin_interface::geyser_plugin_interface::{ + log::*, + solana_geyser_plugin_interface::geyser_plugin_interface::{ ReplicaBlockInfoV3, ReplicaBlockInfoVersions, }, - log::*, solana_measure::measure::Measure, solana_metrics::*, solana_sdk::{clock::UnixTimestamp, pubkey::Pubkey, reward_info::RewardInfo}, diff --git a/geyser-plugin-manager/src/entry_notifier.rs b/geyser-plugin-manager/src/entry_notifier.rs index da9a9698ed1540..ea14592b615db8 100644 --- a/geyser-plugin-manager/src/entry_notifier.rs +++ b/geyser-plugin-manager/src/entry_notifier.rs @@ -1,11 +1,11 @@ /// Module responsible for notifying plugins about entries use { crate::geyser_plugin_manager::GeyserPluginManager, - agave_geyser_plugin_interface::geyser_plugin_interface::{ - ReplicaEntryInfoV2, ReplicaEntryInfoVersions, - }, log::*, solana_entry::entry::EntrySummary, + solana_geyser_plugin_interface::geyser_plugin_interface::{ + ReplicaEntryInfoV2, ReplicaEntryInfoVersions, + }, solana_ledger::entry_notifier_interface::EntryNotifier, solana_measure::measure::Measure, solana_metrics::*, diff --git a/geyser-plugin-manager/src/geyser_plugin_manager.rs b/geyser-plugin-manager/src/geyser_plugin_manager.rs index 3d0abe16899637..a15f9e1318075d 100644 --- a/geyser-plugin-manager/src/geyser_plugin_manager.rs +++ b/geyser-plugin-manager/src/geyser_plugin_manager.rs @@ -1,9 +1,9 @@ use { - agave_geyser_plugin_interface::geyser_plugin_interface::GeyserPlugin, jsonrpc_core::{ErrorCode, Result as JsonRpcResult}, jsonrpc_server_utils::tokio::sync::oneshot::Sender as OneShotSender, libloading::Library, log::*, + solana_geyser_plugin_interface::geyser_plugin_interface::GeyserPlugin, std::{ ops::{Deref, DerefMut}, path::Path, @@ -442,8 +442,8 @@ mod tests { crate::geyser_plugin_manager::{ GeyserPluginManager, LoadedGeyserPlugin, TESTPLUGIN2_CONFIG, TESTPLUGIN_CONFIG, }, - agave_geyser_plugin_interface::geyser_plugin_interface::GeyserPlugin, libloading::Library, + solana_geyser_plugin_interface::geyser_plugin_interface::GeyserPlugin, std::sync::{Arc, RwLock}, }; diff --git a/geyser-plugin-manager/src/slot_status_notifier.rs b/geyser-plugin-manager/src/slot_status_notifier.rs index 1557bb2d4d8c36..587abe2f79d4de 100644 --- a/geyser-plugin-manager/src/slot_status_notifier.rs +++ b/geyser-plugin-manager/src/slot_status_notifier.rs @@ -1,7 +1,7 @@ use { crate::geyser_plugin_manager::GeyserPluginManager, - agave_geyser_plugin_interface::geyser_plugin_interface::SlotStatus, log::*, + solana_geyser_plugin_interface::geyser_plugin_interface::SlotStatus, solana_measure::measure::Measure, solana_metrics::*, solana_sdk::clock::Slot, diff --git a/geyser-plugin-manager/src/transaction_notifier.rs b/geyser-plugin-manager/src/transaction_notifier.rs index b757c1202b377d..ab821e811047d2 100644 --- a/geyser-plugin-manager/src/transaction_notifier.rs +++ b/geyser-plugin-manager/src/transaction_notifier.rs @@ -1,10 +1,10 @@ /// Module responsible for notifying plugins of transactions use { crate::geyser_plugin_manager::GeyserPluginManager, - agave_geyser_plugin_interface::geyser_plugin_interface::{ + log::*, + solana_geyser_plugin_interface::geyser_plugin_interface::{ ReplicaTransactionInfoV2, ReplicaTransactionInfoVersions, }, - log::*, solana_measure::measure::Measure, solana_metrics::*, solana_rpc::transaction_notifier_interface::TransactionNotifier, diff --git a/install/Cargo.toml b/install/Cargo.toml index c40a0ee6e9eee3..588d4315df5f35 100644 --- a/install/Cargo.toml +++ b/install/Cargo.toml @@ -1,7 +1,7 @@ [package] -name = "agave-install" +name = "solana-install" description = "The solana cluster software installer" -documentation = "https://docs.rs/agave-install" +documentation = "https://docs.rs/solana-install" version = { workspace = true } authors = { workspace = true } repository = { workspace = true } diff --git a/install/install-help.sh b/install/install-help.sh index 7604777e378677..9fb08afa6d14c9 100755 --- a/install/install-help.sh +++ b/install/install-help.sh @@ -4,11 +4,11 @@ set -e cd "$(dirname "$0")"/.. cargo="$(readlink -f "./cargo")" -"$cargo" build --package agave-install +"$cargo" build --package solana-install export PATH=$PWD/target/debug:$PATH echo "\`\`\`manpage" -agave-install --help +solana-install --help echo "\`\`\`" echo "" @@ -16,7 +16,7 @@ commands=(init info deploy update run) for x in "${commands[@]}"; do echo "\`\`\`manpage" - agave-install "${x}" --help + solana-install "${x}" --help echo "\`\`\`" echo "" done diff --git a/install/agave-install-init.sh b/install/solana-install-init.sh similarity index 89% rename from install/agave-install-init.sh rename to install/solana-install-init.sh index cf2d1babf3c306..4f28e300be52ab 100755 --- a/install/agave-install-init.sh +++ b/install/solana-install-init.sh @@ -10,7 +10,7 @@ # except according to those terms. # This is just a little script that can be downloaded from the internet to -# install agave-install. It just does platform detection, downloads the installer +# install solana-install. It just does platform detection, downloads the installer # and runs it. { # this ensures the entire script is downloaded # @@ -24,11 +24,11 @@ set -e usage() { cat 1>&2 < --pubkey + solana-install-init [FLAGS] [OPTIONS] --data_dir --pubkey FLAGS: -h, --help Prints help information @@ -81,7 +81,7 @@ main() { esac TARGET="${_cputype}-${_ostype}" - temp_dir="$(mktemp -d 2>/dev/null || ensure mktemp -d -t agave-install-init)" + temp_dir="$(mktemp -d 2>/dev/null || ensure mktemp -d -t solana-install-init)" ensure mkdir -p "$temp_dir" # Check for SOLANA_RELEASE environment variable override. Otherwise fetch @@ -101,8 +101,8 @@ main() { fi fi - download_url="$SOLANA_DOWNLOAD_ROOT/$release/agave-install-init-$TARGET" - solana_install_init="$temp_dir/agave-install-init" + download_url="$SOLANA_DOWNLOAD_ROOT/$release/solana-install-init-$TARGET" + solana_install_init="$temp_dir/solana-install-init" printf 'downloading %s installer\n' "$release" 1>&2 @@ -111,7 +111,7 @@ main() { ensure chmod u+x "$solana_install_init" if [ ! -x "$solana_install_init" ]; then printf '%s\n' "Cannot execute $solana_install_init (likely because of mounting /tmp as noexec)." 1>&2 - printf '%s\n' "Please copy the file to a location where you can execute binaries and run ./agave-install-init." 1>&2 + printf '%s\n' "Please copy the file to a location where you can execute binaries and run ./solana-install-init." 1>&2 exit 1 fi @@ -130,7 +130,7 @@ main() { } err() { - printf 'agave-install-init: %s\n' "$1" >&2 + printf 'solana-install-init: %s\n' "$1" >&2 exit 1 } diff --git a/install/src/bin/agave-install-init.rs b/install/src/bin/solana-install-init.rs similarity index 92% rename from install/src/bin/agave-install-init.rs rename to install/src/bin/solana-install-init.rs index 84c154ac12b35e..ec888d8f452090 100644 --- a/install/src/bin/agave-install-init.rs +++ b/install/src/bin/solana-install-init.rs @@ -16,7 +16,7 @@ fn press_enter() { } fn main() { - agave_install::main_init().unwrap_or_else(|err| { + solana_install::main_init().unwrap_or_else(|err| { println!("Error: {err}"); press_enter(); exit(1); diff --git a/install/src/command.rs b/install/src/command.rs index 4ae9e7ee38cedd..218e815467e9a9 100644 --- a/install/src/command.rs +++ b/install/src/command.rs @@ -540,7 +540,7 @@ pub fn init( explicit_release: Option, ) -> Result<(), String> { let config = { - // Write new config file only if different, so that running |agave-install init| + // Write new config file only if different, so that running |solana-install init| // repeatedly doesn't unnecessarily re-download let mut current_config = Config::load(config_file).unwrap_or_default(); current_config.current_update_manifest = None; @@ -870,7 +870,7 @@ fn check_for_newer_github_release( prerelease_allowed: bool, ) -> Result, String> { let client = reqwest::blocking::Client::builder() - .user_agent("agave-install") + .user_agent("solana-install") .build() .map_err(|err| err.to_string())?; diff --git a/install/src/lib.rs b/install/src/lib.rs index a28b963d65f825..159317edd2e5a8 100644 --- a/install/src/lib.rs +++ b/install/src/lib.rs @@ -281,7 +281,7 @@ pub fn main() -> Result<(), String> { pub fn main_init() -> Result<(), String> { solana_logger::setup(); - let matches = App::new("agave-install-init") + let matches = App::new("solana-install-init") .about("Initializes a new installation") .version(solana_version::version!()) .arg({ diff --git a/install/src/main.rs b/install/src/main.rs index 245f09825ddc6a..c7b15aa6a67206 100644 --- a/install/src/main.rs +++ b/install/src/main.rs @@ -1,3 +1,3 @@ fn main() -> Result<(), String> { - agave_install::main() + solana_install::main() } diff --git a/ledger-tool/Cargo.toml b/ledger-tool/Cargo.toml index cb87a0e16f4a36..6da42940a4ba7f 100644 --- a/ledger-tool/Cargo.toml +++ b/ledger-tool/Cargo.toml @@ -1,7 +1,7 @@ [package] -name = "agave-ledger-tool" +name = "solana-ledger-tool" description = "Blockchain, Rebuilt for Scale" -documentation = "https://docs.rs/agave-ledger-tool" +documentation = "https://docs.rs/solana-ledger-tool" version = { workspace = true } authors = { workspace = true } repository = { workspace = true } diff --git a/ledger-tool/src/blockstore.rs b/ledger-tool/src/blockstore.rs index fed6abde2f2d08..453a801702f864 100644 --- a/ledger-tool/src/blockstore.rs +++ b/ledger-tool/src/blockstore.rs @@ -359,7 +359,7 @@ pub fn blockstore_subcommands<'a, 'b>(hidden: bool) -> Vec> { and timestamps.", ) // This command is important in cluster restart scenarios, so do not hide it ever - // such that the subcommand will be visible as the top level of agave-ledger-tool + // such that the subcommand will be visible as the top level of solana-ledger-tool .arg( Arg::with_name("num_slots") .long("num-slots") diff --git a/ledger-tool/src/ledger_utils.rs b/ledger-tool/src/ledger_utils.rs index 8a8302d7e4e94b..dd7b365eac93e4 100644 --- a/ledger-tool/src/ledger_utils.rs +++ b/ledger-tool/src/ledger_utils.rs @@ -187,14 +187,14 @@ pub fn load_and_process_ledger( } let account_paths = if let Some(account_paths) = arg_matches.value_of("account_paths") { - // If this blockstore access is Primary, no other process (agave-validator) can hold + // If this blockstore access is Primary, no other process (solana-validator) can hold // Primary access. So, allow a custom accounts path without worry of wiping the accounts - // of agave-validator. + // of solana-validator. if !blockstore.is_primary_access() { // Attempt to open the Blockstore in Primary access; if successful, no other process // was holding Primary so allow things to proceed with custom accounts path. Release - // the Primary access instead of holding it to give priority to agave-validator over - // agave-ledger-tool should agave-validator start before we've finished. + // the Primary access instead of holding it to give priority to solana-validator over + // solana-ledger-tool should solana-validator start before we've finished. info!( "Checking if another process currently holding Primary access to {:?}", blockstore.ledger_path() diff --git a/ledger/src/blockstore_db.rs b/ledger/src/blockstore_db.rs index 8b6b44edae61f6..18ba491ea34bd1 100644 --- a/ledger/src/blockstore_db.rs +++ b/ledger/src/blockstore_db.rs @@ -431,7 +431,7 @@ impl Rocks { info!( "Opening Rocks with secondary (read only) access at: {secondary_path:?}. \ This secondary access could temporarily degrade other accesses, such as \ - by agave-validator" + by solana-validator" ); DB::open_cf_descriptors_as_secondary( &db_options, diff --git a/local-cluster/tests/local_cluster.rs b/local-cluster/tests/local_cluster.rs index 20eef0bb0e3e2d..3b18ba44bf2d03 100644 --- a/local-cluster/tests/local_cluster.rs +++ b/local-cluster/tests/local_cluster.rs @@ -2321,13 +2321,13 @@ fn test_hard_fork_with_gap_in_roots() { ); // create hard-forked snapshot only for validator a, emulating the manual cluster restart - // procedure with `agave-ledger-tool create-snapshot` + // procedure with `solana-ledger-tool create-snapshot` let genesis_slot = 0; { let blockstore_a = Blockstore::open(&val_a_ledger_path).unwrap(); create_snapshot_to_hard_fork(&blockstore_a, hard_fork_slot, vec![hard_fork_slot]); - // Intentionally make agave-validator unbootable by replaying blocks from the genesis to + // Intentionally make solana-validator unbootable by replaying blocks from the genesis to // ensure the hard-forked snapshot is used always. Otherwise, we couldn't create a gap // in the ledger roots column family reliably. // There was a bug which caused the hard-forked snapshot at an unrooted slot to forget diff --git a/multinode-demo/bootstrap-validator.sh b/multinode-demo/bootstrap-validator.sh index 2872af5cc426af..5afc543b2f0032 100755 --- a/multinode-demo/bootstrap-validator.sh +++ b/multinode-demo/bootstrap-validator.sh @@ -14,9 +14,9 @@ if [[ "$SOLANA_GPU_MISSING" -eq 1 ]]; then fi if [[ -n $SOLANA_CUDA ]]; then - program=$agave_validator_cuda + program=$solana_validator_cuda else - program=$agave_validator + program=$solana_validator fi no_restart=0 diff --git a/multinode-demo/common.sh b/multinode-demo/common.sh index 1643208947b643..9ae9331cb7a11d 100644 --- a/multinode-demo/common.sh +++ b/multinode-demo/common.sh @@ -40,8 +40,6 @@ else if [[ -z $program ]]; then crate="cli" program="solana" - elif [[ $program == "validator" || $program == "ledger-tool" || $program == "watchtower" || $program == "install" ]]; then - program="agave-$program" else program="solana-$program" fi @@ -65,8 +63,8 @@ fi solana_bench_tps=$(solana_program bench-tps) solana_faucet=$(solana_program faucet) -agave_validator=$(solana_program validator) -agave_validator_cuda="$agave_validator --cuda" +solana_validator=$(solana_program validator) +solana_validator_cuda="$solana_validator --cuda" solana_genesis=$(solana_program genesis) solana_gossip=$(solana_program gossip) solana_keygen=$(solana_program keygen) diff --git a/multinode-demo/validator.sh b/multinode-demo/validator.sh index efb7a6afd56ea0..487154101ac979 100755 --- a/multinode-demo/validator.sh +++ b/multinode-demo/validator.sh @@ -64,7 +64,7 @@ while [[ -n $1 ]]; do elif [[ $1 = --no-airdrop ]]; then airdrops_enabled=0 shift - # agave-validator options + # solana-validator options elif [[ $1 = --expected-genesis-hash ]]; then args+=("$1" "$2") shift 2 @@ -270,9 +270,9 @@ if [[ $maybeRequireTower = true ]]; then fi if [[ -n $SOLANA_CUDA ]]; then - program=$agave_validator_cuda + program=$solana_validator_cuda else - program=$agave_validator + program=$solana_validator fi set -e diff --git a/net/net.sh b/net/net.sh index 36bc48efdb7861..a2d16cef20f417 100755 --- a/net/net.sh +++ b/net/net.sh @@ -122,7 +122,7 @@ Operate a configured testnet sanity/start-specific options: -F - Discard validator nodes that didn't bootup successfully - -o noInstallCheck - Skip agave-install sanity + -o noInstallCheck - Skip solana-install sanity -o rejectExtraNodes - Require the exact number of nodes stop-specific options: @@ -138,7 +138,7 @@ Operate a configured testnet --netem-cmd - Optional command argument to netem. Default is "add". Use "cleanup" to remove rules. update-specific options: - --platform linux|osx|windows - Deploy the tarball using 'agave-install deploy ...' for the + --platform linux|osx|windows - Deploy the tarball using 'solana-install deploy ...' for the given platform (multiple platforms may be specified) (-t option must be supplied as well) @@ -514,11 +514,11 @@ deployUpdate() { declare bootstrapLeader=${validatorIpList[0]} for updatePlatform in $updatePlatforms; do - echo "--- Deploying agave-install update: $updatePlatform" + echo "--- Deploying solana-install update: $updatePlatform" ( set -x - scripts/agave-install-update-manifest-keypair.sh "$updatePlatform" + scripts/solana-install-update-manifest-keypair.sh "$updatePlatform" timeout 30s scp "${sshOptions[@]}" \ update_manifest_keypair.json "$bootstrapLeader:solana/update_manifest_keypair.json" diff --git a/net/remote/remote-deploy-update.sh b/net/remote/remote-deploy-update.sh index 3a71cf5725123e..dd772927c0e119 100755 --- a/net/remote/remote-deploy-update.sh +++ b/net/remote/remote-deploy-update.sh @@ -35,6 +35,6 @@ loadConfigFile PATH="$HOME"/.cargo/bin:"$PATH" set -x -scripts/agave-install-deploy.sh \ +scripts/solana-install-deploy.sh \ --keypair config/faucet.json \ localhost "$releaseChannel" "$updatePlatform" diff --git a/net/remote/remote-node.sh b/net/remote/remote-node.sh index b7d224088da9f9..aeb920bd50bab0 100755 --- a/net/remote/remote-node.sh +++ b/net/remote/remote-node.sh @@ -121,7 +121,7 @@ cat >> ~/solana/on-reboot < system-stats.pid if ${GPU_CUDA_OK} && [[ -e /dev/nvidia0 ]]; then - echo Selecting agave-validator-cuda + echo Selecting solana-validator-cuda export SOLANA_CUDA=1 elif ${GPU_FAIL_IF_NONE} ; then echo "Expected GPU, found none!" @@ -257,13 +257,13 @@ EOF if [[ -n "$maybeWarpSlot" ]]; then # shellcheck disable=SC2086 # Do not want to quote $maybeWarSlot - agave-ledger-tool -l config/bootstrap-validator create-snapshot 0 config/bootstrap-validator $maybeWarpSlot + solana-ledger-tool -l config/bootstrap-validator create-snapshot 0 config/bootstrap-validator $maybeWarpSlot fi - agave-ledger-tool -l config/bootstrap-validator shred-version --max-genesis-archive-unpacked-size 1073741824 | tee config/shred-version + solana-ledger-tool -l config/bootstrap-validator shred-version --max-genesis-archive-unpacked-size 1073741824 | tee config/shred-version if [[ -n "$maybeWaitForSupermajority" ]]; then - bankHash=$(agave-ledger-tool -l config/bootstrap-validator bank-hash --halt-at-slot 0) + bankHash=$(solana-ledger-tool -l config/bootstrap-validator bank-hash --halt-at-slot 0) extraNodeArgs="$extraNodeArgs --expected-bank-hash $bankHash" echo "$bankHash" > config/bank-hash fi diff --git a/net/remote/remote-sanity.sh b/net/remote/remote-sanity.sh index 91dae4b57336fa..8c36e99ffdf936 100755 --- a/net/remote/remote-sanity.sh +++ b/net/remote/remote-sanity.sh @@ -65,7 +65,7 @@ local|tar|skip) export USE_INSTALL=1 solana_cli=solana solana_gossip=solana-gossip - solana_install=agave-install + solana_install=solana-install ;; *) echo "Unknown deployment method: $deployMethod" @@ -122,7 +122,7 @@ else fi if $installCheck && [[ -r update_manifest_keypair.json ]]; then - echo "--- $sanityTargetIp: agave-install test" + echo "--- $sanityTargetIp: solana-install test" ( set -x diff --git a/notifier/src/lib.rs b/notifier/src/lib.rs index 75406d2fbdae33..a369225772492c 100644 --- a/notifier/src/lib.rs +++ b/notifier/src/lib.rs @@ -19,7 +19,7 @@ /// /// To receive a Twilio SMS notification on failure, having a Twilio account, /// and a sending number owned by that account, -/// define environment variable before running `agave-watchtower`: +/// define environment variable before running `solana-watchtower`: /// ```bash /// export TWILIO_CONFIG='ACCOUNT=,TOKEN=,TO=,FROM=' /// ``` @@ -208,7 +208,7 @@ impl Notifier { NotificationType::Resolve { ref incident } => incident.clone().to_string(), }; - let data = json!({"payload":{"summary":msg,"source":"agave-watchtower","severity":"critical"},"routing_key":routing_key,"event_action":event_action,"dedup_key":dedup_key}); + let data = json!({"payload":{"summary":msg,"source":"solana-watchtower","severity":"critical"},"routing_key":routing_key,"event_action":event_action,"dedup_key":dedup_key}); let url = "https://events.pagerduty.com/v2/enqueue"; if let Err(err) = self.client.post(url).json(&data).send() { diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index cb0ad6f1ee448c..9e8a39c79ce1c8 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -63,80 +63,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "agave-geyser-plugin-interface" -version = "1.19.0" -dependencies = [ - "log", - "solana-sdk", - "solana-transaction-status", - "thiserror", -] - -[[package]] -name = "agave-validator" -version = "1.19.0" -dependencies = [ - "agave-geyser-plugin-interface", - "chrono", - "clap 2.33.3", - "console", - "core_affinity", - "crossbeam-channel", - "fd-lock", - "indicatif", - "itertools", - "jsonrpc-core", - "jsonrpc-core-client", - "jsonrpc-derive", - "jsonrpc-ipc-server", - "jsonrpc-server-utils", - "lazy_static", - "libc", - "libloading", - "log", - "num_cpus", - "rand 0.8.5", - "rayon", - "serde", - "serde_json", - "serde_yaml", - "signal-hook", - "solana-accounts-db", - "solana-clap-utils", - "solana-cli-config", - "solana-core", - "solana-download-utils", - "solana-entry", - "solana-faucet", - "solana-genesis-utils", - "solana-geyser-plugin-manager", - "solana-gossip", - "solana-ledger", - "solana-logger", - "solana-metrics", - "solana-net-utils", - "solana-perf", - "solana-poh", - "solana-rpc", - "solana-rpc-client", - "solana-rpc-client-api", - "solana-runtime", - "solana-sdk", - "solana-send-transaction-service", - "solana-storage-bigtable", - "solana-streamer", - "solana-svm", - "solana-test-validator", - "solana-tpu-client", - "solana-unified-scheduler-pool", - "solana-version", - "solana-vote-program", - "symlink", - "thiserror", - "tikv-jemallocator", -] - [[package]] name = "ahash" version = "0.7.6" @@ -5118,11 +5044,20 @@ dependencies = [ "solana-sdk", ] +[[package]] +name = "solana-geyser-plugin-interface" +version = "1.19.0" +dependencies = [ + "log", + "solana-sdk", + "solana-transaction-status", + "thiserror", +] + [[package]] name = "solana-geyser-plugin-manager" version = "1.19.0" dependencies = [ - "agave-geyser-plugin-interface", "bs58", "crossbeam-channel", "json5", @@ -5133,6 +5068,7 @@ dependencies = [ "serde_json", "solana-accounts-db", "solana-entry", + "solana-geyser-plugin-interface", "solana-ledger", "solana-measure", "solana-metrics", @@ -6119,11 +6055,11 @@ dependencies = [ name = "solana-sbf-rust-simulation" version = "1.19.0" dependencies = [ - "agave-validator", "solana-logger", "solana-program", "solana-program-test", "solana-sdk", + "solana-validator", ] [[package]] @@ -6526,6 +6462,70 @@ dependencies = [ "solana-vote", ] +[[package]] +name = "solana-validator" +version = "1.19.0" +dependencies = [ + "chrono", + "clap 2.33.3", + "console", + "core_affinity", + "crossbeam-channel", + "fd-lock", + "indicatif", + "itertools", + "jsonrpc-core", + "jsonrpc-core-client", + "jsonrpc-derive", + "jsonrpc-ipc-server", + "jsonrpc-server-utils", + "lazy_static", + "libc", + "libloading", + "log", + "num_cpus", + "rand 0.8.5", + "rayon", + "serde", + "serde_json", + "serde_yaml", + "signal-hook", + "solana-accounts-db", + "solana-clap-utils", + "solana-cli-config", + "solana-core", + "solana-download-utils", + "solana-entry", + "solana-faucet", + "solana-genesis-utils", + "solana-geyser-plugin-interface", + "solana-geyser-plugin-manager", + "solana-gossip", + "solana-ledger", + "solana-logger", + "solana-metrics", + "solana-net-utils", + "solana-perf", + "solana-poh", + "solana-rpc", + "solana-rpc-client", + "solana-rpc-client-api", + "solana-runtime", + "solana-sdk", + "solana-send-transaction-service", + "solana-storage-bigtable", + "solana-streamer", + "solana-svm", + "solana-test-validator", + "solana-tpu-client", + "solana-unified-scheduler-pool", + "solana-version", + "solana-vote-program", + "symlink", + "thiserror", + "tikv-jemallocator", +] + [[package]] name = "solana-version" version = "1.19.0" diff --git a/programs/sbf/Cargo.toml b/programs/sbf/Cargo.toml index dee6a947b1965d..8a99a0f005471a 100644 --- a/programs/sbf/Cargo.toml +++ b/programs/sbf/Cargo.toml @@ -46,7 +46,7 @@ solana-sbf-rust-realloc = { path = "rust/realloc", version = "=1.19.0", default- solana-sbf-rust-realloc-invoke = { path = "rust/realloc_invoke", version = "=1.19.0" } solana-sdk = { path = "../../sdk", version = "=1.19.0" } solana-transaction-status = { path = "../../transaction-status", version = "=1.19.0" } -agave-validator = { path = "../../validator", version = "=1.19.0" } +solana-validator = { path = "../../validator", version = "=1.19.0" } solana-zk-token-sdk = { path = "../../zk-token-sdk", version = "=1.19.0" } solana-svm = { path = "../../svm", version = "=1.19.0" } solana_rbpf = "=0.8.0" diff --git a/programs/sbf/rust/simulation/Cargo.toml b/programs/sbf/rust/simulation/Cargo.toml index e9728e5916b801..7091ef9d5ade0c 100644 --- a/programs/sbf/rust/simulation/Cargo.toml +++ b/programs/sbf/rust/simulation/Cargo.toml @@ -16,10 +16,10 @@ test-bpf = [] solana-program = { workspace = true } [dev-dependencies] -agave-validator = { workspace = true } solana-logger = { workspace = true } solana-program-test = { workspace = true } solana-sdk = { workspace = true } +solana-validator = { workspace = true } [lib] crate-type = ["cdylib", "lib"] diff --git a/programs/sbf/rust/simulation/tests/validator.rs b/programs/sbf/rust/simulation/tests/validator.rs index 17de51e665e3ec..3044ad9a642629 100644 --- a/programs/sbf/rust/simulation/tests/validator.rs +++ b/programs/sbf/rust/simulation/tests/validator.rs @@ -1,13 +1,13 @@ #![cfg(feature = "test-bpf")] use { - agave_validator::test_validator::*, solana_program::{ instruction::{AccountMeta, Instruction}, pubkey::Pubkey, sysvar, }, solana_sdk::{signature::Signer, transaction::Transaction}, + solana_validator::test_validator::*, }; #[test] diff --git a/pubsub-client/src/nonblocking/pubsub_client.rs b/pubsub-client/src/nonblocking/pubsub_client.rs index b79e91f681b97f..408df60454e4e1 100644 --- a/pubsub-client/src/nonblocking/pubsub_client.rs +++ b/pubsub-client/src/nonblocking/pubsub_client.rs @@ -33,7 +33,7 @@ //! By default the [`block_subscribe`] and [`vote_subscribe`] events are //! disabled on RPC nodes. They can be enabled by passing //! `--rpc-pubsub-enable-block-subscription` and -//! `--rpc-pubsub-enable-vote-subscription` to `agave-validator`. When these +//! `--rpc-pubsub-enable-vote-subscription` to `solana-validator`. When these //! methods are disabled, the RPC server will return a "Method not found" error //! message. //! @@ -381,7 +381,7 @@ impl PubsubClient { /// Receives messages of type [`RpcBlockUpdate`] when a block is confirmed or finalized. /// /// This method is disabled by default. It can be enabled by passing - /// `--rpc-pubsub-enable-block-subscription` to `agave-validator`. + /// `--rpc-pubsub-enable-block-subscription` to `solana-validator`. /// /// # RPC Reference /// @@ -452,7 +452,7 @@ impl PubsubClient { /// votes are observed prior to confirmation and may never be confirmed. /// /// This method is disabled by default. It can be enabled by passing - /// `--rpc-pubsub-enable-vote-subscription` to `agave-validator`. + /// `--rpc-pubsub-enable-vote-subscription` to `solana-validator`. /// /// # RPC Reference /// diff --git a/pubsub-client/src/pubsub_client.rs b/pubsub-client/src/pubsub_client.rs index 70769619db1f4d..e1a2dd34546528 100644 --- a/pubsub-client/src/pubsub_client.rs +++ b/pubsub-client/src/pubsub_client.rs @@ -32,7 +32,7 @@ //! By default the [`block_subscribe`] and [`vote_subscribe`] events are //! disabled on RPC nodes. They can be enabled by passing //! `--rpc-pubsub-enable-block-subscription` and -//! `--rpc-pubsub-enable-vote-subscription` to `agave-validator`. When these +//! `--rpc-pubsub-enable-vote-subscription` to `solana-validator`. When these //! methods are disabled, the RPC server will return a "Method not found" error //! message. //! @@ -416,7 +416,7 @@ impl PubsubClient { /// Receives messages of type [`RpcBlockUpdate`] when a block is confirmed or finalized. /// /// This method is disabled by default. It can be enabled by passing - /// `--rpc-pubsub-enable-block-subscription` to `agave-validator`. + /// `--rpc-pubsub-enable-block-subscription` to `solana-validator`. /// /// # RPC Reference /// @@ -578,7 +578,7 @@ impl PubsubClient { /// votes are observed prior to confirmation and may never be confirmed. /// /// This method is disabled by default. It can be enabled by passing - /// `--rpc-pubsub-enable-vote-subscription` to `agave-validator`. + /// `--rpc-pubsub-enable-vote-subscription` to `solana-validator`. /// /// # RPC Reference /// diff --git a/rbpf-cli/src/main.rs b/rbpf-cli/src/main.rs index 9e243f0836aa0f..e7db982026f82a 100644 --- a/rbpf-cli/src/main.rs +++ b/rbpf-cli/src/main.rs @@ -1,6 +1,6 @@ fn main() { println!( - r##"rbpf-cli is replaced by agave-ledger-tool program run subcommand. -Please, use 'agave-ledger-tool program run --help' for more information."## + r##"rbpf-cli is replaced by solana-ledger-tool program run subcommand. +Please, use 'solana-ledger-tool program run --help' for more information."## ); } diff --git a/rpc/src/rpc.rs b/rpc/src/rpc.rs index 41b26e5fa1e2c2..45038868502771 100644 --- a/rpc/src/rpc.rs +++ b/rpc/src/rpc.rs @@ -2555,7 +2555,7 @@ pub mod rpc_minimal { #[rpc(meta, name = "getVersion")] fn get_version(&self, meta: Self::Metadata) -> Result; - // TODO: Refactor `agave-validator wait-for-restart-window` to not require this method, so + // TODO: Refactor `solana-validator wait-for-restart-window` to not require this method, so // it can be removed from rpc_minimal #[rpc(meta, name = "getVoteAccounts")] fn get_vote_accounts( @@ -2564,7 +2564,7 @@ pub mod rpc_minimal { config: Option, ) -> Result; - // TODO: Refactor `agave-validator wait-for-restart-window` to not require this method, so + // TODO: Refactor `solana-validator wait-for-restart-window` to not require this method, so // it can be removed from rpc_minimal #[rpc(meta, name = "getLeaderSchedule")] fn get_leader_schedule( @@ -2690,7 +2690,7 @@ pub mod rpc_minimal { }) } - // TODO: Refactor `agave-validator wait-for-restart-window` to not require this method, so + // TODO: Refactor `solana-validator wait-for-restart-window` to not require this method, so // it can be removed from rpc_minimal fn get_vote_accounts( &self, @@ -2701,7 +2701,7 @@ pub mod rpc_minimal { meta.get_vote_accounts(config) } - // TODO: Refactor `agave-validator wait-for-restart-window` to not require this method, so + // TODO: Refactor `solana-validator wait-for-restart-window` to not require this method, so // it can be removed from rpc_minimal fn get_leader_schedule( &self, diff --git a/scripts/cargo-install-all.sh b/scripts/cargo-install-all.sh index 029b1fbf27943d..549aa15550b0eb 100755 --- a/scripts/cargo-install-all.sh +++ b/scripts/cargo-install-all.sh @@ -91,8 +91,8 @@ if [[ $CI_OS_NAME = windows ]]; then cargo-test-bpf cargo-test-sbf solana - agave-install - agave-install-init + solana-install + solana-install-init solana-keygen solana-stake-accounts solana-test-validator @@ -106,12 +106,12 @@ else solana-bench-tps solana-faucet solana-gossip - agave-install + solana-install solana-keygen - agave-ledger-tool + solana-ledger-tool solana-log-analyzer solana-net-shaper - agave-validator + solana-validator rbpf-cli ) @@ -123,11 +123,11 @@ else cargo-test-bpf cargo-test-sbf solana-dos - agave-install-init + solana-install-init solana-stake-accounts solana-test-validator solana-tokens - agave-watchtower + solana-watchtower ) fi diff --git a/scripts/check-dev-context-only-utils.sh b/scripts/check-dev-context-only-utils.sh index 6a4f798c633e26..8719af96a212e4 100755 --- a/scripts/check-dev-context-only-utils.sh +++ b/scripts/check-dev-context-only-utils.sh @@ -31,7 +31,7 @@ source ci/rust-version.sh nightly declare tainted_packages=( solana-accounts-bench solana-banking-bench - agave-ledger-tool + solana-ledger-tool ) # convert to comma separeted (ref: https://stackoverflow.com/a/53839433) diff --git a/scripts/run.sh b/scripts/run.sh index 2d8e1ec88ac450..699bfce3e253e3 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -23,11 +23,9 @@ fi PATH=$PWD/target/$profile:$PATH ok=true -for program in solana-{faucet,genesis,keygen}; do +for program in solana-{faucet,genesis,keygen,validator}; do $program -V || ok=false done -agave-validator -V || ok=false - $ok || { echo echo "Unable to locate required programs. Try building them first with:" @@ -117,7 +115,7 @@ args=( --no-os-network-limits-test ) # shellcheck disable=SC2086 -agave-validator "${args[@]}" $SOLANA_RUN_SH_VALIDATOR_ARGS & +solana-validator "${args[@]}" $SOLANA_RUN_SH_VALIDATOR_ARGS & validator=$! wait "$validator" diff --git a/scripts/agave-install-deploy.sh b/scripts/solana-install-deploy.sh similarity index 90% rename from scripts/agave-install-deploy.sh rename to scripts/solana-install-deploy.sh index a8f8eeb65b3857..ea77ca34bc9ea3 100755 --- a/scripts/agave-install-deploy.sh +++ b/scripts/solana-install-deploy.sh @@ -26,7 +26,7 @@ if [[ -z $URL || -z $TAG ]]; then fi if [[ ! -f update_manifest_keypair.json ]]; then - "$SOLANA_ROOT"/scripts/agave-install-update-manifest-keypair.sh "$OS" + "$SOLANA_ROOT"/scripts/solana-install-update-manifest-keypair.sh "$OS" fi case "$OS" in @@ -76,4 +76,4 @@ if [[ $balance = "0 lamports" ]]; then fi # shellcheck disable=SC2086 # Don't want to double quote $maybeKeypair -agave-install deploy $maybeKeypair --url "$URL" "$DOWNLOAD_URL" update_manifest_keypair.json +solana-install deploy $maybeKeypair --url "$URL" "$DOWNLOAD_URL" update_manifest_keypair.json diff --git a/system-test/abi-testcases/mixed-validator-test.sh b/system-test/abi-testcases/mixed-validator-test.sh index c0400560dc519e..8ab673b26a3d21 100755 --- a/system-test/abi-testcases/mixed-validator-test.sh +++ b/system-test/abi-testcases/mixed-validator-test.sh @@ -30,14 +30,14 @@ solanaInstallGlobalOpts=( bootstrapInstall() { declare v=$1 if [[ ! -h $solanaInstallDataDir/active_release ]]; then - sh "$SOLANA_ROOT"/install/agave-install-init.sh "$v" "${solanaInstallGlobalOpts[@]}" + sh "$SOLANA_ROOT"/install/solana-install-init.sh "$v" "${solanaInstallGlobalOpts[@]}" fi export PATH="$solanaInstallDataDir/active_release/bin/:$PATH" } bootstrapInstall "$baselineVersion" for v in "${otherVersions[@]}"; do - agave-install-init "${solanaInstallGlobalOpts[@]}" "$v" + solana-install-init "${solanaInstallGlobalOpts[@]}" "$v" solana -V done @@ -113,7 +113,7 @@ for v in "${otherVersions[@]}"; do ( set -x tmux new-window -t abi -n "$v" " \ - $SOLANA_BIN/agave-validator \ + $SOLANA_BIN/solana-validator \ --ledger $ledger \ --no-snapshot-fetch \ --entrypoint 127.0.0.1:8001 \ diff --git a/system-test/stability-testcases/gossip-dos-test.sh b/system-test/stability-testcases/gossip-dos-test.sh index 68c3c540d5948c..f8afade75dc847 100755 --- a/system-test/stability-testcases/gossip-dos-test.sh +++ b/system-test/stability-testcases/gossip-dos-test.sh @@ -19,14 +19,14 @@ solanaInstallGlobalOpts=( bootstrapInstall() { declare v=$1 if [[ ! -h $solanaInstallDataDir/active_release ]]; then - sh "$SOLANA_ROOT"/install/agave-install-init.sh "$v" "${solanaInstallGlobalOpts[@]}" + sh "$SOLANA_ROOT"/install/solana-install-init.sh "$v" "${solanaInstallGlobalOpts[@]}" fi export PATH="$solanaInstallDataDir/active_release/bin/:$PATH" } bootstrapInstall "edge" -agave-install-init --version -agave-install-init edge +solana-install-init --version +solana-install-init edge solana-gossip --version solana-dos --version diff --git a/validator/Cargo.toml b/validator/Cargo.toml index 362a07343b5e4a..5cc76a810116b3 100644 --- a/validator/Cargo.toml +++ b/validator/Cargo.toml @@ -1,8 +1,8 @@ [package] -name = "agave-validator" +name = "solana-validator" description = "Blockchain, Rebuilt for Scale" -documentation = "https://docs.rs/agave-validator" -default-run = "agave-validator" +documentation = "https://docs.rs/solana-validator" +default-run = "solana-validator" version = { workspace = true } authors = { workspace = true } repository = { workspace = true } @@ -11,7 +11,6 @@ license = { workspace = true } edition = { workspace = true } [dependencies] -agave-geyser-plugin-interface = { workspace = true } chrono = { workspace = true, features = ["default", "serde"] } clap = { workspace = true } console = { workspace = true } @@ -42,6 +41,7 @@ solana-download-utils = { workspace = true } solana-entry = { workspace = true } solana-faucet = { workspace = true } solana-genesis-utils = { workspace = true } +solana-geyser-plugin-interface = { workspace = true } solana-geyser-plugin-manager = { workspace = true } solana-gossip = { workspace = true } solana-ledger = { workspace = true } diff --git a/validator/src/bin/solana-test-validator.rs b/validator/src/bin/solana-test-validator.rs index 68e6bcca4fd96f..42f5a0634c0cfa 100644 --- a/validator/src/bin/solana-test-validator.rs +++ b/validator/src/bin/solana-test-validator.rs @@ -1,8 +1,4 @@ use { - agave_validator::{ - admin_rpc_service, cli, dashboard::Dashboard, ledger_lockfile, lock_ledger, - println_name_value, redirect_stderr_to_file, - }, clap::{crate_name, value_t, value_t_or_exit, values_t_or_exit}, crossbeam_channel::unbounded, itertools::Itertools, @@ -32,6 +28,10 @@ use { }, solana_streamer::socket::SocketAddrSpace, solana_test_validator::*, + solana_validator::{ + admin_rpc_service, cli, dashboard::Dashboard, ledger_lockfile, lock_ledger, + println_name_value, redirect_stderr_to_file, + }, std::{ collections::HashSet, fs, io, diff --git a/validator/src/bootstrap.rs b/validator/src/bootstrap.rs index 12bbd0b21001c9..8d5457744a23b8 100644 --- a/validator/src/bootstrap.rs +++ b/validator/src/bootstrap.rs @@ -447,7 +447,7 @@ pub fn attempt_download_genesis_and_snapshot( ) .unwrap_or_else(|err| { // Consider failures here to be more likely due to user error (eg, - // incorrect `agave-validator` command-line arguments) rather than the + // incorrect `solana-validator` command-line arguments) rather than the // RPC node failing. // // Power users can always use the `--no-check-vote-account` option to diff --git a/validator/src/main.rs b/validator/src/main.rs index 9741a2aecd68a8..ec70796130e7d2 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -2,15 +2,6 @@ #[cfg(not(target_env = "msvc"))] use jemallocator::Jemalloc; use { - agave_validator::{ - admin_rpc_service, - admin_rpc_service::{load_staked_nodes_overrides, StakedNodesOverrides}, - bootstrap, - cli::{app, warn_for_deprecated_arguments, DefaultArgs}, - dashboard::Dashboard, - ledger_lockfile, lock_ledger, new_spinner_progress_bar, println_name_value, - redirect_stderr_to_file, - }, clap::{crate_name, value_t, value_t_or_exit, values_t, values_t_or_exit, ArgMatches}, console::style, crossbeam_channel::unbounded, @@ -69,6 +60,15 @@ use { solana_streamer::socket::SocketAddrSpace, solana_svm::runtime_config::RuntimeConfig, solana_tpu_client::tpu_client::DEFAULT_TPU_ENABLE_UDP, + solana_validator::{ + admin_rpc_service, + admin_rpc_service::{load_staked_nodes_overrides, StakedNodesOverrides}, + bootstrap, + cli::{app, warn_for_deprecated_arguments, DefaultArgs}, + dashboard::Dashboard, + ledger_lockfile, lock_ledger, new_spinner_progress_bar, println_name_value, + redirect_stderr_to_file, + }, std::{ collections::{HashSet, VecDeque}, env, @@ -917,7 +917,7 @@ pub fn main() { let logfile = matches .value_of("logfile") .map(|s| s.into()) - .unwrap_or_else(|| format!("agave-validator-{}.log", identity_keypair.pubkey())); + .unwrap_or_else(|| format!("solana-validator-{}.log", identity_keypair.pubkey())); if logfile == "-" { None diff --git a/watchtower/Cargo.toml b/watchtower/Cargo.toml index 4088ee7d9b51ab..d8bad3cf4d18f0 100644 --- a/watchtower/Cargo.toml +++ b/watchtower/Cargo.toml @@ -1,7 +1,7 @@ [package] -name = "agave-watchtower" +name = "solana-watchtower" description = "Blockchain, Rebuilt for Scale" -documentation = "https://docs.rs/agave-watchtower" +documentation = "https://docs.rs/solana-watchtower" version = { workspace = true } authors = { workspace = true } repository = { workspace = true } diff --git a/watchtower/README.md b/watchtower/README.md index ab219be67575eb..33a13939cd260c 100644 --- a/watchtower/README.md +++ b/watchtower/README.md @@ -1,4 +1,4 @@ -The `agave-watchtower` program is used to monitor the health of a cluster. It +The `solana-watchtower` program is used to monitor the health of a cluster. It periodically polls the cluster over an RPC API to confirm that the transaction count is advancing, new blockhashes are available, and no validators are delinquent. Results are reported as InfluxDB metrics, with an optional push diff --git a/watchtower/src/main.rs b/watchtower/src/main.rs index 341b7903c0a33e..f42acdaadaabc6 100644 --- a/watchtower/src/main.rs +++ b/watchtower/src/main.rs @@ -47,7 +47,7 @@ fn get_config() -> Config { .version(solana_version::version!()) .after_help("ADDITIONAL HELP: To receive a Slack, Discord, PagerDuty and/or Telegram notification on sanity failure, - define environment variables before running `agave-watchtower`: + define environment variables before running `solana-watchtower`: export SLACK_WEBHOOK=... export DISCORD_WEBHOOK=... @@ -63,7 +63,7 @@ fn get_config() -> Config { To receive a Twilio SMS notification on failure, having a Twilio account, and a sending number owned by that account, - define environment variable before running `agave-watchtower`: + define environment variable before running `solana-watchtower`: export TWILIO_CONFIG='ACCOUNT=,TOKEN=,TO=,FROM='") .arg({ @@ -166,7 +166,7 @@ fn get_config() -> Config { .value_name("SUFFIX") .takes_value(true) .default_value("") - .help("Add this string into all notification messages after \"agave-watchtower\"") + .help("Add this string into all notification messages after \"solana-watchtower\"") ) .get_matches(); @@ -381,7 +381,7 @@ fn main() -> Result<(), Box> { if let Some((failure_test_name, failure_error_message)) = &failure { let notification_msg = format!( - "agave-watchtower{}: Error: {}: {}", + "solana-watchtower{}: Error: {}: {}", config.name_suffix, failure_test_name, failure_error_message ); num_consecutive_failures += 1; @@ -415,7 +415,7 @@ fn main() -> Result<(), Box> { ); info!("{}", all_clear_msg); notifier.send( - &format!("agave-watchtower{}: {}", config.name_suffix, all_clear_msg), + &format!("solana-watchtower{}: {}", config.name_suffix, all_clear_msg), &NotificationType::Resolve { incident }, ); } From e72f6ba5036b975243c5d639a485c3de1a836b27 Mon Sep 17 00:00:00 2001 From: Will Hickey Date: Tue, 5 Mar 2024 10:19:12 -0600 Subject: [PATCH 16/71] Revert "[anza migration] ci: removed unused s3 upload in Windows build (#9)" This reverts commit 91e3dd225004927f84638410f32cb98717ffb1de. --- .github/workflows/release-artifacts.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index f32bd22eed8c82..79fa59636c7f7d 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -90,6 +90,17 @@ jobs: run: | gcloud storage cp --recursive windows-release/* gs://anza-release/ + - name: Upload + uses: jakejarvis/s3-sync-action@master + with: + args: --acl public-read --follow-symlinks + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} + AWS_REGION: "us-west-1" + SOURCE_DIR: "windows-release" + windows-gh-release: if: ${{ needs.windows-build.outputs.tag != '' }} needs: [windows-build] From 4cb7c315b8ca0b35e1e5c19ba9d101f5b9b07535 Mon Sep 17 00:00:00 2001 From: Will Hickey Date: Tue, 5 Mar 2024 10:19:20 -0600 Subject: [PATCH 17/71] Revert "[anza migration] ci: fix path (#8)" This reverts commit 58e9a19f11822c93fdece4567d5250cd0b671ad1. --- .github/workflows/release-artifacts.yml | 4 ++-- .mergify.yml | 2 +- sdk/docker-solana/build.sh | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 79fa59636c7f7d..9b2d1965a7dc01 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -70,7 +70,7 @@ jobs: name: windows-artifact path: windows-release/ - windows-gcs-upload: + windows-s3-upload: if: ${{ needs.windows-build.outputs.channel != '' || needs.windows-build.outputs.tag != '' }} needs: [windows-build] runs-on: ubuntu-20.04 @@ -79,7 +79,7 @@ jobs: uses: actions/download-artifact@v3 with: name: windows-artifact - path: ./windows-release + path: .windows-release/ - name: Setup crediential uses: "google-github-actions/auth@v2" diff --git a/.mergify.yml b/.mergify.yml index 19f9b8f116a78a..166f59a5f365d1 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -50,7 +50,7 @@ pull_request_rules: - name: automatic merge (squash) on CI success conditions: - and: - - status-success=buildkite/agave + - status-success=buildkite/solana - status-success=ci-gate - label=automerge - label!=no-automerge diff --git a/sdk/docker-solana/build.sh b/sdk/docker-solana/build.sh index 70e3d0d23e44de..f1c8ee265d6d56 100755 --- a/sdk/docker-solana/build.sh +++ b/sdk/docker-solana/build.sh @@ -29,7 +29,7 @@ cp -f ../../fetch-spl.sh usr/bin/ ./fetch-spl.sh ) -docker build -t anzaxyz/agave:"$CHANNEL_OR_TAG" . +docker build -t solanalabs/solana:"$CHANNEL_OR_TAG" . maybeEcho= if [[ -z $CI ]]; then @@ -43,4 +43,4 @@ else fi ) fi -$maybeEcho docker push anzaxyz/agave:"$CHANNEL_OR_TAG" +$maybeEcho docker push solanalabs/solana:"$CHANNEL_OR_TAG" From 6e01a074dfa6d720a62310f8bf75edb3497b37af Mon Sep 17 00:00:00 2001 From: Will Hickey Date: Tue, 5 Mar 2024 10:19:45 -0600 Subject: [PATCH 18/71] Revert "[anza migration] ci (#5)" This reverts commit b0022d73eadacf9097afffb80aeda6c10596b310. --- .mergify.yml | 4 +- .travis.yml | 94 ++++++++++++++++++++++++++++++++ README.md | 34 +++++++++++- ci/buildkite-pipeline-in-disk.sh | 4 +- ci/buildkite-pipeline.sh | 8 +-- ci/buildkite-solana-private.sh | 2 +- ci/channel-info.sh | 2 +- ci/dependabot-pr.sh | 2 +- ci/rust-version.sh | 2 +- ci/test-coverage.sh | 2 +- 10 files changed, 140 insertions(+), 14 deletions(-) create mode 100644 .travis.yml diff --git a/.mergify.yml b/.mergify.yml index 166f59a5f365d1..ef576943d5d635 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -33,7 +33,7 @@ pull_request_rules: actions: request_reviews: teams: - - "@anza-xyz/community-pr-subscribers" + - "@solana-labs/community-pr-subscribers" - name: label changes from monorepo-triage conditions: - author≠@core-contributors @@ -102,7 +102,7 @@ pull_request_rules: actions: backport: assignees: &BackportAssignee - - "{{ merged_by|replace('mergify[bot]', label|select('equalto', 'community')|first|default(author)|replace('community', '@anza-xyz/community-pr-subscribers')) }}" + - "{{ merged_by|replace('mergify[bot]', label|select('equalto', 'community')|first|default(author)|replace('community', '@solana-labs/community-pr-subscribers')) }}" title: "{{ destination_branch }}: {{ title }} (backport of #{{ number }})" ignore_conflicts: true labels: diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000000000..c2dd13e32551ba --- /dev/null +++ b/.travis.yml @@ -0,0 +1,94 @@ +branches: + only: + - master + - /^v\d+\.\d+/ + +notifications: + email: false + slack: + on_success: change + if: NOT type = pull_request + secure: F4IjOE05MyaMOdPRL+r8qhs7jBvv4yDM3RmFKE1zNXnfUOqV4X38oQM1EI+YVsgpMQLj/pxnEB7wcTE4Bf86N6moLssEULCpvAuMVoXj4QbWdomLX+01WbFa6fLVeNQIg45NHrz2XzVBhoKOrMNnl+QI5mbR2AlS5oqsudHsXDnyLzZtd4Y5SDMdYG1zVWM01+oNNjgNfjcCGmOE/K0CnOMl6GPi3X9C34tJ19P2XT7MTDsz1/IfEF7fro2Q8DHEYL9dchJMoisXSkem5z7IDQkGzXsWdWT4NnndUvmd1MlTCE9qgoXDqRf95Qh8sB1Dz08HtvgfaosP2XjtNTfDI9BBYS15Ibw9y7PchAJE1luteNjF35EOy6OgmCLw/YpnweqfuNViBZz+yOPWXVC0kxnPIXKZ1wyH9ibeH6E4hr7a8o9SV/6SiWIlbYF+IR9jPXyTCLP/cc3sYljPWxDnhWFwFdRVIi3PbVAhVu7uWtVUO17Oc9gtGPgs/GrhOMkJfwQPXaudRJDpVZowxTX4x9kefNotlMAMRgq+Drbmgt4eEBiCNp0ITWgh17BiE1U09WS3myuduhoct85+FoVeaUkp1sxzHVtGsNQH0hcz7WcpZyOM+AwistJA/qzeEDQao5zi1eKWPbO2xAhi2rV1bDH6bPf/4lDBwLRqSiwvlWU= + +os: linux +dist: bionic +language: minimal + +jobs: + include: + - &release-artifacts + if: type IN (api, cron) OR tag IS present + name: "macOS release artifacts" + os: osx + osx_image: xcode12 + language: rust + rust: + - stable + install: + - source ci/rust-version.sh + - PATH="/usr/local/opt/coreutils/libexec/gnubin:$PATH" + - readlink -f . + - brew install gnu-tar + - PATH="/usr/local/opt/gnu-tar/libexec/gnubin:$PATH" + - tar --version + script: + - source ci/env.sh + - rustup set profile default + - ci/publish-tarball.sh + deploy: + - provider: s3 + access_key_id: $AWS_ACCESS_KEY_ID + secret_access_key: $AWS_SECRET_ACCESS_KEY + bucket: release.solana.com + region: us-west-1 + skip_cleanup: true + acl: public_read + local_dir: travis-s3-upload + on: + all_branches: true + - provider: releases + token: $GITHUB_TOKEN + skip_cleanup: true + file_glob: true + file: travis-release-upload/* + on: + tags: true + - <<: *release-artifacts + name: "Windows release artifacts" + os: windows + install: + - choco install openssl + - export OPENSSL_DIR="C:\Program Files\OpenSSL-Win64" + - source ci/rust-version.sh + - PATH="/usr/local/opt/coreutils/libexec/gnubin:$PATH" + - readlink -f . + # Linux release artifacts are still built by ci/buildkite-secondary.yml + #- <<: *release-artifacts + # name: "Linux release artifacts" + # os: linux + # before_install: + # - sudo apt-get install libssl-dev libudev-dev + + # docs pull request + - name: "docs" + if: type IN (push, pull_request) OR tag IS present + language: node_js + node_js: + - "lts/*" + + services: + - docker + + cache: + directories: + - ~/.npm + + before_install: + - source ci/env.sh + - .travis/channel_restriction.sh edge beta || travis_terminate 0 + - .travis/affects.sh docs/ .travis || travis_terminate 0 + - cd docs/ + - source .travis/before_install.sh + + script: + - source .travis/script.sh diff --git a/README.md b/README.md index bbaeb3d019a658..c6183f6ab6183e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@

- Solana + Solana

@@ -113,3 +113,35 @@ problem is solved by this code?" On the other hand, if a test does fail and you better way to solve the same problem, a Pull Request with your solution would most certainly be welcome! Likewise, if rewriting a test can better communicate what code it's protecting, please send us that patch! + +# Disclaimer + +All claims, content, designs, algorithms, estimates, roadmaps, +specifications, and performance measurements described in this project +are done with the Solana Labs, Inc. (“SL”) good faith efforts. It is up to +the reader to check and validate their accuracy and truthfulness. +Furthermore, nothing in this project constitutes a solicitation for +investment. + +Any content produced by SL or developer resources that SL provides are +for educational and inspirational purposes only. SL does not encourage, +induce or sanction the deployment, integration or use of any such +applications (including the code comprising the Solana blockchain +protocol) in violation of applicable laws or regulations and hereby +prohibits any such deployment, integration or use. This includes the use of +any such applications by the reader (a) in violation of export control +or sanctions laws of the United States or any other applicable +jurisdiction, (b) if the reader is located in or ordinarily resident in +a country or territory subject to comprehensive sanctions administered +by the U.S. Office of Foreign Assets Control (OFAC), or (c) if the +reader is or is working on behalf of a Specially Designated National +(SDN) or a person subject to similar blocking or denied party +prohibitions. + +The reader should be aware that U.S. export control and sanctions laws prohibit +U.S. persons (and other persons that are subject to such laws) from transacting +with persons in certain countries and territories or that are on the SDN list. +Accordingly, there is a risk to individuals that other persons using any of the +code contained in this repo, or a derivation thereof, may be sanctioned persons +and that transactions with such persons would be a violation of U.S. export +controls and sanctions law. diff --git a/ci/buildkite-pipeline-in-disk.sh b/ci/buildkite-pipeline-in-disk.sh index 2ce8af0432106b..ad12e1fc000a89 100755 --- a/ci/buildkite-pipeline-in-disk.sh +++ b/ci/buildkite-pipeline-in-disk.sh @@ -289,7 +289,7 @@ if [[ -n $BUILDKITE_TAG ]]; then start_pipeline "Tag pipeline for $BUILDKITE_TAG" annotate --style info --context release-tag \ - "https://github.com/anza-xyz/agave/releases/$BUILDKITE_TAG" + "https://github.com/solana-labs/solana/releases/$BUILDKITE_TAG" # Jump directly to the secondary build to publish release artifacts quickly trigger_secondary_step @@ -307,7 +307,7 @@ if [[ $BUILDKITE_BRANCH =~ ^pull ]]; then # Add helpful link back to the corresponding Github Pull Request annotate --style info --context pr-backlink \ - "Github Pull Request: https://github.com/anza-xyz/agave/$BUILDKITE_BRANCH" + "Github Pull Request: https://github.com/solana-labs/solana/$BUILDKITE_BRANCH" if [[ $GITHUB_USER = "dependabot[bot]" ]]; then command_step dependabot "ci/dependabot-pr.sh" 5 diff --git a/ci/buildkite-pipeline.sh b/ci/buildkite-pipeline.sh index 4ae00c9feab586..d40273863cc7a3 100755 --- a/ci/buildkite-pipeline.sh +++ b/ci/buildkite-pipeline.sh @@ -121,8 +121,8 @@ EOF trigger_secondary_step() { cat >> "$output_file" <<"EOF" - - name: "Trigger Build on agave-secondary" - trigger: "agave-secondary" + - name: "Trigger Build on solana-secondary" + trigger: "solana-secondary" branches: "!pull/*" async: true soft_fail: true @@ -315,7 +315,7 @@ if [[ -n $BUILDKITE_TAG ]]; then start_pipeline "Tag pipeline for $BUILDKITE_TAG" annotate --style info --context release-tag \ - "https://github.com/anza-xyz/agave/releases/$BUILDKITE_TAG" + "https://github.com/solana-labs/solana/releases/$BUILDKITE_TAG" # Jump directly to the secondary build to publish release artifacts quickly trigger_secondary_step @@ -333,7 +333,7 @@ if [[ $BUILDKITE_BRANCH =~ ^pull ]]; then # Add helpful link back to the corresponding Github Pull Request annotate --style info --context pr-backlink \ - "Github Pull Request: https://github.com/anza-xyz/agave/$BUILDKITE_BRANCH" + "Github Pull Request: https://github.com/solana-labs/solana/$BUILDKITE_BRANCH" if [[ $GITHUB_USER = "dependabot[bot]" ]]; then command_step dependabot "ci/dependabot-pr.sh" 5 diff --git a/ci/buildkite-solana-private.sh b/ci/buildkite-solana-private.sh index e5886a314eb27c..70d8e4bfe4f59f 100755 --- a/ci/buildkite-solana-private.sh +++ b/ci/buildkite-solana-private.sh @@ -287,7 +287,7 @@ if [[ $BUILDKITE_BRANCH =~ ^pull ]]; then # Add helpful link back to the corresponding Github Pull Request annotate --style info --context pr-backlink \ - "Github Pull Request: https://github.com/anza-xyz/agave/$BUILDKITE_BRANCH" + "Github Pull Request: https://github.com/solana-labs/solana/$BUILDKITE_BRANCH" if [[ $GITHUB_USER = "dependabot[bot]" ]]; then command_step dependabot "ci/dependabot-pr.sh" 5 diff --git a/ci/channel-info.sh b/ci/channel-info.sh index 2bb808365653c6..c82806454d012c 100755 --- a/ci/channel-info.sh +++ b/ci/channel-info.sh @@ -11,7 +11,7 @@ here="$(dirname "$0")" # shellcheck source=ci/semver_bash/semver.sh source "$here"/semver_bash/semver.sh -remote=https://github.com/anza-xyz/agave.git +remote=https://github.com/solana-labs/solana.git # Fetch all vX.Y.Z tags # diff --git a/ci/dependabot-pr.sh b/ci/dependabot-pr.sh index bb019001a0bcfa..91ecd5948c9a43 100755 --- a/ci/dependabot-pr.sh +++ b/ci/dependabot-pr.sh @@ -21,7 +21,7 @@ fi echo --- "(FAILING) Backpropagating dependabot-triggered Cargo.lock updates" name="dependabot-buildkite" -api_base="https://api.github.com/repos/anza-xyz/agave/pulls" +api_base="https://api.github.com/repos/solana-labs/solana/pulls" pr_num=$(echo "$BUILDKITE_BRANCH" | grep -Eo '[0-9]+') branch=$(curl -s "$api_base/$pr_num" | python3 -c 'import json,sys;print(json.load(sys.stdin)["head"]["ref"])') diff --git a/ci/rust-version.sh b/ci/rust-version.sh index 97ebb1c7935006..3321f1d5ecb6a1 100644 --- a/ci/rust-version.sh +++ b/ci/rust-version.sh @@ -37,7 +37,7 @@ export rust_stable="$stable_version" export rust_nightly=nightly-"$nightly_version" -export ci_docker_image="anzaxyz/ci:rust_${rust_stable}_${rust_nightly}" +export ci_docker_image="solanalabs/ci:rust_${rust_stable}_${rust_nightly}" [[ -z $1 ]] || ( diff --git a/ci/test-coverage.sh b/ci/test-coverage.sh index ffd362acd287b8..44231cd338a13e 100755 --- a/ci/test-coverage.sh +++ b/ci/test-coverage.sh @@ -32,5 +32,5 @@ else codecov -t "${CODECOV_TOKEN}" annotate --style success --context codecov.io \ - "CodeCov report: https://codecov.io/github/anza-xyz/agave/commit/${CI_COMMIT:0:9}" + "CodeCov report: https://codecov.io/github/solana-labs/solana/commit/${CI_COMMIT:0:9}" fi From 9b1243ee4cd13b2bd0f3b5d18bb38ebd39b024b2 Mon Sep 17 00:00:00 2001 From: Will Hickey Date: Tue, 5 Mar 2024 10:19:55 -0600 Subject: [PATCH 19/71] Revert "[anza migration] replace binaries backend (#6)" This reverts commit 935551842450258df90ccd8c747cfb4f37722221. --- .github/workflows/release-artifacts-auto.yml | 6 ++-- .../workflows/release-artifacts-manually.yml | 4 ++- .github/workflows/release-artifacts.yml | 33 ++++++++----------- ci/publish-installer.sh | 12 +++---- ci/publish-tarball.sh | 6 ++-- ci/upload-ci-artifact.sh | 10 ------ ci/upload-github-release-asset.sh | 2 +- install/solana-install-init.sh | 4 +-- install/src/command.rs | 8 ++--- 9 files changed, 37 insertions(+), 48 deletions(-) diff --git a/.github/workflows/release-artifacts-auto.yml b/.github/workflows/release-artifacts-auto.yml index 0cdd176e04396c..a8309cdffc8a72 100644 --- a/.github/workflows/release-artifacts-auto.yml +++ b/.github/workflows/release-artifacts-auto.yml @@ -14,12 +14,14 @@ concurrency: jobs: release-artifacts: - if: github.repository == 'anza-xyz/agave' + if: github.repository == 'solana-labs/solana' uses: ./.github/workflows/release-artifacts.yml with: commit: ${{ github.sha }} secrets: - GCS_RELEASE_BUCKET_WRITER_CREDIENTIAL: ${{ secrets.GCS_RELEASE_BUCKET_WRITER_CREDIENTIAL }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} error_reporting: needs: diff --git a/.github/workflows/release-artifacts-manually.yml b/.github/workflows/release-artifacts-manually.yml index fe5c1b03b638b3..35de72922c32c8 100644 --- a/.github/workflows/release-artifacts-manually.yml +++ b/.github/workflows/release-artifacts-manually.yml @@ -14,4 +14,6 @@ jobs: with: commit: ${{ github.event.inputs.commit }} secrets: - GCS_RELEASE_BUCKET_WRITER_CREDIENTIAL: ${{ secrets.GCS_RELEASE_BUCKET_WRITER_CREDIENTIAL }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 9b2d1965a7dc01..0391a09766e97b 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -7,7 +7,11 @@ on: required: false type: string secrets: - GCS_RELEASE_BUCKET_WRITER_CREDIENTIAL: + AWS_ACCESS_KEY_ID: + required: true + AWS_SECRET_ACCESS_KEY: + required: true + AWS_S3_BUCKET: required: true jobs: @@ -58,17 +62,17 @@ jobs: shell: bash run: | FOLDER_NAME=${{ steps.build.outputs.tag || steps.build.outputs.channel }} - mkdir -p "windows-release/$FOLDER_NAME" - cp -v "solana-release-x86_64-pc-windows-msvc.tar.bz2" "windows-release/$FOLDER_NAME/" - cp -v "solana-release-x86_64-pc-windows-msvc.yml" "windows-release/$FOLDER_NAME/" - cp -v "solana-install-init-x86_64-pc-windows-msvc"* "windows-release/$FOLDER_NAME" + mkdir -p "github-action-s3-upload/$FOLDER_NAME" + cp -v "solana-release-x86_64-pc-windows-msvc.tar.bz2" "github-action-s3-upload/$FOLDER_NAME/" + cp -v "solana-release-x86_64-pc-windows-msvc.yml" "github-action-s3-upload/$FOLDER_NAME/" + cp -v "solana-install-init-x86_64-pc-windows-msvc"* "github-action-s3-upload/$FOLDER_NAME" - name: Upload Artifacts if: ${{ steps.build.outputs.channel != '' || steps.build.outputs.tag != '' }} uses: actions/upload-artifact@v3 with: name: windows-artifact - path: windows-release/ + path: github-action-s3-upload/ windows-s3-upload: if: ${{ needs.windows-build.outputs.channel != '' || needs.windows-build.outputs.tag != '' }} @@ -79,16 +83,7 @@ jobs: uses: actions/download-artifact@v3 with: name: windows-artifact - path: .windows-release/ - - - name: Setup crediential - uses: "google-github-actions/auth@v2" - with: - credentials_json: "${{ secrets.GCS_RELEASE_BUCKET_WRITER_CREDIENTIAL }}" - - - name: Upload files to GCS - run: | - gcloud storage cp --recursive windows-release/* gs://anza-release/ + path: ./github-action-s3-upload - name: Upload uses: jakejarvis/s3-sync-action@master @@ -99,7 +94,7 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} AWS_REGION: "us-west-1" - SOURCE_DIR: "windows-release" + SOURCE_DIR: "github-action-s3-upload" windows-gh-release: if: ${{ needs.windows-build.outputs.tag != '' }} @@ -110,7 +105,7 @@ jobs: uses: actions/download-artifact@v3 with: name: windows-artifact - path: .windows-release/ + path: ./github-action-s3-upload - name: Release uses: softprops/action-gh-release@v1 @@ -118,4 +113,4 @@ jobs: tag_name: ${{ needs.windows-build.outputs.tag }} draft: true files: | - windows-release/${{ needs.windows-build.outputs.tag }}/* + github-action-s3-upload/${{ needs.windows-build.outputs.tag }}/* diff --git a/ci/publish-installer.sh b/ci/publish-installer.sh index e58fd939dd1a40..4b5345ae0d26fe 100755 --- a/ci/publish-installer.sh +++ b/ci/publish-installer.sh @@ -26,14 +26,14 @@ fi # upload install script source ci/upload-ci-artifact.sh -cat >release.anza.xyz-install <release.solana.com-install <>release.anza.xyz-install +cat install/solana-install-init.sh >>release.solana.com-install -echo --- GCS: "install" -upload-gcs-artifact "/solana/release.anza.xyz-install" "gs://anza-release/$CHANNEL_OR_TAG/install" +echo --- AWS S3 Store: "install" +upload-s3-artifact "/solana/release.solana.com-install" "s3://release.solana.com/$CHANNEL_OR_TAG/install" echo Published to: -ci/format-url.sh https://release.anza.xyz/"$CHANNEL_OR_TAG"/install +ci/format-url.sh https://release.solana.com/"$CHANNEL_OR_TAG"/install diff --git a/ci/publish-tarball.sh b/ci/publish-tarball.sh index 5c64f09564fe9f..ff72bb7da2d066 100755 --- a/ci/publish-tarball.sh +++ b/ci/publish-tarball.sh @@ -118,11 +118,11 @@ for file in "${TARBALL_BASENAME}"-$TARGET.tar.bz2 "${TARBALL_BASENAME}"-$TARGET. fi if [[ -n $BUILDKITE ]]; then - echo --- GCS Store: "$file" - upload-gcs-artifact "/solana/$file" gs://anza-release/"$CHANNEL_OR_TAG"/"$file" + echo --- AWS S3 Store: "$file" + upload-s3-artifact "/solana/$file" s3://release.solana.com/"$CHANNEL_OR_TAG"/"$file" echo Published to: - $DRYRUN ci/format-url.sh https://release.anza.xyz/"$CHANNEL_OR_TAG"/"$file" + $DRYRUN ci/format-url.sh https://release.solana.com/"$CHANNEL_OR_TAG"/"$file" if [[ -n $TAG ]]; then ci/upload-github-release-asset.sh "$file" diff --git a/ci/upload-ci-artifact.sh b/ci/upload-ci-artifact.sh index e7cc34ab2b2d8c..1236da9f27114a 100644 --- a/ci/upload-ci-artifact.sh +++ b/ci/upload-ci-artifact.sh @@ -40,13 +40,3 @@ upload-s3-artifact() { docker run "${args[@]}" ) } - -upload-gcs-artifact() { - echo "--- artifact: $1 to $2" - docker run --rm \ - -v "$GCS_RELEASE_BUCKET_WRITER_CREDIENTIAL:/application_default_credentials.json" \ - -v "$PWD:/solana" \ - -e CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=/application_default_credentials.json \ - gcr.io/google.com/cloudsdktool/google-cloud-cli:latest \ - gcloud storage cp "$1" "$2" -} diff --git a/ci/upload-github-release-asset.sh b/ci/upload-github-release-asset.sh index 229fb8993edafd..ca2ae2a8f60443 100755 --- a/ci/upload-github-release-asset.sh +++ b/ci/upload-github-release-asset.sh @@ -26,7 +26,7 @@ fi # Force CI_REPO_SLUG since sometimes # BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG is not set correctly, causing the # artifact upload to fail -CI_REPO_SLUG=anza-xyz/agave +CI_REPO_SLUG=solana-labs/solana #if [[ -z $CI_REPO_SLUG ]]; then # echo Error: CI_REPO_SLUG not defined # exit 1 diff --git a/install/solana-install-init.sh b/install/solana-install-init.sh index 4f28e300be52ab..db36dc61e2ff30 100755 --- a/install/solana-install-init.sh +++ b/install/solana-install-init.sh @@ -16,9 +16,9 @@ { # this ensures the entire script is downloaded # if [ -z "$SOLANA_DOWNLOAD_ROOT" ]; then - SOLANA_DOWNLOAD_ROOT="https://github.com/anza-xyz/agave/releases/download/" + SOLANA_DOWNLOAD_ROOT="https://github.com/solana-labs/solana/releases/download/" fi -GH_LATEST_RELEASE="https://api.github.com/repos/anza-xyz/agave/releases/latest" +GH_LATEST_RELEASE="https://api.github.com/repos/solana-labs/solana/releases/latest" set -e diff --git a/install/src/command.rs b/install/src/command.rs index 218e815467e9a9..d7b92c17690bda 100644 --- a/install/src/command.rs +++ b/install/src/command.rs @@ -572,7 +572,7 @@ pub fn init( fn github_release_download_url(release_semver: &str) -> String { format!( - "https://github.com/anza-xyz/agave/releases/download/v{}/solana-release-{}.tar.bz2", + "https://github.com/solana-labs/solana/releases/download/v{}/solana-release-{}.tar.bz2", release_semver, crate::build_env::TARGET ) @@ -580,7 +580,7 @@ fn github_release_download_url(release_semver: &str) -> String { fn release_channel_download_url(release_channel: &str) -> String { format!( - "https://release.anza.xyz/{}/solana-release-{}.tar.bz2", + "https://release.solana.com/{}/solana-release-{}.tar.bz2", release_channel, crate::build_env::TARGET ) @@ -588,7 +588,7 @@ fn release_channel_download_url(release_channel: &str) -> String { fn release_channel_version_url(release_channel: &str) -> String { format!( - "https://release.anza.xyz/{}/solana-release-{}.yml", + "https://release.solana.com/{}/solana-release-{}.yml", release_channel, crate::build_env::TARGET ) @@ -905,7 +905,7 @@ fn check_for_newer_github_release( while page == 1 || releases.len() == PER_PAGE { let url = reqwest::Url::parse_with_params( - "https://api.github.com/repos/anza-xyz/agave/releases", + "https://api.github.com/repos/solana-labs/solana/releases", &[ ("per_page", &format!("{PER_PAGE}")), ("page", &format!("{page}")), From cb4ddfb5c18b478d95594f333ee82f502a7ca596 Mon Sep 17 00:00:00 2001 From: Yihau Chen Date: Thu, 7 Mar 2024 12:18:39 +0800 Subject: [PATCH 20/71] ci: disable dependabot (#35430) --- .github/dependabot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 95e3fb34445ee0..e730708b694e1d 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -13,4 +13,4 @@ updates: timezone: America/Los_Angeles #labels: # - "automerge" - open-pull-requests-limit: 6 + open-pull-requests-limit: 0 From 556a7499487d32359019e1e3f8fd3a28e2033b0b Mon Sep 17 00:00:00 2001 From: gcmutator <134900551+gcmutator@users.noreply.github.com> Date: Sat, 9 Mar 2024 13:01:33 +0800 Subject: [PATCH 21/71] remove repetitive words (#35434) Signed-off-by: gcmutator <329964069@qq.com> --- core/src/consensus/heaviest_subtree_fork_choice.rs | 6 +++--- core/src/replay_stage.rs | 2 +- docs/static/katex/katex.js | 2 +- docs/static/katex/katex.mjs | 2 +- dos/src/main.rs | 2 +- download-utils/src/lib.rs | 4 ++-- local-cluster/tests/local_cluster.rs | 2 +- sdk/program/src/secp256k1_recover.rs | 2 +- sdk/src/secp256k1_instruction.rs | 2 +- send-transaction-service/src/tpu_info.rs | 2 +- 10 files changed, 13 insertions(+), 13 deletions(-) diff --git a/core/src/consensus/heaviest_subtree_fork_choice.rs b/core/src/consensus/heaviest_subtree_fork_choice.rs index 9a67069c2dfefd..62f507898ccf6c 100644 --- a/core/src/consensus/heaviest_subtree_fork_choice.rs +++ b/core/src/consensus/heaviest_subtree_fork_choice.rs @@ -1167,7 +1167,7 @@ impl HeaviestSubtreeForkChoice { // // In this scenario only 60% of the network has voted before the duplicate proof for Slot 1 and 1' // was viewed. Neither version of the slot will reach the duplicate confirmed threshold, so it is - // critical that a new fork Slot 2 from Slot 0 is created to allow the the validators on Slot 1 and + // critical that a new fork Slot 2 from Slot 0 is created to allow the validators on Slot 1 and // Slot 1' to switch. Since the `best_slot` is an ancestor of the last vote (Slot 0 is ancestor of last // vote Slot 1 or Slot 1'), we will trigger `SwitchForkDecision::FailedSwitchDuplicateRollback`, which // will create an alternate fork off of Slot 0. Once this alternate fork is created, the `best_slot` @@ -3514,7 +3514,7 @@ mod test { (vote_pubkeys[1], duplicate_leaves_descended_from_5[0]), ]; - // The best slot should be the the smallest leaf descended from 4 + // The best slot should be the smallest leaf descended from 4 assert_eq!( heaviest_subtree_fork_choice.add_votes( pubkey_votes.iter(), @@ -3560,7 +3560,7 @@ mod test { .., ) = setup_mark_invalid_forks_duplicate_tests(); - // Marking candidate as valid again will choose the the heaviest leaf of + // Marking candidate as valid again will choose the heaviest leaf of // the newly valid branch let duplicate_slot = duplicate_leaves_descended_from_4[0].0; let duplicate_descendant = (duplicate_slot + 1, Hash::new_unique()); diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs index 46014e3f7912de..5c21cc3c3885e5 100644 --- a/core/src/replay_stage.rs +++ b/core/src/replay_stage.rs @@ -3558,7 +3558,7 @@ impl ReplayStage { SwitchForkDecision::SameFork } else { // If we can't switch and our last vote was on a non-duplicate/confirmed slot, then - // reset to the the next votable bank on the same fork as our last vote, + // reset to the next votable bank on the same fork as our last vote, // but don't vote. // We don't just reset to the heaviest fork when switch threshold fails because diff --git a/docs/static/katex/katex.js b/docs/static/katex/katex.js index e5d316691883bf..b289a482ae5662 100644 --- a/docs/static/katex/katex.js +++ b/docs/static/katex/katex.js @@ -3662,7 +3662,7 @@ function assertSpan(group) { // In TeX, there are actually three sets of dimensions, one for each of // textstyle (size index 5 and higher: >=9pt), scriptstyle (size index 3 and 4: // 7-8pt), and scriptscriptstyle (size index 1 and 2: 5-6pt). These are -// provided in the the arrays below, in that order. +// provided in the arrays below, in that order. // // The font metrics are stored in fonts cmsy10, cmsy7, and cmsy5 respsectively. // This was determined by running the following script: diff --git a/docs/static/katex/katex.mjs b/docs/static/katex/katex.mjs index 488d2101c21232..56abd045543143 100644 --- a/docs/static/katex/katex.mjs +++ b/docs/static/katex/katex.mjs @@ -3686,7 +3686,7 @@ var metricMap = { // In TeX, there are actually three sets of dimensions, one for each of // textstyle (size index 5 and higher: >=9pt), scriptstyle (size index 3 and 4: // 7-8pt), and scriptscriptstyle (size index 1 and 2: 5-6pt). These are -// provided in the the arrays below, in that order. +// provided in the arrays below, in that order. // // The font metrics are stored in fonts cmsy10, cmsy7, and cmsy5 respsectively. // This was determined by running the following script: diff --git a/dos/src/main.rs b/dos/src/main.rs index 8e6c3c5b2b11b5..19a239d145e638 100644 --- a/dos/src/main.rs +++ b/dos/src/main.rs @@ -547,7 +547,7 @@ fn create_payers( // Assume that if we use valid blockhash, we also have a payer if valid_blockhash { // each payer is used to fund transaction - // transactions are built to be invalid so the the amount here is arbitrary + // transactions are built to be invalid so the amount here is arbitrary let funding_key = Keypair::new(); let funding_key = Arc::new(funding_key); let res = generate_and_fund_keypairs( diff --git a/download-utils/src/lib.rs b/download-utils/src/lib.rs index 17d50d31b55628..c4da82a21bb46b 100644 --- a/download-utils/src/lib.rs +++ b/download-utils/src/lib.rs @@ -39,7 +39,7 @@ fn new_spinner_progress_bar() -> ProgressBar { pub struct DownloadProgressRecord { // Duration since the beginning of the download pub elapsed_time: Duration, - // Duration since the the last notification + // Duration since the last notification pub last_elapsed_time: Duration, // the bytes/sec speed measured for the last notification period pub last_throughput: f32, @@ -51,7 +51,7 @@ pub struct DownloadProgressRecord { pub current_bytes: usize, // percentage downloaded pub percentage_done: f32, - // Estimated remaining time (in seconds) to finish the download if it keeps at the the last download speed + // Estimated remaining time (in seconds) to finish the download if it keeps at the last download speed pub estimated_remaining_time: f32, // The times of the progress is being notified, it starts from 1 and increments by 1 each time pub notification_count: u64, diff --git a/local-cluster/tests/local_cluster.rs b/local-cluster/tests/local_cluster.rs index 3b18ba44bf2d03..86d510920d018f 100644 --- a/local-cluster/tests/local_cluster.rs +++ b/local-cluster/tests/local_cluster.rs @@ -5072,7 +5072,7 @@ fn test_boot_from_local_state() { // 0 // \--- 2 // -// 1. > DUPLICATE_THRESHOLD of the nodes vote on some version of the the duplicate block 3, +// 1. > DUPLICATE_THRESHOLD of the nodes vote on some version of the duplicate block 3, // but don't immediately duplicate confirm so they remove 3 from fork choice and reset PoH back to 1. // 2. All the votes on 3 don't land because there are no further blocks building off 3. // 3. Some < SWITCHING_THRESHOLD of nodes vote on 2, making it the heaviest fork because no votes on 3 landed diff --git a/sdk/program/src/secp256k1_recover.rs b/sdk/program/src/secp256k1_recover.rs index f688e7d485497e..6c47cbdc183051 100644 --- a/sdk/program/src/secp256k1_recover.rs +++ b/sdk/program/src/secp256k1_recover.rs @@ -233,7 +233,7 @@ impl Secp256k1Pubkey { /// lengths of `hash` and `signature` beforehand. /// /// When run on-chain this function will not directly validate the lengths of -/// `hash` and `signature`. It will assume they are the the correct lengths and +/// `hash` and `signature`. It will assume they are the correct lengths and /// pass their pointers to the runtime, which will interpret them as 32-byte and /// 64-byte buffers. If the provided slices are too short, the runtime will read /// invalid data and attempt to interpret it, most likely returning an error, diff --git a/sdk/src/secp256k1_instruction.rs b/sdk/src/secp256k1_instruction.rs index a7bdff0b6dfb22..97d61b52df42b3 100644 --- a/sdk/src/secp256k1_instruction.rs +++ b/sdk/src/secp256k1_instruction.rs @@ -98,7 +98,7 @@ //! Many steps must be done manually. //! //! The `solana_program` crate provides no APIs to assist in interpreting -//! the the secp256k1 instruction data. It must be done manually. +//! the secp256k1 instruction data. It must be done manually. //! //! The secp256k1 program is implemented with the [`libsecp256k1`] crate, //! which clients may also want to use. diff --git a/send-transaction-service/src/tpu_info.rs b/send-transaction-service/src/tpu_info.rs index 456a0ced0d3304..9ea4cc782c95c3 100644 --- a/send-transaction-service/src/tpu_info.rs +++ b/send-transaction-service/src/tpu_info.rs @@ -3,7 +3,7 @@ use {solana_client::connection_cache::Protocol, solana_sdk::clock::Slot, std::ne pub trait TpuInfo { fn refresh_recent_peers(&mut self); fn get_leader_tpus(&self, max_count: u64, protocol: Protocol) -> Vec<&SocketAddr>; - /// In addition to the the tpu address, also return the leader slot + /// In addition to the tpu address, also return the leader slot fn get_leader_tpus_with_slots( &self, max_count: u64, From e81ecfbe9fbc2b6f95e6f8f5f88cb1af39cb24aa Mon Sep 17 00:00:00 2001 From: steviez Date: Fri, 8 Mar 2024 23:36:52 -0600 Subject: [PATCH 22/71] Revert "remove repetitive words (#35434)" (#35436) This reverts commit 556a7499487d32359019e1e3f8fd3a28e2033b0b. --- core/src/consensus/heaviest_subtree_fork_choice.rs | 6 +++--- core/src/replay_stage.rs | 2 +- docs/static/katex/katex.js | 2 +- docs/static/katex/katex.mjs | 2 +- dos/src/main.rs | 2 +- download-utils/src/lib.rs | 4 ++-- local-cluster/tests/local_cluster.rs | 2 +- sdk/program/src/secp256k1_recover.rs | 2 +- sdk/src/secp256k1_instruction.rs | 2 +- send-transaction-service/src/tpu_info.rs | 2 +- 10 files changed, 13 insertions(+), 13 deletions(-) diff --git a/core/src/consensus/heaviest_subtree_fork_choice.rs b/core/src/consensus/heaviest_subtree_fork_choice.rs index 62f507898ccf6c..9a67069c2dfefd 100644 --- a/core/src/consensus/heaviest_subtree_fork_choice.rs +++ b/core/src/consensus/heaviest_subtree_fork_choice.rs @@ -1167,7 +1167,7 @@ impl HeaviestSubtreeForkChoice { // // In this scenario only 60% of the network has voted before the duplicate proof for Slot 1 and 1' // was viewed. Neither version of the slot will reach the duplicate confirmed threshold, so it is - // critical that a new fork Slot 2 from Slot 0 is created to allow the validators on Slot 1 and + // critical that a new fork Slot 2 from Slot 0 is created to allow the the validators on Slot 1 and // Slot 1' to switch. Since the `best_slot` is an ancestor of the last vote (Slot 0 is ancestor of last // vote Slot 1 or Slot 1'), we will trigger `SwitchForkDecision::FailedSwitchDuplicateRollback`, which // will create an alternate fork off of Slot 0. Once this alternate fork is created, the `best_slot` @@ -3514,7 +3514,7 @@ mod test { (vote_pubkeys[1], duplicate_leaves_descended_from_5[0]), ]; - // The best slot should be the smallest leaf descended from 4 + // The best slot should be the the smallest leaf descended from 4 assert_eq!( heaviest_subtree_fork_choice.add_votes( pubkey_votes.iter(), @@ -3560,7 +3560,7 @@ mod test { .., ) = setup_mark_invalid_forks_duplicate_tests(); - // Marking candidate as valid again will choose the heaviest leaf of + // Marking candidate as valid again will choose the the heaviest leaf of // the newly valid branch let duplicate_slot = duplicate_leaves_descended_from_4[0].0; let duplicate_descendant = (duplicate_slot + 1, Hash::new_unique()); diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs index 5c21cc3c3885e5..46014e3f7912de 100644 --- a/core/src/replay_stage.rs +++ b/core/src/replay_stage.rs @@ -3558,7 +3558,7 @@ impl ReplayStage { SwitchForkDecision::SameFork } else { // If we can't switch and our last vote was on a non-duplicate/confirmed slot, then - // reset to the next votable bank on the same fork as our last vote, + // reset to the the next votable bank on the same fork as our last vote, // but don't vote. // We don't just reset to the heaviest fork when switch threshold fails because diff --git a/docs/static/katex/katex.js b/docs/static/katex/katex.js index b289a482ae5662..e5d316691883bf 100644 --- a/docs/static/katex/katex.js +++ b/docs/static/katex/katex.js @@ -3662,7 +3662,7 @@ function assertSpan(group) { // In TeX, there are actually three sets of dimensions, one for each of // textstyle (size index 5 and higher: >=9pt), scriptstyle (size index 3 and 4: // 7-8pt), and scriptscriptstyle (size index 1 and 2: 5-6pt). These are -// provided in the arrays below, in that order. +// provided in the the arrays below, in that order. // // The font metrics are stored in fonts cmsy10, cmsy7, and cmsy5 respsectively. // This was determined by running the following script: diff --git a/docs/static/katex/katex.mjs b/docs/static/katex/katex.mjs index 56abd045543143..488d2101c21232 100644 --- a/docs/static/katex/katex.mjs +++ b/docs/static/katex/katex.mjs @@ -3686,7 +3686,7 @@ var metricMap = { // In TeX, there are actually three sets of dimensions, one for each of // textstyle (size index 5 and higher: >=9pt), scriptstyle (size index 3 and 4: // 7-8pt), and scriptscriptstyle (size index 1 and 2: 5-6pt). These are -// provided in the arrays below, in that order. +// provided in the the arrays below, in that order. // // The font metrics are stored in fonts cmsy10, cmsy7, and cmsy5 respsectively. // This was determined by running the following script: diff --git a/dos/src/main.rs b/dos/src/main.rs index 19a239d145e638..8e6c3c5b2b11b5 100644 --- a/dos/src/main.rs +++ b/dos/src/main.rs @@ -547,7 +547,7 @@ fn create_payers( // Assume that if we use valid blockhash, we also have a payer if valid_blockhash { // each payer is used to fund transaction - // transactions are built to be invalid so the amount here is arbitrary + // transactions are built to be invalid so the the amount here is arbitrary let funding_key = Keypair::new(); let funding_key = Arc::new(funding_key); let res = generate_and_fund_keypairs( diff --git a/download-utils/src/lib.rs b/download-utils/src/lib.rs index c4da82a21bb46b..17d50d31b55628 100644 --- a/download-utils/src/lib.rs +++ b/download-utils/src/lib.rs @@ -39,7 +39,7 @@ fn new_spinner_progress_bar() -> ProgressBar { pub struct DownloadProgressRecord { // Duration since the beginning of the download pub elapsed_time: Duration, - // Duration since the last notification + // Duration since the the last notification pub last_elapsed_time: Duration, // the bytes/sec speed measured for the last notification period pub last_throughput: f32, @@ -51,7 +51,7 @@ pub struct DownloadProgressRecord { pub current_bytes: usize, // percentage downloaded pub percentage_done: f32, - // Estimated remaining time (in seconds) to finish the download if it keeps at the last download speed + // Estimated remaining time (in seconds) to finish the download if it keeps at the the last download speed pub estimated_remaining_time: f32, // The times of the progress is being notified, it starts from 1 and increments by 1 each time pub notification_count: u64, diff --git a/local-cluster/tests/local_cluster.rs b/local-cluster/tests/local_cluster.rs index 86d510920d018f..3b18ba44bf2d03 100644 --- a/local-cluster/tests/local_cluster.rs +++ b/local-cluster/tests/local_cluster.rs @@ -5072,7 +5072,7 @@ fn test_boot_from_local_state() { // 0 // \--- 2 // -// 1. > DUPLICATE_THRESHOLD of the nodes vote on some version of the duplicate block 3, +// 1. > DUPLICATE_THRESHOLD of the nodes vote on some version of the the duplicate block 3, // but don't immediately duplicate confirm so they remove 3 from fork choice and reset PoH back to 1. // 2. All the votes on 3 don't land because there are no further blocks building off 3. // 3. Some < SWITCHING_THRESHOLD of nodes vote on 2, making it the heaviest fork because no votes on 3 landed diff --git a/sdk/program/src/secp256k1_recover.rs b/sdk/program/src/secp256k1_recover.rs index 6c47cbdc183051..f688e7d485497e 100644 --- a/sdk/program/src/secp256k1_recover.rs +++ b/sdk/program/src/secp256k1_recover.rs @@ -233,7 +233,7 @@ impl Secp256k1Pubkey { /// lengths of `hash` and `signature` beforehand. /// /// When run on-chain this function will not directly validate the lengths of -/// `hash` and `signature`. It will assume they are the correct lengths and +/// `hash` and `signature`. It will assume they are the the correct lengths and /// pass their pointers to the runtime, which will interpret them as 32-byte and /// 64-byte buffers. If the provided slices are too short, the runtime will read /// invalid data and attempt to interpret it, most likely returning an error, diff --git a/sdk/src/secp256k1_instruction.rs b/sdk/src/secp256k1_instruction.rs index 97d61b52df42b3..a7bdff0b6dfb22 100644 --- a/sdk/src/secp256k1_instruction.rs +++ b/sdk/src/secp256k1_instruction.rs @@ -98,7 +98,7 @@ //! Many steps must be done manually. //! //! The `solana_program` crate provides no APIs to assist in interpreting -//! the secp256k1 instruction data. It must be done manually. +//! the the secp256k1 instruction data. It must be done manually. //! //! The secp256k1 program is implemented with the [`libsecp256k1`] crate, //! which clients may also want to use. diff --git a/send-transaction-service/src/tpu_info.rs b/send-transaction-service/src/tpu_info.rs index 9ea4cc782c95c3..456a0ced0d3304 100644 --- a/send-transaction-service/src/tpu_info.rs +++ b/send-transaction-service/src/tpu_info.rs @@ -3,7 +3,7 @@ use {solana_client::connection_cache::Protocol, solana_sdk::clock::Slot, std::ne pub trait TpuInfo { fn refresh_recent_peers(&mut self); fn get_leader_tpus(&self, max_count: u64, protocol: Protocol) -> Vec<&SocketAddr>; - /// In addition to the tpu address, also return the leader slot + /// In addition to the the tpu address, also return the leader slot fn get_leader_tpus_with_slots( &self, max_count: u64, From f6cd7c278e14fd7985f81d02732d99d2f8265fc0 Mon Sep 17 00:00:00 2001 From: Brooks Date: Tue, 5 Mar 2024 12:02:47 -0500 Subject: [PATCH 23/71] Increases account hash's stack buffer to hold 200 bytes of data (#56) --- accounts-db/Cargo.toml | 2 +- accounts-db/src/accounts_db.rs | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/accounts-db/Cargo.toml b/accounts-db/Cargo.toml index 702f14f9f3b07d..0fc5a381fbda5e 100644 --- a/accounts-db/Cargo.toml +++ b/accounts-db/Cargo.toml @@ -42,7 +42,7 @@ regex = { workspace = true } seqlock = { workspace = true } serde = { workspace = true, features = ["rc"] } serde_derive = { workspace = true } -smallvec = { workspace = true } +smallvec = { workspace = true, features = ["const_generics"] } solana-bucket-map = { workspace = true } solana-config-program = { workspace = true } solana-frozen-abi = { workspace = true } diff --git a/accounts-db/src/accounts_db.rs b/accounts-db/src/accounts_db.rs index 1f3c36876f4531..cf4d17745b1b73 100644 --- a/accounts-db/src/accounts_db.rs +++ b/accounts-db/src/accounts_db.rs @@ -6119,17 +6119,18 @@ impl AccountsDb { } let mut hasher = blake3::Hasher::new(); - // allocate 128 bytes buffer on the stack - const BUFFER_SIZE: usize = 128; - const METADATA_SIZE: usize = 8 /* lamports */ + 8 /* rent_epoch */ + 1 /* executable */ + 32 /* owner */ + 32 /* pubkey */; - const REMAINING_SIZE: usize = BUFFER_SIZE - METADATA_SIZE; + // allocate a buffer on the stack that's big enough + // to hold a token account or a stake account + const META_SIZE: usize = 8 /* lamports */ + 8 /* rent_epoch */ + 1 /* executable */ + 32 /* owner */ + 32 /* pubkey */; + const DATA_SIZE: usize = 200; // stake acounts are 200 B and token accounts are 165-182ish B + const BUFFER_SIZE: usize = META_SIZE + DATA_SIZE; let mut buffer = SmallVec::<[u8; BUFFER_SIZE]>::new(); // collect lamports, rent_epoch into buffer to hash buffer.extend_from_slice(&lamports.to_le_bytes()); buffer.extend_from_slice(&rent_epoch.to_le_bytes()); - if data.len() > REMAINING_SIZE { + if data.len() > DATA_SIZE { // For larger accounts whose data can't fit into the buffer, update the hash now. hasher.update(&buffer); buffer.clear(); From 4753dcae717e90f12a02c33b7e3b05a4a1ec92db Mon Sep 17 00:00:00 2001 From: steviez Date: Tue, 5 Mar 2024 12:09:17 -0600 Subject: [PATCH 24/71] Rename and uniquify QUIC thread names (#28) When viewing in various tools such as gdb and perf, it is not easy to distinguish which threads are serving which function (TPU or TPU FWD) --- client/src/connection_cache.rs | 1 + core/src/tpu.rs | 2 ++ quic-client/src/quic_client.rs | 2 +- quic-client/tests/quic_client.rs | 3 +++ streamer/src/quic.rs | 26 +++++++++++++++----------- 5 files changed, 22 insertions(+), 12 deletions(-) diff --git a/client/src/connection_cache.rs b/client/src/connection_cache.rs index b53b66b155e719..a94bc7cd3d8ca8 100644 --- a/client/src/connection_cache.rs +++ b/client/src/connection_cache.rs @@ -260,6 +260,7 @@ mod tests { thread: response_recv_thread, key_updater: _, } = solana_streamer::quic::spawn_server( + "solQuicTest", "quic_streamer_test", response_recv_socket, &keypair2, diff --git a/core/src/tpu.rs b/core/src/tpu.rs index 548b299148d935..0f5431da4eb43c 100644 --- a/core/src/tpu.rs +++ b/core/src/tpu.rs @@ -153,6 +153,7 @@ impl Tpu { thread: tpu_quic_t, key_updater, } = spawn_server( + "solQuicTpu", "quic_streamer_tpu", transactions_quic_sockets, keypair, @@ -172,6 +173,7 @@ impl Tpu { thread: tpu_forwards_quic_t, key_updater: forwards_key_updater, } = spawn_server( + "solQuicTpuFwd", "quic_streamer_tpu_forwards", transactions_forwards_quic_sockets, keypair, diff --git a/quic-client/src/quic_client.rs b/quic-client/src/quic_client.rs index f057980c79fe06..8c8e8e5338993f 100644 --- a/quic-client/src/quic_client.rs +++ b/quic-client/src/quic_client.rs @@ -69,7 +69,7 @@ lazy_static! { static ref ASYNC_TASK_SEMAPHORE: AsyncTaskSemaphore = AsyncTaskSemaphore::new(MAX_OUTSTANDING_TASK); static ref RUNTIME: Runtime = tokio::runtime::Builder::new_multi_thread() - .thread_name("quic-client") + .thread_name("solQuicClientRt") .enable_all() .build() .unwrap(); diff --git a/quic-client/tests/quic_client.rs b/quic-client/tests/quic_client.rs index 658ee6a57d672d..0237fc21d098dc 100644 --- a/quic-client/tests/quic_client.rs +++ b/quic-client/tests/quic_client.rs @@ -72,6 +72,7 @@ mod tests { thread: t, key_updater: _, } = solana_streamer::quic::spawn_server( + "solQuicTest", "quic_streamer_test", s.try_clone().unwrap(), &keypair, @@ -212,6 +213,7 @@ mod tests { thread: request_recv_thread, key_updater: _, } = solana_streamer::quic::spawn_server( + "solQuicTest", "quic_streamer_test", request_recv_socket.try_clone().unwrap(), &keypair, @@ -239,6 +241,7 @@ mod tests { thread: response_recv_thread, key_updater: _, } = solana_streamer::quic::spawn_server( + "solQuicTest", "quic_streamer_test", response_recv_socket, &keypair2, diff --git a/streamer/src/quic.rs b/streamer/src/quic.rs index 69a75532b8ca68..a7a08c73f4833b 100644 --- a/streamer/src/quic.rs +++ b/streamer/src/quic.rs @@ -100,9 +100,9 @@ pub(crate) fn configure_server( Ok((server_config, cert_chain_pem)) } -fn rt() -> Runtime { +fn rt(name: String) -> Runtime { tokio::runtime::Builder::new_multi_thread() - .thread_name("quic-server") + .thread_name(name) .enable_all() .build() .unwrap() @@ -431,7 +431,8 @@ impl StreamStats { #[allow(clippy::too_many_arguments)] pub fn spawn_server( - name: &'static str, + thread_name: &'static str, + metrics_name: &'static str, sock: UdpSocket, keypair: &Keypair, packet_sender: Sender, @@ -443,11 +444,11 @@ pub fn spawn_server( wait_for_chunk_timeout: Duration, coalesce: Duration, ) -> Result { - let runtime = rt(); + let runtime = rt(format!("{thread_name}Rt")); let (endpoint, _stats, task) = { let _guard = runtime.enter(); crate::nonblocking::quic::spawn_server( - name, + metrics_name, sock, keypair, packet_sender, @@ -461,7 +462,7 @@ pub fn spawn_server( ) }?; let handle = thread::Builder::new() - .name("solQuicServer".into()) + .name(thread_name.into()) .spawn(move || { if let Err(e) = runtime.block_on(task) { warn!("error from runtime.block_on: {:?}", e); @@ -505,6 +506,7 @@ mod test { thread: t, key_updater: _, } = spawn_server( + "solQuicTest", "quic_streamer_test", s, &keypair, @@ -532,7 +534,7 @@ mod test { fn test_quic_timeout() { solana_logger::setup(); let (t, exit, receiver, server_address) = setup_quic_server(); - let runtime = rt(); + let runtime = rt("solQuicTestRt".to_string()); runtime.block_on(check_timeout(receiver, server_address)); exit.store(true, Ordering::Relaxed); t.join().unwrap(); @@ -543,7 +545,7 @@ mod test { solana_logger::setup(); let (t, exit, _receiver, server_address) = setup_quic_server(); - let runtime = rt(); + let runtime = rt("solQuicTestRt".to_string()); runtime.block_on(check_block_multiple_connections(server_address)); exit.store(true, Ordering::Relaxed); t.join().unwrap(); @@ -563,6 +565,7 @@ mod test { thread: t, key_updater: _, } = spawn_server( + "solQuicTest", "quic_streamer_test", s, &keypair, @@ -577,7 +580,7 @@ mod test { ) .unwrap(); - let runtime = rt(); + let runtime = rt("solQuicTestRt".to_string()); runtime.block_on(check_multiple_streams(receiver, server_address)); exit.store(true, Ordering::Relaxed); t.join().unwrap(); @@ -588,7 +591,7 @@ mod test { solana_logger::setup(); let (t, exit, receiver, server_address) = setup_quic_server(); - let runtime = rt(); + let runtime = rt("solQuicTestRt".to_string()); runtime.block_on(check_multiple_writes(receiver, server_address, None)); exit.store(true, Ordering::Relaxed); t.join().unwrap(); @@ -608,6 +611,7 @@ mod test { thread: t, key_updater: _, } = spawn_server( + "solQuicTest", "quic_streamer_test", s, &keypair, @@ -622,7 +626,7 @@ mod test { ) .unwrap(); - let runtime = rt(); + let runtime = rt("solQuicTestRt".to_string()); runtime.block_on(check_unstaked_node_connect_failure(server_address)); exit.store(true, Ordering::Relaxed); t.join().unwrap(); From f3c6c08752e5fbc1b702e91c99eb9ff3fbe748a1 Mon Sep 17 00:00:00 2001 From: steviez Date: Tue, 5 Mar 2024 22:02:04 -0600 Subject: [PATCH 25/71] Give SigVerify and ShredFetch threads unique names (#98) - solTvuFetchPmod ==> solTvuPktMod + solTvuRepPktMod - solSigVerifier ==> solSigVerTpu + solSigVerTpuVot --- core/benches/sigverify_stage.rs | 2 +- core/src/shred_fetch_stage.rs | 5 ++++- core/src/sigverify_stage.rs | 23 +++++++++-------------- core/src/tpu.rs | 9 +++++++-- 4 files changed, 21 insertions(+), 18 deletions(-) diff --git a/core/benches/sigverify_stage.rs b/core/benches/sigverify_stage.rs index 70f33020dd3e70..7013f718e4ab2e 100644 --- a/core/benches/sigverify_stage.rs +++ b/core/benches/sigverify_stage.rs @@ -160,7 +160,7 @@ fn bench_sigverify_stage(bencher: &mut Bencher, use_same_tx: bool) { let (packet_s, packet_r) = unbounded(); let (verified_s, verified_r) = BankingTracer::channel_for_test(); let verifier = TransactionSigVerifier::new(verified_s); - let stage = SigVerifyStage::new(packet_r, verifier, "bench"); + let stage = SigVerifyStage::new(packet_r, verifier, "solSigVerBench", "bench"); bencher.iter(move || { let now = Instant::now(); diff --git a/core/src/shred_fetch_stage.rs b/core/src/shred_fetch_stage.rs index 39cc193adad96e..84f1520e649822 100644 --- a/core/src/shred_fetch_stage.rs +++ b/core/src/shred_fetch_stage.rs @@ -148,6 +148,7 @@ impl ShredFetchStage { #[allow(clippy::too_many_arguments)] fn packet_modifier( receiver_thread_name: &'static str, + modifier_thread_name: &'static str, sockets: Vec>, exit: Arc, sender: Sender, @@ -178,7 +179,7 @@ impl ShredFetchStage { }) .collect(); let modifier_hdl = Builder::new() - .name("solTvuFetchPMod".to_string()) + .name(modifier_thread_name.to_string()) .spawn(move || { let repair_context = repair_context .as_ref() @@ -215,6 +216,7 @@ impl ShredFetchStage { let (mut tvu_threads, tvu_filter) = Self::packet_modifier( "solRcvrShred", + "solTvuPktMod", sockets, exit.clone(), sender.clone(), @@ -229,6 +231,7 @@ impl ShredFetchStage { let (repair_receiver, repair_handler) = Self::packet_modifier( "solRcvrShredRep", + "solTvuRepPktMod", vec![repair_socket.clone()], exit.clone(), sender.clone(), diff --git a/core/src/sigverify_stage.rs b/core/src/sigverify_stage.rs index e5e06a3bc701c9..cde1735611c0d0 100644 --- a/core/src/sigverify_stage.rs +++ b/core/src/sigverify_stage.rs @@ -238,9 +238,11 @@ impl SigVerifyStage { pub fn new( packet_receiver: Receiver, verifier: T, - name: &'static str, + thread_name: &'static str, + metrics_name: &'static str, ) -> Self { - let thread_hdl = Self::verifier_services(packet_receiver, verifier, name); + let thread_hdl = + Self::verifier_service(packet_receiver, verifier, thread_name, metrics_name); Self { thread_hdl } } @@ -407,7 +409,8 @@ impl SigVerifyStage { fn verifier_service( packet_receiver: Receiver, mut verifier: T, - name: &'static str, + thread_name: &'static str, + metrics_name: &'static str, ) -> JoinHandle<()> { let mut stats = SigVerifierStats::default(); let mut last_print = Instant::now(); @@ -415,7 +418,7 @@ impl SigVerifyStage { const DEDUPER_FALSE_POSITIVE_RATE: f64 = 0.001; const DEDUPER_NUM_BITS: u64 = 63_999_979; Builder::new() - .name("solSigVerifier".to_string()) + .name(thread_name.to_string()) .spawn(move || { let mut rng = rand::thread_rng(); let mut deduper = Deduper::<2, [u8]>::new(&mut rng, DEDUPER_NUM_BITS); @@ -440,7 +443,7 @@ impl SigVerifyStage { } } if last_print.elapsed().as_secs() > 2 { - stats.report(name); + stats.report(metrics_name); stats = SigVerifierStats::default(); last_print = Instant::now(); } @@ -449,14 +452,6 @@ impl SigVerifyStage { .unwrap() } - fn verifier_services( - packet_receiver: Receiver, - verifier: T, - name: &'static str, - ) -> JoinHandle<()> { - Self::verifier_service(packet_receiver, verifier, name) - } - pub fn join(self) -> thread::Result<()> { self.thread_hdl.join() } @@ -552,7 +547,7 @@ mod tests { let (packet_s, packet_r) = unbounded(); let (verified_s, verified_r) = BankingTracer::channel_for_test(); let verifier = TransactionSigVerifier::new(verified_s); - let stage = SigVerifyStage::new(packet_r, verifier, "test"); + let stage = SigVerifyStage::new(packet_r, verifier, "solSigVerTest", "test"); let now = Instant::now(); let packets_per_batch = 128; diff --git a/core/src/tpu.rs b/core/src/tpu.rs index 0f5431da4eb43c..640caf64544d45 100644 --- a/core/src/tpu.rs +++ b/core/src/tpu.rs @@ -190,14 +190,19 @@ impl Tpu { let sigverify_stage = { let verifier = TransactionSigVerifier::new(non_vote_sender); - SigVerifyStage::new(packet_receiver, verifier, "tpu-verifier") + SigVerifyStage::new(packet_receiver, verifier, "solSigVerTpu", "tpu-verifier") }; let (tpu_vote_sender, tpu_vote_receiver) = banking_tracer.create_channel_tpu_vote(); let vote_sigverify_stage = { let verifier = TransactionSigVerifier::new_reject_non_vote(tpu_vote_sender); - SigVerifyStage::new(vote_packet_receiver, verifier, "tpu-vote-verifier") + SigVerifyStage::new( + vote_packet_receiver, + verifier, + "solSigVerTpuVot", + "tpu-vote-verifier", + ) }; let (gossip_vote_sender, gossip_vote_receiver) = From e6a71366fa557c4b26b7ba72ec115e1eab0cb647 Mon Sep 17 00:00:00 2001 From: Tyera Date: Wed, 6 Mar 2024 00:23:32 -0700 Subject: [PATCH 26/71] Cli: improve vote-account vote-authority display (#95) * Simplify vote-authority display * Add handling for new vote authority * Add proper None handling, because unit test (shouldn't happen IRL, though) * Unwrap->expect --- cli-output/src/cli_output.rs | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/cli-output/src/cli_output.rs b/cli-output/src/cli_output.rs index 0eca9cde5c1a52..62b66eddf27eb0 100644 --- a/cli-output/src/cli_output.rs +++ b/cli-output/src/cli_output.rs @@ -1648,7 +1648,23 @@ impl VerboseDisplay for CliAuthorizedVoters {} impl fmt::Display for CliAuthorizedVoters { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{:?}", self.authorized_voters) + if let Some((_epoch, current_authorized_voter)) = self.authorized_voters.first_key_value() { + write!(f, "{current_authorized_voter}")?; + } else { + write!(f, "None")?; + } + if self.authorized_voters.len() > 1 { + let (epoch, upcoming_authorized_voter) = self + .authorized_voters + .last_key_value() + .expect("CliAuthorizedVoters::authorized_voters.len() > 1"); + writeln!(f)?; + write!( + f, + " New Vote Authority as of Epoch {epoch}: {upcoming_authorized_voter}" + )?; + } + Ok(()) } } @@ -3379,12 +3395,12 @@ mod tests { ..CliVoteAccount::default() }; let s = format!("{c}"); - assert_eq!(s, "Account Balance: 0.00001 SOL\nValidator Identity: 11111111111111111111111111111111\nVote Authority: {}\nWithdraw Authority: \nCredits: 0\nCommission: 0%\nRoot Slot: ~\nRecent Timestamp: 1970-01-01T00:00:00Z from slot 0\nEpoch Rewards:\n Epoch Reward Slot Time Amount New Balance Percent Change APR Commission\n 1 100 1970-01-01 00:00:00 UTC ◎0.000000010 ◎0.000000100 11.000% 10.00% 1%\n 2 200 1970-01-12 13:46:40 UTC ◎0.000000012 ◎0.000000100 11.000% 13.00% 1%\n"); + assert_eq!(s, "Account Balance: 0.00001 SOL\nValidator Identity: 11111111111111111111111111111111\nVote Authority: None\nWithdraw Authority: \nCredits: 0\nCommission: 0%\nRoot Slot: ~\nRecent Timestamp: 1970-01-01T00:00:00Z from slot 0\nEpoch Rewards:\n Epoch Reward Slot Time Amount New Balance Percent Change APR Commission\n 1 100 1970-01-01 00:00:00 UTC ◎0.000000010 ◎0.000000100 11.000% 10.00% 1%\n 2 200 1970-01-12 13:46:40 UTC ◎0.000000012 ◎0.000000100 11.000% 13.00% 1%\n"); println!("{s}"); c.use_csv = true; let s = format!("{c}"); - assert_eq!(s, "Account Balance: 0.00001 SOL\nValidator Identity: 11111111111111111111111111111111\nVote Authority: {}\nWithdraw Authority: \nCredits: 0\nCommission: 0%\nRoot Slot: ~\nRecent Timestamp: 1970-01-01T00:00:00Z from slot 0\nEpoch Rewards:\nEpoch,Reward Slot,Time,Amount,New Balance,Percent Change,APR,Commission\n1,100,1970-01-01 00:00:00 UTC,0.00000001,0.0000001,11%,10.00%,1%\n2,200,1970-01-12 13:46:40 UTC,0.000000012,0.0000001,11%,13.00%,1%\n"); + assert_eq!(s, "Account Balance: 0.00001 SOL\nValidator Identity: 11111111111111111111111111111111\nVote Authority: None\nWithdraw Authority: \nCredits: 0\nCommission: 0%\nRoot Slot: ~\nRecent Timestamp: 1970-01-01T00:00:00Z from slot 0\nEpoch Rewards:\nEpoch,Reward Slot,Time,Amount,New Balance,Percent Change,APR,Commission\n1,100,1970-01-01 00:00:00 UTC,0.00000001,0.0000001,11%,10.00%,1%\n2,200,1970-01-12 13:46:40 UTC,0.000000012,0.0000001,11%,13.00%,1%\n"); println!("{s}"); } } From 1f7e6f269bae89ae84ee21ce6f24d6c5752cab90 Mon Sep 17 00:00:00 2001 From: Jon C Date: Wed, 6 Mar 2024 15:32:05 +0100 Subject: [PATCH 27/71] frozen-abi: Remove proc_macro_hygiene featurization (#109) --- frozen-abi/build.rs | 5 ----- frozen-abi/src/lib.rs | 1 - perf/build.rs | 5 ----- programs/address-lookup-table/src/lib.rs | 1 - sdk/program/src/lib.rs | 1 - sdk/src/lib.rs | 1 - 6 files changed, 14 deletions(-) diff --git a/frozen-abi/build.rs b/frozen-abi/build.rs index c9550c1c5c4f22..e17ca70cb4718b 100644 --- a/frozen-abi/build.rs +++ b/frozen-abi/build.rs @@ -17,11 +17,6 @@ fn main() { } Channel::Dev => { println!("cargo:rustc-cfg=RUSTC_WITH_SPECIALIZATION"); - // See https://github.com/solana-labs/solana/issues/11055 - // We may be running the custom `rust-bpf-builder` toolchain, - // which currently needs `#![feature(proc_macro_hygiene)]` to - // be applied. - println!("cargo:rustc-cfg=RUSTC_NEEDS_PROC_MACRO_HYGIENE"); } } } diff --git a/frozen-abi/src/lib.rs b/frozen-abi/src/lib.rs index 189535ccddaa74..4747cf64b9e50f 100644 --- a/frozen-abi/src/lib.rs +++ b/frozen-abi/src/lib.rs @@ -1,6 +1,5 @@ #![allow(incomplete_features)] #![cfg_attr(RUSTC_WITH_SPECIALIZATION, feature(specialization))] -#![cfg_attr(RUSTC_NEEDS_PROC_MACRO_HYGIENE, feature(proc_macro_hygiene))] // Allows macro expansion of `use ::solana_frozen_abi::*` to work within this crate extern crate self as solana_frozen_abi; diff --git a/perf/build.rs b/perf/build.rs index 4925ee898eb612..eef20dd887bc42 100644 --- a/perf/build.rs +++ b/perf/build.rs @@ -27,11 +27,6 @@ fn main() { } Channel::Dev => { println!("cargo:rustc-cfg=RUSTC_WITH_SPECIALIZATION"); - // See https://github.com/solana-labs/solana/issues/11055 - // We may be running the custom `rust-bpf-builder` toolchain, - // which currently needs `#![feature(proc_macro_hygiene)]` to - // be applied. - println!("cargo:rustc-cfg=RUSTC_NEEDS_PROC_MACRO_HYGIENE"); } } } diff --git a/programs/address-lookup-table/src/lib.rs b/programs/address-lookup-table/src/lib.rs index 737ec32c8f6782..737c35e4c4b2f4 100644 --- a/programs/address-lookup-table/src/lib.rs +++ b/programs/address-lookup-table/src/lib.rs @@ -1,6 +1,5 @@ #![allow(incomplete_features)] #![cfg_attr(RUSTC_WITH_SPECIALIZATION, feature(specialization))] -#![cfg_attr(RUSTC_NEEDS_PROC_MACRO_HYGIENE, feature(proc_macro_hygiene))] #[cfg(not(target_os = "solana"))] pub mod processor; diff --git a/sdk/program/src/lib.rs b/sdk/program/src/lib.rs index 54de9d817205a8..4d623524772ccb 100644 --- a/sdk/program/src/lib.rs +++ b/sdk/program/src/lib.rs @@ -465,7 +465,6 @@ #![allow(incomplete_features)] #![cfg_attr(RUSTC_WITH_SPECIALIZATION, feature(specialization))] -#![cfg_attr(RUSTC_NEEDS_PROC_MACRO_HYGIENE, feature(proc_macro_hygiene))] // Allows macro expansion of `use ::solana_program::*` to work within this crate extern crate self as solana_program; diff --git a/sdk/src/lib.rs b/sdk/src/lib.rs index 7c6b643884e449..ecc186f0494191 100644 --- a/sdk/src/lib.rs +++ b/sdk/src/lib.rs @@ -31,7 +31,6 @@ #![allow(incomplete_features)] #![cfg_attr(RUSTC_WITH_SPECIALIZATION, feature(specialization))] -#![cfg_attr(RUSTC_NEEDS_PROC_MACRO_HYGIENE, feature(proc_macro_hygiene))] // Allows macro expansion of `use ::solana_sdk::*` to work within this crate extern crate self as solana_sdk; From c9549399fbbc4707698fa2ad6a8437df6bd14a25 Mon Sep 17 00:00:00 2001 From: Tao Zhu <82401714+tao-stones@users.noreply.github.com> Date: Wed, 6 Mar 2024 11:08:49 -0600 Subject: [PATCH 28/71] assert simple vote tx const cost (#100) * assert simple vote tx const cost --- cost-model/src/transaction_cost.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/cost-model/src/transaction_cost.rs b/cost-model/src/transaction_cost.rs index e765eee3bc7038..76865fff30fd57 100644 --- a/cost-model/src/transaction_cost.rs +++ b/cost-model/src/transaction_cost.rs @@ -18,8 +18,19 @@ pub enum TransactionCost { impl TransactionCost { pub fn sum(&self) -> u64 { + #![allow(clippy::assertions_on_constants)] match self { - Self::SimpleVote { .. } => SIMPLE_VOTE_USAGE_COST, + Self::SimpleVote { .. } => { + const _: () = assert!( + SIMPLE_VOTE_USAGE_COST + == solana_vote_program::vote_processor::DEFAULT_COMPUTE_UNITS + + block_cost_limits::SIGNATURE_COST + + 2 * block_cost_limits::WRITE_LOCK_UNITS + + 8 + ); + + SIMPLE_VOTE_USAGE_COST + } Self::Transaction(usage_cost) => usage_cost.sum(), } } From f944227dba88558724c056c4f3a466df877b1c96 Mon Sep 17 00:00:00 2001 From: steviez Date: Wed, 6 Mar 2024 14:49:32 -0600 Subject: [PATCH 29/71] Use tokio directly instead of jsonrpc_server_utils's re-export (#116) --- Cargo.lock | 4 +- Cargo.toml | 1 - geyser-plugin-manager/Cargo.toml | 2 +- .../src/geyser_plugin_manager.rs | 2 +- programs/sbf/Cargo.lock | 76 ++++++++++++++++++- validator/Cargo.toml | 2 +- validator/src/admin_rpc_service.rs | 6 +- 7 files changed, 83 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 528c7d00f33d04..22e78997ed13e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6059,7 +6059,6 @@ dependencies = [ "crossbeam-channel", "json5", "jsonrpc-core", - "jsonrpc-server-utils", "libloading", "log", "serde_json", @@ -6074,6 +6073,7 @@ dependencies = [ "solana-sdk", "solana-transaction-status", "thiserror", + "tokio", ] [[package]] @@ -7480,7 +7480,6 @@ dependencies = [ "jsonrpc-core-client", "jsonrpc-derive", "jsonrpc-ipc-server", - "jsonrpc-server-utils", "lazy_static", "libc", "libloading", @@ -7528,6 +7527,7 @@ dependencies = [ "symlink", "thiserror", "tikv-jemallocator", + "tokio", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 66436c9cfb3fd8..34cd21171785a0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -236,7 +236,6 @@ jsonrpc-derive = "18.0.0" jsonrpc-http-server = "18.0.0" jsonrpc-ipc-server = "18.0.0" jsonrpc-pubsub = "18.0.0" -jsonrpc-server-utils = "18.0.0" lazy_static = "1.4.0" libc = "0.2.153" libloading = "0.7.4" diff --git a/geyser-plugin-manager/Cargo.toml b/geyser-plugin-manager/Cargo.toml index d905248150b717..30d19dbe702288 100644 --- a/geyser-plugin-manager/Cargo.toml +++ b/geyser-plugin-manager/Cargo.toml @@ -14,7 +14,6 @@ bs58 = { workspace = true } crossbeam-channel = { workspace = true } json5 = { workspace = true } jsonrpc-core = { workspace = true } -jsonrpc-server-utils = { workspace = true } libloading = { workspace = true } log = { workspace = true } serde_json = { workspace = true } @@ -29,6 +28,7 @@ solana-runtime = { workspace = true } solana-sdk = { workspace = true } solana-transaction-status = { workspace = true } thiserror = { workspace = true } +tokio = { workspace = true } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] diff --git a/geyser-plugin-manager/src/geyser_plugin_manager.rs b/geyser-plugin-manager/src/geyser_plugin_manager.rs index a15f9e1318075d..d9e556d78a6895 100644 --- a/geyser-plugin-manager/src/geyser_plugin_manager.rs +++ b/geyser-plugin-manager/src/geyser_plugin_manager.rs @@ -1,6 +1,5 @@ use { jsonrpc_core::{ErrorCode, Result as JsonRpcResult}, - jsonrpc_server_utils::tokio::sync::oneshot::Sender as OneShotSender, libloading::Library, log::*, solana_geyser_plugin_interface::geyser_plugin_interface::GeyserPlugin, @@ -8,6 +7,7 @@ use { ops::{Deref, DerefMut}, path::Path, }, + tokio::sync::oneshot::Sender as OneShotSender, }; #[derive(Debug)] diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index 9e8a39c79ce1c8..a31542e3713a6b 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -63,6 +63,80 @@ dependencies = [ "zeroize", ] +[[package]] +name = "agave-geyser-plugin-interface" +version = "1.19.0" +dependencies = [ + "log", + "solana-sdk", + "solana-transaction-status", + "thiserror", +] + +[[package]] +name = "agave-validator" +version = "1.19.0" +dependencies = [ + "agave-geyser-plugin-interface", + "chrono", + "clap 2.33.3", + "console", + "core_affinity", + "crossbeam-channel", + "fd-lock", + "indicatif", + "itertools", + "jsonrpc-core", + "jsonrpc-core-client", + "jsonrpc-derive", + "jsonrpc-ipc-server", + "lazy_static", + "libc", + "libloading", + "log", + "num_cpus", + "rand 0.8.5", + "rayon", + "serde", + "serde_json", + "serde_yaml", + "signal-hook", + "solana-accounts-db", + "solana-clap-utils", + "solana-cli-config", + "solana-core", + "solana-download-utils", + "solana-entry", + "solana-faucet", + "solana-genesis-utils", + "solana-geyser-plugin-manager", + "solana-gossip", + "solana-ledger", + "solana-logger", + "solana-metrics", + "solana-net-utils", + "solana-perf", + "solana-poh", + "solana-rpc", + "solana-rpc-client", + "solana-rpc-client-api", + "solana-runtime", + "solana-sdk", + "solana-send-transaction-service", + "solana-storage-bigtable", + "solana-streamer", + "solana-svm", + "solana-test-validator", + "solana-tpu-client", + "solana-unified-scheduler-pool", + "solana-version", + "solana-vote-program", + "symlink", + "thiserror", + "tikv-jemallocator", + "tokio", +] + [[package]] name = "ahash" version = "0.7.6" @@ -5062,7 +5136,6 @@ dependencies = [ "crossbeam-channel", "json5", "jsonrpc-core", - "jsonrpc-server-utils", "libloading", "log", "serde_json", @@ -5077,6 +5150,7 @@ dependencies = [ "solana-sdk", "solana-transaction-status", "thiserror", + "tokio", ] [[package]] diff --git a/validator/Cargo.toml b/validator/Cargo.toml index 5cc76a810116b3..94e80ffaf90704 100644 --- a/validator/Cargo.toml +++ b/validator/Cargo.toml @@ -23,7 +23,6 @@ jsonrpc-core = { workspace = true } jsonrpc-core-client = { workspace = true, features = ["ipc"] } jsonrpc-derive = { workspace = true } jsonrpc-ipc-server = { workspace = true } -jsonrpc-server-utils = { workspace = true } lazy_static = { workspace = true } libloading = { workspace = true } log = { workspace = true } @@ -66,6 +65,7 @@ solana-version = { workspace = true } solana-vote-program = { workspace = true } symlink = { workspace = true } thiserror = { workspace = true } +tokio = { workspace = true } [dev-dependencies] solana-account-decoder = { workspace = true } diff --git a/validator/src/admin_rpc_service.rs b/validator/src/admin_rpc_service.rs index 57be4cf488865d..3881487882dc2e 100644 --- a/validator/src/admin_rpc_service.rs +++ b/validator/src/admin_rpc_service.rs @@ -6,7 +6,6 @@ use { jsonrpc_ipc_server::{ tokio::sync::oneshot::channel as oneshot_channel, RequestContext, ServerBuilder, }, - jsonrpc_server_utils::tokio, log::*, serde::{de::Deserializer, Deserialize, Serialize}, solana_accounts_db::accounts_index::AccountIndex, @@ -35,6 +34,7 @@ use { thread::{self, Builder}, time::{Duration, SystemTime}, }, + tokio::runtime::Runtime, }; #[derive(Clone)] @@ -815,8 +815,8 @@ pub async fn connect(ledger_path: &Path) -> std::result::Result jsonrpc_server_utils::tokio::runtime::Runtime { - jsonrpc_server_utils::tokio::runtime::Runtime::new().expect("new tokio runtime") +pub fn runtime() -> Runtime { + Runtime::new().expect("new tokio runtime") } #[derive(Default, Deserialize, Clone)] From 54b44a382f10c95fb00d57bfcfb6197dab902977 Mon Sep 17 00:00:00 2001 From: Brennan Date: Wed, 6 Mar 2024 14:03:25 -0800 Subject: [PATCH 30/71] fix typo (#57) --- program-runtime/src/loaded_programs.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/program-runtime/src/loaded_programs.rs b/program-runtime/src/loaded_programs.rs index 926d1179837380..e8a691c537934f 100644 --- a/program-runtime/src/loaded_programs.rs +++ b/program-runtime/src/loaded_programs.rs @@ -67,7 +67,7 @@ pub enum LoadedProgramType { /// /// These can potentially come back alive if the environment changes. FailedVerification(ProgramRuntimeEnvironment), - /// Tombstone for programs which were explicitly undeployoed / closed. + /// Tombstone for programs which were explicitly undeployed / closed. #[default] Closed, /// Tombstone for programs which have recently been modified but the new version is not visible yet. From d0cb5e273aa6b2c9bb4e8ad65c792f926d3b21df Mon Sep 17 00:00:00 2001 From: steviez Date: Wed, 6 Mar 2024 17:03:02 -0600 Subject: [PATCH 31/71] Name previously unnamed thread pool threads (#104) Several rayon and tokio threadpools did not have names; give them names to make tracking them in debug tools easier --- net-utils/src/ip_echo_server.rs | 6 +++++- runtime/src/bank.rs | 5 ++++- runtime/src/snapshot_utils/snapshot_storage_rebuilder.rs | 3 ++- validator/src/admin_rpc_service.rs | 6 +++++- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/net-utils/src/ip_echo_server.rs b/net-utils/src/ip_echo_server.rs index 7d4186ccb6a810..64fbedadc7acf9 100644 --- a/net-utils/src/ip_echo_server.rs +++ b/net-utils/src/ip_echo_server.rs @@ -173,7 +173,11 @@ pub fn ip_echo_server( ) -> IpEchoServer { tcp_listener.set_nonblocking(true).unwrap(); - let runtime = Runtime::new().expect("Failed to create Runtime"); + let runtime = tokio::runtime::Builder::new_multi_thread() + .thread_name("solIpEchoSrvrRt") + .enable_all() + .build() + .expect("new tokio runtime"); runtime.spawn(run_echo_server(tcp_listener, shred_version)); runtime } diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index 3ea316f857a2bc..1abf9403e3fef1 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -1483,7 +1483,10 @@ impl Bank { let epoch = self.epoch(); let slot = self.slot(); let (thread_pool, thread_pool_time) = measure!( - ThreadPoolBuilder::new().build().unwrap(), + ThreadPoolBuilder::new() + .thread_name(|i| format!("solBnkNewEpch{i:02}")) + .build() + .expect("new rayon threadpool"), "thread_pool_creation", ); diff --git a/runtime/src/snapshot_utils/snapshot_storage_rebuilder.rs b/runtime/src/snapshot_utils/snapshot_storage_rebuilder.rs index 0c6116274b1cb1..30cbbd4afd5970 100644 --- a/runtime/src/snapshot_utils/snapshot_storage_rebuilder.rs +++ b/runtime/src/snapshot_utils/snapshot_storage_rebuilder.rs @@ -418,9 +418,10 @@ impl SnapshotStorageRebuilder { /// Builds thread pool to rebuild with fn build_thread_pool(&self) -> ThreadPool { ThreadPoolBuilder::default() + .thread_name(|i| format!("solRbuildSnap{i:02}")) .num_threads(self.num_threads) .build() - .unwrap() + .expect("new rayon threadpool") } } diff --git a/validator/src/admin_rpc_service.rs b/validator/src/admin_rpc_service.rs index 3881487882dc2e..b6d65e3ec4a4df 100644 --- a/validator/src/admin_rpc_service.rs +++ b/validator/src/admin_rpc_service.rs @@ -816,7 +816,11 @@ pub async fn connect(ledger_path: &Path) -> std::result::Result Runtime { - Runtime::new().expect("new tokio runtime") + tokio::runtime::Builder::new_multi_thread() + .thread_name("solAdminRpcRt") + .enable_all() + .build() + .expect("new tokio runtime") } #[derive(Default, Deserialize, Clone)] From 1ff68447d471cd5e46654664bb2eef540614ae8f Mon Sep 17 00:00:00 2001 From: Justin Starry Date: Thu, 7 Mar 2024 09:52:23 +0800 Subject: [PATCH 32/71] Refactor transaction account unlocking (#103) refactor: unlock accounts --- accounts-db/src/accounts.rs | 64 ++++++++++++++++++++++---------- runtime/src/bank.rs | 12 +++--- runtime/src/transaction_batch.rs | 9 ++++- 3 files changed, 57 insertions(+), 28 deletions(-) diff --git a/accounts-db/src/accounts.rs b/accounts-db/src/accounts.rs index 371db9eb08c095..33a57d56461c78 100644 --- a/accounts-db/src/accounts.rs +++ b/accounts-db/src/accounts.rs @@ -80,11 +80,20 @@ impl AccountLocks { if *count == 0 { occupied_entry.remove_entry(); } + } else { + debug_assert!( + false, + "Attempted to remove a read-lock for a key that wasn't read-locked" + ); } } fn unlock_write(&mut self, key: &Pubkey) { - self.write_locks.remove(key); + let removed = self.write_locks.remove(key); + debug_assert!( + removed, + "Attempted to remove a write-lock for a key that wasn't write-locked" + ); } } @@ -618,14 +627,16 @@ impl Accounts { #[allow(clippy::needless_collect)] pub fn unlock_accounts<'a>( &self, - txs: impl Iterator, - results: &[Result<()>], + txs_and_results: impl Iterator)>, ) { - let keys: Vec<_> = txs - .zip(results) + let keys: Vec<_> = txs_and_results .filter(|(_, res)| res.is_ok()) .map(|(tx, _)| tx.get_account_locks_unchecked()) .collect(); + if keys.is_empty() { + return; + } + let mut account_locks = self.account_locks.lock().unwrap(); debug!("bank unlock accounts"); keys.into_iter().for_each(|keys| { @@ -812,6 +823,7 @@ mod tests { }, std::{ borrow::Cow, + iter, sync::atomic::{AtomicBool, AtomicU64, Ordering}, thread, time, }, @@ -1099,8 +1111,8 @@ mod tests { let txs = vec![new_sanitized_tx(&[&keypair], message, Hash::default())]; let results = accounts.lock_accounts(txs.iter(), MAX_TX_ACCOUNT_LOCKS); - assert_eq!(results[0], Ok(())); - accounts.unlock_accounts(txs.iter(), &results); + assert_eq!(results, vec![Ok(())]); + accounts.unlock_accounts(txs.iter().zip(&results)); } // Disallow over MAX_TX_ACCOUNT_LOCKS @@ -1156,7 +1168,7 @@ mod tests { let tx = new_sanitized_tx(&[&keypair0], message, Hash::default()); let results0 = accounts.lock_accounts([tx.clone()].iter(), MAX_TX_ACCOUNT_LOCKS); - assert!(results0[0].is_ok()); + assert_eq!(results0, vec![Ok(())]); assert_eq!( *accounts .account_locks @@ -1190,9 +1202,13 @@ mod tests { let tx1 = new_sanitized_tx(&[&keypair1], message, Hash::default()); let txs = vec![tx0, tx1]; let results1 = accounts.lock_accounts(txs.iter(), MAX_TX_ACCOUNT_LOCKS); - - assert!(results1[0].is_ok()); // Read-only account (keypair1) can be referenced multiple times - assert!(results1[1].is_err()); // Read-only account (keypair1) cannot also be locked as writable + assert_eq!( + results1, + vec![ + Ok(()), // Read-only account (keypair1) can be referenced multiple times + Err(TransactionError::AccountInUse), // Read-only account (keypair1) cannot also be locked as writable + ], + ); assert_eq!( *accounts .account_locks @@ -1204,8 +1220,8 @@ mod tests { 2 ); - accounts.unlock_accounts([tx].iter(), &results0); - accounts.unlock_accounts(txs.iter(), &results1); + accounts.unlock_accounts(iter::once(&tx).zip(&results0)); + accounts.unlock_accounts(txs.iter().zip(&results1)); let instructions = vec![CompiledInstruction::new(2, &(), vec![0, 1])]; let message = Message::new_with_compiled_instructions( 1, @@ -1217,7 +1233,10 @@ mod tests { ); let tx = new_sanitized_tx(&[&keypair1], message, Hash::default()); let results2 = accounts.lock_accounts([tx].iter(), MAX_TX_ACCOUNT_LOCKS); - assert!(results2[0].is_ok()); // Now keypair1 account can be locked as writable + assert_eq!( + results2, + vec![Ok(())] // Now keypair1 account can be locked as writable + ); // Check that read-only lock with zero references is deleted assert!(accounts @@ -1285,7 +1304,7 @@ mod tests { counter_clone.clone().fetch_add(1, Ordering::SeqCst); } } - accounts_clone.unlock_accounts(txs.iter(), &results); + accounts_clone.unlock_accounts(txs.iter().zip(&results)); if exit_clone.clone().load(Ordering::Relaxed) { break; } @@ -1301,7 +1320,7 @@ mod tests { thread::sleep(time::Duration::from_millis(50)); assert_eq!(counter_value, counter_clone.clone().load(Ordering::SeqCst)); } - accounts_arc.unlock_accounts(txs.iter(), &results); + accounts_arc.unlock_accounts(txs.iter().zip(&results)); thread::sleep(time::Duration::from_millis(50)); } exit.store(true, Ordering::Relaxed); @@ -1442,9 +1461,14 @@ mod tests { MAX_TX_ACCOUNT_LOCKS, ); - assert!(results[0].is_ok()); // Read-only account (keypair0) can be referenced multiple times - assert!(results[1].is_err()); // is not locked due to !qos_results[1].is_ok() - assert!(results[2].is_ok()); // Read-only account (keypair0) can be referenced multiple times + assert_eq!( + results, + vec![ + Ok(()), // Read-only account (keypair0) can be referenced multiple times + Err(TransactionError::WouldExceedMaxBlockCostLimit), // is not locked due to !qos_results[1].is_ok() + Ok(()), // Read-only account (keypair0) can be referenced multiple times + ], + ); // verify that keypair0 read-only lock twice (for tx0 and tx2) assert_eq!( @@ -1466,7 +1490,7 @@ mod tests { .get(&keypair2.pubkey()) .is_none()); - accounts.unlock_accounts(txs.iter(), &results); + accounts.unlock_accounts(txs.iter().zip(&results)); // check all locks to be removed assert!(accounts diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index 1abf9403e3fef1..39df91c382feff 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -4379,13 +4379,11 @@ impl Bank { account_overrides } - pub fn unlock_accounts(&self, batch: &mut TransactionBatch) { - if batch.needs_unlock() { - batch.set_needs_unlock(false); - self.rc - .accounts - .unlock_accounts(batch.sanitized_transactions().iter(), batch.lock_results()) - } + pub fn unlock_accounts<'a>( + &self, + txs_and_results: impl Iterator)>, + ) { + self.rc.accounts.unlock_accounts(txs_and_results) } pub fn remove_unrooted_slots(&self, slots: &[(Slot, BankId)]) { diff --git a/runtime/src/transaction_batch.rs b/runtime/src/transaction_batch.rs index 66711fd5a1acd5..9d0ff5fb7ce007 100644 --- a/runtime/src/transaction_batch.rs +++ b/runtime/src/transaction_batch.rs @@ -51,7 +51,14 @@ impl<'a, 'b> TransactionBatch<'a, 'b> { // Unlock all locked accounts in destructor. impl<'a, 'b> Drop for TransactionBatch<'a, 'b> { fn drop(&mut self) { - self.bank.unlock_accounts(self) + if self.needs_unlock() { + self.set_needs_unlock(false); + self.bank.unlock_accounts( + self.sanitized_transactions() + .iter() + .zip(self.lock_results()), + ) + } } } From a8d527b4468366b382bc093ef3f8e37bd70c1570 Mon Sep 17 00:00:00 2001 From: Tyera Date: Wed, 6 Mar 2024 19:31:07 -0700 Subject: [PATCH 33/71] Prep Anchor downstream CI job for v2 bump (#123) * Add new script to patch spl in anchor downstream * Only specify major version for token-2022 * Add update for ahash * Patch spl in anchor * Remove dex and metadata features for now --- scripts/build-downstream-anchor-projects.sh | 15 +++++- scripts/patch-spl-crates-for-anchor.sh | 55 +++++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 scripts/patch-spl-crates-for-anchor.sh diff --git a/scripts/build-downstream-anchor-projects.sh b/scripts/build-downstream-anchor-projects.sh index cdfa0bae10addb..7d75ccc08ab0e2 100755 --- a/scripts/build-downstream-anchor-projects.sh +++ b/scripts/build-downstream-anchor-projects.sh @@ -8,6 +8,7 @@ cd "$(dirname "$0")"/.. source ci/_ source scripts/patch-crates.sh source scripts/read-cargo-variable.sh +source scripts/patch-spl-crates-for-anchor.sh anchor_version=$1 solana_ver=$(readCargoVariable version Cargo.toml) @@ -43,6 +44,14 @@ EOF # NOTE This isn't run in a subshell to get $anchor_dir and $anchor_ver anchor() { set -x + + rm -rf spl + git clone https://github.com/solana-labs/solana-program-library.git spl + cd spl || exit 1 + spl_dir=$PWD + get_spl_versions "$spl_dir" + cd .. + rm -rf anchor git clone https://github.com/coral-xyz/anchor.git cd anchor || exit 1 @@ -57,9 +66,13 @@ anchor() { update_solana_dependencies . "$solana_ver" patch_crates_io_solana Cargo.toml "$solana_dir" + patch_spl_crates . Cargo.toml "$spl_dir" $cargo test - (cd spl && $cargo_build_sbf --features dex metadata stake) + # serum_dex and mpl-token-metadata are using caret versions of solana and SPL dependencies + # rather pull and patch those as well, ignore for now + # (cd spl && $cargo_build_sbf --features dex metadata stake) + (cd spl && $cargo_build_sbf --features stake) (cd client && $cargo test --all-features) anchor_dir=$PWD diff --git a/scripts/patch-spl-crates-for-anchor.sh b/scripts/patch-spl-crates-for-anchor.sh new file mode 100644 index 00000000000000..93ea67b8fceb20 --- /dev/null +++ b/scripts/patch-spl-crates-for-anchor.sh @@ -0,0 +1,55 @@ +spl_memo_version= +spl_token_version= +spl_token_2022_version= +spl_tlv_account_resolution_verison= +spl_transfer_hook_interface_version= + +get_spl_versions() { + declare spl_dir="$1" + spl_memo_version=$(readCargoVariable version "$spl_dir/memo/program/Cargo.toml") + spl_token_version=$(readCargoVariable version "$spl_dir/token/program/Cargo.toml") + spl_token_2022_version=$(readCargoVariable version "$spl_dir/token/program-2022/Cargo.toml"| head -c1) # only use the major version for convenience + spl_tlv_account_resolution_verison=$(readCargoVariable version "$spl_dir/libraries/tlv-account-resolution/Cargo.toml") + spl_transfer_hook_interface_version=$(readCargoVariable version "$spl_dir/token/transfer-hook/interface/Cargo.toml") +} + +patch_spl_crates() { + declare project_root="$1" + declare Cargo_toml="$2" + declare spl_dir="$3" + update_spl_dependencies "$project_root" + patch_crates_io "$Cargo_toml" "$spl_dir" +} + +update_spl_dependencies() { + declare project_root="$1" + declare tomls=() + while IFS='' read -r line; do tomls+=("$line"); done < <(find "$project_root" -name Cargo.toml) + + sed -i -e "s#\(spl-memo = \"\)[^\"]*\(\"\)#\1$spl_memo_version\2#g" "${tomls[@]}" || return $? + sed -i -e "s#\(spl-memo = { version = \"\)[^\"]*\(\"\)#\1$spl_memo_version\2#g" "${tomls[@]}" || return $? + sed -i -e "s#\(spl-token = \"\)[^\"]*\(\"\)#\1$spl_token_version\2#g" "${tomls[@]}" || return $? + sed -i -e "s#\(spl-token = { version = \"\)[^\"]*\(\"\)#\1$spl_token_version\2#g" "${tomls[@]}" || return $? + sed -i -e "s#\(spl-token-2022 = \"\).*\(\"\)#\1$spl_token_2022_version\2#g" "${tomls[@]}" || return $? + sed -i -e "s#\(spl-token-2022 = { version = \"\)[^\"]*\(\"\)#\1$spl_token_2022_version\2#g" "${tomls[@]}" || return $? + sed -i -e "s#\(spl-tlv-account-resolution = \"\)[^\"]*\(\"\)#\1=$spl_tlv_account_resolution_verison\2#g" "${tomls[@]}" || return $? + sed -i -e "s#\(spl-tlv-account-resolution = { version = \"\)[^\"]*\(\"\)#\1=$spl_tlv_account_resolution_verison\2#g" "${tomls[@]}" || return $? + sed -i -e "s#\(spl-transfer-hook-interface = \"\)[^\"]*\(\"\)#\1=$spl_transfer_hook_interface_version\2#g" "${tomls[@]}" || return $? + sed -i -e "s#\(spl-transfer-hook-interface = { version = \"\)[^\"]*\(\"\)#\1=$spl_transfer_hook_interface_version\2#g" "${tomls[@]}" || return $? + + # patch ahash. This is super brittle; putting here for convenience, since we are already iterating through the tomls + ahash_minor_version="0.8" + sed -i -e "s#\(ahash = \"\)[^\"]*\(\"\)#\1$ahash_minor_version\2#g" "${tomls[@]}" || return $? +} + +patch_crates_io() { + declare Cargo_toml="$1" + declare spl_dir="$2" + cat >> "$Cargo_toml" < Date: Wed, 6 Mar 2024 18:51:50 -0800 Subject: [PATCH 34/71] [TieredStorage] Deprecate the use of account-hash in HotStorage (#93) #### Problem TieredStorage stores account hash as an optional field inside its HotStorage. However, the field isn't used and we have already decided to deprecate the account hash. #### Summary of Changes Remove account-hash from the tiered-storage. #### Test Plan Existing tiered-storage tests. Running validators w/ tiered-storage in mainnet-beta w/o storing account-hash. --- accounts-db/src/account_storage/meta.rs | 3 +- accounts-db/src/tiered_storage.rs | 8 +- accounts-db/src/tiered_storage/byte_block.rs | 25 +--- accounts-db/src/tiered_storage/hot.rs | 60 ++------- accounts-db/src/tiered_storage/meta.rs | 128 +++++-------------- accounts-db/src/tiered_storage/readable.rs | 6 - accounts-db/src/tiered_storage/test_utils.rs | 21 +-- 7 files changed, 58 insertions(+), 193 deletions(-) diff --git a/accounts-db/src/account_storage/meta.rs b/accounts-db/src/account_storage/meta.rs index 69c24d7be75f7d..b6c8d72042097a 100644 --- a/accounts-db/src/account_storage/meta.rs +++ b/accounts-db/src/account_storage/meta.rs @@ -128,7 +128,8 @@ impl<'storage> StoredAccountMeta<'storage> { pub fn hash(&self) -> &'storage AccountHash { match self { Self::AppendVec(av) => av.hash(), - Self::Hot(hot) => hot.hash().unwrap_or(&DEFAULT_ACCOUNT_HASH), + // tiered-storage has deprecated the use of AccountHash + Self::Hot(_) => &DEFAULT_ACCOUNT_HASH, } } diff --git a/accounts-db/src/tiered_storage.rs b/accounts-db/src/tiered_storage.rs index a6f4ea89428bf9..2f8ebac65e3b57 100644 --- a/accounts-db/src/tiered_storage.rs +++ b/accounts-db/src/tiered_storage.rs @@ -362,15 +362,15 @@ mod tests { let mut expected_accounts_map = HashMap::new(); for i in 0..num_accounts { - let (account, address, account_hash, _write_version) = storable_accounts.get(i); - expected_accounts_map.insert(address, (account, account_hash)); + let (account, address, _account_hash, _write_version) = storable_accounts.get(i); + expected_accounts_map.insert(address, account); } let mut index_offset = IndexOffset(0); let mut verified_accounts = HashSet::new(); while let Some((stored_meta, next)) = reader.get_account(index_offset).unwrap() { - if let Some((account, account_hash)) = expected_accounts_map.get(stored_meta.pubkey()) { - verify_test_account(&stored_meta, *account, stored_meta.pubkey(), account_hash); + if let Some(account) = expected_accounts_map.get(stored_meta.pubkey()) { + verify_test_account(&stored_meta, *account, stored_meta.pubkey()); verified_accounts.insert(stored_meta.pubkey()); } index_offset = next; diff --git a/accounts-db/src/tiered_storage/byte_block.rs b/accounts-db/src/tiered_storage/byte_block.rs index 1cd80add0c2307..6fc7dec611e9a9 100644 --- a/accounts-db/src/tiered_storage/byte_block.rs +++ b/accounts-db/src/tiered_storage/byte_block.rs @@ -95,9 +95,6 @@ impl ByteBlockWriter { if let Some(rent_epoch) = opt_fields.rent_epoch { size += self.write_pod(&rent_epoch)?; } - if let Some(hash) = opt_fields.account_hash { - size += self.write_pod(hash)?; - } debug_assert_eq!(size, opt_fields.size()); @@ -191,11 +188,7 @@ impl ByteBlockReader { #[cfg(test)] mod tests { - use { - super::*, - crate::accounts_hash::AccountHash, - solana_sdk::{hash::Hash, stake_history::Epoch}, - }; + use {super::*, solana_sdk::stake_history::Epoch}; fn read_type_unaligned(buffer: &[u8], offset: usize) -> (T, usize) { let size = std::mem::size_of::(); @@ -352,19 +345,13 @@ mod tests { let mut writer = ByteBlockWriter::new(format); let mut opt_fields_vec = vec![]; let mut some_count = 0; - let acc_hash = AccountHash(Hash::new_unique()); // prepare a vector of optional fields that contains all combinations // of Some and None. for rent_epoch in [None, Some(test_epoch)] { - for account_hash in [None, Some(&acc_hash)] { - some_count += rent_epoch.iter().count() + account_hash.iter().count(); + some_count += rent_epoch.iter().count(); - opt_fields_vec.push(AccountMetaOptionalFields { - rent_epoch, - account_hash, - }); - } + opt_fields_vec.push(AccountMetaOptionalFields { rent_epoch }); test_epoch += 1; } @@ -396,12 +383,6 @@ mod tests { verified_count += 1; offset += std::mem::size_of::(); } - if let Some(expected_hash) = opt_fields.account_hash { - let hash = read_pod::(&decoded_buffer, offset).unwrap(); - assert_eq!(hash, expected_hash); - verified_count += 1; - offset += std::mem::size_of::(); - } } // make sure the number of Some fields matches the number of fields we diff --git a/accounts-db/src/tiered_storage/hot.rs b/accounts-db/src/tiered_storage/hot.rs index f662c2e062ee11..34f7915186ba9b 100644 --- a/accounts-db/src/tiered_storage/hot.rs +++ b/accounts-db/src/tiered_storage/hot.rs @@ -242,19 +242,6 @@ impl TieredAccountMeta for HotAccountMeta { .flatten() } - /// Returns the account hash by parsing the specified account block. None - /// will be returned if this account does not persist this optional field. - fn account_hash<'a>(&self, account_block: &'a [u8]) -> Option<&'a AccountHash> { - self.flags() - .has_account_hash() - .then(|| { - let offset = self.optional_fields_offset(account_block) - + AccountMetaOptionalFields::account_hash_offset(self.flags()); - byte_block::read_pod::(account_block, offset) - }) - .flatten() - } - /// Returns the offset of the optional fields based on the specified account /// block. fn optional_fields_offset(&self, account_block: &[u8]) -> usize { @@ -488,9 +475,6 @@ fn write_optional_fields( if let Some(rent_epoch) = opt_fields.rent_epoch { size += file.write_pod(&rent_epoch)?; } - if let Some(hash) = opt_fields.account_hash { - size += file.write_pod(hash)?; - } debug_assert_eq!(size, opt_fields.size()); @@ -520,12 +504,8 @@ impl HotStorageWriter { account_data: &[u8], executable: bool, rent_epoch: Option, - account_hash: Option<&AccountHash>, ) -> TieredStorageResult { - let optional_fields = AccountMetaOptionalFields { - rent_epoch, - account_hash, - }; + let optional_fields = AccountMetaOptionalFields { rent_epoch }; let mut flags = AccountMetaFlags::new_from(&optional_fields); flags.set_executable(executable); @@ -574,7 +554,7 @@ impl HotStorageWriter { let total_input_accounts = len - skip; let mut stored_infos = Vec::with_capacity(total_input_accounts); for i in skip..len { - let (account, address, account_hash, _write_version) = accounts.get(i); + let (account, address, _account_hash, _write_version) = accounts.get(i); let index_entry = AccountIndexWriterEntry { address, offset: HotAccountOffset::new(cursor)?, @@ -582,7 +562,7 @@ impl HotStorageWriter { // Obtain necessary fields from the account, or default fields // for a zero-lamport account in the None case. - let (lamports, owner, data, executable, rent_epoch, account_hash) = account + let (lamports, owner, data, executable, rent_epoch) = account .map(|acc| { ( acc.lamports(), @@ -591,19 +571,12 @@ impl HotStorageWriter { acc.executable(), // only persist rent_epoch for those rent-paying accounts (acc.rent_epoch() != RENT_EXEMPT_RENT_EPOCH).then_some(acc.rent_epoch()), - Some(account_hash), ) }) - .unwrap_or((0, &OWNER_NO_OWNER, &[], false, None, None)); + .unwrap_or((0, &OWNER_NO_OWNER, &[], false, None)); let owner_offset = owners_table.insert(owner); - let stored_size = self.write_account( - lamports, - owner_offset, - data, - executable, - rent_epoch, - account_hash, - )?; + let stored_size = + self.write_account(lamports, owner_offset, data, executable, rent_epoch)?; cursor += stored_size; stored_infos.push(StoredAccountInfo { @@ -755,11 +728,9 @@ pub mod tests { const TEST_PADDING: u8 = 5; const TEST_OWNER_OFFSET: OwnerOffset = OwnerOffset(0x1fef_1234); const TEST_RENT_EPOCH: Epoch = 7; - let acc_hash = AccountHash(Hash::new_unique()); let optional_fields = AccountMetaOptionalFields { rent_epoch: Some(TEST_RENT_EPOCH), - account_hash: Some(&acc_hash), }; let flags = AccountMetaFlags::new_from(&optional_fields); @@ -779,7 +750,6 @@ pub mod tests { fn test_hot_account_meta_full() { let account_data = [11u8; 83]; let padding = [0u8; 5]; - let acc_hash = AccountHash(Hash::new_unique()); const TEST_LAMPORT: u64 = 2314232137; const OWNER_OFFSET: u32 = 0x1fef_1234; @@ -787,7 +757,6 @@ pub mod tests { let optional_fields = AccountMetaOptionalFields { rent_epoch: Some(TEST_RENT_EPOCH), - account_hash: Some(&acc_hash), }; let flags = AccountMetaFlags::new_from(&optional_fields); @@ -810,7 +779,6 @@ pub mod tests { let meta = byte_block::read_pod::(&buffer, 0).unwrap(); assert_eq!(expected_meta, *meta); assert!(meta.flags().has_rent_epoch()); - assert!(meta.flags().has_account_hash()); assert_eq!(meta.account_data_padding() as usize, padding.len()); let account_block = &buffer[std::mem::size_of::()..]; @@ -823,10 +791,6 @@ pub mod tests { assert_eq!(account_data.len(), meta.account_data_size(account_block)); assert_eq!(account_data, meta.account_data(account_block)); assert_eq!(meta.rent_epoch(account_block), optional_fields.rent_epoch); - assert_eq!( - (meta.account_hash(account_block).unwrap()), - optional_fields.account_hash.unwrap() - ); } #[test] @@ -1334,8 +1298,8 @@ pub mod tests { .unwrap() .unwrap(); - let (account, address, account_hash, _write_version) = storable_accounts.get(i); - verify_test_account(&stored_meta, account, address, account_hash); + let (account, address, _account_hash, _write_version) = storable_accounts.get(i); + verify_test_account(&stored_meta, account, address); assert_eq!(i + 1, next.0 as usize); } @@ -1352,9 +1316,9 @@ pub mod tests { .unwrap() .unwrap(); - let (account, address, account_hash, _write_version) = + let (account, address, _account_hash, _write_version) = storable_accounts.get(stored_info.offset); - verify_test_account(&stored_meta, account, address, account_hash); + verify_test_account(&stored_meta, account, address); } // verify get_accounts @@ -1362,8 +1326,8 @@ pub mod tests { // first, we verify everything for (i, stored_meta) in accounts.iter().enumerate() { - let (account, address, account_hash, _write_version) = storable_accounts.get(i); - verify_test_account(stored_meta, account, address, account_hash); + let (account, address, _account_hash, _write_version) = storable_accounts.get(i); + verify_test_account(stored_meta, account, address); } // second, we verify various initial position diff --git a/accounts-db/src/tiered_storage/meta.rs b/accounts-db/src/tiered_storage/meta.rs index 4e2bb0d95041ca..2aa53e5a4de1ed 100644 --- a/accounts-db/src/tiered_storage/meta.rs +++ b/accounts-db/src/tiered_storage/meta.rs @@ -1,7 +1,7 @@ //! The account meta and related structs for the tiered storage. use { - crate::{accounts_hash::AccountHash, tiered_storage::owners::OwnerOffset}, + crate::tiered_storage::owners::OwnerOffset, bytemuck::{Pod, Zeroable}, modular_bitfield::prelude::*, solana_sdk::stake_history::Epoch, @@ -14,12 +14,10 @@ use { pub struct AccountMetaFlags { /// whether the account meta has rent epoch pub has_rent_epoch: bool, - /// whether the account meta has account hash - pub has_account_hash: bool, /// whether the account is executable pub executable: bool, /// the reserved bits. - reserved: B29, + reserved: B30, } // Ensure there are no implicit padding bytes @@ -70,10 +68,6 @@ pub trait TieredAccountMeta: Sized { /// does not persist this optional field. fn rent_epoch(&self, _account_block: &[u8]) -> Option; - /// Returns the account hash by parsing the specified account block. None - /// will be returned if this account does not persist this optional field. - fn account_hash<'a>(&self, _account_block: &'a [u8]) -> Option<&'a AccountHash>; - /// Returns the offset of the optional fields based on the specified account /// block. fn optional_fields_offset(&self, _account_block: &[u8]) -> usize; @@ -91,7 +85,6 @@ impl AccountMetaFlags { pub fn new_from(optional_fields: &AccountMetaOptionalFields) -> Self { let mut flags = AccountMetaFlags::default(); flags.set_has_rent_epoch(optional_fields.rent_epoch.is_some()); - flags.set_has_account_hash(optional_fields.account_hash.is_some()); flags.set_executable(false); flags } @@ -102,20 +95,15 @@ impl AccountMetaFlags { /// Note that the storage representation of the optional fields might be /// different from its in-memory representation. #[derive(Debug, PartialEq, Eq, Clone)] -pub struct AccountMetaOptionalFields<'a> { +pub struct AccountMetaOptionalFields { /// the epoch at which its associated account will next owe rent pub rent_epoch: Option, - /// the hash of its associated account - pub account_hash: Option<&'a AccountHash>, } -impl<'a> AccountMetaOptionalFields<'a> { +impl AccountMetaOptionalFields { /// The size of the optional fields in bytes (excluding the boolean flags). pub fn size(&self) -> usize { self.rent_epoch.map_or(0, |_| std::mem::size_of::()) - + self - .account_hash - .map_or(0, |_| std::mem::size_of::()) } /// Given the specified AccountMetaFlags, returns the size of its @@ -125,9 +113,6 @@ impl<'a> AccountMetaOptionalFields<'a> { if flags.has_rent_epoch() { fields_size += std::mem::size_of::(); } - if flags.has_account_hash() { - fields_size += std::mem::size_of::(); - } fields_size } @@ -137,29 +122,17 @@ impl<'a> AccountMetaOptionalFields<'a> { pub fn rent_epoch_offset(_flags: &AccountMetaFlags) -> usize { 0 } - - /// Given the specified AccountMetaFlags, returns the relative offset - /// of its account_hash field to the offset of its optional fields entry. - pub fn account_hash_offset(flags: &AccountMetaFlags) -> usize { - let mut offset = Self::rent_epoch_offset(flags); - // rent_epoch is the previous field to account hash - if flags.has_rent_epoch() { - offset += std::mem::size_of::(); - } - offset - } } #[cfg(test)] pub mod tests { - use {super::*, solana_sdk::hash::Hash}; + use super::*; #[test] fn test_account_meta_flags_new() { let flags = AccountMetaFlags::new(); assert!(!flags.has_rent_epoch()); - assert!(!flags.has_account_hash()); assert_eq!(flags.reserved(), 0u32); assert_eq!( @@ -179,20 +152,11 @@ pub mod tests { flags.set_has_rent_epoch(true); assert!(flags.has_rent_epoch()); - assert!(!flags.has_account_hash()); - assert!(!flags.executable()); - verify_flags_serialization(&flags); - - flags.set_has_account_hash(true); - - assert!(flags.has_rent_epoch()); - assert!(flags.has_account_hash()); assert!(!flags.executable()); verify_flags_serialization(&flags); flags.set_executable(true); assert!(flags.has_rent_epoch()); - assert!(flags.has_account_hash()); assert!(flags.executable()); verify_flags_serialization(&flags); @@ -203,84 +167,58 @@ pub mod tests { fn update_and_verify_flags(opt_fields: &AccountMetaOptionalFields) { let flags: AccountMetaFlags = AccountMetaFlags::new_from(opt_fields); assert_eq!(flags.has_rent_epoch(), opt_fields.rent_epoch.is_some()); - assert_eq!(flags.has_account_hash(), opt_fields.account_hash.is_some()); assert_eq!(flags.reserved(), 0u32); } #[test] fn test_optional_fields_update_flags() { let test_epoch = 5432312; - let acc_hash = AccountHash(Hash::new_unique()); for rent_epoch in [None, Some(test_epoch)] { - for account_hash in [None, Some(&acc_hash)] { - update_and_verify_flags(&AccountMetaOptionalFields { - rent_epoch, - account_hash, - }); - } + update_and_verify_flags(&AccountMetaOptionalFields { rent_epoch }); } } #[test] fn test_optional_fields_size() { let test_epoch = 5432312; - let acc_hash = AccountHash(Hash::new_unique()); for rent_epoch in [None, Some(test_epoch)] { - for account_hash in [None, Some(&acc_hash)] { - let opt_fields = AccountMetaOptionalFields { - rent_epoch, - account_hash, - }; - assert_eq!( - opt_fields.size(), - rent_epoch.map_or(0, |_| std::mem::size_of::()) - + account_hash.map_or(0, |_| std::mem::size_of::()) - ); - assert_eq!( - opt_fields.size(), - AccountMetaOptionalFields::size_from_flags(&AccountMetaFlags::new_from( - &opt_fields - )) - ); - } + let opt_fields = AccountMetaOptionalFields { rent_epoch }; + assert_eq!( + opt_fields.size(), + rent_epoch.map_or(0, |_| std::mem::size_of::()), + ); + assert_eq!( + opt_fields.size(), + AccountMetaOptionalFields::size_from_flags(&AccountMetaFlags::new_from( + &opt_fields + )) + ); } } #[test] fn test_optional_fields_offset() { let test_epoch = 5432312; - let acc_hash = AccountHash(Hash::new_unique()); for rent_epoch in [None, Some(test_epoch)] { - for account_hash in [None, Some(&acc_hash)] { - let rent_epoch_offset = 0; - let account_hash_offset = - rent_epoch_offset + rent_epoch.as_ref().map(std::mem::size_of_val).unwrap_or(0); - let derived_size = account_hash_offset - + account_hash - .as_ref() - .map(|acc_hash| std::mem::size_of_val(*acc_hash)) - .unwrap_or(0); - let opt_fields = AccountMetaOptionalFields { - rent_epoch, - account_hash, - }; - let flags = AccountMetaFlags::new_from(&opt_fields); - assert_eq!( - AccountMetaOptionalFields::rent_epoch_offset(&flags), - rent_epoch_offset - ); - assert_eq!( - AccountMetaOptionalFields::account_hash_offset(&flags), - account_hash_offset - ); - assert_eq!( - AccountMetaOptionalFields::size_from_flags(&flags), - derived_size - ); - } + let rent_epoch_offset = 0; + let derived_size = if rent_epoch.is_some() { + std::mem::size_of::() + } else { + 0 + }; + let opt_fields = AccountMetaOptionalFields { rent_epoch }; + let flags = AccountMetaFlags::new_from(&opt_fields); + assert_eq!( + AccountMetaOptionalFields::rent_epoch_offset(&flags), + rent_epoch_offset + ); + assert_eq!( + AccountMetaOptionalFields::size_from_flags(&flags), + derived_size + ); } } } diff --git a/accounts-db/src/tiered_storage/readable.rs b/accounts-db/src/tiered_storage/readable.rs index 1801b04fcecd80..8f1d2007182a5b 100644 --- a/accounts-db/src/tiered_storage/readable.rs +++ b/accounts-db/src/tiered_storage/readable.rs @@ -2,7 +2,6 @@ use { crate::{ account_storage::meta::StoredAccountMeta, accounts_file::MatchAccountOwnerError, - accounts_hash::AccountHash, tiered_storage::{ footer::{AccountMetaFormat, TieredStorageFooter}, hot::HotStorageReader, @@ -40,11 +39,6 @@ impl<'accounts_file, M: TieredAccountMeta> TieredReadableAccount<'accounts_file, self.address } - /// Returns the hash of this account. - pub fn hash(&self) -> Option<&'accounts_file AccountHash> { - self.meta.account_hash(self.account_block) - } - /// Returns the index to this account in its AccountsFile. pub fn index(&self) -> IndexOffset { self.index diff --git a/accounts-db/src/tiered_storage/test_utils.rs b/accounts-db/src/tiered_storage/test_utils.rs index 2ed2399f30fbaa..f44f20f77cc5dd 100644 --- a/accounts-db/src/tiered_storage/test_utils.rs +++ b/accounts-db/src/tiered_storage/test_utils.rs @@ -48,20 +48,10 @@ pub(super) fn verify_test_account( stored_meta: &StoredAccountMeta<'_>, account: Option<&impl ReadableAccount>, address: &Pubkey, - account_hash: &AccountHash, ) { - let (lamports, owner, data, executable, account_hash) = account - .map(|acc| { - ( - acc.lamports(), - acc.owner(), - acc.data(), - acc.executable(), - // only persist rent_epoch for those rent-paying accounts - Some(*account_hash), - ) - }) - .unwrap_or((0, &OWNER_NO_OWNER, &[], false, None)); + let (lamports, owner, data, executable) = account + .map(|acc| (acc.lamports(), acc.owner(), acc.data(), acc.executable())) + .unwrap_or((0, &OWNER_NO_OWNER, &[], false)); assert_eq!(stored_meta.lamports(), lamports); assert_eq!(stored_meta.data().len(), data.len()); @@ -69,8 +59,5 @@ pub(super) fn verify_test_account( assert_eq!(stored_meta.executable(), executable); assert_eq!(stored_meta.owner(), owner); assert_eq!(stored_meta.pubkey(), address); - assert_eq!( - *stored_meta.hash(), - account_hash.unwrap_or(AccountHash(Hash::default())) - ); + assert_eq!(*stored_meta.hash(), AccountHash(Hash::default())); } From c36b572c361a51b6aae29b176b9504ecd175fdd0 Mon Sep 17 00:00:00 2001 From: Justin Starry Date: Thu, 7 Mar 2024 12:06:52 +0800 Subject: [PATCH 35/71] Add support for partial tx batch unlocking (#110) * Add support for partial tx batch unlocking * add assert * fix build * Add comments --- runtime/src/transaction_batch.rs | 96 ++++++++++++++++++++++++++------ 1 file changed, 78 insertions(+), 18 deletions(-) diff --git a/runtime/src/transaction_batch.rs b/runtime/src/transaction_batch.rs index 9d0ff5fb7ce007..ecec27e02e93aa 100644 --- a/runtime/src/transaction_batch.rs +++ b/runtime/src/transaction_batch.rs @@ -46,6 +46,39 @@ impl<'a, 'b> TransactionBatch<'a, 'b> { pub fn needs_unlock(&self) -> bool { self.needs_unlock } + + /// For every error result, if the corresponding transaction is + /// still locked, unlock the transaction and then record the new error. + pub fn unlock_failures(&mut self, transaction_results: Vec>) { + assert_eq!(self.lock_results.len(), transaction_results.len()); + // Shouldn't happen but if a batch was marked as not needing an unlock, + // don't unlock failures. + if !self.needs_unlock() { + return; + } + + let txs_and_results = transaction_results + .iter() + .enumerate() + .inspect(|(index, result)| { + // It's not valid to update a previously recorded lock error to + // become an "ok" result because this could lead to serious + // account lock violations where accounts are later unlocked + // when they were not currently locked. + assert!(!(result.is_ok() && self.lock_results[*index].is_err())) + }) + .filter(|(index, result)| result.is_err() && self.lock_results[*index].is_ok()) + .map(|(index, _)| (&self.sanitized_txs[index], &self.lock_results[index])); + + // Unlock the accounts for all transactions which will be updated to an + // lock error below. + self.bank.unlock_accounts(txs_and_results); + + // Record all new errors by overwriting lock results. Note that it's + // not valid to update from err -> ok and the assertion above enforces + // that validity constraint. + self.lock_results = transaction_results; + } } // Unlock all locked accounts in destructor. @@ -67,12 +100,12 @@ mod tests { use { super::*, crate::genesis_utils::{create_genesis_config_with_leader, GenesisConfigInfo}, - solana_sdk::{signature::Keypair, system_transaction}, + solana_sdk::{signature::Keypair, system_transaction, transaction::TransactionError}, }; #[test] fn test_transaction_batch() { - let (bank, txs) = setup(); + let (bank, txs) = setup(false); // Test getting locked accounts let batch = bank.prepare_sanitized_batch(&txs); @@ -94,7 +127,7 @@ mod tests { #[test] fn test_simulation_batch() { - let (bank, txs) = setup(); + let (bank, txs) = setup(false); // Prepare batch without locks let batch = bank.prepare_unlocked_batch_from_single_tx(&txs[0]); @@ -109,7 +142,37 @@ mod tests { assert!(batch3.lock_results().iter().all(|x| x.is_ok())); } - fn setup() -> (Bank, Vec) { + #[test] + fn test_unlock_failures() { + let (bank, txs) = setup(true); + + // Test getting locked accounts + let mut batch = bank.prepare_sanitized_batch(&txs); + assert_eq!( + batch.lock_results, + vec![Ok(()), Err(TransactionError::AccountInUse), Ok(())] + ); + + let qos_results = vec![ + Ok(()), + Err(TransactionError::AccountInUse), + Err(TransactionError::WouldExceedMaxBlockCostLimit), + ]; + batch.unlock_failures(qos_results.clone()); + assert_eq!(batch.lock_results, qos_results); + + // Dropping the batch should unlock remaining locked transactions + drop(batch); + + // The next batch should be able to lock all but the conflicting tx + let batch2 = bank.prepare_sanitized_batch(&txs); + assert_eq!( + batch2.lock_results, + vec![Ok(()), Err(TransactionError::AccountInUse), Ok(())] + ); + } + + fn setup(insert_conflicting_tx: bool) -> (Bank, Vec) { let dummy_leader_pubkey = solana_sdk::pubkey::new_rand(); let GenesisConfigInfo { genesis_config, @@ -122,20 +185,17 @@ mod tests { let keypair2 = Keypair::new(); let pubkey2 = solana_sdk::pubkey::new_rand(); - let txs = vec![ - SanitizedTransaction::from_transaction_for_tests(system_transaction::transfer( - &mint_keypair, - &pubkey, - 1, - genesis_config.hash(), - )), - SanitizedTransaction::from_transaction_for_tests(system_transaction::transfer( - &keypair2, - &pubkey2, - 1, - genesis_config.hash(), - )), - ]; + let mut txs = vec![SanitizedTransaction::from_transaction_for_tests( + system_transaction::transfer(&mint_keypair, &pubkey, 1, genesis_config.hash()), + )]; + if insert_conflicting_tx { + txs.push(SanitizedTransaction::from_transaction_for_tests( + system_transaction::transfer(&mint_keypair, &pubkey2, 1, genesis_config.hash()), + )); + } + txs.push(SanitizedTransaction::from_transaction_for_tests( + system_transaction::transfer(&keypair2, &pubkey2, 1, genesis_config.hash()), + )); (bank, txs) } From e5ec7853c6dddf84734e46e1696dbc3d802e018c Mon Sep 17 00:00:00 2001 From: Tao Zhu <82401714+tao-stones@users.noreply.github.com> Date: Thu, 7 Mar 2024 09:23:49 -0600 Subject: [PATCH 36/71] Combine builtin and BPF compute cost in cost model (#29) * Combine builtin and BPF execution cost into programs_execution_cost since VM has started to consume CUs uniformly * update tests * apply suggestions from code review --- core/src/banking_stage.rs | 3 +- core/src/banking_stage/consumer.rs | 15 ++-- core/src/banking_stage/qos_service.rs | 61 +++++---------- cost-model/src/cost_model.rs | 103 ++++++++++++++++---------- cost-model/src/cost_tracker.rs | 16 ++-- cost-model/src/transaction_cost.rs | 25 ++----- 6 files changed, 106 insertions(+), 117 deletions(-) diff --git a/core/src/banking_stage.rs b/core/src/banking_stage.rs index 652f2569f8fd43..603ff55f0003b4 100644 --- a/core/src/banking_stage.rs +++ b/core/src/banking_stage.rs @@ -285,8 +285,7 @@ pub struct BatchedTransactionCostDetails { pub batched_signature_cost: u64, pub batched_write_lock_cost: u64, pub batched_data_bytes_cost: u64, - pub batched_builtins_execute_cost: u64, - pub batched_bpf_execute_cost: u64, + pub batched_programs_execute_cost: u64, } #[derive(Debug, Default)] diff --git a/core/src/banking_stage/consumer.rs b/core/src/banking_stage/consumer.rs index f4ac6c6040eda8..957e190c873f64 100644 --- a/core/src/banking_stage/consumer.rs +++ b/core/src/banking_stage/consumer.rs @@ -1549,16 +1549,17 @@ mod tests { assert_eq!(retryable_transaction_indexes, vec![1]); let expected_block_cost = if !apply_cost_tracker_during_replay_enabled { - let actual_bpf_execution_cost = match commit_transactions_result.first().unwrap() { - CommitTransactionDetails::Committed { compute_units } => *compute_units, - CommitTransactionDetails::NotCommitted => { - unreachable!() - } - }; + let actual_programs_execution_cost = + match commit_transactions_result.first().unwrap() { + CommitTransactionDetails::Committed { compute_units } => *compute_units, + CommitTransactionDetails::NotCommitted => { + unreachable!() + } + }; let mut cost = CostModel::calculate_cost(&transactions[0], &bank.feature_set); if let TransactionCost::Transaction(ref mut usage_cost) = cost { - usage_cost.bpf_execution_cost = actual_bpf_execution_cost; + usage_cost.programs_execution_cost = actual_programs_execution_cost; } block_cost + cost.sum() diff --git a/core/src/banking_stage/qos_service.rs b/core/src/banking_stage/qos_service.rs index 77f05c73a3bc12..8c1507ae3fb91c 100644 --- a/core/src/banking_stage/qos_service.rs +++ b/core/src/banking_stage/qos_service.rs @@ -236,14 +236,10 @@ impl QosService { batched_transaction_details.costs.batched_data_bytes_cost, Ordering::Relaxed, ); - self.metrics.stats.estimated_builtins_execute_cu.fetch_add( + self.metrics.stats.estimated_programs_execute_cu.fetch_add( batched_transaction_details .costs - .batched_builtins_execute_cost, - Ordering::Relaxed, - ); - self.metrics.stats.estimated_bpf_execute_cu.fetch_add( - batched_transaction_details.costs.batched_bpf_execute_cost, + .batched_programs_execute_cost, Ordering::Relaxed, ); @@ -297,7 +293,7 @@ impl QosService { pub fn accumulate_actual_execute_cu(&self, units: u64) { self.metrics .stats - .actual_bpf_execute_cu + .actual_programs_execute_cu .fetch_add(units, Ordering::Relaxed); } @@ -331,12 +327,8 @@ impl QosService { saturating_add_assign!( batched_transaction_details .costs - .batched_builtins_execute_cost, - cost.builtins_execution_cost() - ); - saturating_add_assign!( - batched_transaction_details.costs.batched_bpf_execute_cost, - cost.bpf_execution_cost() + .batched_programs_execute_cost, + cost.programs_execution_cost() ); } Err(transaction_error) => match transaction_error { @@ -427,14 +419,11 @@ struct QosServiceMetricsStats { /// accumulated estimated instruction data Compute Units to be packed into block estimated_data_bytes_cu: AtomicU64, - /// accumulated estimated builtin programs Compute Units to be packed into block - estimated_builtins_execute_cu: AtomicU64, - - /// accumulated estimated SBF program Compute Units to be packed into block - estimated_bpf_execute_cu: AtomicU64, + /// accumulated estimated program Compute Units to be packed into block + estimated_programs_execute_cu: AtomicU64, /// accumulated actual program Compute Units that have been packed into block - actual_bpf_execute_cu: AtomicU64, + actual_programs_execute_cu: AtomicU64, /// accumulated actual program execute micro-sec that have been packed into block actual_execute_time_us: AtomicU64, @@ -515,24 +504,19 @@ impl QosServiceMetrics { i64 ), ( - "estimated_builtins_execute_cu", + "estimated_programs_execute_cu", self.stats - .estimated_builtins_execute_cu + .estimated_programs_execute_cu .swap(0, Ordering::Relaxed), i64 ), ( - "estimated_bpf_execute_cu", + "actual_programs_execute_cu", self.stats - .estimated_bpf_execute_cu + .actual_programs_execute_cu .swap(0, Ordering::Relaxed), i64 ), - ( - "actual_bpf_execute_cu", - self.stats.actual_bpf_execute_cu.swap(0, Ordering::Relaxed), - i64 - ), ( "actual_execute_time_us", self.stats.actual_execute_time_us.swap(0, Ordering::Relaxed), @@ -735,7 +719,7 @@ mod tests { let committed_status: Vec = qos_cost_results .iter() .map(|tx_cost| CommitTransactionDetails::Committed { - compute_units: tx_cost.as_ref().unwrap().bpf_execution_cost() + compute_units: tx_cost.as_ref().unwrap().programs_execution_cost() + execute_units_adjustment, }) .collect(); @@ -862,7 +846,7 @@ mod tests { CommitTransactionDetails::NotCommitted } else { CommitTransactionDetails::Committed { - compute_units: tx_cost.as_ref().unwrap().bpf_execution_cost() + compute_units: tx_cost.as_ref().unwrap().programs_execution_cost() + execute_units_adjustment, } } @@ -898,8 +882,7 @@ mod tests { let signature_cost = 1; let write_lock_cost = 2; let data_bytes_cost = 3; - let builtins_execution_cost = 4; - let bpf_execution_cost = 10; + let programs_execution_cost = 10; let num_txs = 4; let tx_cost_results: Vec<_> = (0..num_txs) @@ -909,8 +892,7 @@ mod tests { signature_cost, write_lock_cost, data_bytes_cost, - builtins_execution_cost, - bpf_execution_cost, + programs_execution_cost, ..UsageCostDetails::default() })) } else { @@ -922,8 +904,7 @@ mod tests { let expected_signatures = signature_cost * (num_txs / 2); let expected_write_locks = write_lock_cost * (num_txs / 2); let expected_data_bytes = data_bytes_cost * (num_txs / 2); - let expected_builtins_execution_costs = builtins_execution_cost * (num_txs / 2); - let expected_bpf_execution_costs = bpf_execution_cost * (num_txs / 2); + let expected_programs_execution_costs = programs_execution_cost * (num_txs / 2); let batched_transaction_details = QosService::accumulate_batched_transaction_costs(tx_cost_results.iter()); assert_eq!( @@ -939,14 +920,10 @@ mod tests { batched_transaction_details.costs.batched_data_bytes_cost ); assert_eq!( - expected_builtins_execution_costs, + expected_programs_execution_costs, batched_transaction_details .costs - .batched_builtins_execute_cost - ); - assert_eq!( - expected_bpf_execution_costs, - batched_transaction_details.costs.batched_bpf_execute_cost + .batched_programs_execute_cost ); } } diff --git a/cost-model/src/cost_model.rs b/cost-model/src/cost_model.rs index 1e15735426737f..b81ea24402d4df 100644 --- a/cost-model/src/cost_model.rs +++ b/cost-model/src/cost_model.rs @@ -93,21 +93,25 @@ impl CostModel { transaction: &SanitizedTransaction, feature_set: &FeatureSet, ) { - let mut builtin_costs = 0u64; - let mut bpf_costs = 0u64; + let mut programs_execution_costs = 0u64; let mut loaded_accounts_data_size_cost = 0u64; let mut data_bytes_len_total = 0u64; let mut compute_unit_limit_is_set = false; + let mut has_user_space_instructions = false; for (program_id, instruction) in transaction.message().program_instructions_iter() { - // to keep the same behavior, look for builtin first - if let Some(builtin_cost) = BUILT_IN_INSTRUCTION_COSTS.get(program_id) { - builtin_costs = builtin_costs.saturating_add(*builtin_cost); - } else { - bpf_costs = bpf_costs - .saturating_add(u64::from(DEFAULT_INSTRUCTION_COMPUTE_UNIT_LIMIT)) - .min(u64::from(MAX_COMPUTE_UNIT_LIMIT)); - } + let ix_execution_cost = + if let Some(builtin_cost) = BUILT_IN_INSTRUCTION_COSTS.get(program_id) { + *builtin_cost + } else { + has_user_space_instructions = true; + u64::from(DEFAULT_INSTRUCTION_COMPUTE_UNIT_LIMIT) + }; + + programs_execution_costs = programs_execution_costs + .saturating_add(ix_execution_cost) + .min(u64::from(MAX_COMPUTE_UNIT_LIMIT)); + data_bytes_len_total = data_bytes_len_total.saturating_add(instruction.data.len() as u64); @@ -120,8 +124,6 @@ impl CostModel { } } - // calculate bpf cost based on compute budget instructions - // if failed to process compute_budget instructions, the transaction will not be executed // by `bank`, therefore it should be considered as no execution cost by cost model. match process_compute_budget_instructions(transaction.message().program_instructions_iter()) @@ -132,8 +134,8 @@ impl CostModel { // 'compute_unit_limit_is_set' flag, because compute_budget does not distinguish // builtin and bpf instructions when calculating default compute-unit-limit. (see // compute_budget.rs test `test_process_mixed_instructions_without_compute_budget`) - if bpf_costs > 0 && compute_unit_limit_is_set { - bpf_costs = u64::from(compute_budget_limits.compute_unit_limit); + if has_user_space_instructions && compute_unit_limit_is_set { + programs_execution_costs = u64::from(compute_budget_limits.compute_unit_limit); } if feature_set @@ -146,13 +148,11 @@ impl CostModel { } } Err(_) => { - builtin_costs = 0; - bpf_costs = 0; + programs_execution_costs = 0; } } - tx_cost.builtins_execution_cost = builtin_costs; - tx_cost.bpf_execution_cost = bpf_costs; + tx_cost.programs_execution_cost = programs_execution_costs; tx_cost.loaded_accounts_data_size_cost = loaded_accounts_data_size_cost; tx_cost.data_bytes_cost = data_bytes_len_total / INSTRUCTION_DATA_BYTES_COST; } @@ -304,8 +304,7 @@ mod tests { &simple_transaction, &FeatureSet::all_enabled(), ); - assert_eq!(*expected_execution_cost, tx_cost.builtins_execution_cost); - assert_eq!(0, tx_cost.bpf_execution_cost); + assert_eq!(*expected_execution_cost, tx_cost.programs_execution_cost); assert_eq!(3, tx_cost.data_bytes_cost); } @@ -333,8 +332,10 @@ mod tests { &token_transaction, &FeatureSet::all_enabled(), ); - assert_eq!(0, tx_cost.builtins_execution_cost); - assert_eq!(200_000, tx_cost.bpf_execution_cost); + assert_eq!( + DEFAULT_INSTRUCTION_COMPUTE_UNIT_LIMIT as u64, + tx_cost.programs_execution_cost + ); assert_eq!(0, tx_cost.data_bytes_cost); } @@ -396,13 +397,8 @@ mod tests { &token_transaction, &FeatureSet::all_enabled(), ); - assert_eq!( - *BUILT_IN_INSTRUCTION_COSTS - .get(&compute_budget::id()) - .unwrap(), - tx_cost.builtins_execution_cost - ); - assert_eq!(12_345, tx_cost.bpf_execution_cost); + // If cu-limit is specified, that would the cost for all programs + assert_eq!(12_345, tx_cost.programs_execution_cost); assert_eq!(1, tx_cost.data_bytes_cost); } @@ -446,8 +442,7 @@ mod tests { &token_transaction, &FeatureSet::all_enabled(), ); - assert_eq!(0, tx_cost.builtins_execution_cost); - assert_eq!(0, tx_cost.bpf_execution_cost); + assert_eq!(0, tx_cost.programs_execution_cost); } #[test] @@ -474,8 +469,7 @@ mod tests { let mut tx_cost = UsageCostDetails::default(); CostModel::get_transaction_cost(&mut tx_cost, &tx, &FeatureSet::all_enabled()); - assert_eq!(expected_cost, tx_cost.builtins_execution_cost); - assert_eq!(0, tx_cost.bpf_execution_cost); + assert_eq!(expected_cost, tx_cost.programs_execution_cost); assert_eq!(6, tx_cost.data_bytes_cost); } @@ -506,8 +500,7 @@ mod tests { let expected_cost = DEFAULT_INSTRUCTION_COMPUTE_UNIT_LIMIT as u64 * 2; let mut tx_cost = UsageCostDetails::default(); CostModel::get_transaction_cost(&mut tx_cost, &tx, &FeatureSet::all_enabled()); - assert_eq!(0, tx_cost.builtins_execution_cost); - assert_eq!(expected_cost, tx_cost.bpf_execution_cost); + assert_eq!(expected_cost, tx_cost.programs_execution_cost); assert_eq!(0, tx_cost.data_bytes_cost); } @@ -567,7 +560,7 @@ mod tests { let tx_cost = CostModel::calculate_cost(&tx, &FeatureSet::all_enabled()); assert_eq!(expected_account_cost, tx_cost.write_lock_cost()); - assert_eq!(*expected_execution_cost, tx_cost.builtins_execution_cost()); + assert_eq!(*expected_execution_cost, tx_cost.programs_execution_cost()); assert_eq!(2, tx_cost.writable_accounts().len()); assert_eq!( expected_loaded_accounts_data_size_cost, @@ -596,7 +589,7 @@ mod tests { let tx_cost = CostModel::calculate_cost(&tx, &feature_set); assert_eq!(expected_account_cost, tx_cost.write_lock_cost()); - assert_eq!(*expected_execution_cost, tx_cost.builtins_execution_cost()); + assert_eq!(*expected_execution_cost, tx_cost.programs_execution_cost()); assert_eq!(2, tx_cost.writable_accounts().len()); assert_eq!( expected_loaded_accounts_data_size_cost, @@ -635,7 +628,7 @@ mod tests { let tx_cost = CostModel::calculate_cost(&tx, &feature_set); assert_eq!(expected_account_cost, tx_cost.write_lock_cost()); - assert_eq!(expected_execution_cost, tx_cost.builtins_execution_cost()); + assert_eq!(expected_execution_cost, tx_cost.programs_execution_cost()); assert_eq!(2, tx_cost.writable_accounts().len()); assert_eq!( expected_loaded_accounts_data_size_cost, @@ -666,7 +659,37 @@ mod tests { let mut tx_cost = UsageCostDetails::default(); CostModel::get_transaction_cost(&mut tx_cost, &transaction, &FeatureSet::all_enabled()); - assert_eq!(expected_builtin_cost, tx_cost.builtins_execution_cost); - assert_eq!(expected_bpf_cost as u64, tx_cost.bpf_execution_cost); + assert_eq!( + expected_builtin_cost + expected_bpf_cost as u64, + tx_cost.programs_execution_cost + ); + } + + #[test] + fn test_transaction_cost_with_mix_instruction_with_cu_limit() { + let (mint_keypair, start_hash) = test_setup(); + + let transaction = + SanitizedTransaction::from_transaction_for_tests(Transaction::new_signed_with_payer( + &[ + system_instruction::transfer(&mint_keypair.pubkey(), &Pubkey::new_unique(), 2), + ComputeBudgetInstruction::set_compute_unit_limit(12_345), + ], + Some(&mint_keypair.pubkey()), + &[&mint_keypair], + start_hash, + )); + // transaction has one builtin instruction, and one ComputeBudget::compute_unit_limit + let expected_cost = *BUILT_IN_INSTRUCTION_COSTS + .get(&solana_system_program::id()) + .unwrap() + + BUILT_IN_INSTRUCTION_COSTS + .get(&compute_budget::id()) + .unwrap(); + + let mut tx_cost = UsageCostDetails::default(); + CostModel::get_transaction_cost(&mut tx_cost, &transaction, &FeatureSet::all_enabled()); + + assert_eq!(expected_cost, tx_cost.programs_execution_cost); } } diff --git a/cost-model/src/cost_tracker.rs b/cost-model/src/cost_tracker.rs index 9d2b3b624afeb4..8fb092c36680a0 100644 --- a/cost-model/src/cost_tracker.rs +++ b/cost-model/src/cost_tracker.rs @@ -105,7 +105,7 @@ impl CostTracker { estimated_tx_cost: &TransactionCost, actual_execution_units: u64, ) { - let estimated_execution_units = estimated_tx_cost.bpf_execution_cost(); + let estimated_execution_units = estimated_tx_cost.programs_execution_cost(); match actual_execution_units.cmp(&estimated_execution_units) { Ordering::Equal => (), Ordering::Greater => { @@ -307,7 +307,7 @@ mod tests { system_transaction::transfer(mint_keypair, &keypair.pubkey(), 2, *start_hash), ); let mut tx_cost = UsageCostDetails::new_with_capacity(1); - tx_cost.bpf_execution_cost = 5; + tx_cost.programs_execution_cost = 5; tx_cost.writable_accounts.push(mint_keypair.pubkey()); (simple_transaction, TransactionCost::Transaction(tx_cost)) @@ -606,7 +606,7 @@ mod tests { { let tx_cost = TransactionCost::Transaction(UsageCostDetails { writable_accounts: vec![acct1, acct2, acct3], - bpf_execution_cost: cost, + programs_execution_cost: cost, ..UsageCostDetails::default() }); assert!(testee.try_add(&tx_cost).is_ok()); @@ -624,7 +624,7 @@ mod tests { { let tx_cost = TransactionCost::Transaction(UsageCostDetails { writable_accounts: vec![acct2], - bpf_execution_cost: cost, + programs_execution_cost: cost, ..UsageCostDetails::default() }); assert!(testee.try_add(&tx_cost).is_ok()); @@ -644,7 +644,7 @@ mod tests { { let tx_cost = TransactionCost::Transaction(UsageCostDetails { writable_accounts: vec![acct1, acct2], - bpf_execution_cost: cost, + programs_execution_cost: cost, ..UsageCostDetails::default() }); assert!(testee.try_add(&tx_cost).is_err()); @@ -668,7 +668,7 @@ mod tests { let mut testee = CostTracker::new(account_max, block_max, block_max); let tx_cost = TransactionCost::Transaction(UsageCostDetails { writable_accounts: vec![acct1, acct2, acct3], - bpf_execution_cost: cost, + programs_execution_cost: cost, ..UsageCostDetails::default() }); let mut expected_block_cost = tx_cost.sum(); @@ -755,7 +755,7 @@ mod tests { let tx_cost = TransactionCost::Transaction(UsageCostDetails { writable_accounts: vec![acct1, acct2, acct3], - bpf_execution_cost: cost, + programs_execution_cost: cost, ..UsageCostDetails::default() }); @@ -802,7 +802,7 @@ mod tests { let cost = 100u64; let tx_cost = TransactionCost::Transaction(UsageCostDetails { writable_accounts: vec![Pubkey::new_unique()], - bpf_execution_cost: cost, + programs_execution_cost: cost, ..UsageCostDetails::default() }); diff --git a/cost-model/src/transaction_cost.rs b/cost-model/src/transaction_cost.rs index 76865fff30fd57..c6e68bfe17b6f4 100644 --- a/cost-model/src/transaction_cost.rs +++ b/cost-model/src/transaction_cost.rs @@ -35,10 +35,10 @@ impl TransactionCost { } } - pub fn bpf_execution_cost(&self) -> u64 { + pub fn programs_execution_cost(&self) -> u64 { match self { - Self::SimpleVote { .. } => 0, - Self::Transaction(usage_cost) => usage_cost.bpf_execution_cost, + Self::SimpleVote { .. } => solana_vote_program::vote_processor::DEFAULT_COMPUTE_UNITS, + Self::Transaction(usage_cost) => usage_cost.programs_execution_cost, } } @@ -85,13 +85,6 @@ impl TransactionCost { } } - pub fn builtins_execution_cost(&self) -> u64 { - match self { - Self::SimpleVote { .. } => solana_vote_program::vote_processor::DEFAULT_COMPUTE_UNITS, - Self::Transaction(usage_cost) => usage_cost.builtins_execution_cost, - } - } - pub fn writable_accounts(&self) -> &[Pubkey] { match self { Self::SimpleVote { writable_accounts } => writable_accounts, @@ -109,8 +102,7 @@ pub struct UsageCostDetails { pub signature_cost: u64, pub write_lock_cost: u64, pub data_bytes_cost: u64, - pub builtins_execution_cost: u64, - pub bpf_execution_cost: u64, + pub programs_execution_cost: u64, pub loaded_accounts_data_size_cost: u64, pub account_data_size: u64, } @@ -122,8 +114,7 @@ impl Default for UsageCostDetails { signature_cost: 0u64, write_lock_cost: 0u64, data_bytes_cost: 0u64, - builtins_execution_cost: 0u64, - bpf_execution_cost: 0u64, + programs_execution_cost: 0u64, loaded_accounts_data_size_cost: 0u64, account_data_size: 0u64, } @@ -140,8 +131,7 @@ impl PartialEq for UsageCostDetails { self.signature_cost == other.signature_cost && self.write_lock_cost == other.write_lock_cost && self.data_bytes_cost == other.data_bytes_cost - && self.builtins_execution_cost == other.builtins_execution_cost - && self.bpf_execution_cost == other.bpf_execution_cost + && self.programs_execution_cost == other.programs_execution_cost && self.loaded_accounts_data_size_cost == other.loaded_accounts_data_size_cost && self.account_data_size == other.account_data_size && to_hash_set(&self.writable_accounts) == to_hash_set(&other.writable_accounts) @@ -168,8 +158,7 @@ impl UsageCostDetails { self.signature_cost .saturating_add(self.write_lock_cost) .saturating_add(self.data_bytes_cost) - .saturating_add(self.builtins_execution_cost) - .saturating_add(self.bpf_execution_cost) + .saturating_add(self.programs_execution_cost) .saturating_add(self.loaded_accounts_data_size_cost) } } From d7efade85a98f37c1916f9577f5f66ded1ce990a Mon Sep 17 00:00:00 2001 From: Lucas Steuernagel <38472950+LucasSte@users.noreply.github.com> Date: Thu, 7 Mar 2024 12:26:31 -0300 Subject: [PATCH 37/71] Add tests for `svm/transaction_processor.rs` (#112) --- Cargo.lock | 1 + programs/bpf_loader/src/lib.rs | 3 +- svm/Cargo.toml | 1 + svm/src/transaction_processor.rs | 529 ++++++++++++++++++++++++++++++- svm/tests/test_program.so | Bin 0 -> 170136 bytes 5 files changed, 531 insertions(+), 3 deletions(-) create mode 100755 svm/tests/test_program.so diff --git a/Cargo.lock b/Cargo.lock index 22e78997ed13e9..1ed3911a83bb23 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7203,6 +7203,7 @@ dependencies = [ name = "solana-svm" version = "1.19.0" dependencies = [ + "bincode", "itertools", "log", "percentage", diff --git a/programs/bpf_loader/src/lib.rs b/programs/bpf_loader/src/lib.rs index 5ba8b26e086c69..a9c34fbabfc6f6 100644 --- a/programs/bpf_loader/src/lib.rs +++ b/programs/bpf_loader/src/lib.rs @@ -39,7 +39,7 @@ use { }, instruction::{AccountMeta, InstructionError}, loader_upgradeable_instruction::UpgradeableLoaderInstruction, - native_loader, + loader_v4, native_loader, program_utils::limited_deserialize, pubkey::Pubkey, saturating_add_assign, @@ -193,6 +193,7 @@ pub fn check_loader_id(id: &Pubkey) -> bool { bpf_loader::check_id(id) || bpf_loader_deprecated::check_id(id) || bpf_loader_upgradeable::check_id(id) + || loader_v4::check_id(id) } /// Only used in macro, do not use directly! diff --git a/svm/Cargo.toml b/svm/Cargo.toml index ac672613c9c4fc..21da2f7105bd73 100644 --- a/svm/Cargo.toml +++ b/svm/Cargo.toml @@ -28,6 +28,7 @@ crate-type = ["lib"] name = "solana_svm" [dev-dependencies] +bincode = { workspace = true } solana-logger = { workspace = true } solana-sdk = { workspace = true, features = ["dev-context-only-utils"] } diff --git a/svm/src/transaction_processor.rs b/svm/src/transaction_processor.rs index 38c5c23affd4de..fa417850699372 100644 --- a/svm/src/transaction_processor.rs +++ b/svm/src/transaction_processor.rs @@ -87,6 +87,7 @@ pub trait TransactionProcessingCallback { } } +#[derive(Debug)] enum ProgramAccountLoadResult { AccountNotFound, InvalidAccountData(ProgramRuntimeEnvironment), @@ -921,8 +922,18 @@ impl TransactionBatchProcessor { mod tests { use { super::*, - solana_program_runtime::loaded_programs::BlockRelation, - solana_sdk::{sysvar::rent::Rent, transaction_context::TransactionContext}, + solana_program_runtime::{ + loaded_programs::BlockRelation, solana_rbpf::program::BuiltinProgram, + }, + solana_sdk::{ + account::WritableAccount, bpf_loader, sysvar::rent::Rent, + transaction_context::TransactionContext, + }, + std::{ + env, + fs::{self, File}, + io::Read, + }, }; struct TestForkGraph {} @@ -933,6 +944,43 @@ mod tests { } } + #[derive(Default)] + pub struct MockBankCallback { + rent_collector: RentCollector, + feature_set: Arc, + pub account_shared_data: HashMap, + } + + impl TransactionProcessingCallback for MockBankCallback { + fn account_matches_owners(&self, account: &Pubkey, owners: &[Pubkey]) -> Option { + if let Some(data) = self.account_shared_data.get(account) { + if data.lamports() == 0 { + None + } else { + owners.iter().position(|entry| data.owner() == entry) + } + } else { + None + } + } + + fn get_account_shared_data(&self, pubkey: &Pubkey) -> Option { + self.account_shared_data.get(pubkey).cloned() + } + + fn get_last_blockhash_and_lamports_per_signature(&self) -> (Hash, u64) { + todo!() + } + + fn get_rent_collector(&self) -> &RentCollector { + &self.rent_collector + } + + fn get_feature_set(&self) -> Arc { + self.feature_set.clone() + } + } + #[test] fn test_inner_instructions_list_from_instruction_trace() { let instruction_trace = [1, 2, 1, 1, 2, 3, 2]; @@ -980,4 +1028,481 @@ mod tests { ] ); } + + #[test] + fn test_load_program_accounts_account_not_found() { + let mut mock_bank = MockBankCallback::default(); + let key = Pubkey::new_unique(); + let environment = ProgramRuntimeEnvironments::default(); + let batch_processor = TransactionBatchProcessor::::default(); + + let result = batch_processor.load_program_accounts(&mock_bank, &key, &environment); + + assert!(matches!(result, ProgramAccountLoadResult::AccountNotFound)); + + let mut account_data = AccountSharedData::default(); + account_data.set_owner(bpf_loader_upgradeable::id()); + let state = UpgradeableLoaderState::Program { + programdata_address: Pubkey::new_unique(), + }; + account_data.set_data(bincode::serialize(&state).unwrap()); + mock_bank + .account_shared_data + .insert(key, account_data.clone()); + + let result = batch_processor.load_program_accounts(&mock_bank, &key, &environment); + assert!(matches!(result, ProgramAccountLoadResult::AccountNotFound)); + + account_data.set_data(Vec::new()); + mock_bank.account_shared_data.insert(key, account_data); + + let result = batch_processor.load_program_accounts(&mock_bank, &key, &environment); + + assert!(matches!( + result, + ProgramAccountLoadResult::InvalidAccountData(_) + )); + } + + #[test] + fn test_load_program_accounts_loader_v4() { + let key = Pubkey::new_unique(); + let mut mock_bank = MockBankCallback::default(); + let mut account_data = AccountSharedData::default(); + account_data.set_owner(loader_v4::id()); + let environment = ProgramRuntimeEnvironments::default(); + let batch_processor = TransactionBatchProcessor::::default(); + mock_bank + .account_shared_data + .insert(key, account_data.clone()); + + let result = batch_processor.load_program_accounts(&mock_bank, &key, &environment); + assert!(matches!( + result, + ProgramAccountLoadResult::InvalidAccountData(_) + )); + + account_data.set_data(vec![0; 64]); + mock_bank + .account_shared_data + .insert(key, account_data.clone()); + let result = batch_processor.load_program_accounts(&mock_bank, &key, &environment); + assert!(matches!( + result, + ProgramAccountLoadResult::InvalidAccountData(_) + )); + + let loader_data = LoaderV4State { + slot: 25, + authority_address: Pubkey::new_unique(), + status: LoaderV4Status::Deployed, + }; + let encoded = unsafe { + std::mem::transmute::<&LoaderV4State, &[u8; LoaderV4State::program_data_offset()]>( + &loader_data, + ) + }; + account_data.set_data(encoded.to_vec()); + mock_bank + .account_shared_data + .insert(key, account_data.clone()); + + let result = batch_processor.load_program_accounts(&mock_bank, &key, &environment); + + match result { + ProgramAccountLoadResult::ProgramOfLoaderV4(data, slot) => { + assert_eq!(data, account_data); + assert_eq!(slot, 25); + } + + _ => panic!("Invalid result"), + } + } + + #[test] + fn test_load_program_accounts_loader_v1_or_v2() { + let key = Pubkey::new_unique(); + let mut mock_bank = MockBankCallback::default(); + let mut account_data = AccountSharedData::default(); + account_data.set_owner(bpf_loader::id()); + let environment = ProgramRuntimeEnvironments::default(); + let batch_processor = TransactionBatchProcessor::::default(); + mock_bank + .account_shared_data + .insert(key, account_data.clone()); + + let result = batch_processor.load_program_accounts(&mock_bank, &key, &environment); + match result { + ProgramAccountLoadResult::ProgramOfLoaderV1orV2(data) => { + assert_eq!(data, account_data); + } + _ => panic!("Invalid result"), + } + } + + #[test] + fn test_load_program_accounts_success() { + let key1 = Pubkey::new_unique(); + let key2 = Pubkey::new_unique(); + let mut mock_bank = MockBankCallback::default(); + let environment = ProgramRuntimeEnvironments::default(); + let batch_processor = TransactionBatchProcessor::::default(); + + let mut account_data = AccountSharedData::default(); + account_data.set_owner(bpf_loader_upgradeable::id()); + + let state = UpgradeableLoaderState::Program { + programdata_address: key2, + }; + account_data.set_data(bincode::serialize(&state).unwrap()); + mock_bank + .account_shared_data + .insert(key1, account_data.clone()); + + let state = UpgradeableLoaderState::ProgramData { + slot: 25, + upgrade_authority_address: None, + }; + let mut account_data2 = AccountSharedData::default(); + account_data2.set_data(bincode::serialize(&state).unwrap()); + mock_bank + .account_shared_data + .insert(key2, account_data2.clone()); + + let result = batch_processor.load_program_accounts(&mock_bank, &key1, &environment); + + match result { + ProgramAccountLoadResult::ProgramOfLoaderV3(data1, data2, slot) => { + assert_eq!(data1, account_data); + assert_eq!(data2, account_data2); + assert_eq!(slot, 25); + } + + _ => panic!("Invalid result"), + } + } + + #[test] + fn test_load_program_from_bytes() { + let mut dir = env::current_dir().unwrap(); + dir.push("tests"); + dir.push("test_program.so"); + let mut file = File::open(dir.clone()).expect("file not found"); + let metadata = fs::metadata(dir).expect("Unable to read metadata"); + let mut buffer = vec![0; metadata.len() as usize]; + file.read_exact(&mut buffer).expect("Buffer overflow"); + + let mut metrics = LoadProgramMetrics::default(); + let loader = bpf_loader_upgradeable::id(); + let size = metadata.len() as usize; + let slot = 2; + let environment = ProgramRuntimeEnvironment::new(BuiltinProgram::new_mock()); + + let result = TransactionBatchProcessor::::load_program_from_bytes( + &mut metrics, + &buffer, + &loader, + size, + slot, + environment.clone(), + false, + ); + + assert!(result.is_ok()); + + let result = TransactionBatchProcessor::::load_program_from_bytes( + &mut metrics, + &buffer, + &loader, + size, + slot, + environment, + true, + ); + + assert!(result.is_ok()); + } + + #[test] + fn test_load_program_not_found() { + let mock_bank = MockBankCallback::default(); + let key = Pubkey::new_unique(); + let batch_processor = TransactionBatchProcessor::::default(); + + let result = batch_processor.load_program(&mock_bank, &key, false, 50); + + let loaded_program = LoadedProgram::new_tombstone(0, LoadedProgramType::Closed); + assert_eq!(result, Arc::new(loaded_program)); + } + + #[test] + fn test_load_program_invalid_account_data() { + let key = Pubkey::new_unique(); + let mut mock_bank = MockBankCallback::default(); + let mut account_data = AccountSharedData::default(); + account_data.set_owner(loader_v4::id()); + let batch_processor = TransactionBatchProcessor::::default(); + mock_bank + .account_shared_data + .insert(key, account_data.clone()); + + let result = batch_processor.load_program(&mock_bank, &key, false, 20); + + let loaded_program = LoadedProgram::new_tombstone( + 0, + LoadedProgramType::FailedVerification( + batch_processor + .loaded_programs_cache + .read() + .unwrap() + .get_environments_for_epoch(20) + .clone() + .program_runtime_v1, + ), + ); + assert_eq!(result, Arc::new(loaded_program)); + } + + #[test] + fn test_load_program_program_loader_v1_or_v2() { + let key = Pubkey::new_unique(); + let mut mock_bank = MockBankCallback::default(); + let mut account_data = AccountSharedData::default(); + account_data.set_owner(bpf_loader::id()); + let batch_processor = TransactionBatchProcessor::::default(); + mock_bank + .account_shared_data + .insert(key, account_data.clone()); + + // This should return an error + let result = batch_processor.load_program(&mock_bank, &key, false, 20); + let loaded_program = LoadedProgram::new_tombstone( + 0, + LoadedProgramType::FailedVerification( + batch_processor + .loaded_programs_cache + .read() + .unwrap() + .get_environments_for_epoch(20) + .clone() + .program_runtime_v1, + ), + ); + assert_eq!(result, Arc::new(loaded_program)); + + let mut dir = env::current_dir().unwrap(); + dir.push("tests"); + dir.push("test_program.so"); + let mut file = File::open(dir.clone()).expect("file not found"); + let metadata = fs::metadata(dir).expect("Unable to read metadata"); + let mut buffer = vec![0; metadata.len() as usize]; + file.read_exact(&mut buffer).expect("buffer overflow"); + account_data.set_data(buffer); + + mock_bank + .account_shared_data + .insert(key, account_data.clone()); + + let result = batch_processor.load_program(&mock_bank, &key, false, 20); + + let environments = ProgramRuntimeEnvironments::default(); + let expected = TransactionBatchProcessor::::load_program_from_bytes( + &mut LoadProgramMetrics::default(), + account_data.data(), + account_data.owner(), + account_data.data().len(), + 0, + environments.program_runtime_v1.clone(), + false, + ); + + assert_eq!(result, Arc::new(expected.unwrap())); + } + + #[test] + fn test_load_program_program_loader_v3() { + let key1 = Pubkey::new_unique(); + let key2 = Pubkey::new_unique(); + let mut mock_bank = MockBankCallback::default(); + let batch_processor = TransactionBatchProcessor::::default(); + + let mut account_data = AccountSharedData::default(); + account_data.set_owner(bpf_loader_upgradeable::id()); + + let state = UpgradeableLoaderState::Program { + programdata_address: key2, + }; + account_data.set_data(bincode::serialize(&state).unwrap()); + mock_bank + .account_shared_data + .insert(key1, account_data.clone()); + + let state = UpgradeableLoaderState::ProgramData { + slot: 0, + upgrade_authority_address: None, + }; + let mut account_data2 = AccountSharedData::default(); + account_data2.set_data(bincode::serialize(&state).unwrap()); + mock_bank + .account_shared_data + .insert(key2, account_data2.clone()); + + // This should return an error + let result = batch_processor.load_program(&mock_bank, &key1, false, 0); + let loaded_program = LoadedProgram::new_tombstone( + 0, + LoadedProgramType::FailedVerification( + batch_processor + .loaded_programs_cache + .read() + .unwrap() + .get_environments_for_epoch(0) + .clone() + .program_runtime_v1, + ), + ); + assert_eq!(result, Arc::new(loaded_program)); + + let mut dir = env::current_dir().unwrap(); + dir.push("tests"); + dir.push("test_program.so"); + let mut file = File::open(dir.clone()).expect("file not found"); + let metadata = fs::metadata(dir).expect("Unable to read metadata"); + let mut buffer = vec![0; metadata.len() as usize]; + file.read_exact(&mut buffer).expect("buffer overflow"); + let mut header = bincode::serialize(&state).unwrap(); + let mut complement = vec![ + 0; + std::cmp::max( + 0, + UpgradeableLoaderState::size_of_programdata_metadata() - header.len() + ) + ]; + header.append(&mut complement); + header.append(&mut buffer); + account_data.set_data(header); + + mock_bank + .account_shared_data + .insert(key2, account_data.clone()); + + let result = batch_processor.load_program(&mock_bank, &key1, false, 20); + + let data = account_data.data(); + account_data + .set_data(data[UpgradeableLoaderState::size_of_programdata_metadata()..].to_vec()); + + let environments = ProgramRuntimeEnvironments::default(); + let expected = TransactionBatchProcessor::::load_program_from_bytes( + &mut LoadProgramMetrics::default(), + account_data.data(), + account_data.owner(), + account_data.data().len(), + 0, + environments.program_runtime_v1.clone(), + false, + ); + assert_eq!(result, Arc::new(expected.unwrap())); + } + + #[test] + fn test_load_program_of_loader_v4() { + let key = Pubkey::new_unique(); + let mut mock_bank = MockBankCallback::default(); + let mut account_data = AccountSharedData::default(); + account_data.set_owner(loader_v4::id()); + let batch_processor = TransactionBatchProcessor::::default(); + + let loader_data = LoaderV4State { + slot: 0, + authority_address: Pubkey::new_unique(), + status: LoaderV4Status::Deployed, + }; + let encoded = unsafe { + std::mem::transmute::<&LoaderV4State, &[u8; LoaderV4State::program_data_offset()]>( + &loader_data, + ) + }; + account_data.set_data(encoded.to_vec()); + mock_bank + .account_shared_data + .insert(key, account_data.clone()); + + let result = batch_processor.load_program(&mock_bank, &key, false, 0); + let loaded_program = LoadedProgram::new_tombstone( + 0, + LoadedProgramType::FailedVerification( + batch_processor + .loaded_programs_cache + .read() + .unwrap() + .get_environments_for_epoch(0) + .clone() + .program_runtime_v1, + ), + ); + assert_eq!(result, Arc::new(loaded_program)); + + let mut header = account_data.data().to_vec(); + let mut complement = + vec![0; std::cmp::max(0, LoaderV4State::program_data_offset() - header.len())]; + header.append(&mut complement); + + let mut dir = env::current_dir().unwrap(); + dir.push("tests"); + dir.push("test_program.so"); + let mut file = File::open(dir.clone()).expect("file not found"); + let metadata = fs::metadata(dir).expect("Unable to read metadata"); + let mut buffer = vec![0; metadata.len() as usize]; + file.read_exact(&mut buffer).expect("buffer overflow"); + header.append(&mut buffer); + + account_data.set_data(header); + mock_bank + .account_shared_data + .insert(key, account_data.clone()); + + let result = batch_processor.load_program(&mock_bank, &key, false, 20); + + let data = account_data.data()[LoaderV4State::program_data_offset()..].to_vec(); + account_data.set_data(data); + mock_bank + .account_shared_data + .insert(key, account_data.clone()); + + let environments = ProgramRuntimeEnvironments::default(); + let expected = TransactionBatchProcessor::::load_program_from_bytes( + &mut LoadProgramMetrics::default(), + account_data.data(), + account_data.owner(), + account_data.data().len(), + 0, + environments.program_runtime_v1.clone(), + false, + ); + assert_eq!(result, Arc::new(expected.unwrap())); + } + + #[test] + fn test_load_program_effective_slot() { + let key = Pubkey::new_unique(); + let mut mock_bank = MockBankCallback::default(); + let mut account_data = AccountSharedData::default(); + account_data.set_owner(loader_v4::id()); + let batch_processor = TransactionBatchProcessor::::default(); + + batch_processor + .loaded_programs_cache + .write() + .unwrap() + .upcoming_environments = Some(ProgramRuntimeEnvironments::default()); + mock_bank + .account_shared_data + .insert(key, account_data.clone()); + + let result = batch_processor.load_program(&mock_bank, &key, false, 20); + + let slot = batch_processor.epoch_schedule.get_first_slot_in_epoch(20); + assert_eq!(result.effective_slot, slot); + } } diff --git a/svm/tests/test_program.so b/svm/tests/test_program.so new file mode 100755 index 0000000000000000000000000000000000000000..9336ced24df6b4eec7b37c9530c99003016c28d6 GIT binary patch literal 170136 zcmeFa51d_9buWHql1l>`XiP$gAVw2luKdwBxtWaNkCEI= z_;WbcOaKM_{rcpOAhotK6Clv)gV&~pzRvTemNsbH_ptU4eYQ2Vwm)8LYg((OR+!)S zyZ)TB@45Gep!jP0aYJU`z4zK{ueJ8tYp?z1oU>>BhBtS1bVMGeMBj~=TJtK+Sz`qc z+}CNpP!_F-rsD66=oEo1prm9d^6`@OTHtXsQ_(Q@B>da-W-ZUhM_#RP9*6a&qUgx5 z*2~A!7ic|?y&kD&drPcdK7K;0MIKMLbamcx`89E_W(0p{6K2_rZ zFQ0m{WDHD!jPT3r6{j@T%iuM?00}dE5RF~i6Gepx2rRv5F4GFf^2?CEPfDYxK2Lu* z9q}yX#M6X-YcYfWap3RmIsRtk_&YD)@5h1O$ArGDKkomj#{PH;Y3~o0du_J%%K=nl z&sC;c|0|k^`XTvKsHNz~K8d0^P~4xjF~XdKfTL!xAG?6F5WPYGr=c?IC*8*;nNfd8 zvZHALJ~lZy8RDz8z&8haedH=)&_Mj30RB85%>Ujiv;3FZADN#2@Z(}yPH%oE*AGX7 zesDfqK6Ak8(LfGK_fe5svUvJYN`U1%-7JS9uL$~oxA5D(>gJ!~#xRG(z|Ga-E|wO?D&c|{vKx4cYrhImNdRRNu$Jfj9Wn&mmO zp*)l4=*vWBh=+8p59q9)X8i-X(jq1%&-%VBpW`H1x#%c!DE~ttT`QNQ`P&-$Wloy^ zU<17JmvpN>9Ynu#q`JN^%@L4vm!y8(^dlUnJ|~hMj(ft-!FawhXrFxkJa}{cJ_!8- zXq5D!0D)=zkF?GMOj8AuK9*+n>Q`zyx%fe~A1gFnUugN=4@f@8#Rr4-iI1Xny{*?2 zp6Mq*aD2b=ALc)f{0S0)Q0cf;;`rk_j)?z(9R5XsNb&C#@F;%F=)6bM^%0A=X-xP- zuxT7`M?ZDX{X^ZKR=&oxoc8h=j?-~VgO5)m9opBy(?S2(Gg7Z;K2jmlhT^ejB|qtY zLgIM*h^7}mY4Ot%7dtz|UKgT78mdxQ2nWy}NQmeMkUpjv1}wjVdJKi={Ss5o==w|! z0Uu9*Zs8YkJm&N$9bL1eI`}vw^msa1tn^hnZx+9(c=B5XupZzQ`P{1YiT_<7!0E0A za_I(0d-~561TG(!TYRedCG!;jG$<`OPL^q!?VStVtE(7f9@G;4Ev;es&RluVRomXc z(ywm(lme{tDY=(GTCQ(o=}mOG}i)7cdSOLp>(h z9{VdE*YP}L`cZ%{&rluzBhiz1pXzt~km_~(@N=5~pw>$^%@O!y4G<$FOSC_SO4QOQ z8q(~{E-6`Jk62Fox+s_rNZ)Sw6;3DZz_k}zCBRevUu*Ra(7e_ux*$B(F;iT7K7 zjGKKj`x)<3c_k~={`-8H$Cu8JE$sXvwCCffYWQh)#V%RD9F*P($EW(u@tD%*evI2m z_p{i z>{Pt#n2xu3!~%H8lJIUHD^rkx5YN+bmzA4cwSJ*LD)ssVJgRhlQ}NomFe}%3h3N04 zoc->BKVuF)1dfAfl)u!^@O5lm&3sh9Ov`#t)7aO$<;#>$rWw3HT5YzYMQa*KA7ggwZAk^VoFx)}p2!pTFnDjL8S*6zpxob7~EqqqxXH@2d zgHT_u>AK~6KX4o_#d@{AMaxq;SpVel`g*mgpN+42(b~C0<6i5pw4X0gdYby#`L}g% z%D?yX0>#(NzX~!>XX1QX?EVM&J+}qFN%v0CYu7iQ4~X~Kz`iBjccpOTBr(4?>3&r5 z!|{HI6f}b_x>LMFXX@SGVEX7216X~U+p}TX=2y;{+Ez;dSi7fqWU9J zEOzoa%BQMn^7}{NFZqbt$NGMQgPO=bhr!i%X+G`P1Mkwdn8W&C!Md=%*Xp5&rEdSd zo#WO2a;`pA_@&hUyk8y<1hqNCJwj?bsJbnvHyWaXwbf8Z@){)#kz$-BdR-Iod5Edi77S}bG$u-agdI; z({=Wb&k!_GIDgO%5`H=Kx_FZMugNWD2geki?uFRPaZ7^^+Qn9Oa3$>s>?Wzq{LybW z%}?I$F9JTLGq4v+f^pVL56Dq_J;?Ukp4^C5zQBX!aX)opaFqp5)pmMXt%U&GSFkOI+VFuw8Dw zv7M(ubNr~{i666icL-S2^>K}TU&H+$)<4hc&&K{Y*F_8?XD}E1e?tA0`aP12=>N05 zc%SkSFV%ead$_L9{?l^GaV0xEo$2R zv1wRWJPkP<&}`s+jr>}*RT(HmpOLiF*|kmqYv*cQJPDH-g8R93lT&R(%eNaJUA8{) z^~!l@m;6jzsc`e5Buvk}SJNXZwrrdr8)Qxvgx8uMqu{#H$u+h59Hjry_PgjCxB<)ccFEFA?t3P~P;@zjvXw#fCUus39DmTLS-%>z2_VO?;2R5HKd5 zE?akHdi6GaruKz zQr)vaX)i=-U#a&fipo{f+p1KHZbCjG>nhHu>sHP8b-L42Tdw)FevL;iv-tc*c#dD< zV|S8&U&j`r?+U-ZFB8r)&j7A&?be4BALaM6;K}#HD;=uP@tCfck}3cYlA6w6$zHAJ z^F?*+ucW;pTNhQwzHaH?li2rNYqoz!Iy-^F?b-kmXwbGzpxom_iF z<6E>m`N%VpuG&7&P`B2vj%m9?-Txxxp}vv-i_fwFs>#6(cgu5uG&;JWswGKx;rTrjWl!NO&e=2|v z;g;m!P73IJsebCqxvsoW{82Kl@+e^K7_KXqAnp3sF82EoZjao53GLC7&u8oH6M_HL z06tlKO!d+Abfun$B^Rr|LOpvjkgxlf;eNnTj>m_D&Fnn?VNLt|&i)VaP(L0H`qTZd zWbsj>$HvRWN)PEc8t8F&-hJ#v=o5c4pik&+LEp0h9@2MrNZ$#_X-+Wiei%80cu2=+ z105z8+uuztw*Fgba=R*%*YrVJBn>Y`UPp5M*-l=e+)3wk4RpS^UafeUm0{ewurraEZFa;yPvg4>-m0UaXOG9#A|gPD|TRU z!}R%R5TVkk<;7_*J_zoA`+P~d=3x96BV8aBB6T>D8@0asyNRvy3PK&*6LgW!@Aci9 zCcd);pE`&I+`3~ZPSbtiR_#w0c-75(et zmL%psxjl8c(LW(RKZoJ@3xnII0c)?)p#qPqinnh5(oNM=Iw3CO4`a@^Ymf8;K-nw5RX;Vz(6-$+j2ttB{zq5HP|oNAK1TSWwu}PYy*7rehGWP_eawC z)#mZgzTOw$r+vk7u&l$zgM9MGCAr&)_;lmn#(Ow!e4P9l;9rh?t*^rWekuD}=fk8r zom}stdK8~!^v>6~GF8W^_b1m?Ea5%~^*gcoAz7;Mw1?7ql?T_gUfv zC`727#9GbvDY;hl{vgO8CCNv0o#Xw|wO6X8`W5d}yB0t6fcksO6>q%H{F~**-#%L} zEm!_{uE1Z;XV-0tH(6eiAIa=R5_@~;eIm0j&KLd32K!vi?AH2ld~8?z3t=~w;HUF< z5Wh)BviJ$1KR@4b-NSO~+2Zfh{wZ8VW($>{@UaU(26ti5uFEqyMfuOq2fn{W{boq6 zeN^ZuL^xbTsIQeMohRzbUgYwq+P#61S&Id*YWJ^3W}R*6rv=~0ER|<+X-V>vW!g{4 zJ=#CXR&s&%jgRWsze&BJS+9}&+SsJwIY-jG?<@B5V(WjuZ(;X&()|FSCFXcc`6@4E zE&2n$F8K0#Ra18ccIjCuPcD8&;$p`eBK|1P_WGX^JU)NsBUZ-)kuFUj|%I*b?CgPr5!A z(C6da^_F&?^tnCqeyaGr5VI$44-Np9@M$@IGx&a0ns4J^=#)d!FGDNUe$=Z9PrgZC zVtU1WVxgPJnSVx3erK?h`4Xk);az>2CcZ1A-0acEOH$DFfR&^AAR4?Jv7EbGJ)ak1 zT^_-{sJ?EVt?jkb)1j+WPbuFUpr6%;HJ!;(^)hdFU2p3qSKoJb`Hh^Gv%jb5cpJG4 z#3Lm4mLyK^d7Wnbj;Or+ydpVG=}(3PT=roo+QjT^A-YV*i;X+3Pwsj*AvBP$D}ciF zx^Ckso~7ln;g4rqyQdl*wqKfDto@#>d{Wx+d6Dz%FNNoseotk;;-w#S7wy5grJ=`< zGF{8)+K>5@c(&-2A_YnJo!ZVVqY|g%(C*O?el*}O>E4vWk?W-U3a#ILwZvh+kD~pg zdyD1=bZsJC)r_v+fMYoUJ!bIxs_8zR?K_A2bn7sFC$d)o(Z|d6ZQQh>jIQ{KX z!PmcBR#Wl|(N{|l&q4bjOK3&3ryyP*=qZ-89+^d;eURAn(ESJRZ_4Ybiw)S|(KV?) z#CJ>eG@bgdpoetOFM9?A6sNBcK;l~y^gG+9-|Fpof3>5lenVq>#CKngz7SvSKGn+? z!|Ux6--9`PG+|yI->r7R(}aIG2Y*TfypI3a)=}m3JVnQ|`>XCxxE~(dc#h9ie$(e7 z&>llO`Gj;mo};UXdR`vieN1#CqiaIa7ayDypI^(N0o11 z7rCDLxmlNu;~I%VXxBgCeiHTB^=tEu-w}H6)9P~1a8r)%BLUs{am;;+!dp$4Y#;jn z(}>B(EAZR*8EWHdNBp^l=KY52g@SnO=Y<~J16H|?-EHZ}(jT_;F}cGZkL|VeQAxL= zZ)qF)z6;v(^tIo&QXUJ@_8gsyj4r3Ml%sQwrJc^1l5R!k<~DSG?FXWBFh}Q3qr>S` zyBv?XJk=g&@>KgBk12mi_eEA;`AfRjSX%i@x>s0Q`AfQ&NZRFloy`x@?-1p6-8V}) z^_6yk`cDZ{Z+3nU^MKj?%1N0Y#&W{B9r-yF`FJ5-=8v4N@X~JQl&M3uU&(eZ$h9-X zc4&kVLVwTOAw2Pq2JNKsI78>>c+BKs{#86?@-Y7@9y58EpB0Z?A$%uioF(nXV=DLL z4E0muG3}S+jAd4?{g#};Q$E#K3hKHu&QH^%_qsDml0Jy}Lgb0eyj<6lU(6#6mFYSW z79xG#qh{l1#GhZ-r*O2VoOfui+MT~Y5uAhYJoK(Dq(}qh_ZyJ0_tVSS&ds@YKK&zV zXG^Y~PlfGNRo_N<5dooERlbK*J>dUN*Zwv?E*^_4eMsVXOzoKa?a3z|)pEP1lYHa> zNmGBJ`tYmnvif&wZ2nn1ruNV8HOJ$c?&)K#%+C39KHje1$Jjhe_Z6JJcwFh*JnNXW zBjXl$;&J6C-uJZRZ=R>;=e{q`b9-q|n{Dwt+fpfy$CUmG_74c^_Xt>@>4J<_DF88;VV0`B9FSvhCwLFxY&o70)@8huv@0j65H$FkUK|4ot z_j1H80RN*o{5YyOA^dl?pqu{0k>L4H(tlBI{MAkn|6M1De@PDiaDd^GE(g zdq^`3c>WaA!;$kP-|4CNdnOG30Aj^sdN{J~1mWq8>-@_&S^SL9AL_>yt>lgVdp4&Z zoq>FPeO>9;C-ar>N74>({whSDW^=F)4@i4eTQAp617d{j;J<`;qfVDmb%?pRhqcYh z5tWeMsBA_(^HhF*pV0f4e#HfV3)hqJze4}_B9r+jm-!U!5d+814J@xpam(@=%g3~w z`oncE_4~tZ{Je(&oFvAbUxy}U}hy(YXiyEQY`DYMfe%g@hZ~M!cTQN2+#Ik6VT!Pn3!Lfd^|^o`K!rx zVgVi7B}(N#()~Trm+S1g6yl?Y>gTW_pB(r8d}MOV5vdoJbAIx9EBVN?s)u`R-nH?} zex@E4qIU@)ex4fl8QfVV$***tF8Wsem?WdBt$TgnwP(Ewy4gOW!w?*w%%Z4i9vInv5emCwVH9r4~X?Lz%tO>^If zXYxSoo^$O+CkFn@-emUEDCf5%461^RcZ-!*C z=|do2pC_Kn_1^1&9#T6QQBTMKOK{HQ-92c)&OGutaPZs z=DK-BZlAA&a-I*Y$hC7m6yN#E+WA-I&;7pRw9^?pS491qXYE{_YvdC2bu93c5RkA4!mY0}T7DjhmM#kRi7$~C{z0R=$_ z{kTO)hxpe3f8F-iDHqxomQ$2$|BWd3{ZIEVhdWP}n|8yUT>?McIbYVL#m;k4OV0)5 z2mSyh)_YaK$1&~!O`z_(B{|$VBjQvy+*yin%?0qwIRTp(B8H!8^6Kok$#59{-uO(D{CbC}QmuG+exIOR1+L2q6s5#Z03A0a&NjTWK> zP5JD;r5tzyz!jX?(yHE&liUNuFNwJE4}`lo!>|F=i-QW7eL(3SI2E0gS;pS z<}+mT$$Uw>9d$ZGIqd^qem|)lz1jSDVssNf1N~BxM7rOJ_{8Z}MhaqG(7)q4Z+gF! zcWHUOkFB5`^b2!1pO+AOJJ-=@ArvB&gX9A+%U1+1hM)TiCBef{272~NgmukxGS9|) zRWbbiF7aN~&!pSdEg#nP$npAN7y2Q&Ul+iB?=(4`qYLf%`-x7cwPjk{O-=_h2;_@l zB;{8PUtSK!Ykw=+zfaMm`s-=;59xn&KKh-z3HJ6o$KJPjnbBwaRxE%0Re!aJGb`pLZp6)-(U87Xxv9x5%@dB4sDqEYlD31FDYZcGcZj%?e|Ei&yN7W+pS+D zU{SrKu|L0CoDxfh>q*`EscWgi`MNm1RL50(iP;BRKi5tj6~H0Wqx9aPiVb?rSp{?! z)Gn@CP?5Cvb5~W8(a0ql`*WbYuRv1B6Jz&>H_tTty8np!k5iUN#j4$d_Iq`OXrbiS zjGu|3B%^JpPECKxVqfR_du_I(!4k?3Tf7}`lD2rq<%*E?7_xP<&dV$&Vevz157Or+ z*9$o0aE_F(+6sgSzRyN|imHm6`Ru>tn1~qDZ!Dhlr^2tp*VT$!Y-X13#7$MLGIC8^-_k)P+bH1g9O z@KfHZWd-r(Ko8{29G^%0`RbH@G{bV9+lf4;=<|++=huo>X;m8@XeVBsm z!k}OLp76Z}2O#1dn#Wp*&5tTXYUg}ji5B+RrJ(vo`vMU~LwyRLRF%*BG%5PT z9M>zxgb&li7s4~0FVCkrFFzdGC(!XThWFc6pZ-Hw58Hof+V8p0&dteH@d&!`JVR$`7VFz zJN4ZCgmhkICc*$>{;YJ&lXTnXdKPMh@cAo_gCED|dN^bLDCEv5-N&ErLo>ZP$WPZF z=BE`q%|BYJcBI&Wmv10s&kb@F1UXuN9K>(_EKYkhvm5p=n9ugQFfQFsF3N5J$DsP3 z{yd-K^XCbfW_^F|FkVaS>h~}|ya)E`2K+2#J+A7oFBv?q$P&lz>qc)kmT-KjofPkS1L-(1$MPiJTs~9#B6< zfp4OQKM3X1v@O?*ted*Uly`h!*d}>y|Pw=PpM87(Y)1K1d`)W=f z=PlaZ3lNVY9_fBT`q#^MBcB$FDUcFnI885VyTaKl;yWPulj*BNh(B(@E3Zt z6#`^ze&FU=LU6k7pj>A`FETg*kNkOw5tFCS!^r~0SBT02==3)8qx3kv7l5uqs4;Sj z^1*o*DgZjwZ*GrA@2Gn@Hk(f4s}m-`+JPABnx1bpT5{W;+R ztY`@D_5Aq}e}0~L>Trk-m|f27lQQo1G@M7D4D82=%%fKXcFE__c&m+{YKde<`EzK} z^~wf&sdDBz7Nn+rkDvb%{t@so!9s+IQHfG{(Jo*5cVGM*bB}GmW>M0ofXx4!UEimQ zMi1Zp^;45R{>tPW_m!B$bG=6Ie4RH~?=t~k+#lk($Isy(^*WWe`FXU@&s~iI3+``( z^9Q;!y$=z-pY%!U??YOV`wV>U+5IBY@pRB1qytSw2O%!Tl*{K)$KgYHeFgw5XS?m} zzT1VmPT$dX;3@&{VT*_HTqEtM+w;thE8}j*eV%QGdpl}p{s!T&Ol*N~V-p?`J4e=H_@&51VEMbSFe*euL4io~r6a$S*znW;i{^ z3gOOb38#emdCo-OPmw~>O^EjEHFPZT5(u;&mwyo8=Wroi3t7L3&&5P5`O-f6i#KnY z;gsM3)5nxvzgNi02m^@u)6ZjF->8>2<6mZv?hyE}{2LH(R;~yOa+;6*nH!;?2yAy0 zakeks&+HSE3sk5O={oC#` z)_u_s4=uFAhxyd(G(Vt=`2%R2>-&ij{6W|a{iA)N9!mZVk{#`aeI)(?#FD?$@)42F z0sNEvn=K!}G=T$%nLh_cYDw@M9sTSu-F3Uxn*;cU{6&@zIvet>|GBT*kbjxMgU*J0 zq#5X6k)AF+W{3{p_s`((3=#Dwe)V{l^*?2?!(Rhu6-s^kThtT)WlN zx^AqU?`geX@(`3k<`3(C${+5|_hyBg| z@O;+mLYR-Z1A*=E+@AaGY?tdE&co$doAH-=f4Rgz(chowFBRcGj{fQZr#{njk@Krz zD7gjBkG}=m;{IE>-odugq`yyu>+l41M*MjLq~8S_c%9jsYp^&W%Ip078sln*6FNBv zw9{;VWC2GNV)LVLA4lrdb-Ih<1tlqu7pj4a?S=gHs$|Q1t^eq48U6HAX_x6Q)8F%X zoc4=Q6IcT=R62D0#&+*F+mE?R+Nb~fY~cSUi+4&s;m`!`N!~2!WbvI!@3^3qeX$22 zr}Q4B*(a{UY3Gy0hf@5D{!Z)guYdn4vF~;H_dr;W^H2JGJ<`X%8P
9FrUgzKH1 zpu^=<-=%ffuRFj%UM`i6a|OWf7Zf{4E!wr`<{b~^awI2jWVMxdr9=I?`lEs_;`tZl zoOPSpHkdtN(!IpL}}!x8i+Uv|P76qK)%d0zK9!eLsxv zS3CVLU0$4(z{iO3V{|({*X#SNU5-QQ^>y}M`~3Sf$;Wg%vt<=uCo{is{rs%(7s~Bf z$c^&e6+9o~_imDlpH}#Xln|~Tc~0o>h4B64CZH@YwzvJzn3(m;}H9A znYKfE{>tb=H0k%{viF`p*u;m7M|-ax`9Agy#l^q=UN_%=4cp~CbC*}RZhnI827Di3 zx~4eU--`lyQ?EHbxNc@l`oDr74j-_k_rY^&Y%S zaq_Q!Hz&D8^+fn;!25vI3-N6dd^#_Ld^|%w0zJ5k=|B%|&glVEU-W>n?<=$Ye+lH8 z-d{xicY^#--qa(v%W)qP2$ha6H_@%~%jD1~zb6IWJf4@y5DmHcMCc&-Va{r!X3hbMD< zvp+-oaCsZP&ky;2y|4|wm0tQ&49Q}p)9;D-eT!uAvqHb$xAW&E<5MAM*0c8u`?*a_TTK6zEb)g7G+wQ0KtXm=J-gOd1UFU1eam3*7 z&mF*T=?{BPBA;(#+VS~!01I;K3Hc=yB6Ml@MKzPB>uvF*Z>yXY4DIOmHXL8inf+4I zLD(0V@3d+shthM!;8%wKmr?f0#9otfUx$=NCTZF_^~uIQIgv@d4ADjg~} zmsfg!)b!EUEv~1elcDFLcPO7rHSOoNwb{DAmhv<8676TJSBN%BJ@@}77HIw0p5yg< zh@1~tCI7sAe5MXS|DF@=tiaiO#J$~#t%^6_uOl`-<9ka2;Qo%oQ(n|8f6gQRm{xGO zmn`QK?q9VRE*HNq=6YYV_m&>OrU3bMKHJSJm5vVxf3fLpb?YkmCi*{p4@V$>Vfqy<+J$hmHRs7Rg9fn*5lF;Qf(4|7`fYAH2VPUQOlH zg6?y2bh8Pk+x?kJhdN@eA0wZrOMk|)@yJKiUJjk2_SC;0!+G`Dz|VC1nq2j)w&&~al}}sy zha?Wy0qPH{pXd80@jjI|{qM)IF`mi6`e{talJy(w%X&B`M|KJ8?{BDYeA;^y(|w$A z!-q z7p4CI{2`9scJ{^X$pC-|SLslPF1|+FsorgNM%^?yM{2M$1=h;!jO%r3Usk$(nJsYS z`@DefBJRO4Fh8^>9GApgoUZtNoW^&T2p+{BTcmN`jwm|#WoqAiJXbp3pzuC^?~!CM zeqSdU-d;K$y}f&^y}*vF2>5XN(*9GjAa zW2UF~*?7EbiQujMyp6|CytMJyZoIwNIQ92a{a+s^=I@W#eO!N@A^DJqA^qN6I!;t9 z>HX8%km`BUIB9!+&!_vKf7d8}@7D=izj2>R#s~gT1yX;9nFzHZn}@x8>gkg2<1?Kn zOn-y+=vrw|5c)9j}t8v63;d<$I)A`Np)fzz*-==udd%R8Kgii%wyc_)~ zy~dyKdt}cuDI+H`p5BlfPps*5XLPZ54BlIgc zkFzL++oW)0k93x&C>`Zqk`IRiy@}y4-l1p?vBx`{f!SS}bFZgJ9K4P_V*d{S4u;qTm$+0a&Z3P_v7RFCda;#0Q&pw3D4S2H^(hZ zR5W7W?c#nDYxsM*eZMZ1&EZh1!ivRl+Ii&wA~>aBc2@+;GOB^@tSc@;aZ*YvoS z$9CU$JBXpF@b!0>f=BV$0v_>sP43%#ZLc56_=4|HS7Rw=0#-w@AIPocuZ+>AsvmK@Rp@da_v8 zXZ{}X0hE*Oit$m9w_L*R+4qKgKC5)ve%_gj1Y>-9N#c4{qA0yDzuwZjHKu(&grCj+ zn4WX`^XIXB53l?om8ZWKWdP%pn(Xs$(cVAEG=s|}+h*TnA^(1j z+4*Z^Pb}=pFda1(|yznB^ibD(_do@7NS0d z`vK(l0BSbN?nk$UgXQVxUPSN)jtNWgn ze7p}-;@WJjzYR)12S1;(n0&Ck=KJmcFj+_YI=M^OYTP$qA_X5_l+G-TRoz15h@*@)q2)nRhzFz z`cxiOI|p<*QeF&&=y9p%|K~zaaGa`yCwK_0;t+e#rd9ht;mI|K*t;-;mPtB69KdlJ652JJlaxK7*fo z`gkaxJ|gnV`n#t6{SJk9`%ypJ(j2@9X*e|^I2z&6C>;3 zydy`Sw?6=Kq#tcxvc65o!MI^=c26L&bC)##%?g)XtK-J~`1HB#y(IyNZ`FCj_hHuMh z+TLECC-6Ks<#;Q|>J0sQzhpPf`|cMcpU`!u+p+40>D{2+JFGwNEJ>Q>v=gacG@|)# z$93DQLe7^Si z+wW<)AK9~2J2rk1K54&%KMEb8?fi{0!g)L8Sxmvc$Y0nu#LP+O$=M|-iBGkDD4Lws zO1Ai)FbOHVzqiQ!xA;`8&uR0+_+Gi=_4|dXpP*#Qe4aF zMXe+NjwjWNTVA4G?ALbaPm5mY_CVy%`Lmz-z0eoc3#&)?Ul8~eM~OKeTBGr{i!>hI zB(cvAgum}6dIWv{5$Vx|+Q0SJTYQ3c#>Y##4`t&y)ibj{Zg;Zt5uL}v^ACCc-~4g+ zx7hDrL%+Ly$o!dh_F;)IKzqQD-m9Fa>C{eOP?FU4=T{4*g8Mo2p9o(Yvvtla!-q`4 z)8>A#-#3h_C8?0UA0jyax-Gb8@An{lytwR%ZdyzgJCtphwjFK9iCVn_T;>_-8)8NH;@q+N-48-}j!}@~FW1 zzQ2DTjdGIp3gQy#9kw{>en9cU?lbtlw!d#GvG1}Kr|z$F zUCw%rw;%9dMcUWv@wn1m?`Li7AEEP!D!EGcw`wKUR{w7`V2zfOPC?8E`I&o9e!JY9(QH&Z{^bZz|Wlca$5XB08>8Aum@Ryv-L^4gd_e^49O z@6OM){-Hh)!0{0e>ow!qFL=s#=snQlG+l4TV>*7yHSLd{GhT(-?ANzQ?BmSGm){Sk zUWoo$?Cozyx`djCEspo;c-t|LRJEe(jgnuq_q3<<>@7=v4=)-Z`1b;Qy!t*@dLP{O zxBNTb>3RE@+7X|Bd>*KD+$?zGF_pjT8TCZu3HsSG?d&LGAAi+-`u@2ge?HXCfqh*1 zJYVSq0@j0BA4Ob<^!{G$F?(Obetl2Fs*hhru=G7099C@Czt7Y^ZN5Br9qJYR)qH=X z_we^UBmM05)#;(1OS~EXSFl;orTUci^Ipl0UTQxNL+|ReQ)~ob05N}7I=&$EIR1EC z^{sCGIrgR@tBiUaC#%sHjDdD?s`banVwGElpH>BS-28CvpK=GB&*9I>J%^81 zQD@J&jY&1RSNl@r>GE^ARyxiRM85yfYCMhtqFF9F&X`ZGYvVb+Sg5jSuJ=l9Zr zxSyD3X&5i^UtVteoQekhtnbaPj_dn5eO~qVTTxFLhU~s&A^NKHo6|@AA4Pe2StDIM zw_wfy{xFUodr<0CI_5Oh=Rt3)`Wz`9qY`END|c%;9rtR!BhF)Dhb<;vQobF3N{HR>8OFYx zj86(1AL zSn6M%q2=j&D*DvH@^ipYehUFgIa6dlE`2?e?e}dGgjE|ac{?ieG8iTu19%~a>wC@4 zH@WXJ3JlC=$n;bNnc4S#DQ~v#t0dpw!_;iwXP%IKcfOt9W_zv#79w3Ixg7lcYA=WV z{674av|q=Bt+w_vPp|h+r9A3uBf6}2bU&xJ_s=ZV|PztVA1Rk?5+hWeT6w_ZoX zJrlK){+@+UFZDeMx}Qb&=S1JH@F3<%zxNnESN{a^!~6M1m^M4d_2JdQ`1R)i!sn+S z=Xb>oK0F8cM5qrxhJ4-mB7A+U4PSeH%=vosMfi%_@U<=A%b!Pgdpr$#_D|3YuHQHW z%1ikdc2B1*`92WSm4fE``FC;38zm!LUt;QxE=N5+=iRGjQQjlX`n02mke{EQ+z;XV z+C~uw%h=D^cu$NC`8|Zn$qS`jzsFRU8kh@n^~+NP4D-0c`?|bKvm<-GbTqM-En@wM z{D%1y?gBymzQcAfS3>%*#d$tismX(!ShYka(0qO0fB50~Q-yH;3gV?H512xu^3>3jDLD<3P;whUyhp+rpJ|2x%TM)j`X5Wa>xn$M2TLke`$dQ^$d6MvaKlbDK zIgz`Po%&x+cctSI!Q=h39rTotbJ${kZ~KnB6+yoLD%00W{X+DMibscEa%qMF20foCUv(VjQ-w~r^FTY~a( zoBm4iPS<;Gh3Gwk*Yzgf|E?FqxL5p9&TnXkVZhr1pX8u2eY4~fKZAe&3&URYZ`IN( zHGhFbQI{UOMzyM@o9wFLZ5p@MZn_>bdPqO@-07-Jf2$x)=`lFk6WXg)D=w7s6SgDm z=s`b_|CAo28AuP?@p-p$vL2-2{t&xO^65vq++81dZuTJaiIi}N`7`ae4(~VJH}U#S z{bu#pZ=X6YKJAcJa-uvlyW2-{vUzo{B-`8FA3$Exea{oQ@!}kk%2Q;7S z5vqQ9-G7mcZr@c2yGCi+L?`7Sq`J|65WaCc< zLj16{=kI+fc6LjJ`WmfwIU?c5_~>$fS&Jb*oiJ<0Z_~X?>(~6fx^#Yrzg?&P+qm?5 zQtqd7o?vi4y;As_N#;;r7z)u}OFwsQk!1S*r0l-F5_}V!C60H$r^fHV^1Z#R#n`_m zTqDB>P4_zqkNoYBV+pa(8^d^ylF*NG2KNi&MVhYPq47G2QTKE0nGynV{Qg!-Fj{iXDe zf3MR0toSk0a~&2D^_=}vUx7aey{5O$zd!%T_d!m|%&PI=g$jQY*{h>85=+gt)wSKK>#N zVh$np=aGCrFk4ru|LyhseU-j0%PI6mVo3h&RGa6HR+Lbi+t#9%ktxCVa^237r|u976v=QrtJh&_XR8M60DC|ztf z)UUe14bD>dn-Bp-6MmsrI5z#V_6aZgWqNg=qLKYr$|v1#e1{e!7q5_hNmee={y~4! zN|pQ7pQshJktREf>cp3~v%_v*6}`MSkbsp;DuHp-ujRCgAJ|06X((T+ zaLf;%7w7p7hids!g=Lx{x#}YA-|jUU-*SbBlYwgHOyZj`G|U=o{)OszU(OS z!+lxVr`2VWycgi<+&&-e8TVxw`#sU}Jmuf*XR>mqwlk)*(axwGl%HlfJS*@{e@M5= zp&h?ZHnp?!2jue#E}ksV-=m!)pNu*Fzlwd!R{PIi{FmT7!t6=cbk7zH7m^+o|FdC1lk+-H)pm6|U=Cjcc1U-iC-{;wg*W zpW`~_5b)*c+YY!A{vEb>2RjzQ{k{Rni#h53>^dpNeXA|nPxCdsYM#zxKL7JQmbjiR z-=|{z@^Ztg;A#)?+n*m!pKDyA02DK8tlGrc26DMXqR8h3p2Oc6^e^R9M1w9bzvtU^ zxz(GkaTn-f=-sVxUB#NtpFFn{IUz0q#P3V_eNxf|wkF9R;{jxl&I%nb1b?rizt^QW z)xNXA$qQi{V(A}?ef`S$w}kS;7Vo(2pGCgD{`B`Sx;-pTMaLkN&)54Op5MD(vZJ2! zNfpPlW)ur5*sir&?)NYz`Ze8T&%r-2w3>g&H|4~@do6^kOj#vq=VziM$*3QtU&T-7 zqgGV9s8_3z&vW5k#1r!rp7*iX;~Os1^gt?KtyhSaN`bdSI*ua8_v5`C@9ms# z^A+GXn#&4KIqjtjcZ2#OX;tA&|i5y$@=d%C)tAAdtWby_UKvM*Yxq_ z{l)W0_6+wAMiILo&3uNTmG2e2^qUDs`7rkP0)+MH;N|PH9#LiWbO1A-J)!O39*5l@ zN&WMSb-r?aPr7eb{>D%VA)CLBN`CYFbxiX8xy$DH>quJf$?b}he_PG_I)7#J%Hi&@O|oy=d{i9;}(H`LhG69vOf%FLGFR70)UP_3M(}lO){Yi9 z9)C`@yj$;0IzIn?Y@AxzvY zk?iBKV`>*^`hZ61^PD|%^!c-lE?r;c<>Y+y%u+e{y#(T8j^Fz$c9@*EYD1py@7Iz2 zb`Vq>((<;?0fSG5CjSHCEg}D~#eUy+$K^_w-}7}lG!1Yp$?oT}_t1Z~j`MlCr(X>- z<+%stoL4Bxn(6$8b*zQ{$0R`vb9Ea!m7}+te-5>NpHyp_7k6+%KybNd{iyUkgfc>G zUZDG=1C};)at_&QUZ9`zU@%T;e{cU*W{3Sep{~m(xu5uC4*s9;nBX_8V)a=ZZ)(xE~nSd-qbU$D6KcyW5-FWzGf)fbnvT{rLj*8$!QA$K{0iTT?;o zuYcqF@_YpGkUp@Itz*a(LgttJA-| z9nq0TI@9~&S-bkatgs&3w1#@GezH|PeP7m85!*M@bsF0{+VT8kd6xPa2gnpcA=3A@ z_B0UC;id-m z`2VK-&jUUGndN`!3CjPEaPQ6MPnW-a&Rf}y9u~}D*X4Dt z1ABa~MDjiw>aowqzVF90!vJEM$OyHQ&F_%<=Rmfb;kG{l^_w zXGf#pYRgBrCKp&ui3 z8ed7b`Y+r!DPqMpMy>Yj_d6?y(Hf1sZy--Br?jgOOoN^-p}XH z106F_?%!{rtR=l(>(7Os$uO}=;{inc5)Z|`jrs5o8RjEqdhW0!8~jJ%-~30yh&-p| z`X6bAYj$dU-AVK02iG%%zZ@|&f_wQ5dSi55V{~5kj((}fc^;1=P4-Xy+T`SsZx@&b zoA5t7q9MfoUT4=|_Pg+fm^4nT7c|j);AML5C7N!~W732226{}t*dEsf?eyT9isHM@ z@cR7)`Zd(k@@zf-P4y4?4ejXgi(a&y3#^^B8vA|?>w&z6{mO7WO@+{xpo)*Dc6|IO z^!b%7^x5BgEPfw4@+bQI6Md#e|3}d0U&Hy*eLDQ`9cKOE&wu#yr|!@2xj+o{?B3;k zgSYP-_4yS5No<=x8|c>fd?tCEXz7sQ+@ zik;I0f7hF}!1wX0L(~Gmp`OB_VFbVL>F0_(FD5;H4qja^V6v{jQ(Niz{eA#ZZr(q| zDa4x1Yn;cn*%<)y{hlD@%|N@$;P(OiyORB#uZsQzxP3p|^ZmPsGZtt)mwWkiN;i}H z0O~Tw-?PATex}Ro)c@*Qg9L*2=Y5(?{UiFXlq>zS=QufA;JEnI(QJL|{uJT+07Lj^ za_~PN!k?~){C>qvpbJ&fdxi_q)W1@m=D+oK%#nRH`gIH(XFUHXpUc@?(~Rp|_!sg8 zeGL8{o~|=A9p9vJvQy(}z(+iO4pScZF4Z>rS^GYSzkjo9w!*t!bZPfSwTkACP=E-; zM-q3Orsu6rPb$}{fQes8eJMoxJ_Emh?RM%WAQ5?3y9*&7k;iha-#^vlVS1CvgS*HW zxBUt?06CE%`dPdO$$fm%$H4fU_3|^-zk`~L{QH?*7ivawxyIBVkr$GzU+(yx%4@l< zpZ$4eKc^n)%jyw+$2#TLpNFG<5`ovFUQsT2xg7uu>vJ5j96uTF@%#0IW=ClUvv71z zc}NB5=l_EPT3X(w4*W)wC-E~Br>gw?J&l7lpA1haNV|k%`#buSU#3Y{PoFmI^JNM* zOFJloBQ)*0#>{7M{UHA=@wC(Z_-p}_aizcH@95xiO(k}&E%%yJ`E*4oAL|s(-$yn0 z7EAN61%da-iHi4UCoxyJKJq=}l8;#Ow+Xns|EB{NV5xVVh|B9UJ_+IPD-nBx^8Ts! zM1-dur~C+mXpi6f8U!8&-mlIA`tJtQtNEHcsLkNL6fQ>{|Ge*zh3U~oUS2`)c?I$o z2%cAE0dReqa()YP8yj?FK=^fZPmev%TM+F>d4rxE+tV}k+fsveF9@T9Hs3#P^!UDg z&(wbruwvn;##28p@d2yc$j{8EY-ewUNDhGaQ`Lp&} zj{h^hgM5m%6+Fj#t9+n}eE=Tv$={f@uzupG!Mue0HHeE-teJmY-<$dqBPBm*$@n^yf_c`MyTIDMjU3>Nk)dq07cw zyzhv#dyetJd$G$wPwyfjOYZATg64R?-lrSJI_m}SR*>FMyP}Z%dvs6yh_1H^vYU!} zhTqpysT|P`?Nr##3utE;-;pPNw!`4>ZzMkupU_JCHP|P`vK}M78(l8I%bNb3?SrWI z`>3~Lt3B_xRqL_8Snqlp7ldQIUMt^Y_?zfwM(>%lv#^#3427XY$tfzc%@2<|w$IL4T=)dj33>`{Dkba`AWJ_e|Kh z_4^#$KPSolK7Xbe4810zqpJrDt}te2&R37to8g>~UlP3ZOR{iu zbzRj_*3nnuAi(>^Ugd}-KYKJ-^kHWbf;rq-O+>Q(& z$sAmDlaZO;Q-mF2zUQ+T zk7@nNv~j`b^^A_Sj(4I@3Ed4Msc}QyIv)>!KGN;aEyeqd?~d<@JgZ|0$8p3kbd~no ziH|E{Xa9UGrH@N`$2-^dg-Ay)^4(s){|44VTFprG-dNMlkIR#3wy?QPe`fO0f#>6t z^~m5{+gC~PfluNy`?K2l_Ij0q?gu;nshu+UWN^-Rj}Y$fk)vOS6UajwZk6m*Zxl}} zeg8F;8_*FW?fm5AH&?rUxjwmGRyzJhTJU|O;m%*7xtMGmztL#?ppK8nxi~>6M2|>2 z^-)Q}AJg*Tj(siaWBeQa90z}f=cEeJFDad z9Nl~T`RM|l{v#po$5DTNANK$c2o~&b+@tg|zXJ!${@qx&3yuEvnsw~=KZVY_9nRzN z?~vStHcM>R^ugPixC@yG6DI$s5YsLV;2-_p^gP7)rMy1+z!dm{s+Z~WSwcVTiG4q5 z#&Ti>UW{*s3nYf$W_Z%)gv^g7-djHViuBwhk2kBQggRXp2=*hZdGDE$^Y(@=R(qV7 zo%HXI`FdtMfJ=b)b=Sr-1yQQcTz_$bOBSo$mHvDSThQR^bbrrUa_u4Q*Sodc{d4xu zH-RfTPvKa#^^awpvV&;$P-)^6Y=wR|pd z7UCJUe&GAFuybAV=MuwqX<)ov-xqTI@$&<=^I__n^AWWFhxlE${Xnl5kE>i*?|-t~ z+j8|Lf93kG zu)WXj_=?oyC$U@6`Q}``De1oDUrW93LpQuX()z(TcvDb6^lRV1{Jo?AG33TTx%@Nn zS--@On7*{=pVzX!Zl9-gH`>wZYl!bT;$fk^5OjYcX0zfJw24)t(lzuAdpk4zd=2o!RF-?^8Y(8fq&nBC*Pae z@b@<62jk%RDJ%b+)Si6!A4N}|hb?zLL%Dvt2t(?jBqXV;&IT+etO~} zYXo)TN7at|dm#OuPbl9<$tV6XgzX}Egn3Ti|3SNMXPuAG-hay4KNPf|w(I*OE$DuL z?PF{)RL6AwBA<6*B1-$GrsbiW?+)s{n0|)!<_7%`_V<}VzJDi-bezuosOWeE#&qD>*ef1 zSpPk&kADn)4$SvPm5>(y4Dp1GBl-)>KWy=^tUo`xhHwl6i21Xk$8YeXS4zIu*Y(KG z`Ud06@wZ=(C}RIUQu2{wI$!P;v@*ZEaO-P^nYys}@i(%cHiYu1DqN@!{~IV(u z`{49UfIo&-{4zh!Z~UmzmGS*%g2BG&`jvM0lQ%?nGVhIFKkoyi^Jjzp%$FCU*9zlY zFYrgYe_{JIyr<6m=Jmw?$0pK!8Gfh3(~PtA$ql?VoXO!Aiz=y}nH)&(V)*I! zF2?hO^Zum2ii@FpTyC-t@OyV1M<3>~dRkxfVfpo|^ZHO}yB}lc^;JQWo`aIUPm%7u zmLJf4_Ycvpm5x^mLhnc1YmzkAA81aVGvxZ%-yh3-hRlyA6xWkqkeK`84E~9Kn5DQ(>-qFe5U#9imHA`b3kM38o-?yNm+c)1w2*>*uIq#c&8?x`zaC;`{}_n>o3H<#jKT8_|G$Cv z)Q$&qy#Bq9@<=+WzN;kuz;_6@mpO<7{GTD8`FY@0g`=DvW4X<{KJSO~&)*YoZk`~W z&ph-wvHsVb_ntQq&)*U*;NzdJ`@`R1D{KG9Zwh^%WIK0jyCHpF&DHzqzwCdUP}jcW zRFUs}ZR$Ont9S4J_?-0Cdw=Uab<*=++SL0}uHKu{`v|`y?cLF)-WOTV=g~mEr-A2G zK3*R5`){+J&!fhFs7GK`{N$V2uJvyypATSvB#+;pb0?oP<=*erM|Iyctj~!tZQtqJ zN(rm}Jx=;swELbma`-mtwUWa>Gas}tH1c;;7qP&H`wJ@h%OMg&@Nyf zyk3azlKIZc?;jR; z-(SV8wC@x00OZQ{?Hq13(+srN%=h`TAg96^{KSxeHOZOZ(_?7H^PJcy_VbnBI`(%Z(CknWrO zHQRAJV)T-q1*q4k-$xqu@!ki1oPQr*tk=u-e12)n`nBNZx*Hb((c6#uA zJ{dm~3j}>SZYQW<2=x(-^L9_ZZwb1a$KzR2f%cLb&hk$9<*v7fkuD+M@540N>6c z^-!OI{uXm4wn{SnPHy_1GUyS@`=@n8D{!CF{m@2z+A@Xx``2jO?SbQ?zxf5UF<^Kq z(^U|y>VGAupMTHJFxuaVi(I@v&zSGuxCeco+8M*|{!CsT9Cy42)0+Mxdq{t{KO54| z^NEapm0$Qi6qOwRDxS{=e6W9>$3?3=J$e3i?6Q6VUxX)ryk{HI!Fxl*Lsy&h{yKj9 zd2s$7>f_6lD#8(S2uJ;8DWW|-zxusmzo$8PuGVkFJ9E^Z^Y{1l@*a&+PWYLl<&#vg z>T9&V!VEfkw?e&;C`+E)JvMxvwVr9@xDUMhx{3eQ_Gs3 zr|I&5BqP5sm!|u)JbX@q{mpgNB=r|!V$G)eBIt*#UvUrI`hRe%;WIiop9wz1#OL#( zKPR8wH%2>a08{L=d#wFPAo%@V*PD$4nqOb9G0UNrQLB8g&@!J>RV#Rs+$z5mNi@H{4^je9CDB>T~@p_l-@CQ<2Z1cRAwKh_AmCe-P-mkPm;~DANq-cpcWX z`$3gyssIO}7i&;dELcCjL*aODjre?AZQP>yzJ6f2^ka*B^4$tv`mqJP^ka+sQh;Ya zeoH^1-G+WNKGS~O4nTz7ofLq8fm<~Q`C z#uo48QZb4_jskYqn|-dlsXd^1x>%U|rz*t|*gQ~mmXq-5h(1>pBj*e^ccxnJk! z!j1EI)RF3Azt*2vqj9mYN`CNp6Sl*3AmfdDwcO9O{d>l}w`%@DjbMj9D8Cw?2ilW9 zx907XFI^&V{@hk-ulf|x#!EDII#YYqB46~pMZV~Hi+s`Z7WpMK>Fw?Z)1v3-=LS7z zWd!Og&pGmO`tc-9#qx9Uw{r-*fwV=M%V)IvSLc3xofROlD%6(eb`@!`JPjk5( z|2CRF(e0pVaCz|k4W!BW+_(e@gp6NR0AFtu9iEL74l!BqbB2PvMM|HSVSZN$e-PY$ zOzhP3?HbAb8|+BpGcqIlKKUMAwK6$Qx_ir(jXNq(9tW z!t-7NH)wM7@&cZ=B&7GJ?{lZ`1LEL9aCF*)-_jyjRvk+L$CR^pRzgp!}bglAd2Kf(xKI%)@4(VWj(;m%ThXjI`GoN;6 zH|mid_BYcrjLr#zXF2KeeAc6VnU9$Eg#6HkIXw6Ksae;XT-e_|fSJ1nKM-7RU4M3i zNXm6P;?;;Zk|~6=pMC#+tLAfjl2fc-M%yr7P=DC3B*pPMysw*Geg_P$y!Jaf z&P5-HU%&gl8~kJ09XUMZ&HZTBV@T<5+h68?%~yD>KRvCN?V zd#>V{SZerCD0apA|GgA8!tF*M<-qdyzfQ8FiK@mM`Yiosi_Pv)p4`{)_q)3sozK+n zktl@ZDm`zY-3!kH2T)Fs3ciHJ;Cy-eyHQBGHdwptH^RRd+yRk)N#<3-YxQU+81TJ4 ziRbQ;D4Jn%Nbzpb{N0G8ebaAme>d=x4yS)&iNaSp&=5kh((aur-cjE-qI?*Jy7j!j z5Y3bH#9E1>8Sej?AClkKt1nVGzn6FzI$lE7j@y(#f1lQj^$O?fydAe`!xLv%eI!}$ zu*L20y8iCiYxUO|UL={*KlM*189QHv=#M7}k;>8Zjq(yXp3hQ!j|S_xJX1N|AjQy! zM!(DNRp$4-{5c(e4#MxP9>#Dg0af}PP0L1oPUw>Hc(PRHot4Hs$@jLfYg-`X^ z`n_uR;b++RcYT|y>KRR3uJsDC>I7d#-{x73^tpT-zA{x2xO`UGbAe9R0ICzi#6rbi z>G%`T2Zv`n)YS6w-%RrfS%`i^()oKF?e6vX{nXf=%c5uh?z!-3EIg*pz)_eY~#@LYv`A2{8|g*?}wN^z>~|E<&d?0-ta*OeR1&iH;b%Q?AncsZP*u~lZW8@IDRhCYx69}1=k~lVeIt#zCpS#V(|W6 z%<#QC>)^;2(ITzxp)YN}fOIFTCgD1p8}lyM9`x@}*9&KfQmFYtOPJaFB&&zWi z`vT?>_cwh#QJHe8RCGMbUNL0Z&N&6|5AP|XBa?C`BbJVe#(vI>G{}fLSg)y{Pvig-DvrH zj9$0H?#KIcksG&2_1rzHXtdnNRymJx)a#xeb9`TPbDt9E`|Vw$gkrhYbAIpB>~y`8 z*t*8|bEzLCz|f8|u3v;d2<|u2zbGMH{}6NWo)42}vRL)e_hHVle(>|yn*dwFzdHJl z;T+RX9~WJBXujXua60N&Ykrqa-1>Yq`UCXg|ET;S{?#<&@`$BvT>5xEhsuEv+jD&9 z&{#`a*L8{Yx8H+ze}1Dq7wqSL=ai&c)ODA|7`F`my+!|SxXbwf%1BY8Klh*Sy=LT- zN3xBu_& znWS6F(gs@SP+ADIO=q7wT_9~~Nogs8E`&9+cS74_$U;*D9Li!4k$_ti@jD<(1i>gS zQ4pdm21SXu5JbNOM2U(LL@|oO?>YB;&ScI*B7xuk_xj(yntPu2dG2$bv!8qJotehH zsphf&x%t!ETlS^gYu38kTRT$QTAEXCI_g_ncDKg%xto%y=Juv^Q>t~v|#?JJ$#)$s%EsW!K$6LBn( zC#w!bu(hMDePvs$xvBlY%C^?T%DqkTn%1^hTU)BtN+KO=+M7zQa(inc5i^ty1>9@g z%^myP#~o*#*0_B&9-@iZUQ%;M&q}p*>}_AQs-t;-Yb;xRLex!e#+uzxDvPL_%AzCn zZrjJiv+2gYEwLo!cU2;jO6+NDN>*oOVNmddl{?9uwv~H35_EBUYGq9#*1Ef8Wov48 z6BUMZys0^v>a0n$#wfQnO)V=zX>T~`^Tp#aZ_wlMrdP(>g60ZOO`yhOWi-{CXi26h zukXY4^&RPSs&z+8%l3VgU#leFotqPz+8W!Mb~mS58?)`LmRs%?s-~8-J5E)ZY+L1S z&!pVc4BSm^?*H0OHnq0wD=1j?Xf+{Jp>--9dV0g3{JFZ!T&kqJd2K z8Qq&ox4Tbp@6V72ZF`#%P0hRAQF-Z@X!3N7pWGU2-kox%tOiE~>vp#`?as7E-Ksr! zygq**7z&$_SUizTrO{3|H>X<)`KQ4C)~5DYd~b@^;3+MMj(ya~+g8SQ$1X{&Y)kH0 zi3+!BCnlSHV-wknl9S2uel|Aacf(9$0hqa4_x|wGfrjFdLY}<)Z9+lUg6%{=9X<~ zg?mftzzX-)*uHE_YdgiLfk9NRXiU4?Gb`L@?5Ae;ev?GGQ+lUG>d)DbXxY-yR(D{_ zS*JHNuixqo@7mF@!GHR$jXMrpy#7?*mIGTGc3l!$*Hm}OfxxMCyX$wfoV%@M-ciP3-OV2+0qTM_8H?(f*NN-7Zp3}5t&#u!>IsL#{Tg}!Ddt#~ZzSxHCmjuEc z*=-%`Pv4aEpT4`L&6~c|x9*a2F5PwMIiVKwtjM;uot@j-_H6Q=o<93@Z`v0M9ti9= zL#Le+-?;zc4f}S)63MN5>axD}i?#$B4s6>Q3^XLSMJ{UD(-{bzdFj>z@%YwVr|v)V z%u9BpGl`D%4eKsCYoY_FCVXP5GnMGDny2Wg?^ZZQbY<0og$m02^i+E++YzTBneOLX zTbg%oZ@M(Kx!D@&dEx(mF;pc}7^bGg86o$<`$_ zsJF&woV2d6`d4dgf8!;ogf;wSV+k7l-7S}-TGM-5_Q%}mmezf-_I5EYP_Z<%yPNi9 z_ohU5m8rF++B;gCX}G6>-5QYIZ9lzFv|sW-jjm^8g`<38a7V?|hl$~YCRv1T8r*4I zk}fA&T4{8kiHR((+}DyMt2H(2TWBWTZ)GjEw>1?@9&qooD)Imgu_8*$SgX#<(zKS? zBm8W4uc~mzX%H48uN;zT>|ML|n6;EqnsM6n;jO|=Ju2P4%3UG32!O0|yFcV!PJb$v zS6uGCytJuiqQ+2E8F~*lT(t zK9A4q^ZER~fG_9^`NBTa7x8=iUcZmFu>$^}KjaVlO@Acd33vm(fIkoj1OuT!IA8`M zK~K;d^acIFKrk2#1;arz7zueo-jFZk4+TQOP$(1*nW0G76ZVFEVShLf4u(VFaM%n- zOpoa`eWu?Gm_ajShE3CqM99SmB_AQv5xOiwC>`5Nv#>QSw6(`tsa1#>*R9%|jG?Au z%_%7{-N{i2vedS&q=iho+o}$lUo1ha^#1=mtgiBZ^~e5S4+lG(;wVb*yqcPt3*xk# zX%Im>O$yOS?PfPGn-Oav4GWd9_>g%{EeMqZK1f=~U)X0kq|YMc>sz zY2AeG^scB5>NT`9qWPF+Au)N2o?L3PIc!dQNm-fQhU# z_uT)L?*>N>`N(x2|AKSY?74@W;0>(WxM}lg+s;U)&j04O7A!1tRm?bSekii4_e;ai zm4~};>Mg5SeNwvV#+&E1G(Pf|*UpXq?ceWg-|@LyYE~X!z4O-mZC||O?%w;p^U!0Z zGiDxs)T)y=o_W{ZPyIMwcEpj3m!5R;E3dxx&f`xw-N!CHuG(*|syl6K{q~)^8qPic z!e}Cu-qY53`G-Gx$30)^8yLF(E6ptrf8xT$A1ZM;S2)rR+sc~U!J{1BISZXj%8xEt zR&t7S_KCTBN|!j7IICTO8QV5o87`k!;i_7_G2%$L%02T+j&Up~vDKQ+EhQ_R6=mgR zweI7cGs{DcRV7E3IcJvDZw~us`O9it6;~d2)|O?i6XzXy+`_}=mv5s4PMLK?Sw(4` z>-h4H8S76vv2=AwMd_KPwvtLmN$&di(RHqh++7zg-Z-P8bk?D(N-ILko%3_wUX$ED zv#z{i)5ZmLuI;lnmsRB6+*EOt+2;6WU3E*kV@B?= zk8VxOy4q7Y@5Xx$o_gE24@SyPbe>;&T*an}>XJhb_MMm7;*6Bdtrbn;GvltSe|cj0 z?XO(vpJO|!bhguV<;Sjb?kSn&C@-tLIeKb&`YBU;hQc!Z02Dz&n`bA_mL}4 zbzHN4&f!=>HfN^O zq|8;j7H_+9$IPS1myl~Vsg{@JezdgW>QckzC@Co|wU?E;%F5?fESzz~%p+%2&Yn5P zS?M_BkVDJo+2%W|Y)3ebEL&h(XkRqX?O5hmF{8%jad_=M+uimr+3#`Q>-v}dt&;!P z-*&uHeqZN->upH{JZDuYTvj$Dere zhyVNX@7^(-R)Y?&TD^AjX%}4maf-Xb;C_xqDvls@`aawH#TeT#?8r8?&_~S^zb9Ucy{ddtFF2J zj=LUy~kDhdhkDvbIRtGSCxwfcTs6Y>5--9lr1aSRI%Ji zZNuT6vD|rN=?q8i%M`WJd%7cchii>vj$=)k=~`BD@SV9;u9b6FIF6Zf%$(fEoCiO1 z#EiqcK3TG|WHq(4s`A{!i`!@BetG1~lH5Bbxff>s^;So?{L1qW%k{f*KPsuHTJ5MP zHC>xrGfUfN9PK#Qc}{ums;Y$*^UAk6b000e_l}wKo!;A=SN`g_vY90%xw|T_{Ck{#HKGi%BL8UuBsxCNjw`dxY{y0^_*qc=@8v`=!C({0>T5;ZP3G;bU- z-@Rz2JG$t#<@seNdfdxf?iyKc?~SfF`mJb<@wPkkg?FN%|JYs(*(#0+FPZgX_}iO&F~yvX-iQB+U+*yv9_ZZoHt`t zc{%+;g{_?Cnv!LXHLeq@Z0;~!;B-+%l~ve}vaJ%AJ6)twVL#Glw?}AFa@wh3*p9Y4 zY%_#hLJGFS?DJ^KB5ktmvXwb1>_^*H)3r0{%4#xCRvjhOWXkL_EUUsLO4Kgoh4u*f zr&2k}R%dh4?WE1-vYlzOm(6s=ZT9jRWt;5_$ezs>o^7KPOJ>-Xl-tryTPZnZKf>;G zR61u-u+%n3d}FBNDEra$uhwoWbJ^@O%5BtKZ5{T-wo4pNd%3OD@jMj*d0Qqd+Fhj; zcAICB*Xbd-#8zEC)9$8B+8icD(uIyyF1x+QVVh+u6A3!(Pt+Q=M~^WaH`twrDLji zYvcF7#1Fm4^|e?^PGvtzeP0|?#rI)zaNj2R?zhyYT_BS2_{OLus2*h&^X?!o?rr=`ZN!b^Eial&`6qlp;o-%P1rz1AL)aF+)7u&dMPqd!$QGDEbs?E3nU@t|< zZ&WpmV!n^K@}Zb#h$}yf`A*_)*evE35|`f?Lw+xw@PcsR7(7k}i?91VAvZKe={i*o z>MWv0@QiG#co!;fs`zeHoQPL>Ew5&|~HD1B9r>MX7pLcKK7_hw>4 z)!0NF#c34t^N2&;Ft-+4q2F0Q&1S3>2I+`V6uD^~KO!!_MT_1KAgQhDv^wnDn%{bRSp#?{1UA}mh#l1$Nsqn~7C)nW}K7az}%Tuiv!hx~|ZiuL+3 z(naY58%N@gWl|1*{D9>x@`UNs*G|^f4-c9*XoPvdZTg-!$F;(`(yoUH< z*gXCWy}TkMt2m)j$@0(R&m(;`vM3LNbgFwyqgt^>Y*H7uWO6 zB&$9luD_LJOwmF#hRI&>Jn|^Tms&CM&wJT>lJtxH=lU;Dyej8%atzay@h{>_k&YY={2H2?pKhVw16A`{J#iH-=Ax@9UmhiWRd)g6stk&GoVb{Wj;8s?1+U6)Bqf1LTeVv?W(=ieyvV0GTIH#b2Z+-?32TQ? z?T^CYVO%NNkKnavNW<_`%;72zP_<6tx%y;Wl=);_tP3XNTc+S*Jw2Je7=kC`MuHyf zx9n@Qo`Dz>PZfDN!z^7b&fmdTw=&NF$C z$>U6RW8zl+<;3Jbhmr@G?81DY;@wQnFu9k>V@x)e>gm-pxrfQaOm-iqUms<1FO!Fu z+>gmtsOzo%e73dXL2`_dzswNzF_5b-A0#c_xoB z+1Q|8A3a5vvm2SbNtcH=>+&d*&C~SwIwlV?d4$PhO!kNc3LUC^>X_WX3V#W$(>9dWbz1;J==79bxf|_uE!5Exqp`)Kg8s|20cD^jxJ}<)8&39 zSK;1Qm2Y2E*B@cBE2hU+C3Lx)$-PYOXY%-N{rc*RE?YN*gp_}tCOy83$%eJcpU|&l zay64{ncTqSERzQ=*3%zi@+gzl4TJK(3O5i+_AoiyrrYv9*9N15#I(Dn079=b%2 zuivlB877Y~Iohe~4=}msfF56UsV>(uIm_fhCXX@M^&#C}1CzU%Jj&#T%k=BJm^{Md z&V#!C$W^**T%*fXOdeu#*R{HS-A8nJh{+9IdVKwLx}0Hh)kpRCVJ7FU*W>#>rpuWd zbh-NDx;)6_?2USSp2^kSdVB+ub4(ujgsz|aq%POqrOTa6?&{U!2bi3_M~@$6a_(L| z-uSXES2MZyYkK@RlcV3%UG9Eem-9@H{#B26zoEEnLNs5<9yx!DkgiFT*u@nlRKH5V{#9Z`+9j?q%`-lZTl+#$;Ec?!TMKCX?%#oMCc~$vsT&WAY%AhnPId zWb4Opipx8y`|n|L9h0L>?qqT|lY5ywz~n(Dk1%t=E_lT9YqF}Z=s876l!xr@m?OzvfJ zKa&TUJj~=VCXX}ONa*=@G1<*zlgaf=&M>)?$z4qDVRA2%`@Jy*9TYT6 zb$Nuz-81$0K_=JE(&KYX9$>P1aX|QEr9WH0e)MQvZa7AlyP4e2-6gz*6VUVldCr9@txSuSN7^o z)#HbloZqa+8>i`Vl*xG}ySC{1StbuKxqqv!UwyhRcQbjM$2g<_F8A!$`=?5FU#?%@)2+(`*e_H1l@I9go!CE9@%7k0 zQ*sphVM;b1)UU66NS6nm(&dpMUC#VemwT{ZrR+zsU!~-6Cg+~lukT~>2$Qof>iT0Z z>2d@1b5wc**uPP7C-!HQY`&pipJ8$xZiIxslm7QIxvzZUdJ*4us4nNix;(^Wx2eZx znVe^`Sv0xwWZz4}Hyud+XLy+}?1FrzT`YPx;42F;3(4ceuhz z0`$R{!e=}Bps?RM+YEYDPdlyv?_>NL@Gizb2A*g96W{}ke-3<*@!N+@ILvEpr2>@zkHZjz|-JExQ|nIn!&S7 z{}S*z+{Y>XE5N&%{pxUiL?NZd^MfBM@arlw2 z_?;}FulD(s{?X7s8T!khzXJN|If5H}4Y+!KpzLgco!PMCh5otFuf_8NrQbwccr=Rp zcBmT{gE!1kM4k_TcQJktyqoclf@c~31o$xHw}98HfdR>X5xkS}d%=4ce*k=h@dv@p zxq7}H1&=cRBk&yKKLa0Q{MX<^jK2(S9-^oFDtMIfzk%l%|2Oyu<4ziPM7vVYi&VA` zwYc3-66n&6$}i4p9MY&?gEd2k0~zA(R?HAjH4XJpnqFb zau*D!{VFf{GugUPUS=}})W8WlxumqCIFdKkO<_m;Zi48K|6={?5{o+w_1wS>`_~Y+ z#&e{r>^}uQ0^SGx!>s(!=a3Eqz-b|PmhmOvb&M|uk1`$r&oRCRyo>QuhztMKXI48= zFCQZ=+KGCO=Z2rRKwo_ZR*et81$U!?3_^dIH6PfFI@NH%{oq}UuLd7sd;@W79tT(D zc|G*2=EE>fp8)S<{1)(f#=i)jVfN z68;|wUig3+_yX`MrhhEB$@ntxTE_jvg`euP#`MuG>-Z3HQSa(`k{juM)Z%ty2YrT2 z9BSJ8EcDg$MYk18>q_WvfxcB-6!$IYtLIrt{|V?X#b( zx*uF9x%&Hk@H)nS2tIz4(v;_C!26i~ufVGo>-sM#`^?Utz(<&!H^A$e{=dPqcrKt) zDY4dz)W42VM4o4Z4>CLR!AF_C8+?q}KLNa&aUXa+F6hJxu>};-dYi_bJ@S z#`)mtJqpzh_rQ*8k)Gdn@GMLBa&SW##%0%nyP2IE!3UU~9`H`4e;ar|#b?*W zLDRH#Y@+qF;Nmmy;!yk`_&DP~1h+nX;}Gl0racbc)l0J9uuRTupXF09! zMZOxC{u<)KfAt+G9{6*!rEfQuVt%oj4aMCK{c+ewwDAk#B8ANHFo4rvsMQKFajp2CriJW!8GyW{flaN^lR;KZ3X@hg$HwO{SlQeun9{fcIlQRo)%| z&oljl;C+l=4?fKJC&4S3{V#ymGTsZ`!}!<0yBU|ik4p8<_!BDK6IG7n`Dt)-na-aB zuVeZzfj2PzC-82@{|4UA_}k!BEM1p%e?s?1j2{AC$M{0xqJE>`s<G9|-S`yrX}YnF!@+N7c9s$seyZ=r>4pB) z#D%4?yLsDFgve<=a~M#z{eSPSod|bZdU^pl9>f=uGG_=2R_QUy#J)b~t2lyYU6ITRhG-kUDg0d>(aHl{vqJCC+mD6ajQK?WjR}Fq_P}by|=6Q8gTV~ui{&+0Ige? z{axUrj7P!iYIT1u0*^A@4&KT572w8t-A%-cpc-z;Ju8$1fF61Rq!6h-voDW(ESl#Dkth)y~l2;kxC`Fdf#1f zH+UUuCmwM1KDg4aB`#8EVCkL)UdQ+j@Lt9*1Rr2L1MWFR&sQt>7}LKD+|Bf_1U-pzWt_bATnd>!0%ny&vaxUo^^gWwsa{|s?4&R@sInU|p7$Lzcc z-plN~30}KNPgi__ld%5{X1|iSi0NbcM}iMBz688-i*COLy#7?3%Y8|kG0gaS==ap= z`dh&}nf+bh8CEY*@DXO`B5;H0w}X!{{VTw0nZEqq78-9EzZv=sjDH?{kokWXanatY zqBbkTs=a*^++h3(aP_$jrTM*4j+@!}E_jCVLD+v0{qY#s zABO#UW`7iRMwy*Al>IYw|NjG?XS~$fzoz#P>UI4&;3JG50p4(yu751}5VOAmyp!1p zgIAuZ+gS%*&G;5@gW1^$Ud!|w!5bKFQudjhHt;OdzZ|@a@sEJ#nVp*yXZp8*4>0{Z z!TXv1SHOoD|2Ft2wd}@h#xpjPC?Db}A3$xeqv$GKT z>oD%A`_1LhA7}bO=!@?{7RLqFU&Gi4F22)P9P0L9C%E_yV{xpt{u)LKTzsdlI97qb z7NvaAdm8Hf-AeEiVv>)cJo~}Z;3FXk)O(u01g~5nc^3NT#HAe%5>($)w>=?w4*KIL zh);r7V%+O>NJDodsWj~q<}SuYnIR z{w8=Gv;Q`D_8i?#=^`Q1emmo{z%$IB!@$Rw{zC97mhKYpA!cVeaj{-fpKGmxTdS0P zroSFM%F;azJkR)9;GN8$bBT*|)#qzfx*6#AF*{lC%JcMclh0YHoiqD6%f8+E4ry_P z8XvZzomZm1)c1ehNcxkttNW(lPffu`r{E6vI$bEn{!Ou=s=F2-l|ISDn6 zeV4cxpYOzYTVu$>PvtAdjcSy$(!T}cNuKfQwA8NySM~VQX35p(p_HAH7Rjraoo|7Q z_u9naktgF1?#at-<}s7=6;trjh>LRQgZ*mQ{}6HO{wQX%h)VZPi`$K1ln>O6r(tIp zc3g;TvTza6}n>7NJQ&v*iSjPX6-Rm^@najU&G$^_N?ayRrlnSLL5mho?c_cH!{ z*dG8_@7>K=BFkZv>CXqRgg+<-Lw?UEJ@2?s5qYkLzK3xSc!qHkyqocL;A4!R20qI8 zc5pZI^L+4r#*^S?RQG=`_z2T)2k&J1mw{I?ehqjx;~yt3`b!=9i|W_k23Ox-pvHlh zVW&T?`}wM}AJh5Yz-yWQf569?e#ugCJw5ki`m?}?8J`DkGQJ3Wl<{itEaP7ATtfGM z6>-sjE93`@3h$+-EKZ*jLAkkcSvR=)9tTx!4-yysXjs{I$O~SDees(H;+QM3G4D7L zpmIn8;B+*&E2Z<}!Rwg*O7INR4}**Mf{Txn!Fw351Mg#Y&IGSZ7h5rmbHPofp8y|X z`g_5<8Sfx&l^b|3lKC0*&E0ys&qH7RW`rvL)yK2}qUv|A5Eq{2!C}Vu z3-rZrEr>(SQ_IK!VMqO@gyL<)MZ4v{-Eb@P2X2&hM&!wO+R~@@IoUYy7WDg(?l|hJf!hZ+ANcrD}4fDh1pO!4tNc!uf!7JQ89zY5;X^#2at%lO;i)yz)W zGT{Z?x6=Jn@gd&}p>+t|w-p~pK!2F=#o!*smw~&P{Q!6a(_agoWqcEO594Qmk1;O4 z=YjecOE(7nDpn2`f%h~0Ht;&8e;Ifu+Z0!-ga zT;$gkm!7*}?gH>?#;=2&`U6nJ>1J^CJ+dnH7Vs{ne+Rhwo>`@TKXEaBj>G>lMEnZ+ z!_3ah;I$u87UcOa#6|fJvhokn4VuVrC-PN`%i`9J8};kUbUS-3eY^2DR-Qf3uVU%m z2K_gg{*NtA>pi5a>iwVK>id=4NPg)`Sq_WoH*myJ0)D2&sr@iJmlGFP7*{|Mr)$CU zm+QP6y!tAge@<~`=ZoNDSL*urf!8zrZ-V#dbo~MF5vKn`@Tw2%`ac67VEVrSuV?%|<>@)r~ z#k+Jn4}lLd{l}I4YjypniHq^)M2zPSM85_7>T7g;r&sE)VDo}}uZ}+7#PnA|U;LJr zIMn!@CN3;>GXJyS4b0C=!Ly8C4L-p5$BB!4tz`N74!Do`^LyACVg9@Z-g~|9cjEXb zxaYbFCPH?f_)G7XGCq^I@F$FPWogj568fD?e;s%?<6FS<%%7d$#z#f^6GtO>J>yNp zg+C@MhfjfDh4EbZ`91Je%>O^bzUyNXX^POB;A0=xxxBHZ^%&DH_X|B*SKpxP9|E3Z z`U}9v7+(rL%FbLEx!4C_F0F_%e08U2{7xR8EyRX>{uD)+jt@k=$r}}2y&K2POpVawB zz$=-4H+b$QUH`M-b)VAt7s30P{+GdBpV0Nc0p7{deFVIo=|2hH$N0~{2N?eic;zSc zbYB6lXZ&^WDC2K|4>Mj86n1G}ot48J@Os9N0PkjeF?eN zz^gv3+n4Wc)B5}~IzI>cT}(egT(sNF=cWEIF1rW%W6aLi!3P+B7(CDH4}xcz{xixB z)BiR2Fyp@mH$JQ9YYcpx@qdB$d`{PQhJ;M}Y0RJ5;MJ^Ns=zxLUku*Q_;T3H#~JSdcipNi z$n$OBm5kp5p8X$P{{ir!yw1N1ZhS%KKLBrFT)t0F`zI{jUqQczrTa2?AG7mk@Nvff z0d6uowy^Mz`r{qCpEJP6nEv75BaANs@5$?SP5`fB+z*~-d@Xp6@l(P37(Wwykn!`t zt8O#2BLyC1yal|M@dMyvj9&%b#rO^2=IvUt#%I8@jNb;HXZ&999OK^rA7uOyaQ7GW zbe{z8W&CI0os9nm+{N0%D@y-P-ToMOKjZ%d@3~9YcbGz^=L<}KCU_Oop9dagTz;P) zeGdlXCqjSVZarN;c;O4Q5WNO`q*vG91ny@14Dcx9=YaPzo&X?q-m-KWmQTmL3 zSn1!Z>t6?M+@tfG!5bL=yt2diUEn>8_kj;E{vB|W`STcYvA+EoTmO$if0XI}6Wsk} z-G6&T$n@Nq@fqO#-_Z3B2d`s%5qRf)x_-5?&$v(7VSEkv*w=MCo4|*d{u$u)U(xl? z0XOc~c^o{$_#W^qv(o|I#q>W6-p}-}12_8gbU&f&d`;)Kg7-50yTNN2|0;M7;}3!l zG5$DsHB0wt@DWzd&w89eiVuKy44Jkz(W5_;6$Sh_R7N1308gLg8% zh`6|KQs1{;h2*zDe~{_#1g~O#Ho}hjo^WO7dgz->|5M=IjQi2GzZ<-P@r%Jb8UGOY0OQwy*D^nE1RrPm zpCvBF=LYN#*ijshTb!P2u=0N$JOl2A{!wdWeRVOu8N8SA4&vg9x`$u@rz^mtjDJMw zKcVa21YY@FoqrB|klDEdyo%YmAAE%A_k-t{{qKPfJgletBk*3ve*r$m?7RTp&-fp} zGtAB#;C+n$2Yj5_DO)T2qkVX$KbN?u_bc#xgaqqY3w_rkQdWOY1s`YpOmGk5=PCX@ zWkH@(;0;W_1-ysx1K{of-Og3uwT#~Y-u-=D|1*j+{oBAtnf|@tLrni0;8iSNkAQm~ z)zf_vyn*Te47`)^-zfdZbUUwr4>JAN!ShW2E$~W~uac95UAn(xd=7Zm<9fPBfM=Qh zvEZXD-4)=2Og{`>#mZ+LcrW8yz)faxbvsXi*E9Z0@F8aB1@JM(N5Ngp z&R@ZMf2gPXZ}1+bUwX38qt9hLsq4$%lcs(9r*wWK^!r)4EdlRi`ZeGq%pduC6=a{; zSr7eg#tr0^!-}QpBco( zdT$Wx1WS!H!&ZPkU&->j4t$KIyM?&0qkiAVLpH>*zgGOEeR7uWmEhxyUk^UY_@}^) zXUJU9@jtafpy!W2D}ogKQmv5bK7#2#0N%j(cfs?F|A4s2m--!|DzYk$H)_RS8qZm} z{{bIocFNXC{VJwE7rc-01>l)qi0dYfrQpMiuLM`Wm#VIafV+OKh&*oqH-4$})0G{j ze>QlI@tCsD?Cb&0GyM+Y*18w_63dW$9{T+(U-y7lGy7jx_8ET|Jj3`PaZ!&h{2o+2 z()}B8@myo{SpcUzpRit>X@BfFaGZSL`C*-}2Jd8iBlrO0+rX=seffJn^!;hi>-OW& z?`M23c!t@z1iX&%9C$6W^HK0VrvFLs%3taJ+zOs${BH0(<6i|IV*EjH^*g*OmB+!| z%+IHZi+WVQ1E%`pTj1*Vz!X1zgRn@?Re!B4%d-c(_cuCU1zykeH-P6D-wLjN*H>BF z1>VQ>qu_&#Uj*LI{AnjH`pfFGY?kTo%af4@Uv#Df4M_MY@crOwSvC&)d1l{!inPCX zr?j&Y_G`fZvt0t!U(N+TUH(9Zu?YIt5*L=nnf-3?>i>mjyTfy^;?*unr*6myfKE(KL@Gi!) z;I7~4b`F5&89xX<&iM7<^}pBcd=k8x@ms-b8NVC6`xV{JSHZg&e-M0t*?A1SlkuN` zk2C%(c*Cfk?u+1+EZskW8%+Q2;QdVh9dMKBmv0j3(faKVdb)>zyBJ>p-pBY7@F?Re z!0Y~`+Yf=)|54{BgEuh!I`C1(&s26^)$N=Mo?|=#UiBAUe=oTE&pPh_uV(sJfcG>0 z5oPB!-A*@nJ>#DxF6KM+JC}V=c*HDT8}Y1hk#cyI}5>wnf`I$1 z1NsZMNPlhxKMuSO{88{y@OJPg!IhtR@E5>UIXngaFYu+XBfc+5)YmQON7djsKQ}7v_M!qh!N;-g%7H%%-ua@`S3J}x?U;AKPkA!-fRFV`kU=?o72MS> zx%wWAs;IQ%I-v7&!MkC<3U+RfN&Tvaq<#kFBRxi~q<$arHG*<@ z3cUNMO{2zt6<)lBE zKW9(DpF@A|g8fR^IXH#>eefsq58eN-gO9(Z^M}Aknf{~T1C0L=yi!dFF!xjNVWvL} zKK72D?g+St*?9##%lK>HRa7^{$3MVDofRMNfcLyzOzHcfw+Wd(SI+Fu0k8d+u0J2V zit$C@1FSrc2k&R;t^^-rJPba{%I9QogYi?rO~%gv?`8aK@L|Rew#)X}gLb8U@3E2e zCu^S{oq}&fJFiChtM>VgDfCC+PY8HRu8Q%xqRjQ}kp*YjO96ZnXHQ>XH-vHk2((Qj5e2nRTLD_fc`genyjDH2Zmf8On zc$Dcs0^Y^=55Pwlm*4McGu*74pM$<LHX6Fs?VaDH5_Dgg-j(XY7 zqm0i0&m5)e9|}Ii^cR4SGJY(0J!=ok!1Ijz!Mn=zbXS89GW`wUBaClR`pnLD@Jhzd z12-8@fOj6P`?Cjpl{&$0gU4tO`y zcbzHQ`7q;i!N(at61=uT_vcvfLB^MZ_b?s=uRcV#a}s#Np*lYmJj-}Jcr6Le z{Wy3(<9om}tUa`Y*D-!Mcn_g3#(Ti4Sbg0FKF;*-0nf1fJ^-F${2}l@ z#vcdwF#mr7?q>X1rO(E_5%4I}{{wg@<9`JoVEKIue2D2g&ywwPl<`^MX-E-v-{p z?B4_KX7%y__z>d{D*KE-2HwT&KLy^0{L0)K#?O@fWAyU;6}XG>7s0C;A651j>vqP# zYnlEy_yFU^cG*sPnH?9n!T22TI>zUN=NW$+>wFXIXVWc9ViD<2w$2~wkoN^cu%q^! zPMbo%1L<}itCz!NiZ9XmRp32L|9bEN#y3W3-}Pz|04J}{b#`Qj6VlH#P|rfd#Rq^QSiRwbUp^&#rQb*Amhdk+1@&mo5nZ600XS^0Xx=c^E4!nW!dhlMx<@cV`_ZKl9 zh5i`h8F0^XJ>4vL9pjzg{fy_pdl~NnuVUrX4L;0x4|pBpdGILXz2N@VCY+ij&sw{t9bl<^Z4 zH+6jQ@hb2f<8E*d^RpU!l<9lG ztC+qC-pP0^xQq3pI`9FeUk^Uc_<;_rD=-e^U2+}LK>Cx7Cs$3u@1KG{HwAZ|J-Pjr zQ}A=A;MYvSH(-B0i}IP zuU)0vFKv+ixYp@>w&G0xaPWG@j{=V}z7)Kd@fF|(OE(BU%JkQQcdpU>+z4LJ`02_% zE6<(EKI0dFyHD2br@*Tj-v=ILyaRlk@yo&c8NUYHU8|>i19+bCPl4Alek=G8<9C9O zFuof7F^6{BjsAEa=}*=lhhb-w)z=H)j_AKkVdsc*CohLnrr<48@Xt-bAD@D+fuFVTUyYMvQ|P})^=R0g zh9{-xcfomvafm^mgE&OA7sD8qC+l}01h1Vf!7wh{5vAXBgii74_fzoO$j{02|4Hv> zOvW##c0L)uZVKK{`I=1MsA*4iwi`9AEy-AW%%~B6Yg$u#36OB0IkvAUVFlU_?33sA zR?;Q~iZp5xE&KMRn(2CqZjZ(3oM~%sl_7aH+FJI;nq!S^@pNNrM_YSiTdcXM{XmT` znTmPSX-_Pe428Wh?~26kjv9~Ac<#1{^@DOpy2nWA5&=$9$_Wr}{8qW=S@m<|Md@lZ14j|7uc zYC$WH0hwZ;F(4`;Kq=O2ZEWmlrWTh2h4QX zXXPa%^AeJ!7m}qHlBE}tr58f!S&n4eTYaHqYfHAVskxE-PozYHFC;|`IvMcAQi)j7 z8;Hk(iG*h&7m+OWpTwS~=G|U@tgS88+TNIsHSJ9$=_+3+Zl;3qP$))CDj7*{)XG#Z z;ZUJm{+q>cDijZT62VX~5(@{?l-=T5GBrsyNRh$+X3-2LyrFo^7c)a~f6zmpvnaOc z_2{Bk7k$F%kZcxmIH1v{rY6-Q2V~b2K$bxOg%gxr z*Bg}M5S?X71Q8!bd>HXz#D|rkFm-HEs3sImD4I|-$&lu_F8cK%@du5jNc@sLO&zJ$ zy~&GW0iT*bv#l+=TVwlb;vsJ!7z}tkF={j3hzIkgmc+XXQVfA!YcP$sw8&9!8k1_A zM$lj??u`UOiDV@3L8mx!l{$OG8>6AbpH9mu&WkaV&a%KUYLYCg65}SxA9SMg1d|?L zD4g_rgNa}`MPq1jA^GJL^ZwTwzCa`$NK0^SnU&tQ|gd)LsEX1mtI#O}%2lOry z(8n735?M`IA8P_(J=Xky6-28OU&I$Qy@8l7oleqhP2-3U;|QI-AJDGp^NN3Q)9(w? z0-NUS57t1_o;Y=WT9o>e-n8FjGj&KGxI%jQhV=3c>E#=eZ93i-G()N8L`yQ|^KG=w z>pPnECR43qIkZ`lra#n2ru2e$U*s)~8l6AsPxHUyfrAtyiZ?dHk;l@~VvN1z{_NK&&D`BQX{$wB= zN~dGK0L{Qu8vzg1f;?MO9m#UJ>XD-s$yPI_GtGQb(VBDxu*NY8_*IPGntDW(HKmGx zx>`Ur-mVW1;SUIe>lCa)N!g?zS2dtL*|8}h&z^x9QWocl!M6xVFESpH4SkU@u z#qRZZ-?d~H_o!CmtEr)uS5t##w`6O*!m{Po)NHp}Ff~W0Gk#Y6646YpCLhkEQb{wG z@OvZableQ2ViT)>;fOEzER>@p z8=B$ByBuUCwyCvc-^5B^IvWpoeKa43L$PFtvQGs@Z8#{F7|Qm4yCw0^92pG8JwabQ z7z;50h6?7fVHeeqbzpGx|YiEw}xFM5eZ zrdMKde;^r%`>5KIURnXM67zbj%^<2oF_4R?g6{LIIijZK{oLevf*#Whc>F0^E5|~9 zR=Hk}wG0eeZLBbZc+*YIO>LRR-K{PAsZ04hDSy}(p(aMnBo>cbtt)~%13Fvd2A!?( zgwECt0iETbE&|rzDgxG+Dgu)wJK4nL4XN3Rb|f0J9c>wM$WJpAm3`O~j?f(FrKJuv zcPcR4f=~c=ZxoVuW)!fz6iV`bMJUO8BN37}aw1gF_sY9G!Q_3USJ|fk)F>qNDIo1r zK-w2d@>WZP)a^sT z$t>%m#}o;$-9^)9#=NO?JWM?%6bsUgP;uw=()_N=`ufnz=54QEpE11wtD)=e7SASy zMa(p}HR7}Y44JfMONSzUYWuQv6mGaEpijzPwvwPFpPnI_mqq&uh!(Hql*TQJ@OgrA z2SNm_>JheAx#sXg`f|8(sEC~uw zH_$F(dquSG%i{F~mQU`x2-`BB6ku1-Hj8kWBBVbQV2)B7Bn`g@j*5WZYJJoyp<$JS z(6EX@1b7Z)#fX@~MY3`QlI3k(X4Rw>vI@fr6|D1Y2lQ6yV|Q#mx@CidvOp|r6WJ=H zAa|!ktZY+O$SM~rRPb1?#s!-w9J%JTnAMW3P{Cu_E-Y5CLbrpV4-Y6R#OmZ2Mc))7QAIQZM2o4e5M;Vcu)6gDpbf@ zsE{TbD}2F1dZB`?aN#;OlK3V(mSc+WGhFb~EaWF#$hTRrZx%eH-;#odasU!m%n2(K z*A~2v6f8#y_Gk^Jtgu?3($qR^P``VnMF5LuwHz#X5Sj3x;GbC7AYrekkQ1+mCsHU* zFO4#qKU9ka1BIl#ypvAUs8>$9qL{oitLwJ7T&O#*+!K~pu_1}Kf{FG)8xPVs+P{Er zrWz%%CF=K>jdV|GWx;GzO@j74P(&0$)zDrl3W!1lCKnTyl|l+?^dieBUAm%RkGd67 zQ+cA;f}WfLggw>C$V%Z_)#)h~9fLvz?^UNKhWer$6+A^36uZtcR&7lCDIk5KKp{%b zav~0+DTSmT0?CpPA?b$*$^26Yr7l>(uACKuw54q!5wNZl0c-6Q4qGcEJfu(?%r)7L z_@2~(nwt7pYg?+`65h{(+q%v3#eyN9FOg1)ZLQ)47nEfejEId+J;KVWh|rrPt-3VD za9E%H!Za`x8#1X=#rMYcr2I7zTJ^_Fx-+CFfnlFFh`n-TINjQEX{xy~7H_HvhXP4& zA{B}x;`BHumV}~p*OBr>%w!@Nk9qvzU_2Z^95pCfNLfA?9{;O_M{St*q)kDn4ZHW z)AS(D^oD$?n96XIyuN(%>X?UAQd0t_O|S< ziKoJJV{gVHrk@_CdDJugcuQ+rrY1#C=INnkgdPS5BB_YFPDaLJfq?0WMG_vLN&BLv z^2fR|<_(GGkn|YAOHbQVDv~zzlBpVciy$61<0-nCjfFymDsE5Kgv_9y9`Pk2^aRsS zuNlB5-5U7Y+LQja18shKDIrX)p^==Y+bY^KG5zrX-Qm&RM}nRuQW3|RTbgUA$9U=X zKAkjEv_FtYr~Py<8BBQmp-3_kOC)LnG(=m^WvMl_w6@pyg8^?Mod|g1^z0;=Qf(#^ zYs2)aPAvaQ~P%t&&c;PvA@a)#6_EcoQ z_}eJof$-6;dxAD==pH34mj4CseaT2D7Eh+bu~axt&n=R{q@Suc?)RsBNi!X*5&g-^ z=X+|(XC`VBX!@H;L0Tfyt!hMU(8m)YJTp;t0>1a!^TGX~-0bm((k6A*m_JSX{Nc2` zzrkJ)?I(%t0$~}iioBnpE07A(Ry=Ls1XGD{(qlbBm|nqHdoMwK7h$s7`S;$3riVh_ zFzrMq%n-d7kksv)AizIw_ zBuCd;Qy^e= zpJ_UgsO)1O1k{rLaVQM zkY02vDzyJ+Z@1AFyfyFp2_gVv-KKYxcxvj)z~CnN28t-qt=!7gcII0 z75wyTMs7)3ZTh`Snw}!6mJ<+tDZDS$o@q(8(Yp?!fG6Jbqd4mUTPW3*h>?@j@H%!k z((A9Hrm0;dg8{M2OA8gRTvS-SoCb9|%khQ+)(}N!dH+ZOYuKT)TBb3C~GzpQR*sr^DKypqF;!UuFLyi zijud%6bLHE*^}vah#5DXUWP^I>w2LET zby|`6={Ig#T+TNVDGvVpmr8mx}Z&z-e(ygm}heZVBCYlJzer|=ViWEpT1PaL& zGzH{1W0{caZ(_0~Q9zD!mI>Jy1S?#M`xIfs8e1$Qve*SHTp@=k5i1`YSRq-d#N-fa zrD2UnR>&F-txzE~xrHy79GAWHk~fl7J(39WpzfWBATxODjh>0Ct0;hEFyxCk%JH;o zRc=y?tKLTwlY87&QtxAQl)L4Y^T;8&IuSx)5}STQ^O^6X0h?$z*a!2{f zehN?S*Ne;1V#p#|426&(3L%pe5*uC99Bi$3F{Y>MwYK`Fr|Xjs6ufjxfR;zMb7)_* zDJ**#ZUTi8az;uM>JtpDonp}pDO9+5d+%XUJ{a-R({*W_0`H?j>9NIhOVdwvpjKaE z6+Owu6lIp)RG_^$+CB>hV>G&pnZp#T%IT&;CZ?_H#~5Tr4uS%hRgi_pXsg;=y#ibbtPRQ*suwrUE<3n?I- zrGPX_0c4Fr(ldc3-q^A3Vm7t3?u)gzr&_(fY%ED1FlkTGd*ge(K6$g5ZB3<{I>mxF zK-+-<`iw~+6b!`!^6gW4M+Ofvtk6UvvTP_8SrsAKL`A6JD28jn3c0`lFW5vrWYH1B zG$BH=m_$f6c?!u#E|xD7_6lCf6}O0$i(V^K(8CNTtiV&6VyvcTJz@}TkOFW?&3O?u zZBRh2w!IX>h(jT$Tl->`1=2@e)Epx8ksyVT6`F(!mttZeJymd&R>6q8R}iVn8vzkg zMMw-qO`(E4%zdPXxle=&v2tZEtf<1ITuj8GVu_(_C{*xXKFJc-$|qS?2*po1D(J~a zjot{gdRf5Y{R69bCR)1cgrtO_T!aexvTOM0Z6gbbfMr4iWFd->)kiF3m6sJN=qnSG zy@Vort$57?Cg^n(Tv+^K%7h8oF2x1Z!KOP*`D9C@*Kd#u3Rn$55Sbk>y_R5=IlcCw zUV5RBRjmTa=i*kVZ~?rc*a=jyVO59FFA&)v1(NMhgk-n1LIoRgF({b4i?l*;mzdR& z1e5K?3dvF@CLh{cOcp3H`6__L3ckp;B$#}yzzWHN7nY~Lnx%KyMT@eox7wW*n*Izz zpY$qRUnnx!4=hU)xhTZaLvEx z6Jf;`@`AyLu8^27 zQVJnMs>jlmaEan%-xbD?3F?Z4D^&$ir!Ux0RYeS0EOc#Q1sN-JXkv1e}vSLI?R;38NkNY8c7br?bzD*=TXze0Y$d7zc z&x$RS8~Ot2VaA~l`mbOU*A}jjFJ1{Nvq6rx(VjzXtLb zeO@BmK_6A8Pj^MK^zkkFQi#2&U?P)B>=CygF@MbKp^q}q>(8nG)7qKrNRkv`xC8PmEqyxi}_||zS!?$r@Lw#i7;sxCeh-f>Y})GM2VMGMVKD@U`997 zZ);!@v+;qSRhgeg`--wTxsSR;m=iWxDTG+f;)<78xr-l7c_<-+TpMxbBSlaKa2Sz% zGlFng`EV^Uf^n8-gM>_Sbv;Lc%6@zgvdUF#Xt(p-n3d8zW5*1*c}v)+Qs z*v7(jan0%3$>EY-B@LJ1@ZZGYvl54IEphlnIWrUjTic3{1tx7x$3#n3GyCyAIP?<^ zO$UdU7g%P7FszqqV4Kd$CksR8)lW8xFbptKacDOZFYQ|1 zvQs%;v9bF6@zZB7R!`r5|M`0_R?j~D_&NL(YoIbKI62p{bDkG!5+oVSdHUKFA79@- zZ(l9mnD?7WgLB7nfCObS zK5Tq*C;p6l+>)(AIn(%=nlW5dSNn>|3-WlUjW>mhrH(g#hR!!g!)8XmB1lDl{sazsdMJ#cD=^Glc4z#ZMJ zI-mk>FRr%dO;WXK3V=@mYEm^edYDN?y~#+EEhc?z9nl0S6Bl9C=R9GQ1Z z7jq@wyL5(0FE{FRn-)5-dG-NnOL*yzmb02I(LXI<GA zEKQwP^KO~y+6`4@-*slpyJewkPrO@BIuB;(TB&t4i_-Z%fxo4rYe!UiD5Ux8emsQE zg$hMbBP_mzis$5?R#2j=FMQI~V+j*Jsp^kUC;+EPELJEARutx=vj|Biw6%6^@v-tG zp@KN1WU+KvjWD6Eu!FEqx|+@>wYtNhlXV=}sQT9B+@m>kX-__d=F>S3KB}SP2%Sw% z-f8qup`zrN&-^K55h|M#i54a_7kl^Sqf2;XF?Tmz-VvV=ORe|t(Ulu<_*3j0ggs>U zIlG&$^7Tb`S#pYR-CfxOrLzE;3CoMdv8XUKm7+jMTA=`8#L0HD7D?Aw2@?iO68hO( zqA(b=8fNzBi#IbuUl16Y`RZWc-FE<6;PnCYIAAPEPK|Nj~k*Goij0keBaO zJpFiJf*42(-+d)5qz2NpKJYi*YbCJ~gmT+c`xRMRX3FY#x5$B!&kWp%2#j6_0}S56 z+a9hRPece=y0(>R%maTTM;nb+v2Z#7dACB^Hs(rO8-JtQwlZxajLM0Yh4B`+l`e0>w2egtb{0nAJP56F12fTf zL?}d8b3)gO9yn!e(1ePLOjQ_sbkKNyK+D9Y%3z5O`T#CtjKJ829c9opoLO5mc4(ta zd(*5#22M0{CL5$-aL>%tq6ZE%GZ|wZcC2L98UwSLnTjz3?c6XIOHXAwv>HdGGso6b zknr=^jgDvo&>kj7<8@OQRcu!AJPfoDO+=xG=@;vW#bokb1K1Ihlv6m!lPhyKU@`4n zU$e7%#1KXCcaCN7;pU6+=JEKY2edhvLn`pR-B9IenRIe+|F9Uv=^Q@2zJ7ZSRg2{i z_RS;EyMUCMRjL9KVfz&cw=?z*jKr~R8%(LH6dcb(I(>B?8l$|Wy31TlRpdmUtfqo# zUJWg1Y@HY~?3zjis~@#{jj!yz^s}z2XwR&~kO~O3V^&3{(?4F-d+CGlo^^GdWmQ`b zQ-NXi(JE3~sbgh0+9N%Ew@B&~EZf6pJRqUc7Jxn0p8H)67RY_v{kusRq>+dx>@$Tt8s3v|Fi5B;W$Q zozv~z^%`68i*Y?9JOtp3u2tuG%lcW;1IgWGm5wsZe-_Yk@o;L7UGIX5J(#cXsTb_c7j*~2sE=;{@xSgTi!18)~lS>}X8=aKcik^QOS zVs9^IT?}BLwY9wIQeKPz0dnXtOH2Ez$hEvDfx6ka-t^;k3oV)+_K;9CwOu~|f`AER z!$t>*7>K}1h)VK(M2%7bHA;oj$P{a!E&)oY3u+~s;GYWdco>rn^6Lq30*3re?RU=A zu+EJWU=?Xr^Z+=p*y?NZFtX1`ND*`cgbd)(TWC*!IIRZk@$(M+f?{{9Q^dO8Tzw9v z{I&rd<8pNeqPoNZaBeKd3OLCUV{RQ?hxq}I*Y}&dt6M5-+9b&%eUomIG|09U$OvE` ztS;|vbz=)0y8=VTZmO$cav0tTEvt#*-rije7sJF`JAfdPon3Ff;N|5lw#Gcy`n+G$ zm0_5GJR-*oR^AkmPIY&AEtafOe1IodMNW&Bf{pN{rRhX)-+`vU=G*PoH>5{f*y*3O zj)9Is4%<~Y$eVb&6 zd^Y@cK>eo{8~~R)0Peb?JW=^>(I1jz5U|^ecFolVlIKwNMLux@11o+woRvweVVvu>2COfrX$X)#uDB5H5LJ7i~7S@Cv!yVpnxivJCV0 zMn1h7Oi#tO=<;9Ua5YmeWiK@RtW%! z0ymq~4wHhVTQ6fkT?STCB;yF+a)rruhxh`3g67?YWtm3|ShifWF>+H0(g`$kOt%Zd{R#byfg&^46H; zEYYWgwP5fnAv7Pb?bwgHqZb?zl-VLE`<(Gux}@TgjsnhcDwzx`6B!1=!yB3jHF&5K zt07g2=(wl)-@R4>$QPg=SY6&0nC$Lo{l#{D(T@ewq4CQ!_b)4sLooF5Sgk>Fl6z1f zC99wU%dLU6A5dzK=z?EIAluU&~ZVsqu1&1J7 zLt^)^H!_$@D}*rIK>WbysjC9pem9j>r>&=BZ-%F>uwHL_E>T)-kktrG0jbTIjP>Op zcJ@V+wsG0@h<2$B@KSTT zpnh_TLfe9AR+pf_3VFlD=SEd|97)Y!$ll0-?~QeI_#mijmbb5 z(oWY+N3fegZOOW%ZC#>@=ZWOsE&&~i?Q^P?iR%a90Vf~k8l zFk~%*=BWIR%ivxa(h9J;g1x;KeVOT@FN59xA^;hunDIefv$Vrr8rOxjAPRTYjs(~n zGseZXw=Gvu-DAc;*{4joQefK}^txRZNcH&ZxK1xE9G#e7>@6*11sXL6-{ zd3*XtXXg9qPgyzi3`3b{)ALji?htjS_{B!ZRJXZ1DVkaHiY)X-DLdyK=qR{QmBmq`-6I8o5%?d};^6jW)W-nvm3T z3Oh+JwyXI}ws1}XNCySqH9c2x7|#)kRr1t+MG{S!pc4RjEVl4laLJ2Jpbf=aVQxfX zoS(nrXL}DpUrQJmj;@=58kzL9j&068FG~@AK+P~cr@HaI@h}!J?wUms_Rm38(~it} z+=GOJ1}N(r!=s_Sra@2SaU3ZMv>S|RLD=ISFGQ*8F8`l=4zQxw=$5+_=$Tv(=f(pS zXI3n0mG$GIS2hhIjA>+*aqC$xfNVj~?~CRG<6*iN7eNi^F_g_^z1&tbM?{q={KTJK z_svR|0EErRbbCOOxeCbJDH$Dzkvq}C@Oy7l2eXsi{%!y{YbSY8u$n^! zYvo5>SM(ET2`+O#E#V*W|W$a01cFxpY$2E1IB`)=1q z+uPaMcW_AvyQcT-6T9~}Jpc2 Date: Thu, 7 Mar 2024 17:56:40 +0000 Subject: [PATCH 38/71] adds api to obtain the parent node in the turbine retransmit tree (#115) Following commits will use this api to check retransmitter's signature on incoming shreds. --- Cargo.lock | 1 + turbine/Cargo.toml | 1 + turbine/src/cluster_nodes.rs | 199 +++++++++++++++++++++++++++++++---- 3 files changed, 179 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1ed3911a83bb23..7fa4b07b4346f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7416,6 +7416,7 @@ dependencies = [ "solana-runtime", "solana-sdk", "solana-streamer", + "test-case", "thiserror", "tokio", ] diff --git a/turbine/Cargo.toml b/turbine/Cargo.toml index bedd870952af99..2d2b0a79574d27 100644 --- a/turbine/Cargo.toml +++ b/turbine/Cargo.toml @@ -43,6 +43,7 @@ tokio = { workspace = true } assert_matches = { workspace = true } solana-logger = { workspace = true } solana-runtime = { workspace = true, features = ["dev-context-only-utils"] } +test-case = { workspace = true } [[bench]] name = "cluster_info" diff --git a/turbine/src/cluster_nodes.rs b/turbine/src/cluster_nodes.rs index 0c55cb41e56472..6036907cd7dc5c 100644 --- a/turbine/src/cluster_nodes.rs +++ b/turbine/src/cluster_nodes.rs @@ -152,8 +152,7 @@ impl ClusterNodes { } pub(crate) fn get_broadcast_peer(&self, shred: &ShredId) -> Option<&ContactInfo> { - let shred_seed = shred.seed(&self.pubkey); - let mut rng = ChaChaRng::from_seed(shred_seed); + let mut rng = get_seeded_rng(/*leader:*/ &self.pubkey, shred); let index = self.weighted_shuffle.first(&mut rng)?; self.nodes[index].contact_info() } @@ -187,7 +186,6 @@ impl ClusterNodes { shred: &ShredId, fanout: usize, ) -> Result { - let shred_seed = shred.seed(slot_leader); let mut weighted_shuffle = self.weighted_shuffle.clone(); // Exclude slot leader from list of nodes. if slot_leader == &self.pubkey { @@ -200,7 +198,7 @@ impl ClusterNodes { weighted_shuffle.remove_index(*index); } let mut addrs = HashMap::::with_capacity(self.nodes.len()); - let mut rng = ChaChaRng::from_seed(shred_seed); + let mut rng = get_seeded_rng(slot_leader, shred); let protocol = get_broadcast_protocol(shred); let nodes: Vec<_> = weighted_shuffle .shuffle(&mut rng) @@ -233,6 +231,43 @@ impl ClusterNodes { addrs, }) } + + // Returns the parent node in the turbine broadcast tree. + // Returns None if the node is the root of the tree or if it is not staked. + #[allow(unused)] + fn get_retransmit_parent( + &self, + leader: &Pubkey, + shred: &ShredId, + fanout: usize, + ) -> Result, Error> { + // Exclude slot leader from list of nodes. + if leader == &self.pubkey { + return Err(Error::Loopback { + leader: *leader, + shred: *shred, + }); + } + // Unstaked nodes' position in the turbine tree is not deterministic + // and depends on gossip propagation of contact-infos. Therefore, if + // this node is not staked return None. + if self.nodes[self.index[&self.pubkey]].stake == 0 { + return Ok(None); + } + let mut weighted_shuffle = self.weighted_shuffle.clone(); + if let Some(index) = self.index.get(leader).copied() { + weighted_shuffle.remove_index(index); + } + let mut rng = get_seeded_rng(leader, shred); + // Only need shuffled nodes until this node itself. + let nodes: Vec<_> = weighted_shuffle + .shuffle(&mut rng) + .map(|index| &self.nodes[index]) + .take_while(|node| node.pubkey() != self.pubkey) + .collect(); + let parent = get_retransmit_parent(fanout, nodes.len(), &nodes); + Ok(parent.map(Node::pubkey)) + } } pub fn new_cluster_nodes( @@ -296,6 +331,11 @@ fn get_nodes(cluster_info: &ClusterInfo, stakes: &HashMap) -> Vec ChaChaRng { + let seed = shred.seed(leader); + ChaChaRng::from_seed(seed) +} + // root : [0] // 1st layer: [1, 2, ..., fanout] // 2nd layer: [[fanout + 1, ..., fanout * 2], @@ -327,6 +367,21 @@ fn get_retransmit_peers( .copied() } +// Returns the parent node in the turbine broadcast tree. +// Returns None if the node is the root of the tree. +fn get_retransmit_parent( + fanout: usize, + index: usize, // Local node's index within the nodes slice. + nodes: &[T], +) -> Option { + // Node's index within its neighborhood. + let offset = index.saturating_sub(1) % fanout; + let index = index.checked_sub(1)? / fanout; + let index = index - index.saturating_sub(1) % fanout; + let index = if index == 0 { index } else { index + offset }; + nodes.get(index).copied() +} + impl ClusterNodesCache { pub fn new( // Capacity of underlying LRU-cache in terms of number of epochs. @@ -516,7 +571,11 @@ pub fn check_feature_activation(feature: &Pubkey, shred_slot: Slot, root_bank: & #[cfg(test)] mod tests { - use super::*; + use { + super::*, + std::{fmt::Debug, hash::Hash}, + test_case::test_case, + }; #[test] fn test_cluster_nodes_retransmit() { @@ -589,10 +648,42 @@ mod tests { } } + // Checks (1) computed retransmit children against expected children and + // (2) computed parent of each child against the expected parent. + fn check_retransmit_nodes(fanout: usize, nodes: &[T], peers: Vec>) + where + T: Copy + Eq + PartialEq + Debug + Hash, + { + // Map node identities to their index within the shuffled tree. + let index: HashMap<_, _> = nodes + .iter() + .copied() + .enumerate() + .map(|(k, node)| (node, k)) + .collect(); + let offset = peers.len(); + // Root node's parent is None. + assert_eq!(get_retransmit_parent(fanout, /*index:*/ 0, nodes), None); + for (k, peers) in peers.into_iter().enumerate() { + assert_eq!( + get_retransmit_peers(fanout, k, nodes).collect::>(), + peers + ); + let parent = Some(nodes[k]); + for peer in peers { + assert_eq!(get_retransmit_parent(fanout, index[&peer], nodes), parent); + } + } + // Remaining nodes have no children. + for k in offset..=nodes.len() { + assert_eq!(get_retransmit_peers(fanout, k, nodes).next(), None); + } + } + #[test] - fn test_get_retransmit_peers() { + fn test_get_retransmit_nodes() { // fanout 2 - let index = vec![ + let nodes = [ 7, // root 6, 10, // 1st layer // 2nd layer @@ -620,16 +711,9 @@ mod tests { vec![16, 9], vec![8], ]; - for (k, peers) in peers.into_iter().enumerate() { - let retransmit_peers = get_retransmit_peers(/*fanout:*/ 2, k, &index); - assert_eq!(retransmit_peers.collect::>(), peers); - } - for k in 10..=index.len() { - let mut retransmit_peers = get_retransmit_peers(/*fanout:*/ 2, k, &index); - assert_eq!(retransmit_peers.next(), None); - } + check_retransmit_nodes(/*fanout:*/ 2, &nodes, peers); // fanout 3 - let index = vec![ + let nodes = [ 19, // root 14, 15, 28, // 1st layer // 2nd layer @@ -661,13 +745,84 @@ mod tests { vec![24, 32], vec![34], ]; - for (k, peers) in peers.into_iter().enumerate() { - let retransmit_peers = get_retransmit_peers(/*fanout:*/ 3, k, &index); - assert_eq!(retransmit_peers.collect::>(), peers); + check_retransmit_nodes(/*fanout:*/ 3, &nodes, peers); + let nodes = [ + 5, // root + 34, 52, 8, // 1st layer + // 2nd layar + 44, 18, 2, // 1st neigborhood + 42, 47, 46, // 2nd + 11, 26, 28, // 3rd + // 3rd layer + 53, 23, 37, // 1st neighborhood + 40, 13, 7, // 2nd + 50, 35, 22, // 3rd + 3, 27, 31, // 4th + 10, 48, 15, // 5th + 19, 6, 30, // 6th + 36, 45, 1, // 7th + 38, 12, 17, // 8th + 4, 32, 16, // 9th + // 4th layer + 41, 49, 24, // 1st neighborhood + 14, 9, 0, // 2nd + 29, 21, 39, // 3rd + 43, 51, 33, // 4th + 25, 20, // 5th + ]; + let peers = vec![ + vec![34, 52, 8], + vec![44, 42, 11], + vec![18, 47, 26], + vec![2, 46, 28], + vec![53, 40, 50], + vec![23, 13, 35], + vec![37, 7, 22], + vec![3, 10, 19], + vec![27, 48, 6], + vec![31, 15, 30], + vec![36, 38, 4], + vec![45, 12, 32], + vec![1, 17, 16], + vec![41, 14, 29], + vec![49, 9, 21], + vec![24, 0, 39], + vec![43, 25], + vec![51, 20], + vec![33], + ]; + check_retransmit_nodes(/*fanout:*/ 3, &nodes, peers); + } + + #[test_case(2, 1_347)] + #[test_case(3, 1_359)] + #[test_case(4, 4_296)] + #[test_case(5, 3_925)] + #[test_case(6, 8_778)] + #[test_case(7, 9_879)] + fn test_get_retransmit_nodes_round_trip(fanout: usize, size: usize) { + let mut rng = rand::thread_rng(); + let mut nodes: Vec<_> = (0..size).collect(); + nodes.shuffle(&mut rng); + // Map node identities to their index within the shuffled tree. + let index: HashMap<_, _> = nodes + .iter() + .copied() + .enumerate() + .map(|(k, node)| (node, k)) + .collect(); + // Root node's parent is None. + assert_eq!(get_retransmit_parent(fanout, /*index:*/ 0, &nodes), None); + for k in 1..size { + let parent = get_retransmit_parent(fanout, k, &nodes).unwrap(); + let mut peers = get_retransmit_peers(fanout, index[&parent], &nodes); + assert_eq!(peers.find(|&peer| peer == nodes[k]), Some(nodes[k])); } - for k in 13..=index.len() { - let mut retransmit_peers = get_retransmit_peers(/*fanout:*/ 3, k, &index); - assert_eq!(retransmit_peers.next(), None); + for k in 0..size { + let parent = Some(nodes[k]); + for peer in get_retransmit_peers(fanout, k, &nodes) { + assert_eq!(get_retransmit_parent(fanout, index[&peer], &nodes), parent); + } } } } From 264f4dfdd0662810762db066f3c461f90c9972a8 Mon Sep 17 00:00:00 2001 From: Dmitri Makarov Date: Thu, 7 Mar 2024 13:16:16 -0500 Subject: [PATCH 39/71] [SVM] Move RuntimeConfig to program-runtime (#96) RuntimeConfig doesn't use anything SVM specific and logically belongs in program runtime rather than SVM. This change moves the definition of RuntimeConfig struct from the SVM crate to program-runtime and adjusts `use` statements accordingly. --- Cargo.lock | 4 +--- core/src/validator.rs | 2 +- core/tests/epoch_accounts_hash.rs | 2 +- core/tests/snapshots.rs | 2 +- ledger-tool/Cargo.toml | 1 - ledger-tool/src/args.rs | 2 +- ledger/src/blockstore_processor.rs | 12 ++++++------ program-runtime/src/lib.rs | 1 + program-runtime/src/runtime_config.rs | 17 +++++++++++++++++ program-test/src/lib.rs | 4 ++-- programs/sbf/Cargo.lock | 3 +-- runtime/src/bank.rs | 2 +- runtime/src/bank/serde_snapshot.rs | 2 +- runtime/src/serde_snapshot.rs | 2 +- runtime/src/snapshot_bank_utils.rs | 2 +- svm/src/lib.rs | 1 - svm/src/runtime_config.rs | 9 --------- svm/src/transaction_processor.rs | 2 +- test-validator/Cargo.toml | 1 - test-validator/src/lib.rs | 3 +-- validator/Cargo.toml | 2 +- validator/src/main.rs | 2 +- 22 files changed, 40 insertions(+), 38 deletions(-) create mode 100644 program-runtime/src/runtime_config.rs delete mode 100644 svm/src/runtime_config.rs diff --git a/Cargo.lock b/Cargo.lock index 7fa4b07b4346f0..79fcfb624d1b8f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6290,7 +6290,6 @@ dependencies = [ "solana-stake-program", "solana-storage-bigtable", "solana-streamer", - "solana-svm", "solana-transaction-status", "solana-unified-scheduler-pool", "solana-version", @@ -7260,7 +7259,6 @@ dependencies = [ "solana-runtime", "solana-sdk", "solana-streamer", - "solana-svm", "solana-tpu-client", "tokio", ] @@ -7511,6 +7509,7 @@ dependencies = [ "solana-net-utils", "solana-perf", "solana-poh", + "solana-program-runtime", "solana-rpc", "solana-rpc-client", "solana-rpc-client-api", @@ -7519,7 +7518,6 @@ dependencies = [ "solana-send-transaction-service", "solana-storage-bigtable", "solana-streamer", - "solana-svm", "solana-test-validator", "solana-tpu-client", "solana-unified-scheduler-pool", diff --git a/core/src/validator.rs b/core/src/validator.rs index 196dad5f25d17a..a1c8293f86cb3a 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -74,6 +74,7 @@ use { poh_recorder::PohRecorder, poh_service::{self, PohService}, }, + solana_program_runtime::runtime_config::RuntimeConfig, solana_rpc::{ max_slots::MaxSlots, optimistically_confirmed_bank_tracker::{ @@ -116,7 +117,6 @@ use { }, solana_send_transaction_service::send_transaction_service, solana_streamer::{socket::SocketAddrSpace, streamer::StakedNodes}, - solana_svm::runtime_config::RuntimeConfig, solana_turbine::{self, broadcast_stage::BroadcastStageType}, solana_unified_scheduler_pool::DefaultSchedulerPool, solana_vote_program::vote_state, diff --git a/core/tests/epoch_accounts_hash.rs b/core/tests/epoch_accounts_hash.rs index 62e31f0a88b766..25e97689923bb0 100755 --- a/core/tests/epoch_accounts_hash.rs +++ b/core/tests/epoch_accounts_hash.rs @@ -16,6 +16,7 @@ use { snapshot_packager_service::SnapshotPackagerService, }, solana_gossip::{cluster_info::ClusterInfo, contact_info::ContactInfo}, + solana_program_runtime::runtime_config::RuntimeConfig, solana_runtime::{ accounts_background_service::{ AbsRequestHandlers, AbsRequestSender, AccountsBackgroundService, DroppedSlotsReceiver, @@ -39,7 +40,6 @@ use { timing::timestamp, }, solana_streamer::socket::SocketAddrSpace, - solana_svm::runtime_config::RuntimeConfig, std::{ mem::ManuallyDrop, sync::{ diff --git a/core/tests/snapshots.rs b/core/tests/snapshots.rs index e67c942f07ab0b..1607ebd3fa2094 100644 --- a/core/tests/snapshots.rs +++ b/core/tests/snapshots.rs @@ -18,6 +18,7 @@ use { snapshot_packager_service::SnapshotPackagerService, }, solana_gossip::{cluster_info::ClusterInfo, contact_info::ContactInfo}, + solana_program_runtime::runtime_config::RuntimeConfig, solana_runtime::{ accounts_background_service::{ AbsRequestHandlers, AbsRequestSender, AccountsBackgroundService, @@ -50,7 +51,6 @@ use { timing::timestamp, }, solana_streamer::socket::SocketAddrSpace, - solana_svm::runtime_config::RuntimeConfig, std::{ collections::HashSet, fs, diff --git a/ledger-tool/Cargo.toml b/ledger-tool/Cargo.toml index 6da42940a4ba7f..ddc1ca9b564e94 100644 --- a/ledger-tool/Cargo.toml +++ b/ledger-tool/Cargo.toml @@ -44,7 +44,6 @@ solana-sdk = { workspace = true } solana-stake-program = { workspace = true } solana-storage-bigtable = { workspace = true } solana-streamer = { workspace = true } -solana-svm = { workspace = true } solana-transaction-status = { workspace = true } solana-unified-scheduler-pool = { workspace = true } solana-version = { workspace = true } diff --git a/ledger-tool/src/args.rs b/ledger-tool/src/args.rs index 80ea6f9715bf35..1f0c06966deffc 100644 --- a/ledger-tool/src/args.rs +++ b/ledger-tool/src/args.rs @@ -12,8 +12,8 @@ use { blockstore_processor::ProcessOptions, use_snapshot_archives_at_startup::{self, UseSnapshotArchivesAtStartup}, }, + solana_program_runtime::runtime_config::RuntimeConfig, solana_sdk::clock::Slot, - solana_svm::runtime_config::RuntimeConfig, std::{ collections::HashSet, path::{Path, PathBuf}, diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs index 2e172870d6e5f7..c999eab1a56fd4 100644 --- a/ledger/src/blockstore_processor.rs +++ b/ledger/src/blockstore_processor.rs @@ -27,7 +27,10 @@ use { }, solana_measure::{measure, measure::Measure}, solana_metrics::datapoint_error, - solana_program_runtime::timings::{ExecuteTimingType, ExecuteTimings, ThreadExecuteTimings}, + solana_program_runtime::{ + runtime_config::RuntimeConfig, + timings::{ExecuteTimingType, ExecuteTimings, ThreadExecuteTimings}, + }, solana_rayon_threadlimit::{get_max_thread_count, get_thread_count}, solana_runtime::{ accounts_background_service::{AbsRequestSender, SnapshotRequestKind}, @@ -54,11 +57,8 @@ use { VersionedTransaction, }, }, - solana_svm::{ - runtime_config::RuntimeConfig, - transaction_results::{ - TransactionExecutionDetails, TransactionExecutionResult, TransactionResults, - }, + solana_svm::transaction_results::{ + TransactionExecutionDetails, TransactionExecutionResult, TransactionResults, }, solana_transaction_status::token_balances::TransactionTokenBalancesSet, solana_vote::{vote_account::VoteAccountsHashMap, vote_sender_types::ReplayVoteSender}, diff --git a/program-runtime/src/lib.rs b/program-runtime/src/lib.rs index 5797626a00a756..079f214fa236f0 100644 --- a/program-runtime/src/lib.rs +++ b/program-runtime/src/lib.rs @@ -17,6 +17,7 @@ pub mod loaded_programs; pub mod log_collector; pub mod message_processor; pub mod prioritization_fee; +pub mod runtime_config; pub mod stable_log; pub mod sysvar_cache; pub mod timings; diff --git a/program-runtime/src/runtime_config.rs b/program-runtime/src/runtime_config.rs new file mode 100644 index 00000000000000..da6fc1dfba4db1 --- /dev/null +++ b/program-runtime/src/runtime_config.rs @@ -0,0 +1,17 @@ +use crate::compute_budget::ComputeBudget; + +#[cfg(RUSTC_WITH_SPECIALIZATION)] +impl ::solana_frozen_abi::abi_example::AbiExample for RuntimeConfig { + fn example() -> Self { + // RuntimeConfig is not Serialize so just rely on Default. + RuntimeConfig::default() + } +} + +/// Encapsulates flags that can be used to tweak the runtime behavior. +#[derive(Debug, Default, Clone)] +pub struct RuntimeConfig { + pub compute_budget: Option, + pub log_messages_bytes_limit: Option, + pub transaction_account_lock_limit: Option, +} diff --git a/program-test/src/lib.rs b/program-test/src/lib.rs index 20b9f5806e29c3..669cb15a595afb 100644 --- a/program-test/src/lib.rs +++ b/program-test/src/lib.rs @@ -17,7 +17,8 @@ use { solana_bpf_loader_program::serialization::serialize_parameters, solana_program_runtime::{ compute_budget::ComputeBudget, ic_msg, invoke_context::BuiltinFunctionWithContext, - loaded_programs::LoadedProgram, stable_log, timings::ExecuteTimings, + loaded_programs::LoadedProgram, runtime_config::RuntimeConfig, stable_log, + timings::ExecuteTimings, }, solana_runtime::{ accounts_background_service::{AbsRequestSender, SnapshotRequestKind}, @@ -45,7 +46,6 @@ use { stable_layout::stable_instruction::StableInstruction, sysvar::{Sysvar, SysvarId}, }, - solana_svm::runtime_config::RuntimeConfig, solana_vote_program::vote_state::{self, VoteState, VoteStateVersions}, std::{ cell::RefCell, diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index a31542e3713a6b..3b2bc66e262336 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -117,6 +117,7 @@ dependencies = [ "solana-net-utils", "solana-perf", "solana-poh", + "solana-program-runtime", "solana-rpc", "solana-rpc-client", "solana-rpc-client-api", @@ -125,7 +126,6 @@ dependencies = [ "solana-send-transaction-service", "solana-storage-bigtable", "solana-streamer", - "solana-svm", "solana-test-validator", "solana-tpu-client", "solana-unified-scheduler-pool", @@ -6403,7 +6403,6 @@ dependencies = [ "solana-runtime", "solana-sdk", "solana-streamer", - "solana-svm", "solana-tpu-client", "tokio", ] diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index 39df91c382feff..ee04f20787cb9a 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -99,6 +99,7 @@ use { compute_budget_processor::process_compute_budget_instructions, invoke_context::BuiltinFunctionWithContext, loaded_programs::{LoadedProgram, LoadedProgramType, LoadedPrograms}, + runtime_config::RuntimeConfig, timings::{ExecuteTimingType, ExecuteTimings}, }, solana_sdk::{ @@ -163,7 +164,6 @@ use { solana_svm::{ account_loader::{TransactionCheckResult, TransactionLoadResult}, account_overrides::AccountOverrides, - runtime_config::RuntimeConfig, transaction_error_metrics::TransactionErrorMetrics, transaction_processor::{ TransactionBatchProcessor, TransactionLogMessages, TransactionProcessingCallback, diff --git a/runtime/src/bank/serde_snapshot.rs b/runtime/src/bank/serde_snapshot.rs index 8b78efbcf3e11a..f5b1653e8d6311 100644 --- a/runtime/src/bank/serde_snapshot.rs +++ b/runtime/src/bank/serde_snapshot.rs @@ -31,6 +31,7 @@ mod tests { epoch_accounts_hash::EpochAccountsHash, stake_rewards::StakeReward, }, + solana_program_runtime::runtime_config::RuntimeConfig, solana_sdk::{ epoch_schedule::EpochSchedule, genesis_config::create_genesis_config, @@ -38,7 +39,6 @@ mod tests { pubkey::Pubkey, signature::{Keypair, Signer}, }, - solana_svm::runtime_config::RuntimeConfig, std::{ io::{Cursor, Read, Write}, num::NonZeroUsize, diff --git a/runtime/src/serde_snapshot.rs b/runtime/src/serde_snapshot.rs index 4b066976d49048..8e678044e23670 100644 --- a/runtime/src/serde_snapshot.rs +++ b/runtime/src/serde_snapshot.rs @@ -27,6 +27,7 @@ use { epoch_accounts_hash::EpochAccountsHash, }, solana_measure::measure::Measure, + solana_program_runtime::runtime_config::RuntimeConfig, solana_sdk::{ clock::{Epoch, Slot, UnixTimestamp}, deserialize_utils::default_on_eof, @@ -39,7 +40,6 @@ use { pubkey::Pubkey, rent_collector::RentCollector, }, - solana_svm::runtime_config::RuntimeConfig, std::{ collections::{HashMap, HashSet}, io::{self, BufReader, BufWriter, Read, Write}, diff --git a/runtime/src/snapshot_bank_utils.rs b/runtime/src/snapshot_bank_utils.rs index 721021142f9258..ab3a76fc80945a 100644 --- a/runtime/src/snapshot_bank_utils.rs +++ b/runtime/src/snapshot_bank_utils.rs @@ -37,6 +37,7 @@ use { utils::delete_contents_of_path, }, solana_measure::{measure, measure::Measure}, + solana_program_runtime::runtime_config::RuntimeConfig, solana_sdk::{ clock::Slot, feature_set, @@ -45,7 +46,6 @@ use { pubkey::Pubkey, slot_history::{Check, SlotHistory}, }, - solana_svm::runtime_config::RuntimeConfig, std::{ collections::HashSet, fs, diff --git a/svm/src/lib.rs b/svm/src/lib.rs index 5505e34bea9d61..d0f679a15c448b 100644 --- a/svm/src/lib.rs +++ b/svm/src/lib.rs @@ -4,7 +4,6 @@ pub mod account_loader; pub mod account_overrides; pub mod account_rent_state; -pub mod runtime_config; pub mod transaction_account_state_info; pub mod transaction_error_metrics; pub mod transaction_processor; diff --git a/svm/src/runtime_config.rs b/svm/src/runtime_config.rs deleted file mode 100644 index 2439dd85c2e46f..00000000000000 --- a/svm/src/runtime_config.rs +++ /dev/null @@ -1,9 +0,0 @@ -use solana_program_runtime::compute_budget::ComputeBudget; - -/// Encapsulates flags that can be used to tweak the runtime behavior. -#[derive(AbiExample, Debug, Default, Clone)] -pub struct RuntimeConfig { - pub compute_budget: Option, - pub log_messages_bytes_limit: Option, - pub transaction_account_lock_limit: Option, -} diff --git a/svm/src/transaction_processor.rs b/svm/src/transaction_processor.rs index fa417850699372..e44b426df96b0d 100644 --- a/svm/src/transaction_processor.rs +++ b/svm/src/transaction_processor.rs @@ -4,7 +4,6 @@ use { load_accounts, LoadedTransaction, TransactionCheckResult, TransactionLoadResult, }, account_overrides::AccountOverrides, - runtime_config::RuntimeConfig, transaction_account_state_info::TransactionAccountStateInfo, transaction_error_metrics::TransactionErrorMetrics, transaction_results::{ @@ -23,6 +22,7 @@ use { }, log_collector::LogCollector, message_processor::MessageProcessor, + runtime_config::RuntimeConfig, sysvar_cache::SysvarCache, timings::{ExecuteDetailsTimings, ExecuteTimingType, ExecuteTimings}, }, diff --git a/test-validator/Cargo.toml b/test-validator/Cargo.toml index 2bc8deb5fc200e..60f299d01e58a0 100644 --- a/test-validator/Cargo.toml +++ b/test-validator/Cargo.toml @@ -32,7 +32,6 @@ solana-rpc-client = { workspace = true } solana-runtime = { workspace = true } solana-sdk = { workspace = true } solana-streamer = { workspace = true } -solana-svm = { workspace = true } solana-tpu-client = { workspace = true } tokio = { workspace = true, features = ["full"] } diff --git a/test-validator/src/lib.rs b/test-validator/src/lib.rs index c658b53305bf74..f551cb97820d06 100644 --- a/test-validator/src/lib.rs +++ b/test-validator/src/lib.rs @@ -29,7 +29,7 @@ use { create_new_tmp_ledger, }, solana_net_utils::PortRange, - solana_program_runtime::compute_budget::ComputeBudget, + solana_program_runtime::{compute_budget::ComputeBudget, runtime_config::RuntimeConfig}, solana_rpc::{rpc::JsonRpcConfig, rpc_pubsub_service::PubSubConfig}, solana_rpc_client::{nonblocking, rpc_client::RpcClient}, solana_runtime::{ @@ -54,7 +54,6 @@ use { signature::{read_keypair_file, write_keypair_file, Keypair, Signer}, }, solana_streamer::socket::SocketAddrSpace, - solana_svm::runtime_config::RuntimeConfig, solana_tpu_client::tpu_client::{ DEFAULT_TPU_CONNECTION_POOL_SIZE, DEFAULT_TPU_ENABLE_UDP, DEFAULT_TPU_USE_QUIC, }, diff --git a/validator/Cargo.toml b/validator/Cargo.toml index 94e80ffaf90704..dd8246fafd3c6e 100644 --- a/validator/Cargo.toml +++ b/validator/Cargo.toml @@ -49,6 +49,7 @@ solana-metrics = { workspace = true } solana-net-utils = { workspace = true } solana-perf = { workspace = true } solana-poh = { workspace = true } +solana-program-runtime = { workspace = true } solana-rpc = { workspace = true } solana-rpc-client = { workspace = true } solana-rpc-client-api = { workspace = true } @@ -57,7 +58,6 @@ solana-sdk = { workspace = true } solana-send-transaction-service = { workspace = true } solana-storage-bigtable = { workspace = true } solana-streamer = { workspace = true } -solana-svm = { workspace = true } solana-test-validator = { workspace = true } solana-tpu-client = { workspace = true } solana-unified-scheduler-pool = { workspace = true } diff --git a/validator/src/main.rs b/validator/src/main.rs index ec70796130e7d2..ad89bc4886d4ed 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -38,6 +38,7 @@ use { }, solana_perf::recycler::enable_recycler_warming, solana_poh::poh_service, + solana_program_runtime::runtime_config::RuntimeConfig, solana_rpc::{ rpc::{JsonRpcConfig, RpcBigtableConfig}, rpc_pubsub_service::PubSubConfig, @@ -58,7 +59,6 @@ use { }, solana_send_transaction_service::send_transaction_service, solana_streamer::socket::SocketAddrSpace, - solana_svm::runtime_config::RuntimeConfig, solana_tpu_client::tpu_client::DEFAULT_TPU_ENABLE_UDP, solana_validator::{ admin_rpc_service, From ca1af7b7022c2da53c5e89579c736c15c8174497 Mon Sep 17 00:00:00 2001 From: kirill lykov Date: Thu, 7 Mar 2024 12:51:44 -0800 Subject: [PATCH 40/71] Add get_blocks and get_slot methods to bench-tps-client (#94) * add get_block(s)/slot methods to BenchTpsClient * Update Cargo.lock * add commitment level for get_slot/blocks --- Cargo.lock | 1 + bench-tps/Cargo.toml | 1 + bench-tps/src/bench_tps_client.rs | 22 +++++++++++-- bench-tps/src/bench_tps_client/bank_client.rs | 24 ++++++++++++++ bench-tps/src/bench_tps_client/rpc_client.rs | 27 +++++++++++++++- bench-tps/src/bench_tps_client/thin_client.rs | 30 +++++++++++++++++ bench-tps/src/bench_tps_client/tpu_client.rs | 32 ++++++++++++++++++- 7 files changed, 132 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 79fcfb624d1b8f..06a2ad5ca5250c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5380,6 +5380,7 @@ dependencies = [ "solana-test-validator", "solana-thin-client", "solana-tpu-client", + "solana-transaction-status", "solana-version", "spl-instruction-padding", "tempfile", diff --git a/bench-tps/Cargo.toml b/bench-tps/Cargo.toml index cd40eb1c833c1c..2fc48c9e296d50 100644 --- a/bench-tps/Cargo.toml +++ b/bench-tps/Cargo.toml @@ -37,6 +37,7 @@ solana-sdk = { workspace = true } solana-streamer = { workspace = true } solana-thin-client = { workspace = true } solana-tpu-client = { workspace = true } +solana-transaction-status = { workspace = true } solana-version = { workspace = true } spl-instruction-padding = { workspace = true } thiserror = { workspace = true } diff --git a/bench-tps/src/bench_tps_client.rs b/bench-tps/src/bench_tps_client.rs index 3ab15bec11f7ee..0715d739879165 100644 --- a/bench-tps/src/bench_tps_client.rs +++ b/bench-tps/src/bench_tps_client.rs @@ -1,11 +1,12 @@ use { - solana_rpc_client_api::client_error::Error as ClientError, + solana_rpc_client_api::{client_error::Error as ClientError, config::RpcBlockConfig}, solana_sdk::{ account::Account, commitment_config::CommitmentConfig, epoch_info::EpochInfo, hash::Hash, - message::Message, pubkey::Pubkey, signature::Signature, transaction::Transaction, - transport::TransportError, + message::Message, pubkey::Pubkey, signature::Signature, slot_history::Slot, + transaction::Transaction, transport::TransportError, }, solana_tpu_client::tpu_client::TpuSenderError, + solana_transaction_status::UiConfirmedBlock, thiserror::Error, }; @@ -93,6 +94,21 @@ pub trait BenchTpsClient { ) -> Result; fn get_multiple_accounts(&self, pubkeys: &[Pubkey]) -> Result>>; + + fn get_slot_with_commitment(&self, commitment_config: CommitmentConfig) -> Result; + + fn get_blocks_with_commitment( + &self, + start_slot: Slot, + end_slot: Option, + commitment_config: CommitmentConfig, + ) -> Result>; + + fn get_block_with_config( + &self, + slot: Slot, + rpc_block_config: RpcBlockConfig, + ) -> Result; } mod bank_client; diff --git a/bench-tps/src/bench_tps_client/bank_client.rs b/bench-tps/src/bench_tps_client/bank_client.rs index 1aef7284c01ed6..3ea9080e51398a 100644 --- a/bench-tps/src/bench_tps_client/bank_client.rs +++ b/bench-tps/src/bench_tps_client/bank_client.rs @@ -1,5 +1,6 @@ use { crate::bench_tps_client::{BenchTpsClient, BenchTpsError, Result}, + solana_rpc_client_api::config::RpcBlockConfig, solana_runtime::bank_client::BankClient, solana_sdk::{ account::Account, @@ -10,8 +11,10 @@ use { message::Message, pubkey::Pubkey, signature::Signature, + slot_history::Slot, transaction::Transaction, }, + solana_transaction_status::UiConfirmedBlock, }; impl BenchTpsClient for BankClient { @@ -111,4 +114,25 @@ impl BenchTpsClient for BankClient { fn get_multiple_accounts(&self, _pubkeys: &[Pubkey]) -> Result>> { unimplemented!("BankClient doesn't support get_multiple_accounts"); } + + fn get_slot_with_commitment(&self, commitment_config: CommitmentConfig) -> Result { + SyncClient::get_slot_with_commitment(self, commitment_config).map_err(|err| err.into()) + } + + fn get_blocks_with_commitment( + &self, + _start_slot: Slot, + _end_slot: Option, + _commitment_config: CommitmentConfig, + ) -> Result> { + unimplemented!("BankClient doesn't support get_blocks"); + } + + fn get_block_with_config( + &self, + _slot: Slot, + _rpc_block_config: RpcBlockConfig, + ) -> Result { + unimplemented!("BankClient doesn't support get_block_with_config"); + } } diff --git a/bench-tps/src/bench_tps_client/rpc_client.rs b/bench-tps/src/bench_tps_client/rpc_client.rs index 2535099b464351..87ec1b8690c417 100644 --- a/bench-tps/src/bench_tps_client/rpc_client.rs +++ b/bench-tps/src/bench_tps_client/rpc_client.rs @@ -1,10 +1,13 @@ use { crate::bench_tps_client::{BenchTpsClient, BenchTpsError, Result}, solana_rpc_client::rpc_client::RpcClient, + solana_rpc_client_api::config::RpcBlockConfig, solana_sdk::{ account::Account, commitment_config::CommitmentConfig, epoch_info::EpochInfo, hash::Hash, - message::Message, pubkey::Pubkey, signature::Signature, transaction::Transaction, + message::Message, pubkey::Pubkey, signature::Signature, slot_history::Slot, + transaction::Transaction, }, + solana_transaction_status::UiConfirmedBlock, }; impl BenchTpsClient for RpcClient { @@ -104,4 +107,26 @@ impl BenchTpsClient for RpcClient { fn get_multiple_accounts(&self, pubkeys: &[Pubkey]) -> Result>> { RpcClient::get_multiple_accounts(self, pubkeys).map_err(|err| err.into()) } + + fn get_slot_with_commitment(&self, commitment_config: CommitmentConfig) -> Result { + RpcClient::get_slot_with_commitment(self, commitment_config).map_err(|err| err.into()) + } + + fn get_blocks_with_commitment( + &self, + start_slot: Slot, + end_slot: Option, + commitment_config: CommitmentConfig, + ) -> Result> { + RpcClient::get_blocks_with_commitment(self, start_slot, end_slot, commitment_config) + .map_err(|err| err.into()) + } + + fn get_block_with_config( + &self, + slot: Slot, + rpc_block_config: RpcBlockConfig, + ) -> Result { + RpcClient::get_block_with_config(self, slot, rpc_block_config).map_err(|err| err.into()) + } } diff --git a/bench-tps/src/bench_tps_client/thin_client.rs b/bench-tps/src/bench_tps_client/thin_client.rs index 6696774d679a8a..22945c4494f453 100644 --- a/bench-tps/src/bench_tps_client/thin_client.rs +++ b/bench-tps/src/bench_tps_client/thin_client.rs @@ -1,6 +1,7 @@ use { crate::bench_tps_client::{BenchTpsClient, BenchTpsError, Result}, solana_client::thin_client::ThinClient, + solana_rpc_client_api::config::RpcBlockConfig, solana_sdk::{ account::Account, client::{AsyncClient, Client, SyncClient}, @@ -10,8 +11,10 @@ use { message::Message, pubkey::Pubkey, signature::Signature, + slot_history::Slot, transaction::Transaction, }, + solana_transaction_status::UiConfirmedBlock, }; impl BenchTpsClient for ThinClient { @@ -110,4 +113,31 @@ impl BenchTpsClient for ThinClient { .get_multiple_accounts(pubkeys) .map_err(|err| err.into()) } + + fn get_slot_with_commitment(&self, commitment_config: CommitmentConfig) -> Result { + self.rpc_client() + .get_slot_with_commitment(commitment_config) + .map_err(|err| err.into()) + } + + fn get_blocks_with_commitment( + &self, + start_slot: Slot, + end_slot: Option, + commitment_config: CommitmentConfig, + ) -> Result> { + self.rpc_client() + .get_blocks_with_commitment(start_slot, end_slot, commitment_config) + .map_err(|err| err.into()) + } + + fn get_block_with_config( + &self, + slot: Slot, + rpc_block_config: RpcBlockConfig, + ) -> Result { + self.rpc_client() + .get_block_with_config(slot, rpc_block_config) + .map_err(|err| err.into()) + } } diff --git a/bench-tps/src/bench_tps_client/tpu_client.rs b/bench-tps/src/bench_tps_client/tpu_client.rs index c56da2ae6e880b..6c053271ad3eec 100644 --- a/bench-tps/src/bench_tps_client/tpu_client.rs +++ b/bench-tps/src/bench_tps_client/tpu_client.rs @@ -4,10 +4,13 @@ use { solana_connection_cache::connection_cache::{ ConnectionManager, ConnectionPool, NewConnectionConfig, }, + solana_rpc_client_api::config::RpcBlockConfig, solana_sdk::{ account::Account, commitment_config::CommitmentConfig, epoch_info::EpochInfo, hash::Hash, - message::Message, pubkey::Pubkey, signature::Signature, transaction::Transaction, + message::Message, pubkey::Pubkey, signature::Signature, slot_history::Slot, + transaction::Transaction, }, + solana_transaction_status::UiConfirmedBlock, }; impl BenchTpsClient for TpuClient @@ -130,4 +133,31 @@ where .get_multiple_accounts(pubkeys) .map_err(|err| err.into()) } + + fn get_slot_with_commitment(&self, commitment_config: CommitmentConfig) -> Result { + self.rpc_client() + .get_slot_with_commitment(commitment_config) + .map_err(|err| err.into()) + } + + fn get_blocks_with_commitment( + &self, + start_slot: Slot, + end_slot: Option, + commitment_config: CommitmentConfig, + ) -> Result> { + self.rpc_client() + .get_blocks_with_commitment(start_slot, end_slot, commitment_config) + .map_err(|err| err.into()) + } + + fn get_block_with_config( + &self, + slot: Slot, + rpc_block_config: RpcBlockConfig, + ) -> Result { + self.rpc_client() + .get_block_with_config(slot, rpc_block_config) + .map_err(|err| err.into()) + } } From 9decf2a320bea1f61c6acbb0dc9fec9990c06cc5 Mon Sep 17 00:00:00 2001 From: steviez Date: Thu, 7 Mar 2024 16:06:31 -0600 Subject: [PATCH 41/71] blockstore: Remove unnecessary function and threadpool (#122) In a previous change, we removed the threadpool used to fetch entries in parallel in favor of combining all fetches into a single rocksdb multi_get() call. This change does the same thing, except for a threadpool that was used to fetch entries when we needed them to purge the transaction status and address signatures columns. --- ledger/src/blockstore.rs | 36 +---------------------- ledger/src/blockstore/blockstore_purge.rs | 3 +- 2 files changed, 3 insertions(+), 36 deletions(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index f8c8330843dfce..f15976abdb241b 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -31,10 +31,7 @@ use { itertools::Itertools, log::*, rand::Rng, - rayon::{ - iter::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator}, - ThreadPool, - }, + rayon::iter::{IntoParallelIterator, ParallelIterator}, rocksdb::{DBRawIterator, LiveFile}, solana_accounts_db::hardened_unpack::{ unpack_genesis_archive, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, @@ -94,16 +91,6 @@ pub use { rocksdb::properties as RocksProperties, }; -// get_max_thread_count to match number of threads in the old code. -// see: https://github.com/solana-labs/solana/pull/24853 -lazy_static! { - static ref PAR_THREAD_POOL_ALL_CPUS: ThreadPool = rayon::ThreadPoolBuilder::new() - .num_threads(num_cpus::get()) - .thread_name(|i| format!("solBstoreAll{i:02}")) - .build() - .unwrap(); -} - pub const MAX_REPLAY_WAKE_UP_SIGNALS: usize = 1; pub const MAX_COMPLETED_SLOTS_IN_CHANNEL: usize = 100_000; @@ -3283,27 +3270,6 @@ impl Blockstore { self.get_slot_entries_in_block(slot, vec![(start_index, end_index)], slot_meta) } - fn get_any_valid_slot_entries(&self, slot: Slot, start_index: u64) -> Vec { - let (completed_ranges, slot_meta) = self - .get_completed_ranges(slot, start_index) - .unwrap_or_default(); - if completed_ranges.is_empty() { - return vec![]; - } - let slot_meta = slot_meta.unwrap(); - - let entries: Vec> = PAR_THREAD_POOL_ALL_CPUS.install(|| { - completed_ranges - .par_iter() - .map(|(start_index, end_index)| { - self.get_entries_in_data_block(slot, *start_index, *end_index, Some(&slot_meta)) - .unwrap_or_default() - }) - .collect() - }); - entries.into_iter().flatten().collect() - } - /// Returns a mapping from each elements of `slots` to a list of the /// element's children slots. pub fn get_slots_since(&self, slots: &[Slot]) -> Result>> { diff --git a/ledger/src/blockstore/blockstore_purge.rs b/ledger/src/blockstore/blockstore_purge.rs index d8b4c7424cd8c1..d442732303fa2a 100644 --- a/ledger/src/blockstore/blockstore_purge.rs +++ b/ledger/src/blockstore/blockstore_purge.rs @@ -455,7 +455,8 @@ impl Blockstore { for slot in from_slot..=to_slot { let primary_indexes = slot_indexes(slot); - let slot_entries = self.get_any_valid_slot_entries(slot, 0); + let (slot_entries, _, _) = + self.get_slot_entries_with_shred_info(slot, 0, true /* allow_dead_slots */)?; let transactions = slot_entries .into_iter() .flat_map(|entry| entry.transactions); From ca41b16e47526146a25c9ea6f8b96c331bb2767e Mon Sep 17 00:00:00 2001 From: Tao Zhu <82401714+tao-stones@users.noreply.github.com> Date: Thu, 7 Mar 2024 18:48:35 -0600 Subject: [PATCH 42/71] add precompile signature metrics to cost tracker (#133) --- cost-model/src/block_cost_limits.rs | 8 +- cost-model/src/cost_model.rs | 24 +++++- cost-model/src/cost_tracker.rs | 42 ++++++++++ cost-model/src/transaction_cost.rs | 31 +++++++ sdk/program/src/message/sanitized.rs | 116 ++++++++++++++++++++++++--- 5 files changed, 205 insertions(+), 16 deletions(-) diff --git a/cost-model/src/block_cost_limits.rs b/cost-model/src/block_cost_limits.rs index 328d89cd04198b..b04f289e0553af 100644 --- a/cost-model/src/block_cost_limits.rs +++ b/cost-model/src/block_cost_limits.rs @@ -24,6 +24,10 @@ pub const MAX_CONCURRENCY: u64 = 4; pub const COMPUTE_UNIT_TO_US_RATIO: u64 = 30; /// Number of compute units for one signature verification. pub const SIGNATURE_COST: u64 = COMPUTE_UNIT_TO_US_RATIO * 24; +/// Number of compute units for one secp256k1 signature verification. +pub const SECP256K1_VERIFY_COST: u64 = COMPUTE_UNIT_TO_US_RATIO * 223; +/// Number of compute units for one ed25519 signature verification. +pub const ED25519_VERIFY_COST: u64 = COMPUTE_UNIT_TO_US_RATIO * 76; /// Number of compute units for one write lock pub const WRITE_LOCK_UNITS: u64 = COMPUTE_UNIT_TO_US_RATIO * 10; /// Number of data bytes per compute units @@ -43,8 +47,8 @@ lazy_static! { (bpf_loader::id(), solana_bpf_loader_program::DEFAULT_LOADER_COMPUTE_UNITS), (loader_v4::id(), solana_loader_v4_program::DEFAULT_COMPUTE_UNITS), // Note: These are precompile, run directly in bank during sanitizing; - (secp256k1_program::id(), COMPUTE_UNIT_TO_US_RATIO * 24), - (ed25519_program::id(), COMPUTE_UNIT_TO_US_RATIO * 24), + (secp256k1_program::id(), 0), + (ed25519_program::id(), 0), ] .iter() .cloned() diff --git a/cost-model/src/cost_model.rs b/cost-model/src/cost_model.rs index b81ea24402d4df..fa12a7343bc7e0 100644 --- a/cost-model/src/cost_model.rs +++ b/cost-model/src/cost_model.rs @@ -43,7 +43,7 @@ impl CostModel { } else { let mut tx_cost = UsageCostDetails::new_with_default_capacity(); - tx_cost.signature_cost = Self::get_signature_cost(transaction); + Self::get_signature_cost(&mut tx_cost, transaction); Self::get_write_lock_cost(&mut tx_cost, transaction, feature_set); Self::get_transaction_cost(&mut tx_cost, transaction, feature_set); tx_cost.account_data_size = Self::calculate_account_data_size(transaction); @@ -53,8 +53,26 @@ impl CostModel { } } - fn get_signature_cost(transaction: &SanitizedTransaction) -> u64 { - transaction.signatures().len() as u64 * SIGNATURE_COST + fn get_signature_cost(tx_cost: &mut UsageCostDetails, transaction: &SanitizedTransaction) { + let signatures_count_detail = transaction.message().get_signature_details(); + tx_cost.num_transaction_signatures = signatures_count_detail.num_transaction_signatures(); + tx_cost.num_secp256k1_instruction_signatures = + signatures_count_detail.num_secp256k1_instruction_signatures(); + tx_cost.num_ed25519_instruction_signatures = + signatures_count_detail.num_ed25519_instruction_signatures(); + tx_cost.signature_cost = signatures_count_detail + .num_transaction_signatures() + .saturating_mul(SIGNATURE_COST) + .saturating_add( + signatures_count_detail + .num_secp256k1_instruction_signatures() + .saturating_mul(SECP256K1_VERIFY_COST), + ) + .saturating_add( + signatures_count_detail + .num_ed25519_instruction_signatures() + .saturating_mul(ED25519_VERIFY_COST), + ); } fn get_writable_accounts(transaction: &SanitizedTransaction) -> Vec { diff --git a/cost-model/src/cost_tracker.rs b/cost-model/src/cost_tracker.rs index 8fb092c36680a0..b5e3f9f4932a59 100644 --- a/cost-model/src/cost_tracker.rs +++ b/cost-model/src/cost_tracker.rs @@ -58,6 +58,9 @@ pub struct CostTracker { vote_cost: u64, transaction_count: u64, account_data_size: u64, + transaction_signature_count: u64, + secp256k1_instruction_signature_count: u64, + ed25519_instruction_signature_count: u64, } impl Default for CostTracker { @@ -77,6 +80,9 @@ impl Default for CostTracker { vote_cost: 0, transaction_count: 0, account_data_size: 0, + transaction_signature_count: 0, + secp256k1_instruction_signature_count: 0, + ed25519_instruction_signature_count: 0, } } } @@ -153,6 +159,21 @@ impl CostTracker { ("costliest_account", costliest_account.to_string(), String), ("costliest_account_cost", costliest_account_cost as i64, i64), ("account_data_size", self.account_data_size, i64), + ( + "transaction_signature_count", + self.transaction_signature_count, + i64 + ), + ( + "secp256k1_instruction_signature_count", + self.secp256k1_instruction_signature_count, + i64 + ), + ( + "ed25519_instruction_signature_count", + self.ed25519_instruction_signature_count, + i64 + ), ); } @@ -213,6 +234,18 @@ impl CostTracker { self.add_transaction_execution_cost(tx_cost, tx_cost.sum()); saturating_add_assign!(self.account_data_size, tx_cost.account_data_size()); saturating_add_assign!(self.transaction_count, 1); + saturating_add_assign!( + self.transaction_signature_count, + tx_cost.num_transaction_signatures() + ); + saturating_add_assign!( + self.secp256k1_instruction_signature_count, + tx_cost.num_secp256k1_instruction_signatures() + ); + saturating_add_assign!( + self.ed25519_instruction_signature_count, + tx_cost.num_ed25519_instruction_signatures() + ); } fn remove_transaction_cost(&mut self, tx_cost: &TransactionCost) { @@ -222,6 +255,15 @@ impl CostTracker { .account_data_size .saturating_sub(tx_cost.account_data_size()); self.transaction_count = self.transaction_count.saturating_sub(1); + self.transaction_signature_count = self + .transaction_signature_count + .saturating_sub(tx_cost.num_transaction_signatures()); + self.secp256k1_instruction_signature_count = self + .secp256k1_instruction_signature_count + .saturating_sub(tx_cost.num_secp256k1_instruction_signatures()); + self.ed25519_instruction_signature_count = self + .ed25519_instruction_signature_count + .saturating_sub(tx_cost.num_ed25519_instruction_signatures()); } /// Apply additional actual execution units to cost_tracker diff --git a/cost-model/src/transaction_cost.rs b/cost-model/src/transaction_cost.rs index c6e68bfe17b6f4..c92639676958ae 100644 --- a/cost-model/src/transaction_cost.rs +++ b/cost-model/src/transaction_cost.rs @@ -91,6 +91,27 @@ impl TransactionCost { Self::Transaction(usage_cost) => &usage_cost.writable_accounts, } } + + pub fn num_transaction_signatures(&self) -> u64 { + match self { + Self::SimpleVote { .. } => 1, + Self::Transaction(usage_cost) => usage_cost.num_transaction_signatures, + } + } + + pub fn num_secp256k1_instruction_signatures(&self) -> u64 { + match self { + Self::SimpleVote { .. } => 0, + Self::Transaction(usage_cost) => usage_cost.num_secp256k1_instruction_signatures, + } + } + + pub fn num_ed25519_instruction_signatures(&self) -> u64 { + match self { + Self::SimpleVote { .. } => 0, + Self::Transaction(usage_cost) => usage_cost.num_ed25519_instruction_signatures, + } + } } const MAX_WRITABLE_ACCOUNTS: usize = 256; @@ -105,6 +126,9 @@ pub struct UsageCostDetails { pub programs_execution_cost: u64, pub loaded_accounts_data_size_cost: u64, pub account_data_size: u64, + pub num_transaction_signatures: u64, + pub num_secp256k1_instruction_signatures: u64, + pub num_ed25519_instruction_signatures: u64, } impl Default for UsageCostDetails { @@ -117,6 +141,9 @@ impl Default for UsageCostDetails { programs_execution_cost: 0u64, loaded_accounts_data_size_cost: 0u64, account_data_size: 0u64, + num_transaction_signatures: 0u64, + num_secp256k1_instruction_signatures: 0u64, + num_ed25519_instruction_signatures: 0u64, } } } @@ -134,6 +161,10 @@ impl PartialEq for UsageCostDetails { && self.programs_execution_cost == other.programs_execution_cost && self.loaded_accounts_data_size_cost == other.loaded_accounts_data_size_cost && self.account_data_size == other.account_data_size + && self.num_transaction_signatures == other.num_transaction_signatures + && self.num_secp256k1_instruction_signatures + == other.num_secp256k1_instruction_signatures + && self.num_ed25519_instruction_signatures == other.num_ed25519_instruction_signatures && to_hash_set(&self.writable_accounts) == to_hash_set(&other.writable_accounts) } } diff --git a/sdk/program/src/message/sanitized.rs b/sdk/program/src/message/sanitized.rs index d4c7638e136a72..ce276a60ef69e7 100644 --- a/sdk/program/src/message/sanitized.rs +++ b/sdk/program/src/message/sanitized.rs @@ -345,17 +345,7 @@ impl SanitizedMessage { } pub fn num_signatures(&self) -> u64 { - let mut num_signatures = u64::from(self.header().num_required_signatures); - // This next part is really calculating the number of pre-processor - // operations being done and treating them like a signature - for (program_id, instruction) in self.program_instructions_iter() { - if secp256k1_program::check_id(program_id) || ed25519_program::check_id(program_id) { - if let Some(num_verifies) = instruction.data.first() { - num_signatures = num_signatures.saturating_add(u64::from(*num_verifies)); - } - } - } - num_signatures + self.get_signature_details().total_signatures() } /// Returns the number of requested write-locks in this message. @@ -365,6 +355,68 @@ impl SanitizedMessage { .len() .saturating_sub(self.num_readonly_accounts()) as u64 } + + /// return detailed signature counts + pub fn get_signature_details(&self) -> TransactionSignatureDetails { + let mut transaction_signature_details = TransactionSignatureDetails { + num_transaction_signatures: u64::from(self.header().num_required_signatures), + ..TransactionSignatureDetails::default() + }; + + // counting the number of pre-processor operations separately + for (program_id, instruction) in self.program_instructions_iter() { + if secp256k1_program::check_id(program_id) { + if let Some(num_verifies) = instruction.data.first() { + transaction_signature_details.num_secp256k1_instruction_signatures = + transaction_signature_details + .num_secp256k1_instruction_signatures + .saturating_add(u64::from(*num_verifies)); + } + } else if ed25519_program::check_id(program_id) { + if let Some(num_verifies) = instruction.data.first() { + transaction_signature_details.num_ed25519_instruction_signatures = + transaction_signature_details + .num_ed25519_instruction_signatures + .saturating_add(u64::from(*num_verifies)); + } + } + } + + transaction_signature_details + } +} + +#[derive(Default)] +/// Transaction signature details including the number of transaction signatures +/// and precompile signatures. +pub struct TransactionSignatureDetails { + num_transaction_signatures: u64, + num_secp256k1_instruction_signatures: u64, + num_ed25519_instruction_signatures: u64, +} + +impl TransactionSignatureDetails { + /// return total number of signature, treating pre-processor operations as signature + pub(crate) fn total_signatures(&self) -> u64 { + self.num_transaction_signatures + .saturating_add(self.num_secp256k1_instruction_signatures) + .saturating_add(self.num_ed25519_instruction_signatures) + } + + /// return the number of transaction signatures + pub fn num_transaction_signatures(&self) -> u64 { + self.num_transaction_signatures + } + + /// return the number of secp256k1 instruction signatures + pub fn num_secp256k1_instruction_signatures(&self) -> u64 { + self.num_secp256k1_instruction_signatures + } + + /// return the number of ed25519 instruction signatures + pub fn num_ed25519_instruction_signatures(&self) -> u64 { + self.num_ed25519_instruction_signatures + } } #[cfg(test)] @@ -563,4 +615,46 @@ mod tests { } } } + + #[test] + fn test_get_signature_details() { + let key0 = Pubkey::new_unique(); + let key1 = Pubkey::new_unique(); + let loader_key = Pubkey::new_unique(); + + let loader_instr = CompiledInstruction::new(2, &(), vec![0, 1]); + let mock_secp256k1_instr = CompiledInstruction::new(3, &[1u8; 10], vec![]); + let mock_ed25519_instr = CompiledInstruction::new(4, &[5u8; 10], vec![]); + + let message = SanitizedMessage::try_from_legacy_message( + legacy::Message::new_with_compiled_instructions( + 2, + 1, + 2, + vec![ + key0, + key1, + loader_key, + secp256k1_program::id(), + ed25519_program::id(), + ], + Hash::default(), + vec![ + loader_instr, + mock_secp256k1_instr.clone(), + mock_ed25519_instr, + mock_secp256k1_instr, + ], + ), + ) + .unwrap(); + + let signature_details = message.get_signature_details(); + // expect 2 required transaction signatures + assert_eq!(2, signature_details.num_transaction_signatures); + // expect 2 secp256k1 instruction signatures - 1 for each mock_secp2561k1_instr + assert_eq!(2, signature_details.num_secp256k1_instruction_signatures); + // expect 5 ed25519 instruction signatures from mock_ed25519_instr + assert_eq!(5, signature_details.num_ed25519_instruction_signatures); + } } From ed57e6aa108dacbc2e53e44ce1c2cd1514ddba10 Mon Sep 17 00:00:00 2001 From: Yihau Chen Date: Fri, 8 Mar 2024 10:48:39 +0800 Subject: [PATCH 43/71] ci: rename script (#125) --- ...nifest-keypair.sh => agave-install-update-manifest-keypair.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/{solana-install-update-manifest-keypair.sh => agave-install-update-manifest-keypair.sh} (100%) diff --git a/scripts/solana-install-update-manifest-keypair.sh b/scripts/agave-install-update-manifest-keypair.sh similarity index 100% rename from scripts/solana-install-update-manifest-keypair.sh rename to scripts/agave-install-update-manifest-keypair.sh From ce10dc10430d0926aa9b003be702806584cf5ef4 Mon Sep 17 00:00:00 2001 From: Jon C Date: Fri, 8 Mar 2024 11:02:33 +0100 Subject: [PATCH 44/71] runtime: Move `From` from sdk (#141) sdk: Move `From` into runtime --- runtime/src/bank/address_lookup_table.rs | 28 ++++++++++++++----- sdk/program/src/address_lookup_table/error.rs | 14 ---------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/runtime/src/bank/address_lookup_table.rs b/runtime/src/bank/address_lookup_table.rs index 483ec7cea00ea1..51eee794803e14 100644 --- a/runtime/src/bank/address_lookup_table.rs +++ b/runtime/src/bank/address_lookup_table.rs @@ -10,6 +10,17 @@ use { }, }; +fn into_address_loader_error(err: AddressLookupError) -> AddressLoaderError { + match err { + AddressLookupError::LookupTableAccountNotFound => { + AddressLoaderError::LookupTableAccountNotFound + } + AddressLookupError::InvalidAccountOwner => AddressLoaderError::InvalidAccountOwner, + AddressLookupError::InvalidAccountData => AddressLoaderError::InvalidAccountData, + AddressLookupError::InvalidLookupIndex => AddressLoaderError::InvalidLookupIndex, + } +} + impl AddressLoader for &Bank { fn load_addresses( self, @@ -23,15 +34,18 @@ impl AddressLoader for &Bank { .get_slot_hashes() .map_err(|_| AddressLoaderError::SlotHashesSysvarNotFound)?; - Ok(address_table_lookups + address_table_lookups .iter() .map(|address_table_lookup| { - self.rc.accounts.load_lookup_table_addresses( - &self.ancestors, - address_table_lookup, - &slot_hashes, - ) + self.rc + .accounts + .load_lookup_table_addresses( + &self.ancestors, + address_table_lookup, + &slot_hashes, + ) + .map_err(into_address_loader_error) }) - .collect::>()?) + .collect::>() } } diff --git a/sdk/program/src/address_lookup_table/error.rs b/sdk/program/src/address_lookup_table/error.rs index b427067afc386c..9925dee4dbbf4c 100644 --- a/sdk/program/src/address_lookup_table/error.rs +++ b/sdk/program/src/address_lookup_table/error.rs @@ -1,5 +1,3 @@ -#[cfg(not(target_os = "solana"))] -use solana_program::message::AddressLoaderError; use thiserror::Error; #[derive(Debug, Error, PartialEq, Eq, Clone)] @@ -20,15 +18,3 @@ pub enum AddressLookupError { #[error("Address lookup contains an invalid index")] InvalidLookupIndex, } - -#[cfg(not(target_os = "solana"))] -impl From for AddressLoaderError { - fn from(err: AddressLookupError) -> Self { - match err { - AddressLookupError::LookupTableAccountNotFound => Self::LookupTableAccountNotFound, - AddressLookupError::InvalidAccountOwner => Self::InvalidAccountOwner, - AddressLookupError::InvalidAccountData => Self::InvalidAccountData, - AddressLookupError::InvalidLookupIndex => Self::InvalidLookupIndex, - } - } -} From 2c55819e7bfdeeb6cc798afb18373cd86573af63 Mon Sep 17 00:00:00 2001 From: Lucas Steuernagel <38472950+LucasSte@users.noreply.github.com> Date: Fri, 8 Mar 2024 09:28:04 -0300 Subject: [PATCH 45/71] Gather recording booleans in a data structure (#134) --- core/src/banking_stage/consumer.rs | 5 ++-- ledger/src/blockstore_processor.rs | 16 +++++------ programs/sbf/tests/programs.rs | 13 +++++---- runtime/src/bank.rs | 37 +++++++++++------------- runtime/src/bank/tests.rs | 24 ++++++++-------- svm/src/transaction_processor.rs | 45 +++++++++++++++++++----------- 6 files changed, 74 insertions(+), 66 deletions(-) diff --git a/core/src/banking_stage/consumer.rs b/core/src/banking_stage/consumer.rs index 957e190c873f64..c5ed22a34278ce 100644 --- a/core/src/banking_stage/consumer.rs +++ b/core/src/banking_stage/consumer.rs @@ -34,6 +34,7 @@ use { solana_svm::{ account_loader::{validate_fee_payer, TransactionCheckResult}, transaction_error_metrics::TransactionErrorMetrics, + transaction_processor::ExecutionRecordingConfig, }, std::{ sync::{atomic::Ordering, Arc}, @@ -593,9 +594,7 @@ impl Consumer { .load_and_execute_transactions( batch, MAX_PROCESSING_AGE, - transaction_status_sender_enabled, - transaction_status_sender_enabled, - transaction_status_sender_enabled, + ExecutionRecordingConfig::new_single_setting(transaction_status_sender_enabled), &mut execute_and_commit_timings.execute_timings, None, // account_overrides self.log_messages_bytes_limit, diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs index c999eab1a56fd4..e4ae5f368b2afd 100644 --- a/ledger/src/blockstore_processor.rs +++ b/ledger/src/blockstore_processor.rs @@ -57,8 +57,11 @@ use { VersionedTransaction, }, }, - solana_svm::transaction_results::{ - TransactionExecutionDetails, TransactionExecutionResult, TransactionResults, + solana_svm::{ + transaction_processor::ExecutionRecordingConfig, + transaction_results::{ + TransactionExecutionDetails, TransactionExecutionResult, TransactionResults, + }, }, solana_transaction_status::token_balances::TransactionTokenBalancesSet, solana_vote::{vote_account::VoteAccountsHashMap, vote_sender_types::ReplayVoteSender}, @@ -163,9 +166,7 @@ pub fn execute_batch( batch, MAX_PROCESSING_AGE, transaction_status_sender.is_some(), - transaction_status_sender.is_some(), - transaction_status_sender.is_some(), - transaction_status_sender.is_some(), + ExecutionRecordingConfig::new_single_setting(transaction_status_sender.is_some()), timings, log_messages_bytes_limit, ); @@ -1972,6 +1973,7 @@ pub mod tests { system_transaction, transaction::{Transaction, TransactionError}, }, + solana_svm::transaction_processor::ExecutionRecordingConfig, solana_vote::vote_account::VoteAccount, solana_vote_program::{ self, @@ -3962,9 +3964,7 @@ pub mod tests { &batch, MAX_PROCESSING_AGE, false, - false, - false, - false, + ExecutionRecordingConfig::new_single_setting(false), &mut ExecuteTimings::default(), None, ); diff --git a/programs/sbf/tests/programs.rs b/programs/sbf/tests/programs.rs index dc4867ce7e40fd..22969bc482a28e 100644 --- a/programs/sbf/tests/programs.rs +++ b/programs/sbf/tests/programs.rs @@ -48,6 +48,7 @@ use { sysvar::{self, clock}, transaction::VersionedTransaction, }, + solana_svm::transaction_processor::ExecutionRecordingConfig, solana_svm::transaction_results::{ DurableNonceFee, InnerInstruction, TransactionExecutionDetails, TransactionExecutionResult, TransactionResults, @@ -104,9 +105,11 @@ fn process_transaction_and_record_inner( &tx_batch, MAX_PROCESSING_AGE, false, - true, - true, - false, + ExecutionRecordingConfig { + enable_cpi_recording: true, + enable_log_recording: true, + enable_return_data_recording: false, + }, &mut ExecuteTimings::default(), None, ) @@ -152,9 +155,7 @@ fn execute_transactions( &batch, std::usize::MAX, true, - true, - true, - true, + ExecutionRecordingConfig::new_single_setting(true), &mut timings, None, ); diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index ee04f20787cb9a..3e504d470de744 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -269,6 +269,7 @@ pub struct BankRc { #[cfg(RUSTC_WITH_SPECIALIZATION)] use solana_frozen_abi::abi_example::AbiExample; +use solana_svm::transaction_processor::ExecutionRecordingConfig; #[cfg(RUSTC_WITH_SPECIALIZATION)] impl AbiExample for BankRc { @@ -4297,9 +4298,11 @@ impl Bank { // for processing. During forwarding, the transaction could expire if the // delay is not accounted for. MAX_PROCESSING_AGE - MAX_TRANSACTION_FORWARDING_DELAY, - enable_cpi_recording, - true, - true, + ExecutionRecordingConfig { + enable_cpi_recording, + enable_log_recording: true, + enable_return_data_recording: true, + }, &mut timings, Some(&account_overrides), None, @@ -4548,9 +4551,7 @@ impl Bank { &self, batch: &TransactionBatch, max_age: usize, - enable_cpi_recording: bool, - enable_log_recording: bool, - enable_return_data_recording: bool, + recording_config: ExecutionRecordingConfig, timings: &mut ExecuteTimings, account_overrides: Option<&AccountOverrides>, log_messages_bytes_limit: Option, @@ -4614,9 +4615,7 @@ impl Bank { sanitized_txs, &mut check_results, &mut error_counters, - enable_cpi_recording, - enable_log_recording, - enable_return_data_recording, + recording_config, timings, account_overrides, self.builtin_programs.iter(), @@ -5642,9 +5641,7 @@ impl Bank { batch: &TransactionBatch, max_age: usize, collect_balances: bool, - enable_cpi_recording: bool, - enable_log_recording: bool, - enable_return_data_recording: bool, + recording_config: ExecutionRecordingConfig, timings: &mut ExecuteTimings, log_messages_bytes_limit: Option, ) -> (TransactionResults, TransactionBalancesSet) { @@ -5665,9 +5662,7 @@ impl Bank { } = self.load_and_execute_transactions( batch, max_age, - enable_cpi_recording, - enable_log_recording, - enable_return_data_recording, + recording_config, timings, None, log_messages_bytes_limit, @@ -5735,9 +5730,11 @@ impl Bank { &batch, MAX_PROCESSING_AGE, false, // collect_balances - false, // enable_cpi_recording - true, // enable_log_recording - true, // enable_return_data_recording + ExecutionRecordingConfig { + enable_cpi_recording: false, + enable_log_recording: true, + enable_return_data_recording: true, + }, &mut ExecuteTimings::default(), Some(1000 * 1000), ); @@ -5773,9 +5770,7 @@ impl Bank { batch, MAX_PROCESSING_AGE, false, - false, - false, - false, + ExecutionRecordingConfig::new_single_setting(false), &mut ExecuteTimings::default(), None, ) diff --git a/runtime/src/bank/tests.rs b/runtime/src/bank/tests.rs index 753116ff878e18..f9b846d85b1512 100644 --- a/runtime/src/bank/tests.rs +++ b/runtime/src/bank/tests.rs @@ -3122,9 +3122,7 @@ fn test_interleaving_locks() { &lock_result, MAX_PROCESSING_AGE, false, - false, - false, - false, + ExecutionRecordingConfig::new_single_setting(false), &mut ExecuteTimings::default(), None, ) @@ -5948,9 +5946,7 @@ fn test_pre_post_transaction_balances() { &lock_result, MAX_PROCESSING_AGE, true, - false, - false, - false, + ExecutionRecordingConfig::new_single_setting(false), &mut ExecuteTimings::default(), None, ); @@ -9230,9 +9226,11 @@ fn test_tx_log_order() { &batch, MAX_PROCESSING_AGE, false, - false, - true, - false, + ExecutionRecordingConfig { + enable_cpi_recording: false, + enable_log_recording: true, + enable_return_data_recording: false, + }, &mut ExecuteTimings::default(), None, ) @@ -9338,9 +9336,11 @@ fn test_tx_return_data() { &batch, MAX_PROCESSING_AGE, false, - false, - false, - true, + ExecutionRecordingConfig { + enable_cpi_recording: false, + enable_log_recording: false, + enable_return_data_recording: true, + }, &mut ExecuteTimings::default(), None, ) diff --git a/svm/src/transaction_processor.rs b/svm/src/transaction_processor.rs index e44b426df96b0d..d90afb0a428ea3 100644 --- a/svm/src/transaction_processor.rs +++ b/svm/src/transaction_processor.rs @@ -65,6 +65,24 @@ pub struct LoadAndExecuteSanitizedTransactionsOutput { pub execution_results: Vec, } +/// Configuration of the recording capabilities for transaction execution +#[derive(Copy, Clone)] +pub struct ExecutionRecordingConfig { + pub enable_cpi_recording: bool, + pub enable_log_recording: bool, + pub enable_return_data_recording: bool, +} + +impl ExecutionRecordingConfig { + pub fn new_single_setting(option: bool) -> Self { + ExecutionRecordingConfig { + enable_return_data_recording: option, + enable_log_recording: option, + enable_cpi_recording: option, + } + } +} + pub trait TransactionProcessingCallback { fn account_matches_owners(&self, account: &Pubkey, owners: &[Pubkey]) -> Option; @@ -184,9 +202,7 @@ impl TransactionBatchProcessor { sanitized_txs: &[SanitizedTransaction], check_results: &mut [TransactionCheckResult], error_counters: &mut TransactionErrorMetrics, - enable_cpi_recording: bool, - enable_log_recording: bool, - enable_return_data_recording: bool, + recording_config: ExecutionRecordingConfig, timings: &mut ExecuteTimings, account_overrides: Option<&AccountOverrides>, builtin_programs: impl Iterator, @@ -266,9 +282,7 @@ impl TransactionBatchProcessor { loaded_transaction, compute_budget, nonce.as_ref().map(DurableNonceFee::from), - enable_cpi_recording, - enable_log_recording, - enable_return_data_recording, + recording_config, timings, error_counters, log_messages_bytes_limit, @@ -466,9 +480,7 @@ impl TransactionBatchProcessor { loaded_transaction: &mut LoadedTransaction, compute_budget: ComputeBudget, durable_nonce_fee: Option, - enable_cpi_recording: bool, - enable_log_recording: bool, - enable_return_data_recording: bool, + recording_config: ExecutionRecordingConfig, timings: &mut ExecuteTimings, error_counters: &mut TransactionErrorMetrics, log_messages_bytes_limit: Option, @@ -506,7 +518,7 @@ impl TransactionBatchProcessor { tx.message(), ); - let log_collector = if enable_log_recording { + let log_collector = if recording_config.enable_log_recording { match log_messages_bytes_limit { None => Some(LogCollector::new_ref()), Some(log_messages_bytes_limit) => Some(LogCollector::new_ref_with_limit(Some( @@ -585,7 +597,7 @@ impl TransactionBatchProcessor { .ok() }); - let inner_instructions = if enable_cpi_recording { + let inner_instructions = if recording_config.enable_cpi_recording { Some(Self::inner_instructions_list_from_instruction_trace( &transaction_context, )) @@ -616,11 +628,12 @@ impl TransactionBatchProcessor { ); saturating_add_assign!(timings.details.changed_account_count, touched_account_count); - let return_data = if enable_return_data_recording && !return_data.data.is_empty() { - Some(return_data) - } else { - None - }; + let return_data = + if recording_config.enable_return_data_recording && !return_data.data.is_empty() { + Some(return_data) + } else { + None + }; TransactionExecutionResult::Executed { details: TransactionExecutionDetails { From d5ffeefb3d6642228101f39e2d76d36393cba479 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 9 Mar 2024 00:59:43 +0800 Subject: [PATCH 46/71] build(deps): bump cc from 1.0.83 to 1.0.89 (#40) * build(deps): bump cc from 1.0.83 to 1.0.89 Bumps [cc](https://github.com/rust-lang/cc-rs) from 1.0.83 to 1.0.89. - [Release notes](https://github.com/rust-lang/cc-rs/releases) - [Commits](https://github.com/rust-lang/cc-rs/compare/1.0.83...1.0.89) --- updated-dependencies: - dependency-name: cc dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * [auto-commit] Update all Cargo lock files --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot-buildkite --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- programs/sbf/Cargo.lock | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 06a2ad5ca5250c..17e6d5e3a82f04 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1021,9 +1021,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.0.83" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +checksum = "a0ba8f7aaa012f30d5b2861462f6708eccd49c3c39863fe083a308035f63d723" dependencies = [ "jobserver", "libc", diff --git a/Cargo.toml b/Cargo.toml index 34cd21171785a0..e227ba5ff1578f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -170,7 +170,7 @@ bytes = "1.5" bzip2 = "0.4.4" caps = "0.5.5" cargo_metadata = "0.15.4" -cc = "1.0.83" +cc = "1.0.89" chrono = { version = "0.4.34", default-features = false } chrono-humanize = "0.2.3" clap = "2.33.1" diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index 3b2bc66e262336..04b6a37b4291f7 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -971,9 +971,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.83" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +checksum = "a0ba8f7aaa012f30d5b2861462f6708eccd49c3c39863fe083a308035f63d723" dependencies = [ "jobserver", "libc", From 3cba0722748e2cebfef38b63aa88301ad5052404 Mon Sep 17 00:00:00 2001 From: Justin Starry Date: Sat, 9 Mar 2024 01:02:21 +0800 Subject: [PATCH 47/71] Use agave prefix in scripts for pre-installed binaries (#155) --- multinode-demo/common.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/multinode-demo/common.sh b/multinode-demo/common.sh index 9ae9331cb7a11d..9914bff0bac15a 100644 --- a/multinode-demo/common.sh +++ b/multinode-demo/common.sh @@ -30,7 +30,11 @@ if [[ -n $USE_INSTALL || ! -f "$SOLANA_ROOT"/Cargo.toml ]]; then if [[ -z $program ]]; then printf "solana" else - printf "solana-%s" "$program" + if [[ $program == "validator" || $program == "ledger-tool" || $program == "watchtower" || $program == "install" ]]; then + printf "agave-%s" "$program" + else + printf "solana-%s" "$program" + fi fi } else From f8c5e15020810e073d6f484a12018edc21841bee Mon Sep 17 00:00:00 2001 From: HaoranYi Date: Fri, 8 Mar 2024 12:14:40 -0600 Subject: [PATCH 48/71] Move delta hash test function to dev-context-utils (#151) move delta hash test function to dev-context-utils Co-authored-by: HaoranYi --- accounts-db/src/accounts_db.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/accounts-db/src/accounts_db.rs b/accounts-db/src/accounts_db.rs index cf4d17745b1b73..41ec05dce0e4a5 100644 --- a/accounts-db/src/accounts_db.rs +++ b/accounts-db/src/accounts_db.rs @@ -7882,6 +7882,7 @@ impl AccountsDb { /// /// As part of calculating the accounts delta hash, get a list of accounts modified this slot /// (aka dirty pubkeys) and add them to `self.uncleaned_pubkeys` for future cleaning. + #[cfg(feature = "dev-context-only-utils")] pub fn calculate_accounts_delta_hash(&self, slot: Slot) -> AccountsDeltaHash { self.calculate_accounts_delta_hash_internal(slot, None, HashMap::default()) } From 1fcef5171451de0101656d18ada15fcaf4a427a6 Mon Sep 17 00:00:00 2001 From: steviez Date: Fri, 8 Mar 2024 12:52:35 -0600 Subject: [PATCH 49/71] Make ReplayStage create the parallel fork replay threadpool (#137) ReplayStage owning the pool allows for subsequent work to configure the size of the pool; configuring the size of the pool inside of the lazy_static would have been a little messy --- core/src/replay_stage.rs | 136 +++++++++++++++++++++------------------ 1 file changed, 73 insertions(+), 63 deletions(-) diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs index 46014e3f7912de..3683e257ed10a8 100644 --- a/core/src/replay_stage.rs +++ b/core/src/replay_stage.rs @@ -33,7 +33,6 @@ use { window_service::DuplicateSlotReceiver, }, crossbeam_channel::{Receiver, RecvTimeoutError, Sender}, - lazy_static::lazy_static, rayon::{prelude::*, ThreadPool}, solana_entry::entry::VerifyRecyclers, solana_geyser_plugin_manager::block_metadata_notifier_interface::BlockMetadataNotifierArc, @@ -102,14 +101,6 @@ const MAX_VOTE_REFRESH_INTERVAL_MILLIS: usize = 5000; const MAX_CONCURRENT_FORKS_TO_REPLAY: usize = 4; const MAX_REPAIR_RETRY_LOOP_ATTEMPTS: usize = 10; -lazy_static! { - static ref PAR_THREAD_POOL: ThreadPool = rayon::ThreadPoolBuilder::new() - .num_threads(MAX_CONCURRENT_FORKS_TO_REPLAY) - .thread_name(|i| format!("solReplay{i:02}")) - .build() - .unwrap(); -} - #[derive(PartialEq, Eq, Debug)] pub enum HeaviestForkFailures { LockedOut(u64), @@ -131,6 +122,11 @@ pub enum HeaviestForkFailures { ), } +enum ForkReplayMode { + Serial, + Parallel(ThreadPool), +} + #[derive(PartialEq, Eq, Debug)] enum ConfirmationType { SupermajorityVoted, @@ -656,6 +652,16 @@ impl ReplayStage { r_bank_forks.get_vote_only_mode_signal(), ) }; + let replay_mode = if replay_slots_concurrently { + ForkReplayMode::Serial + } else { + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(MAX_CONCURRENT_FORKS_TO_REPLAY) + .thread_name(|i| format!("solReplay{i:02}")) + .build() + .expect("new rayon threadpool"); + ForkReplayMode::Parallel(pool) + }; Self::reset_poh_recorder( &my_pubkey, @@ -717,7 +723,7 @@ impl ReplayStage { block_metadata_notifier.clone(), &mut replay_timing, log_messages_bytes_limit, - replay_slots_concurrently, + &replay_mode, &prioritization_fee_cache, &mut purge_repair_slot_counter, ); @@ -2706,6 +2712,7 @@ impl ReplayStage { fn replay_active_banks_concurrently( blockstore: &Blockstore, bank_forks: &RwLock, + thread_pool: &ThreadPool, my_pubkey: &Pubkey, vote_account: &Pubkey, progress: &mut ProgressMap, @@ -2723,7 +2730,7 @@ impl ReplayStage { let longest_replay_time_us = AtomicU64::new(0); // Allow for concurrent replaying of slots from different forks. - let replay_result_vec: Vec = PAR_THREAD_POOL.install(|| { + let replay_result_vec: Vec = thread_pool.install(|| { active_bank_slots .into_par_iter() .map(|bank_slot| { @@ -2737,7 +2744,7 @@ impl ReplayStage { trace!( "Replay active bank: slot {}, thread_idx {}", bank_slot, - PAR_THREAD_POOL.current_thread_index().unwrap_or_default() + thread_pool.current_thread_index().unwrap_or_default() ); let mut progress_lock = progress.write().unwrap(); if progress_lock @@ -3175,7 +3182,7 @@ impl ReplayStage { block_metadata_notifier: Option, replay_timing: &mut ReplayLoopTiming, log_messages_bytes_limit: Option, - replay_slots_concurrently: bool, + replay_mode: &ForkReplayMode, prioritization_fee_cache: &PrioritizationFeeCache, purge_repair_slot_counter: &mut PurgeRepairSlotCounter, ) -> bool /* completed a bank */ { @@ -3186,11 +3193,17 @@ impl ReplayStage { num_active_banks, active_bank_slots ); - if num_active_banks > 0 { - let replay_result_vec = if num_active_banks > 1 && replay_slots_concurrently { + if active_bank_slots.is_empty() { + return false; + } + + let replay_result_vec = match replay_mode { + // Skip the overhead of the threadpool if there is only one bank to play + ForkReplayMode::Parallel(thread_pool) if num_active_banks > 1 => { Self::replay_active_banks_concurrently( blockstore, bank_forks, + thread_pool, my_pubkey, vote_account, progress, @@ -3203,55 +3216,52 @@ impl ReplayStage { &active_bank_slots, prioritization_fee_cache, ) - } else { - active_bank_slots - .iter() - .map(|bank_slot| { - Self::replay_active_bank( - blockstore, - bank_forks, - my_pubkey, - vote_account, - progress, - transaction_status_sender, - entry_notification_sender, - verify_recyclers, - replay_vote_sender, - replay_timing, - log_messages_bytes_limit, - *bank_slot, - prioritization_fee_cache, - ) - }) - .collect() - }; + } + ForkReplayMode::Serial | ForkReplayMode::Parallel(_) => active_bank_slots + .iter() + .map(|bank_slot| { + Self::replay_active_bank( + blockstore, + bank_forks, + my_pubkey, + vote_account, + progress, + transaction_status_sender, + entry_notification_sender, + verify_recyclers, + replay_vote_sender, + replay_timing, + log_messages_bytes_limit, + *bank_slot, + prioritization_fee_cache, + ) + }) + .collect(), + }; - Self::process_replay_results( - blockstore, - bank_forks, - progress, - transaction_status_sender, - cache_block_meta_sender, - heaviest_subtree_fork_choice, - bank_notification_sender, - rewards_recorder_sender, - rpc_subscriptions, - duplicate_slots_tracker, - duplicate_confirmed_slots, - epoch_slots_frozen_slots, - unfrozen_gossip_verified_vote_hashes, - latest_validator_votes_for_frozen_banks, - cluster_slots_update_sender, - cost_update_sender, - duplicate_slots_to_repair, - ancestor_hashes_replay_update_sender, - block_metadata_notifier, - &replay_result_vec, - purge_repair_slot_counter, - ) - } else { - false - } + Self::process_replay_results( + blockstore, + bank_forks, + progress, + transaction_status_sender, + cache_block_meta_sender, + heaviest_subtree_fork_choice, + bank_notification_sender, + rewards_recorder_sender, + rpc_subscriptions, + duplicate_slots_tracker, + duplicate_confirmed_slots, + epoch_slots_frozen_slots, + unfrozen_gossip_verified_vote_hashes, + latest_validator_votes_for_frozen_banks, + cluster_slots_update_sender, + cost_update_sender, + duplicate_slots_to_repair, + ancestor_hashes_replay_update_sender, + block_metadata_notifier, + &replay_result_vec, + purge_repair_slot_counter, + ) } #[allow(clippy::too_many_arguments)] From 85ccadf35b55c3a3f5ff02d6ba55aa97d133f20e Mon Sep 17 00:00:00 2001 From: Dmitri Makarov Date: Fri, 8 Mar 2024 14:04:07 -0500 Subject: [PATCH 50/71] SVM: Add doc comments, restrict visibility of some xfaces to crate (#136) --- runtime/src/bank.rs | 2 +- svm/src/account_loader.rs | 385 ++++++++++++++++++---- svm/src/account_overrides.rs | 1 + svm/src/account_rent_state.rs | 33 +- svm/src/transaction_account_state_info.rs | 4 +- svm/src/transaction_processor.rs | 231 ++++++------- svm/tests/account_loader.rs | 214 ------------ svm/tests/rent_state.rs | 90 ----- 8 files changed, 463 insertions(+), 497 deletions(-) delete mode 100644 svm/tests/account_loader.rs delete mode 100644 svm/tests/rent_state.rs diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index 3e504d470de744..f0ba75defa0517 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -7499,7 +7499,7 @@ impl Bank { effective_epoch: Epoch, ) -> Arc { self.transaction_processor - .load_program(self, pubkey, reload, effective_epoch) + .load_program_with_pubkey(self, pubkey, reload, effective_epoch) } } diff --git a/svm/src/account_loader.rs b/svm/src/account_loader.rs index 1c02ded24665ff..bf9b5b9c40bfee 100644 --- a/svm/src/account_loader.rs +++ b/svm/src/account_loader.rs @@ -38,8 +38,11 @@ use { }; // for the load instructions -pub type TransactionRent = u64; -pub type TransactionProgramIndices = Vec>; +pub(crate) type TransactionRent = u64; +pub(crate) type TransactionProgramIndices = Vec>; +pub type TransactionCheckResult = (transaction::Result<()>, Option, Option); +pub type TransactionLoadResult = (Result, Option); + #[derive(PartialEq, Eq, Debug, Clone)] pub struct LoadedTransaction { pub accounts: Vec, @@ -48,10 +51,66 @@ pub struct LoadedTransaction { pub rent_debits: RentDebits, } -pub type TransactionLoadResult = (Result, Option); -pub type TransactionCheckResult = (transaction::Result<()>, Option, Option); +/// Check whether the payer_account is capable of paying the fee. The +/// side effect is to subtract the fee amount from the payer_account +/// balance of lamports. If the payer_acount is not able to pay the +/// fee, the error_counters is incremented, and a specific error is +/// returned. +pub fn validate_fee_payer( + payer_address: &Pubkey, + payer_account: &mut AccountSharedData, + payer_index: IndexOfAccount, + error_counters: &mut TransactionErrorMetrics, + rent_collector: &RentCollector, + fee: u64, +) -> Result<()> { + if payer_account.lamports() == 0 { + error_counters.account_not_found += 1; + return Err(TransactionError::AccountNotFound); + } + let system_account_kind = get_system_account_kind(payer_account).ok_or_else(|| { + error_counters.invalid_account_for_fee += 1; + TransactionError::InvalidAccountForFee + })?; + let min_balance = match system_account_kind { + SystemAccountKind::System => 0, + SystemAccountKind::Nonce => { + // Should we ever allow a fees charge to zero a nonce account's + // balance. The state MUST be set to uninitialized in that case + rent_collector.rent.minimum_balance(NonceState::size()) + } + }; + + payer_account + .lamports() + .checked_sub(min_balance) + .and_then(|v| v.checked_sub(fee)) + .ok_or_else(|| { + error_counters.insufficient_funds += 1; + TransactionError::InsufficientFundsForFee + })?; -pub fn load_accounts( + let payer_pre_rent_state = RentState::from_account(payer_account, &rent_collector.rent); + payer_account + .checked_sub_lamports(fee) + .map_err(|_| TransactionError::InsufficientFundsForFee)?; + + let payer_post_rent_state = RentState::from_account(payer_account, &rent_collector.rent); + RentState::check_rent_state_with_account( + &payer_pre_rent_state, + &payer_post_rent_state, + payer_address, + payer_account, + payer_index, + ) +} + +/// Collect information about accounts used in txs transactions and +/// return vector of tuples, one for each transaction in the +/// batch. Each tuple contains struct of information about accounts as +/// its first element and an optional transaction nonce info as its +/// second element. +pub(crate) fn load_accounts( callbacks: &CB, txs: &[SanitizedTransaction], lock_results: &[TransactionCheckResult], @@ -399,55 +458,6 @@ fn accumulate_and_check_loaded_account_data_size( } } -pub fn validate_fee_payer( - payer_address: &Pubkey, - payer_account: &mut AccountSharedData, - payer_index: IndexOfAccount, - error_counters: &mut TransactionErrorMetrics, - rent_collector: &RentCollector, - fee: u64, -) -> Result<()> { - if payer_account.lamports() == 0 { - error_counters.account_not_found += 1; - return Err(TransactionError::AccountNotFound); - } - let system_account_kind = get_system_account_kind(payer_account).ok_or_else(|| { - error_counters.invalid_account_for_fee += 1; - TransactionError::InvalidAccountForFee - })?; - let min_balance = match system_account_kind { - SystemAccountKind::System => 0, - SystemAccountKind::Nonce => { - // Should we ever allow a fees charge to zero a nonce account's - // balance. The state MUST be set to uninitialized in that case - rent_collector.rent.minimum_balance(NonceState::size()) - } - }; - - payer_account - .lamports() - .checked_sub(min_balance) - .and_then(|v| v.checked_sub(fee)) - .ok_or_else(|| { - error_counters.insufficient_funds += 1; - TransactionError::InsufficientFundsForFee - })?; - - let payer_pre_rent_state = RentState::from_account(payer_account, &rent_collector.rent); - payer_account - .checked_sub_lamports(fee) - .map_err(|_| TransactionError::InsufficientFundsForFee)?; - - let payer_post_rent_state = RentState::from_account(payer_account, &rent_collector.rent); - RentState::check_rent_state_with_account( - &payer_pre_rent_state, - &payer_post_rent_state, - payer_address, - payer_account, - payer_index, - ) -} - fn construct_instructions_account(message: &SanitizedMessage) -> AccountSharedData { AccountSharedData::from(Account { data: construct_instructions_data(&message.decompile_instructions()), @@ -460,11 +470,15 @@ fn construct_instructions_account(message: &SanitizedMessage) -> AccountSharedDa mod tests { use { super::*, - crate::transaction_processor::TransactionProcessingCallback, + crate::{ + transaction_account_state_info::TransactionAccountStateInfo, + transaction_processor::TransactionProcessingCallback, + }, nonce::state::Versions as NonceVersions, solana_program_runtime::{ + compute_budget::ComputeBudget, compute_budget_processor, - loaded_programs::LoadedProgram, + loaded_programs::{LoadedProgram, LoadedProgramsForTxBatch}, prioritization_fee::{PrioritizationFeeDetails, PrioritizationFeeType}, }, solana_sdk::{ @@ -473,22 +487,27 @@ mod tests { compute_budget::ComputeBudgetInstruction, epoch_schedule::EpochSchedule, feature_set::FeatureSet, + fee::FeeStructure, hash::Hash, instruction::CompiledInstruction, message::{ v0::{LoadedAddresses, LoadedMessage}, LegacyMessage, Message, MessageHeader, SanitizedMessage, }, + native_loader, + native_token::sol_to_lamports, nonce, + nonce_info::{NonceFull, NoncePartial}, pubkey::Pubkey, rent::Rent, - rent_collector::RentCollector, + rent_collector::{RentCollector, RENT_EXEMPT_RENT_EPOCH}, + rent_debits::RentDebits, signature::{Keypair, Signature, Signer}, - system_program, sysvar, - transaction::{Result, Transaction, TransactionError}, - transaction_context::TransactionAccount, + system_program, system_transaction, sysvar, + transaction::{Result, SanitizedTransaction, Transaction, TransactionError}, + transaction_context::{TransactionAccount, TransactionContext}, }, - std::{borrow::Cow, convert::TryFrom, sync::Arc}, + std::{borrow::Cow, collections::HashMap, convert::TryFrom, sync::Arc}, }; #[derive(Default)] @@ -2017,4 +2036,248 @@ mod tests { } ); } + + #[test] + fn test_rent_state_list_len() { + let mint_keypair = Keypair::new(); + let mut bank = TestCallbacks::default(); + let recipient = Pubkey::new_unique(); + let last_block_hash = Hash::new_unique(); + + let mut system_data = AccountSharedData::default(); + system_data.set_executable(true); + system_data.set_owner(native_loader::id()); + bank.accounts_map + .insert(Pubkey::new_from_array([0u8; 32]), system_data); + + let mut mint_data = AccountSharedData::default(); + mint_data.set_lamports(2); + bank.accounts_map.insert(mint_keypair.pubkey(), mint_data); + + bank.accounts_map + .insert(recipient, AccountSharedData::default()); + + let tx = system_transaction::transfer( + &mint_keypair, + &recipient, + sol_to_lamports(1.), + last_block_hash, + ); + let num_accounts = tx.message().account_keys.len(); + let sanitized_tx = SanitizedTransaction::from_transaction_for_tests(tx); + let mut error_counters = TransactionErrorMetrics::default(); + let loaded_txs = load_accounts( + &bank, + &[sanitized_tx.clone()], + &[(Ok(()), None, Some(0))], + &mut error_counters, + &FeeStructure::default(), + None, + &HashMap::new(), + &LoadedProgramsForTxBatch::default(), + ); + + let compute_budget = ComputeBudget::new(u64::from( + compute_budget_processor::DEFAULT_INSTRUCTION_COMPUTE_UNIT_LIMIT, + )); + let transaction_context = TransactionContext::new( + loaded_txs[0].0.as_ref().unwrap().accounts.clone(), + Rent::default(), + compute_budget.max_invoke_stack_height, + compute_budget.max_instruction_trace_length, + ); + + assert_eq!( + TransactionAccountStateInfo::new( + &Rent::default(), + &transaction_context, + sanitized_tx.message() + ) + .len(), + num_accounts, + ); + } + + #[test] + fn test_load_accounts_success() { + let key1 = Keypair::new(); + let key2 = Keypair::new(); + let key3 = Keypair::new(); + let key4 = Keypair::new(); + + let message = Message { + account_keys: vec![key2.pubkey(), key1.pubkey(), key4.pubkey()], + header: MessageHeader::default(), + instructions: vec![ + CompiledInstruction { + program_id_index: 1, + accounts: vec![0], + data: vec![], + }, + CompiledInstruction { + program_id_index: 1, + accounts: vec![2], + data: vec![], + }, + ], + recent_blockhash: Hash::default(), + }; + + let legacy = LegacyMessage::new(message); + let sanitized_message = SanitizedMessage::Legacy(legacy); + let mut mock_bank = TestCallbacks::default(); + let mut account_data = AccountSharedData::default(); + account_data.set_executable(true); + account_data.set_owner(key3.pubkey()); + mock_bank.accounts_map.insert(key1.pubkey(), account_data); + + let mut account_data = AccountSharedData::default(); + account_data.set_lamports(200); + mock_bank.accounts_map.insert(key2.pubkey(), account_data); + + let mut account_data = AccountSharedData::default(); + account_data.set_executable(true); + account_data.set_owner(native_loader::id()); + mock_bank.accounts_map.insert(key3.pubkey(), account_data); + + let mut error_counter = TransactionErrorMetrics::default(); + let loaded_programs = LoadedProgramsForTxBatch::default(); + + let sanitized_transaction = SanitizedTransaction::new_for_tests( + sanitized_message, + vec![Signature::new_unique()], + false, + ); + let lock_results = + (Ok(()), Some(NoncePartial::default()), Some(20u64)) as TransactionCheckResult; + + let results = load_accounts( + &mock_bank, + &[sanitized_transaction], + &[lock_results], + &mut error_counter, + &FeeStructure::default(), + None, + &HashMap::new(), + &loaded_programs, + ); + + let mut account_data = AccountSharedData::default(); + account_data.set_rent_epoch(RENT_EXEMPT_RENT_EPOCH); + + assert_eq!(results.len(), 1); + let (loaded_result, nonce) = results[0].clone(); + assert_eq!( + loaded_result.unwrap(), + LoadedTransaction { + accounts: vec![ + ( + key2.pubkey(), + mock_bank.accounts_map[&key2.pubkey()].clone() + ), + ( + key1.pubkey(), + mock_bank.accounts_map[&key1.pubkey()].clone() + ), + (key4.pubkey(), account_data), + ( + key3.pubkey(), + mock_bank.accounts_map[&key3.pubkey()].clone() + ), + ], + program_indices: vec![vec![3, 1], vec![3, 1]], + rent: 0, + rent_debits: RentDebits::default() + } + ); + + assert_eq!( + nonce.unwrap(), + NonceFull::new( + Pubkey::from([0; 32]), + AccountSharedData::default(), + Some(mock_bank.accounts_map[&key2.pubkey()].clone()) + ) + ); + } + + #[test] + fn test_load_accounts_error() { + let mock_bank = TestCallbacks::default(); + let message = Message { + account_keys: vec![Pubkey::new_from_array([0; 32])], + header: MessageHeader::default(), + instructions: vec![CompiledInstruction { + program_id_index: 0, + accounts: vec![], + data: vec![], + }], + recent_blockhash: Hash::default(), + }; + + let legacy = LegacyMessage::new(message); + let sanitized_message = SanitizedMessage::Legacy(legacy); + let sanitized_transaction = SanitizedTransaction::new_for_tests( + sanitized_message, + vec![Signature::new_unique()], + false, + ); + + let lock_results = (Ok(()), Some(NoncePartial::default()), None) as TransactionCheckResult; + let fee_structure = FeeStructure::default(); + + let result = load_accounts( + &mock_bank, + &[sanitized_transaction.clone()], + &[lock_results], + &mut TransactionErrorMetrics::default(), + &fee_structure, + None, + &HashMap::new(), + &LoadedProgramsForTxBatch::default(), + ); + + assert_eq!( + result, + vec![(Err(TransactionError::BlockhashNotFound), None)] + ); + + let lock_results = + (Ok(()), Some(NoncePartial::default()), Some(20u64)) as TransactionCheckResult; + + let result = load_accounts( + &mock_bank, + &[sanitized_transaction.clone()], + &[lock_results.clone()], + &mut TransactionErrorMetrics::default(), + &fee_structure, + None, + &HashMap::new(), + &LoadedProgramsForTxBatch::default(), + ); + + assert_eq!(result, vec![(Err(TransactionError::AccountNotFound), None)]); + + let lock_results = ( + Err(TransactionError::InvalidWritableAccount), + Some(NoncePartial::default()), + Some(20u64), + ) as TransactionCheckResult; + + let result = load_accounts( + &mock_bank, + &[sanitized_transaction.clone()], + &[lock_results], + &mut TransactionErrorMetrics::default(), + &fee_structure, + None, + &HashMap::new(), + &LoadedProgramsForTxBatch::default(), + ); + + assert_eq!( + result, + vec![(Err(TransactionError::InvalidWritableAccount), None)] + ); + } } diff --git a/svm/src/account_overrides.rs b/svm/src/account_overrides.rs index c88d77d54f30a9..8a205a798f66b1 100644 --- a/svm/src/account_overrides.rs +++ b/svm/src/account_overrides.rs @@ -10,6 +10,7 @@ pub struct AccountOverrides { } impl AccountOverrides { + /// Insert or remove an account with a given pubkey to/from the list of overrides. pub fn set_account(&mut self, pubkey: &Pubkey, account: Option) { match account { Some(account) => self.accounts.insert(*pubkey, account), diff --git a/svm/src/account_rent_state.rs b/svm/src/account_rent_state.rs index 6fae6e9033bd39..7e3501d0d6c649 100644 --- a/svm/src/account_rent_state.rs +++ b/svm/src/account_rent_state.rs @@ -23,6 +23,7 @@ pub enum RentState { } impl RentState { + /// Return a new RentState instance for a given account and rent. pub fn from_account(account: &AccountSharedData, rent: &Rent) -> Self { if account.lamports() == 0 { Self::Uninitialized @@ -36,6 +37,8 @@ impl RentState { } } + /// Check whether a transition from the pre_rent_state to this + /// state is valid. pub fn transition_allowed_from(&self, pre_rent_state: &RentState) -> bool { match self { Self::Uninitialized | Self::RentExempt => true, @@ -57,21 +60,6 @@ impl RentState { } } - fn submit_rent_state_metrics(pre_rent_state: &Self, post_rent_state: &Self) { - match (pre_rent_state, post_rent_state) { - (&RentState::Uninitialized, &RentState::RentPaying { .. }) => { - inc_new_counter_info!("rent_paying_err-new_account", 1); - } - (&RentState::RentPaying { .. }, &RentState::RentPaying { .. }) => { - inc_new_counter_info!("rent_paying_ok-legacy", 1); - } - (_, &RentState::RentPaying { .. }) => { - inc_new_counter_info!("rent_paying_err-other", 1); - } - _ => {} - } - } - pub(crate) fn check_rent_state( pre_rent_state: Option<&Self>, post_rent_state: Option<&Self>, @@ -118,6 +106,21 @@ impl RentState { Ok(()) } } + + fn submit_rent_state_metrics(pre_rent_state: &Self, post_rent_state: &Self) { + match (pre_rent_state, post_rent_state) { + (&RentState::Uninitialized, &RentState::RentPaying { .. }) => { + inc_new_counter_info!("rent_paying_err-new_account", 1); + } + (&RentState::RentPaying { .. }, &RentState::RentPaying { .. }) => { + inc_new_counter_info!("rent_paying_ok-legacy", 1); + } + (_, &RentState::RentPaying { .. }) => { + inc_new_counter_info!("rent_paying_err-other", 1); + } + _ => {} + } + } } #[cfg(test)] diff --git a/svm/src/transaction_account_state_info.rs b/svm/src/transaction_account_state_info.rs index ff5b93f6a6c459..0631050fe0e765 100644 --- a/svm/src/transaction_account_state_info.rs +++ b/svm/src/transaction_account_state_info.rs @@ -11,12 +11,12 @@ use { }; #[derive(PartialEq, Debug)] -pub struct TransactionAccountStateInfo { +pub(crate) struct TransactionAccountStateInfo { rent_state: Option, // None: readonly account } impl TransactionAccountStateInfo { - pub fn new( + pub(crate) fn new( rent: &Rent, transaction_context: &TransactionContext, message: &SanitizedMessage, diff --git a/svm/src/transaction_processor.rs b/svm/src/transaction_processor.rs index d90afb0a428ea3..fec908619f14f8 100644 --- a/svm/src/transaction_processor.rs +++ b/svm/src/transaction_processor.rs @@ -195,6 +195,7 @@ impl TransactionBatchProcessor { } } + /// Main entrypoint to the SVM. #[allow(clippy::too_many_arguments)] pub fn load_and_execute_sanitized_transactions<'a, CB: TransactionProcessingCallback>( &self, @@ -377,6 +378,112 @@ impl TransactionBatchProcessor { result } + /// Load program with a specific pubkey from loaded programs + /// cache, and update the program's access slot as a side-effect. + pub fn load_program_with_pubkey( + &self, + callbacks: &CB, + pubkey: &Pubkey, + reload: bool, + effective_epoch: Epoch, + ) -> Arc { + let loaded_programs_cache = self.loaded_programs_cache.read().unwrap(); + let environments = loaded_programs_cache.get_environments_for_epoch(effective_epoch); + let mut load_program_metrics = LoadProgramMetrics { + program_id: pubkey.to_string(), + ..LoadProgramMetrics::default() + }; + + let mut loaded_program = + match self.load_program_accounts(callbacks, pubkey, environments) { + ProgramAccountLoadResult::AccountNotFound => Ok(LoadedProgram::new_tombstone( + self.slot, + LoadedProgramType::Closed, + )), + + ProgramAccountLoadResult::InvalidAccountData(env) => Err((self.slot, env)), + + ProgramAccountLoadResult::ProgramOfLoaderV1orV2(program_account) => { + Self::load_program_from_bytes( + &mut load_program_metrics, + program_account.data(), + program_account.owner(), + program_account.data().len(), + 0, + environments.program_runtime_v1.clone(), + reload, + ) + .map_err(|_| (0, environments.program_runtime_v1.clone())) + } + + ProgramAccountLoadResult::ProgramOfLoaderV3( + program_account, + programdata_account, + slot, + ) => programdata_account + .data() + .get(UpgradeableLoaderState::size_of_programdata_metadata()..) + .ok_or(Box::new(InstructionError::InvalidAccountData).into()) + .and_then(|programdata| { + Self::load_program_from_bytes( + &mut load_program_metrics, + programdata, + program_account.owner(), + program_account + .data() + .len() + .saturating_add(programdata_account.data().len()), + slot, + environments.program_runtime_v1.clone(), + reload, + ) + }) + .map_err(|_| (slot, environments.program_runtime_v1.clone())), + + ProgramAccountLoadResult::ProgramOfLoaderV4(program_account, slot) => { + program_account + .data() + .get(LoaderV4State::program_data_offset()..) + .ok_or(Box::new(InstructionError::InvalidAccountData).into()) + .and_then(|elf_bytes| { + Self::load_program_from_bytes( + &mut load_program_metrics, + elf_bytes, + &loader_v4::id(), + program_account.data().len(), + slot, + environments.program_runtime_v2.clone(), + reload, + ) + }) + .map_err(|_| (slot, environments.program_runtime_v2.clone())) + } + } + .unwrap_or_else(|(slot, env)| { + LoadedProgram::new_tombstone(slot, LoadedProgramType::FailedVerification(env)) + }); + + let mut timings = ExecuteDetailsTimings::default(); + load_program_metrics.submit_datapoint(&mut timings); + if !Arc::ptr_eq( + &environments.program_runtime_v1, + &loaded_programs_cache.environments.program_runtime_v1, + ) || !Arc::ptr_eq( + &environments.program_runtime_v2, + &loaded_programs_cache.environments.program_runtime_v2, + ) { + // There can be two entries per program when the environment changes. + // One for the old environment before the epoch boundary and one for the new environment after the epoch boundary. + // These two entries have the same deployment slot, so they must differ in their effective slot instead. + // This is done by setting the effective slot of the entry for the new environment to the epoch boundary. + loaded_program.effective_slot = loaded_program + .effective_slot + .max(self.epoch_schedule.get_first_slot_in_epoch(effective_epoch)); + } + loaded_program.update_access_slot(self.slot); + Arc::new(loaded_program) + } + fn replenish_program_cache( &self, callback: &CB, @@ -454,7 +561,7 @@ impl TransactionBatchProcessor { if let Some((key, count)) = program_to_load { // Load, verify and compile one program. - let program = self.load_program(callback, &key, false, self.epoch); + let program = self.load_program_with_pubkey(callback, &key, false, self.epoch); program.tx_usage_counter.store(count, Ordering::Relaxed); program_to_store = Some((key, program)); } else if missing_programs.is_empty() { @@ -683,110 +790,6 @@ impl TransactionBatchProcessor { } } - pub fn load_program( - &self, - callbacks: &CB, - pubkey: &Pubkey, - reload: bool, - effective_epoch: Epoch, - ) -> Arc { - let loaded_programs_cache = self.loaded_programs_cache.read().unwrap(); - let environments = loaded_programs_cache.get_environments_for_epoch(effective_epoch); - let mut load_program_metrics = LoadProgramMetrics { - program_id: pubkey.to_string(), - ..LoadProgramMetrics::default() - }; - - let mut loaded_program = - match self.load_program_accounts(callbacks, pubkey, environments) { - ProgramAccountLoadResult::AccountNotFound => Ok(LoadedProgram::new_tombstone( - self.slot, - LoadedProgramType::Closed, - )), - - ProgramAccountLoadResult::InvalidAccountData(env) => Err((self.slot, env)), - - ProgramAccountLoadResult::ProgramOfLoaderV1orV2(program_account) => { - Self::load_program_from_bytes( - &mut load_program_metrics, - program_account.data(), - program_account.owner(), - program_account.data().len(), - 0, - environments.program_runtime_v1.clone(), - reload, - ) - .map_err(|_| (0, environments.program_runtime_v1.clone())) - } - - ProgramAccountLoadResult::ProgramOfLoaderV3( - program_account, - programdata_account, - slot, - ) => programdata_account - .data() - .get(UpgradeableLoaderState::size_of_programdata_metadata()..) - .ok_or(Box::new(InstructionError::InvalidAccountData).into()) - .and_then(|programdata| { - Self::load_program_from_bytes( - &mut load_program_metrics, - programdata, - program_account.owner(), - program_account - .data() - .len() - .saturating_add(programdata_account.data().len()), - slot, - environments.program_runtime_v1.clone(), - reload, - ) - }) - .map_err(|_| (slot, environments.program_runtime_v1.clone())), - - ProgramAccountLoadResult::ProgramOfLoaderV4(program_account, slot) => { - program_account - .data() - .get(LoaderV4State::program_data_offset()..) - .ok_or(Box::new(InstructionError::InvalidAccountData).into()) - .and_then(|elf_bytes| { - Self::load_program_from_bytes( - &mut load_program_metrics, - elf_bytes, - &loader_v4::id(), - program_account.data().len(), - slot, - environments.program_runtime_v2.clone(), - reload, - ) - }) - .map_err(|_| (slot, environments.program_runtime_v2.clone())) - } - } - .unwrap_or_else(|(slot, env)| { - LoadedProgram::new_tombstone(slot, LoadedProgramType::FailedVerification(env)) - }); - - let mut timings = ExecuteDetailsTimings::default(); - load_program_metrics.submit_datapoint(&mut timings); - if !Arc::ptr_eq( - &environments.program_runtime_v1, - &loaded_programs_cache.environments.program_runtime_v1, - ) || !Arc::ptr_eq( - &environments.program_runtime_v2, - &loaded_programs_cache.environments.program_runtime_v2, - ) { - // There can be two entries per program when the environment changes. - // One for the old environment before the epoch boundary and one for the new environment after the epoch boundary. - // These two entries have the same deployment slot, so they must differ in their effective slot instead. - // This is done by setting the effective slot of the entry for the new environment to the epoch boundary. - loaded_program.effective_slot = loaded_program - .effective_slot - .max(self.epoch_schedule.get_first_slot_in_epoch(effective_epoch)); - } - loaded_program.update_access_slot(self.slot); - Arc::new(loaded_program) - } - fn load_program_from_bytes( load_program_metrics: &mut LoadProgramMetrics, programdata: &[u8], @@ -1242,7 +1245,7 @@ mod tests { let key = Pubkey::new_unique(); let batch_processor = TransactionBatchProcessor::::default(); - let result = batch_processor.load_program(&mock_bank, &key, false, 50); + let result = batch_processor.load_program_with_pubkey(&mock_bank, &key, false, 50); let loaded_program = LoadedProgram::new_tombstone(0, LoadedProgramType::Closed); assert_eq!(result, Arc::new(loaded_program)); @@ -1259,7 +1262,7 @@ mod tests { .account_shared_data .insert(key, account_data.clone()); - let result = batch_processor.load_program(&mock_bank, &key, false, 20); + let result = batch_processor.load_program_with_pubkey(&mock_bank, &key, false, 20); let loaded_program = LoadedProgram::new_tombstone( 0, @@ -1288,7 +1291,7 @@ mod tests { .insert(key, account_data.clone()); // This should return an error - let result = batch_processor.load_program(&mock_bank, &key, false, 20); + let result = batch_processor.load_program_with_pubkey(&mock_bank, &key, false, 20); let loaded_program = LoadedProgram::new_tombstone( 0, LoadedProgramType::FailedVerification( @@ -1316,7 +1319,7 @@ mod tests { .account_shared_data .insert(key, account_data.clone()); - let result = batch_processor.load_program(&mock_bank, &key, false, 20); + let result = batch_processor.load_program_with_pubkey(&mock_bank, &key, false, 20); let environments = ProgramRuntimeEnvironments::default(); let expected = TransactionBatchProcessor::::load_program_from_bytes( @@ -1361,7 +1364,7 @@ mod tests { .insert(key2, account_data2.clone()); // This should return an error - let result = batch_processor.load_program(&mock_bank, &key1, false, 0); + let result = batch_processor.load_program_with_pubkey(&mock_bank, &key1, false, 0); let loaded_program = LoadedProgram::new_tombstone( 0, LoadedProgramType::FailedVerification( @@ -1399,7 +1402,7 @@ mod tests { .account_shared_data .insert(key2, account_data.clone()); - let result = batch_processor.load_program(&mock_bank, &key1, false, 20); + let result = batch_processor.load_program_with_pubkey(&mock_bank, &key1, false, 20); let data = account_data.data(); account_data @@ -1441,7 +1444,7 @@ mod tests { .account_shared_data .insert(key, account_data.clone()); - let result = batch_processor.load_program(&mock_bank, &key, false, 0); + let result = batch_processor.load_program_with_pubkey(&mock_bank, &key, false, 0); let loaded_program = LoadedProgram::new_tombstone( 0, LoadedProgramType::FailedVerification( @@ -1475,7 +1478,7 @@ mod tests { .account_shared_data .insert(key, account_data.clone()); - let result = batch_processor.load_program(&mock_bank, &key, false, 20); + let result = batch_processor.load_program_with_pubkey(&mock_bank, &key, false, 20); let data = account_data.data()[LoaderV4State::program_data_offset()..].to_vec(); account_data.set_data(data); @@ -1513,7 +1516,7 @@ mod tests { .account_shared_data .insert(key, account_data.clone()); - let result = batch_processor.load_program(&mock_bank, &key, false, 20); + let result = batch_processor.load_program_with_pubkey(&mock_bank, &key, false, 20); let slot = batch_processor.epoch_schedule.get_first_slot_in_epoch(20); assert_eq!(result.effective_slot, slot); diff --git a/svm/tests/account_loader.rs b/svm/tests/account_loader.rs deleted file mode 100644 index dd4cd046046399..00000000000000 --- a/svm/tests/account_loader.rs +++ /dev/null @@ -1,214 +0,0 @@ -use { - crate::mock_bank::MockBankCallback, - solana_program_runtime::loaded_programs::LoadedProgramsForTxBatch, - solana_sdk::{ - account::{AccountSharedData, WritableAccount}, - fee::FeeStructure, - hash::Hash, - instruction::CompiledInstruction, - message::{LegacyMessage, Message, MessageHeader, SanitizedMessage}, - native_loader, - nonce_info::{NonceFull, NoncePartial}, - pubkey::Pubkey, - rent_collector::RENT_EXEMPT_RENT_EPOCH, - rent_debits::RentDebits, - signature::{Keypair, Signature, Signer}, - transaction::{SanitizedTransaction, TransactionError}, - }, - solana_svm::{ - account_loader::{load_accounts, LoadedTransaction, TransactionCheckResult}, - transaction_error_metrics::TransactionErrorMetrics, - }, - std::collections::HashMap, -}; - -mod mock_bank; - -#[test] -fn test_load_accounts_success() { - let key1 = Keypair::new(); - let key2 = Keypair::new(); - let key3 = Keypair::new(); - let key4 = Keypair::new(); - - let message = Message { - account_keys: vec![key2.pubkey(), key1.pubkey(), key4.pubkey()], - header: MessageHeader::default(), - instructions: vec![ - CompiledInstruction { - program_id_index: 1, - accounts: vec![0], - data: vec![], - }, - CompiledInstruction { - program_id_index: 1, - accounts: vec![2], - data: vec![], - }, - ], - recent_blockhash: Hash::default(), - }; - - let legacy = LegacyMessage::new(message); - let sanitized_message = SanitizedMessage::Legacy(legacy); - let mut mock_bank = MockBankCallback::default(); - let mut account_data = AccountSharedData::default(); - account_data.set_executable(true); - account_data.set_owner(key3.pubkey()); - mock_bank - .account_shared_data - .insert(key1.pubkey(), account_data); - - let mut account_data = AccountSharedData::default(); - account_data.set_lamports(200); - mock_bank - .account_shared_data - .insert(key2.pubkey(), account_data); - - let mut account_data = AccountSharedData::default(); - account_data.set_executable(true); - account_data.set_owner(native_loader::id()); - mock_bank - .account_shared_data - .insert(key3.pubkey(), account_data); - - let mut error_counter = TransactionErrorMetrics::default(); - let loaded_programs = LoadedProgramsForTxBatch::default(); - - let sanitized_transaction = SanitizedTransaction::new_for_tests( - sanitized_message, - vec![Signature::new_unique()], - false, - ); - let lock_results = - (Ok(()), Some(NoncePartial::default()), Some(20u64)) as TransactionCheckResult; - - let results = load_accounts( - &mock_bank, - &[sanitized_transaction], - &[lock_results], - &mut error_counter, - &FeeStructure::default(), - None, - &HashMap::new(), - &loaded_programs, - ); - - let mut account_data = AccountSharedData::default(); - account_data.set_rent_epoch(RENT_EXEMPT_RENT_EPOCH); - - assert_eq!(results.len(), 1); - let (loaded_result, nonce) = results[0].clone(); - assert_eq!( - loaded_result.unwrap(), - LoadedTransaction { - accounts: vec![ - ( - key2.pubkey(), - mock_bank.account_shared_data[&key2.pubkey()].clone() - ), - ( - key1.pubkey(), - mock_bank.account_shared_data[&key1.pubkey()].clone() - ), - (key4.pubkey(), account_data), - ( - key3.pubkey(), - mock_bank.account_shared_data[&key3.pubkey()].clone() - ), - ], - program_indices: vec![vec![3, 1], vec![3, 1]], - rent: 0, - rent_debits: RentDebits::default() - } - ); - - assert_eq!( - nonce.unwrap(), - NonceFull::new( - Pubkey::from([0; 32]), - AccountSharedData::default(), - Some(mock_bank.account_shared_data[&key2.pubkey()].clone()) - ) - ); -} - -#[test] -fn test_load_accounts_error() { - let mock_bank = MockBankCallback::default(); - let message = Message { - account_keys: vec![Pubkey::new_from_array([0; 32])], - header: MessageHeader::default(), - instructions: vec![CompiledInstruction { - program_id_index: 0, - accounts: vec![], - data: vec![], - }], - recent_blockhash: Hash::default(), - }; - - let legacy = LegacyMessage::new(message); - let sanitized_message = SanitizedMessage::Legacy(legacy); - let sanitized_transaction = SanitizedTransaction::new_for_tests( - sanitized_message, - vec![Signature::new_unique()], - false, - ); - - let lock_results = (Ok(()), Some(NoncePartial::default()), None) as TransactionCheckResult; - let fee_structure = FeeStructure::default(); - - let result = load_accounts( - &mock_bank, - &[sanitized_transaction.clone()], - &[lock_results], - &mut TransactionErrorMetrics::default(), - &fee_structure, - None, - &HashMap::new(), - &LoadedProgramsForTxBatch::default(), - ); - - assert_eq!( - result, - vec![(Err(TransactionError::BlockhashNotFound), None)] - ); - - let lock_results = - (Ok(()), Some(NoncePartial::default()), Some(20u64)) as TransactionCheckResult; - - let result = load_accounts( - &mock_bank, - &[sanitized_transaction.clone()], - &[lock_results.clone()], - &mut TransactionErrorMetrics::default(), - &fee_structure, - None, - &HashMap::new(), - &LoadedProgramsForTxBatch::default(), - ); - - assert_eq!(result, vec![(Err(TransactionError::AccountNotFound), None)]); - - let lock_results = ( - Err(TransactionError::InvalidWritableAccount), - Some(NoncePartial::default()), - Some(20u64), - ) as TransactionCheckResult; - - let result = load_accounts( - &mock_bank, - &[sanitized_transaction.clone()], - &[lock_results], - &mut TransactionErrorMetrics::default(), - &fee_structure, - None, - &HashMap::new(), - &LoadedProgramsForTxBatch::default(), - ); - - assert_eq!( - result, - vec![(Err(TransactionError::InvalidWritableAccount), None)] - ); -} diff --git a/svm/tests/rent_state.rs b/svm/tests/rent_state.rs deleted file mode 100644 index f3ea728f6b874f..00000000000000 --- a/svm/tests/rent_state.rs +++ /dev/null @@ -1,90 +0,0 @@ -#![cfg(test)] - -use { - solana_program_runtime::{ - compute_budget::ComputeBudget, compute_budget_processor, - loaded_programs::LoadedProgramsForTxBatch, - }, - solana_sdk::{ - account::{AccountSharedData, WritableAccount}, - fee::FeeStructure, - hash::Hash, - native_loader, - native_token::sol_to_lamports, - pubkey::Pubkey, - rent::Rent, - signature::{Keypair, Signer}, - system_transaction, - transaction::SanitizedTransaction, - transaction_context::TransactionContext, - }, - solana_svm::{ - account_loader::load_accounts, transaction_account_state_info::TransactionAccountStateInfo, - transaction_error_metrics::TransactionErrorMetrics, - }, - std::collections::HashMap, -}; - -mod mock_bank; - -#[test] -fn test_rent_state_list_len() { - let mint_keypair = Keypair::new(); - let mut bank = mock_bank::MockBankCallback::default(); - let recipient = Pubkey::new_unique(); - let last_block_hash = Hash::new_unique(); - - let mut system_data = AccountSharedData::default(); - system_data.set_executable(true); - system_data.set_owner(native_loader::id()); - bank.account_shared_data - .insert(Pubkey::new_from_array([0u8; 32]), system_data); - - let mut mint_data = AccountSharedData::default(); - mint_data.set_lamports(2); - bank.account_shared_data - .insert(mint_keypair.pubkey(), mint_data); - - bank.account_shared_data - .insert(recipient, AccountSharedData::default()); - - let tx = system_transaction::transfer( - &mint_keypair, - &recipient, - sol_to_lamports(1.), - last_block_hash, - ); - let num_accounts = tx.message().account_keys.len(); - let sanitized_tx = SanitizedTransaction::from_transaction_for_tests(tx); - let mut error_counters = TransactionErrorMetrics::default(); - let loaded_txs = load_accounts( - &bank, - &[sanitized_tx.clone()], - &[(Ok(()), None, Some(0))], - &mut error_counters, - &FeeStructure::default(), - None, - &HashMap::new(), - &LoadedProgramsForTxBatch::default(), - ); - - let compute_budget = ComputeBudget::new(u64::from( - compute_budget_processor::DEFAULT_INSTRUCTION_COMPUTE_UNIT_LIMIT, - )); - let transaction_context = TransactionContext::new( - loaded_txs[0].0.as_ref().unwrap().accounts.clone(), - Rent::default(), - compute_budget.max_invoke_stack_height, - compute_budget.max_instruction_trace_length, - ); - - assert_eq!( - TransactionAccountStateInfo::new( - &Rent::default(), - &transaction_context, - sanitized_tx.message() - ) - .len(), - num_accounts, - ); -} From f11d0530c387bc08ec14b90b3d1d91dcfb1b872b Mon Sep 17 00:00:00 2001 From: Lijun Wang <83639177+lijunwangs@users.noreply.github.com> Date: Fri, 5 Jan 2024 18:23:33 -0800 Subject: [PATCH 51/71] output txn signature for debug purpose trying txn mask matching output txn to figure out why txn is not exactly matched Use 62 and 61 portion track fetch performance using random txn mask track sigverify performance using random txn mask track banking stage performance using random txn mask adding missing cargo lock file add debug messages Revert "add debug messages" This reverts commit 96aead5cbc4acb5b2fc9d8a37fcd506c73ddf552. fixed some clippy issues check-crate issue Fix a clippy issue Fix a clippy issue debug why txns in banking stage shows fewer performance tracking points debug why txns in banking stage shows fewer performance tracking points debug why txns in banking stage shows fewer performance tracking points debug why txns in banking stage shows fewer performance tracking points get higher PPS for testing purpose more debug messages on why txn is skipped display if tracer packet in log add debug before calling processing_function debug at the initial of banking stage track if a txn is forwarded dependency order missing cargo file clean up debug messages Do not use TRACER_PACKET, use its own bit rename some functions addressed some comments from Trent Update core/src/banking_stage/immutable_deserialized_packet.rs Co-authored-by: Trent Nelson addressed some comments from Trent Do not use binary_search, do simple compare in one loop --- Cargo.lock | 16 ++++++ Cargo.toml | 2 + core/Cargo.toml | 1 + core/src/banking_stage/consumer.rs | 27 ++++++++++ .../immutable_deserialized_packet.rs | 13 ++++- .../unprocessed_transaction_storage.rs | 1 + core/src/sigverify_stage.rs | 29 +++++++++- programs/sbf/Cargo.lock | 16 ++++++ sdk/src/packet.rs | 13 +++++ sdk/src/transaction/versioned/sanitized.rs | 4 ++ streamer/Cargo.toml | 1 + streamer/src/nonblocking/quic.rs | 42 ++++++++++++++- transaction-metrics-tracker/Cargo.toml | 25 +++++++++ transaction-metrics-tracker/src/lib.rs | 54 +++++++++++++++++++ 14 files changed, 240 insertions(+), 4 deletions(-) create mode 100644 transaction-metrics-tracker/Cargo.toml create mode 100644 transaction-metrics-tracker/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 17e6d5e3a82f04..823572dbc95441 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5830,6 +5830,7 @@ dependencies = [ "solana-streamer", "solana-svm", "solana-tpu-client", + "solana-transaction-metrics-tracker", "solana-transaction-status", "solana-turbine", "solana-unified-scheduler-pool", @@ -7194,6 +7195,7 @@ dependencies = [ "solana-metrics", "solana-perf", "solana-sdk", + "solana-transaction-metrics-tracker", "thiserror", "tokio", "x509-parser", @@ -7360,6 +7362,20 @@ dependencies = [ "solana-version", ] +[[package]] +name = "solana-transaction-metrics-tracker" +version = "1.19.0" +dependencies = [ + "Inflector", + "base64 0.21.7", + "bincode", + "lazy_static", + "log", + "rand 0.8.5", + "solana-perf", + "solana-sdk", +] + [[package]] name = "solana-transaction-status" version = "1.19.0" diff --git a/Cargo.toml b/Cargo.toml index e227ba5ff1578f..44869153632241 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -106,6 +106,7 @@ members = [ "tokens", "tpu-client", "transaction-dos", + "transaction-metrics-tracker", "transaction-status", "turbine", "udp-client", @@ -377,6 +378,7 @@ solana-system-program = { path = "programs/system", version = "=1.19.0" } solana-test-validator = { path = "test-validator", version = "=1.19.0" } solana-thin-client = { path = "thin-client", version = "=1.19.0" } solana-tpu-client = { path = "tpu-client", version = "=1.19.0", default-features = false } +solana-transaction-metrics-tracker = { path = "transaction-metrics-tracker", version = "=1.19.0" } solana-transaction-status = { path = "transaction-status", version = "=1.19.0" } solana-turbine = { path = "turbine", version = "=1.19.0" } solana-udp-client = { path = "udp-client", version = "=1.19.0" } diff --git a/core/Cargo.toml b/core/Cargo.toml index e2a936cdabc4c1..1fd25ec38a8d3b 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -67,6 +67,7 @@ solana-send-transaction-service = { workspace = true } solana-streamer = { workspace = true } solana-svm = { workspace = true } solana-tpu-client = { workspace = true } +solana-transaction-metrics-tracker = { workspace = true } solana-transaction-status = { workspace = true } solana-turbine = { workspace = true } solana-unified-scheduler-pool = { workspace = true } diff --git a/core/src/banking_stage/consumer.rs b/core/src/banking_stage/consumer.rs index c5ed22a34278ce..f417363c7e4409 100644 --- a/core/src/banking_stage/consumer.rs +++ b/core/src/banking_stage/consumer.rs @@ -209,6 +209,33 @@ impl Consumer { .slot_metrics_tracker .increment_retryable_packets_count(retryable_transaction_indexes.len() as u64); + // Now we track the performance for the interested transactions which is not in the retryable_transaction_indexes + // We assume the retryable_transaction_indexes is already sorted. + let mut retryable_idx = 0; + for (index, packet) in packets_to_process.iter().enumerate() { + if packet.original_packet().meta().is_perf_track_packet() { + if let Some(start_time) = packet.start_time() { + if retryable_idx >= retryable_transaction_indexes.len() + || retryable_transaction_indexes[retryable_idx] != index + { + let duration = Instant::now().duration_since(*start_time); + + debug!( + "Banking stage processing took {duration:?} for transaction {:?}", + packet.transaction().get_signatures().first() + ); + inc_new_counter_info!( + "txn-metrics-banking-stage-process-us", + duration.as_micros() as usize + ); + } else { + // This packet is retried, advance the retry index to the next, as the next packet's index will + // certainly be > than this. + retryable_idx += 1; + } + } + } + } Some(retryable_transaction_indexes) } diff --git a/core/src/banking_stage/immutable_deserialized_packet.rs b/core/src/banking_stage/immutable_deserialized_packet.rs index 26ede7045d3480..6eb5d68ecaaca5 100644 --- a/core/src/banking_stage/immutable_deserialized_packet.rs +++ b/core/src/banking_stage/immutable_deserialized_packet.rs @@ -13,7 +13,7 @@ use { VersionedTransaction, }, }, - std::{cmp::Ordering, mem::size_of, sync::Arc}, + std::{cmp::Ordering, mem::size_of, sync::Arc, time::Instant}, thiserror::Error, }; @@ -41,10 +41,16 @@ pub struct ImmutableDeserializedPacket { message_hash: Hash, is_simple_vote: bool, compute_budget_details: ComputeBudgetDetails, + banking_stage_start_time: Option, } impl ImmutableDeserializedPacket { pub fn new(packet: Packet) -> Result { + let banking_stage_start_time = packet + .meta() + .is_perf_track_packet() + .then_some(Instant::now()); + let versioned_transaction: VersionedTransaction = packet.deserialize_slice(..)?; let sanitized_transaction = SanitizedVersionedTransaction::try_from(versioned_transaction)?; let message_bytes = packet_message(&packet)?; @@ -67,6 +73,7 @@ impl ImmutableDeserializedPacket { message_hash, is_simple_vote, compute_budget_details, + banking_stage_start_time, }) } @@ -98,6 +105,10 @@ impl ImmutableDeserializedPacket { self.compute_budget_details.clone() } + pub fn start_time(&self) -> &Option { + &self.banking_stage_start_time + } + // This function deserializes packets into transactions, computes the blake3 hash of transaction // messages, and verifies secp256k1 instructions. pub fn build_sanitized_transaction( diff --git a/core/src/banking_stage/unprocessed_transaction_storage.rs b/core/src/banking_stage/unprocessed_transaction_storage.rs index fcc68050b72d4c..52706f8c2bf63b 100644 --- a/core/src/banking_stage/unprocessed_transaction_storage.rs +++ b/core/src/banking_stage/unprocessed_transaction_storage.rs @@ -924,6 +924,7 @@ impl ThreadLocalUnprocessedPackets { .iter() .map(|p| (*p).clone()) .collect_vec(); + let retryable_packets = if let Some(retryable_transaction_indexes) = processing_function(&packets_to_process, payload) { diff --git a/core/src/sigverify_stage.rs b/core/src/sigverify_stage.rs index cde1735611c0d0..18bb9df6d5b96b 100644 --- a/core/src/sigverify_stage.rs +++ b/core/src/sigverify_stage.rs @@ -18,9 +18,11 @@ use { count_discarded_packets, count_packets_in_batches, count_valid_packets, shrink_batches, }, }, - solana_sdk::timing, + solana_sdk::{signature::Signature, timing}, solana_streamer::streamer::{self, StreamerError}, + solana_transaction_metrics_tracker::get_signature_from_packet, std::{ + collections::HashMap, thread::{self, Builder, JoinHandle}, time::Instant, }, @@ -298,8 +300,20 @@ impl SigVerifyStage { verifier: &mut T, stats: &mut SigVerifierStats, ) -> Result<(), T::SendType> { - let (mut batches, num_packets, recv_duration) = streamer::recv_packet_batches(recvr)?; + let mut packet_perf_measure: HashMap<[u8; 64], std::time::Instant> = HashMap::default(); + let (mut batches, num_packets, recv_duration) = streamer::recv_packet_batches(recvr)?; + // track sigverify start time for interested packets + for batch in &batches { + for packet in batch.iter() { + if packet.meta().is_perf_track_packet() { + let signature = get_signature_from_packet(packet); + if let Ok(signature) = signature { + packet_perf_measure.insert(*signature, Instant::now()); + } + } + } + } let batches_len = batches.len(); debug!( "@{:?} verifier: verifying: {}", @@ -372,6 +386,17 @@ impl SigVerifyStage { (num_packets as f32 / verify_time.as_s()) ); + for (signature, start_time) in packet_perf_measure.drain() { + let duration = Instant::now().duration_since(start_time); + debug!( + "Sigverify took {duration:?} for transaction {:?}", + Signature::from(signature) + ); + inc_new_counter_info!( + "txn-metrics-sigverify-packet-verify-us", + duration.as_micros() as usize + ); + } stats .recv_batches_us_hist .increment(recv_duration.as_micros() as u64) diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index 04b6a37b4291f7..cc360e25dbfc58 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -4982,6 +4982,7 @@ dependencies = [ "solana-streamer", "solana-svm", "solana-tpu-client", + "solana-transaction-metrics-tracker", "solana-transaction-status", "solana-turbine", "solana-unified-scheduler-pool", @@ -6341,6 +6342,7 @@ dependencies = [ "solana-metrics", "solana-perf", "solana-sdk", + "solana-transaction-metrics-tracker", "thiserror", "tokio", "x509-parser", @@ -6442,6 +6444,20 @@ dependencies = [ "tokio", ] +[[package]] +name = "solana-transaction-metrics-tracker" +version = "1.19.0" +dependencies = [ + "Inflector", + "base64 0.21.7", + "bincode", + "lazy_static", + "log", + "rand 0.8.5", + "solana-perf", + "solana-sdk", +] + [[package]] name = "solana-transaction-status" version = "1.19.0" diff --git a/sdk/src/packet.rs b/sdk/src/packet.rs index faea9ab4753c67..8300b57218c696 100644 --- a/sdk/src/packet.rs +++ b/sdk/src/packet.rs @@ -33,6 +33,8 @@ bitflags! { /// the packet is built. /// This field can be removed when the above feature gate is adopted by mainnet-beta. const ROUND_COMPUTE_UNIT_PRICE = 0b0010_0000; + /// For tracking performance + const PERF_TRACK_PACKET = 0b0100_0000; } } @@ -228,6 +230,12 @@ impl Meta { self.flags.set(PacketFlags::TRACER_PACKET, is_tracer); } + #[inline] + pub fn set_track_performance(&mut self, is_performance_track: bool) { + self.flags + .set(PacketFlags::PERF_TRACK_PACKET, is_performance_track); + } + #[inline] pub fn set_simple_vote(&mut self, is_simple_vote: bool) { self.flags.set(PacketFlags::SIMPLE_VOTE_TX, is_simple_vote); @@ -261,6 +269,11 @@ impl Meta { self.flags.contains(PacketFlags::TRACER_PACKET) } + #[inline] + pub fn is_perf_track_packet(&self) -> bool { + self.flags.contains(PacketFlags::PERF_TRACK_PACKET) + } + #[inline] pub fn round_compute_unit_price(&self) -> bool { self.flags.contains(PacketFlags::ROUND_COMPUTE_UNIT_PRICE) diff --git a/sdk/src/transaction/versioned/sanitized.rs b/sdk/src/transaction/versioned/sanitized.rs index 61ecdfea56bb2a..b6311d5886b0e3 100644 --- a/sdk/src/transaction/versioned/sanitized.rs +++ b/sdk/src/transaction/versioned/sanitized.rs @@ -33,6 +33,10 @@ impl SanitizedVersionedTransaction { &self.message } + pub fn get_signatures(&self) -> &Vec { + &self.signatures + } + /// Consumes the SanitizedVersionedTransaction, returning the fields individually. pub fn destruct(self) -> (Vec, SanitizedVersionedMessage) { (self.signatures, self.message) diff --git a/streamer/Cargo.toml b/streamer/Cargo.toml index 8e1eb12dff1d42..22170e6426c433 100644 --- a/streamer/Cargo.toml +++ b/streamer/Cargo.toml @@ -29,6 +29,7 @@ rustls = { workspace = true, features = ["dangerous_configuration"] } solana-metrics = { workspace = true } solana-perf = { workspace = true } solana-sdk = { workspace = true } +solana-transaction-metrics-tracker = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true, features = ["full"] } x509-parser = { workspace = true } diff --git a/streamer/src/nonblocking/quic.rs b/streamer/src/nonblocking/quic.rs index 225412dd08b315..23234231fd2750 100644 --- a/streamer/src/nonblocking/quic.rs +++ b/streamer/src/nonblocking/quic.rs @@ -27,10 +27,14 @@ use { QUIC_MIN_STAKED_CONCURRENT_STREAMS, QUIC_MIN_STAKED_RECEIVE_WINDOW_RATIO, QUIC_TOTAL_STAKED_CONCURRENT_STREAMS, QUIC_UNSTAKED_RECEIVE_WINDOW_RATIO, }, - signature::Keypair, + signature::{Keypair, Signature}, timing, }, + solana_transaction_metrics_tracker::{ + get_signature_from_packet, signature_if_should_track_packet, + }, std::{ + collections::HashMap, iter::repeat_with, net::{IpAddr, SocketAddr, UdpSocket}, sync::{ @@ -81,6 +85,7 @@ struct PacketChunk { struct PacketAccumulator { pub meta: Meta, pub chunks: Vec, + pub start_time: Instant, } #[derive(Copy, Clone, Debug)] @@ -628,6 +633,7 @@ async fn packet_batch_sender( trace!("enter packet_batch_sender"); let mut batch_start_time = Instant::now(); loop { + let mut packet_perf_measure: HashMap<[u8; 64], std::time::Instant> = HashMap::default(); let mut packet_batch = PacketBatch::with_capacity(PACKETS_PER_BATCH); let mut total_bytes: usize = 0; @@ -647,6 +653,7 @@ async fn packet_batch_sender( || (!packet_batch.is_empty() && elapsed >= coalesce) { let len = packet_batch.len(); + track_streamer_fetch_packet_performance(&packet_batch, &mut packet_perf_measure); if let Err(e) = packet_sender.send(packet_batch) { stats .total_packet_batch_send_err @@ -692,6 +699,13 @@ async fn packet_batch_sender( total_bytes += packet_batch[i].meta().size; + if let Some(signature) = + signature_if_should_track_packet(&packet_batch[i]).unwrap_or(None) + { + packet_perf_measure.insert(*signature, packet_accumulator.start_time); + // we set the PERF_TRACK_PACKET on + packet_batch[i].meta_mut().set_track_performance(true); + } stats .total_chunks_processed_by_batcher .fetch_add(num_chunks, Ordering::Relaxed); @@ -700,6 +714,30 @@ async fn packet_batch_sender( } } +fn track_streamer_fetch_packet_performance( + packet_batch: &PacketBatch, + packet_perf_measure: &mut HashMap<[u8; 64], Instant>, +) { + for packet in packet_batch.iter() { + if packet.meta().is_perf_track_packet() { + let signature = get_signature_from_packet(packet); + if let Ok(signature) = signature { + if let Some(start_time) = packet_perf_measure.remove(signature) { + let duration = Instant::now().duration_since(start_time); + debug!( + "QUIC streamer fetch stage took {duration:?} for transaction {:?}", + Signature::from(*signature) + ); + inc_new_counter_info!( + "txn-metrics-quic-streamer-packet-fetch-us", + duration.as_micros() as usize + ); + } + } + } + } +} + async fn handle_connection( connection: Connection, remote_addr: SocketAddr, @@ -854,6 +892,7 @@ async fn handle_chunk( *packet_accum = Some(PacketAccumulator { meta, chunks: Vec::new(), + start_time: Instant::now(), }); } @@ -1453,6 +1492,7 @@ pub mod test { offset, end_of_chunk: size, }], + start_time: Instant::now(), }; ptk_sender.send(packet_accum).await.unwrap(); } diff --git a/transaction-metrics-tracker/Cargo.toml b/transaction-metrics-tracker/Cargo.toml new file mode 100644 index 00000000000000..9bd82702a3ebb4 --- /dev/null +++ b/transaction-metrics-tracker/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "solana-transaction-metrics-tracker" +description = "Solana transaction metrics tracker" +documentation = "https://docs.rs/solana-transaction-metrics-tracker" +version = { workspace = true } +authors = { workspace = true } +repository = { workspace = true } +homepage = { workspace = true } +license = { workspace = true } +edition = { workspace = true } +publish = false + +[dependencies] +Inflector = { workspace = true } +base64 = { workspace = true } +bincode = { workspace = true } +# Update this borsh dependency to the workspace version once +lazy_static = { workspace = true } +log = { workspace = true } +rand = { workspace = true } +solana-perf = { workspace = true } +solana-sdk = { workspace = true } + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/transaction-metrics-tracker/src/lib.rs b/transaction-metrics-tracker/src/lib.rs new file mode 100644 index 00000000000000..02ae2c14fa8ca5 --- /dev/null +++ b/transaction-metrics-tracker/src/lib.rs @@ -0,0 +1,54 @@ +use { + lazy_static::lazy_static, + log::*, + rand::Rng, + solana_perf::sigverify::PacketError, + solana_sdk::{packet::Packet, short_vec::decode_shortu16_len, signature::SIGNATURE_BYTES}, +}; + +// The mask is 12 bits long (1<<12 = 4096), it means the probability of matching +// the transaction is 1/4096 assuming the portion being matched is random. +lazy_static! { + static ref TXN_MASK: u16 = rand::thread_rng().gen_range(0..4096); +} + +/// Check if a transaction given its signature matches the randomly selected mask. +/// The signaure should be from the reference of Signature +pub fn should_track_transaction(signature: &[u8; SIGNATURE_BYTES]) -> bool { + // We do not use the highest signature byte as it is not really random + let match_portion: u16 = u16::from_le_bytes([signature[61], signature[62]]) >> 4; + trace!("Matching txn: {match_portion:b} {:b}", *TXN_MASK); + *TXN_MASK == match_portion +} + +/// Check if a transaction packet's signature matches the mask. +/// This does a rudimentry verification to make sure the packet at least +/// contains the signature data and it returns the reference to the signature. +pub fn signature_if_should_track_packet( + packet: &Packet, +) -> Result, PacketError> { + let signature = get_signature_from_packet(packet)?; + Ok(should_track_transaction(signature).then_some(signature)) +} + +/// Get the signature of the transaction packet +/// This does a rudimentry verification to make sure the packet at least +/// contains the signature data and it returns the reference to the signature. +pub fn get_signature_from_packet(packet: &Packet) -> Result<&[u8; SIGNATURE_BYTES], PacketError> { + let (sig_len_untrusted, sig_start) = packet + .data(..) + .and_then(|bytes| decode_shortu16_len(bytes).ok()) + .ok_or(PacketError::InvalidShortVec)?; + + if sig_len_untrusted < 1 { + return Err(PacketError::InvalidSignatureLen); + } + + let signature = packet + .data(sig_start..sig_start.saturating_add(SIGNATURE_BYTES)) + .ok_or(PacketError::InvalidSignatureLen)?; + let signature = signature + .try_into() + .map_err(|_| PacketError::InvalidSignatureLen)?; + Ok(signature) +} From 1321919b000be3755f9be07289fe61ba20b6d7b7 Mon Sep 17 00:00:00 2001 From: Lijun Wang <83639177+lijunwangs@users.noreply.github.com> Date: Thu, 29 Feb 2024 03:03:57 -0800 Subject: [PATCH 52/71] Use datapoint as opposed to counters --- core/src/banking_stage/consumer.rs | 7 ++-- core/src/banking_stage/leader_slot_metrics.rs | 11 ++++++ .../leader_slot_timing_metrics.rs | 25 +++++++++++++ core/src/sigverify_stage.rs | 35 +++++++++++++++---- streamer/src/nonblocking/quic.rs | 23 ++++++++---- streamer/src/quic.rs | 26 +++++++++++++- 6 files changed, 109 insertions(+), 18 deletions(-) diff --git a/core/src/banking_stage/consumer.rs b/core/src/banking_stage/consumer.rs index f417363c7e4409..9e9475f7678e3c 100644 --- a/core/src/banking_stage/consumer.rs +++ b/core/src/banking_stage/consumer.rs @@ -224,10 +224,9 @@ impl Consumer { "Banking stage processing took {duration:?} for transaction {:?}", packet.transaction().get_signatures().first() ); - inc_new_counter_info!( - "txn-metrics-banking-stage-process-us", - duration.as_micros() as usize - ); + payload + .slot_metrics_tracker + .increment_process_sampled_packets_us(duration.as_micros() as u64); } else { // This packet is retried, advance the retry index to the next, as the next packet's index will // certainly be > than this. diff --git a/core/src/banking_stage/leader_slot_metrics.rs b/core/src/banking_stage/leader_slot_metrics.rs index 88ea6b5ee340cf..1c255ca019bfe7 100644 --- a/core/src/banking_stage/leader_slot_metrics.rs +++ b/core/src/banking_stage/leader_slot_metrics.rs @@ -936,6 +936,17 @@ impl LeaderSlotMetricsTracker { ); } } + + pub(crate) fn increment_process_sampled_packets_us(&mut self, us: u64) { + if let Some(leader_slot_metrics) = &mut self.leader_slot_metrics { + leader_slot_metrics + .timing_metrics + .process_packets_timings + .process_sampled_packets_us_hist + .increment(us) + .unwrap(); + } + } } #[cfg(test)] diff --git a/core/src/banking_stage/leader_slot_timing_metrics.rs b/core/src/banking_stage/leader_slot_timing_metrics.rs index 7727b6cf6c6563..34ce64b31c34f3 100644 --- a/core/src/banking_stage/leader_slot_timing_metrics.rs +++ b/core/src/banking_stage/leader_slot_timing_metrics.rs @@ -244,6 +244,9 @@ pub(crate) struct ProcessPacketsTimings { // Time spent running the cost model in processing transactions before executing // transactions pub cost_model_us: u64, + + // banking stage processing time histogram for sampled packets + pub process_sampled_packets_us_hist: histogram::Histogram, } impl ProcessPacketsTimings { @@ -264,6 +267,28 @@ impl ProcessPacketsTimings { i64 ), ("cost_model_us", self.cost_model_us, i64), + ( + "process_sampled_packets_us_90pct", + self.process_sampled_packets_us_hist + .percentile(90.0) + .unwrap_or(0), + i64 + ), + ( + "process_sampled_packets_us_min", + self.process_sampled_packets_us_hist.minimum().unwrap_or(0), + i64 + ), + ( + "process_sampled_packets_us_max", + self.process_sampled_packets_us_hist.maximum().unwrap_or(0), + i64 + ), + ( + "process_sampled_packets_us_mean", + self.process_sampled_packets_us_hist.mean().unwrap_or(0), + i64 + ), ); } } diff --git a/core/src/sigverify_stage.rs b/core/src/sigverify_stage.rs index 18bb9df6d5b96b..5b9112f87a1622 100644 --- a/core/src/sigverify_stage.rs +++ b/core/src/sigverify_stage.rs @@ -80,8 +80,9 @@ struct SigVerifierStats { verify_batches_pp_us_hist: histogram::Histogram, // per-packet time to call verify_batch discard_packets_pp_us_hist: histogram::Histogram, // per-packet time to call verify_batch dedup_packets_pp_us_hist: histogram::Histogram, // per-packet time to call verify_batch - batches_hist: histogram::Histogram, // number of packet batches per verify call - packets_hist: histogram::Histogram, // number of packets per verify call + sampled_packets_pp_us_hist: histogram::Histogram, // per-packet time do do overall verify for sampled packets + batches_hist: histogram::Histogram, // number of packet batches per verify call + packets_hist: histogram::Histogram, // number of packets per verify call num_deduper_saturations: usize, total_batches: usize, total_packets: usize, @@ -183,6 +184,28 @@ impl SigVerifierStats { self.dedup_packets_pp_us_hist.mean().unwrap_or(0), i64 ), + ( + "sampled_verify_packets_pp_us_90pct", + self.sampled_packets_pp_us_hist + .percentile(90.0) + .unwrap_or(0), + i64 + ), + ( + "sampled_verify_packets_pp_us_min", + self.sampled_packets_pp_us_hist.minimum().unwrap_or(0), + i64 + ), + ( + "sampled_verify_packets_pp_us_max", + self.sampled_packets_pp_us_hist.maximum().unwrap_or(0), + i64 + ), + ( + "sampled_verify_packets_pp_us_mean", + self.sampled_packets_pp_us_hist.mean().unwrap_or(0), + i64 + ), ( "batches_90pct", self.batches_hist.percentile(90.0).unwrap_or(0), @@ -392,10 +415,10 @@ impl SigVerifyStage { "Sigverify took {duration:?} for transaction {:?}", Signature::from(signature) ); - inc_new_counter_info!( - "txn-metrics-sigverify-packet-verify-us", - duration.as_micros() as usize - ); + stats + .sampled_packets_pp_us_hist + .increment(duration.as_micros() as u64) + .unwrap(); } stats .recv_batches_us_hist diff --git a/streamer/src/nonblocking/quic.rs b/streamer/src/nonblocking/quic.rs index 23234231fd2750..b3d02d8287b77e 100644 --- a/streamer/src/nonblocking/quic.rs +++ b/streamer/src/nonblocking/quic.rs @@ -653,7 +653,12 @@ async fn packet_batch_sender( || (!packet_batch.is_empty() && elapsed >= coalesce) { let len = packet_batch.len(); - track_streamer_fetch_packet_performance(&packet_batch, &mut packet_perf_measure); + track_streamer_fetch_packet_performance( + &packet_batch, + &mut packet_perf_measure, + stats.clone(), + ); + if let Err(e) = packet_sender.send(packet_batch) { stats .total_packet_batch_send_err @@ -699,8 +704,9 @@ async fn packet_batch_sender( total_bytes += packet_batch[i].meta().size; - if let Some(signature) = - signature_if_should_track_packet(&packet_batch[i]).unwrap_or(None) + if let Some(signature) = signature_if_should_track_packet(&packet_batch[i]) + .ok() + .flatten() { packet_perf_measure.insert(*signature, packet_accumulator.start_time); // we set the PERF_TRACK_PACKET on @@ -717,6 +723,7 @@ async fn packet_batch_sender( fn track_streamer_fetch_packet_performance( packet_batch: &PacketBatch, packet_perf_measure: &mut HashMap<[u8; 64], Instant>, + stats: Arc, ) { for packet in packet_batch.iter() { if packet.meta().is_perf_track_packet() { @@ -728,10 +735,12 @@ fn track_streamer_fetch_packet_performance( "QUIC streamer fetch stage took {duration:?} for transaction {:?}", Signature::from(*signature) ); - inc_new_counter_info!( - "txn-metrics-quic-streamer-packet-fetch-us", - duration.as_micros() as usize - ); + stats + .process_sampled_packets_us_hist + .lock() + .unwrap() + .increment(duration.as_micros() as u64) + .unwrap(); } } } diff --git a/streamer/src/quic.rs b/streamer/src/quic.rs index a7a08c73f4833b..152572d1fe218a 100644 --- a/streamer/src/quic.rs +++ b/streamer/src/quic.rs @@ -17,7 +17,7 @@ use { net::UdpSocket, sync::{ atomic::{AtomicBool, AtomicUsize, Ordering}, - Arc, RwLock, + Arc, Mutex, RwLock, }, thread, time::{Duration, SystemTime}, @@ -175,10 +175,12 @@ pub struct StreamStats { pub(crate) stream_load_ema: AtomicUsize, pub(crate) stream_load_ema_overflow: AtomicUsize, pub(crate) stream_load_capacity_overflow: AtomicUsize, + pub(crate) process_sampled_packets_us_hist: Mutex, } impl StreamStats { pub fn report(&self, name: &'static str) { + let process_sampled_packets_us_hist = self.process_sampled_packets_us_hist.lock().unwrap(); datapoint_info!( name, ( @@ -425,6 +427,28 @@ impl StreamStats { self.stream_load_capacity_overflow.load(Ordering::Relaxed), i64 ), + ( + "process_sampled_packets_us_90pct", + process_sampled_packets_us_hist + .percentile(90.0) + .unwrap_or(0), + i64 + ), + ( + "process_sampled_packets_us_min", + process_sampled_packets_us_hist.minimum().unwrap_or(0), + i64 + ), + ( + "process_sampled_packets_us_max", + process_sampled_packets_us_hist.maximum().unwrap_or(0), + i64 + ), + ( + "process_sampled_packets_us_mean", + process_sampled_packets_us_hist.mean().unwrap_or(0), + i64 + ), ); } } From f91bfccf64f71b44822a527d2076ce8691bad375 Mon Sep 17 00:00:00 2001 From: Lijun Wang <83639177+lijunwangs@users.noreply.github.com> Date: Thu, 29 Feb 2024 04:07:20 -0800 Subject: [PATCH 53/71] Add a unit test --- transaction-metrics-tracker/src/lib.rs | 29 ++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/transaction-metrics-tracker/src/lib.rs b/transaction-metrics-tracker/src/lib.rs index 02ae2c14fa8ca5..4b770eddb6c59b 100644 --- a/transaction-metrics-tracker/src/lib.rs +++ b/transaction-metrics-tracker/src/lib.rs @@ -40,6 +40,7 @@ pub fn get_signature_from_packet(packet: &Packet) -> Result<&[u8; SIGNATURE_BYTE .and_then(|bytes| decode_shortu16_len(bytes).ok()) .ok_or(PacketError::InvalidShortVec)?; + println!("Sig length is okay!!"); if sig_len_untrusted < 1 { return Err(PacketError::InvalidSignatureLen); } @@ -52,3 +53,31 @@ pub fn get_signature_from_packet(packet: &Packet) -> Result<&[u8; SIGNATURE_BYTE .map_err(|_| PacketError::InvalidSignatureLen)?; Ok(signature) } + +#[cfg(test)] +mod tests { + use { + super::*, + solana_sdk::{hash::Hash, signature::Keypair, system_transaction}, + }; + + #[test] + fn test_get_signature_from_packet() { + // default invalid txn packet + let packet = Packet::default(); + let sig = get_signature_from_packet(&packet); + assert_eq!(sig, Err(PacketError::InvalidShortVec)); + + // Use a valid transaction, it should succeed + let tx = system_transaction::transfer( + &Keypair::new(), + &solana_sdk::pubkey::new_rand(), + 1, + Hash::new_unique(), + ); + let packet = Packet::from_data(None, tx).unwrap(); + + let sig = get_signature_from_packet(&packet); + assert!(sig.is_ok()); + } +} From bcc8217629075f3b33a1b3e6af82daf621e22045 Mon Sep 17 00:00:00 2001 From: Lijun Wang <83639177+lijunwangs@users.noreply.github.com> Date: Thu, 29 Feb 2024 04:10:07 -0800 Subject: [PATCH 54/71] removed a print --- transaction-metrics-tracker/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/transaction-metrics-tracker/src/lib.rs b/transaction-metrics-tracker/src/lib.rs index 4b770eddb6c59b..02d7c03d94f068 100644 --- a/transaction-metrics-tracker/src/lib.rs +++ b/transaction-metrics-tracker/src/lib.rs @@ -40,7 +40,6 @@ pub fn get_signature_from_packet(packet: &Packet) -> Result<&[u8; SIGNATURE_BYTE .and_then(|bytes| decode_shortu16_len(bytes).ok()) .ok_or(PacketError::InvalidShortVec)?; - println!("Sig length is okay!!"); if sig_len_untrusted < 1 { return Err(PacketError::InvalidSignatureLen); } From 3f52b9e788fea0e14b8e30af667070bb77ada4ae Mon Sep 17 00:00:00 2001 From: Lijun Wang <83639177+lijunwangs@users.noreply.github.com> Date: Thu, 29 Feb 2024 12:29:26 -0800 Subject: [PATCH 55/71] Added more unit tests --- transaction-metrics-tracker/src/lib.rs | 83 ++++++++++++++++++++++++-- 1 file changed, 79 insertions(+), 4 deletions(-) diff --git a/transaction-metrics-tracker/src/lib.rs b/transaction-metrics-tracker/src/lib.rs index 02d7c03d94f068..4ffe7fda9d7ece 100644 --- a/transaction-metrics-tracker/src/lib.rs +++ b/transaction-metrics-tracker/src/lib.rs @@ -17,7 +17,7 @@ lazy_static! { pub fn should_track_transaction(signature: &[u8; SIGNATURE_BYTES]) -> bool { // We do not use the highest signature byte as it is not really random let match_portion: u16 = u16::from_le_bytes([signature[61], signature[62]]) >> 4; - trace!("Matching txn: {match_portion:b} {:b}", *TXN_MASK); + trace!("Matching txn: {match_portion:016b} {:016b}", *TXN_MASK); *TXN_MASK == match_portion } @@ -57,12 +57,16 @@ pub fn get_signature_from_packet(packet: &Packet) -> Result<&[u8; SIGNATURE_BYTE mod tests { use { super::*, - solana_sdk::{hash::Hash, signature::Keypair, system_transaction}, + solana_sdk::{ + hash::Hash, + signature::{Keypair, Signature}, + system_transaction, + }, }; #[test] fn test_get_signature_from_packet() { - // default invalid txn packet + // Default invalid txn packet let packet = Packet::default(); let sig = get_signature_from_packet(&packet); assert_eq!(sig, Err(PacketError::InvalidShortVec)); @@ -74,9 +78,80 @@ mod tests { 1, Hash::new_unique(), ); - let packet = Packet::from_data(None, tx).unwrap(); + let mut packet = Packet::from_data(None, tx).unwrap(); let sig = get_signature_from_packet(&packet); assert!(sig.is_ok()); + + // Invalid signature length + packet.buffer_mut()[0] = 0x0; + let sig = get_signature_from_packet(&packet); + assert_eq!(sig, Err(PacketError::InvalidSignatureLen)); + } + + #[test] + fn test_should_track_transaction() { + let mut sig = [0x0; SIGNATURE_BYTES]; + let track = should_track_transaction(&sig); + assert!(!track); + + // Intentionally matching the randomly generated mask + // The lower four bits are ignored as only 12 highest bits from + // signature's 61 and 62 u8 are used for matching. + // We generate a random one + let mut rng = rand::thread_rng(); + let random_number: u8 = rng.gen_range(0..=15); + sig[61] = ((*TXN_MASK & 0xf as u16) << 4) as u8 | random_number; + sig[62] = (*TXN_MASK >> 4) as u8; + + let track = should_track_transaction(&sig); + assert!(track); + } + + #[test] + fn test_signature_if_should_track_packet() { + // Default invalid txn packet + let packet = Packet::default(); + let sig = signature_if_should_track_packet(&packet); + assert_eq!(sig, Err(PacketError::InvalidShortVec)); + + // Use a valid transaction which is not matched + let tx = system_transaction::transfer( + &Keypair::new(), + &solana_sdk::pubkey::new_rand(), + 1, + Hash::new_unique(), + ); + let packet = Packet::from_data(None, tx).unwrap(); + let sig = signature_if_should_track_packet(&packet); + assert_eq!(Ok(None), sig); + + // Now simulate a txn matching the signature mask + let mut tx = system_transaction::transfer( + &Keypair::new(), + &solana_sdk::pubkey::new_rand(), + 1, + Hash::new_unique(), + ); + let mut sig = [0x0; SIGNATURE_BYTES]; + sig[61] = ((*TXN_MASK & 0xf as u16) << 4) as u8; + sig[62] = (*TXN_MASK >> 4) as u8; + + let sig = Signature::from(sig); + tx.signatures[0] = sig; + let mut packet = Packet::from_data(None, tx).unwrap(); + let sig2 = signature_if_should_track_packet(&packet); + + match sig2 { + Ok(sig) => { + assert!(sig.is_some()); + } + Err(_) => assert!(false, "Expected to get a matching signature!"), + } + + // Invalid signature length + packet.buffer_mut()[0] = 0x0; + let sig = signature_if_should_track_packet(&packet); + assert_eq!(sig, Err(PacketError::InvalidSignatureLen)); } } From da5253885eb29df4565ad7d0ce7042f5034ac7d0 Mon Sep 17 00:00:00 2001 From: Lijun Wang <83639177+lijunwangs@users.noreply.github.com> Date: Thu, 29 Feb 2024 13:08:46 -0800 Subject: [PATCH 56/71] Making stats names consistent across layers --- core/src/sigverify_stage.rs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/core/src/sigverify_stage.rs b/core/src/sigverify_stage.rs index 5b9112f87a1622..7cce095695f445 100644 --- a/core/src/sigverify_stage.rs +++ b/core/src/sigverify_stage.rs @@ -80,9 +80,9 @@ struct SigVerifierStats { verify_batches_pp_us_hist: histogram::Histogram, // per-packet time to call verify_batch discard_packets_pp_us_hist: histogram::Histogram, // per-packet time to call verify_batch dedup_packets_pp_us_hist: histogram::Histogram, // per-packet time to call verify_batch - sampled_packets_pp_us_hist: histogram::Histogram, // per-packet time do do overall verify for sampled packets - batches_hist: histogram::Histogram, // number of packet batches per verify call - packets_hist: histogram::Histogram, // number of packets per verify call + process_sampled_packets_us_hist: histogram::Histogram, // per-packet time do do overall verify for sampled packets + batches_hist: histogram::Histogram, // number of packet batches per verify call + packets_hist: histogram::Histogram, // number of packets per verify call num_deduper_saturations: usize, total_batches: usize, total_packets: usize, @@ -185,25 +185,25 @@ impl SigVerifierStats { i64 ), ( - "sampled_verify_packets_pp_us_90pct", - self.sampled_packets_pp_us_hist + "process_sampled_packets_us_90pct", + self.process_sampled_packets_us_hist .percentile(90.0) .unwrap_or(0), i64 ), ( - "sampled_verify_packets_pp_us_min", - self.sampled_packets_pp_us_hist.minimum().unwrap_or(0), + "process_sampled_packets_us_min", + self.process_sampled_packets_us_hist.minimum().unwrap_or(0), i64 ), ( - "sampled_verify_packets_pp_us_max", - self.sampled_packets_pp_us_hist.maximum().unwrap_or(0), + "process_sampled_packets_us_max", + self.process_sampled_packets_us_hist.maximum().unwrap_or(0), i64 ), ( - "sampled_verify_packets_pp_us_mean", - self.sampled_packets_pp_us_hist.mean().unwrap_or(0), + "process_sampled_packets_us_mean", + self.process_sampled_packets_us_hist.mean().unwrap_or(0), i64 ), ( @@ -416,7 +416,7 @@ impl SigVerifyStage { Signature::from(signature) ); stats - .sampled_packets_pp_us_hist + .process_sampled_packets_us_hist .increment(duration.as_micros() as u64) .unwrap(); } From 486972f7804bfd1cfb61fbe1766c53f7b2fc1de0 Mon Sep 17 00:00:00 2001 From: Lijun Wang <83639177+lijunwangs@users.noreply.github.com> Date: Thu, 29 Feb 2024 14:17:44 -0800 Subject: [PATCH 57/71] Added more unit tests --- transaction-metrics-tracker/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/transaction-metrics-tracker/src/lib.rs b/transaction-metrics-tracker/src/lib.rs index 4ffe7fda9d7ece..36c0349620275e 100644 --- a/transaction-metrics-tracker/src/lib.rs +++ b/transaction-metrics-tracker/src/lib.rs @@ -134,7 +134,7 @@ mod tests { Hash::new_unique(), ); let mut sig = [0x0; SIGNATURE_BYTES]; - sig[61] = ((*TXN_MASK & 0xf as u16) << 4) as u8; + sig[61] = ((*TXN_MASK & 0xf_u16) << 4) as u8; sig[62] = (*TXN_MASK >> 4) as u8; let sig = Signature::from(sig); @@ -146,7 +146,7 @@ mod tests { Ok(sig) => { assert!(sig.is_some()); } - Err(_) => assert!(false, "Expected to get a matching signature!"), + Err(_) => panic!("Expected to get a matching signature!"), } // Invalid signature length From 93caf8b209d6d5446b274ccb6a5cb1d1f6570eb0 Mon Sep 17 00:00:00 2001 From: Lijun Wang <83639177+lijunwangs@users.noreply.github.com> Date: Thu, 29 Feb 2024 14:55:09 -0800 Subject: [PATCH 58/71] Added more unit tests --- transaction-metrics-tracker/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transaction-metrics-tracker/src/lib.rs b/transaction-metrics-tracker/src/lib.rs index 36c0349620275e..2baec195de9b84 100644 --- a/transaction-metrics-tracker/src/lib.rs +++ b/transaction-metrics-tracker/src/lib.rs @@ -101,7 +101,7 @@ mod tests { // We generate a random one let mut rng = rand::thread_rng(); let random_number: u8 = rng.gen_range(0..=15); - sig[61] = ((*TXN_MASK & 0xf as u16) << 4) as u8 | random_number; + sig[61] = ((*TXN_MASK & 0xf_u16) << 4) as u8 | random_number; sig[62] = (*TXN_MASK >> 4) as u8; let track = should_track_transaction(&sig); From 02aecd22681bb51e6359fc7415eead5cc8b29aa1 Mon Sep 17 00:00:00 2001 From: Lijun Wang <83639177+lijunwangs@users.noreply.github.com> Date: Thu, 29 Feb 2024 21:31:14 -0800 Subject: [PATCH 59/71] Do not use binary_search, do simple compare in one loop --- streamer/src/nonblocking/quic.rs | 53 ++++++++++++-------------------- 1 file changed, 20 insertions(+), 33 deletions(-) diff --git a/streamer/src/nonblocking/quic.rs b/streamer/src/nonblocking/quic.rs index b3d02d8287b77e..901076da10f9d0 100644 --- a/streamer/src/nonblocking/quic.rs +++ b/streamer/src/nonblocking/quic.rs @@ -30,11 +30,8 @@ use { signature::{Keypair, Signature}, timing, }, - solana_transaction_metrics_tracker::{ - get_signature_from_packet, signature_if_should_track_packet, - }, + solana_transaction_metrics_tracker::signature_if_should_track_packet, std::{ - collections::HashMap, iter::repeat_with, net::{IpAddr, SocketAddr, UdpSocket}, sync::{ @@ -633,7 +630,7 @@ async fn packet_batch_sender( trace!("enter packet_batch_sender"); let mut batch_start_time = Instant::now(); loop { - let mut packet_perf_measure: HashMap<[u8; 64], std::time::Instant> = HashMap::default(); + let mut packet_perf_measure: Vec<([u8; 64], std::time::Instant)> = Vec::default(); let mut packet_batch = PacketBatch::with_capacity(PACKETS_PER_BATCH); let mut total_bytes: usize = 0; @@ -653,11 +650,7 @@ async fn packet_batch_sender( || (!packet_batch.is_empty() && elapsed >= coalesce) { let len = packet_batch.len(); - track_streamer_fetch_packet_performance( - &packet_batch, - &mut packet_perf_measure, - stats.clone(), - ); + track_streamer_fetch_packet_performance(&mut packet_perf_measure, &stats); if let Err(e) = packet_sender.send(packet_batch) { stats @@ -708,7 +701,7 @@ async fn packet_batch_sender( .ok() .flatten() { - packet_perf_measure.insert(*signature, packet_accumulator.start_time); + packet_perf_measure.push((*signature, packet_accumulator.start_time)); // we set the PERF_TRACK_PACKET on packet_batch[i].meta_mut().set_track_performance(true); } @@ -721,29 +714,23 @@ async fn packet_batch_sender( } fn track_streamer_fetch_packet_performance( - packet_batch: &PacketBatch, - packet_perf_measure: &mut HashMap<[u8; 64], Instant>, - stats: Arc, + packet_perf_measure: &mut Vec<([u8; 64], Instant)>, + stats: &Arc, ) { - for packet in packet_batch.iter() { - if packet.meta().is_perf_track_packet() { - let signature = get_signature_from_packet(packet); - if let Ok(signature) = signature { - if let Some(start_time) = packet_perf_measure.remove(signature) { - let duration = Instant::now().duration_since(start_time); - debug!( - "QUIC streamer fetch stage took {duration:?} for transaction {:?}", - Signature::from(*signature) - ); - stats - .process_sampled_packets_us_hist - .lock() - .unwrap() - .increment(duration.as_micros() as u64) - .unwrap(); - } - } - } + if packet_perf_measure.is_empty() { + return; + } + let mut process_sampled_packets_us_hist = stats.process_sampled_packets_us_hist.lock().unwrap(); + + for (signature, start_time) in packet_perf_measure.iter() { + let duration = Instant::now().duration_since(*start_time); + debug!( + "QUIC streamer fetch stage took {duration:?} for transaction {:?}", + Signature::from(*signature) + ); + process_sampled_packets_us_hist + .increment(duration.as_micros() as u64) + .unwrap(); } } From babf345c2efbcfed97f285758162a3829b6fa5f5 Mon Sep 17 00:00:00 2001 From: Lijun Wang <83639177+lijunwangs@users.noreply.github.com> Date: Thu, 29 Feb 2024 21:47:32 -0800 Subject: [PATCH 60/71] measure perf track overhead --- Cargo.lock | 1 + streamer/Cargo.toml | 1 + streamer/src/nonblocking/quic.rs | 6 ++++++ streamer/src/quic.rs | 8 +++++++- 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 823572dbc95441..2a3e591580da49 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7192,6 +7192,7 @@ dependencies = [ "rand 0.8.5", "rustls", "solana-logger", + "solana-measure", "solana-metrics", "solana-perf", "solana-sdk", diff --git a/streamer/Cargo.toml b/streamer/Cargo.toml index 22170e6426c433..55d0030e734607 100644 --- a/streamer/Cargo.toml +++ b/streamer/Cargo.toml @@ -26,6 +26,7 @@ quinn = { workspace = true } quinn-proto = { workspace = true } rand = { workspace = true } rustls = { workspace = true, features = ["dangerous_configuration"] } +solana-measure = { workspace = true } solana-metrics = { workspace = true } solana-perf = { workspace = true } solana-sdk = { workspace = true } diff --git a/streamer/src/nonblocking/quic.rs b/streamer/src/nonblocking/quic.rs index 901076da10f9d0..0a5b0ef3fe6c03 100644 --- a/streamer/src/nonblocking/quic.rs +++ b/streamer/src/nonblocking/quic.rs @@ -17,6 +17,7 @@ use { quinn::{Connecting, Connection, Endpoint, EndpointConfig, TokioRuntime, VarInt}, quinn_proto::VarIntBoundsExceeded, rand::{thread_rng, Rng}, + solana_measure::measure::Measure, solana_perf::packet::{PacketBatch, PACKETS_PER_BATCH}, solana_sdk::{ packet::{Meta, PACKET_DATA_SIZE}, @@ -720,6 +721,7 @@ fn track_streamer_fetch_packet_performance( if packet_perf_measure.is_empty() { return; } + let mut measure = Measure::start("track_perf"); let mut process_sampled_packets_us_hist = stats.process_sampled_packets_us_hist.lock().unwrap(); for (signature, start_time) in packet_perf_measure.iter() { @@ -732,6 +734,10 @@ fn track_streamer_fetch_packet_performance( .increment(duration.as_micros() as u64) .unwrap(); } + measure.stop(); + stats + .perf_track_overhead_us + .fetch_add(measure.as_us(), Ordering::Relaxed); } async fn handle_connection( diff --git a/streamer/src/quic.rs b/streamer/src/quic.rs index 152572d1fe218a..65e1c571a05467 100644 --- a/streamer/src/quic.rs +++ b/streamer/src/quic.rs @@ -16,7 +16,7 @@ use { std::{ net::UdpSocket, sync::{ - atomic::{AtomicBool, AtomicUsize, Ordering}, + atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}, Arc, Mutex, RwLock, }, thread, @@ -176,6 +176,7 @@ pub struct StreamStats { pub(crate) stream_load_ema_overflow: AtomicUsize, pub(crate) stream_load_capacity_overflow: AtomicUsize, pub(crate) process_sampled_packets_us_hist: Mutex, + pub(crate) perf_track_overhead_us: AtomicU64, } impl StreamStats { @@ -449,6 +450,11 @@ impl StreamStats { process_sampled_packets_us_hist.mean().unwrap_or(0), i64 ), + ( + "perf_track_overhead_us", + self.perf_track_overhead_us.swap(0, Ordering::Relaxed), + i64 + ), ); } } From f562877b88b628f40240df47c50945d881fcc208 Mon Sep 17 00:00:00 2001 From: Lijun Wang <83639177+lijunwangs@users.noreply.github.com> Date: Thu, 29 Feb 2024 21:48:45 -0800 Subject: [PATCH 61/71] missing cargo.lock --- programs/sbf/Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index cc360e25dbfc58..6aa6553f0f8ccb 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -6339,6 +6339,7 @@ dependencies = [ "quinn-proto", "rand 0.8.5", "rustls", + "solana-measure", "solana-metrics", "solana-perf", "solana-sdk", From 1d4272cc048e4fa16bfbf80e634b73b65289fd45 Mon Sep 17 00:00:00 2001 From: Lijun Wang <83639177+lijunwangs@users.noreply.github.com> Date: Fri, 1 Mar 2024 01:47:03 -0800 Subject: [PATCH 62/71] Do not use Hashmap for perf track. Using vec. Measure the overhead of perf track --- core/src/sigverify_stage.rs | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/core/src/sigverify_stage.rs b/core/src/sigverify_stage.rs index 7cce095695f445..aa9f4b7c4a2869 100644 --- a/core/src/sigverify_stage.rs +++ b/core/src/sigverify_stage.rs @@ -22,7 +22,6 @@ use { solana_streamer::streamer::{self, StreamerError}, solana_transaction_metrics_tracker::get_signature_from_packet, std::{ - collections::HashMap, thread::{self, Builder, JoinHandle}, time::Instant, }, @@ -96,6 +95,7 @@ struct SigVerifierStats { total_discard_random_time_us: usize, total_verify_time_us: usize, total_shrink_time_us: usize, + perf_track_overhead_us: usize, } impl SigVerifierStats { @@ -239,6 +239,7 @@ impl SigVerifierStats { ), ("total_verify_time_us", self.total_verify_time_us, i64), ("total_shrink_time_us", self.total_shrink_time_us, i64), + ("perf_track_overhead_us", self.perf_track_overhead_us, i64), ); } } @@ -323,20 +324,26 @@ impl SigVerifyStage { verifier: &mut T, stats: &mut SigVerifierStats, ) -> Result<(), T::SendType> { - let mut packet_perf_measure: HashMap<[u8; 64], std::time::Instant> = HashMap::default(); + let mut packet_perf_measure: Vec<([u8; 64], std::time::Instant)> = Vec::default(); let (mut batches, num_packets, recv_duration) = streamer::recv_packet_batches(recvr)?; + + let mut start_perf_track_measure = Measure::start("start_perf_track"); // track sigverify start time for interested packets for batch in &batches { for packet in batch.iter() { if packet.meta().is_perf_track_packet() { let signature = get_signature_from_packet(packet); if let Ok(signature) = signature { - packet_perf_measure.insert(*signature, Instant::now()); + packet_perf_measure.push((*signature, Instant::now())); } } } } + start_perf_track_measure.stop(); + + stats.perf_track_overhead_us = start_perf_track_measure.as_us() as usize; + let batches_len = batches.len(); debug!( "@{:?} verifier: verifying: {}", @@ -409,17 +416,22 @@ impl SigVerifyStage { (num_packets as f32 / verify_time.as_s()) ); - for (signature, start_time) in packet_perf_measure.drain() { - let duration = Instant::now().duration_since(start_time); + let mut perf_track_end_measure = Measure::start("perf_track_end"); + for (signature, start_time) in packet_perf_measure.iter() { + let duration = Instant::now().duration_since(*start_time); debug!( "Sigverify took {duration:?} for transaction {:?}", - Signature::from(signature) + Signature::from(*signature) ); stats .process_sampled_packets_us_hist .increment(duration.as_micros() as u64) .unwrap(); } + + perf_track_end_measure.stop(); + stats.perf_track_overhead_us += perf_track_end_measure.as_us() as usize; + stats .recv_batches_us_hist .increment(recv_duration.as_micros() as u64) From 9ef4f07c1cdbee14a624c92742d7d0bb0abaed0d Mon Sep 17 00:00:00 2001 From: Lijun Wang <83639177+lijunwangs@users.noreply.github.com> Date: Fri, 1 Mar 2024 02:03:37 -0800 Subject: [PATCH 63/71] Clippy issue --- streamer/src/nonblocking/quic.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/streamer/src/nonblocking/quic.rs b/streamer/src/nonblocking/quic.rs index 0a5b0ef3fe6c03..3485e4fe585d06 100644 --- a/streamer/src/nonblocking/quic.rs +++ b/streamer/src/nonblocking/quic.rs @@ -715,7 +715,7 @@ async fn packet_batch_sender( } fn track_streamer_fetch_packet_performance( - packet_perf_measure: &mut Vec<([u8; 64], Instant)>, + packet_perf_measure: &mut [([u8; 64], Instant)], stats: &Arc, ) { if packet_perf_measure.is_empty() { From 56697929d66f9570104e3fb1648197cb53f99bd0 Mon Sep 17 00:00:00 2001 From: Lijun Wang <83639177+lijunwangs@users.noreply.github.com> Date: Tue, 5 Mar 2024 10:21:04 -0800 Subject: [PATCH 64/71] clear histogram for streamer stage --- streamer/src/quic.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/streamer/src/quic.rs b/streamer/src/quic.rs index 65e1c571a05467..a8d4cd01291cd2 100644 --- a/streamer/src/quic.rs +++ b/streamer/src/quic.rs @@ -181,7 +181,8 @@ pub struct StreamStats { impl StreamStats { pub fn report(&self, name: &'static str) { - let process_sampled_packets_us_hist = self.process_sampled_packets_us_hist.lock().unwrap(); + let mut process_sampled_packets_us_hist = + self.process_sampled_packets_us_hist.lock().unwrap(); datapoint_info!( name, ( @@ -456,6 +457,7 @@ impl StreamStats { i64 ), ); + process_sampled_packets_us_hist.clear(); } } From 1a64dbff0d33ce0ce5ee103bdb959564d7e46381 Mon Sep 17 00:00:00 2001 From: Lijun Wang <83639177+lijunwangs@users.noreply.github.com> Date: Sat, 2 Mar 2024 03:28:32 -0800 Subject: [PATCH 65/71] Fixed a rebase issue --- accounts-db/src/accounts.rs | 29 +++--- accounts-db/src/accounts_db.rs | 35 +++---- banks-server/src/banks_server.rs | 10 +- core/src/banking_stage/consumer.rs | 21 +++-- .../forward_packet_batches_by_accounts.rs | 8 +- .../banking_stage/latest_unprocessed_votes.rs | 3 +- core/src/banking_stage/qos_service.rs | 12 ++- core/src/banking_stage/scheduler_messages.rs | 4 +- .../prio_graph_scheduler.rs | 26 +++--- .../scheduler_controller.rs | 27 ++++-- .../transaction_state.rs | 4 +- .../unprocessed_transaction_storage.rs | 46 ++++++---- entry/src/entry.rs | 27 ++++-- ledger/src/blockstore.rs | 2 +- ledger/src/blockstore_processor.rs | 16 ++-- ledger/src/token_balances.rs | 6 +- rpc/src/rpc.rs | 13 ++- rpc/src/transaction_status_service.rs | 25 +++-- runtime/src/bank.rs | 91 +++++++++++-------- runtime/src/bank_utils.rs | 11 ++- runtime/src/installed_scheduler_pool.rs | 8 +- runtime/src/prioritization_fee_cache.rs | 15 ++- runtime/src/transaction_batch.rs | 8 +- sdk/src/transaction/sanitized.rs | 17 ++++ svm/src/account_loader.rs | 6 +- svm/src/transaction_processor.rs | 13 +-- unified-scheduler-logic/src/lib.rs | 8 +- unified-scheduler-pool/src/lib.rs | 8 +- 28 files changed, 303 insertions(+), 196 deletions(-) diff --git a/accounts-db/src/accounts.rs b/accounts-db/src/accounts.rs index 33a57d56461c78..5702353f9fc142 100644 --- a/accounts-db/src/accounts.rs +++ b/accounts-db/src/accounts.rs @@ -23,7 +23,9 @@ use { nonce_info::{NonceFull, NonceInfo}, pubkey::Pubkey, slot_hashes::SlotHashes, - transaction::{Result, SanitizedTransaction, TransactionAccountLocks, TransactionError}, + transaction::{ + ExtendedSanitizedTransaction, Result, TransactionAccountLocks, TransactionError, + }, transaction_context::TransactionAccount, }, solana_svm::{ @@ -31,10 +33,7 @@ use { }, std::{ cmp::Reverse, - collections::{ - hash_map::{self}, - BinaryHeap, HashMap, HashSet, - }, + collections::{hash_map, BinaryHeap, HashMap, HashSet}, ops::RangeBounds, sync::{ atomic::{AtomicUsize, Ordering}, @@ -577,11 +576,11 @@ impl Accounts { #[allow(clippy::needless_collect)] pub fn lock_accounts<'a>( &self, - txs: impl Iterator, + txs: impl Iterator, tx_account_lock_limit: usize, ) -> Vec> { let tx_account_locks_results: Vec> = txs - .map(|tx| tx.get_account_locks(tx_account_lock_limit)) + .map(|tx| tx.transaction.get_account_locks(tx_account_lock_limit)) .collect(); self.lock_accounts_inner(tx_account_locks_results) } @@ -590,14 +589,14 @@ impl Accounts { #[allow(clippy::needless_collect)] pub fn lock_accounts_with_results<'a>( &self, - txs: impl Iterator, + txs: impl Iterator, results: impl Iterator>, tx_account_lock_limit: usize, ) -> Vec> { let tx_account_locks_results: Vec> = txs .zip(results) .map(|(tx, result)| match result { - Ok(()) => tx.get_account_locks(tx_account_lock_limit), + Ok(()) => tx.transaction.get_account_locks(tx_account_lock_limit), Err(err) => Err(err), }) .collect(); @@ -627,11 +626,11 @@ impl Accounts { #[allow(clippy::needless_collect)] pub fn unlock_accounts<'a>( &self, - txs_and_results: impl Iterator)>, + txs_and_results: impl Iterator)>, ) { let keys: Vec<_> = txs_and_results .filter(|(_, res)| res.is_ok()) - .map(|(tx, _)| tx.get_account_locks_unchecked()) + .map(|(tx, _)| tx.transaction.get_account_locks_unchecked()) .collect(); if keys.is_empty() { return; @@ -650,7 +649,7 @@ impl Accounts { pub fn store_cached( &self, slot: Slot, - txs: &[SanitizedTransaction], + txs: &[ExtendedSanitizedTransaction], res: &[TransactionExecutionResult], loaded: &mut [TransactionLoadResult], durable_nonce: &DurableNonce, @@ -677,14 +676,14 @@ impl Accounts { #[allow(clippy::too_many_arguments)] fn collect_accounts_to_store<'a>( &self, - txs: &'a [SanitizedTransaction], + txs: &'a [ExtendedSanitizedTransaction], execution_results: &'a [TransactionExecutionResult], load_results: &'a mut [TransactionLoadResult], durable_nonce: &DurableNonce, lamports_per_signature: u64, ) -> ( Vec<(&'a Pubkey, &'a AccountSharedData)>, - Vec>, + Vec>, ) { let mut accounts = Vec::with_capacity(load_results.len()); let mut transactions = Vec::with_capacity(load_results.len()); @@ -712,7 +711,7 @@ impl Accounts { } }; - let message = tx.message(); + let message = tx.transaction.message(); let loaded_transaction = tx_load_result.as_mut().unwrap(); let mut fee_payer_index = None; for (i, (address, account)) in (0..message.account_keys().len()) diff --git a/accounts-db/src/accounts_db.rs b/accounts-db/src/accounts_db.rs index 41ec05dce0e4a5..44ee7a1fdf4cb0 100644 --- a/accounts-db/src/accounts_db.rs +++ b/accounts-db/src/accounts_db.rs @@ -93,7 +93,7 @@ use { rent_collector::RentCollector, saturating_add_assign, timing::AtomicInterval, - transaction::SanitizedTransaction, + transaction::ExtendedSanitizedTransaction, }, std::{ borrow::{Borrow, Cow}, @@ -6622,7 +6622,7 @@ impl AccountsDb { &self, slot: Slot, accounts_and_meta_to_store: &impl StorableAccounts<'b, T>, - txn_iter: Box> + 'a>, + txn_iter: Box> + 'a>, mut write_version_producer: P, ) -> Vec where @@ -6640,7 +6640,7 @@ impl AccountsDb { self.notify_account_at_accounts_update( slot, &account, - txn, + &txn.map(|txn| &txn.transaction), accounts_and_meta_to_store.pubkey(i), &mut write_version_producer, ); @@ -6672,7 +6672,7 @@ impl AccountsDb { hashes: Option>>, mut write_version_producer: P, store_to: &StoreTo, - transactions: Option<&[Option<&'a SanitizedTransaction>]>, + transactions: Option<&[Option<&'a ExtendedSanitizedTransaction>]>, ) -> Vec { let mut calc_stored_meta_time = Measure::start("calc_stored_meta"); let slot = accounts.target_slot(); @@ -6687,14 +6687,15 @@ impl AccountsDb { match store_to { StoreTo::Cache => { - let txn_iter: Box>> = - match transactions { - Some(transactions) => { - assert_eq!(transactions.len(), accounts.len()); - Box::new(transactions.iter()) - } - None => Box::new(std::iter::repeat(&None).take(accounts.len())), - }; + let txn_iter: Box< + dyn std::iter::Iterator>, + > = match transactions { + Some(transactions) => { + assert_eq!(transactions.len(), accounts.len()); + Box::new(transactions.iter()) + } + None => Box::new(std::iter::repeat(&None).take(accounts.len())), + }; self.write_accounts_to_cache(slot, accounts, txn_iter, write_version_producer) } @@ -8379,7 +8380,7 @@ impl AccountsDb { pub fn store_cached<'a, T: ReadableAccount + Sync + ZeroLamport + 'a>( &self, accounts: impl StorableAccounts<'a, T>, - transactions: Option<&'a [Option<&'a SanitizedTransaction>]>, + transactions: Option<&'a [Option<&'a ExtendedSanitizedTransaction>]>, ) { self.store( accounts, @@ -8396,7 +8397,7 @@ impl AccountsDb { >( &self, accounts: impl StorableAccounts<'a, T>, - transactions: Option<&'a [Option<&'a SanitizedTransaction>]>, + transactions: Option<&'a [Option<&'a ExtendedSanitizedTransaction>]>, ) { self.store( accounts, @@ -8424,7 +8425,7 @@ impl AccountsDb { &self, accounts: impl StorableAccounts<'a, T>, store_to: &StoreTo, - transactions: Option<&'a [Option<&'a SanitizedTransaction>]>, + transactions: Option<&'a [Option<&'a ExtendedSanitizedTransaction>]>, reclaim: StoreReclaims, update_index_thread_selection: UpdateIndexThreadSelection, ) { @@ -8615,7 +8616,7 @@ impl AccountsDb { accounts: impl StorableAccounts<'a, T>, hashes: Option>>, store_to: &StoreTo, - transactions: Option<&'a [Option<&'a SanitizedTransaction>]>, + transactions: Option<&'a [Option<&'a ExtendedSanitizedTransaction>]>, reclaim: StoreReclaims, update_index_thread_selection: UpdateIndexThreadSelection, ) { @@ -8670,7 +8671,7 @@ impl AccountsDb { write_version_producer: Option>>, store_to: &StoreTo, reset_accounts: bool, - transactions: Option<&[Option<&SanitizedTransaction>]>, + transactions: Option<&[Option<&ExtendedSanitizedTransaction>]>, reclaim: StoreReclaims, update_index_thread_selection: UpdateIndexThreadSelection, ) -> StoreAccountsTiming { diff --git a/banks-server/src/banks_server.rs b/banks-server/src/banks_server.rs index b3028c0132ed48..973c591e622edb 100644 --- a/banks-server/src/banks_server.rs +++ b/banks-server/src/banks_server.rs @@ -23,7 +23,10 @@ use { message::{Message, SanitizedMessage}, pubkey::Pubkey, signature::Signature, - transaction::{self, MessageHash, SanitizedTransaction, VersionedTransaction}, + transaction::{ + self, ExtendedSanitizedTransaction, MessageHash, SanitizedTransaction, + VersionedTransaction, + }, }, solana_send_transaction_service::{ send_transaction_service::{SendTransactionService, TransactionInfo}, @@ -194,7 +197,10 @@ fn simulate_transaction( units_consumed, return_data, inner_instructions, - } = bank.simulate_transaction_unchecked(&sanitized_transaction, false); + } = bank.simulate_transaction_unchecked( + &ExtendedSanitizedTransaction::from(sanitized_transaction), + false, + ); let simulation_details = TransactionSimulationDetails { logs, diff --git a/core/src/banking_stage/consumer.rs b/core/src/banking_stage/consumer.rs index 9e9475f7678e3c..0c5ae73c5ba651 100644 --- a/core/src/banking_stage/consumer.rs +++ b/core/src/banking_stage/consumer.rs @@ -29,7 +29,7 @@ use { message::SanitizedMessage, saturating_add_assign, timing::timestamp, - transaction::{self, AddressLoader, SanitizedTransaction, TransactionError}, + transaction::{self, AddressLoader, ExtendedSanitizedTransaction, TransactionError}, }, solana_svm::{ account_loader::{validate_fee_payer, TransactionCheckResult}, @@ -242,7 +242,7 @@ impl Consumer { &self, bank: &Arc, bank_creation_time: &Instant, - sanitized_transactions: &[SanitizedTransaction], + sanitized_transactions: &[ExtendedSanitizedTransaction], banking_stage_stats: &BankingStageStats, slot_metrics_tracker: &mut LeaderSlotMetricsTracker, ) -> ProcessTransactionsSummary { @@ -298,7 +298,7 @@ impl Consumer { &self, bank: &Arc, bank_creation_time: &Instant, - transactions: &[SanitizedTransaction], + transactions: &[ExtendedSanitizedTransaction], ) -> ProcessTransactionsSummary { let mut chunk_start = 0; let mut all_retryable_tx_indexes = vec![]; @@ -432,7 +432,7 @@ impl Consumer { pub fn process_and_record_transactions( &self, bank: &Arc, - txs: &[SanitizedTransaction], + txs: &[ExtendedSanitizedTransaction], chunk_offset: usize, ) -> ProcessTransactionBatchOutput { let mut error_counters = TransactionErrorMetrics::default(); @@ -460,7 +460,7 @@ impl Consumer { pub fn process_and_record_aged_transactions( &self, bank: &Arc, - txs: &[SanitizedTransaction], + txs: &[ExtendedSanitizedTransaction], max_slot_ages: &[Slot], ) -> ProcessTransactionBatchOutput { // Need to filter out transactions since they were sanitized earlier. @@ -472,7 +472,7 @@ impl Consumer { // Re-sanitized transaction should be equal to the original transaction, // but whether it will pass sanitization needs to be checked. let resanitized_tx = - bank.fully_verify_transaction(tx.to_versioned_transaction())?; + bank.fully_verify_transaction(tx.transaction.to_versioned_transaction())?; if resanitized_tx != *tx { // Sanitization before/after epoch give different transaction data - do not execute. return Err(TransactionError::ResanitizationNeeded); @@ -480,7 +480,7 @@ impl Consumer { } else { // Any transaction executed between sanitization time and now may have closed the lookup table(s). // Above re-sanitization already loads addresses, so don't need to re-check in that case. - let lookup_tables = tx.message().message_address_table_lookups(); + let lookup_tables = tx.transaction.message().message_address_table_lookups(); if !lookup_tables.is_empty() { bank.load_addresses(lookup_tables)?; } @@ -493,7 +493,7 @@ impl Consumer { fn process_and_record_transactions_with_pre_results( &self, bank: &Arc, - txs: &[SanitizedTransaction], + txs: &[ExtendedSanitizedTransaction], chunk_offset: usize, pre_results: impl Iterator>, ) -> ProcessTransactionBatchOutput { @@ -609,6 +609,7 @@ impl Consumer { .filter_map(|transaction| { let round_compute_unit_price_enabled = false; // TODO get from working_bank.feature_set transaction + .transaction .get_compute_budget_details(round_compute_unit_price_enabled) .map(|details| details.compute_unit_price) }) @@ -647,7 +648,7 @@ impl Consumer { .zip(batch.sanitized_transactions()) .filter_map(|(execution_result, tx)| { if execution_result.was_executed() { - Some(tx.to_versioned_transaction()) + Some(tx.transaction.to_versioned_transaction()) } else { None } @@ -811,7 +812,7 @@ impl Consumer { /// * `pending_indexes` - identifies which indexes in the `transactions` list are still pending fn filter_pending_packets_from_pending_txs( bank: &Bank, - transactions: &[SanitizedTransaction], + transactions: &[ExtendedSanitizedTransaction], pending_indexes: &[usize], ) -> Vec { let filter = diff --git a/core/src/banking_stage/forward_packet_batches_by_accounts.rs b/core/src/banking_stage/forward_packet_batches_by_accounts.rs index 54efb27ce56129..ab8f52820e8eac 100644 --- a/core/src/banking_stage/forward_packet_batches_by_accounts.rs +++ b/core/src/banking_stage/forward_packet_batches_by_accounts.rs @@ -6,7 +6,7 @@ use { cost_tracker::{CostTracker, CostTrackerError}, }, solana_perf::packet::Packet, - solana_sdk::{feature_set::FeatureSet, transaction::SanitizedTransaction}, + solana_sdk::{feature_set::FeatureSet, transaction::ExtendedSanitizedTransaction}, std::sync::Arc, }; @@ -58,11 +58,11 @@ impl ForwardBatch { fn try_add( &mut self, - sanitized_transaction: &SanitizedTransaction, + sanitized_transaction: &ExtendedSanitizedTransaction, immutable_packet: Arc, feature_set: &FeatureSet, ) -> Result { - let tx_cost = CostModel::calculate_cost(sanitized_transaction, feature_set); + let tx_cost = CostModel::calculate_cost(&sanitized_transaction.transaction, feature_set); let res = self.cost_tracker.try_add(&tx_cost); if res.is_ok() { self.forwardable_packets.push(immutable_packet); @@ -112,7 +112,7 @@ impl ForwardPacketBatchesByAccounts { /// packets are filled into first available 'batch' that have space to fit it. pub fn try_add_packet( &mut self, - sanitized_transaction: &SanitizedTransaction, + sanitized_transaction: &ExtendedSanitizedTransaction, immutable_packet: Arc, feature_set: &FeatureSet, ) -> bool { diff --git a/core/src/banking_stage/latest_unprocessed_votes.rs b/core/src/banking_stage/latest_unprocessed_votes.rs index a62e5bf9b3e455..fecec111bd660c 100644 --- a/core/src/banking_stage/latest_unprocessed_votes.rs +++ b/core/src/banking_stage/latest_unprocessed_votes.rs @@ -11,6 +11,7 @@ use { clock::{Slot, UnixTimestamp}, program_utils::limited_deserialize, pubkey::Pubkey, + transaction::ExtendedSanitizedTransaction, }, solana_vote_program::vote_instruction::VoteInstruction, std::{ @@ -286,7 +287,7 @@ impl LatestUnprocessedVotes { ) { if forward_packet_batches_by_accounts.try_add_packet( - &sanitized_vote_transaction, + &ExtendedSanitizedTransaction::from(sanitized_vote_transaction), deserialized_vote_packet, &bank.feature_set, ) { diff --git a/core/src/banking_stage/qos_service.rs b/core/src/banking_stage/qos_service.rs index 8c1507ae3fb91c..5532d36eb39ecd 100644 --- a/core/src/banking_stage/qos_service.rs +++ b/core/src/banking_stage/qos_service.rs @@ -12,7 +12,7 @@ use { clock::Slot, feature_set::FeatureSet, saturating_add_assign, - transaction::{self, SanitizedTransaction, TransactionError}, + transaction::{self, ExtendedSanitizedTransaction, TransactionError}, }, std::sync::atomic::{AtomicU64, Ordering}, }; @@ -40,7 +40,7 @@ impl QosService { pub fn select_and_accumulate_transaction_costs( &self, bank: &Bank, - transactions: &[SanitizedTransaction], + transactions: &[ExtendedSanitizedTransaction], pre_results: impl Iterator>, ) -> (Vec>, usize) { let transaction_costs = @@ -67,13 +67,15 @@ impl QosService { fn compute_transaction_costs<'a>( &self, feature_set: &FeatureSet, - transactions: impl Iterator, + transactions: impl Iterator, pre_results: impl Iterator>, ) -> Vec> { let mut compute_cost_time = Measure::start("compute_cost_time"); let txs_costs: Vec<_> = transactions .zip(pre_results) - .map(|(tx, pre_result)| pre_result.map(|()| CostModel::calculate_cost(tx, feature_set))) + .map(|(tx, pre_result)| { + pre_result.map(|()| CostModel::calculate_cost(&tx.transaction, feature_set)) + }) .collect(); compute_cost_time.stop(); self.metrics @@ -92,7 +94,7 @@ impl QosService { /// and a count of the number of transactions that would fit in the block fn select_transactions_per_cost<'a>( &self, - transactions: impl Iterator, + transactions: impl Iterator, transactions_costs: impl Iterator>, bank: &Bank, ) -> (Vec>, usize) { diff --git a/core/src/banking_stage/scheduler_messages.rs b/core/src/banking_stage/scheduler_messages.rs index 172087e2cf8e82..359f4cac710b71 100644 --- a/core/src/banking_stage/scheduler_messages.rs +++ b/core/src/banking_stage/scheduler_messages.rs @@ -1,6 +1,6 @@ use { super::immutable_deserialized_packet::ImmutableDeserializedPacket, - solana_sdk::{clock::Slot, transaction::SanitizedTransaction}, + solana_sdk::{clock::Slot, transaction::ExtendedSanitizedTransaction}, std::{fmt::Display, sync::Arc}, }; @@ -41,7 +41,7 @@ impl Display for TransactionId { pub struct ConsumeWork { pub batch_id: TransactionBatchId, pub ids: Vec, - pub transactions: Vec, + pub transactions: Vec, pub max_age_slots: Vec, } diff --git a/core/src/banking_stage/transaction_scheduler/prio_graph_scheduler.rs b/core/src/banking_stage/transaction_scheduler/prio_graph_scheduler.rs index d983fcf4d163c3..95408800852b84 100644 --- a/core/src/banking_stage/transaction_scheduler/prio_graph_scheduler.rs +++ b/core/src/banking_stage/transaction_scheduler/prio_graph_scheduler.rs @@ -18,7 +18,7 @@ use { solana_measure::measure_us, solana_sdk::{ pubkey::Pubkey, saturating_add_assign, slot_history::Slot, - transaction::SanitizedTransaction, + transaction::ExtendedSanitizedTransaction, }, }; @@ -64,8 +64,8 @@ impl PrioGraphScheduler { pub(crate) fn schedule( &mut self, container: &mut TransactionStateContainer, - pre_graph_filter: impl Fn(&[&SanitizedTransaction], &mut [bool]), - pre_lock_filter: impl Fn(&SanitizedTransaction) -> bool, + pre_graph_filter: impl Fn(&[&ExtendedSanitizedTransaction], &mut [bool]), + pre_lock_filter: impl Fn(&ExtendedSanitizedTransaction) -> bool, ) -> Result { let num_threads = self.consume_work_senders.len(); let mut batches = Batches::new(num_threads); @@ -161,15 +161,15 @@ impl PrioGraphScheduler { } // Check if this transaction conflicts with any blocked transactions - if !blocking_locks.check_locks(transaction.message()) { - blocking_locks.take_locks(transaction.message()); + if !blocking_locks.check_locks(transaction.transaction.message()) { + blocking_locks.take_locks(transaction.transaction.message()); unschedulable_ids.push(id); saturating_add_assign!(num_unschedulable, 1); continue; } // Schedule the transaction if it can be. - let transaction_locks = transaction.get_account_locks_unchecked(); + let transaction_locks = transaction.transaction.get_account_locks_unchecked(); let Some(thread_id) = self.account_locks.try_lock_accounts( transaction_locks.writable.into_iter(), transaction_locks.readonly.into_iter(), @@ -182,7 +182,7 @@ impl PrioGraphScheduler { ) }, ) else { - blocking_locks.take_locks(transaction.message()); + blocking_locks.take_locks(transaction.transaction.message()); unschedulable_ids.push(id); saturating_add_assign!(num_unschedulable, 1); continue; @@ -329,11 +329,11 @@ impl PrioGraphScheduler { fn complete_batch( &mut self, batch_id: TransactionBatchId, - transactions: &[SanitizedTransaction], + transactions: &[ExtendedSanitizedTransaction], ) { let thread_id = self.in_flight_tracker.complete_batch(batch_id); for transaction in transactions { - let account_locks = transaction.get_account_locks_unchecked(); + let account_locks = transaction.transaction.get_account_locks_unchecked(); self.account_locks.unlock_accounts( account_locks.writable.into_iter(), account_locks.readonly.into_iter(), @@ -392,7 +392,7 @@ impl PrioGraphScheduler { /// on `ThreadAwareAccountLocks::try_lock_accounts`. fn select_thread( thread_set: ThreadSet, - batches_per_thread: &[Vec], + batches_per_thread: &[Vec], in_flight_per_thread: &[usize], ) -> ThreadId { thread_set @@ -412,7 +412,7 @@ impl PrioGraphScheduler { fn get_transaction_account_access( transaction: &SanitizedTransactionTTL, ) -> impl Iterator + '_ { - let message = transaction.transaction.message(); + let message = transaction.transaction.transaction.message(); message .account_keys() .iter() @@ -442,7 +442,7 @@ pub(crate) struct SchedulingSummary { struct Batches { ids: Vec>, - transactions: Vec>, + transactions: Vec>, max_age_slots: Vec>, total_cus: Vec, } @@ -462,7 +462,7 @@ impl Batches { thread_id: ThreadId, ) -> ( Vec, - Vec, + Vec, Vec, u64, ) { diff --git a/core/src/banking_stage/transaction_scheduler/scheduler_controller.rs b/core/src/banking_stage/transaction_scheduler/scheduler_controller.rs index 12e8f7bf8bf0bf..600c7ef6c3f3a3 100644 --- a/core/src/banking_stage/transaction_scheduler/scheduler_controller.rs +++ b/core/src/banking_stage/transaction_scheduler/scheduler_controller.rs @@ -31,7 +31,7 @@ use { }, fee::FeeBudgetLimits, saturating_add_assign, - transaction::SanitizedTransaction, + transaction::{ExtendedSanitizedTransaction, SanitizedTransaction}, }, solana_svm::transaction_error_metrics::TransactionErrorMetrics, std::{ @@ -189,7 +189,11 @@ impl SchedulerController { Ok(()) } - fn pre_graph_filter(transactions: &[&SanitizedTransaction], results: &mut [bool], bank: &Bank) { + fn pre_graph_filter( + transactions: &[&ExtendedSanitizedTransaction], + results: &mut [bool], + bank: &Bank, + ) { let lock_results = vec![Ok(()); transactions.len()]; let mut error_counters = TransactionErrorMetrics::default(); let check_results = bank.check_transactions( @@ -204,7 +208,11 @@ impl SchedulerController { .zip(transactions) .map(|((result, _nonce, _lamports), tx)| { result?; // if there's already error do nothing - Consumer::check_fee_payer_unlocked(bank, tx.message(), &mut error_counters) + Consumer::check_fee_payer_unlocked( + bank, + tx.transaction.message(), + &mut error_counters, + ) }) .collect(); @@ -387,7 +395,12 @@ impl SchedulerController { }) .filter_map(|tx| { process_compute_budget_instructions(tx.message().program_instructions_iter()) - .map(|compute_budget| (tx, compute_budget.into())) + .map(|compute_budget| { + ( + ExtendedSanitizedTransaction::from(tx), + compute_budget.into(), + ) + }) .ok() }) .unzip(); @@ -478,13 +491,13 @@ impl SchedulerController { /// Any difference in the prioritization is negligible for /// the current transaction costs. fn calculate_priority_and_cost( - transaction: &SanitizedTransaction, + transaction: &ExtendedSanitizedTransaction, fee_budget_limits: &FeeBudgetLimits, bank: &Bank, ) -> (u64, u64) { - let cost = CostModel::calculate_cost(transaction, &bank.feature_set).sum(); + let cost = CostModel::calculate_cost(&transaction.transaction, &bank.feature_set).sum(); let fee = bank.fee_structure.calculate_fee( - transaction.message(), + transaction.transaction.message(), 5_000, // this just needs to be non-zero fee_budget_limits, bank.feature_set diff --git a/core/src/banking_stage/transaction_scheduler/transaction_state.rs b/core/src/banking_stage/transaction_scheduler/transaction_state.rs index 727140545ab656..7ae094258a9bad 100644 --- a/core/src/banking_stage/transaction_scheduler/transaction_state.rs +++ b/core/src/banking_stage/transaction_scheduler/transaction_state.rs @@ -1,8 +1,8 @@ -use solana_sdk::{clock::Slot, transaction::SanitizedTransaction}; +use solana_sdk::{clock::Slot, transaction::ExtendedSanitizedTransaction}; /// Simple wrapper type to tie a sanitized transaction to max age slot. pub(crate) struct SanitizedTransactionTTL { - pub(crate) transaction: SanitizedTransaction, + pub(crate) transaction: ExtendedSanitizedTransaction, pub(crate) max_age_slot: Slot, } diff --git a/core/src/banking_stage/unprocessed_transaction_storage.rs b/core/src/banking_stage/unprocessed_transaction_storage.rs index 52706f8c2bf63b..7c50e045b89f62 100644 --- a/core/src/banking_stage/unprocessed_transaction_storage.rs +++ b/core/src/banking_stage/unprocessed_transaction_storage.rs @@ -20,8 +20,11 @@ use { solana_measure::{measure, measure_us}, solana_runtime::bank::Bank, solana_sdk::{ - clock::FORWARD_TRANSACTIONS_TO_LEADER_AT_SLOT_OFFSET, feature_set::FeatureSet, hash::Hash, - saturating_add_assign, transaction::SanitizedTransaction, + clock::FORWARD_TRANSACTIONS_TO_LEADER_AT_SLOT_OFFSET, + feature_set::FeatureSet, + hash::Hash, + saturating_add_assign, + transaction::{ExtendedSanitizedTransaction, SanitizedTransaction}, }, solana_svm::transaction_error_metrics::TransactionErrorMetrics, std::{ @@ -135,7 +138,7 @@ fn filter_processed_packets<'a, F>( pub struct ConsumeScannerPayload<'a> { pub reached_end_of_slot: bool, pub account_locks: ReadWriteAccountSet, - pub sanitized_transactions: Vec, + pub sanitized_transactions: Vec, pub slot_metrics_tracker: &'a mut LeaderSlotMetricsTracker, pub message_hash_to_transaction: &'a mut HashMap, pub error_counters: TransactionErrorMetrics, @@ -208,7 +211,9 @@ fn consume_scan_should_process_packet( return ProcessingDecision::Later; } - payload.sanitized_transactions.push(sanitized_transaction); + payload + .sanitized_transactions + .push(ExtendedSanitizedTransaction::from(sanitized_transaction)); ProcessingDecision::Now } else { payload @@ -635,7 +640,7 @@ impl ThreadLocalUnprocessedPackets { (sanitized_transactions, transaction_to_packet_indexes), packet_conversion_time, ): ( - (Vec, Vec), + (Vec, Vec), _, ) = measure!( self.sanitize_unforwarded_packets( @@ -762,18 +767,25 @@ impl ThreadLocalUnprocessedPackets { packets_to_process: &[Arc], bank: &Bank, total_dropped_packets: &mut usize, - ) -> (Vec, Vec) { + ) -> (Vec, Vec) { // Get ref of ImmutableDeserializedPacket let deserialized_packets = packets_to_process.iter().map(|p| &**p); - let (transactions, transaction_to_packet_indexes): (Vec, Vec) = - deserialized_packets - .enumerate() - .filter_map(|(packet_index, deserialized_packet)| { - deserialized_packet - .build_sanitized_transaction(&bank.feature_set, bank.vote_only_bank(), bank) - .map(|transaction| (transaction, packet_index)) - }) - .unzip(); + let (transactions, transaction_to_packet_indexes): ( + Vec, + Vec, + ) = deserialized_packets + .enumerate() + .filter_map(|(packet_index, deserialized_packet)| { + deserialized_packet + .build_sanitized_transaction(&bank.feature_set, bank.vote_only_bank(), bank) + .map(|transaction| { + ( + ExtendedSanitizedTransaction::from(transaction), + packet_index, + ) + }) + }) + .unzip(); let filtered_count = packets_to_process.len().saturating_sub(transactions.len()); saturating_add_assign!(*total_dropped_packets, filtered_count); @@ -783,7 +795,7 @@ impl ThreadLocalUnprocessedPackets { /// Checks sanitized transactions against bank, returns valid transaction indexes fn filter_invalid_transactions( - transactions: &[SanitizedTransaction], + transactions: &[ExtendedSanitizedTransaction], bank: &Bank, total_dropped_packets: &mut usize, ) -> Vec { @@ -821,7 +833,7 @@ impl ThreadLocalUnprocessedPackets { fn add_filtered_packets_to_forward_buffer( forward_buffer: &mut ForwardPacketBatchesByAccounts, packets_to_process: &[Arc], - transactions: &[SanitizedTransaction], + transactions: &[ExtendedSanitizedTransaction], transaction_to_packet_indexes: &[usize], forwardable_transaction_indexes: &[usize], total_dropped_packets: &mut usize, diff --git a/entry/src/entry.rs b/entry/src/entry.rs index af3fdca9518e83..0029148dcee18a 100644 --- a/entry/src/entry.rs +++ b/entry/src/entry.rs @@ -27,7 +27,7 @@ use { packet::Meta, timing, transaction::{ - Result, SanitizedTransaction, Transaction, TransactionError, + ExtendedSanitizedTransaction, Result, Transaction, TransactionError, TransactionVerificationMode, VersionedTransaction, }, }, @@ -163,7 +163,7 @@ impl From<&Entry> for EntrySummary { /// Typed entry to distinguish between transaction and tick entries pub enum EntryType { - Transactions(Vec), + Transactions(Vec), Tick(Hash), } @@ -405,7 +405,7 @@ impl EntryVerificationState { pub fn verify_transactions( entries: Vec, - verify: Arc Result + Send + Sync>, + verify: Arc Result + Send + Sync>, ) -> Result> { PAR_THREAD_POOL.install(|| { entries @@ -432,7 +432,10 @@ pub fn start_verify_transactions( skip_verification: bool, verify_recyclers: VerifyRecyclers, verify: Arc< - dyn Fn(VersionedTransaction, TransactionVerificationMode) -> Result + dyn Fn( + VersionedTransaction, + TransactionVerificationMode, + ) -> Result + Send + Sync, >, @@ -469,7 +472,10 @@ fn start_verify_transactions_cpu( entries: Vec, skip_verification: bool, verify: Arc< - dyn Fn(VersionedTransaction, TransactionVerificationMode) -> Result + dyn Fn( + VersionedTransaction, + TransactionVerificationMode, + ) -> Result + Send + Sync, >, @@ -498,13 +504,16 @@ fn start_verify_transactions_gpu( entries: Vec, verify_recyclers: VerifyRecyclers, verify: Arc< - dyn Fn(VersionedTransaction, TransactionVerificationMode) -> Result + dyn Fn( + VersionedTransaction, + TransactionVerificationMode, + ) -> Result + Send + Sync, >, ) -> Result { let verify_func = { - move |versioned_tx: VersionedTransaction| -> Result { + move |versioned_tx: VersionedTransaction| -> Result { verify( versioned_tx, TransactionVerificationMode::HashAndVerifyPrecompiles, @@ -514,7 +523,7 @@ fn start_verify_transactions_gpu( let entries = verify_transactions(entries, Arc::new(verify_func))?; - let entry_txs: Vec<&SanitizedTransaction> = entries + let entry_txs: Vec<&ExtendedSanitizedTransaction> = entries .iter() .filter_map(|entry_type| match entry_type { EntryType::Tick(_) => None, @@ -552,7 +561,7 @@ fn start_verify_transactions_gpu( } let entry_tx_iter = slice .into_par_iter() - .map(|tx| tx.to_versioned_transaction()); + .map(|tx| tx.transaction.to_versioned_transaction()); let res = packet_batch .par_iter_mut() diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index f15976abdb241b..24f94f762b137a 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -3095,7 +3095,7 @@ impl Blockstore { // Attempt to verify transaction and load addresses from the current bank, // or manually scan the transaction for addresses if the transaction. if let Ok(tx) = bank.fully_verify_transaction(tx.clone()) { - add_to_set(&result, tx.message().account_keys().iter()); + add_to_set(&result, tx.transaction.message().account_keys().iter()); } else { add_to_set(&result, tx.message.static_account_keys()); diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs index e4ae5f368b2afd..d478c8fa6dfa1b 100644 --- a/ledger/src/blockstore_processor.rs +++ b/ledger/src/blockstore_processor.rs @@ -53,7 +53,7 @@ use { signature::{Keypair, Signature}, timing, transaction::{ - Result, SanitizedTransaction, TransactionError, TransactionVerificationMode, + ExtendedSanitizedTransaction, Result, TransactionError, TransactionVerificationMode, VersionedTransaction, }, }, @@ -120,7 +120,7 @@ fn get_first_error( { if let Err(ref err) = result { if first_err.is_none() { - first_err = Some((result.clone(), *transaction.signature())); + first_err = Some((result.clone(), *transaction.transaction.signature())); } warn!( "Unexpected validator error: {:?}, transaction: {:?}", @@ -379,7 +379,7 @@ fn schedule_batches_for_execution( fn rebatch_transactions<'a>( lock_results: &'a [Result<()>], bank: &'a Arc, - sanitized_txs: &'a [SanitizedTransaction], + sanitized_txs: &'a [ExtendedSanitizedTransaction], start: usize, end: usize, transaction_indexes: &'a [usize], @@ -427,7 +427,7 @@ fn rebatch_and_execute_batches( let tx_costs = sanitized_txs .iter() .map(|tx| { - let tx_cost = CostModel::calculate_cost(tx, &bank.feature_set); + let tx_cost = CostModel::calculate_cost(&tx.transaction, &bank.feature_set); let cost = tx_cost.sum(); minimal_tx_cost = std::cmp::min(minimal_tx_cost, cost); total_cost = total_cost.saturating_add(cost); @@ -508,7 +508,7 @@ pub fn process_entries_for_tests( ) -> Result<()> { let verify_transaction = { let bank = bank.clone_with_scheduler(); - move |versioned_tx: VersionedTransaction| -> Result { + move |versioned_tx: VersionedTransaction| -> Result { bank.verify_transaction(versioned_tx, TransactionVerificationMode::FullVerification) } }; @@ -1287,7 +1287,7 @@ fn confirm_slot_entries( let bank = bank.clone_with_scheduler(); move |versioned_tx: VersionedTransaction, verification_mode: TransactionVerificationMode| - -> Result { + -> Result { bank.verify_transaction(versioned_tx, verification_mode) } }; @@ -1834,7 +1834,7 @@ pub enum TransactionStatusMessage { pub struct TransactionStatusBatch { pub bank: Arc, - pub transactions: Vec, + pub transactions: Vec, pub execution_results: Vec>, pub balances: TransactionBalancesSet, pub token_balances: TransactionTokenBalancesSet, @@ -1851,7 +1851,7 @@ impl TransactionStatusSender { pub fn send_transaction_status_batch( &self, bank: Arc, - transactions: Vec, + transactions: Vec, execution_results: Vec, balances: TransactionBalancesSet, token_balances: TransactionTokenBalancesSet, diff --git a/ledger/src/token_balances.rs b/ledger/src/token_balances.rs index 204bd4335972aa..ff133127a14b4a 100644 --- a/ledger/src/token_balances.rs +++ b/ledger/src/token_balances.rs @@ -43,13 +43,15 @@ pub fn collect_token_balances( let mut collect_time = Measure::start("collect_token_balances"); for transaction in batch.sanitized_transactions() { - let account_keys = transaction.message().account_keys(); + let account_keys = transaction.transaction.message().account_keys(); let has_token_program = account_keys.iter().any(is_known_spl_token_id); let mut transaction_balances: Vec = vec![]; if has_token_program { for (index, account_id) in account_keys.iter().enumerate() { - if transaction.message().is_invoked(index) || is_known_spl_token_id(account_id) { + if transaction.transaction.message().is_invoked(index) + || is_known_spl_token_id(account_id) + { continue; } diff --git a/rpc/src/rpc.rs b/rpc/src/rpc.rs index 45038868502771..c8100e29a622c5 100644 --- a/rpc/src/rpc.rs +++ b/rpc/src/rpc.rs @@ -3252,7 +3252,10 @@ pub mod rpc_accounts_scan { pub mod rpc_full { use { super::*, - solana_sdk::message::{SanitizedVersionedMessage, VersionedMessage}, + solana_sdk::{ + message::{SanitizedVersionedMessage, VersionedMessage}, + transaction::ExtendedSanitizedTransaction, + }, solana_transaction_status::UiInnerInstructions, }; #[rpc] @@ -3665,7 +3668,8 @@ pub mod rpc_full { units_consumed, return_data, inner_instructions: _, // Always `None` due to `enable_cpi_recording = false` - } = preflight_bank.simulate_transaction(&transaction, false) + } = preflight_bank + .simulate_transaction(&ExtendedSanitizedTransaction::from(transaction), false) { match err { TransactionError::BlockhashNotFound => { @@ -3741,8 +3745,9 @@ pub mod rpc_full { } let transaction = sanitize_transaction(unsanitized_tx, bank)?; + let transaction = ExtendedSanitizedTransaction::from(transaction); if sig_verify { - verify_transaction(&transaction, &bank.feature_set)?; + verify_transaction(&transaction.transaction, &bank.feature_set)?; } let TransactionSimulationResult { @@ -3754,7 +3759,7 @@ pub mod rpc_full { inner_instructions, } = bank.simulate_transaction(&transaction, enable_cpi_recording); - let account_keys = transaction.message().account_keys(); + let account_keys = transaction.transaction.message().account_keys(); let number_of_accounts = account_keys.len(); let accounts = if let Some(config_accounts) = config_accounts { diff --git a/rpc/src/transaction_status_service.rs b/rpc/src/transaction_status_service.rs index 8730fb2ed0f3d8..29a20542379ba3 100644 --- a/rpc/src/transaction_status_service.rs +++ b/rpc/src/transaction_status_service.rs @@ -110,15 +110,16 @@ impl TransactionStatusService { } Some(DurableNonceFee::Invalid) => None, None => bank.get_lamports_per_signature_for_blockhash( - transaction.message().recent_blockhash(), + transaction.transaction.message().recent_blockhash(), ), } .expect("lamports_per_signature must be available"); let fee = bank.get_fee_for_message_with_lamports_per_signature( - transaction.message(), + transaction.transaction.message(), lamports_per_signature, ); - let tx_account_locks = transaction.get_account_locks_unchecked(); + let tx_account_locks = + transaction.transaction.get_account_locks_unchecked(); let inner_instructions = inner_instructions.map(|inner_instructions| { map_inner_instructions(inner_instructions).collect() @@ -138,7 +139,7 @@ impl TransactionStatusService { }) .collect(), ); - let loaded_addresses = transaction.get_loaded_addresses(); + let loaded_addresses = transaction.transaction.get_loaded_addresses(); let mut transaction_status_meta = TransactionStatusMeta { status, fee, @@ -158,9 +159,9 @@ impl TransactionStatusService { transaction_notifier.notify_transaction( slot, transaction_index, - transaction.signature(), + transaction.transaction.signature(), &transaction_status_meta, - &transaction, + &transaction.transaction, ); } @@ -172,16 +173,22 @@ impl TransactionStatusService { } if enable_rpc_transaction_history { - if let Some(memos) = extract_and_fmt_memos(transaction.message()) { + if let Some(memos) = + extract_and_fmt_memos(transaction.transaction.message()) + { blockstore - .write_transaction_memos(transaction.signature(), slot, memos) + .write_transaction_memos( + transaction.transaction.signature(), + slot, + memos, + ) .expect("Expect database write to succeed: TransactionMemos"); } blockstore .write_transaction_status( slot, - *transaction.signature(), + *transaction.transaction.signature(), tx_account_locks.writable, tx_account_locks.readonly, transaction_status_meta, diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index f0ba75defa0517..b568fc809420a7 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -152,8 +152,9 @@ use { sysvar::{self, last_restart_slot::LastRestartSlot, Sysvar, SysvarId}, timing::years_as_slots, transaction::{ - self, MessageHash, Result, SanitizedTransaction, Transaction, TransactionError, - TransactionVerificationMode, VersionedTransaction, MAX_TX_ACCOUNT_LOCKS, + self, ExtendedSanitizedTransaction, MessageHash, Result, SanitizedTransaction, + Transaction, TransactionError, TransactionVerificationMode, VersionedTransaction, + MAX_TX_ACCOUNT_LOCKS, }, transaction_context::{TransactionAccount, TransactionReturnData}, }, @@ -4081,7 +4082,7 @@ impl Bank { fn update_transaction_statuses( &self, - sanitized_txs: &[SanitizedTransaction], + sanitized_txs: &[ExtendedSanitizedTransaction], execution_results: &[TransactionExecutionResult], ) { let mut status_cache = self.status_cache.write().unwrap(); @@ -4091,8 +4092,8 @@ impl Bank { // Add the message hash to the status cache to ensure that this message // won't be processed again with a different signature. status_cache.insert( - tx.message().recent_blockhash(), - tx.message_hash(), + tx.transaction.message().recent_blockhash(), + tx.transaction.message_hash(), self.slot(), details.status.clone(), ); @@ -4100,8 +4101,8 @@ impl Bank { // can be queried by transaction signature over RPC. In the future, this should // only be added for API nodes because voting validators don't need to do this. status_cache.insert( - tx.message().recent_blockhash(), - tx.signature(), + tx.transaction.message().recent_blockhash(), + tx.transaction.signature(), self.slot(), details.status.clone(), ); @@ -4202,7 +4203,16 @@ impl Bank { pub fn prepare_entry_batch(&self, txs: Vec) -> Result { let sanitized_txs = txs .into_iter() - .map(|tx| SanitizedTransaction::try_create(tx, MessageHash::Compute, None, self)) + .map(|tx| { + SanitizedTransaction::try_create(tx, MessageHash::Compute, None, self).and_then( + |txn| { + Ok(ExtendedSanitizedTransaction { + transaction: txn, + start_time: None, + }) + }, + ) + }) .collect::>>()?; let tx_account_lock_limit = self.get_transaction_account_lock_limit(); let lock_results = self @@ -4219,7 +4229,7 @@ impl Bank { /// Prepare a locked transaction batch from a list of sanitized transactions. pub fn prepare_sanitized_batch<'a, 'b>( &'a self, - txs: &'b [SanitizedTransaction], + txs: &'b [ExtendedSanitizedTransaction], ) -> TransactionBatch<'a, 'b> { let tx_account_lock_limit = self.get_transaction_account_lock_limit(); let lock_results = self @@ -4233,7 +4243,7 @@ impl Bank { /// limited packing status pub fn prepare_sanitized_batch_with_results<'a, 'b>( &'a self, - transactions: &'b [SanitizedTransaction], + transactions: &'b [ExtendedSanitizedTransaction], transaction_results: impl Iterator>, ) -> TransactionBatch<'a, 'b> { // this lock_results could be: Ok, AccountInUse, WouldExceedBlockMaxLimit or WouldExceedAccountMaxLimit @@ -4249,10 +4259,11 @@ impl Bank { /// Prepare a transaction batch from a single transaction without locking accounts pub fn prepare_unlocked_batch_from_single_tx<'a>( &'a self, - transaction: &'a SanitizedTransaction, + transaction: &'a ExtendedSanitizedTransaction, ) -> TransactionBatch<'_, '_> { let tx_account_lock_limit = self.get_transaction_account_lock_limit(); let lock_result = transaction + .transaction .get_account_locks(tx_account_lock_limit) .map(|_| ()); let mut batch = TransactionBatch::new( @@ -4267,7 +4278,7 @@ impl Bank { /// Run transactions against a frozen bank without committing the results pub fn simulate_transaction( &self, - transaction: &SanitizedTransaction, + transaction: &ExtendedSanitizedTransaction, enable_cpi_recording: bool, ) -> TransactionSimulationResult { assert!(self.is_frozen(), "simulation bank must be frozen"); @@ -4279,10 +4290,10 @@ impl Bank { /// is frozen, enabling use in single-Bank test frameworks pub fn simulate_transaction_unchecked( &self, - transaction: &SanitizedTransaction, + transaction: &ExtendedSanitizedTransaction, enable_cpi_recording: bool, ) -> TransactionSimulationResult { - let account_keys = transaction.message().account_keys(); + let account_keys = transaction.transaction.message().account_keys(); let number_of_accounts = account_keys.len(); let account_overrides = self.get_account_overrides_for_simulation(&account_keys); let batch = self.prepare_unlocked_batch_from_single_tx(transaction); @@ -4395,7 +4406,7 @@ impl Bank { fn check_age( &self, - sanitized_txs: &[impl core::borrow::Borrow], + sanitized_txs: &[impl core::borrow::Borrow], lock_results: &[Result<()>], max_age: usize, error_counters: &mut TransactionErrorMetrics, @@ -4409,7 +4420,7 @@ impl Bank { .zip(lock_results) .map(|(tx, lock_res)| match lock_res { Ok(()) => self.check_transaction_age( - tx.borrow(), + &tx.borrow().transaction, max_age, &next_durable_nonce, &hash_queue, @@ -4461,7 +4472,7 @@ impl Bank { fn check_status_cache( &self, - sanitized_txs: &[impl core::borrow::Borrow], + sanitized_txs: &[impl core::borrow::Borrow], lock_results: Vec, error_counters: &mut TransactionErrorMetrics, ) -> Vec { @@ -4472,7 +4483,7 @@ impl Bank { .map(|(sanitized_tx, (lock_result, nonce, lamports))| { let sanitized_tx = sanitized_tx.borrow(); if lock_result.is_ok() - && self.is_transaction_already_processed(sanitized_tx, &rcache) + && self.is_transaction_already_processed(&sanitized_tx.transaction, &rcache) { error_counters.already_processed += 1; return (Err(TransactionError::AlreadyProcessed), None, None); @@ -4525,7 +4536,7 @@ impl Bank { pub fn check_transactions( &self, - sanitized_txs: &[impl core::borrow::Borrow], + sanitized_txs: &[impl core::borrow::Borrow], lock_results: &[Result<()>], max_age: usize, error_counters: &mut TransactionErrorMetrics, @@ -4538,7 +4549,7 @@ impl Bank { let mut balances: TransactionBalances = vec![]; for transaction in batch.sanitized_transactions() { let mut transaction_balances: Vec = vec![]; - for account_key in transaction.message().account_keys().iter() { + for account_key in transaction.transaction.message().account_keys().iter() { transaction_balances.push(self.get_balance(account_key)); } balances.push(transaction_balances); @@ -4635,7 +4646,7 @@ impl Bank { let mut collect_logs_time = Measure::start("collect_logs_time"); for (execution_result, tx) in sanitized_output.execution_results.iter().zip(sanitized_txs) { if let Some(debug_keys) = &self.transaction_debug_keys { - for key in tx.message().account_keys().iter() { + for key in tx.transaction.message().account_keys().iter() { if debug_keys.contains(key) { let result = execution_result.flattened_result(); info!("slot: {} result: {:?} tx: {:?}", self.slot, result, tx); @@ -4644,7 +4655,7 @@ impl Bank { } } - let is_vote = tx.is_simple_vote_transaction(); + let is_vote = tx.transaction.is_simple_vote_transaction(); if execution_result.was_executed() // Skip log collection for unprocessed transactions && transaction_log_collector_config.filter != TransactionLogCollectorFilter::None @@ -4654,7 +4665,7 @@ impl Bank { .mentioned_addresses .is_empty() { - for key in tx.message().account_keys().iter() { + for key in tx.transaction.message().account_keys().iter() { if transaction_log_collector_config .mentioned_addresses .contains(key) @@ -4687,7 +4698,7 @@ impl Bank { let transaction_log_index = transaction_log_collector.logs.len(); transaction_log_collector.logs.push(TransactionLogInfo { - signature: *tx.signature(), + signature: *tx.transaction.signature(), result: status.clone(), is_vote, log_messages: log_messages.clone(), @@ -4707,7 +4718,8 @@ impl Bank { // Signature count must be accumulated only if the transaction // is executed, otherwise a mismatched count between banking and // replay could occur - signature_count += u64::from(tx.message().header().num_required_signatures); + signature_count += + u64::from(tx.transaction.message().header().num_required_signatures); executed_transactions_count += 1; } @@ -4817,7 +4829,7 @@ impl Bank { fn filter_program_errors_and_collect_fee( &self, - txs: &[SanitizedTransaction], + txs: &[ExtendedSanitizedTransaction], execution_results: &[TransactionExecutionResult], ) -> Vec> { let hash_queue = self.blockhash_queue.read().unwrap(); @@ -4839,7 +4851,9 @@ impl Bank { .map(|maybe_lamports_per_signature| (maybe_lamports_per_signature, true)) .unwrap_or_else(|| { ( - hash_queue.get_lamports_per_signature(tx.message().recent_blockhash()), + hash_queue.get_lamports_per_signature( + tx.transaction.message().recent_blockhash(), + ), false, ) }); @@ -4847,7 +4861,7 @@ impl Bank { let lamports_per_signature = lamports_per_signature.ok_or(TransactionError::BlockhashNotFound)?; let fee = self.get_fee_for_message_with_lamports_per_signature( - tx.message(), + tx.transaction.message(), lamports_per_signature, ); @@ -4859,7 +4873,7 @@ impl Bank { // post-load, fee deducted, pre-execute account state // stored if execution_status.is_err() && !is_nonce { - self.withdraw(tx.message().fee_payer(), fee)?; + self.withdraw(tx.transaction.message().fee_payer(), fee)?; } fees += fee; @@ -4877,7 +4891,7 @@ impl Bank { /// a failure result. pub fn commit_transactions( &self, - sanitized_txs: &[SanitizedTransaction], + sanitized_txs: &[ExtendedSanitizedTransaction], loaded_txs: &mut [TransactionLoadResult], execution_results: Vec, last_blockhash: Hash, @@ -6546,7 +6560,7 @@ impl Bank { &self, tx: VersionedTransaction, verification_mode: TransactionVerificationMode, - ) -> Result { + ) -> Result { let sanitized_tx = { let size = bincode::serialized_size(&tx).map_err(|_| TransactionError::SanitizeFailure)?; @@ -6569,13 +6583,13 @@ impl Bank { sanitized_tx.verify_precompiles(&self.feature_set)?; } - Ok(sanitized_tx) + Ok(ExtendedSanitizedTransaction::from(sanitized_tx)) } pub fn fully_verify_transaction( &self, tx: VersionedTransaction, - ) -> Result { + ) -> Result { self.verify_transaction(tx, TransactionVerificationMode::FullVerification) } @@ -6917,7 +6931,7 @@ impl Bank { /// a bank-level cache of vote accounts and stake delegation info fn update_stakes_cache( &self, - txs: &[SanitizedTransaction], + txs: &[ExtendedSanitizedTransaction], execution_results: &[TransactionExecutionResult], loaded_txs: &[TransactionLoadResult], ) { @@ -6928,7 +6942,7 @@ impl Bank { .filter(|(_, execution_result, _)| execution_result.was_executed_successfully()) .flat_map(|(tx, _, (load_result, _))| { load_result.iter().flat_map(|loaded_transaction| { - let num_account_keys = tx.message().account_keys().len(); + let num_account_keys = tx.transaction.message().account_keys().len(); loaded_transaction.accounts.iter().take(num_account_keys) }) }) @@ -7450,7 +7464,7 @@ impl Bank { /// Checks a batch of sanitized transactions again bank for age and status pub fn check_transactions_with_forwarding_delay( &self, - transactions: &[SanitizedTransaction], + transactions: &[ExtendedSanitizedTransaction], filter: &[transaction::Result<()>], forward_transactions_to_leader_at_slot_offset: u64, ) -> Vec { @@ -7673,7 +7687,10 @@ impl Bank { let transaction_account_lock_limit = self.get_transaction_account_lock_limit(); let sanitized_txs = txs .into_iter() - .map(SanitizedTransaction::from_transaction_for_tests) + .map(|txn| ExtendedSanitizedTransaction { + transaction: SanitizedTransaction::from_transaction_for_tests(txn), + start_time: None, + }) .collect::>(); let lock_results = self .rc diff --git a/runtime/src/bank_utils.rs b/runtime/src/bank_utils.rs index 10835afb82dc49..645630a89a938e 100644 --- a/runtime/src/bank_utils.rs +++ b/runtime/src/bank_utils.rs @@ -7,7 +7,7 @@ use { solana_sdk::{pubkey::Pubkey, signature::Signer}, }; use { - solana_sdk::transaction::SanitizedTransaction, + solana_sdk::transaction::ExtendedSanitizedTransaction, solana_svm::transaction_results::TransactionResults, solana_vote::{vote_parser, vote_sender_types::ReplayVoteSender}, }; @@ -37,7 +37,7 @@ pub fn setup_bank_and_vote_pubkeys_for_tests( } pub fn find_and_send_votes( - sanitized_txs: &[SanitizedTransaction], + sanitized_txs: &[ExtendedSanitizedTransaction], tx_results: &TransactionResults, vote_sender: Option<&ReplayVoteSender>, ) { @@ -49,8 +49,11 @@ pub fn find_and_send_votes( .iter() .zip(execution_results.iter()) .for_each(|(tx, result)| { - if tx.is_simple_vote_transaction() && result.was_executed_successfully() { - if let Some(parsed_vote) = vote_parser::parse_sanitized_vote_transaction(tx) { + if tx.transaction.is_simple_vote_transaction() && result.was_executed_successfully() + { + if let Some(parsed_vote) = + vote_parser::parse_sanitized_vote_transaction(&tx.transaction) + { if parsed_vote.1.last_voted_slot().is_some() { let _ = vote_sender.send(parsed_vote); } diff --git a/runtime/src/installed_scheduler_pool.rs b/runtime/src/installed_scheduler_pool.rs index d39a18d567232a..ff62b69a8d87d9 100644 --- a/runtime/src/installed_scheduler_pool.rs +++ b/runtime/src/installed_scheduler_pool.rs @@ -27,7 +27,7 @@ use { solana_sdk::{ hash::Hash, slot_history::Slot, - transaction::{Result, SanitizedTransaction}, + transaction::{ExtendedSanitizedTransaction, Result}, }, std::{ fmt::Debug, @@ -104,7 +104,7 @@ pub trait InstalledScheduler: Send + Sync + Debug + 'static { // Calling this is illegal as soon as wait_for_termination is called. fn schedule_execution<'a>( &'a self, - transaction_with_index: &'a (&'a SanitizedTransaction, usize), + transaction_with_index: &'a (&'a ExtendedSanitizedTransaction, usize), ); /// Wait for a scheduler to terminate after processing. @@ -289,7 +289,9 @@ impl BankWithScheduler { // 'a is needed; anonymous_lifetime_in_impl_trait isn't stabilized yet... pub fn schedule_transaction_executions<'a>( &self, - transactions_with_indexes: impl ExactSizeIterator, + transactions_with_indexes: impl ExactSizeIterator< + Item = (&'a ExtendedSanitizedTransaction, &'a usize), + >, ) { trace!( "schedule_transaction_executions(): {} txs", diff --git a/runtime/src/prioritization_fee_cache.rs b/runtime/src/prioritization_fee_cache.rs index 0490f594451b9c..363379178174bd 100644 --- a/runtime/src/prioritization_fee_cache.rs +++ b/runtime/src/prioritization_fee_cache.rs @@ -8,7 +8,7 @@ use { solana_sdk::{ clock::{BankId, Slot}, pubkey::Pubkey, - transaction::SanitizedTransaction, + transaction::ExtendedSanitizedTransaction, }, std::{ collections::HashMap, @@ -208,20 +208,29 @@ impl PrioritizationFeeCache { /// Update with a list of non-vote transactions' compute_budget_details and account_locks; Only /// transactions have both valid compute_budget_details and account_locks will be used to update /// fee_cache asynchronously. - pub fn update<'a>(&self, bank: &Bank, txs: impl Iterator) { + pub fn update<'a>( + &self, + bank: &Bank, + txs: impl Iterator, + ) { let (_, send_updates_time) = measure!( { for sanitized_transaction in txs { // Vote transactions are not prioritized, therefore they are excluded from // updating fee_cache. - if sanitized_transaction.is_simple_vote_transaction() { + if sanitized_transaction + .transaction + .is_simple_vote_transaction() + { continue; } let round_compute_unit_price_enabled = false; // TODO: bank.feture_set.is_active(round_compute_unit_price) let compute_budget_details = sanitized_transaction + .transaction .get_compute_budget_details(round_compute_unit_price_enabled); let account_locks = sanitized_transaction + .transaction .get_account_locks(bank.get_transaction_account_lock_limit()); if compute_budget_details.is_none() || account_locks.is_err() { diff --git a/runtime/src/transaction_batch.rs b/runtime/src/transaction_batch.rs index ecec27e02e93aa..fe1c8bc3210ea9 100644 --- a/runtime/src/transaction_batch.rs +++ b/runtime/src/transaction_batch.rs @@ -1,6 +1,6 @@ use { crate::bank::Bank, - solana_sdk::transaction::{Result, SanitizedTransaction}, + solana_sdk::transaction::{ExtendedSanitizedTransaction, Result}, std::borrow::Cow, }; @@ -8,7 +8,7 @@ use { pub struct TransactionBatch<'a, 'b> { lock_results: Vec>, bank: &'a Bank, - sanitized_txs: Cow<'b, [SanitizedTransaction]>, + sanitized_txs: Cow<'b, [ExtendedSanitizedTransaction]>, needs_unlock: bool, } @@ -16,7 +16,7 @@ impl<'a, 'b> TransactionBatch<'a, 'b> { pub fn new( lock_results: Vec>, bank: &'a Bank, - sanitized_txs: Cow<'b, [SanitizedTransaction]>, + sanitized_txs: Cow<'b, [ExtendedSanitizedTransaction]>, ) -> Self { assert_eq!(lock_results.len(), sanitized_txs.len()); Self { @@ -31,7 +31,7 @@ impl<'a, 'b> TransactionBatch<'a, 'b> { &self.lock_results } - pub fn sanitized_transactions(&self) -> &[SanitizedTransaction] { + pub fn sanitized_transactions(&self) -> &[ExtendedSanitizedTransaction] { &self.sanitized_txs } diff --git a/sdk/src/transaction/sanitized.rs b/sdk/src/transaction/sanitized.rs index b7383b4a0a454c..2d4e2bd21018dc 100644 --- a/sdk/src/transaction/sanitized.rs +++ b/sdk/src/transaction/sanitized.rs @@ -19,6 +19,7 @@ use { transaction::{Result, Transaction, TransactionError, VersionedTransaction}, }, solana_program::message::SanitizedVersionedMessage, + std::time::Instant, }; /// Maximum number of accounts that a transaction may lock. @@ -35,6 +36,22 @@ pub struct SanitizedTransaction { signatures: Vec, } +/// Sanitized transaction with option start_time +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct ExtendedSanitizedTransaction { + pub transaction: SanitizedTransaction, + pub start_time: Option, +} + +impl From for ExtendedSanitizedTransaction { + fn from(value: SanitizedTransaction) -> Self { + Self { + transaction: value, + start_time: None, + } + } +} + /// Set of accounts that must be locked for safe transaction processing #[derive(Debug, Clone, Default, Eq, PartialEq)] pub struct TransactionAccountLocks<'a> { diff --git a/svm/src/account_loader.rs b/svm/src/account_loader.rs index bf9b5b9c40bfee..5b9b2a1f8f0835 100644 --- a/svm/src/account_loader.rs +++ b/svm/src/account_loader.rs @@ -30,7 +30,7 @@ use { rent_debits::RentDebits, saturating_add_assign, sysvar::{self, instructions::construct_instructions_data}, - transaction::{self, Result, SanitizedTransaction, TransactionError}, + transaction::{self, ExtendedSanitizedTransaction, Result, TransactionError}, transaction_context::{IndexOfAccount, TransactionAccount}, }, solana_system_program::{get_system_account_kind, SystemAccountKind}, @@ -112,7 +112,7 @@ pub fn validate_fee_payer( /// second element. pub(crate) fn load_accounts( callbacks: &CB, - txs: &[SanitizedTransaction], + txs: &[ExtendedSanitizedTransaction], lock_results: &[TransactionCheckResult], error_counters: &mut TransactionErrorMetrics, fee_structure: &FeeStructure, @@ -125,7 +125,7 @@ pub(crate) fn load_accounts( .zip(lock_results) .map(|etx| match etx { (tx, (Ok(()), nonce, lamports_per_signature)) => { - let message = tx.message(); + let message = tx.transaction.message(); let fee = if let Some(lamports_per_signature) = lamports_per_signature { fee_structure.calculate_fee( message, diff --git a/svm/src/transaction_processor.rs b/svm/src/transaction_processor.rs index fec908619f14f8..de99526587ef57 100644 --- a/svm/src/transaction_processor.rs +++ b/svm/src/transaction_processor.rs @@ -43,7 +43,7 @@ use { pubkey::Pubkey, rent_collector::RentCollector, saturating_add_assign, - transaction::{self, SanitizedTransaction, TransactionError}, + transaction::{self, ExtendedSanitizedTransaction, SanitizedTransaction, TransactionError}, transaction_context::{ExecutionRecord, TransactionContext}, }, std::{ @@ -200,7 +200,7 @@ impl TransactionBatchProcessor { pub fn load_and_execute_sanitized_transactions<'a, CB: TransactionProcessingCallback>( &self, callbacks: &CB, - sanitized_txs: &[SanitizedTransaction], + sanitized_txs: &[ExtendedSanitizedTransaction], check_results: &mut [TransactionCheckResult], error_counters: &mut TransactionErrorMetrics, recording_config: ExecutionRecordingConfig, @@ -262,7 +262,7 @@ impl TransactionBatchProcessor { let mut compute_budget_process_transaction_time = Measure::start("compute_budget_process_transaction_time"); let maybe_compute_budget = ComputeBudget::try_from_instructions( - tx.message().program_instructions_iter(), + tx.transaction.message().program_instructions_iter(), ); compute_budget_process_transaction_time.stop(); saturating_add_assign!( @@ -279,7 +279,7 @@ impl TransactionBatchProcessor { let result = self.execute_loaded_transaction( callbacks, - tx, + &tx.transaction, loaded_transaction, compute_budget, nonce.as_ref().map(DurableNonceFee::from), @@ -341,7 +341,7 @@ impl TransactionBatchProcessor { /// blockhash or nonce. pub fn filter_executable_program_accounts<'a, CB: TransactionProcessingCallback>( callbacks: &CB, - txs: &[SanitizedTransaction], + txs: &[ExtendedSanitizedTransaction], lock_results: &mut [TransactionCheckResult], program_owners: &'a [Pubkey], ) -> HashMap { @@ -349,7 +349,8 @@ impl TransactionBatchProcessor { lock_results.iter_mut().zip(txs).for_each(|etx| { if let ((Ok(()), _nonce, lamports_per_signature), tx) = etx { if lamports_per_signature.is_some() { - tx.message() + tx.transaction + .message() .account_keys() .iter() .for_each(|key| match result.entry(*key) { diff --git a/unified-scheduler-logic/src/lib.rs b/unified-scheduler-logic/src/lib.rs index 997c6c1745a7c9..e587c3ba06f38d 100644 --- a/unified-scheduler-logic/src/lib.rs +++ b/unified-scheduler-logic/src/lib.rs @@ -1,12 +1,12 @@ -use solana_sdk::transaction::SanitizedTransaction; +use solana_sdk::transaction::ExtendedSanitizedTransaction; pub struct Task { - transaction: SanitizedTransaction, + transaction: ExtendedSanitizedTransaction, index: usize, } impl Task { - pub fn create_task(transaction: SanitizedTransaction, index: usize) -> Self { + pub fn create_task(transaction: ExtendedSanitizedTransaction, index: usize) -> Self { Task { transaction, index } } @@ -14,7 +14,7 @@ impl Task { self.index } - pub fn transaction(&self) -> &SanitizedTransaction { + pub fn transaction(&self) -> &ExtendedSanitizedTransaction { &self.transaction } } diff --git a/unified-scheduler-pool/src/lib.rs b/unified-scheduler-pool/src/lib.rs index 09ded82ee88e7d..4d8780c278a3c9 100644 --- a/unified-scheduler-pool/src/lib.rs +++ b/unified-scheduler-pool/src/lib.rs @@ -26,7 +26,7 @@ use { }, prioritization_fee_cache::PrioritizationFeeCache, }, - solana_sdk::transaction::{Result, SanitizedTransaction}, + solana_sdk::transaction::{ExtendedSanitizedTransaction, Result}, solana_unified_scheduler_logic::Task, solana_vote::vote_sender_types::ReplayVoteSender, std::{ @@ -203,7 +203,7 @@ pub trait TaskHandler: Send + Sync + Debug + Sized + 'static { result: &mut Result<()>, timings: &mut ExecuteTimings, bank: &Arc, - transaction: &SanitizedTransaction, + transaction: &ExtendedSanitizedTransaction, index: usize, handler_context: &HandlerContext, ); @@ -217,7 +217,7 @@ impl TaskHandler for DefaultTaskHandler { result: &mut Result<()>, timings: &mut ExecuteTimings, bank: &Arc, - transaction: &SanitizedTransaction, + transaction: &ExtendedSanitizedTransaction, index: usize, handler_context: &HandlerContext, ) { @@ -740,7 +740,7 @@ impl InstalledScheduler for PooledScheduler { &self.context } - fn schedule_execution(&self, &(transaction, index): &(&SanitizedTransaction, usize)) { + fn schedule_execution(&self, &(transaction, index): &(&ExtendedSanitizedTransaction, usize)) { let task = Task::create_task(transaction.clone(), index); self.inner.thread_manager.send_task(task); } From 27e7d7ef54344a970b6da7b5a490c83898cf05a9 Mon Sep 17 00:00:00 2001 From: Lijun Wang <83639177+lijunwangs@users.noreply.github.com> Date: Sat, 2 Mar 2024 03:22:27 -0800 Subject: [PATCH 66/71] Fixed a clippy issue --- runtime/src/bank.rs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index b568fc809420a7..52a3950cc9fe7b 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -4204,14 +4204,12 @@ impl Bank { let sanitized_txs = txs .into_iter() .map(|tx| { - SanitizedTransaction::try_create(tx, MessageHash::Compute, None, self).and_then( - |txn| { - Ok(ExtendedSanitizedTransaction { - transaction: txn, - start_time: None, - }) - }, - ) + SanitizedTransaction::try_create(tx, MessageHash::Compute, None, self).map(|txn| { + ExtendedSanitizedTransaction { + transaction: txn, + start_time: None, + } + }) }) .collect::>>()?; let tx_account_lock_limit = self.get_transaction_account_lock_limit(); From 798a3676b0273c40e31aed64a53b4f305f449ea2 Mon Sep 17 00:00:00 2001 From: Lijun Wang <83639177+lijunwangs@users.noreply.github.com> Date: Sun, 3 Mar 2024 02:53:26 -0800 Subject: [PATCH 67/71] instrument to pass the perf_track_metrics down the transaction processing stack --- Cargo.lock | 2 + core/src/banking_stage/consume_worker.rs | 17 ++++-- core/src/banking_stage/consumer.rs | 55 ++++++++----------- core/src/banking_stage/leader_slot_metrics.rs | 20 ++++--- ledger/src/blockstore_processor.rs | 1 + runtime/Cargo.toml | 1 + runtime/src/bank.rs | 8 +++ svm/Cargo.toml | 1 + svm/src/transaction_processor.rs | 12 ++++ 9 files changed, 72 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2a3e591580da49..16aa434486236a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6909,6 +6909,7 @@ dependencies = [ "ed25519-dalek", "flate2", "fnv", + "histogram", "im", "index_list", "itertools", @@ -7207,6 +7208,7 @@ name = "solana-svm" version = "1.19.0" dependencies = [ "bincode", + "histogram", "itertools", "log", "percentage", diff --git a/core/src/banking_stage/consume_worker.rs b/core/src/banking_stage/consume_worker.rs index 92fb07ddfab18c..779e9db4e4ea02 100644 --- a/core/src/banking_stage/consume_worker.rs +++ b/core/src/banking_stage/consume_worker.rs @@ -34,6 +34,7 @@ pub(crate) struct ConsumeWorker { leader_bank_notifier: Arc, metrics: Arc, + perf_track_metrics: histogram::Histogram, } #[allow(dead_code)] @@ -51,6 +52,7 @@ impl ConsumeWorker { consumed_sender, leader_bank_notifier, metrics: Arc::new(ConsumeWorkerMetrics::new(id)), + perf_track_metrics: histogram::Histogram::default(), } } @@ -58,19 +60,23 @@ impl ConsumeWorker { self.metrics.clone() } - pub fn run(self) -> Result<(), ConsumeWorkerError> { + pub fn run(mut self) -> Result<(), ConsumeWorkerError> { loop { let work = self.consume_receiver.recv()?; self.consume_loop(work)?; } } - fn consume_loop(&self, work: ConsumeWork) -> Result<(), ConsumeWorkerError> { + fn consume_loop(&mut self, work: ConsumeWork) -> Result<(), ConsumeWorkerError> { let Some(mut bank) = self.get_consume_bank() else { return self.retry_drain(work); }; - for work in try_drain_iter(work, &self.consume_receiver) { + // We need to have a mutable borrow in consume, so move the receiver out + let (_, dummy_receiver) = crossbeam_channel::unbounded(); + let consume_receiver = std::mem::replace(&mut self.consume_receiver, dummy_receiver); + + for work in try_drain_iter(work, &consume_receiver) { if bank.is_complete() { if let Some(new_bank) = self.get_consume_bank() { bank = new_bank; @@ -81,15 +87,18 @@ impl ConsumeWorker { self.consume(&bank, work)?; } + // restore it + self.consume_receiver = consume_receiver; Ok(()) } /// Consume a single batch. - fn consume(&self, bank: &Arc, work: ConsumeWork) -> Result<(), ConsumeWorkerError> { + fn consume(&mut self, bank: &Arc, work: ConsumeWork) -> Result<(), ConsumeWorkerError> { let output = self.consumer.process_and_record_aged_transactions( bank, &work.transactions, &work.max_age_slots, + Some(&mut self.perf_track_metrics), ); self.metrics.update_for_consume(&output); diff --git a/core/src/banking_stage/consumer.rs b/core/src/banking_stage/consumer.rs index 0c5ae73c5ba651..b9b80f01bb0dc3 100644 --- a/core/src/banking_stage/consumer.rs +++ b/core/src/banking_stage/consumer.rs @@ -208,33 +208,6 @@ impl Consumer { payload .slot_metrics_tracker .increment_retryable_packets_count(retryable_transaction_indexes.len() as u64); - - // Now we track the performance for the interested transactions which is not in the retryable_transaction_indexes - // We assume the retryable_transaction_indexes is already sorted. - let mut retryable_idx = 0; - for (index, packet) in packets_to_process.iter().enumerate() { - if packet.original_packet().meta().is_perf_track_packet() { - if let Some(start_time) = packet.start_time() { - if retryable_idx >= retryable_transaction_indexes.len() - || retryable_transaction_indexes[retryable_idx] != index - { - let duration = Instant::now().duration_since(*start_time); - - debug!( - "Banking stage processing took {duration:?} for transaction {:?}", - packet.transaction().get_signatures().first() - ); - payload - .slot_metrics_tracker - .increment_process_sampled_packets_us(duration.as_micros() as u64); - } else { - // This packet is retried, advance the retry index to the next, as the next packet's index will - // certainly be > than this. - retryable_idx += 1; - } - } - } - } Some(retryable_transaction_indexes) } @@ -246,9 +219,13 @@ impl Consumer { banking_stage_stats: &BankingStageStats, slot_metrics_tracker: &mut LeaderSlotMetricsTracker, ) -> ProcessTransactionsSummary { - let (mut process_transactions_summary, process_transactions_us) = measure_us!( - self.process_transactions(bank, bank_creation_time, sanitized_transactions) - ); + let (mut process_transactions_summary, process_transactions_us) = measure_us!(self + .process_transactions( + bank, + bank_creation_time, + sanitized_transactions, + &mut slot_metrics_tracker.get_transaction_perf_track_metrics_ref() + )); slot_metrics_tracker.increment_process_transactions_us(process_transactions_us); banking_stage_stats .transaction_processing_elapsed @@ -299,6 +276,7 @@ impl Consumer { bank: &Arc, bank_creation_time: &Instant, transactions: &[ExtendedSanitizedTransaction], + perf_track_metrics: &mut Option<&mut histogram::Histogram>, ) -> ProcessTransactionsSummary { let mut chunk_start = 0; let mut all_retryable_tx_indexes = vec![]; @@ -328,6 +306,7 @@ impl Consumer { bank, &transactions[chunk_start..chunk_end], chunk_start, + perf_track_metrics, ); let ProcessTransactionBatchOutput { @@ -434,6 +413,7 @@ impl Consumer { bank: &Arc, txs: &[ExtendedSanitizedTransaction], chunk_offset: usize, + perf_track_metrics: &mut Option<&mut histogram::Histogram>, ) -> ProcessTransactionBatchOutput { let mut error_counters = TransactionErrorMetrics::default(); let pre_results = vec![Ok(()); txs.len()]; @@ -447,6 +427,7 @@ impl Consumer { txs, chunk_offset, check_results, + perf_track_metrics, ); // Accumulate error counters from the initial checks into final results @@ -462,6 +443,7 @@ impl Consumer { bank: &Arc, txs: &[ExtendedSanitizedTransaction], max_slot_ages: &[Slot], + mut perf_track_metrics: Option<&mut histogram::Histogram>, ) -> ProcessTransactionBatchOutput { // Need to filter out transactions since they were sanitized earlier. // This means that the transaction may cross and epoch boundary (not allowed), @@ -487,7 +469,13 @@ impl Consumer { } Ok(()) }); - self.process_and_record_transactions_with_pre_results(bank, txs, 0, pre_results) + self.process_and_record_transactions_with_pre_results( + bank, + txs, + 0, + pre_results, + &mut perf_track_metrics, + ) } fn process_and_record_transactions_with_pre_results( @@ -496,6 +484,7 @@ impl Consumer { txs: &[ExtendedSanitizedTransaction], chunk_offset: usize, pre_results: impl Iterator>, + perf_track_metrics: &mut Option<&mut histogram::Histogram>, ) -> ProcessTransactionBatchOutput { let ( (transaction_qos_cost_results, cost_model_throttled_transactions_count), @@ -521,7 +510,7 @@ impl Consumer { // WouldExceedMaxAccountCostLimit, WouldExceedMaxVoteCostLimit // and WouldExceedMaxAccountDataCostLimit let mut execute_and_commit_transactions_output = - self.execute_and_commit_transactions_locked(bank, &batch); + self.execute_and_commit_transactions_locked(bank, &batch, perf_track_metrics); // Once the accounts are new transactions can enter the pipeline to process them let (_, unlock_us) = measure_us!(drop(batch)); @@ -587,6 +576,7 @@ impl Consumer { &self, bank: &Arc, batch: &TransactionBatch, + perf_track_metrics: &mut Option<&mut histogram::Histogram>, ) -> ExecuteAndCommitTransactionsOutput { let transaction_status_sender_enabled = self.committer.transaction_status_sender_enabled(); let mut execute_and_commit_timings = LeaderExecuteAndCommitTimings::default(); @@ -626,6 +616,7 @@ impl Consumer { None, // account_overrides self.log_messages_bytes_limit, true, + perf_track_metrics, )); execute_and_commit_timings.load_execute_us = load_execute_us; diff --git a/core/src/banking_stage/leader_slot_metrics.rs b/core/src/banking_stage/leader_slot_metrics.rs index 1c255ca019bfe7..301df48a4f2582 100644 --- a/core/src/banking_stage/leader_slot_metrics.rs +++ b/core/src/banking_stage/leader_slot_metrics.rs @@ -937,15 +937,17 @@ impl LeaderSlotMetricsTracker { } } - pub(crate) fn increment_process_sampled_packets_us(&mut self, us: u64) { - if let Some(leader_slot_metrics) = &mut self.leader_slot_metrics { - leader_slot_metrics - .timing_metrics - .process_packets_timings - .process_sampled_packets_us_hist - .increment(us) - .unwrap(); - } + pub(crate) fn get_transaction_perf_track_metrics_ref( + &mut self, + ) -> Option<&mut histogram::Histogram> { + self.leader_slot_metrics + .as_mut() + .map(|leader_slot_metrics| { + &mut leader_slot_metrics + .timing_metrics + .process_packets_timings + .process_sampled_packets_us_hist + }) } } diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs index d478c8fa6dfa1b..64f37f9ad80fbf 100644 --- a/ledger/src/blockstore_processor.rs +++ b/ledger/src/blockstore_processor.rs @@ -169,6 +169,7 @@ pub fn execute_batch( ExecutionRecordingConfig::new_single_setting(transaction_status_sender.is_some()), timings, log_messages_bytes_limit, + None, ); bank_utils::find_and_send_votes( diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index 02553d4215909d..85600e9de319da 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -24,6 +24,7 @@ dashmap = { workspace = true, features = ["rayon", "raw-api"] } dir-diff = { workspace = true } flate2 = { workspace = true } fnv = { workspace = true } +histogram = { workspace = true } im = { workspace = true, features = ["rayon", "serde"] } index_list = { workspace = true } itertools = { workspace = true } diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index 52a3950cc9fe7b..100c28fe71285f 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -4316,6 +4316,7 @@ impl Bank { Some(&account_overrides), None, true, + &mut None, ); let post_simulation_accounts = loaded_transactions @@ -4565,6 +4566,7 @@ impl Bank { account_overrides: Option<&AccountOverrides>, log_messages_bytes_limit: Option, limit_to_load_programs: bool, + perf_track_metrics: &mut Option<&mut histogram::Histogram>, ) -> LoadAndExecuteTransactionsOutput { let sanitized_txs = batch.sanitized_transactions(); debug!("processing transactions: {}", sanitized_txs.len()); @@ -4625,6 +4627,7 @@ impl Bank { &mut check_results, &mut error_counters, recording_config, + perf_track_metrics, timings, account_overrides, self.builtin_programs.iter(), @@ -5648,6 +5651,7 @@ impl Bank { /// Process a batch of transactions. #[must_use] + #[allow(clippy::too_many_arguments)] pub fn load_execute_and_commit_transactions( &self, batch: &TransactionBatch, @@ -5656,6 +5660,7 @@ impl Bank { recording_config: ExecutionRecordingConfig, timings: &mut ExecuteTimings, log_messages_bytes_limit: Option, + mut perf_track_metrics: Option<&mut histogram::Histogram>, ) -> (TransactionResults, TransactionBalancesSet) { let pre_balances = if collect_balances { self.collect_balances(batch) @@ -5679,6 +5684,7 @@ impl Bank { None, log_messages_bytes_limit, false, + &mut perf_track_metrics, ); let (last_blockhash, lamports_per_signature) = @@ -5749,6 +5755,7 @@ impl Bank { }, &mut ExecuteTimings::default(), Some(1000 * 1000), + None, ); execution_results.remove(0) @@ -5785,6 +5792,7 @@ impl Bank { ExecutionRecordingConfig::new_single_setting(false), &mut ExecuteTimings::default(), None, + None, ) .0 .fee_collection_results diff --git a/svm/Cargo.toml b/svm/Cargo.toml index 21da2f7105bd73..ca4fba3a6e2b06 100644 --- a/svm/Cargo.toml +++ b/svm/Cargo.toml @@ -11,6 +11,7 @@ edition = { workspace = true } [dependencies] itertools = { workspace = true } +histogram = { workspace = true } log = { workspace = true } percentage = { workspace = true } solana-bpf-loader-program = { workspace = true } diff --git a/svm/src/transaction_processor.rs b/svm/src/transaction_processor.rs index de99526587ef57..7e40b27ecb5908 100644 --- a/svm/src/transaction_processor.rs +++ b/svm/src/transaction_processor.rs @@ -43,6 +43,7 @@ use { pubkey::Pubkey, rent_collector::RentCollector, saturating_add_assign, + timing::duration_as_us, transaction::{self, ExtendedSanitizedTransaction, SanitizedTransaction, TransactionError}, transaction_context::{ExecutionRecord, TransactionContext}, }, @@ -52,6 +53,7 @@ use { fmt::{Debug, Formatter}, rc::Rc, sync::{atomic::Ordering, Arc, RwLock}, + time::Instant, }, }; @@ -204,6 +206,7 @@ impl TransactionBatchProcessor { check_results: &mut [TransactionCheckResult], error_counters: &mut TransactionErrorMetrics, recording_config: ExecutionRecordingConfig, + perf_track_metrics: &mut Option<&mut histogram::Histogram>, timings: &mut ExecuteTimings, account_overrides: Option<&AccountOverrides>, builtin_programs: impl Iterator, @@ -295,6 +298,15 @@ impl TransactionBatchProcessor { programs_modified_by_tx, } = &result { + if let Some(perf_track_metrics) = perf_track_metrics.as_mut() { + if let Some(start_time) = &tx.start_time { + // measure the time from start of banking stage to the execution of the transaction + let duration = Instant::now().duration_since(*start_time); + perf_track_metrics + .increment(duration_as_us(&duration)) + .unwrap(); + } + } // Update batch specific cache of the loaded programs with the modifications // made by the transaction, if it executed successfully. if details.status.is_ok() { From 1236c49d95593c0b5c246073c6b82e6fe5a1b34c Mon Sep 17 00:00:00 2001 From: Lijun Wang <83639177+lijunwangs@users.noreply.github.com> Date: Sun, 10 Mar 2024 22:08:28 -0700 Subject: [PATCH 68/71] move ExtendedSanitizedTransaction to solana-runtime-transaction crate --- Cargo.lock | 10 ++++++++++ accounts-db/Cargo.toml | 1 + accounts-db/src/accounts.rs | 5 ++--- accounts-db/src/accounts_db.rs | 2 +- banks-server/Cargo.toml | 1 + banks-server/src/banks_server.rs | 6 ++---- core/Cargo.toml | 1 + core/src/banking_stage/consumer.rs | 3 ++- .../forward_packet_batches_by_accounts.rs | 3 ++- .../banking_stage/latest_unprocessed_votes.rs | 2 +- core/src/banking_stage/qos_service.rs | 3 ++- core/src/banking_stage/scheduler_messages.rs | 3 ++- .../prio_graph_scheduler.rs | 6 ++---- .../scheduler_controller.rs | 3 ++- .../transaction_scheduler/transaction_state.rs | 5 ++++- .../unprocessed_transaction_storage.rs | 8 +++----- entry/Cargo.toml | 1 + entry/src/entry.rs | 5 +++-- ledger/Cargo.toml | 1 + ledger/src/blockstore_processor.rs | 4 ++-- rpc/Cargo.toml | 1 + rpc/src/rpc.rs | 6 ++---- runtime-transaction/src/lib.rs | 1 + runtime/Cargo.toml | 1 + runtime/src/bank.rs | 6 +++--- runtime/src/bank_utils.rs | 2 +- runtime/src/installed_scheduler_pool.rs | 7 ++----- runtime/src/prioritization_fee_cache.rs | 2 +- runtime/src/transaction_batch.rs | 4 ++-- sdk/src/transaction/sanitized.rs | 17 ----------------- svm/Cargo.toml | 1 + svm/src/account_loader.rs | 3 ++- svm/src/transaction_processor.rs | 3 ++- unified-scheduler-logic/Cargo.toml | 1 + unified-scheduler-logic/src/lib.rs | 2 +- unified-scheduler-pool/Cargo.toml | 1 + unified-scheduler-pool/src/lib.rs | 3 ++- 37 files changed, 69 insertions(+), 65 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 16aa434486236a..f57a028d98a8fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5222,6 +5222,7 @@ dependencies = [ "solana-nohash-hasher", "solana-program-runtime", "solana-rayon-threadlimit", + "solana-runtime-transaction", "solana-sdk", "solana-stake-program", "solana-svm", @@ -5326,6 +5327,7 @@ dependencies = [ "solana-banks-interface", "solana-client", "solana-runtime", + "solana-runtime-transaction", "solana-sdk", "solana-send-transaction-service", "solana-svm", @@ -5824,6 +5826,7 @@ dependencies = [ "solana-rpc", "solana-rpc-client-api", "solana-runtime", + "solana-runtime-transaction", "solana-sdk", "solana-send-transaction-service", "solana-stake-program", @@ -5948,6 +5951,7 @@ dependencies = [ "solana-metrics", "solana-perf", "solana-rayon-threadlimit", + "solana-runtime-transaction", "solana-sdk", ] @@ -6229,6 +6233,7 @@ dependencies = [ "solana-program-runtime", "solana-rayon-threadlimit", "solana-runtime", + "solana-runtime-transaction", "solana-sdk", "solana-stake-program", "solana-storage-bigtable", @@ -6775,6 +6780,7 @@ dependencies = [ "solana-rayon-threadlimit", "solana-rpc-client-api", "solana-runtime", + "solana-runtime-transaction", "solana-sdk", "solana-send-transaction-service", "solana-stake-program", @@ -6953,6 +6959,7 @@ dependencies = [ "solana-program-runtime", "solana-rayon-threadlimit", "solana-runtime", + "solana-runtime-transaction", "solana-sdk", "solana-stake-program", "solana-svm", @@ -7221,6 +7228,7 @@ dependencies = [ "solana-measure", "solana-metrics", "solana-program-runtime", + "solana-runtime-transaction", "solana-sdk", "solana-system-program", ] @@ -7456,6 +7464,7 @@ dependencies = [ name = "solana-unified-scheduler-logic" version = "1.19.0" dependencies = [ + "solana-runtime-transaction", "solana-sdk", ] @@ -7471,6 +7480,7 @@ dependencies = [ "solana-logger", "solana-program-runtime", "solana-runtime", + "solana-runtime-transaction", "solana-sdk", "solana-unified-scheduler-logic", "solana-vote", diff --git a/accounts-db/Cargo.toml b/accounts-db/Cargo.toml index 0fc5a381fbda5e..b575e47db07299 100644 --- a/accounts-db/Cargo.toml +++ b/accounts-db/Cargo.toml @@ -52,6 +52,7 @@ solana-metrics = { workspace = true } solana-nohash-hasher = { workspace = true } solana-program-runtime = { workspace = true } solana-rayon-threadlimit = { workspace = true } +solana-runtime-transaction = { workspace = true } solana-sdk = { workspace = true } solana-stake-program = { workspace = true } solana-svm = { workspace = true } diff --git a/accounts-db/src/accounts.rs b/accounts-db/src/accounts.rs index 5702353f9fc142..b72b47f715394c 100644 --- a/accounts-db/src/accounts.rs +++ b/accounts-db/src/accounts.rs @@ -10,6 +10,7 @@ use { }, dashmap::DashMap, log::*, + solana_runtime_transaction::extended_transaction::ExtendedSanitizedTransaction, solana_sdk::{ account::{AccountSharedData, ReadableAccount}, account_utils::StateMut, @@ -23,9 +24,7 @@ use { nonce_info::{NonceFull, NonceInfo}, pubkey::Pubkey, slot_hashes::SlotHashes, - transaction::{ - ExtendedSanitizedTransaction, Result, TransactionAccountLocks, TransactionError, - }, + transaction::{Result, TransactionAccountLocks, TransactionError}, transaction_context::TransactionAccount, }, solana_svm::{ diff --git a/accounts-db/src/accounts_db.rs b/accounts-db/src/accounts_db.rs index 44ee7a1fdf4cb0..22add846bb863d 100644 --- a/accounts-db/src/accounts_db.rs +++ b/accounts-db/src/accounts_db.rs @@ -83,6 +83,7 @@ use { solana_measure::{measure::Measure, measure_us}, solana_nohash_hasher::{IntMap, IntSet}, solana_rayon_threadlimit::get_thread_count, + solana_runtime_transaction::extended_transaction::ExtendedSanitizedTransaction, solana_sdk::{ account::{Account, AccountSharedData, ReadableAccount, WritableAccount}, clock::{BankId, Epoch, Slot}, @@ -93,7 +94,6 @@ use { rent_collector::RentCollector, saturating_add_assign, timing::AtomicInterval, - transaction::ExtendedSanitizedTransaction, }, std::{ borrow::{Borrow, Cow}, diff --git a/banks-server/Cargo.toml b/banks-server/Cargo.toml index 6cf5f77f92548b..357a07bddd44b3 100644 --- a/banks-server/Cargo.toml +++ b/banks-server/Cargo.toml @@ -16,6 +16,7 @@ futures = { workspace = true } solana-banks-interface = { workspace = true } solana-client = { workspace = true } solana-runtime = { workspace = true } +solana-runtime-transaction = { workspace = true } solana-sdk = { workspace = true } solana-send-transaction-service = { workspace = true } solana-svm = { workspace = true } diff --git a/banks-server/src/banks_server.rs b/banks-server/src/banks_server.rs index 973c591e622edb..8cca21009edc79 100644 --- a/banks-server/src/banks_server.rs +++ b/banks-server/src/banks_server.rs @@ -13,6 +13,7 @@ use { bank_forks::BankForks, commitment::BlockCommitmentCache, }, + solana_runtime_transaction::extended_transaction::ExtendedSanitizedTransaction, solana_sdk::{ account::Account, clock::Slot, @@ -23,10 +24,7 @@ use { message::{Message, SanitizedMessage}, pubkey::Pubkey, signature::Signature, - transaction::{ - self, ExtendedSanitizedTransaction, MessageHash, SanitizedTransaction, - VersionedTransaction, - }, + transaction::{self, MessageHash, SanitizedTransaction, VersionedTransaction}, }, solana_send_transaction_service::{ send_transaction_service::{SendTransactionService, TransactionInfo}, diff --git a/core/Cargo.toml b/core/Cargo.toml index 1fd25ec38a8d3b..4308dc3fba3a43 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -62,6 +62,7 @@ solana-rayon-threadlimit = { workspace = true } solana-rpc = { workspace = true } solana-rpc-client-api = { workspace = true } solana-runtime = { workspace = true } +solana-runtime-transaction = { workspace = true } solana-sdk = { workspace = true } solana-send-transaction-service = { workspace = true } solana-streamer = { workspace = true } diff --git a/core/src/banking_stage/consumer.rs b/core/src/banking_stage/consumer.rs index b9b80f01bb0dc3..b55019130c3ca1 100644 --- a/core/src/banking_stage/consumer.rs +++ b/core/src/banking_stage/consumer.rs @@ -23,13 +23,14 @@ use { compute_budget_details::GetComputeBudgetDetails, transaction_batch::TransactionBatch, }, + solana_runtime_transaction::extended_transaction::ExtendedSanitizedTransaction, solana_sdk::{ clock::{Slot, FORWARD_TRANSACTIONS_TO_LEADER_AT_SLOT_OFFSET, MAX_PROCESSING_AGE}, feature_set, message::SanitizedMessage, saturating_add_assign, timing::timestamp, - transaction::{self, AddressLoader, ExtendedSanitizedTransaction, TransactionError}, + transaction::{self, AddressLoader, TransactionError}, }, solana_svm::{ account_loader::{validate_fee_payer, TransactionCheckResult}, diff --git a/core/src/banking_stage/forward_packet_batches_by_accounts.rs b/core/src/banking_stage/forward_packet_batches_by_accounts.rs index ab8f52820e8eac..6b1685c8a46483 100644 --- a/core/src/banking_stage/forward_packet_batches_by_accounts.rs +++ b/core/src/banking_stage/forward_packet_batches_by_accounts.rs @@ -6,7 +6,8 @@ use { cost_tracker::{CostTracker, CostTrackerError}, }, solana_perf::packet::Packet, - solana_sdk::{feature_set::FeatureSet, transaction::ExtendedSanitizedTransaction}, + solana_runtime_transaction::extended_transaction::ExtendedSanitizedTransaction, + solana_sdk::feature_set::FeatureSet, std::sync::Arc, }; diff --git a/core/src/banking_stage/latest_unprocessed_votes.rs b/core/src/banking_stage/latest_unprocessed_votes.rs index fecec111bd660c..f550c680112c98 100644 --- a/core/src/banking_stage/latest_unprocessed_votes.rs +++ b/core/src/banking_stage/latest_unprocessed_votes.rs @@ -7,11 +7,11 @@ use { rand::{thread_rng, Rng}, solana_perf::packet::Packet, solana_runtime::bank::Bank, + solana_runtime_transaction::extended_transaction::ExtendedSanitizedTransaction, solana_sdk::{ clock::{Slot, UnixTimestamp}, program_utils::limited_deserialize, pubkey::Pubkey, - transaction::ExtendedSanitizedTransaction, }, solana_vote_program::vote_instruction::VoteInstruction, std::{ diff --git a/core/src/banking_stage/qos_service.rs b/core/src/banking_stage/qos_service.rs index 5532d36eb39ecd..91b89a9e619cc1 100644 --- a/core/src/banking_stage/qos_service.rs +++ b/core/src/banking_stage/qos_service.rs @@ -8,11 +8,12 @@ use { solana_cost_model::{cost_model::CostModel, transaction_cost::TransactionCost}, solana_measure::measure::Measure, solana_runtime::bank::Bank, + solana_runtime_transaction::extended_transaction::ExtendedSanitizedTransaction, solana_sdk::{ clock::Slot, feature_set::FeatureSet, saturating_add_assign, - transaction::{self, ExtendedSanitizedTransaction, TransactionError}, + transaction::{self, TransactionError}, }, std::sync::atomic::{AtomicU64, Ordering}, }; diff --git a/core/src/banking_stage/scheduler_messages.rs b/core/src/banking_stage/scheduler_messages.rs index 359f4cac710b71..b1836b4db54f22 100644 --- a/core/src/banking_stage/scheduler_messages.rs +++ b/core/src/banking_stage/scheduler_messages.rs @@ -1,6 +1,7 @@ use { super::immutable_deserialized_packet::ImmutableDeserializedPacket, - solana_sdk::{clock::Slot, transaction::ExtendedSanitizedTransaction}, + solana_runtime_transaction::extended_transaction::ExtendedSanitizedTransaction, + solana_sdk::clock::Slot, std::{fmt::Display, sync::Arc}, }; diff --git a/core/src/banking_stage/transaction_scheduler/prio_graph_scheduler.rs b/core/src/banking_stage/transaction_scheduler/prio_graph_scheduler.rs index 95408800852b84..a80cdabb668ea6 100644 --- a/core/src/banking_stage/transaction_scheduler/prio_graph_scheduler.rs +++ b/core/src/banking_stage/transaction_scheduler/prio_graph_scheduler.rs @@ -16,10 +16,8 @@ use { itertools::izip, prio_graph::{AccessKind, PrioGraph}, solana_measure::measure_us, - solana_sdk::{ - pubkey::Pubkey, saturating_add_assign, slot_history::Slot, - transaction::ExtendedSanitizedTransaction, - }, + solana_runtime_transaction::extended_transaction::ExtendedSanitizedTransaction, + solana_sdk::{pubkey::Pubkey, saturating_add_assign, slot_history::Slot}, }; pub(crate) struct PrioGraphScheduler { diff --git a/core/src/banking_stage/transaction_scheduler/scheduler_controller.rs b/core/src/banking_stage/transaction_scheduler/scheduler_controller.rs index 600c7ef6c3f3a3..e3aa895fddc54c 100644 --- a/core/src/banking_stage/transaction_scheduler/scheduler_controller.rs +++ b/core/src/banking_stage/transaction_scheduler/scheduler_controller.rs @@ -23,6 +23,7 @@ use { solana_measure::measure_us, solana_program_runtime::compute_budget_processor::process_compute_budget_instructions, solana_runtime::{bank::Bank, bank_forks::BankForks}, + solana_runtime_transaction::extended_transaction::ExtendedSanitizedTransaction, solana_sdk::{ clock::MAX_PROCESSING_AGE, feature_set::{ @@ -31,7 +32,7 @@ use { }, fee::FeeBudgetLimits, saturating_add_assign, - transaction::{ExtendedSanitizedTransaction, SanitizedTransaction}, + transaction::SanitizedTransaction, }, solana_svm::transaction_error_metrics::TransactionErrorMetrics, std::{ diff --git a/core/src/banking_stage/transaction_scheduler/transaction_state.rs b/core/src/banking_stage/transaction_scheduler/transaction_state.rs index 7ae094258a9bad..22bf8a7b87344a 100644 --- a/core/src/banking_stage/transaction_scheduler/transaction_state.rs +++ b/core/src/banking_stage/transaction_scheduler/transaction_state.rs @@ -1,4 +1,7 @@ -use solana_sdk::{clock::Slot, transaction::ExtendedSanitizedTransaction}; +use { + solana_runtime_transaction::extended_transaction::ExtendedSanitizedTransaction, + solana_sdk::clock::Slot, +}; /// Simple wrapper type to tie a sanitized transaction to max age slot. pub(crate) struct SanitizedTransactionTTL { diff --git a/core/src/banking_stage/unprocessed_transaction_storage.rs b/core/src/banking_stage/unprocessed_transaction_storage.rs index 7c50e045b89f62..02d61c42b894b7 100644 --- a/core/src/banking_stage/unprocessed_transaction_storage.rs +++ b/core/src/banking_stage/unprocessed_transaction_storage.rs @@ -19,12 +19,10 @@ use { min_max_heap::MinMaxHeap, solana_measure::{measure, measure_us}, solana_runtime::bank::Bank, + solana_runtime_transaction::extended_transaction::ExtendedSanitizedTransaction, solana_sdk::{ - clock::FORWARD_TRANSACTIONS_TO_LEADER_AT_SLOT_OFFSET, - feature_set::FeatureSet, - hash::Hash, - saturating_add_assign, - transaction::{ExtendedSanitizedTransaction, SanitizedTransaction}, + clock::FORWARD_TRANSACTIONS_TO_LEADER_AT_SLOT_OFFSET, feature_set::FeatureSet, hash::Hash, + saturating_add_assign, transaction::SanitizedTransaction, }, solana_svm::transaction_error_metrics::TransactionErrorMetrics, std::{ diff --git a/entry/Cargo.toml b/entry/Cargo.toml index a9bde85d833e7c..984c08a2a93e18 100644 --- a/entry/Cargo.toml +++ b/entry/Cargo.toml @@ -23,6 +23,7 @@ solana-merkle-tree = { workspace = true } solana-metrics = { workspace = true } solana-perf = { workspace = true } solana-rayon-threadlimit = { workspace = true } +solana-runtime-transaction = { workspace = true } solana-sdk = { workspace = true } [dev-dependencies] diff --git a/entry/src/entry.rs b/entry/src/entry.rs index 0029148dcee18a..3ea1ca7e51551a 100644 --- a/entry/src/entry.rs +++ b/entry/src/entry.rs @@ -22,13 +22,14 @@ use { sigverify, }, solana_rayon_threadlimit::get_max_thread_count, + solana_runtime_transaction::extended_transaction::ExtendedSanitizedTransaction, solana_sdk::{ hash::Hash, packet::Meta, timing, transaction::{ - ExtendedSanitizedTransaction, Result, Transaction, TransactionError, - TransactionVerificationMode, VersionedTransaction, + Result, Transaction, TransactionError, TransactionVerificationMode, + VersionedTransaction, }, }, std::{ diff --git a/ledger/Cargo.toml b/ledger/Cargo.toml index 7665428981ed82..8d3de706183b4e 100644 --- a/ledger/Cargo.toml +++ b/ledger/Cargo.toml @@ -49,6 +49,7 @@ solana-metrics = { workspace = true } solana-perf = { workspace = true } solana-program-runtime = { workspace = true } solana-rayon-threadlimit = { workspace = true } +solana-runtime-transaction = { workspace = true } solana-runtime = { workspace = true } solana-sdk = { workspace = true } solana-stake-program = { workspace = true } diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs index 64f37f9ad80fbf..b9f85dab6a0660 100644 --- a/ledger/src/blockstore_processor.rs +++ b/ledger/src/blockstore_processor.rs @@ -42,6 +42,7 @@ use { prioritization_fee_cache::PrioritizationFeeCache, transaction_batch::TransactionBatch, }, + solana_runtime_transaction::extended_transaction::ExtendedSanitizedTransaction, solana_sdk::{ clock::{Slot, MAX_PROCESSING_AGE}, feature_set, @@ -53,8 +54,7 @@ use { signature::{Keypair, Signature}, timing, transaction::{ - ExtendedSanitizedTransaction, Result, TransactionError, TransactionVerificationMode, - VersionedTransaction, + Result, TransactionError, TransactionVerificationMode, VersionedTransaction, }, }, solana_svm::{ diff --git a/rpc/Cargo.toml b/rpc/Cargo.toml index d4f2648b6b1078..8a5c7f57c5b1ea 100644 --- a/rpc/Cargo.toml +++ b/rpc/Cargo.toml @@ -43,6 +43,7 @@ solana-poh = { workspace = true } solana-rayon-threadlimit = { workspace = true } solana-rpc-client-api = { workspace = true } solana-runtime = { workspace = true } +solana-runtime-transaction = { workspace = true } solana-sdk = { workspace = true } solana-send-transaction-service = { workspace = true } solana-stake-program = { workspace = true } diff --git a/rpc/src/rpc.rs b/rpc/src/rpc.rs index c8100e29a622c5..f022bd9b9198a5 100644 --- a/rpc/src/rpc.rs +++ b/rpc/src/rpc.rs @@ -3252,10 +3252,8 @@ pub mod rpc_accounts_scan { pub mod rpc_full { use { super::*, - solana_sdk::{ - message::{SanitizedVersionedMessage, VersionedMessage}, - transaction::ExtendedSanitizedTransaction, - }, + solana_runtime_transaction::extended_transaction::ExtendedSanitizedTransaction, + solana_sdk::message::{SanitizedVersionedMessage, VersionedMessage}, solana_transaction_status::UiInnerInstructions, }; #[rpc] diff --git a/runtime-transaction/src/lib.rs b/runtime-transaction/src/lib.rs index 0fdeb7c5b6bd65..4868e1f77275fc 100644 --- a/runtime-transaction/src/lib.rs +++ b/runtime-transaction/src/lib.rs @@ -1,5 +1,6 @@ #![cfg_attr(RUSTC_WITH_SPECIALIZATION, feature(min_specialization))] #![allow(clippy::arithmetic_side_effects)] +pub mod extended_transaction; pub mod runtime_transaction; pub mod transaction_meta; diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index 85600e9de319da..e439f92ef008c0 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -62,6 +62,7 @@ solana-metrics = { workspace = true } solana-perf = { workspace = true } solana-program-runtime = { workspace = true } solana-rayon-threadlimit = { workspace = true } +solana-runtime-transaction = { workspace = true } solana-sdk = { workspace = true } solana-stake-program = { workspace = true } solana-svm = { workspace = true } diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index 100c28fe71285f..e557475aea56f4 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -102,6 +102,7 @@ use { runtime_config::RuntimeConfig, timings::{ExecuteTimingType, ExecuteTimings}, }, + solana_runtime_transaction::extended_transaction::ExtendedSanitizedTransaction, solana_sdk::{ account::{ create_account_shared_data_with_fields as create_account, create_executable_meta, @@ -152,9 +153,8 @@ use { sysvar::{self, last_restart_slot::LastRestartSlot, Sysvar, SysvarId}, timing::years_as_slots, transaction::{ - self, ExtendedSanitizedTransaction, MessageHash, Result, SanitizedTransaction, - Transaction, TransactionError, TransactionVerificationMode, VersionedTransaction, - MAX_TX_ACCOUNT_LOCKS, + self, MessageHash, Result, SanitizedTransaction, Transaction, TransactionError, + TransactionVerificationMode, VersionedTransaction, MAX_TX_ACCOUNT_LOCKS, }, transaction_context::{TransactionAccount, TransactionReturnData}, }, diff --git a/runtime/src/bank_utils.rs b/runtime/src/bank_utils.rs index 645630a89a938e..618089e10b55b7 100644 --- a/runtime/src/bank_utils.rs +++ b/runtime/src/bank_utils.rs @@ -7,7 +7,7 @@ use { solana_sdk::{pubkey::Pubkey, signature::Signer}, }; use { - solana_sdk::transaction::ExtendedSanitizedTransaction, + solana_runtime_transaction::extended_transaction::ExtendedSanitizedTransaction, solana_svm::transaction_results::TransactionResults, solana_vote::{vote_parser, vote_sender_types::ReplayVoteSender}, }; diff --git a/runtime/src/installed_scheduler_pool.rs b/runtime/src/installed_scheduler_pool.rs index ff62b69a8d87d9..5f3679060a93e3 100644 --- a/runtime/src/installed_scheduler_pool.rs +++ b/runtime/src/installed_scheduler_pool.rs @@ -24,11 +24,8 @@ use { crate::bank::Bank, log::*, solana_program_runtime::timings::ExecuteTimings, - solana_sdk::{ - hash::Hash, - slot_history::Slot, - transaction::{ExtendedSanitizedTransaction, Result}, - }, + solana_runtime_transaction::extended_transaction::ExtendedSanitizedTransaction, + solana_sdk::{hash::Hash, slot_history::Slot, transaction::Result}, std::{ fmt::Debug, ops::Deref, diff --git a/runtime/src/prioritization_fee_cache.rs b/runtime/src/prioritization_fee_cache.rs index 363379178174bd..9ff3c3d651be74 100644 --- a/runtime/src/prioritization_fee_cache.rs +++ b/runtime/src/prioritization_fee_cache.rs @@ -5,10 +5,10 @@ use { log::*, lru::LruCache, solana_measure::measure, + solana_runtime_transaction::extended_transaction::ExtendedSanitizedTransaction, solana_sdk::{ clock::{BankId, Slot}, pubkey::Pubkey, - transaction::ExtendedSanitizedTransaction, }, std::{ collections::HashMap, diff --git a/runtime/src/transaction_batch.rs b/runtime/src/transaction_batch.rs index fe1c8bc3210ea9..935b8a34a91113 100644 --- a/runtime/src/transaction_batch.rs +++ b/runtime/src/transaction_batch.rs @@ -1,7 +1,7 @@ use { crate::bank::Bank, - solana_sdk::transaction::{ExtendedSanitizedTransaction, Result}, - std::borrow::Cow, + solana_runtime_transaction::extended_transaction::ExtendedSanitizedTransaction, + solana_sdk::transaction::Result, std::borrow::Cow, }; // Represents the results of trying to lock a set of accounts diff --git a/sdk/src/transaction/sanitized.rs b/sdk/src/transaction/sanitized.rs index 2d4e2bd21018dc..b7383b4a0a454c 100644 --- a/sdk/src/transaction/sanitized.rs +++ b/sdk/src/transaction/sanitized.rs @@ -19,7 +19,6 @@ use { transaction::{Result, Transaction, TransactionError, VersionedTransaction}, }, solana_program::message::SanitizedVersionedMessage, - std::time::Instant, }; /// Maximum number of accounts that a transaction may lock. @@ -36,22 +35,6 @@ pub struct SanitizedTransaction { signatures: Vec, } -/// Sanitized transaction with option start_time -#[derive(Debug, Clone, Eq, PartialEq)] -pub struct ExtendedSanitizedTransaction { - pub transaction: SanitizedTransaction, - pub start_time: Option, -} - -impl From for ExtendedSanitizedTransaction { - fn from(value: SanitizedTransaction) -> Self { - Self { - transaction: value, - start_time: None, - } - } -} - /// Set of accounts that must be locked for safe transaction processing #[derive(Debug, Clone, Default, Eq, PartialEq)] pub struct TransactionAccountLocks<'a> { diff --git a/svm/Cargo.toml b/svm/Cargo.toml index ca4fba3a6e2b06..b436236ac5ba0d 100644 --- a/svm/Cargo.toml +++ b/svm/Cargo.toml @@ -21,6 +21,7 @@ solana-loader-v4-program = { workspace = true } solana-measure = { workspace = true } solana-metrics = { workspace = true } solana-program-runtime = { workspace = true } +solana-runtime-transaction = { workspace = true } solana-sdk = { workspace = true } solana-system-program = { workspace = true } diff --git a/svm/src/account_loader.rs b/svm/src/account_loader.rs index 5b9b2a1f8f0835..bf586649104510 100644 --- a/svm/src/account_loader.rs +++ b/svm/src/account_loader.rs @@ -10,6 +10,7 @@ use { compute_budget_processor::process_compute_budget_instructions, loaded_programs::LoadedProgramsForTxBatch, }, + solana_runtime_transaction::extended_transaction::ExtendedSanitizedTransaction, solana_sdk::{ account::{ create_executable_meta, is_builtin, is_executable, Account, AccountSharedData, @@ -30,7 +31,7 @@ use { rent_debits::RentDebits, saturating_add_assign, sysvar::{self, instructions::construct_instructions_data}, - transaction::{self, ExtendedSanitizedTransaction, Result, TransactionError}, + transaction::{self, Result, TransactionError}, transaction_context::{IndexOfAccount, TransactionAccount}, }, solana_system_program::{get_system_account_kind, SystemAccountKind}, diff --git a/svm/src/transaction_processor.rs b/svm/src/transaction_processor.rs index 7e40b27ecb5908..5cf7f07277196a 100644 --- a/svm/src/transaction_processor.rs +++ b/svm/src/transaction_processor.rs @@ -26,6 +26,7 @@ use { sysvar_cache::SysvarCache, timings::{ExecuteDetailsTimings, ExecuteTimingType, ExecuteTimings}, }, + solana_runtime_transaction::extended_transaction::ExtendedSanitizedTransaction, solana_sdk::{ account::{AccountSharedData, ReadableAccount, PROGRAM_OWNERS}, account_utils::StateMut, @@ -44,7 +45,7 @@ use { rent_collector::RentCollector, saturating_add_assign, timing::duration_as_us, - transaction::{self, ExtendedSanitizedTransaction, SanitizedTransaction, TransactionError}, + transaction::{self, SanitizedTransaction, TransactionError}, transaction_context::{ExecutionRecord, TransactionContext}, }, std::{ diff --git a/unified-scheduler-logic/Cargo.toml b/unified-scheduler-logic/Cargo.toml index b2e80c79c7a08f..0cc315b159af39 100644 --- a/unified-scheduler-logic/Cargo.toml +++ b/unified-scheduler-logic/Cargo.toml @@ -11,3 +11,4 @@ edition = { workspace = true } [dependencies] solana-sdk = { workspace = true } +solana-runtime-transaction = { workspace = true } diff --git a/unified-scheduler-logic/src/lib.rs b/unified-scheduler-logic/src/lib.rs index e587c3ba06f38d..e1c9dc64205a84 100644 --- a/unified-scheduler-logic/src/lib.rs +++ b/unified-scheduler-logic/src/lib.rs @@ -1,4 +1,4 @@ -use solana_sdk::transaction::ExtendedSanitizedTransaction; +use solana_runtime_transaction::extended_transaction::ExtendedSanitizedTransaction; pub struct Task { transaction: ExtendedSanitizedTransaction, diff --git a/unified-scheduler-pool/Cargo.toml b/unified-scheduler-pool/Cargo.toml index 7626215b1e1126..b5a6920bb93753 100644 --- a/unified-scheduler-pool/Cargo.toml +++ b/unified-scheduler-pool/Cargo.toml @@ -17,6 +17,7 @@ log = { workspace = true } solana-ledger = { workspace = true } solana-program-runtime = { workspace = true } solana-runtime = { workspace = true } +solana-runtime-transaction = { workspace = true } solana-sdk = { workspace = true } solana-unified-scheduler-logic = { workspace = true } solana-vote = { workspace = true } diff --git a/unified-scheduler-pool/src/lib.rs b/unified-scheduler-pool/src/lib.rs index 4d8780c278a3c9..6ee3460390c2a9 100644 --- a/unified-scheduler-pool/src/lib.rs +++ b/unified-scheduler-pool/src/lib.rs @@ -26,7 +26,8 @@ use { }, prioritization_fee_cache::PrioritizationFeeCache, }, - solana_sdk::transaction::{ExtendedSanitizedTransaction, Result}, + solana_runtime_transaction::extended_transaction::ExtendedSanitizedTransaction, + solana_sdk::transaction::Result, solana_unified_scheduler_logic::Task, solana_vote::vote_sender_types::ReplayVoteSender, std::{ From 7e26c64a323540476c81bbc2be25c03412a507f2 Mon Sep 17 00:00:00 2001 From: Lijun Wang <83639177+lijunwangs@users.noreply.github.com> Date: Mon, 11 Mar 2024 00:32:09 -0700 Subject: [PATCH 69/71] Fixed unit test failures --- accounts-db/src/accounts.rs | 38 ++-- core/src/banking_stage.rs | 7 +- core/src/banking_stage/consumer.rs | 39 ++-- .../forward_packet_batches_by_accounts.rs | 16 +- core/src/banking_stage/qos_service.rs | 74 ++++-- .../prio_graph_scheduler.rs | 22 +- .../scheduler_controller.rs | 16 +- .../transaction_state.rs | 11 +- .../transaction_state_container.rs | 3 +- entry/src/entry.rs | 9 +- ledger/src/blockstore_processor.rs | 14 +- rpc/src/rpc.rs | 2 +- rpc/src/transaction_status_service.rs | 2 +- runtime/src/bank/tests.rs | 10 +- runtime/src/installed_scheduler_pool.rs | 4 +- runtime/src/prioritization_fee_cache.rs | 33 +-- runtime/src/transaction_batch.rs | 11 +- svm/src/account_loader.rs | 4 +- svm/tests/account_loader.rs | 214 ++++++++++++++++++ svm/tests/rent_state.rs | 90 ++++++++ svm/tests/transaction_processor.rs | 4 +- unified-scheduler-pool/src/lib.rs | 16 +- 22 files changed, 514 insertions(+), 125 deletions(-) create mode 100644 svm/tests/account_loader.rs create mode 100644 svm/tests/rent_state.rs diff --git a/accounts-db/src/accounts.rs b/accounts-db/src/accounts.rs index b72b47f715394c..52674ea37824eb 100644 --- a/accounts-db/src/accounts.rs +++ b/accounts-db/src/accounts.rs @@ -813,7 +813,7 @@ mod tests { rent_debits::RentDebits, signature::{keypair_from_seed, signers::Signers, Keypair, Signer}, system_instruction, system_program, - transaction::{Transaction, MAX_TX_ACCOUNT_LOCKS}, + transaction::{SanitizedTransaction, Transaction, MAX_TX_ACCOUNT_LOCKS}, }, solana_svm::{ account_loader::LoadedTransaction, @@ -1080,7 +1080,7 @@ mod tests { }; let tx = new_sanitized_tx(&[&keypair], message, Hash::default()); - let results = accounts.lock_accounts([tx].iter(), MAX_TX_ACCOUNT_LOCKS); + let results = accounts.lock_accounts([tx.into()].iter(), MAX_TX_ACCOUNT_LOCKS); assert_eq!(results[0], Err(TransactionError::AccountLoadedTwice)); } @@ -1107,7 +1107,7 @@ mod tests { ..Message::default() }; - let txs = vec![new_sanitized_tx(&[&keypair], message, Hash::default())]; + let txs = vec![new_sanitized_tx(&[&keypair], message, Hash::default()).into()]; let results = accounts.lock_accounts(txs.iter(), MAX_TX_ACCOUNT_LOCKS); assert_eq!(results, vec![Ok(())]); accounts.unlock_accounts(txs.iter().zip(&results)); @@ -1129,7 +1129,7 @@ mod tests { ..Message::default() }; - let txs = vec![new_sanitized_tx(&[&keypair], message, Hash::default())]; + let txs = vec![new_sanitized_tx(&[&keypair], message, Hash::default()).into()]; let results = accounts.lock_accounts(txs.iter(), MAX_TX_ACCOUNT_LOCKS); assert_eq!(results[0], Err(TransactionError::TooManyAccountLocks)); } @@ -1164,7 +1164,7 @@ mod tests { instructions, ); let tx = new_sanitized_tx(&[&keypair0], message, Hash::default()); - let results0 = accounts.lock_accounts([tx.clone()].iter(), MAX_TX_ACCOUNT_LOCKS); + let results0 = accounts.lock_accounts([tx.clone().into()].iter(), MAX_TX_ACCOUNT_LOCKS); assert_eq!(results0, vec![Ok(())]); assert_eq!( @@ -1198,7 +1198,7 @@ mod tests { instructions, ); let tx1 = new_sanitized_tx(&[&keypair1], message, Hash::default()); - let txs = vec![tx0, tx1]; + let txs = vec![tx0.into(), tx1.into()]; let results1 = accounts.lock_accounts(txs.iter(), MAX_TX_ACCOUNT_LOCKS); assert_eq!( results1, @@ -1218,7 +1218,7 @@ mod tests { 2 ); - accounts.unlock_accounts(iter::once(&tx).zip(&results0)); + accounts.unlock_accounts(iter::once(&tx.into()).zip(&results0)); accounts.unlock_accounts(txs.iter().zip(&results1)); let instructions = vec![CompiledInstruction::new(2, &(), vec![0, 1])]; let message = Message::new_with_compiled_instructions( @@ -1230,7 +1230,7 @@ mod tests { instructions, ); let tx = new_sanitized_tx(&[&keypair1], message, Hash::default()); - let results2 = accounts.lock_accounts([tx].iter(), MAX_TX_ACCOUNT_LOCKS); + let results2 = accounts.lock_accounts([tx.into()].iter(), MAX_TX_ACCOUNT_LOCKS); assert_eq!( results2, vec![Ok(())] // Now keypair1 account can be locked as writable @@ -1293,7 +1293,7 @@ mod tests { let accounts_clone = accounts_arc.clone(); let exit_clone = exit.clone(); thread::spawn(move || loop { - let txs = vec![writable_tx.clone()]; + let txs = vec![writable_tx.clone().into()]; let results = accounts_clone .clone() .lock_accounts(txs.iter(), MAX_TX_ACCOUNT_LOCKS); @@ -1309,7 +1309,7 @@ mod tests { }); let counter_clone = counter; for _ in 0..5 { - let txs = vec![readonly_tx.clone()]; + let txs = vec![readonly_tx.clone().into()]; let results = accounts_arc .clone() .lock_accounts(txs.iter(), MAX_TX_ACCOUNT_LOCKS); @@ -1353,7 +1353,7 @@ mod tests { instructions, ); let tx = new_sanitized_tx(&[&keypair0], message, Hash::default()); - let results0 = accounts.lock_accounts([tx].iter(), MAX_TX_ACCOUNT_LOCKS); + let results0 = accounts.lock_accounts([tx.into()].iter(), MAX_TX_ACCOUNT_LOCKS); assert!(results0[0].is_ok()); // Instruction program-id account demoted to readonly @@ -1445,7 +1445,7 @@ mod tests { instructions, ); let tx2 = new_sanitized_tx(&[&keypair3], message, Hash::default()); - let txs = vec![tx0, tx1, tx2]; + let txs = vec![tx0.into(), tx1.into(), tx2.into()]; let qos_results = vec![ Ok(()), @@ -1575,7 +1575,7 @@ mod tests { .unwrap() .insert_new_readonly(&pubkey); } - let txs = vec![tx0.clone(), tx1.clone()]; + let txs = vec![tx0.clone().into(), tx1.clone().into()]; let execution_results = vec![new_execution_result(Ok(()), None); 2]; let (collected_accounts, transactions) = accounts.collect_accounts_to_store( &txs, @@ -1593,8 +1593,12 @@ mod tests { .any(|(pubkey, _account)| *pubkey == &keypair1.pubkey())); assert_eq!(transactions.len(), 2); - assert!(transactions.iter().any(|txn| txn.unwrap().eq(&tx0))); - assert!(transactions.iter().any(|txn| txn.unwrap().eq(&tx1))); + assert!(transactions + .iter() + .any(|txn| txn.unwrap().eq(&tx0.clone().into()))); + assert!(transactions + .iter() + .any(|txn| txn.unwrap().eq(&tx1.clone().into()))); // Ensure readonly_lock reflects lock assert_eq!( @@ -1947,7 +1951,7 @@ mod tests { let durable_nonce = DurableNonce::from_blockhash(&Hash::new_unique()); let accounts_db = AccountsDb::new_single_for_tests(); let accounts = Accounts::new(Arc::new(accounts_db)); - let txs = vec![tx]; + let txs = vec![tx.into()]; let execution_results = vec![new_execution_result( Err(TransactionError::InstructionError( 1, @@ -2053,7 +2057,7 @@ mod tests { let durable_nonce = DurableNonce::from_blockhash(&Hash::new_unique()); let accounts_db = AccountsDb::new_single_for_tests(); let accounts = Accounts::new(Arc::new(accounts_db)); - let txs = vec![tx]; + let txs = vec![tx.into()]; let execution_results = vec![new_execution_result( Err(TransactionError::InstructionError( 1, diff --git a/core/src/banking_stage.rs b/core/src/banking_stage.rs index 603ff55f0003b4..9d30eb5a0ccaa1 100644 --- a/core/src/banking_stage.rs +++ b/core/src/banking_stage.rs @@ -804,6 +804,7 @@ mod tests { poh_service::PohService, }, solana_runtime::{bank::Bank, genesis_utils::bootstrap_validator_stake_lamports}, + solana_runtime_transaction::extended_transaction::ExtendedSanitizedTransaction, solana_sdk::{ hash::Hash, poh_config::PohConfig, @@ -830,9 +831,11 @@ mod tests { (node, cluster_info) } - pub(crate) fn sanitize_transactions(txs: Vec) -> Vec { + pub(crate) fn sanitize_transactions( + txs: Vec, + ) -> Vec { txs.into_iter() - .map(SanitizedTransaction::from_transaction_for_tests) + .map(|t| SanitizedTransaction::from_transaction_for_tests(t).into()) .collect() } diff --git a/core/src/banking_stage/consumer.rs b/core/src/banking_stage/consumer.rs index b55019130c3ca1..fcd428582c4957 100644 --- a/core/src/banking_stage/consumer.rs +++ b/core/src/banking_stage/consumer.rs @@ -888,7 +888,7 @@ mod tests { signature::Keypair, signer::Signer, system_instruction, system_program, system_transaction, - transaction::{MessageHash, Transaction, VersionedTransaction}, + transaction::{MessageHash, SanitizedTransaction, Transaction, VersionedTransaction}, }, solana_transaction_status::{TransactionStatusMeta, VersionedTransactionWithStatusMeta}, std::{ @@ -941,7 +941,7 @@ mod tests { ); let consumer = Consumer::new(committer, recorder, QosService::new(1), None); let process_transactions_summary = - consumer.process_transactions(&bank, &Instant::now(), &transactions); + consumer.process_transactions(&bank, &Instant::now(), &transactions, &mut None); poh_recorder .read() @@ -1117,7 +1117,7 @@ mod tests { let consumer = Consumer::new(committer, recorder, QosService::new(1), None); let process_transactions_batch_output = - consumer.process_and_record_transactions(&bank, &transactions, 0); + consumer.process_and_record_transactions(&bank, &transactions, 0, &mut None); let ExecuteAndCommitTransactionsOutput { transactions_attempted_execution_count, @@ -1162,7 +1162,7 @@ mod tests { )]); let process_transactions_batch_output = - consumer.process_and_record_transactions(&bank, &transactions, 0); + consumer.process_and_record_transactions(&bank, &transactions, 0, &mut None); let ExecuteAndCommitTransactionsOutput { transactions_attempted_execution_count, @@ -1302,7 +1302,7 @@ mod tests { let consumer = Consumer::new(committer, recorder, QosService::new(1), None); let process_transactions_batch_output = - consumer.process_and_record_transactions(&bank, &transactions, 0); + consumer.process_and_record_transactions(&bank, &transactions, 0, &mut None); let ExecuteAndCommitTransactionsOutput { transactions_attempted_execution_count, executed_transactions_count, @@ -1404,7 +1404,7 @@ mod tests { let consumer = Consumer::new(committer, recorder, QosService::new(1), None); let process_transactions_batch_output = - consumer.process_and_record_transactions(&bank, &transactions, 0); + consumer.process_and_record_transactions(&bank, &transactions, 0, &mut None); let ExecuteAndCommitTransactionsOutput { transactions_attempted_execution_count, @@ -1510,7 +1510,7 @@ mod tests { )]); let process_transactions_batch_output = - consumer.process_and_record_transactions(&bank, &transactions, 0); + consumer.process_and_record_transactions(&bank, &transactions, 0, &mut None); let ExecuteAndCommitTransactionsOutput { executed_with_successful_result_count, @@ -1541,7 +1541,7 @@ mod tests { ]); let process_transactions_batch_output = - consumer.process_and_record_transactions(&bank, &transactions, 0); + consumer.process_and_record_transactions(&bank, &transactions, 0, &mut None); let ExecuteAndCommitTransactionsOutput { executed_with_successful_result_count, @@ -1575,14 +1575,17 @@ mod tests { } }; - let mut cost = CostModel::calculate_cost(&transactions[0], &bank.feature_set); + let mut cost = + CostModel::calculate_cost(&transactions[0].transaction, &bank.feature_set); if let TransactionCost::Transaction(ref mut usage_cost) = cost { usage_cost.programs_execution_cost = actual_programs_execution_cost; } block_cost + cost.sum() } else { - block_cost + CostModel::calculate_cost(&transactions[0], &bank.feature_set).sum() + block_cost + + CostModel::calculate_cost(&transactions[0].transaction, &bank.feature_set) + .sum() }; assert_eq!(get_block_cost(), expected_block_cost); @@ -1650,7 +1653,7 @@ mod tests { let consumer = Consumer::new(committer, recorder, QosService::new(1), None); let process_transactions_batch_output = - consumer.process_and_record_transactions(&bank, &transactions, 0); + consumer.process_and_record_transactions(&bank, &transactions, 0, &mut None); poh_recorder .read() @@ -1847,7 +1850,7 @@ mod tests { let consumer = Consumer::new(committer, recorder.clone(), QosService::new(1), None); let process_transactions_summary = - consumer.process_transactions(&bank, &Instant::now(), &transactions); + consumer.process_transactions(&bank, &Instant::now(), &transactions, &mut None); let ProcessTransactionsSummary { reached_max_poh_height, @@ -1974,7 +1977,7 @@ mod tests { ); let consumer = Consumer::new(committer, recorder, QosService::new(1), None); - let _ = consumer.process_and_record_transactions(&bank, &transactions, 0); + let _ = consumer.process_and_record_transactions(&bank, &transactions, 0, &mut None); drop(consumer); // drop/disconnect transaction_status_sender transaction_status_service.join().unwrap(); @@ -2119,7 +2122,12 @@ mod tests { ); let consumer = Consumer::new(committer, recorder, QosService::new(1), None); - let _ = consumer.process_and_record_transactions(&bank, &[sanitized_tx.clone()], 0); + let _ = consumer.process_and_record_transactions( + &bank, + &[sanitized_tx.clone().into()], + 0, + &mut None, + ); drop(consumer); // drop/disconnect transaction_status_sender transaction_status_service.join().unwrap(); @@ -2328,7 +2336,8 @@ mod tests { &lock_account, 1, bank.last_blockhash(), - )); + )) + .into(); let _ = bank_start.working_bank.accounts().lock_accounts( std::iter::once(&manual_lock_tx), bank_start.working_bank.get_transaction_account_lock_limit(), diff --git a/core/src/banking_stage/forward_packet_batches_by_accounts.rs b/core/src/banking_stage/forward_packet_batches_by_accounts.rs index 6b1685c8a46483..3db9797efcad8b 100644 --- a/core/src/banking_stage/forward_packet_batches_by_accounts.rs +++ b/core/src/banking_stage/forward_packet_batches_by_accounts.rs @@ -139,8 +139,12 @@ mod tests { super::*, crate::banking_stage::unprocessed_packet_batches::DeserializedPacket, solana_sdk::{ - compute_budget::ComputeBudgetInstruction, feature_set::FeatureSet, message::Message, - pubkey::Pubkey, system_instruction, transaction::Transaction, + compute_budget::ComputeBudgetInstruction, + feature_set::FeatureSet, + message::Message, + pubkey::Pubkey, + system_instruction, + transaction::{SanitizedTransaction, Transaction}, }, }; @@ -149,7 +153,7 @@ mod tests { fn build_test_transaction_and_packet( priority: u64, write_to_account: &Pubkey, - ) -> (SanitizedTransaction, DeserializedPacket, u32) { + ) -> (ExtendedSanitizedTransaction, DeserializedPacket, u32) { let from_account = solana_sdk::pubkey::new_rand(); let transaction = Transaction::new_unsigned(Message::new( @@ -169,7 +173,11 @@ mod tests { // set limit ratio so each batch can only have one test transaction let limit_ratio: u32 = ((block_cost_limits::MAX_WRITABLE_ACCOUNT_UNITS - cost + 1) / cost) as u32; - (sanitized_transaction, deserialized_packet, limit_ratio) + ( + sanitized_transaction.into(), + deserialized_packet, + limit_ratio, + ) } #[test] diff --git a/core/src/banking_stage/qos_service.rs b/core/src/banking_stage/qos_service.rs index 91b89a9e619cc1..78cd5bfd8109c8 100644 --- a/core/src/banking_stage/qos_service.rs +++ b/core/src/banking_stage/qos_service.rs @@ -582,6 +582,7 @@ mod tests { hash::Hash, signature::{Keypair, Signer}, system_transaction, + transaction::SanitizedTransaction, }, solana_vote_program::vote_transaction, std::sync::Arc, @@ -593,20 +594,27 @@ mod tests { // make a vec of txs let keypair = Keypair::new(); - let transfer_tx = SanitizedTransaction::from_transaction_for_tests( - system_transaction::transfer(&keypair, &keypair.pubkey(), 1, Hash::default()), - ); - let vote_tx = SanitizedTransaction::from_transaction_for_tests( - vote_transaction::new_vote_transaction( - vec![42], - Hash::default(), - Hash::default(), - &keypair, + let transfer_tx: ExtendedSanitizedTransaction = + SanitizedTransaction::from_transaction_for_tests(system_transaction::transfer( &keypair, - &keypair, - None, - ), - ); + &keypair.pubkey(), + 1, + Hash::default(), + )) + .into(); + let vote_tx: ExtendedSanitizedTransaction = + SanitizedTransaction::from_transaction_for_tests( + vote_transaction::new_vote_transaction( + vec![42], + Hash::default(), + Hash::default(), + &keypair, + &keypair, + &keypair, + None, + ), + ) + .into(); let txs = vec![transfer_tx.clone(), vote_tx.clone(), vote_tx, transfer_tx]; let qos_service = QosService::new(1); @@ -624,7 +632,8 @@ mod tests { .map(|(index, cost)| { assert_eq!( cost.as_ref().unwrap().sum(), - CostModel::calculate_cost(&txs[index], &FeatureSet::all_enabled()).sum() + CostModel::calculate_cost(&txs[index].transaction, &FeatureSet::all_enabled()) + .sum() ); }) .collect_vec(); @@ -656,7 +665,12 @@ mod tests { let vote_tx_cost = CostModel::calculate_cost(&vote_tx, &FeatureSet::all_enabled()).sum(); // make a vec of txs - let txs = vec![transfer_tx.clone(), vote_tx.clone(), transfer_tx, vote_tx]; + let txs = vec![ + transfer_tx.clone().into(), + vote_tx.clone().into(), + transfer_tx.into(), + vote_tx.into(), + ]; let qos_service = QosService::new(1); let txs_costs = qos_service.compute_transaction_costs( @@ -695,8 +709,8 @@ mod tests { let transfer_tx = SanitizedTransaction::from_transaction_for_tests( system_transaction::transfer(&keypair, &keypair.pubkey(), 1, Hash::default()), ); - let txs: Vec = (0..transaction_count) - .map(|_| transfer_tx.clone()) + let txs: Vec = (0..transaction_count) + .map(|_| transfer_tx.clone().into()) .collect(); let execute_units_adjustment = 10u64; @@ -761,10 +775,15 @@ mod tests { // calculate their costs, apply to cost_tracker let transaction_count = 5; let keypair = Keypair::new(); - let transfer_tx = SanitizedTransaction::from_transaction_for_tests( - system_transaction::transfer(&keypair, &keypair.pubkey(), 1, Hash::default()), - ); - let txs: Vec = (0..transaction_count) + let transfer_tx: ExtendedSanitizedTransaction = + SanitizedTransaction::from_transaction_for_tests(system_transaction::transfer( + &keypair, + &keypair.pubkey(), + 1, + Hash::default(), + )) + .into(); + let txs: Vec = (0..transaction_count) .map(|_| transfer_tx.clone()) .collect(); @@ -814,10 +833,15 @@ mod tests { // calculate their costs, apply to cost_tracker let transaction_count = 5; let keypair = Keypair::new(); - let transfer_tx = SanitizedTransaction::from_transaction_for_tests( - system_transaction::transfer(&keypair, &keypair.pubkey(), 1, Hash::default()), - ); - let txs: Vec = (0..transaction_count) + let transfer_tx: ExtendedSanitizedTransaction = + SanitizedTransaction::from_transaction_for_tests(system_transaction::transfer( + &keypair, + &keypair.pubkey(), + 1, + Hash::default(), + )) + .into(); + let txs: Vec = (0..transaction_count) .map(|_| transfer_tx.clone()) .collect(); let execute_units_adjustment = 10u64; diff --git a/core/src/banking_stage/transaction_scheduler/prio_graph_scheduler.rs b/core/src/banking_stage/transaction_scheduler/prio_graph_scheduler.rs index a80cdabb668ea6..59bc13c6fe29a6 100644 --- a/core/src/banking_stage/transaction_scheduler/prio_graph_scheduler.rs +++ b/core/src/banking_stage/transaction_scheduler/prio_graph_scheduler.rs @@ -490,8 +490,14 @@ mod tests { crossbeam_channel::{unbounded, Receiver}, itertools::Itertools, solana_sdk::{ - compute_budget::ComputeBudgetInstruction, hash::Hash, message::Message, pubkey::Pubkey, - signature::Keypair, signer::Signer, system_instruction, transaction::Transaction, + compute_budget::ComputeBudgetInstruction, + hash::Hash, + message::Message, + pubkey::Pubkey, + signature::Keypair, + signer::Signer, + system_instruction, + transaction::{SanitizedTransaction, Transaction}, }, std::borrow::Borrow, }; @@ -567,7 +573,8 @@ mod tests { to_pubkeys, lamports, compute_unit_price, - ); + ) + .into(); let transaction_ttl = SanitizedTransactionTTL { transaction, max_age_slot: Slot::MAX, @@ -596,11 +603,11 @@ mod tests { .unzip() } - fn test_pre_graph_filter(_txs: &[&SanitizedTransaction], results: &mut [bool]) { + fn test_pre_graph_filter(_txs: &[&ExtendedSanitizedTransaction], results: &mut [bool]) { results.fill(true); } - fn test_pre_lock_filter(_tx: &SanitizedTransaction) -> bool { + fn test_pre_lock_filter(_tx: &ExtendedSanitizedTransaction) -> bool { true } @@ -775,8 +782,9 @@ mod tests { ]); // 2nd transaction should be filtered out and dropped before locking. - let pre_lock_filter = - |tx: &SanitizedTransaction| tx.message().fee_payer() != &keypair.pubkey(); + let pre_lock_filter = |tx: &ExtendedSanitizedTransaction| { + tx.transaction.message().fee_payer() != &keypair.pubkey() + }; let scheduling_summary = scheduler .schedule(&mut container, test_pre_graph_filter, pre_lock_filter) .unwrap(); diff --git a/core/src/banking_stage/transaction_scheduler/scheduler_controller.rs b/core/src/banking_stage/transaction_scheduler/scheduler_controller.rs index e3aa895fddc54c..b5a7a786424ec9 100644 --- a/core/src/banking_stage/transaction_scheduler/scheduler_controller.rs +++ b/core/src/banking_stage/transaction_scheduler/scheduler_controller.rs @@ -749,7 +749,7 @@ mod tests { let message_hashes = consume_work .transactions .iter() - .map(|tx| tx.message_hash()) + .map(|tx| tx.transaction.message_hash()) .collect_vec(); assert_eq!(message_hashes, vec![&tx2_hash, &tx1_hash]); } @@ -807,7 +807,11 @@ mod tests { let num_txs_per_batch = consume_works.iter().map(|cw| cw.ids.len()).collect_vec(); let message_hashes = consume_works .iter() - .flat_map(|cw| cw.transactions.iter().map(|tx| tx.message_hash())) + .flat_map(|cw| { + cw.transactions + .iter() + .map(|tx| tx.transaction.message_hash()) + }) .collect_vec(); assert_eq!(num_txs_per_batch, vec![1; 2]); assert_eq!(message_hashes, vec![&tx2_hash, &tx1_hash]); @@ -930,14 +934,14 @@ mod tests { .unwrap() .transactions .iter() - .map(|tx| *tx.message_hash()) + .map(|tx| *tx.transaction.message_hash()) .collect_vec(); let t1_actual = consume_work_receivers[1] .try_recv() .unwrap() .transactions .iter() - .map(|tx| *tx.message_hash()) + .map(|tx| *tx.transaction.message_hash()) .collect_vec(); assert_eq!(t0_actual, t0_expected); @@ -996,7 +1000,7 @@ mod tests { let message_hashes = consume_work .transactions .iter() - .map(|tx| tx.message_hash()) + .map(|tx| tx.transaction.message_hash()) .collect_vec(); assert_eq!(message_hashes, vec![&tx2_hash, &tx1_hash]); @@ -1016,7 +1020,7 @@ mod tests { let message_hashes = consume_work .transactions .iter() - .map(|tx| tx.message_hash()) + .map(|tx| tx.transaction.message_hash()) .collect_vec(); assert_eq!(message_hashes, vec![&tx1_hash]); } diff --git a/core/src/banking_stage/transaction_scheduler/transaction_state.rs b/core/src/banking_stage/transaction_scheduler/transaction_state.rs index 22bf8a7b87344a..642bed2a7639bd 100644 --- a/core/src/banking_stage/transaction_scheduler/transaction_state.rs +++ b/core/src/banking_stage/transaction_scheduler/transaction_state.rs @@ -141,8 +141,13 @@ mod tests { use { super::*, solana_sdk::{ - compute_budget::ComputeBudgetInstruction, hash::Hash, message::Message, - signature::Keypair, signer::Signer, system_instruction, transaction::Transaction, + compute_budget::ComputeBudgetInstruction, + hash::Hash, + message::Message, + signature::Keypair, + signer::Signer, + system_instruction, + transaction::{SanitizedTransaction, Transaction}, }, }; @@ -160,7 +165,7 @@ mod tests { let tx = Transaction::new(&[&from_keypair], message, Hash::default()); let transaction_ttl = SanitizedTransactionTTL { - transaction: SanitizedTransaction::from_transaction_for_tests(tx), + transaction: SanitizedTransaction::from_transaction_for_tests(tx).into(), max_age_slot: Slot::MAX, }; const TEST_TRANSACTION_COST: u64 = 5000; diff --git a/core/src/banking_stage/transaction_scheduler/transaction_state_container.rs b/core/src/banking_stage/transaction_scheduler/transaction_state_container.rs index a627375a03e6ba..23873da4c44167 100644 --- a/core/src/banking_stage/transaction_scheduler/transaction_state_container.rs +++ b/core/src/banking_stage/transaction_scheduler/transaction_state_container.rs @@ -187,7 +187,8 @@ mod tests { &[&from_keypair], message, Hash::default(), - )); + )) + .into(); let transaction_ttl = SanitizedTransactionTTL { transaction: tx, max_age_slot: Slot::MAX, diff --git a/entry/src/entry.rs b/entry/src/entry.rs index 3ea1ca7e51551a..2a279408324d83 100644 --- a/entry/src/entry.rs +++ b/entry/src/entry.rs @@ -982,7 +982,7 @@ mod tests { dyn Fn( VersionedTransaction, TransactionVerificationMode, - ) -> Result + ) -> Result + Send + Sync, >, @@ -994,7 +994,7 @@ mod tests { } else { TransactionVerificationMode::FullVerification }; - move |versioned_tx: VersionedTransaction| -> Result { + move |versioned_tx: VersionedTransaction| -> Result { verify(versioned_tx, verification_mode) } }; @@ -1035,7 +1035,7 @@ mod tests { let verify_transaction = { move |versioned_tx: VersionedTransaction, verification_mode: TransactionVerificationMode| - -> Result { + -> Result { let sanitized_tx = { let message_hash = if verification_mode == TransactionVerificationMode::FullVerification { @@ -1050,7 +1050,8 @@ mod tests { None, SimpleAddressLoader::Disabled, ) - }?; + }? + .into(); Ok(sanitized_tx) } diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs index b9f85dab6a0660..3182fb9d2ea635 100644 --- a/ledger/src/blockstore_processor.rs +++ b/ledger/src/blockstore_processor.rs @@ -1972,7 +1972,7 @@ pub mod tests { signature::{Keypair, Signer}, system_instruction::SystemError, system_transaction, - transaction::{Transaction, TransactionError}, + transaction::{SanitizedTransaction, Transaction, TransactionError}, }, solana_svm::transaction_processor::ExecutionRecordingConfig, solana_vote::vote_account::VoteAccount, @@ -3968,6 +3968,7 @@ pub mod tests { ExecutionRecordingConfig::new_single_setting(false), &mut ExecuteTimings::default(), None, + None, ); let (err, signature) = get_first_error(&batch, fee_collection_results).unwrap(); assert_eq!(err.unwrap_err(), TransactionError::AccountNotFound); @@ -4361,7 +4362,7 @@ pub mod tests { fn create_test_transactions( mint_keypair: &Keypair, genesis_hash: &Hash, - ) -> Vec { + ) -> Vec { let pubkey = solana_sdk::pubkey::new_rand(); let keypair2 = Keypair::new(); let pubkey2 = solana_sdk::pubkey::new_rand(); @@ -4374,19 +4375,22 @@ pub mod tests { &pubkey, 1, *genesis_hash, - )), + )) + .into(), SanitizedTransaction::from_transaction_for_tests(system_transaction::transfer( &keypair2, &pubkey2, 1, *genesis_hash, - )), + )) + .into(), SanitizedTransaction::from_transaction_for_tests(system_transaction::transfer( &keypair3, &pubkey3, 1, *genesis_hash, - )), + )) + .into(), ] } diff --git a/rpc/src/rpc.rs b/rpc/src/rpc.rs index f022bd9b9198a5..f5e51ad47f51ea 100644 --- a/rpc/src/rpc.rs +++ b/rpc/src/rpc.rs @@ -5081,7 +5081,7 @@ pub mod tests { let prioritization_fee_cache = &self.meta.prioritization_fee_cache; let transactions: Vec<_> = transactions .into_iter() - .map(SanitizedTransaction::from_transaction_for_tests) + .map(|t| SanitizedTransaction::from_transaction_for_tests(t).into()) .collect(); prioritization_fee_cache.update(&bank, transactions.iter()); } diff --git a/rpc/src/transaction_status_service.rs b/rpc/src/transaction_status_service.rs index 29a20542379ba3..055daeea04e976 100644 --- a/rpc/src/transaction_status_service.rs +++ b/rpc/src/transaction_status_service.rs @@ -415,7 +415,7 @@ pub(crate) mod tests { let transaction_index: usize = bank.transaction_count().try_into().unwrap(); let transaction_status_batch = TransactionStatusBatch { bank, - transactions: vec![transaction], + transactions: vec![transaction.into()], execution_results: vec![transaction_result], balances, token_balances, diff --git a/runtime/src/bank/tests.rs b/runtime/src/bank/tests.rs index f9b846d85b1512..2d4dadd3c808a7 100644 --- a/runtime/src/bank/tests.rs +++ b/runtime/src/bank/tests.rs @@ -2923,7 +2923,7 @@ fn test_filter_program_errors_and_collect_fee() { ]; let initial_balance = bank.get_balance(&leader); - let results = bank.filter_program_errors_and_collect_fee(&[tx1, tx2], &results); + let results = bank.filter_program_errors_and_collect_fee(&[tx1.into(), tx2.into()], &results); bank.freeze(); assert_eq!( bank.get_balance(&leader), @@ -2974,7 +2974,7 @@ fn test_filter_program_errors_and_collect_compute_unit_fee() { ]; let initial_balance = bank.get_balance(&leader); - let results = bank.filter_program_errors_and_collect_fee(&[tx1, tx2], &results); + let results = bank.filter_program_errors_and_collect_fee(&[tx1.into(), tx2.into()], &results); bank.freeze(); assert_eq!( bank.get_balance(&leader), @@ -3125,6 +3125,7 @@ fn test_interleaving_locks() { ExecutionRecordingConfig::new_single_setting(false), &mut ExecuteTimings::default(), None, + None, ) .0 .fee_collection_results; @@ -5949,6 +5950,7 @@ fn test_pre_post_transaction_balances() { ExecutionRecordingConfig::new_single_setting(false), &mut ExecuteTimings::default(), None, + None, ); assert_eq!(transaction_balances_set.pre_balances.len(), 3); @@ -9233,6 +9235,7 @@ fn test_tx_log_order() { }, &mut ExecuteTimings::default(), None, + None, ) .0 .execution_results; @@ -9343,6 +9346,7 @@ fn test_tx_return_data() { }, &mut ExecuteTimings::default(), None, + None, ) .0 .execution_results[0] @@ -13810,7 +13814,7 @@ fn test_failed_simulation_compute_units() { let transaction = Transaction::new(&[&mint_keypair], message, bank.last_blockhash()); bank.freeze(); - let sanitized = SanitizedTransaction::from_transaction_for_tests(transaction); + let sanitized = SanitizedTransaction::from_transaction_for_tests(transaction).into(); let simulation = bank.simulate_transaction(&sanitized, false); assert_eq!(expected_consumed_units, simulation.units_consumed); } diff --git a/runtime/src/installed_scheduler_pool.rs b/runtime/src/installed_scheduler_pool.rs index 5f3679060a93e3..df77af1718d934 100644 --- a/runtime/src/installed_scheduler_pool.rs +++ b/runtime/src/installed_scheduler_pool.rs @@ -427,7 +427,7 @@ mod tests { }, assert_matches::assert_matches, mockall::Sequence, - solana_sdk::system_transaction, + solana_sdk::{system_transaction, transaction::SanitizedTransaction}, std::sync::Mutex, }; @@ -574,6 +574,6 @@ mod tests { ); let bank = BankWithScheduler::new(bank, Some(mocked_scheduler)); - bank.schedule_transaction_executions([(&tx0, &0)].into_iter()); + bank.schedule_transaction_executions([(&tx0.into(), &0)].into_iter()); } } diff --git a/runtime/src/prioritization_fee_cache.rs b/runtime/src/prioritization_fee_cache.rs index 9ff3c3d651be74..37800293db68e3 100644 --- a/runtime/src/prioritization_fee_cache.rs +++ b/runtime/src/prioritization_fee_cache.rs @@ -475,7 +475,7 @@ mod tests { fn sync_update<'a>( prioritization_fee_cache: &PrioritizationFeeCache, bank: Arc, - txs: impl Iterator + ExactSizeIterator, + txs: impl Iterator + ExactSizeIterator, ) { let expected_update_count = prioritization_fee_cache .metrics @@ -534,9 +534,9 @@ mod tests { // [2, a, c ] --> [2, 2, 5, 2 ] // let txs = vec![ - build_sanitized_transaction_for_test(5, &write_account_a, &write_account_b), - build_sanitized_transaction_for_test(9, &write_account_b, &write_account_c), - build_sanitized_transaction_for_test(2, &write_account_a, &write_account_c), + build_sanitized_transaction_for_test(5, &write_account_a, &write_account_b).into(), + build_sanitized_transaction_for_test(9, &write_account_b, &write_account_c).into(), + build_sanitized_transaction_for_test(2, &write_account_a, &write_account_c).into(), ]; let bank = Arc::new(Bank::default_for_tests()); @@ -651,12 +651,13 @@ mod tests { // Assert after add one transaction for slot 1 { let txs = vec![ - build_sanitized_transaction_for_test(2, &write_account_a, &write_account_b), + build_sanitized_transaction_for_test(2, &write_account_a, &write_account_b).into(), build_sanitized_transaction_for_test( 1, &Pubkey::new_unique(), &Pubkey::new_unique(), - ), + ) + .into(), ]; sync_update(&prioritization_fee_cache, bank1.clone(), txs.iter()); // before block is marked as completed @@ -714,12 +715,13 @@ mod tests { // Assert after add one transaction for slot 2 { let txs = vec![ - build_sanitized_transaction_for_test(4, &write_account_b, &write_account_c), + build_sanitized_transaction_for_test(4, &write_account_b, &write_account_c).into(), build_sanitized_transaction_for_test( 3, &Pubkey::new_unique(), &Pubkey::new_unique(), - ), + ) + .into(), ]; sync_update(&prioritization_fee_cache, bank2.clone(), txs.iter()); // before block is marked as completed @@ -788,12 +790,13 @@ mod tests { // Assert after add one transaction for slot 3 { let txs = vec![ - build_sanitized_transaction_for_test(6, &write_account_a, &write_account_c), + build_sanitized_transaction_for_test(6, &write_account_a, &write_account_c).into(), build_sanitized_transaction_for_test( 5, &Pubkey::new_unique(), &Pubkey::new_unique(), - ), + ) + .into(), ]; sync_update(&prioritization_fee_cache, bank3.clone(), txs.iter()); // before block is marked as completed @@ -883,12 +886,13 @@ mod tests { // Assert after add transactions for bank1 of slot 1 { let txs = vec![ - build_sanitized_transaction_for_test(2, &write_account_a, &write_account_b), + build_sanitized_transaction_for_test(2, &write_account_a, &write_account_b).into(), build_sanitized_transaction_for_test( 1, &Pubkey::new_unique(), &Pubkey::new_unique(), - ), + ) + .into(), ]; sync_update(&prioritization_fee_cache, bank1.clone(), txs.iter()); @@ -903,12 +907,13 @@ mod tests { // Assert after add transactions for bank2 of slot 1 { let txs = vec![ - build_sanitized_transaction_for_test(4, &write_account_b, &write_account_c), + build_sanitized_transaction_for_test(4, &write_account_b, &write_account_c).into(), build_sanitized_transaction_for_test( 3, &Pubkey::new_unique(), &Pubkey::new_unique(), - ), + ) + .into(), ]; sync_update(&prioritization_fee_cache, bank2.clone(), txs.iter()); diff --git a/runtime/src/transaction_batch.rs b/runtime/src/transaction_batch.rs index 935b8a34a91113..9ab4be21047682 100644 --- a/runtime/src/transaction_batch.rs +++ b/runtime/src/transaction_batch.rs @@ -100,7 +100,7 @@ mod tests { use { super::*, crate::genesis_utils::{create_genesis_config_with_leader, GenesisConfigInfo}, - solana_sdk::{signature::Keypair, system_transaction, transaction::TransactionError}, + solana_sdk::{signature::Keypair, system_transaction, transaction::{SanitizedTransaction, TransactionError}}, }; #[test] @@ -172,7 +172,7 @@ mod tests { ); } - fn setup(insert_conflicting_tx: bool) -> (Bank, Vec) { + fn setup(insert_conflicting_tx: bool) -> (Bank, Vec) { let dummy_leader_pubkey = solana_sdk::pubkey::new_rand(); let GenesisConfigInfo { genesis_config, @@ -187,16 +187,15 @@ mod tests { let mut txs = vec![SanitizedTransaction::from_transaction_for_tests( system_transaction::transfer(&mint_keypair, &pubkey, 1, genesis_config.hash()), - )]; + ).into()]; if insert_conflicting_tx { txs.push(SanitizedTransaction::from_transaction_for_tests( system_transaction::transfer(&mint_keypair, &pubkey2, 1, genesis_config.hash()), - )); + ).into()); } txs.push(SanitizedTransaction::from_transaction_for_tests( system_transaction::transfer(&keypair2, &pubkey2, 1, genesis_config.hash()), - )); - + ).into()); (bank, txs) } } diff --git a/svm/src/account_loader.rs b/svm/src/account_loader.rs index bf586649104510..a9ef7eb4db9d19 100644 --- a/svm/src/account_loader.rs +++ b/svm/src/account_loader.rs @@ -550,7 +550,7 @@ mod tests { fee_structure: &FeeStructure, ) -> Vec { feature_set.deactivate(&feature_set::disable_rent_fees_collection::id()); - let sanitized_tx = SanitizedTransaction::from_transaction_for_tests(tx); + let sanitized_tx = SanitizedTransaction::from_transaction_for_tests(tx).into(); let mut accounts_map = HashMap::new(); for (pubkey, account) in ka { accounts_map.insert(*pubkey, account.clone()); @@ -1026,7 +1026,7 @@ mod tests { tx: Transaction, account_overrides: Option<&AccountOverrides>, ) -> Vec { - let tx = SanitizedTransaction::from_transaction_for_tests(tx); + let tx = SanitizedTransaction::from_transaction_for_tests(tx).into(); let mut error_counters = TransactionErrorMetrics::default(); let mut accounts_map = HashMap::new(); diff --git a/svm/tests/account_loader.rs b/svm/tests/account_loader.rs new file mode 100644 index 00000000000000..f23f2def69f909 --- /dev/null +++ b/svm/tests/account_loader.rs @@ -0,0 +1,214 @@ +use { + crate::mock_bank::MockBankCallback, + solana_program_runtime::loaded_programs::LoadedProgramsForTxBatch, + solana_sdk::{ + account::{AccountSharedData, WritableAccount}, + fee::FeeStructure, + hash::Hash, + instruction::CompiledInstruction, + message::{LegacyMessage, Message, MessageHeader, SanitizedMessage}, + native_loader, + nonce_info::{NonceFull, NoncePartial}, + pubkey::Pubkey, + rent_collector::RENT_EXEMPT_RENT_EPOCH, + rent_debits::RentDebits, + signature::{Keypair, Signature, Signer}, + transaction::{SanitizedTransaction, TransactionError}, + }, + solana_svm::{ + account_loader::{load_accounts, LoadedTransaction, TransactionCheckResult}, + transaction_error_metrics::TransactionErrorMetrics, + }, + std::collections::HashMap, +}; + +mod mock_bank; + +#[test] +fn test_load_accounts_success() { + let key1 = Keypair::new(); + let key2 = Keypair::new(); + let key3 = Keypair::new(); + let key4 = Keypair::new(); + + let message = Message { + account_keys: vec![key2.pubkey(), key1.pubkey(), key4.pubkey()], + header: MessageHeader::default(), + instructions: vec![ + CompiledInstruction { + program_id_index: 1, + accounts: vec![0], + data: vec![], + }, + CompiledInstruction { + program_id_index: 1, + accounts: vec![2], + data: vec![], + }, + ], + recent_blockhash: Hash::default(), + }; + + let legacy = LegacyMessage::new(message); + let sanitized_message = SanitizedMessage::Legacy(legacy); + let mut mock_bank = MockBankCallback::default(); + let mut account_data = AccountSharedData::default(); + account_data.set_executable(true); + account_data.set_owner(key3.pubkey()); + mock_bank + .account_shared_data + .insert(key1.pubkey(), account_data); + + let mut account_data = AccountSharedData::default(); + account_data.set_lamports(200); + mock_bank + .account_shared_data + .insert(key2.pubkey(), account_data); + + let mut account_data = AccountSharedData::default(); + account_data.set_executable(true); + account_data.set_owner(native_loader::id()); + mock_bank + .account_shared_data + .insert(key3.pubkey(), account_data); + + let mut error_counter = TransactionErrorMetrics::default(); + let loaded_programs = LoadedProgramsForTxBatch::default(); + + let sanitized_transaction = SanitizedTransaction::new_for_tests( + sanitized_message, + vec![Signature::new_unique()], + false, + ); + let lock_results = + (Ok(()), Some(NoncePartial::default()), Some(20u64)) as TransactionCheckResult; + + let results = load_accounts( + &mock_bank, + &[sanitized_transaction.into()], + &[lock_results], + &mut error_counter, + &FeeStructure::default(), + None, + &HashMap::new(), + &loaded_programs, + ); + + let mut account_data = AccountSharedData::default(); + account_data.set_rent_epoch(RENT_EXEMPT_RENT_EPOCH); + + assert_eq!(results.len(), 1); + let (loaded_result, nonce) = results[0].clone(); + assert_eq!( + loaded_result.unwrap(), + LoadedTransaction { + accounts: vec![ + ( + key2.pubkey(), + mock_bank.account_shared_data[&key2.pubkey()].clone() + ), + ( + key1.pubkey(), + mock_bank.account_shared_data[&key1.pubkey()].clone() + ), + (key4.pubkey(), account_data), + ( + key3.pubkey(), + mock_bank.account_shared_data[&key3.pubkey()].clone() + ), + ], + program_indices: vec![vec![3, 1], vec![3, 1]], + rent: 0, + rent_debits: RentDebits::default() + } + ); + + assert_eq!( + nonce.unwrap(), + NonceFull::new( + Pubkey::from([0; 32]), + AccountSharedData::default(), + Some(mock_bank.account_shared_data[&key2.pubkey()].clone()) + ) + ); +} + +#[test] +fn test_load_accounts_error() { + let mock_bank = MockBankCallback::default(); + let message = Message { + account_keys: vec![Pubkey::new_from_array([0; 32])], + header: MessageHeader::default(), + instructions: vec![CompiledInstruction { + program_id_index: 0, + accounts: vec![], + data: vec![], + }], + recent_blockhash: Hash::default(), + }; + + let legacy = LegacyMessage::new(message); + let sanitized_message = SanitizedMessage::Legacy(legacy); + let sanitized_transaction = SanitizedTransaction::new_for_tests( + sanitized_message, + vec![Signature::new_unique()], + false, + ); + + let lock_results = (Ok(()), Some(NoncePartial::default()), None) as TransactionCheckResult; + let fee_structure = FeeStructure::default(); + + let result = load_accounts( + &mock_bank, + &[sanitized_transaction.clone().into()], + &[lock_results], + &mut TransactionErrorMetrics::default(), + &fee_structure, + None, + &HashMap::new(), + &LoadedProgramsForTxBatch::default(), + ); + + assert_eq!( + result, + vec![(Err(TransactionError::BlockhashNotFound), None)] + ); + + let lock_results = + (Ok(()), Some(NoncePartial::default()), Some(20u64)) as TransactionCheckResult; + + let result = load_accounts( + &mock_bank, + &[sanitized_transaction.clone().into()], + &[lock_results.clone()], + &mut TransactionErrorMetrics::default(), + &fee_structure, + None, + &HashMap::new(), + &LoadedProgramsForTxBatch::default(), + ); + + assert_eq!(result, vec![(Err(TransactionError::AccountNotFound), None)]); + + let lock_results = ( + Err(TransactionError::InvalidWritableAccount), + Some(NoncePartial::default()), + Some(20u64), + ) as TransactionCheckResult; + + let result = load_accounts( + &mock_bank, + &[sanitized_transaction.clone().into()], + &[lock_results], + &mut TransactionErrorMetrics::default(), + &fee_structure, + None, + &HashMap::new(), + &LoadedProgramsForTxBatch::default(), + ); + + assert_eq!( + result, + vec![(Err(TransactionError::InvalidWritableAccount), None)] + ); +} diff --git a/svm/tests/rent_state.rs b/svm/tests/rent_state.rs new file mode 100644 index 00000000000000..d958f6f2f12c61 --- /dev/null +++ b/svm/tests/rent_state.rs @@ -0,0 +1,90 @@ +#![cfg(test)] + +use { + solana_program_runtime::{ + compute_budget::ComputeBudget, compute_budget_processor, + loaded_programs::LoadedProgramsForTxBatch, + }, + solana_sdk::{ + account::{AccountSharedData, WritableAccount}, + fee::FeeStructure, + hash::Hash, + native_loader, + native_token::sol_to_lamports, + pubkey::Pubkey, + rent::Rent, + signature::{Keypair, Signer}, + system_transaction, + transaction::SanitizedTransaction, + transaction_context::TransactionContext, + }, + solana_svm::{ + account_loader::load_accounts, transaction_account_state_info::TransactionAccountStateInfo, + transaction_error_metrics::TransactionErrorMetrics, + }, + std::collections::HashMap, +}; + +mod mock_bank; + +#[test] +fn test_rent_state_list_len() { + let mint_keypair = Keypair::new(); + let mut bank = mock_bank::MockBankCallback::default(); + let recipient = Pubkey::new_unique(); + let last_block_hash = Hash::new_unique(); + + let mut system_data = AccountSharedData::default(); + system_data.set_executable(true); + system_data.set_owner(native_loader::id()); + bank.account_shared_data + .insert(Pubkey::new_from_array([0u8; 32]), system_data); + + let mut mint_data = AccountSharedData::default(); + mint_data.set_lamports(2); + bank.account_shared_data + .insert(mint_keypair.pubkey(), mint_data); + + bank.account_shared_data + .insert(recipient, AccountSharedData::default()); + + let tx = system_transaction::transfer( + &mint_keypair, + &recipient, + sol_to_lamports(1.), + last_block_hash, + ); + let num_accounts = tx.message().account_keys.len(); + let sanitized_tx = SanitizedTransaction::from_transaction_for_tests(tx); + let mut error_counters = TransactionErrorMetrics::default(); + let loaded_txs = load_accounts( + &bank, + &[sanitized_tx.clone().into()], + &[(Ok(()), None, Some(0))], + &mut error_counters, + &FeeStructure::default(), + None, + &HashMap::new(), + &LoadedProgramsForTxBatch::default(), + ); + + let compute_budget = ComputeBudget::new(u64::from( + compute_budget_processor::DEFAULT_INSTRUCTION_COMPUTE_UNIT_LIMIT, + )); + let transaction_context = TransactionContext::new( + loaded_txs[0].0.as_ref().unwrap().accounts.clone(), + Rent::default(), + compute_budget.max_invoke_stack_height, + compute_budget.max_instruction_trace_length, + ); + + assert_eq!( + TransactionAccountStateInfo::new( + &Rent::default(), + &transaction_context, + sanitized_tx.message() + ) + .len(), + num_accounts, + ); +} diff --git a/svm/tests/transaction_processor.rs b/svm/tests/transaction_processor.rs index 1704054246748d..ff8e543c4d5c5e 100644 --- a/svm/tests/transaction_processor.rs +++ b/svm/tests/transaction_processor.rs @@ -95,7 +95,7 @@ fn test_filter_executable_program_accounts() { let owners = &[program1_pubkey, program2_pubkey]; let programs = TransactionBatchProcessor::::filter_executable_program_accounts( &bank, - &[sanitized_tx1, sanitized_tx2], + &[sanitized_tx1.into(), sanitized_tx2.into()], &mut [(Ok(()), None, Some(0)), (Ok(()), None, Some(0))], owners, ); @@ -189,7 +189,7 @@ fn test_filter_executable_program_accounts_invalid_blockhash() { let mut lock_results = vec![(Ok(()), None, Some(0)), (Ok(()), None, None)]; let programs = TransactionBatchProcessor::::filter_executable_program_accounts( &bank, - &[sanitized_tx1, sanitized_tx2], + &[sanitized_tx1.into(), sanitized_tx2.into()], &mut lock_results, owners, ); diff --git a/unified-scheduler-pool/src/lib.rs b/unified-scheduler-pool/src/lib.rs index 6ee3460390c2a9..8821c28257d0f3 100644 --- a/unified-scheduler-pool/src/lib.rs +++ b/unified-scheduler-pool/src/lib.rs @@ -963,7 +963,8 @@ mod tests { &solana_sdk::pubkey::new_rand(), 2, genesis_config.hash(), - )); + )) + .into(); let bank = Bank::new_for_tests(&genesis_config); let bank = setup_dummy_fork_graph(bank); let ignored_prioritization_fee_cache = Arc::new(PrioritizationFeeCache::new(0u64)); @@ -1004,7 +1005,8 @@ mod tests { &solana_sdk::pubkey::new_rand(), 2, genesis_config.hash(), - )); + )) + .into(); assert_eq!(bank.transaction_count(), 0); scheduler.schedule_execution(&(bad_tx, 0)); // simulate the task-sending thread is stalled for some reason. @@ -1017,7 +1019,8 @@ mod tests { &solana_sdk::pubkey::new_rand(), 3, genesis_config.hash(), - )); + )) + .into(); // make sure this tx is really a good one to execute. assert_matches!( bank.simulate_transaction_unchecked(good_tx_after_bad_tx, false) @@ -1080,7 +1083,10 @@ mod tests { &self.2 } - fn schedule_execution(&self, &(transaction, index): &(&SanitizedTransaction, usize)) { + fn schedule_execution( + &self, + &(transaction, index): &(&ExtendedSanitizedTransaction, usize), + ) { let transaction_and_index = (transaction.clone(), index); let context = self.context().clone(); let pool = self.3.clone(); @@ -1208,7 +1214,7 @@ mod tests { assert_eq!(bank.transaction_count(), 0); // schedule but not immediately execute transaction - bank.schedule_transaction_executions([(&very_old_valid_tx, &0)].into_iter()); + bank.schedule_transaction_executions([(&very_old_valid_tx.into(), &0)].into_iter()); // this calls register_recent_blockhash internally bank.fill_bank_with_ticks_for_tests(); From aa6489bb686319087d5d0b0a406ac5d30c24b79e Mon Sep 17 00:00:00 2001 From: Lijun Wang <83639177+lijunwangs@users.noreply.github.com> Date: Mon, 11 Mar 2024 01:10:02 -0700 Subject: [PATCH 70/71] Fixed unit test failures and merge issues --- runtime/src/bank.rs | 2 +- runtime/src/transaction_batch.rs | 43 +++++-- svm/src/account_loader.rs | 10 +- svm/tests/account_loader.rs | 214 ------------------------------- svm/tests/rent_state.rs | 90 ------------- 5 files changed, 39 insertions(+), 320 deletions(-) delete mode 100644 svm/tests/account_loader.rs delete mode 100644 svm/tests/rent_state.rs diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index e557475aea56f4..ad8f242153eab7 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -4394,7 +4394,7 @@ impl Bank { pub fn unlock_accounts<'a>( &self, - txs_and_results: impl Iterator)>, + txs_and_results: impl Iterator)>, ) { self.rc.accounts.unlock_accounts(txs_and_results) } diff --git a/runtime/src/transaction_batch.rs b/runtime/src/transaction_batch.rs index 9ab4be21047682..5cbfbe5818ac33 100644 --- a/runtime/src/transaction_batch.rs +++ b/runtime/src/transaction_batch.rs @@ -100,7 +100,11 @@ mod tests { use { super::*, crate::genesis_utils::{create_genesis_config_with_leader, GenesisConfigInfo}, - solana_sdk::{signature::Keypair, system_transaction, transaction::{SanitizedTransaction, TransactionError}}, + solana_sdk::{ + signature::Keypair, + system_transaction, + transaction::{SanitizedTransaction, TransactionError}, + }, }; #[test] @@ -185,17 +189,36 @@ mod tests { let keypair2 = Keypair::new(); let pubkey2 = solana_sdk::pubkey::new_rand(); - let mut txs = vec![SanitizedTransaction::from_transaction_for_tests( - system_transaction::transfer(&mint_keypair, &pubkey, 1, genesis_config.hash()), - ).into()]; + let mut txs = + vec![ + SanitizedTransaction::from_transaction_for_tests(system_transaction::transfer( + &mint_keypair, + &pubkey, + 1, + genesis_config.hash(), + )) + .into(), + ]; if insert_conflicting_tx { - txs.push(SanitizedTransaction::from_transaction_for_tests( - system_transaction::transfer(&mint_keypair, &pubkey2, 1, genesis_config.hash()), - ).into()); + txs.push( + SanitizedTransaction::from_transaction_for_tests(system_transaction::transfer( + &mint_keypair, + &pubkey2, + 1, + genesis_config.hash(), + )) + .into(), + ); } - txs.push(SanitizedTransaction::from_transaction_for_tests( - system_transaction::transfer(&keypair2, &pubkey2, 1, genesis_config.hash()), - ).into()); + txs.push( + SanitizedTransaction::from_transaction_for_tests(system_transaction::transfer( + &keypair2, + &pubkey2, + 1, + genesis_config.hash(), + )) + .into(), + ); (bank, txs) } } diff --git a/svm/src/account_loader.rs b/svm/src/account_loader.rs index a9ef7eb4db9d19..0e266d30f10d13 100644 --- a/svm/src/account_loader.rs +++ b/svm/src/account_loader.rs @@ -2069,7 +2069,7 @@ mod tests { let mut error_counters = TransactionErrorMetrics::default(); let loaded_txs = load_accounts( &bank, - &[sanitized_tx.clone()], + &[sanitized_tx.clone().into()], &[(Ok(()), None, Some(0))], &mut error_counters, &FeeStructure::default(), @@ -2154,7 +2154,7 @@ mod tests { let results = load_accounts( &mock_bank, - &[sanitized_transaction], + &[sanitized_transaction.into()], &[lock_results], &mut error_counter, &FeeStructure::default(), @@ -2229,7 +2229,7 @@ mod tests { let result = load_accounts( &mock_bank, - &[sanitized_transaction.clone()], + &[sanitized_transaction.clone().into()], &[lock_results], &mut TransactionErrorMetrics::default(), &fee_structure, @@ -2248,7 +2248,7 @@ mod tests { let result = load_accounts( &mock_bank, - &[sanitized_transaction.clone()], + &[sanitized_transaction.clone().into()], &[lock_results.clone()], &mut TransactionErrorMetrics::default(), &fee_structure, @@ -2267,7 +2267,7 @@ mod tests { let result = load_accounts( &mock_bank, - &[sanitized_transaction.clone()], + &[sanitized_transaction.clone().into()], &[lock_results], &mut TransactionErrorMetrics::default(), &fee_structure, diff --git a/svm/tests/account_loader.rs b/svm/tests/account_loader.rs deleted file mode 100644 index f23f2def69f909..00000000000000 --- a/svm/tests/account_loader.rs +++ /dev/null @@ -1,214 +0,0 @@ -use { - crate::mock_bank::MockBankCallback, - solana_program_runtime::loaded_programs::LoadedProgramsForTxBatch, - solana_sdk::{ - account::{AccountSharedData, WritableAccount}, - fee::FeeStructure, - hash::Hash, - instruction::CompiledInstruction, - message::{LegacyMessage, Message, MessageHeader, SanitizedMessage}, - native_loader, - nonce_info::{NonceFull, NoncePartial}, - pubkey::Pubkey, - rent_collector::RENT_EXEMPT_RENT_EPOCH, - rent_debits::RentDebits, - signature::{Keypair, Signature, Signer}, - transaction::{SanitizedTransaction, TransactionError}, - }, - solana_svm::{ - account_loader::{load_accounts, LoadedTransaction, TransactionCheckResult}, - transaction_error_metrics::TransactionErrorMetrics, - }, - std::collections::HashMap, -}; - -mod mock_bank; - -#[test] -fn test_load_accounts_success() { - let key1 = Keypair::new(); - let key2 = Keypair::new(); - let key3 = Keypair::new(); - let key4 = Keypair::new(); - - let message = Message { - account_keys: vec![key2.pubkey(), key1.pubkey(), key4.pubkey()], - header: MessageHeader::default(), - instructions: vec![ - CompiledInstruction { - program_id_index: 1, - accounts: vec![0], - data: vec![], - }, - CompiledInstruction { - program_id_index: 1, - accounts: vec![2], - data: vec![], - }, - ], - recent_blockhash: Hash::default(), - }; - - let legacy = LegacyMessage::new(message); - let sanitized_message = SanitizedMessage::Legacy(legacy); - let mut mock_bank = MockBankCallback::default(); - let mut account_data = AccountSharedData::default(); - account_data.set_executable(true); - account_data.set_owner(key3.pubkey()); - mock_bank - .account_shared_data - .insert(key1.pubkey(), account_data); - - let mut account_data = AccountSharedData::default(); - account_data.set_lamports(200); - mock_bank - .account_shared_data - .insert(key2.pubkey(), account_data); - - let mut account_data = AccountSharedData::default(); - account_data.set_executable(true); - account_data.set_owner(native_loader::id()); - mock_bank - .account_shared_data - .insert(key3.pubkey(), account_data); - - let mut error_counter = TransactionErrorMetrics::default(); - let loaded_programs = LoadedProgramsForTxBatch::default(); - - let sanitized_transaction = SanitizedTransaction::new_for_tests( - sanitized_message, - vec![Signature::new_unique()], - false, - ); - let lock_results = - (Ok(()), Some(NoncePartial::default()), Some(20u64)) as TransactionCheckResult; - - let results = load_accounts( - &mock_bank, - &[sanitized_transaction.into()], - &[lock_results], - &mut error_counter, - &FeeStructure::default(), - None, - &HashMap::new(), - &loaded_programs, - ); - - let mut account_data = AccountSharedData::default(); - account_data.set_rent_epoch(RENT_EXEMPT_RENT_EPOCH); - - assert_eq!(results.len(), 1); - let (loaded_result, nonce) = results[0].clone(); - assert_eq!( - loaded_result.unwrap(), - LoadedTransaction { - accounts: vec![ - ( - key2.pubkey(), - mock_bank.account_shared_data[&key2.pubkey()].clone() - ), - ( - key1.pubkey(), - mock_bank.account_shared_data[&key1.pubkey()].clone() - ), - (key4.pubkey(), account_data), - ( - key3.pubkey(), - mock_bank.account_shared_data[&key3.pubkey()].clone() - ), - ], - program_indices: vec![vec![3, 1], vec![3, 1]], - rent: 0, - rent_debits: RentDebits::default() - } - ); - - assert_eq!( - nonce.unwrap(), - NonceFull::new( - Pubkey::from([0; 32]), - AccountSharedData::default(), - Some(mock_bank.account_shared_data[&key2.pubkey()].clone()) - ) - ); -} - -#[test] -fn test_load_accounts_error() { - let mock_bank = MockBankCallback::default(); - let message = Message { - account_keys: vec![Pubkey::new_from_array([0; 32])], - header: MessageHeader::default(), - instructions: vec![CompiledInstruction { - program_id_index: 0, - accounts: vec![], - data: vec![], - }], - recent_blockhash: Hash::default(), - }; - - let legacy = LegacyMessage::new(message); - let sanitized_message = SanitizedMessage::Legacy(legacy); - let sanitized_transaction = SanitizedTransaction::new_for_tests( - sanitized_message, - vec![Signature::new_unique()], - false, - ); - - let lock_results = (Ok(()), Some(NoncePartial::default()), None) as TransactionCheckResult; - let fee_structure = FeeStructure::default(); - - let result = load_accounts( - &mock_bank, - &[sanitized_transaction.clone().into()], - &[lock_results], - &mut TransactionErrorMetrics::default(), - &fee_structure, - None, - &HashMap::new(), - &LoadedProgramsForTxBatch::default(), - ); - - assert_eq!( - result, - vec![(Err(TransactionError::BlockhashNotFound), None)] - ); - - let lock_results = - (Ok(()), Some(NoncePartial::default()), Some(20u64)) as TransactionCheckResult; - - let result = load_accounts( - &mock_bank, - &[sanitized_transaction.clone().into()], - &[lock_results.clone()], - &mut TransactionErrorMetrics::default(), - &fee_structure, - None, - &HashMap::new(), - &LoadedProgramsForTxBatch::default(), - ); - - assert_eq!(result, vec![(Err(TransactionError::AccountNotFound), None)]); - - let lock_results = ( - Err(TransactionError::InvalidWritableAccount), - Some(NoncePartial::default()), - Some(20u64), - ) as TransactionCheckResult; - - let result = load_accounts( - &mock_bank, - &[sanitized_transaction.clone().into()], - &[lock_results], - &mut TransactionErrorMetrics::default(), - &fee_structure, - None, - &HashMap::new(), - &LoadedProgramsForTxBatch::default(), - ); - - assert_eq!( - result, - vec![(Err(TransactionError::InvalidWritableAccount), None)] - ); -} diff --git a/svm/tests/rent_state.rs b/svm/tests/rent_state.rs deleted file mode 100644 index d958f6f2f12c61..00000000000000 --- a/svm/tests/rent_state.rs +++ /dev/null @@ -1,90 +0,0 @@ -#![cfg(test)] - -use { - solana_program_runtime::{ - compute_budget::ComputeBudget, compute_budget_processor, - loaded_programs::LoadedProgramsForTxBatch, - }, - solana_sdk::{ - account::{AccountSharedData, WritableAccount}, - fee::FeeStructure, - hash::Hash, - native_loader, - native_token::sol_to_lamports, - pubkey::Pubkey, - rent::Rent, - signature::{Keypair, Signer}, - system_transaction, - transaction::SanitizedTransaction, - transaction_context::TransactionContext, - }, - solana_svm::{ - account_loader::load_accounts, transaction_account_state_info::TransactionAccountStateInfo, - transaction_error_metrics::TransactionErrorMetrics, - }, - std::collections::HashMap, -}; - -mod mock_bank; - -#[test] -fn test_rent_state_list_len() { - let mint_keypair = Keypair::new(); - let mut bank = mock_bank::MockBankCallback::default(); - let recipient = Pubkey::new_unique(); - let last_block_hash = Hash::new_unique(); - - let mut system_data = AccountSharedData::default(); - system_data.set_executable(true); - system_data.set_owner(native_loader::id()); - bank.account_shared_data - .insert(Pubkey::new_from_array([0u8; 32]), system_data); - - let mut mint_data = AccountSharedData::default(); - mint_data.set_lamports(2); - bank.account_shared_data - .insert(mint_keypair.pubkey(), mint_data); - - bank.account_shared_data - .insert(recipient, AccountSharedData::default()); - - let tx = system_transaction::transfer( - &mint_keypair, - &recipient, - sol_to_lamports(1.), - last_block_hash, - ); - let num_accounts = tx.message().account_keys.len(); - let sanitized_tx = SanitizedTransaction::from_transaction_for_tests(tx); - let mut error_counters = TransactionErrorMetrics::default(); - let loaded_txs = load_accounts( - &bank, - &[sanitized_tx.clone().into()], - &[(Ok(()), None, Some(0))], - &mut error_counters, - &FeeStructure::default(), - None, - &HashMap::new(), - &LoadedProgramsForTxBatch::default(), - ); - - let compute_budget = ComputeBudget::new(u64::from( - compute_budget_processor::DEFAULT_INSTRUCTION_COMPUTE_UNIT_LIMIT, - )); - let transaction_context = TransactionContext::new( - loaded_txs[0].0.as_ref().unwrap().accounts.clone(), - Rent::default(), - compute_budget.max_invoke_stack_height, - compute_budget.max_instruction_trace_length, - ); - - assert_eq!( - TransactionAccountStateInfo::new( - &Rent::default(), - &transaction_context, - sanitized_tx.message() - ) - .len(), - num_accounts, - ); -} From 3c6e0b3aba1fe6769387ea997e334d8a8baabee8 Mon Sep 17 00:00:00 2001 From: Lijun Wang <83639177+lijunwangs@users.noreply.github.com> Date: Mon, 11 Mar 2024 22:10:56 -0700 Subject: [PATCH 71/71] Use Arc, metrics: Arc, - perf_track_metrics: histogram::Histogram, + perf_track_metrics: Arc>, } #[allow(dead_code)] @@ -52,7 +52,7 @@ impl ConsumeWorker { consumed_sender, leader_bank_notifier, metrics: Arc::new(ConsumeWorkerMetrics::new(id)), - perf_track_metrics: histogram::Histogram::default(), + perf_track_metrics: Arc::new(Mutex::new(histogram::Histogram::default())), } } @@ -98,7 +98,7 @@ impl ConsumeWorker { bank, &work.transactions, &work.max_age_slots, - Some(&mut self.perf_track_metrics), + Some(&mut self.perf_track_metrics.lock().unwrap()), ); self.metrics.update_for_consume(&output);