Sync fork #6
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchstat | |
| on: | |
| pull_request: | |
| branches: | |
| - master | |
| - "auto-cadence-upgrade/**" | |
| - "feature/**" | |
| - "v[0-9]+.[0-9]+" | |
| paths: | |
| - ".github/workflows/bench.yml" | |
| - "fvm/**" | |
| - "engine/execution/**" | |
| - "go.sum" | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} | |
| cancel-in-progress: true | |
| jobs: | |
| benchstat: | |
| name: Performance regression check | |
| runs-on: ubuntu-latest | |
| # Check if the event is not triggered by a fork | |
| # peter-evans/find-comment@v1 does not work on forks. | |
| # see https://github.com/peter-evans/create-pull-request/blob/main/docs/concepts-guidelines.md#restrictions-on-repository-forks for details. | |
| # Ideally we would like to still run the benchmark on forks, but we can't do that with the current setup. | |
| if: github.event.pull_request.head.repo.full_name == github.repository | |
| continue-on-error: true | |
| steps: | |
| - name: Set benchmark repetitions | |
| # reducing repetition will speed up execution, | |
| # but will be more inaccurate at detecting change | |
| run: echo "::set-output name=benchmark_repetitions::7" | |
| id: settings | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup go | |
| uses: actions/setup-go@v3 | |
| with: | |
| go-version: "1.20" | |
| cache: true | |
| - name: Build relic | |
| run: make crypto_setup_gopath | |
| - name: Run benchmark on current branch | |
| run: | | |
| (for i in {1..${{ steps.settings.outputs.benchmark_repetitions }}}; do go test ./fvm ./engine/execution/computation --bench . --tags relic -shuffle=on --benchmem --run ^$; done) | tee new.txt | |
| - name: Checkout base branch | |
| run: git checkout ${{ github.event.pull_request.base.sha }} | |
| - name: Run benchmark on base branch | |
| run: | | |
| (for i in {1..${{ steps.settings.outputs.benchmark_repetitions }}}; do go test ./fvm ./engine/execution/computation --bench . --tags relic -shuffle=on --benchmem --run ^$; done) | tee old.txt | |
| # see https://trstringer.com/github-actions-multiline-strings/ to see why this part is complex | |
| - name: Use benchstat for comparison | |
| run: | | |
| export PATH=$PATH:$(go env GOPATH)/bin | |
| go install golang.org/x/perf/cmd/benchstat@91a04616dc65ba76dbe9e5cf746b923b1402d303 | |
| echo "BENCHSTAT<<EOF" >> $GITHUB_ENV | |
| echo "$(benchstat -html -sort delta old.txt new.txt | sed '/<title/,/<\/style>/d' | sed 's/<!doctype html>//g')" >> $GITHUB_ENV | |
| echo "EOF" >> $GITHUB_ENV | |
| - name: Find existing comment on PR | |
| uses: peter-evans/find-comment@v1 | |
| id: fc | |
| with: | |
| issue-number: ${{ github.event.pull_request.number }} | |
| comment-author: "github-actions[bot]" | |
| body-includes: "## FVM [Benchstat](https://pkg.go.dev/golang.org/x/perf/cmd/benchstat) comparison" | |
| - name: Create or update comment | |
| uses: peter-evans/create-or-update-comment@v1 | |
| with: | |
| comment-id: ${{ steps.fc.outputs.comment-id }} | |
| issue-number: ${{ github.event.pull_request.number }} | |
| body: | | |
| ## FVM [Benchstat](https://pkg.go.dev/golang.org/x/perf/cmd/benchstat) comparison | |
| This branch with compared with the base branch ${{ github.event.pull_request.base.label }} commit ${{ github.event.pull_request.base.sha }} | |
| The command `(for i in {1..${{ steps.settings.outputs.benchmark_repetitions }}}; do go test ./fvm ./engine/execution/computation --bench . --tags relic -shuffle=on --benchmem --run ^$; done)` was used. | |
| <details> | |
| <summary>Collapsed results for better readability</summary> | |
| <p> | |
| ${{ env.BENCHSTAT }} | |
| </p> | |
| </details> | |
| edit-mode: replace |