diff --git a/.github/workflows/paton-comment.yml b/.github/workflows/paton-comment.yml new file mode 100644 index 0000000000..bc84bbbce1 --- /dev/null +++ b/.github/workflows/paton-comment.yml @@ -0,0 +1,38 @@ +name: "paton comment" +on: + workflow_run: + workflows: ["paton benchmark"] + types: + - completed + +jobs: + paton-comment: + runs-on: ubuntu-latest + if: > + github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.conclusion == 'success' + steps: + - id: download-artifact + uses: dawidd6/action-download-artifact@v2 + with: paton.yml + name: alerting_benchmarks.json + path: /tmp/alerting_benchmarks.json + - name: Run benchmark + run: if [ ! -s /tmp/alerting_benchmarks.json ]; then rm /tmp/alerting_benchmarks.json + shell: bash + - name: Check files + id: check_files + uses: andstor/file-existence-action@v1 + with: + files: "/tmp/alerting_benchmarks.json" + - uses: peter-evans/create-or-update-comment@v2 + with: + issue-number: 5 # TODO Provide dynamically + body-file: /tmp/alerting_benchmarks.json + + # - name: Slack Notification + # env: + # SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + # run: | + # curl -X POST --data-urlencode "payload={\"text\": \"Benchmark workflow failed. ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} \"}" $SLACK_WEBHOOK + # if: ${{ failure() && (contains(github.ref_name, 'rel/nightly') || contains(github.ref_name, 'rel/beta') || contains(github.ref_name, 'rel/stable') || contains(github.ref_name, 'master')) }} diff --git a/.github/workflows/paton.yml b/.github/workflows/paton.yml new file mode 100644 index 0000000000..c7e25f5897 --- /dev/null +++ b/.github/workflows/paton.yml @@ -0,0 +1,58 @@ +name: "paton benchmark" +on: + push: + branches: + - master + pull_request: +# For demonstration purposes, allow paton to run on all PRs. +# branches: +# - master +permissions: + contents: write + deployments: write +jobs: + paton: + name: paton + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: actions/setup-go@v3 + with: + go-version-file: 'go.mod' + cache: true + - name: Restore libsodium from cache + id: cache-libsodium + uses: actions/cache@v3 + with: + path: crypto/libs + key: libsodium-fork-v1-${{ runner.os }}-${{ hashFiles('crypto/libsodium-fork/**') }} + - run: go install golang.org/x/perf/cmd/benchstat@latest + - run: sudo apt-get update + - run: sudo apt-get -y -q install jq python3-pip + - run: pip3 install jc # Use pip to install jc because aptitude is stale (https://repology.org/project/jc/versions). + - run: ./scripts/configure_dev.sh + shell: bash + - run: ./scripts/buildtools/install_buildtools.sh + shell: bash + - run: ./scripts/travis/before_build.sh # Installs libsodium. + shell: bash + - name: Run benchmark + run: ./scripts/paton.sh --alert-threshold-pct 10 --test-cmd "-run XXX -benchtime 5s -bench BenchmarkControl ./data/transactions/logic" + shell: bash + - uses: actions/upload-artifact@v3 + with: + name: alerting_benchmarks.json + path: /tmp/alerting_benchmarks.json +# - name: Check files +# id: check_files +# uses: andstor/file-existence-action@v1 +# with: +# files: "/tmp/alerting_benchmarks.json" + # - name: Slack Notification + # env: + # SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + # run: | + # curl -X POST --data-urlencode "payload={\"text\": \"Benchmark workflow failed. ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} \"}" $SLACK_WEBHOOK + # if: ${{ failure() && (contains(github.ref_name, 'rel/nightly') || contains(github.ref_name, 'rel/beta') || contains(github.ref_name, 'rel/stable') || contains(github.ref_name, 'master')) }} diff --git a/data/transactions/logic/eval_test.go b/data/transactions/logic/eval_test.go index 8d23b20416..34581a03f8 100644 --- a/data/transactions/logic/eval_test.go +++ b/data/transactions/logic/eval_test.go @@ -25,6 +25,7 @@ import ( "strconv" "strings" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -3727,6 +3728,28 @@ main: } } +func BenchmarkControl1(b *testing.B) { + var arr []int + for i := 0; i < b.N; i++ { + time.Sleep(225 * time.Millisecond) + arr = make([]int, 50) + } + if b.N < 0 { + fmt.Println(arr) + } +} + +func BenchmarkControl2(b *testing.B) { + var arr []int + for i := 0; i < b.N; i++ { + time.Sleep(45 * time.Millisecond) + arr = make([]int, 25) + } + if b.N < 0 { + fmt.Println(arr) + } +} + func BenchmarkByteLogic(b *testing.B) { benches := [][]string{ {"b&", "", "byte 0x012345678901feab; byte 0x01ffffffffffffff; b&; pop", "int 1"}, diff --git a/scripts/paton.sh b/scripts/paton.sh new file mode 100755 index 0000000000..cf23effab8 --- /dev/null +++ b/scripts/paton.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash + +set -euxf -o pipefail + +# Primarily intended for continuous benchmarking, paton.sh compares Go +# benchmarks between 2 git commits and outputs the comparison in a format +# compatible with https://github.com/benchmark-action/github-action-benchmark. +# +# paton.sh is inspired by https://github.com/knqyf263/cob. cob minimizes +# benchmarking variance by running provided benchmarks against 2 commits in 1 +# invocation rather than comparing against a previously stored result. +# Comparing against a previously stored result requires a stable benchmark +# environment across time. Particularly in fully managed CI environments, a +# stable benchmark environment cannot be guaranteed. +# +# paton.sh's namesake is https://en.wikipedia.org/wiki/Paton_Bridge. +# +# paton.sh requires these dependencies. No attempt is made to install prerequisites: +# * https://pkg.go.dev/golang.org/x/perf/cmd/benchstat +# * https://github.com/stedolan/jq +# * https://github.com/kellyjonbrazil/jc + +if [[ $# -lt 1 ]]; then + echo "Must provide required flags" + exit 1 +fi + +# Argument parsing crafted with help from https://stackoverflow.com/a/14203146. +while [[ $# -gt 0 ]]; do + case $1 in + -c|--test-cmd) + GO_TEST_CMD=("$2") + shift # past argument + shift # past value + ;; + -a|--alert-threshold-pct) + ALERT_THRESHOLD_PCT="$2" + shift # past argument + shift # past value + ;; + *) + echo "Unknown flag" + exit 1 + ;; + esac +done + +BASE="HEAD~1" +COMPARE="HEAD" + +COMPARE_COMMIT=$(git rev-parse "$COMPARE") + +git -c advice.detachedHead=false checkout "$BASE" +go test ${GO_TEST_CMD[*]} | tee /tmp/base.txt + +git -c advice.detachedHead=false checkout "$COMPARE_COMMIT" +go test ${GO_TEST_CMD[*]} | tee /tmp/compare.txt + +benchstat -delta-test none /tmp/base.txt /tmp/compare.txt | tee /tmp/benchstat.txt + +cat /tmp/benchstat.txt | + awk '/old time\/op/{f=1} /^$/{f=0} f' | + jc -p --asciitable | + # Remove symbols (+, %) preventing conversion to JavaScript number. + sed '/delta/s/+//g' | + sed '/delta/s/%//g' | + tee /tmp/benchstat_time.json + +cat /tmp/benchstat_time.json | + jq '.[] | { + name: .name, + old_time_op: .old_time_op, + new_time_op: .new_time_op, + delta: .delta | tonumber + }' | + jq -s > /tmp/benchstat_time_jq.json + +cat /tmp/benchstat_time_jq.json | + jq ".[] | select(.delta >= ${ALERT_THRESHOLD_PCT})" | + tee /tmp/alerting_benchmarks.json + +#if [ ! -s /tmp/alerting_benchmarks.json ]; then +# rm /tmp/alerting_benchmarks.json +#fi