Add paton for continuous benchmarking

.github/workflows/paton-comment.yml

@@ -0,0 +1,38 @@
+name: "paton comment"
+on:
+  workflow_run:
+    workflows: ["paton benchmark"]
+    types:
+      - completed
+
+jobs:
+  paton-comment:
+    runs-on: ubuntu-latest
+    if: >
+      github.event.workflow_run.event == 'pull_request' &&
+      github.event.workflow_run.conclusion == 'success'
+    steps:
+      - id: download-artifact
+        uses: dawidd6/action-download-artifact@v2
+        with: paton.yml
+        name: alerting_benchmarks.json
+        path: /tmp/alerting_benchmarks.json
+      - name: Run benchmark
+        run: if [ ! -s /tmp/alerting_benchmarks.json ]; then rm /tmp/alerting_benchmarks.json
+        shell: bash
+      - name: Check files
+        id: check_files
+        uses: andstor/file-existence-action@v1
+        with:
+          files: "/tmp/alerting_benchmarks.json"
+      - uses: peter-evans/create-or-update-comment@v2
+        with:
+          issue-number: 5 # TODO Provide dynamically
+          body-file: /tmp/alerting_benchmarks.json
+
+ #     - name: Slack Notification
+ #       env:
+ #         SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+ #       run: |
+ #         curl -X POST --data-urlencode "payload={\"text\": \"Benchmark workflow failed. ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} \"}" $SLACK_WEBHOOK
+  #      if: ${{ failure() && (contains(github.ref_name, 'rel/nightly') || contains(github.ref_name, 'rel/beta') || contains(github.ref_name, 'rel/stable') || contains(github.ref_name, 'master')) }}

.github/workflows/paton.yml

@@ -0,0 +1,58 @@
+name: "paton benchmark"
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+# For demonstration purposes, allow paton to run on all PRs.
+#    branches:
+#      - master
+permissions:
+  contents: write
+  deployments: write
+jobs:
+  paton:
+    name: paton
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-go@v3
+        with:
+          go-version-file: 'go.mod'
+          cache: true
+      - name: Restore libsodium from cache
+        id: cache-libsodium
+        uses: actions/cache@v3
+        with:
+          path: crypto/libs
+          key: libsodium-fork-v1-${{ runner.os }}-${{ hashFiles('crypto/libsodium-fork/**') }}
+      - run: go install golang.org/x/perf/cmd/benchstat@latest
+      - run: sudo apt-get update
+      - run: sudo apt-get -y -q install jq python3-pip
+      - run: pip3 install jc # Use pip to install jc because aptitude is stale (https://repology.org/project/jc/versions).
+      - run: ./scripts/configure_dev.sh
+        shell: bash
+      - run: ./scripts/buildtools/install_buildtools.sh
+        shell: bash
+      - run: ./scripts/travis/before_build.sh # Installs libsodium.
+        shell: bash
+      - name: Run benchmark
+        run: ./scripts/paton.sh --alert-threshold-pct 10 --test-cmd "-run XXX -benchtime 5s -bench BenchmarkControl ./data/transactions/logic"
+        shell: bash
+      - uses: actions/upload-artifact@v3
+        with:
+          name: alerting_benchmarks.json
+          path: /tmp/alerting_benchmarks.json
+#      - name: Check files
+#        id: check_files
+#        uses: andstor/file-existence-action@v1
+#        with:
+#          files: "/tmp/alerting_benchmarks.json"
+ #     - name: Slack Notification
+ #       env:
+ #         SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+ #       run: |
+ #         curl -X POST --data-urlencode "payload={\"text\": \"Benchmark workflow failed. ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} \"}" $SLACK_WEBHOOK
+  #      if: ${{ failure() && (contains(github.ref_name, 'rel/nightly') || contains(github.ref_name, 'rel/beta') || contains(github.ref_name, 'rel/stable') || contains(github.ref_name, 'master')) }}

data/transactions/logic/eval_test.go

@@ -25,6 +25,7 @@ import (
 	"strconv"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -3727,6 +3728,28 @@ main:
 	}
 }
 
+func BenchmarkControl1(b *testing.B) {
+	var arr []int
+	for i := 0; i < b.N; i++ {
+		time.Sleep(225 * time.Millisecond)
+		arr = make([]int, 50)
+	}
+	if b.N < 0 {
+		fmt.Println(arr)
+	}
+}
+
+func BenchmarkControl2(b *testing.B) {
+	var arr []int
+	for i := 0; i < b.N; i++ {
+		time.Sleep(45 * time.Millisecond)
+		arr = make([]int, 25)
+	}
+	if b.N < 0 {
+		fmt.Println(arr)
+	}
+}
+
 func BenchmarkByteLogic(b *testing.B) {
 	benches := [][]string{
 		{"b&", "", "byte 0x012345678901feab; byte 0x01ffffffffffffff; b&; pop", "int 1"},

scripts/paton.sh

@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+
+set -euxf -o pipefail
+
+# Primarily intended for continuous benchmarking, paton.sh compares Go
+# benchmarks between 2 git commits and outputs the comparison in a format
+# compatible with https://github.com/benchmark-action/github-action-benchmark.
+#
+# paton.sh is inspired by https://github.com/knqyf263/cob.  cob minimizes
+# benchmarking variance by running provided benchmarks against 2 commits in 1
+# invocation rather than comparing against a previously stored result.
+# Comparing against a previously stored result requires a stable benchmark
+# environment across time.  Particularly in fully managed CI environments, a
+# stable benchmark environment cannot be guaranteed.
+#
+# paton.sh's namesake is https://en.wikipedia.org/wiki/Paton_Bridge.
+#
+# paton.sh requires these dependencies.  No attempt is made to install prerequisites:
+# * https://pkg.go.dev/golang.org/x/perf/cmd/benchstat
+# * https://github.com/stedolan/jq
+# * https://github.com/kellyjonbrazil/jc
+
+if [[ $# -lt 1 ]]; then
+  echo "Must provide required flags"
+  exit 1
+fi
+
+# Argument parsing crafted with help from https://stackoverflow.com/a/14203146.
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -c|--test-cmd)
+      GO_TEST_CMD=("$2")
+      shift # past argument
+      shift # past value
+      ;;
+    -a|--alert-threshold-pct)
+      ALERT_THRESHOLD_PCT="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    *)
+      echo "Unknown flag"
+      exit 1
+      ;;
+  esac
+done
+
+BASE="HEAD~1"
+COMPARE="HEAD"
+
+COMPARE_COMMIT=$(git rev-parse "$COMPARE")
+
+git -c advice.detachedHead=false checkout "$BASE"
+go test ${GO_TEST_CMD[*]} | tee /tmp/base.txt
+
+git -c advice.detachedHead=false checkout "$COMPARE_COMMIT"
+go test ${GO_TEST_CMD[*]} | tee /tmp/compare.txt
+
+benchstat -delta-test none /tmp/base.txt /tmp/compare.txt | tee /tmp/benchstat.txt
+
+cat /tmp/benchstat.txt |
+  awk '/old time\/op/{f=1} /^$/{f=0} f' |
+  jc -p --asciitable |
+  # Remove symbols (+, %) preventing conversion to JavaScript number.
+  sed '/delta/s/+//g' |
+  sed '/delta/s/%//g' |
+  tee /tmp/benchstat_time.json
+
+cat /tmp/benchstat_time.json |
+  jq '.[] | {
+    name: .name,
+    old_time_op: .old_time_op,
+    new_time_op: .new_time_op,
+    delta: .delta | tonumber
+    }' |
+  jq -s > /tmp/benchstat_time_jq.json
+
+cat /tmp/benchstat_time_jq.json |
+  jq ".[] | select(.delta >= ${ALERT_THRESHOLD_PCT})" |
+  tee /tmp/alerting_benchmarks.json
+
+#if [ ! -s /tmp/alerting_benchmarks.json ]; then
+#  rm /tmp/alerting_benchmarks.json
+#fi