forked from bytecodealliance/wasmtime
-
Notifications
You must be signed in to change notification settings - Fork 0
148 lines (128 loc) · 5.71 KB
/
benchfx.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
name: "Check for performance regressions"
on:
pull_request:
push:
branches:
- main
# Configuration
env:
# benchfx main as of September 6, 2024
benchfx_commit: db338e74ff9bc6c4c22b54c169c05dd592429cf0
wasmtime_features: unsafe_wasmfx_stacks
benchmark_filters: "--filter=**/*_wasmfx --filter=**/*_asyncify --filter=micro/**/*"
# Maximum allowed performance regression of any benchmark in percent.
max_allowed_regression: 7
jobs:
bench:
# TODO(dhil): Disable this workflow until we have brought our
# toolchains up-to-date with the new binary format.
if: false
name: Setup and run benchmarks
runs-on: ubuntu-latest
steps:
- name: Checkout wasmtime
uses: actions/checkout@v4
with:
submodules: true
path: wasmtime
fetch-depth: 0
- name: Checkout benchfx
uses: actions/checkout@v4
with:
repository: 'wasmfx/benchfx'
ref: ${{ env.benchfx_commit }}
submodules: true
path: benchfx
fetch-depth: 0
- name: Install hyperfine
run: cargo install hyperfine
- name: Install other deps
run: |
sudo apt-get update
sudo apt-get install cmake ocaml dune menhir libmenhir-ocaml-dev
- name: Show some context info
working-directory: ./wasmtime
run: |
echo "cargo --version: $(cargo --version)"
echo "rustc --version: $(rustc --version)"
echo "GITHUB_SHA:"
echo "$GITHUB_SHA"
git show --no-patch --oneline $GITHUB_SHA
echo "Will compare against following commit:"
echo "GITHUB_SHA^1:"
git rev-parse "${GITHUB_SHA}^1"
git show --no-patch --oneline "${GITHUB_SHA}^1"
echo "parent_github_sha=$(git rev-parse ${GITHUB_SHA}^1)" >> $GITHUB_ENV
- name: Initialize benchfx
working-directory: ./benchfx
shell: bash
run: |
./harness.py --verbose setup \
--wasmtime-create-worktree-from-development-repo=$GITHUB_WORKSPACE/wasmtime
# The following is only necessary because we are about to restore the
# build directory inside the binaryen repo inside benchfx from the
# cache. But the binaryen repo is currently a bare repo and doesn't have
# the .gitignore file with an entry for "build", yet. But the harness
# refuses to work on anything other than a clean repo!
# Instead of checking out any revision to get just a proper .gitignore
# file, we check out the revision that the harness actually wants to
# use. This way, the harness will not "modify" any of the binaryen
# source files by switching to a different binaryen revision later. This
# helps with reusing the cached binaryen build artifacts.
BINARYEN_REVISION="$(./harness.py print-config | jq -r .BINARYEN_REVISION)"
pushd tools/external/binaryen
git switch --detach --recurse-submodules "$BINARYEN_REVISION"
popd
# We include the SHA of the base/parent commit in the cache key. This
# ensures that new cache entries are generated frequently instead of reusing
# an old version of the build artifacts. If no matching cache entry is
# found, the restore-keys value is chosen so that we may fall back to using
# cached data from a different Wasmtime commit. We always include the
# benchfx commit in all of the cache keys, meaning that updating benchfx
# itself effectively nukes the cache.
- name: Restore build artifacts from cache
uses: actions/cache@v4
id: cache
with:
key: benchfx-${{ env.benchfx_commit }}-wasmtime-${{ env.parent_github_sha }}
restore-keys: benchfx-${{ env.benchfx_commit }}
path: |
benchfx/tools/external/binaryen/build
benchfx/tools/external/wasmtime1/target
benchfx/tools/external/wasmtime2/target
# This is basically just a smoke test to make sure that the caching still
# does something sensible. If the internal folder structure that the harness
# uses ever changes, we would notice that here because the expected folders
# are not created anymore.
- name: Inspect caches
if: ${{ steps.cache.outputs.cache-hit }}
run: |
ls -la benchfx/tools/external/binaryen/build
ls -la benchfx/tools/external/wasmtime1/target
ls -la benchfx/tools/external/wasmtime2/target
- name: Prepare benchmarking (build binaryen, Wasmtime revs, benchmarks)
shell: bash
working-directory: ./benchfx
run: |
# The cache preserves mtimes, meaning that (the Makefile created by)
# cmake would consider all files from the cache as outdated. Since the
# harness fixes the binaryen commit, we know that the binaryen build
# artifacts are for the exact binaryen commit we want. Thus, we make
# them eligible for re-use by making them look sufficiently new. Note
# that cmake would detect any external changes that invalidate the
# cached build files (e.g., compiler/stdlib update, ...).
find tools/external/binaryen/build -exec touch {} \; || true
./harness.py --verbose compare-revs --prepare-only \
--rev1-wasmtime-cargo-build-args="--features=$wasmtime_features" \
--rev2-wasmtime-cargo-build-args="--features=$wasmtime_features" \
"${GITHUB_SHA}^1" "$GITHUB_SHA"
- name: Run benchmarks
shell: bash
working-directory: ./benchfx
run: |
./harness.py compare-revs \
$benchmark_filters \
--max-allowed-regression=$max_allowed_regression \
--rev1-wasmtime-cargo-build-args="--features=$wasmtime_features" \
--rev2-wasmtime-cargo-build-args="--features=$wasmtime_features" \
"${GITHUB_SHA}^1" "$GITHUB_SHA"