From b48134843315a4026e2e385d0dd13ce8981668da Mon Sep 17 00:00:00 2001 From: Yi Duan Date: Mon, 12 Aug 2024 16:49:09 +0800 Subject: [PATCH] build: add CI on performance (#687) --- .github/workflows/benchmark-linux.yml | 30 ----------- .github/workflows/benchmark.yml | 55 +++++++++++++++++++ generic_test/benchmark_test.go | 10 ++++ scripts/bench.py | 76 +++++++++++++++++++++++++-- 4 files changed, 137 insertions(+), 34 deletions(-) delete mode 100644 .github/workflows/benchmark-linux.yml create mode 100644 .github/workflows/benchmark.yml diff --git a/.github/workflows/benchmark-linux.yml b/.github/workflows/benchmark-linux.yml deleted file mode 100644 index 1e1f4488a..000000000 --- a/.github/workflows/benchmark-linux.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: Benchmark Linux-X64 - -on: pull_request - -jobs: - build: - strategy: - matrix: - os: [X64, arm] - runs-on: ${{ matrix.os }} - steps: - - name: Clear repository - run: sudo rm -fr $GITHUB_WORKSPACE && mkdir $GITHUB_WORKSPACE - - - uses: actions/checkout@v2 - - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: 1.22 - - - uses: actions/cache@v2 - with: - path: ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- - - - name: Benchmark sonic - run: sh scripts/bench.sh \ No newline at end of file diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 000000000..2ee1030d2 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,55 @@ +name: Benchmark + +on: pull_request + +jobs: + build: + strategy: + matrix: + os: [X64, arm] + runs-on: ${{ matrix.os }} + steps: + - name: Clear repository + run: sudo rm -fr $GITHUB_WORKSPACE && mkdir $GITHUB_WORKSPACE + + - uses: actions/checkout@v2 + + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.22 + + - uses: actions/cache@v2 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Benchmark Target + run: | + export SONIC_NO_ASYNC_GC=1 + export SONIC_BENCH_SINGLE=1 + go test -run ^$ -count=10 -benchmem -bench 'Benchmark(Encoder|Decoder)_(Generic|Binding)_Sonic' ./decoder >> /var/tmp/sonic_bench_target.out + go test -run ^$ -count=10 -benchmem -bench 'Benchmark(Get|Set)One_Sonic|BenchmarkParseSeven_Sonic' ./ast >> /var/tmp/sonic_bench_target.out + + - name: Clear repository + run: sudo rm -fr $GITHUB_WORKSPACE && mkdir $GITHUB_WORKSPACE + + - name: Checkout main + uses: actions/checkout@v2 + with: + ref: main + + - name: Benchmark main + run: | + export SONIC_NO_ASYNC_GC=1 + export SONIC_BENCH_SINGLE=1 + go test -run ^$ -count=10 -benchmem -bench 'Benchmark(Encoder|Decoder)_(Generic|Binding)_Sonic' ./decoder >> /var/tmp/sonic_bench_main.out + go test -run ^$ -count=10 -benchmem -bench 'Benchmark(Get|Set)One_Sonic|BenchmarkParseSeven_Sonic' ./ast >> /var/tmp/sonic_bench_main.out + + - name: Diff bench + run: | + go get golang.org/x/perf/cmd/benchstat && go install golang.org/x/perf/cmd/benchstat + benchstat -format=csv /var/tmp/sonic_bench_target.out /var/tmp/sonic_bench_main.out + # run: ./scripts/bench.py -t 0.05 -d /var/tmp/sonic_bench_target.out,/var/tmp/sonic_bench_main.out x diff --git a/generic_test/benchmark_test.go b/generic_test/benchmark_test.go index 377aef025..c3a2b4641 100644 --- a/generic_test/benchmark_test.go +++ b/generic_test/benchmark_test.go @@ -33,6 +33,7 @@ import ( var ( validFlag = os.Getenv("SONIC_VALID_GENERIC_BENCH") != "" pretouchFlag = os.Getenv("SONIC_NO_PRETOUCH_BENCH") == "" + benchSingle = os.Getenv("SONIC_BENCH_SINGLE") != "" ) type jsonLibEntry struct { @@ -51,6 +52,15 @@ var jsonLibs = []jsonLibEntry { {"JsonIterStd", jsoniter.ConfigCompatibleWithStandardLibrary.Marshal, jsoniter.ConfigCompatibleWithStandardLibrary.Unmarshal}, } +func init() { + if benchSingle { + jsonLibs = []jsonLibEntry { + {"Sonic", sonic.Marshal, sonic.Unmarshal}, + {"SonicStd", sonic.ConfigStd.Marshal, sonic.ConfigStd.Unmarshal}, + } + } +} + func BenchmarkUnmarshalConcrete(b *testing.B) { runUnmarshalC(b) } diff --git a/scripts/bench.py b/scripts/bench.py index 80e8f186b..41e1b5a08 100755 --- a/scripts/bench.py +++ b/scripts/bench.py @@ -14,6 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import csv +import io import tempfile import os import subprocess @@ -81,10 +83,6 @@ def compare(args): # benchmark main branch main = run_bench(args, "main") - - # diff the result - # benchstat = "go get golang.org/x/perf/cmd/benchstat && go install golang.org/x/perf/cmd/benchstat" - run( "benchstat %s %s"%(main, target)) run("git checkout -- .") # restore branch @@ -92,6 +90,9 @@ def compare(args): run("git checkout %s"%(current_branch)) run("patch -p1 < %s" % (diff)) + + # diff the result + bench_diff(main, target, args.threshold) return target def run_bench(args, name): @@ -99,16 +100,83 @@ def run_bench(args, name): run("%s 2>&1 | tee %s" %(args.cmd, fname)) return fname +def bench_diff(main, target, threshold=0.05): + run("go get golang.org/x/perf/cmd/benchstat && go install golang.org/x/perf/cmd/benchstat") + csv_content = run_s( "benchstat -format=csv %s %s"%(main, target)) + print("benchstat: %s"%csv_content) + + # filter out invalid lines + valid_headers = {',sec/op,CI,sec/op,CI,vs base,P', ',B/s,CI,B/s,CI,vs base,P'} + valid_blocks = [] + current_block = [] + parsing = False + + # Filter out valid CSV blocks + for line in csv_content.splitlines(): + if line in valid_headers: + if current_block: + valid_blocks.append('\n'.join(current_block)) + current_block = [line] + parsing = True + elif parsing: + if line.strip() == '': + parsing = False + if current_block: + valid_blocks.append('\n'.join(current_block)) + current_block = [] + else: + current_block.append(line) + + if current_block: + valid_blocks.append('\n'.join(current_block)) + + # Parse each valid CSV block + significant_decline = [] + for block in valid_blocks: + csv_reader = csv.DictReader(io.StringIO(block)) + for row in csv_reader: + benchmark = row[''] + vs_base = row['vs base'] + if benchmark == 'geomean': + continue + + # Skip rows without a valid "vs base" value + if not vs_base or vs_base == '~': + continue + + # Convert "vs base" to a float percentage + try: + vs_base_percentage = float(vs_base.strip('%')) / 100 + except ValueError: + continue + + # Check if the decline is significant + if vs_base_percentage < 0 and -vs_base_percentage > threshold: + significant_decline.append({ + 'benchmark': benchmark, + 'vs_base': vs_base_percentage + }) + + if significant_decline: + print("found significant decline! %s %f"%(significant_decline[0]['benchmark'], significant_decline[0]['vs_base'])) + exit(2) + + return def main(): argparser = argparse.ArgumentParser(description='Tools to test the performance. Example: ./bench.py "go test -bench=. ./..."') argparser.add_argument('cmd', type=str, help='Golang benchmark command') + argparser.add_argument('-d', type=str, dest='diff', help='diff bench') + argparser.add_argument('-t', type=float, dest='threshold', default=0.1, help='diff bench threshold') argparser.add_argument('-c', '--compare', dest='compare', action='store_true', help='Compare the current branch with the main branch') args = argparser.parse_args() if args.compare: compare(args) + elif args.diff: + target, base = args.diff.split(',') + bench_diff(target, base, args.threshold) else: run_bench(args, "target")