From 44197c5eebaa42f24cf9aab4e6ea33da5b720fe8 Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Wed, 29 Jun 2022 01:31:29 +0000 Subject: [PATCH 1/3] benchmark prs Signed-off-by: Sertac Ozercan --- .github/workflows/workflow.yaml | 40 +++++++++++++++++++++++++++++++++ Makefile | 6 +++++ 2 files changed, 46 insertions(+) diff --git a/.github/workflows/workflow.yaml b/.github/workflows/workflow.yaml index 70395c07de0..99e3d85409e 100644 --- a/.github/workflows/workflow.yaml +++ b/.github/workflows/workflow.yaml @@ -317,6 +317,40 @@ jobs: done done + benchmark: + name: "Benchmark" + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + # TODO + contents: write + pull-requests: write + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v3 + + - name: Download release benchmark file + run: ...#TODO + + - name: Run benchmarks + run: make benchmark-test BENCHMARK_FILE_NAME=pr_benchmarks.txt + + - name: Compare benchmarks + id: get-comment-body + run: | + go install golang.org/x/perf/cmd/benchstat@latest + benchstat pr_benchmarks.txt release_benchmarks.txt > results.txt + body=$(cat results.txt) + echo ::set-output name=body::$body + + - name: Create commit comment + uses: peter-evans/commit-comment@v1 + with: + body: ${{ steps.get-comment-body.outputs.body }} + sha: ${{ github.event.pull_request.head.sha }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + pre-release: name: "Pre Release" runs-on: "ubuntu-latest" @@ -458,6 +492,11 @@ jobs: env: PLATFORMS: "linux-amd64 linux-arm64 darwin-amd64 darwin-arm64" + - name: Run benchmarks + run: | + make benchmark-test BENCHMARK_FILE_NAME=release_benchmarks.txt + mv benchmarks.txt _dist/release_benchmarks.txt + - name: Create GitHub release uses: "marvinpinto/action-automatic-releases@v1.2.1" with: @@ -466,6 +505,7 @@ jobs: files: | _dist/sha256sums.txt _dist/*.tar.gz + _dist/release_benchmarks.txt - name: Publish Helm chart uses: stefanprodan/helm-gh-pages@v1.7.0 diff --git a/Makefile b/Makefile index e6b06c2761c..3b690dcba51 100644 --- a/Makefile +++ b/Makefile @@ -35,6 +35,8 @@ GOLANGCI_LINT_VERSION := v1.45.2 # Detects the location of the user golangci-lint cache. GOLANGCI_LINT_CACHE := $(shell pwd)/.tmp/golangci-lint +BENCHMARK_FILE_NAME ?= benchmarks.txt + ROOT_DIR := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) BIN_DIR := $(abspath $(ROOT_DIR)/bin) @@ -103,6 +105,10 @@ all: lint test manager native-test: GO111MODULE=on go test -mod vendor ./pkg/... ./apis/... -race -bench . -coverprofile cover.out +.PHONY: benchmark-test +benchmark-test: + go test ./pkg/... -bench . -run="^#" -count 5 > ${BENCHMARK_FILE_NAME} + # Hook to run docker tests .PHONY: test test: __test-image From 884acc6a900c03fc75b719cf6bbbeade513ed6be Mon Sep 17 00:00:00 2001 From: Jaydip Gabani Date: Sat, 3 Dec 2022 23:52:28 +0000 Subject: [PATCH 2/3] Download previous released benchmarks Signed-off-by: Jaydip Gabani --- .github/workflows/workflow.yaml | 49 +++++++++++++++++++++++++-------- 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/.github/workflows/workflow.yaml b/.github/workflows/workflow.yaml index 99e3d85409e..9201386d24f 100644 --- a/.github/workflows/workflow.yaml +++ b/.github/workflows/workflow.yaml @@ -319,10 +319,10 @@ jobs: benchmark: name: "Benchmark" + if: github.event_name == 'pull_request' runs-on: ubuntu-latest timeout-minutes: 30 permissions: - # TODO contents: write pull-requests: write steps: @@ -330,26 +330,46 @@ jobs: uses: actions/checkout@v3 - name: Download release benchmark file - run: ...#TODO + uses: robinraju/release-downloader@v1.6 + id: get-latest-benchmark + continue-on-error: true + with: + latest: true + fileName: "release_benchmarks.txt" - name: Run benchmarks - run: make benchmark-test BENCHMARK_FILE_NAME=pr_benchmarks.txt + run: | + curl -L -O "https://github.com/kubernetes-sigs/kubebuilder/releases/download/v${KUBEBUILDER_VERSION}/kubebuilder_${KUBEBUILDER_VERSION}_linux_amd64.tar.gz" &&\ + tar -zxvf kubebuilder_${KUBEBUILDER_VERSION}_linux_amd64.tar.gz &&\ + sudo mv kubebuilder_${KUBEBUILDER_VERSION}_linux_amd64 /usr/local/kubebuilder + make benchmark-test BENCHMARK_FILE_NAME=pr_benchmarks.txt + if: ${{ steps.get-latest-benchmark.outcome == 'success' }} + env: + KUBEBUILDER_VERSION: 2.3.1 - name: Compare benchmarks id: get-comment-body + if: ${{ steps.get-latest-benchmark.outcome == 'success' }} run: | + export GOPATH="$HOME/go" + PATH="$GOPATH/bin:$PATH" go install golang.org/x/perf/cmd/benchstat@latest - benchstat pr_benchmarks.txt release_benchmarks.txt > results.txt - body=$(cat results.txt) - echo ::set-output name=body::$body - + benchstat release_benchmarks.txt pr_benchmarks.txt > benchstat.txt + delimiter=$(openssl rand -hex 8) + echo 'msg<<$delimiter' >> $GITHUB_OUTPUT + cat benchstat.txt >> $GITHUB_OUTPUT + echo '$delimiter' >> $GITHUB_OUTPUT + - name: Create commit comment - uses: peter-evans/commit-comment@v1 + if: ${{ steps.get-latest-benchmark.outcome == 'success' }} + uses: peter-evans/create-or-update-comment@v2 with: - body: ${{ steps.get-comment-body.outputs.body }} - sha: ${{ github.event.pull_request.head.sha }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + issue-number: ${{ github.event.pull_request.number }} + body: | + Here is how this PR compares againt latest released version in terms of benchmark. Unless this PR containts huge updates, if the performace is significantly lower consider optimizing your changes to improve the performace closest to latest released benchmark. + ``` + ${{ steps.get-comment-body.outputs.msg }} + ``` pre-release: name: "Pre Release" @@ -494,8 +514,13 @@ jobs: - name: Run benchmarks run: | + curl -L -O "https://github.com/kubernetes-sigs/kubebuilder/releases/download/v${KUBEBUILDER_VERSION}/kubebuilder_${KUBEBUILDER_VERSION}_linux_amd64.tar.gz" &&\ + tar -zxvf kubebuilder_${KUBEBUILDER_VERSION}_linux_amd64.tar.gz &&\ + sudo mv kubebuilder_${KUBEBUILDER_VERSION}_linux_amd64 /usr/local/kubebuilder make benchmark-test BENCHMARK_FILE_NAME=release_benchmarks.txt mv benchmarks.txt _dist/release_benchmarks.txt + env: + KUBEBUILDER_VERSION: 2.3.1 - name: Create GitHub release uses: "marvinpinto/action-automatic-releases@v1.2.1" From e25b243cd747b2daaedead81f97b427edda4ca78 Mon Sep 17 00:00:00 2001 From: Jaydip Gabani Date: Tue, 6 Dec 2022 20:59:12 +0000 Subject: [PATCH 3/3] fetching latest semver version Signed-off-by: Jaydip Gabani --- .github/workflows/workflow.yaml | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/.github/workflows/workflow.yaml b/.github/workflows/workflow.yaml index 9201386d24f..bff76d21b68 100644 --- a/.github/workflows/workflow.yaml +++ b/.github/workflows/workflow.yaml @@ -328,13 +328,37 @@ jobs: steps: - name: Check out code into the Go module directory uses: actions/checkout@v3 - + + - name: Get all releases + uses: octokit/request-action@v2.x + id: get-releases + with: + route: GET /repos/${{ github.repository }}/releases + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Create release data json + run: | + cat < semver.json + ${{ steps.get-releases.outputs.data }} + EOF + arr=( $(jq -r '.[] | select(.prerelease == false and .draft == false) | .tag_name' semver.json) ) + echo "${arr[@]}" + + - name: Get the latest release + id: get-latest-semver-release + run: | + npm install --global semver + echo "release=v3.10.0" >> $GITHUB_OUTPUT + version=( $(semver "${arr[@]}") ) + rm semver.json + - name: Download release benchmark file uses: robinraju/release-downloader@v1.6 id: get-latest-benchmark continue-on-error: true with: - latest: true + tag: ${{ steps.get-latest-semver-release.outputs.release }} fileName: "release_benchmarks.txt" - name: Run benchmarks @@ -366,7 +390,7 @@ jobs: with: issue-number: ${{ github.event.pull_request.number }} body: | - Here is how this PR compares againt latest released version in terms of benchmark. Unless this PR containts huge updates, if the performace is significantly lower consider optimizing your changes to improve the performace closest to latest released benchmark. + This PR compares its performance to the latest released version. If it performs significantly lower, consider optimizing your changes to improve the performance. ``` ${{ steps.get-comment-body.outputs.msg }} ```