From b40aac861f8d51a640bc48468cb27d354b039dcd Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Wed, 29 Jun 2022 01:31:29 +0000 Subject: [PATCH 1/6] benchmark prs Signed-off-by: Sertac Ozercan --- .github/workflows/workflow.yaml | 40 +++++++++++++++++++++++++++++++++ Makefile | 6 +++++ 2 files changed, 46 insertions(+) diff --git a/.github/workflows/workflow.yaml b/.github/workflows/workflow.yaml index b0e6ddacd9a..ea4cfcf7e1c 100644 --- a/.github/workflows/workflow.yaml +++ b/.github/workflows/workflow.yaml @@ -273,6 +273,40 @@ jobs: done done + benchmark: + name: "Benchmark" + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + # TODO + contents: write + pull-requests: write + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v3 + + - name: Download release benchmark file + run: ...#TODO + + - name: Run benchmarks + run: make benchmark-test BENCHMARK_FILE_NAME=pr_benchmarks.txt + + - name: Compare benchmarks + id: get-comment-body + run: | + go install golang.org/x/perf/cmd/benchstat@latest + benchstat pr_benchmarks.txt release_benchmarks.txt > results.txt + body=$(cat results.txt) + echo ::set-output name=body::$body + + - name: Create commit comment + uses: peter-evans/commit-comment@v1 + with: + body: ${{ steps.get-comment-body.outputs.body }} + sha: ${{ github.event.pull_request.head.sha }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + pre-release: name: "Pre Release" runs-on: "ubuntu-latest" @@ -414,6 +448,11 @@ jobs: env: PLATFORMS: "linux-amd64 linux-arm64 darwin-amd64 darwin-arm64" + - name: Run benchmarks + run: | + make benchmark-test BENCHMARK_FILE_NAME=release_benchmarks.txt + mv benchmarks.txt _dist/release_benchmarks.txt + - name: Create GitHub release uses: "marvinpinto/action-automatic-releases@v1.2.1" with: @@ -422,6 +461,7 @@ jobs: files: | _dist/sha256sums.txt _dist/*.tar.gz + _dist/release_benchmarks.txt - name: Publish Helm chart uses: stefanprodan/helm-gh-pages@v1.7.0 diff --git a/Makefile b/Makefile index 48de3ddf276..522e25b39bd 100644 --- a/Makefile +++ b/Makefile @@ -35,6 +35,8 @@ GOLANGCI_LINT_VERSION := v1.45.2 # Detects the location of the user golangci-lint cache. GOLANGCI_LINT_CACHE := $(shell pwd)/.tmp/golangci-lint +BENCHMARK_FILE_NAME ?= benchmarks.txt + ROOT_DIR := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) BIN_DIR := $(abspath $(ROOT_DIR)/bin) @@ -103,6 +105,10 @@ all: lint test manager native-test: GO111MODULE=on go test -mod vendor ./pkg/... ./apis/... ./cmd/gator/... -race -bench . -coverprofile cover.out +.PHONY: benchmark-test +benchmark-test: + go test ./pkg/... -bench . -run="^#" -count 5 > ${BENCHMARK_FILE_NAME} + # Hook to run docker tests .PHONY: test test: __test-image From 45ef223a23c3916cda7a177c1f85e0732aa90260 Mon Sep 17 00:00:00 2001 From: Jaydip Gabani Date: Fri, 16 Dec 2022 09:47:10 +0000 Subject: [PATCH 2/6] Benchmarking against base ref on PR comment Signed-off-by: Jaydip Gabani --- .github/workflows/benchmark.yaml | 80 ++++++++++++++++++++++++++++++++ .github/workflows/workflow.yaml | 40 ---------------- 2 files changed, 80 insertions(+), 40 deletions(-) create mode 100644 .github/workflows/benchmark.yaml diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml new file mode 100644 index 00000000000..0019a79631a --- /dev/null +++ b/.github/workflows/benchmark.yaml @@ -0,0 +1,80 @@ +name: benchmark_changes +on: + issue_comment: + types: [created] + +jobs: + benchmark: + name: "Benchmark" + if: github.event.issue.pull_request && github.event.comment.body == '/benchmark' + runs-on: ubuntu-latest + timeout-minutes: 60 + permissions: + contents: write + pull-requests: write + steps: + - uses: izhangzhihao/delete-comment@master + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + delete_user_name: github-actions[bot] + issue_number: ${{ github.event.issue.number }} + + - name: echo output + run: | + echo ${{ github.event.comment.body }} + + - name: Update status + uses: peter-evans/create-or-update-comment@v2 + with: + issue-number: ${{ github.event.issue.number }} + body: | + [Running benchmark here...](${{ github.server.url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) + + - name: Check out base code into the Go module directory + uses: actions/checkout@v3 + with: + ref: ${{ github.base_ref }} + + - name: Run benchmarks on base ref + run: | + curl -L -O "https://github.com/kubernetes-sigs/kubebuilder/releases/download/v${KUBEBUILDER_VERSION}/kubebuilder_${KUBEBUILDER_VERSION}_linux_amd64.tar.gz" &&\ + tar -zxvf kubebuilder_${KUBEBUILDER_VERSION}_linux_amd64.tar.gz &&\ + sudo mv kubebuilder_${KUBEBUILDER_VERSION}_linux_amd64 /usr/local/kubebuilder + GOMAXPROCS=1 go test ./pkg/... -bench . -run="^#" -count 10 > ../base_benchmarks.txt + env: + KUBEBUILDER_VERSION: 2.3.1 + + - name: Check out code into the Go module directory + uses: actions/checkout@v3 + + - name: Run benchmark with incoming changes + run: | + curl -L -O "https://github.com/kubernetes-sigs/kubebuilder/releases/download/v${KUBEBUILDER_VERSION}/kubebuilder_${KUBEBUILDER_VERSION}_linux_amd64.tar.gz" &&\ + tar -zxvf kubebuilder_${KUBEBUILDER_VERSION}_linux_amd64.tar.gz &&\ + sudo mv kubebuilder_${KUBEBUILDER_VERSION}_linux_amd64 /usr/local/kubebuilder + GOMAXPROCS=1 go test ./pkg/... -bench . -run="^#" -count 10 > pr_benchmarks.txt + env: + KUBEBUILDER_VERSION: 2.3.1 + + - name: Compare benchmarks + id: get-comment-body + run: | + export GOPATH="$HOME/go" + PATH="$GOPATH/bin:$PATH" + go install golang.org/x/perf/cmd/benchstat@latest + benchstat ../base_benchmarks.txt pr_benchmarks.txt > benchstat.txt + delimiter=$(openssl rand -hex 8) + echo 'msg<<$delimiter' >> $GITHUB_OUTPUT + cat benchstat.txt >> $GITHUB_OUTPUT + echo '$delimiter' >> $GITHUB_OUTPUT + + - name: Create commit comment + uses: peter-evans/create-or-update-comment@v2 + with: + issue-number: ${{ github.event.issue.number }} + body: | + This PR compares its performance to the latest released version. If it performs significantly lower, consider optimizing your changes to improve the performance. + ``` + ${{ steps.get-comment-body.outputs.msg }} + ``` + \ No newline at end of file diff --git a/.github/workflows/workflow.yaml b/.github/workflows/workflow.yaml index ea4cfcf7e1c..b0e6ddacd9a 100644 --- a/.github/workflows/workflow.yaml +++ b/.github/workflows/workflow.yaml @@ -273,40 +273,6 @@ jobs: done done - benchmark: - name: "Benchmark" - runs-on: ubuntu-latest - timeout-minutes: 30 - permissions: - # TODO - contents: write - pull-requests: write - steps: - - name: Check out code into the Go module directory - uses: actions/checkout@v3 - - - name: Download release benchmark file - run: ...#TODO - - - name: Run benchmarks - run: make benchmark-test BENCHMARK_FILE_NAME=pr_benchmarks.txt - - - name: Compare benchmarks - id: get-comment-body - run: | - go install golang.org/x/perf/cmd/benchstat@latest - benchstat pr_benchmarks.txt release_benchmarks.txt > results.txt - body=$(cat results.txt) - echo ::set-output name=body::$body - - - name: Create commit comment - uses: peter-evans/commit-comment@v1 - with: - body: ${{ steps.get-comment-body.outputs.body }} - sha: ${{ github.event.pull_request.head.sha }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - pre-release: name: "Pre Release" runs-on: "ubuntu-latest" @@ -448,11 +414,6 @@ jobs: env: PLATFORMS: "linux-amd64 linux-arm64 darwin-amd64 darwin-arm64" - - name: Run benchmarks - run: | - make benchmark-test BENCHMARK_FILE_NAME=release_benchmarks.txt - mv benchmarks.txt _dist/release_benchmarks.txt - - name: Create GitHub release uses: "marvinpinto/action-automatic-releases@v1.2.1" with: @@ -461,7 +422,6 @@ jobs: files: | _dist/sha256sums.txt _dist/*.tar.gz - _dist/release_benchmarks.txt - name: Publish Helm chart uses: stefanprodan/helm-gh-pages@v1.7.0 From 5532e100723336c5d035de5880dc0cf1c82682d7 Mon Sep 17 00:00:00 2001 From: Jaydip Gabani Date: Sat, 17 Dec 2022 00:35:00 +0000 Subject: [PATCH 3/6] running benchmark with manual trigger by commenting on PR Signed-off-by: Jaydip Gabani --- .github/workflows/benchmark.yaml | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 0019a79631a..e2bc5552ce4 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -19,10 +19,14 @@ jobs: delete_user_name: github-actions[bot] issue_number: ${{ github.event.issue.number }} - - name: echo output + - name: install kubebuilder run: | - echo ${{ github.event.comment.body }} - + curl -L -O "https://github.com/kubernetes-sigs/kubebuilder/releases/download/v${KUBEBUILDER_VERSION}/kubebuilder_${KUBEBUILDER_VERSION}_linux_amd64.tar.gz" &&\ + tar -zxvf kubebuilder_${KUBEBUILDER_VERSION}_linux_amd64.tar.gz &&\ + sudo mv kubebuilder_${KUBEBUILDER_VERSION}_linux_amd64 /usr/local/kubebuilder + env: + KUBEBUILDER_VERSION: 2.3.1 + - name: Update status uses: peter-evans/create-or-update-comment@v2 with: @@ -36,38 +40,27 @@ jobs: ref: ${{ github.base_ref }} - name: Run benchmarks on base ref - run: | - curl -L -O "https://github.com/kubernetes-sigs/kubebuilder/releases/download/v${KUBEBUILDER_VERSION}/kubebuilder_${KUBEBUILDER_VERSION}_linux_amd64.tar.gz" &&\ - tar -zxvf kubebuilder_${KUBEBUILDER_VERSION}_linux_amd64.tar.gz &&\ - sudo mv kubebuilder_${KUBEBUILDER_VERSION}_linux_amd64 /usr/local/kubebuilder - GOMAXPROCS=1 go test ./pkg/... -bench . -run="^#" -count 10 > ../base_benchmarks.txt - env: - KUBEBUILDER_VERSION: 2.3.1 + run: GOMAXPROCS=1 go test ./pkg/... -bench . -run="^#" -count 10 > ../base_benchmarks.txt - name: Check out code into the Go module directory uses: actions/checkout@v3 - name: Run benchmark with incoming changes run: | - curl -L -O "https://github.com/kubernetes-sigs/kubebuilder/releases/download/v${KUBEBUILDER_VERSION}/kubebuilder_${KUBEBUILDER_VERSION}_linux_amd64.tar.gz" &&\ - tar -zxvf kubebuilder_${KUBEBUILDER_VERSION}_linux_amd64.tar.gz &&\ - sudo mv kubebuilder_${KUBEBUILDER_VERSION}_linux_amd64 /usr/local/kubebuilder GOMAXPROCS=1 go test ./pkg/... -bench . -run="^#" -count 10 > pr_benchmarks.txt - env: - KUBEBUILDER_VERSION: 2.3.1 - name: Compare benchmarks id: get-comment-body run: | export GOPATH="$HOME/go" PATH="$GOPATH/bin:$PATH" - go install golang.org/x/perf/cmd/benchstat@latest + go install golang.org/x/perf/cmd/benchstat@v0.0.0-20220920022801-e8d778a60d07 benchstat ../base_benchmarks.txt pr_benchmarks.txt > benchstat.txt delimiter=$(openssl rand -hex 8) echo 'msg<<$delimiter' >> $GITHUB_OUTPUT cat benchstat.txt >> $GITHUB_OUTPUT echo '$delimiter' >> $GITHUB_OUTPUT - + - name: Create commit comment uses: peter-evans/create-or-update-comment@v2 with: @@ -77,4 +70,3 @@ jobs: ``` ${{ steps.get-comment-body.outputs.msg }} ``` - \ No newline at end of file From b9ff7331b6fc4df96641c408bcf41cd6379054f1 Mon Sep 17 00:00:00 2001 From: Jaydip Gabani Date: Mon, 19 Dec 2022 18:02:53 +0000 Subject: [PATCH 4/6] runnint more iteration of test in benchmark Signed-off-by: Jaydip Gabani --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 522e25b39bd..1aa2bf68416 100644 --- a/Makefile +++ b/Makefile @@ -107,7 +107,7 @@ native-test: .PHONY: benchmark-test benchmark-test: - go test ./pkg/... -bench . -run="^#" -count 5 > ${BENCHMARK_FILE_NAME} + go test ./pkg/... -bench . -run="^#" -count 10 > ${BENCHMARK_FILE_NAME} # Hook to run docker tests .PHONY: test From dea980ff9815ffc1648adc0179d83abf7698c762 Mon Sep 17 00:00:00 2001 From: Jaydip Gabani Date: Thu, 5 Jan 2023 01:19:02 +0000 Subject: [PATCH 5/6] pinning pr comment delete action digest Signed-off-by: Jaydip Gabani --- .github/workflows/benchmark.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index e2bc5552ce4..7539cd4ea32 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -13,7 +13,7 @@ jobs: contents: write pull-requests: write steps: - - uses: izhangzhihao/delete-comment@master + - uses: izhangzhihao/delete-comment@98aa1ea5c6304048edf951c20b3114e03c785c79 with: github_token: ${{ secrets.GITHUB_TOKEN }} delete_user_name: github-actions[bot] From 7e7e4f997ce7be987a5e262b05972b8382a7e1e6 Mon Sep 17 00:00:00 2001 From: Jaydip Gabani Date: Thu, 5 Jan 2023 21:53:35 +0000 Subject: [PATCH 6/6] Using make rules for benchmarking Signed-off-by: Jaydip Gabani --- .github/workflows/benchmark.yaml | 5 ++--- Makefile | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 7539cd4ea32..8b9fcbd76aa 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -40,14 +40,13 @@ jobs: ref: ${{ github.base_ref }} - name: Run benchmarks on base ref - run: GOMAXPROCS=1 go test ./pkg/... -bench . -run="^#" -count 10 > ../base_benchmarks.txt + run: make benchmark-test BENCHMARK_FILE_NAME="../base_benchmarks.txt" - name: Check out code into the Go module directory uses: actions/checkout@v3 - name: Run benchmark with incoming changes - run: | - GOMAXPROCS=1 go test ./pkg/... -bench . -run="^#" -count 10 > pr_benchmarks.txt + run: make benchmark-test BENCHMARK_FILE_NAME="pr_benchmarks.txt" - name: Compare benchmarks id: get-comment-body diff --git a/Makefile b/Makefile index 1aa2bf68416..96d4c5a7e24 100644 --- a/Makefile +++ b/Makefile @@ -107,7 +107,7 @@ native-test: .PHONY: benchmark-test benchmark-test: - go test ./pkg/... -bench . -run="^#" -count 10 > ${BENCHMARK_FILE_NAME} + GOMAXPROCS=1 go test ./pkg/... -bench . -run="^#" -count 10 > ${BENCHMARK_FILE_NAME} # Hook to run docker tests .PHONY: test