diff --git a/.github/workflows/workflow.yaml b/.github/workflows/workflow.yaml index 70395c07de0..bff76d21b68 100644 --- a/.github/workflows/workflow.yaml +++ b/.github/workflows/workflow.yaml @@ -317,6 +317,84 @@ jobs: done done + benchmark: + name: "Benchmark" + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: write + pull-requests: write + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v3 + + - name: Get all releases + uses: octokit/request-action@v2.x + id: get-releases + with: + route: GET /repos/${{ github.repository }}/releases + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Create release data json + run: | + cat < semver.json + ${{ steps.get-releases.outputs.data }} + EOF + arr=( $(jq -r '.[] | select(.prerelease == false and .draft == false) | .tag_name' semver.json) ) + echo "${arr[@]}" + + - name: Get the latest release + id: get-latest-semver-release + run: | + npm install --global semver + echo "release=v3.10.0" >> $GITHUB_OUTPUT + version=( $(semver "${arr[@]}") ) + rm semver.json + + - name: Download release benchmark file + uses: robinraju/release-downloader@v1.6 + id: get-latest-benchmark + continue-on-error: true + with: + tag: ${{ steps.get-latest-semver-release.outputs.release }} + fileName: "release_benchmarks.txt" + + - name: Run benchmarks + run: | + curl -L -O "https://github.com/kubernetes-sigs/kubebuilder/releases/download/v${KUBEBUILDER_VERSION}/kubebuilder_${KUBEBUILDER_VERSION}_linux_amd64.tar.gz" &&\ + tar -zxvf kubebuilder_${KUBEBUILDER_VERSION}_linux_amd64.tar.gz &&\ + sudo mv kubebuilder_${KUBEBUILDER_VERSION}_linux_amd64 /usr/local/kubebuilder + make benchmark-test BENCHMARK_FILE_NAME=pr_benchmarks.txt + if: ${{ steps.get-latest-benchmark.outcome == 'success' }} + env: + KUBEBUILDER_VERSION: 2.3.1 + + - name: Compare benchmarks + id: get-comment-body + if: ${{ steps.get-latest-benchmark.outcome == 'success' }} + run: | + export GOPATH="$HOME/go" + PATH="$GOPATH/bin:$PATH" + go install golang.org/x/perf/cmd/benchstat@latest + benchstat release_benchmarks.txt pr_benchmarks.txt > benchstat.txt + delimiter=$(openssl rand -hex 8) + echo 'msg<<$delimiter' >> $GITHUB_OUTPUT + cat benchstat.txt >> $GITHUB_OUTPUT + echo '$delimiter' >> $GITHUB_OUTPUT + + - name: Create commit comment + if: ${{ steps.get-latest-benchmark.outcome == 'success' }} + uses: peter-evans/create-or-update-comment@v2 + with: + issue-number: ${{ github.event.pull_request.number }} + body: | + This PR compares its performance to the latest released version. If it performs significantly lower, consider optimizing your changes to improve the performance. + ``` + ${{ steps.get-comment-body.outputs.msg }} + ``` + pre-release: name: "Pre Release" runs-on: "ubuntu-latest" @@ -458,6 +536,16 @@ jobs: env: PLATFORMS: "linux-amd64 linux-arm64 darwin-amd64 darwin-arm64" + - name: Run benchmarks + run: | + curl -L -O "https://github.com/kubernetes-sigs/kubebuilder/releases/download/v${KUBEBUILDER_VERSION}/kubebuilder_${KUBEBUILDER_VERSION}_linux_amd64.tar.gz" &&\ + tar -zxvf kubebuilder_${KUBEBUILDER_VERSION}_linux_amd64.tar.gz &&\ + sudo mv kubebuilder_${KUBEBUILDER_VERSION}_linux_amd64 /usr/local/kubebuilder + make benchmark-test BENCHMARK_FILE_NAME=release_benchmarks.txt + mv benchmarks.txt _dist/release_benchmarks.txt + env: + KUBEBUILDER_VERSION: 2.3.1 + - name: Create GitHub release uses: "marvinpinto/action-automatic-releases@v1.2.1" with: @@ -466,6 +554,7 @@ jobs: files: | _dist/sha256sums.txt _dist/*.tar.gz + _dist/release_benchmarks.txt - name: Publish Helm chart uses: stefanprodan/helm-gh-pages@v1.7.0 diff --git a/Makefile b/Makefile index e6b06c2761c..3b690dcba51 100644 --- a/Makefile +++ b/Makefile @@ -35,6 +35,8 @@ GOLANGCI_LINT_VERSION := v1.45.2 # Detects the location of the user golangci-lint cache. GOLANGCI_LINT_CACHE := $(shell pwd)/.tmp/golangci-lint +BENCHMARK_FILE_NAME ?= benchmarks.txt + ROOT_DIR := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) BIN_DIR := $(abspath $(ROOT_DIR)/bin) @@ -103,6 +105,10 @@ all: lint test manager native-test: GO111MODULE=on go test -mod vendor ./pkg/... ./apis/... -race -bench . -coverprofile cover.out +.PHONY: benchmark-test +benchmark-test: + go test ./pkg/... -bench . -run="^#" -count 5 > ${BENCHMARK_FILE_NAME} + # Hook to run docker tests .PHONY: test test: __test-image