From d6272483765cddce6cb0d69a7fb2247a45c57d1f Mon Sep 17 00:00:00 2001 From: Saylor Berman Date: Tue, 18 Jul 2023 10:43:54 -0600 Subject: [PATCH 1/3] Run conformance tests nightly on main branch In order to ensure we are staying ahead of Gateway API changes, we'll now run conformance tests off of the main branch on a nightly schedule. Devs can also run from the main branch locally by setting the GW_API_VERSION variable. --- .github/workflows/ci.yml | 86 ----------------- .github/workflows/conformance.yml | 113 +++++++++++++++++++++++ Makefile | 2 +- conformance/Makefile | 5 +- conformance/README.md | 1 + conformance/scripts/install-gateway.sh | 17 ++++ conformance/scripts/uninstall-gateway.sh | 17 ++++ 7 files changed, 152 insertions(+), 89 deletions(-) create mode 100644 .github/workflows/conformance.yml create mode 100755 conformance/scripts/install-gateway.sh create mode 100755 conformance/scripts/uninstall-gateway.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b3b1213d39..a9a0b74a41 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -147,92 +147,6 @@ jobs: path: ${{ github.workspace }}/dist key: nginx-kubernetes-gateway-${{ github.run_id }}-${{ github.run_number }} - conformance-tests: - name: Gateway Conformance Tests - runs-on: ubuntu-22.04 - needs: vars - if: ${{ github.ref_type != 'tag' }} - steps: - - name: Checkout Repository - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - - - name: Setup Golang Environment - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 - with: - go-version-file: go.mod - - - name: Docker Buildx - uses: docker/setup-buildx-action@4c0219f9ac95b02789c1075625400b2acbff50b1 # v2.9.1 - - - name: Docker meta - id: meta - uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v4.6.0 - with: - images: | - name=ghcr.io/nginxinc/nginx-kubernetes-gateway - tags: | - type=semver,pattern={{version}} - type=edge - type=ref,event=pr - type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} - - - name: Prepare NKG files - run: | - nkg_prefix=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 1) - nkg_tag=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 2) - make update-nkg-manifest NKG_PREFIX=${nkg_prefix} NKG_TAG=${nkg_tag} - working-directory: ./conformance - - - name: Build binary - uses: goreleaser/goreleaser-action@336e29918d653399e599bfca99fadc1d7ffbc9f7 # v4.3.0 - with: - version: latest - args: ${{ startsWith(github.ref, 'refs/tags/') && 'release' || 'build --snapshot' }} --clean - env: - GOPATH: ${{ needs.vars.outputs.go_path }} - - - name: Build Docker Image - uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v4.1.1 - with: - file: build/Dockerfile - tags: ${{ steps.meta.outputs.tags }} - context: "." - target: goreleaser - load: true - cache-from: type=gha - cache-to: type=gha,mode=max - pull: true - - - name: Build Test Docker Image - uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v4.1.1 - with: - file: conformance/tests/Dockerfile - tags: conformance-test-runner:${{ github.sha }} - context: "." - load: true - cache-from: type=gha - cache-to: type=gha,mode=max - pull: true - - - name: Deploy Kubernetes - id: k8s - run: | - make create-kind-cluster KIND_KUBE_CONFIG=kube-${{ github.run_id }} - echo "KUBECONFIG=kube-${{ github.run_id }}" >> "$GITHUB_ENV" - working-directory: ./conformance - - - name: Setup conformance tests - run: | - nkg_prefix=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 1) - nkg_tag=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 2) - make install-nkg-local-no-build NKG_PREFIX=${nkg_prefix} NKG_TAG=${nkg_tag} - working-directory: ./conformance - - - name: Run conformance tests - run: | - make run-conformance-tests TAG=${{ github.sha }} - working-directory: ./conformance - build: name: Build Image runs-on: ubuntu-22.04 diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml new file mode 100644 index 0000000000..5703424c75 --- /dev/null +++ b/.github/workflows/conformance.yml @@ -0,0 +1,113 @@ +name: Conformance Testing + +on: + push: + branches: + - main + - release-* + pull_request: + branches: + - main + - release-* + types: + - opened + - reopened + - synchronize + schedule: + - cron: '0 4 * * *' + +concurrency: + group: ${{ github.ref_name }}-conformance + cancel-in-progress: true + +permissions: + contents: read + +jobs: + conformance-tests: + name: Gateway Conformance Tests + runs-on: ubuntu-22.04 + steps: + - name: Checkout Repository + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + + - name: Setup Golang Environment + uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 + with: + go-version-file: go.mod + + - name: Set GOPATH + run: echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV + + - name: Docker Buildx + uses: docker/setup-buildx-action@4c0219f9ac95b02789c1075625400b2acbff50b1 # v2.9.1 + + - name: Docker meta + id: meta + uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v4.6.0 + with: + images: | + name=ghcr.io/nginxinc/nginx-kubernetes-gateway + tags: | + type=semver,pattern={{version}} + type=edge + type=ref,event=pr + type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} + + - name: Prepare NKG files + run: | + nkg_prefix=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 1) + nkg_tag=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 2) + make update-nkg-manifest NKG_PREFIX=${nkg_prefix} NKG_TAG=${nkg_tag} + working-directory: ./conformance + + - name: Build binary + uses: goreleaser/goreleaser-action@336e29918d653399e599bfca99fadc1d7ffbc9f7 # v4.3.0 + with: + version: latest + args: build --snapshot --clean + + - name: Build Docker Image + uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v4.1.1 + with: + file: build/Dockerfile + tags: ${{ steps.meta.outputs.tags }} + context: "." + target: goreleaser + load: true + cache-from: type=gha + cache-to: type=gha,mode=max + pull: true + + - name: Build Test Docker Image + uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v4.1.1 + with: + file: conformance/tests/Dockerfile + tags: conformance-test-runner:${{ github.sha }} + context: "." + load: true + cache-from: type=gha + cache-to: type=gha,mode=max + pull: true + + - name: Deploy Kubernetes + id: k8s + run: | + make create-kind-cluster KIND_KUBE_CONFIG=${{ github.workspace }}/kube-${{ github.run_id }} + echo "KUBECONFIG=${{ github.workspace }}/kube-${{ github.run_id }}" >> "$GITHUB_ENV" + working-directory: ./conformance + + - name: Setup conformance tests + run: | + nkg_prefix=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 1) + nkg_tag=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 2) + if [ ${{ github.event_name }} == "schedule" ]; then + export GW_API_VERSION=main + fi + make install-nkg-local-no-build NKG_PREFIX=${nkg_prefix} NKG_TAG=${nkg_tag} + working-directory: ./conformance + + - name: Run conformance tests + run: | + make run-conformance-tests TAG=${{ github.sha }} + working-directory: ./conformance diff --git a/Makefile b/Makefile index 6434357af9..b73de929d5 100644 --- a/Makefile +++ b/Makefile @@ -56,7 +56,7 @@ deps: ## Add missing and remove unused modules, verify deps and download them to create-kind-cluster: ## Create a kind cluster $(eval KIND_IMAGE=$(shell grep -m1 'FROM kindest/node' Date: Thu, 20 Jul 2023 09:45:33 -0600 Subject: [PATCH 2/3] Add ability to update to main branch of Go deps --- .github/workflows/conformance.yml | 8 ++++++-- Makefile | 2 +- conformance/Makefile | 9 +++++++++ conformance/README.md | 27 ++++++++++++++++++++++++--- 4 files changed, 40 insertions(+), 6 deletions(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 5703424c75..3451325f68 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -79,6 +79,11 @@ jobs: cache-to: type=gha,mode=max pull: true + - name: Update Go Modules + if: ${{ github.event_name == 'schedule' }} + run: make update-go-modules + working-directory: ./conformance + - name: Build Test Docker Image uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v4.1.1 with: @@ -108,6 +113,5 @@ jobs: working-directory: ./conformance - name: Run conformance tests - run: | - make run-conformance-tests TAG=${{ github.sha }} + run: make run-conformance-tests TAG=${{ github.sha }} working-directory: ./conformance diff --git a/Makefile b/Makefile index b73de929d5..6434357af9 100644 --- a/Makefile +++ b/Makefile @@ -56,7 +56,7 @@ deps: ## Add missing and remove unused modules, verify deps and download them to create-kind-cluster: ## Create a kind cluster $(eval KIND_IMAGE=$(shell grep -m1 'FROM kindest/node' \033[0m\n\nTargets:\n\n"}; {printf " \033[36m%-30s\033[0m %s\n", $$1, $$2}' +.PHONY: update-go-modules +update-go-modules: ## Update the gateway-api go modules to latest main version + go get -u sigs.k8s.io/gateway-api@main + go mod tidy + .PHONY: build-test-runner-image build-test-runner-image: ## Build conformance test runner image docker build -t $(PREFIX):$(TAG) -f tests/Dockerfile .. @@ -94,6 +99,10 @@ uninstall-nkg: ## Uninstall NKG on configured kind cluster undo-image-update: ## Undo the NKG image name and tag in deployment manifest git checkout -- $(NKG_DEPLOYMENT_MANIFEST) +.PHONY: reset-go-modules +reset-go-modules: ## Reset the go modules changes + git checkout -- ../go.mod ../go.sum + .PHONY: delete-kind-cluster delete-kind-cluster: ## Delete kind cluster kind delete cluster diff --git a/conformance/README.md b/conformance/README.md index a6e3392dce..31836b3544 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -29,9 +29,11 @@ install-nkg-local-no-build Install NKG from local build with provisioner on load-images Load NKG and NGINX containers on configured kind cluster preload-nginx-container Preload NGINX container on configured kind cluster prepare-nkg-dependencies Install NKG dependencies on configured kind cluster +reset-go-modules Reset the go modules changes run-conformance-tests Run conformance tests undo-image-update Undo the NKG image name and tag in deployment manifest uninstall-nkg Uninstall NKG on configured kind cluster +update-go-modules Update the gateway-api go modules to latest main version update-nkg-manifest Update the NKG deployment manifest image name and imagePullPolicy ``` @@ -59,6 +61,13 @@ make create-kind-cluster ### Step 2 - Install Nginx Kubernetes Gateway to configured kind cluster +> Note: If you want to run the latest conformance tests from the Gateway API `main` branch, set the following +> environment variable before deploying NKG: +> ```bash +> export GW_API_VERSION=main +>``` +> Otherwise, the latest stable version will be used by default. + #### *Option 1* Build and install Nginx Kubernetes Gateway from local to configured kind cluster ```makefile @@ -97,6 +106,13 @@ make install-nkg-edge ### Step 3 - Build conformance test runner image +> Note: If you want to run the latest conformance tests from the Gateway API `main` branch, run the following +> make command to update the Go modules to `main`: +> ```makefile +> make update-go-modules +>``` +> Otherwise, the latest stable version will be used by default. + ```makefile make build-test-runner-image ``` @@ -117,7 +133,13 @@ make cleanup-conformance-tests make uninstall-nkg ``` -### Step 6 - Revert changes to the NKG deployment manifest +### Step 6 - Revert changes to Go modules +**Optional** Not required if you aren't running the `main` Gateway API tests. +```makefile +make reset-go-modules +``` + +### Step 7 - Revert changes to the NKG deployment manifest **Optional** Not required if using `edge` image **Warning**: `make undo-image-update` will hard reset changes to the deploy/manifests/deployment.yaml file! @@ -125,8 +147,7 @@ make uninstall-nkg make undo-image-update ``` -### Step 7 - Delete kind cluster - +### Step 8 - Delete kind cluster ```makefile make delete-kind-cluster ``` From 6c2e1e913e9342bfe46589ecfb6baf0d7fbfc8c5 Mon Sep 17 00:00:00 2001 From: Saylor Berman Date: Thu, 20 Jul 2023 15:08:52 -0600 Subject: [PATCH 3/3] Add steps for fork --- conformance/README.md | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/conformance/README.md b/conformance/README.md index 31836b3544..b375fef3b8 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -63,9 +63,11 @@ make create-kind-cluster > Note: If you want to run the latest conformance tests from the Gateway API `main` branch, set the following > environment variable before deploying NKG: -> ```bash -> export GW_API_VERSION=main ->``` + +```bash + export GW_API_VERSION=main +``` + > Otherwise, the latest stable version will be used by default. #### *Option 1* Build and install Nginx Kubernetes Gateway from local to configured kind cluster @@ -108,9 +110,20 @@ make install-nkg-edge > Note: If you want to run the latest conformance tests from the Gateway API `main` branch, run the following > make command to update the Go modules to `main`: -> ```makefile -> make update-go-modules ->``` + + ```makefile + make update-go-modules + ``` + +> You can also point to a specific fork/branch by running: + + ```bash + go mod edit -replace=sigs.k8s.io/gateway-api=@ + go mod download + go mod verify + go mod tidy + ``` + > Otherwise, the latest stable version will be used by default. ```makefile @@ -135,6 +148,7 @@ make uninstall-nkg ### Step 6 - Revert changes to Go modules **Optional** Not required if you aren't running the `main` Gateway API tests. + ```makefile make reset-go-modules ``` @@ -148,6 +162,7 @@ make undo-image-update ``` ### Step 8 - Delete kind cluster + ```makefile make delete-kind-cluster ```