From ecf4546d59d8fb048f70910dd4380b06b657758a Mon Sep 17 00:00:00 2001 From: Luca Comellini Date: Tue, 14 May 2024 15:30:32 -0700 Subject: [PATCH] Move conformance tests to tests directory --- .github/dependabot.yml | 3 +- .github/workflows/conformance.yml | 24 +- .github/workflows/functional.yml | 2 + .github/workflows/release-pr.yml | 2 - .gitignore | 2 +- Makefile | 51 ++- .../tests}/static-deployment.yaml | 0 conformance/Makefile | 141 -------- conformance/README.md | 194 ----------- conformance/tests/Dockerfile | 13 - docs/developer/quickstart.md | 2 +- docs/developer/release-process.md | 2 +- docs/developer/testing.md | 2 +- embedded.go | 4 +- go.mod | 8 +- go.sum | 11 - tests/Dockerfile | 3 + tests/Makefile | 150 ++++++--- tests/README.md | 306 ++++++++++++------ tests/conformance/Dockerfile | 10 + .../conformance}/conformance-rbac.yaml | 0 .../conformance}/conformance_test.go | 18 +- .../conformance}/provisioner/README.md | 4 +- .../conformance}/provisioner/provisioner.yaml | 0 tests/go.mod | 14 +- tests/go.sum | 26 +- .../scripts/check-pod-exit-code.sh | 0 .../scripts/install-gateway.sh | 0 .../scripts/uninstall-gateway.sh | 0 29 files changed, 418 insertions(+), 574 deletions(-) rename {conformance/provisioner => config/tests}/static-deployment.yaml (100%) delete mode 100644 conformance/Makefile delete mode 100644 conformance/README.md delete mode 100644 conformance/tests/Dockerfile create mode 100644 tests/Dockerfile create mode 100644 tests/conformance/Dockerfile rename {conformance/tests => tests/conformance}/conformance-rbac.yaml (100%) rename {conformance/tests => tests/conformance}/conformance_test.go (87%) rename {conformance => tests/conformance}/provisioner/README.md (92%) rename {conformance => tests/conformance}/provisioner/provisioner.yaml (100%) rename {conformance => tests}/scripts/check-pod-exit-code.sh (100%) rename {conformance => tests}/scripts/install-gateway.sh (100%) rename {conformance => tests}/scripts/uninstall-gateway.sh (100%) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index ba400b73a2..ef0d0fd5d7 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -15,7 +15,8 @@ updates: - package-ecosystem: docker directories: - /build - - /conformance/tests + - /tests + - /tests/conformance schedule: interval: daily diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 51bb7d69c4..72b403a70e 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -29,6 +29,8 @@ jobs: steps: - name: Checkout Repository uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + with: + fetch-depth: 0 - name: Setup Golang Environment uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1 @@ -67,12 +69,12 @@ jobs: type=ref,event=pr type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} - - name: Prepare NGF files + - name: Generate static deployment run: | ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric ngf_tag=${{ steps.ngf-meta.outputs.version }} - make update-ngf-manifest${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} - working-directory: ./conformance + make generate-static-deployment PLUS_ENABLED=${{ inputs.image == 'plus' && 'true' || 'false' }} PREFIX=${ngf_prefix} TAG=${ngf_tag} + working-directory: ./tests - name: Build binary uses: goreleaser/goreleaser-action@5742e2a039330cbb23ebf35f046f814d4c6ff811 # v5.1.0 @@ -111,14 +113,14 @@ jobs: - name: Update Go Modules if: ${{ github.event_name == 'schedule' }} run: make update-go-modules - working-directory: ./conformance + working-directory: ./tests - name: Build Test Docker Image uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0 with: - file: conformance/tests/Dockerfile + file: tests/conformance/Dockerfile tags: conformance-test-runner:${{ github.sha }} - context: "." + context: "tests" load: true cache-from: type=gha cache-to: type=gha,mode=max @@ -130,7 +132,7 @@ jobs: k8s_version=${{ inputs.k8s-version }} make create-kind-cluster KIND_KUBE_CONFIG=${{ github.workspace }}/kube-${{ github.run_id }} ${{ ! contains(inputs.k8s-version, 'latest') && 'KIND_IMAGE=kindest/node:v${k8s_version}' || '' }} echo "KUBECONFIG=${{ github.workspace }}/kube-${{ github.run_id }}" >> "$GITHUB_ENV" - working-directory: ./conformance + working-directory: ./tests - name: Setup conformance tests run: | @@ -139,19 +141,19 @@ jobs: if [ ${{ github.event_name }} == "schedule" ]; then export GW_API_VERSION=main; fi if [ ${{ inputs.enable-experimental }} == "true" ]; then export ENABLE_EXPERIMENTAL=true; fi make install-ngf-local-no-build${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} - working-directory: ./conformance + working-directory: ./tests - name: Run conformance tests run: | - make run-conformance-tests CONFORMANCE_TAG=${{ github.sha }} VERSION=${{ github.ref_name }} + make run-conformance-tests CONFORMANCE_TAG=${{ github.sha }} NGF_VERSION=${{ github.ref_name }} core_result=$(cat conformance-profile.yaml | yq '.profiles[0].core.result') extended_result=$(cat conformance-profile.yaml | yq '.profiles[0].extended.result') if [ "${core_result}" == "failure" ] || [ "${extended_result}" == "failure" ]; then echo "Conformance test failed, see above for details." && exit 2; fi - working-directory: ./conformance + working-directory: ./tests - name: Upload profile to release if: ${{ inputs.k8s-version == 'latest' && startsWith(github.ref, 'refs/tags/') }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: gh release upload ${{ github.ref_name }} conformance-profile.yaml --clobber - working-directory: ./conformance + working-directory: ./tests diff --git a/.github/workflows/functional.yml b/.github/workflows/functional.yml index caa4e3c238..0163227394 100644 --- a/.github/workflows/functional.yml +++ b/.github/workflows/functional.yml @@ -24,6 +24,8 @@ jobs: steps: - name: Checkout Repository uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + with: + fetch-depth: 0 - name: Setup Golang Environment uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1 diff --git a/.github/workflows/release-pr.yml b/.github/workflows/release-pr.yml index b91c5c0e16..2d9c5d7667 100644 --- a/.github/workflows/release-pr.yml +++ b/.github/workflows/release-pr.yml @@ -74,8 +74,6 @@ jobs: sed -i -e "s/v${{ steps.vars.outputs.current_version }}/v${{ inputs.version }}/g" README.md sed -i -e "s/\[${{ steps.vars.outputs.current_version }}\]/\[${{ inputs.version }}\]/g" README.md sed -i -e "s/VERSION = edge/VERSION = ${{ inputs.version }}/g" Makefile - sed -i -e "s/VERSION = ${{ steps.vars.outputs.current_version }}/VERSION = ${{ inputs.version }}/g" conformance/Makefile - sed -i -e "s/TAG = edge/TAG = ${{ inputs.version }}/g" **/Makefile sed -i "6r .github/CHANGELOG_TEMPLATE.md" CHANGELOG.md sed -i -e "s/%%VERSION%%/${{ inputs.version }}/g" CHANGELOG.md make generate-manifests diff --git a/.gitignore b/.gitignore index a87ca2ab36..d2e7d33e22 100644 --- a/.gitignore +++ b/.gitignore @@ -12,7 +12,7 @@ *.out cover.html cmd-cover.html -conformance/conformance-profile.yaml +conformance-profile.yaml # Dependency directories (remove the comment below to include it) # vendor/ diff --git a/Makefile b/Makefile index a372adf11f..b9c042c4db 100644 --- a/Makefile +++ b/Makefile @@ -2,12 +2,14 @@ VERSION = edge GIT_COMMIT = $(shell git rev-parse HEAD || echo "unknown") DATE = $(shell date -u +"%Y-%m-%dT%H:%M:%SZ") +SELF_DIR := $(dir $(lastword $(MAKEFILE_LIST))) MANIFEST_DIR = $(CURDIR)/deploy/manifests -CHART_DIR = $(CURDIR)/charts/nginx-gateway-fabric +CHART_DIR = $(SELF_DIR)charts/nginx-gateway-fabric NGINX_CONF_DIR = internal/mode/static/nginx/conf NJS_DIR = internal/mode/static/nginx/modules/src NGINX_DOCKER_BUILD_PLUS_ARGS = --secret id=nginx-repo.crt,src=nginx-repo.crt --secret id=nginx-repo.key,src=nginx-repo.key BUILD_AGENT=local +PLUS_ENABLED ?= false PROD_TELEMETRY_ENDPOINT = oss.edge.df.f5.com:443 # the telemetry related variables below are also configured in goreleaser.yml @@ -16,7 +18,7 @@ TELEMETRY_ENDPOINT=# if empty, NGF will report telemetry in its logs at debug le TELEMETRY_ENDPOINT_INSECURE = false GW_API_VERSION = 1.1.0 -ENABLE_EXPERIMENTAL = false +ENABLE_EXPERIMENTAL ?= false NODE_VERSION = $(shell cat .nvmrc) # go build flags - should not be overridden by the user @@ -31,18 +33,23 @@ NGINX_PLUS_PREFIX ?= $(PREFIX)/nginx-plus## The name of the nginx plus image. Fo TAG ?= $(VERSION:v%=%)## The tag of the image. For example, 1.1.0 TARGET ?= local## The target of the build. Possible values: local and container KIND_KUBE_CONFIG=$${HOME}/.kube/kind/config## The location of the kind kubeconfig -OUT_DIR ?= $(CURDIR)/build/out## The folder where the binary will be stored +OUT_DIR ?= build/out## The folder where the binary will be stored GOARCH ?= amd64## The architecture of the image and/or binary. For example: amd64 or arm64 GOOS ?= linux## The OS of the image and/or binary. For example: linux or darwin override HELM_TEMPLATE_COMMON_ARGS += --set creator=template --set nameOverride=nginx-gateway## The common options for the Helm template command. override HELM_TEMPLATE_EXTRA_ARGS_FOR_ALL_MANIFESTS_FILE += --set service.create=false## The options to be passed to the full Helm templating command only. override NGINX_DOCKER_BUILD_OPTIONS += --build-arg NJS_DIR=$(NJS_DIR) --build-arg NGINX_CONF_DIR=$(NGINX_CONF_DIR) --build-arg BUILD_AGENT=$(BUILD_AGENT) + .DEFAULT_GOAL := help +ifneq (,$(findstring plus,$(MAKECMDGOALS))) + PLUS_ENABLED = true +endif + .PHONY: help help: Makefile ## Display this help - @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "; printf "Usage:\n\n make \033[36m\033[0m [VARIABLE=value...]\n\nTargets:\n\n"}; {printf " \033[36m%-30s\033[0m %s\n", $$1, $$2}' - @grep -E '^(override )?[a-zA-Z_-]+ \??\+?= .*?## .*$$' $< | sort | awk 'BEGIN {FS = " \\??\\+?= .*?## "; printf "\nVariables:\n\n"}; {gsub(/override /, "", $$1); printf " \033[36m%-30s\033[0m %s\n", $$1, $$2}' + @grep -hE '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "; printf "Usage:\n\n make \033[36m\033[0m [VARIABLE=value...]\n\nTargets:\n\n"}; {printf " \033[36m%-30s\033[0m %s\n", $$1, $$2}' + @grep -hE '^(override )?[a-zA-Z_-]+ \??\+?= .*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = " \\??\\+?= .*?## "; printf "\nVariables:\n\n"}; {gsub(/override /, "", $$1); printf " \033[36m%-30s\033[0m %s\n", $$1, $$2}' .PHONY: build-prod-images build-prod-images: build-prod-ngf-image build-prod-nginx-image ## Build the NGF and nginx docker images for production @@ -62,21 +69,21 @@ build-prod-ngf-image: build-ngf-image ## Build the NGF docker image for producti .PHONY: build-ngf-image build-ngf-image: check-for-docker build ## Build the NGF docker image - docker build --platform linux/$(GOARCH) --build-arg BUILD_AGENT=$(BUILD_AGENT) --target $(strip $(TARGET)) -f build/Dockerfile -t $(strip $(PREFIX)):$(strip $(TAG)) . + docker build --platform linux/$(GOARCH) --build-arg BUILD_AGENT=$(BUILD_AGENT) --target $(strip $(TARGET)) -f $(SELF_DIR)build/Dockerfile -t $(strip $(PREFIX)):$(strip $(TAG)) $(strip $(SELF_DIR)) .PHONY: build-prod-nginx-image build-prod-nginx-image: build-nginx-image ## Build the custom nginx image for production .PHONY: build-nginx-image build-nginx-image: check-for-docker ## Build the custom nginx image - docker build --platform linux/$(GOARCH) $(strip $(NGINX_DOCKER_BUILD_OPTIONS)) -f build/Dockerfile.nginx -t $(strip $(NGINX_PREFIX)):$(strip $(TAG)) . + docker build --platform linux/$(GOARCH) $(strip $(NGINX_DOCKER_BUILD_OPTIONS)) -f $(SELF_DIR)build/Dockerfile.nginx -t $(strip $(NGINX_PREFIX)):$(strip $(TAG)) $(strip $(SELF_DIR)) .PHONY: build-prod-nginx-plus-image build-prod-nginx-plus-image: build-nginx-plus-image ## Build the custom nginx plus image for production .PHONY: build-nginx-plus-image build-nginx-plus-image: check-for-docker ## Build the custom nginx plus image - docker build --platform linux/$(GOARCH) $(strip $(NGINX_DOCKER_BUILD_OPTIONS)) $(strip $(NGINX_DOCKER_BUILD_PLUS_ARGS)) -f build/Dockerfile.nginxplus -t $(strip $(NGINX_PLUS_PREFIX)):$(strip $(TAG)) . + docker build --platform linux/$(GOARCH) $(strip $(NGINX_DOCKER_BUILD_OPTIONS)) $(strip $(NGINX_DOCKER_BUILD_PLUS_ARGS)) -f $(SELF_DIR)build/Dockerfile.nginxplus -t $(strip $(NGINX_PLUS_PREFIX)):$(strip $(TAG)) $(strip $(SELF_DIR)) .PHONY: check-for-docker check-for-docker: ## Check if Docker is installed @@ -86,7 +93,7 @@ check-for-docker: ## Check if Docker is installed build: ## Build the binary ifeq (${TARGET},local) @go version || (code=$$?; printf "\033[0;31mError\033[0m: unable to build locally\n"; exit $$code) - CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH) go build -trimpath -a -ldflags "$(GO_LINKER_FLAGS)" $(ADDITIONAL_GO_BUILD_FLAGS) -o $(OUT_DIR)/gateway github.com/nginxinc/nginx-gateway-fabric/cmd/gateway + CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH) go build -C $(SELF_DIR) -trimpath -a -ldflags "$(GO_LINKER_FLAGS)" $(ADDITIONAL_GO_BUILD_FLAGS) -o $(OUT_DIR)/gateway github.com/nginxinc/nginx-gateway-fabric/cmd/gateway endif .PHONY: build-goreleaser @@ -103,13 +110,25 @@ generate-crds: ## Generate CRDs and Go types using kubebuilder go run sigs.k8s.io/controller-tools/cmd/controller-gen crd object paths=./apis/... output:crd:artifacts:config=config/crd/bases kubectl kustomize config/crd >deploy/crds.yaml +.PHONY: install-crds +install-crds: ## Install CRDs + kubectl kustomize config/crd | kubectl apply -f - + +.PHONY: install-gateway-crds +install-gateway-crds: ## Install Gateway API CRDs + $(SELF_DIR)tests/scripts/install-gateway.sh $(GW_API_VERSION) $(ENABLE_EXPERIMENTAL) + +.PHONY: uninstall-gateway-crds +uninstall-gateway-crds: ## Uninstall Gateway API CRDs + $(SELF_DIR)tests/scripts/uninstall-gateway.sh $(GW_API_VERSION) $(ENABLE_EXPERIMENTAL) + .PHONY: generate-manifests generate-manifests: ## Generate manifests using Helm. helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) $(HELM_TEMPLATE_EXTRA_ARGS_FOR_ALL_MANIFESTS_FILE) -n nginx-gateway | cat $(strip $(MANIFEST_DIR))/namespace.yaml - > $(strip $(MANIFEST_DIR))/nginx-gateway.yaml helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) $(HELM_TEMPLATE_EXTRA_ARGS_FOR_ALL_MANIFESTS_FILE) --set nginx.plus=true --set nginx.image.repository=$(NGINX_PLUS_PREFIX) -n nginx-gateway | cat $(strip $(MANIFEST_DIR))/namespace.yaml - > $(strip $(MANIFEST_DIR))/nginx-plus-gateway.yaml helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) $(HELM_TEMPLATE_EXTRA_ARGS_FOR_ALL_MANIFESTS_FILE) --set nginxGateway.gwAPIExperimentalFeatures.enable=true -n nginx-gateway | cat $(strip $(MANIFEST_DIR))/namespace.yaml - > $(strip $(MANIFEST_DIR))/nginx-gateway-experimental.yaml helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) $(HELM_TEMPLATE_EXTRA_ARGS_FOR_ALL_MANIFESTS_FILE) --set nginxGateway.gwAPIExperimentalFeatures.enable=true --set nginx.plus=true --set nginx.image.repository=$(NGINX_PLUS_PREFIX) -n nginx-gateway | cat $(strip $(MANIFEST_DIR))/namespace.yaml - > $(strip $(MANIFEST_DIR))/nginx-plus-gateway-experimental.yaml - helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) --set metrics.enable=false --set nginxGateway.productTelemetry.enable=false -n nginx-gateway -s templates/deployment.yaml > conformance/provisioner/static-deployment.yaml + helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) --set metrics.enable=false --set nginxGateway.productTelemetry.enable=false -n nginx-gateway -s templates/deployment.yaml > config/tests/static-deployment.yaml helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) -n nginx-gateway -s templates/service.yaml > $(strip $(MANIFEST_DIR))/service/loadbalancer.yaml helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) --set service.annotations.'service\.beta\.kubernetes\.io\/aws-load-balancer-type'="nlb" -n nginx-gateway -s templates/service.yaml > $(strip $(MANIFEST_DIR))/service/loadbalancer-aws-nlb.yaml helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) --set service.type=NodePort --set service.externalTrafficPolicy="" -n nginx-gateway -s templates/service.yaml > $(strip $(MANIFEST_DIR))/service/nodeport.yaml @@ -128,7 +147,7 @@ deps: ## Add missing and remove unused modules, verify deps and download them to .PHONY: create-kind-cluster create-kind-cluster: ## Create a kind cluster - $(eval KIND_IMAGE=$(shell grep -m1 'FROM kindest/node' \033[0m\n\nTargets:\n\n"}; {printf " \033[36m%-30s\033[0m %s\n", $$1, $$2}' - -.PHONY: update-go-modules -update-go-modules: ## Update the gateway-api go modules to latest main version - go get -u sigs.k8s.io/gateway-api@main - go mod tidy - -.PHONY: build-test-runner-image -build-test-runner-image: ## Build conformance test runner image - docker build -t $(CONFORMANCE_PREFIX):$(CONFORMANCE_TAG) -f tests/Dockerfile .. - -.PHONY: create-kind-cluster -create-kind-cluster: ## Create a kind cluster - kind create cluster --image $(KIND_IMAGE) - kind export kubeconfig --kubeconfig $(KIND_KUBE_CONFIG) - -.PHONY: update-ngf-manifest -update-ngf-manifest: ## Update the NGF deployment manifest image names and imagePullPolicies - cd .. && make generate-manifests HELM_TEMPLATE_EXTRA_ARGS_FOR_ALL_MANIFESTS_FILE="--set nginxGateway.kind=skip" HELM_TEMPLATE_COMMON_ARGS="--set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginxGateway.image.pullPolicy=Never --set nginx.image.repository=$(NGINX_PREFIX) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never --set nginxGateway.gwAPIExperimentalFeatures.enable=$(ENABLE_EXPERIMENTAL)" && cd - - -.PHONY: update-ngf-manifest-with-plus -update-ngf-manifest-with-plus: ## Update the NGF deployment manifest image names and imagePullPolicies including nginx-plus - cd .. && make generate-manifests HELM_TEMPLATE_EXTRA_ARGS_FOR_ALL_MANIFESTS_FILE="--set nginxGateway.kind=skip" HELM_TEMPLATE_COMMON_ARGS="--set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginxGateway.image.pullPolicy=Never --set nginx.image.repository=$(NGINX_PLUS_PREFIX) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never --set nginx.plus=true" && cd - - -.PHONY: build-images -build-images: ## Build NGF and NGINX images - cd .. && make PREFIX=$(PREFIX) TAG=$(TAG) build-images - -.PHONY: load-images -load-images: ## Load NGF and NGINX images on configured kind cluster - cd .. && make PREFIX=$(PREFIX) TAG=$(TAG) load-images - -.PHONY: build-images-with-plus -build-images-with-plus: ## Build NGF and NGINX Plus images - cd .. && make PREFIX=$(PREFIX) TAG=$(TAG) build-images-with-plus - -.PHONY: load-images-with-plus -load-images-with-plus: ## Load NGF and NGINX Plus images on configured kind cluster - cd .. && make PREFIX=$(PREFIX) TAG=$(TAG) load-images-with-plus - -.PHONY: prepare-ngf-dependencies -prepare-ngf-dependencies: update-ngf-manifest ## Install NGF dependencies on configured kind cluster - ./scripts/install-gateway.sh $(GW_API_VERSION) $(ENABLE_EXPERIMENTAL) - kubectl kustomize ../config/crd | kubectl apply -f - - kubectl apply -f $(NGF_MANIFEST) - -.PHONY: prepare-ngf-dependencies-with-plus -prepare-ngf-dependencies-with-plus: update-ngf-manifest-with-plus ## Install NGF dependencies with Plus on configured kind cluster - ./scripts/install-gateway.sh $(GW_API_VERSION) - kubectl kustomize ../config/crd | kubectl apply -f - - kubectl apply -f $(NGF_MANIFEST) - -.PHONY: deploy-updated-provisioner -deploy-updated-provisioner: ## Update provisioner manifest and deploy to the configured kind cluster - yq '(select(di != 3))' $(PROVISIONER_MANIFEST) | kubectl apply -f - - yq '(select(.spec.template.spec.containers[].image) | .spec.template.spec.containers[].image="$(PREFIX):$(TAG)" | .spec.template.spec.containers[].imagePullPolicy = "Never")' $(PROVISIONER_MANIFEST) | kubectl apply -f - - -.PHONY: install-ngf-local-build -install-ngf-local-build: prepare-ngf-dependencies build-images load-images deploy-updated-provisioner ## Install NGF from local build with provisioner on configured kind cluster - -.PHONY: install-ngf-local-no-build -install-ngf-local-no-build: prepare-ngf-dependencies load-images deploy-updated-provisioner ## Install NGF from local build with provisioner on configured kind cluster but do not build the NGF image - -.PHONY: install-ngf-local-build-with-plus -install-ngf-local-build-with-plus: prepare-ngf-dependencies-with-plus build-images-with-plus load-images-with-plus deploy-updated-provisioner ## Install NGF with Plus from local build with provisioner on configured kind cluster - -.PHONY: install-ngf-local-no-build-with-plus -install-ngf-local-no-build-with-plus: prepare-ngf-dependencies-with-plus load-images-with-plus deploy-updated-provisioner ## Install NGF with Plus from local build with provisioner on configured kind cluster but do not build the NGF image - -.PHONY: install-ngf-edge -install-ngf-edge: prepare-ngf-dependencies ## Install NGF with provisioner from edge on configured kind cluster - kubectl apply -f $(PROVISIONER_MANIFEST) - -.PHONY: run-conformance-tests -run-conformance-tests: ## Run conformance tests - kind load docker-image $(CONFORMANCE_PREFIX):$(CONFORMANCE_TAG) - kubectl apply -f tests/conformance-rbac.yaml - kubectl run -i conformance \ - --image=$(CONFORMANCE_PREFIX):$(CONFORMANCE_TAG) --image-pull-policy=Never \ - --overrides='{ "spec": { "serviceAccountName": "conformance" } }' \ - --restart=Never -- sh -c "go test -v . -tags conformance,experimental -args --gateway-class=$(GATEWAY_CLASS) \ - --supported-features=$(SUPPORTED_FEATURES) --version=$(VERSION) \ - --report-output=output.txt; cat output.txt" | tee output.txt - bash scripts/check-pod-exit-code.sh - sed -e '1,/CONFORMANCE PROFILE/d' output.txt > conformance-profile.yaml - rm output.txt - grpc_core_result=`yq '.profiles[0].core.result' conformance-profile.yaml`; \ - http_core_result=`yq '.profiles[1].core.result' conformance-profile.yaml`; \ - http_extended_result=`yq '.profiles[1].extended.result' conformance-profile.yaml`; \ - if [ "$$grpc_core_result" != "failure" ] && [ "$$http_core_result" != "failure" ] && [ "$$http_extended_result" != "failure" ] ; then \ - exit 0; \ - else \ - exit 2; \ - fi - -.PHONY: cleanup-conformance-tests -cleanup-conformance-tests: ## Clean up conformance tests fixtures - kubectl delete pod conformance - kubectl delete -f tests/conformance-rbac.yaml - -.PHONY: uninstall-ngf -uninstall-ngf: uninstall-k8s-components undo-manifests-update ## Uninstall NGF on configured kind cluster and undo manifest changes - -.PHONY: uninstall-k8s-components -uninstall-k8s-components: ## Uninstall installed components on configured kind cluster - -kubectl delete -f $(NGF_MANIFEST) - ./scripts/uninstall-gateway.sh $(GW_API_VERSION) $(ENABLE_EXPERIMENTAL) - kubectl delete clusterrole nginx-gateway-provisioner - kubectl delete clusterrolebinding nginx-gateway-provisioner - -.PHONY: undo-manifests-update -undo-manifests-update: ## Undo the changes in the manifest files - cd .. && make generate-manifests && cd - - -.PHONY: reset-go-modules -reset-go-modules: ## Reset the go modules changes - git checkout -- ../go.mod ../go.sum - -.PHONY: delete-kind-cluster -delete-kind-cluster: ## Delete kind cluster - kind delete cluster diff --git a/conformance/README.md b/conformance/README.md deleted file mode 100644 index fd4f316eb3..0000000000 --- a/conformance/README.md +++ /dev/null @@ -1,194 +0,0 @@ -# Running [Gateway Conformance Tests](https://gateway-api.sigs.k8s.io/concepts/conformance/#3-conformance-tests) in kind - -## Prerequisites - -- [kind](https://kind.sigs.k8s.io/). -- Docker. -- Golang. -- [yq](https://github.com/mikefarah/yq/#install) - -**Note**: all commands in steps below are executed from the ```conformance``` directory - -List available commands: - -```shell -make -``` - -```text -build-images-with-plus Build NGF and NGINX Plus images -build-images Build NGF and NGINX images -build-test-runner-image Build conformance test runner image -cleanup-conformance-tests Clean up conformance tests fixtures -create-kind-cluster Create a kind cluster -delete-kind-cluster Delete kind cluster -deploy-updated-provisioner Update provisioner manifest and deploy to the configured kind cluster -help Display this help -install-ngf-edge Install NGF with provisioner from edge on configured kind cluster -install-ngf-local-build-with-plus Install NGF with Plus from local build with provisioner on configured kind cluster -install-ngf-local-build Install NGF from local build with provisioner on configured kind cluster -install-ngf-local-no-build-with-plus Install NGF with Plus from local build with provisioner on configured kind cluster but do not build the NGF image -install-ngf-local-no-build Install NGF from local build with provisioner on configured kind cluster but do not build the NGF image -load-images-with-plus Load NGF and NGINX Plus images on configured kind cluster -load-images Load NGF and NGINX images on configured kind cluster -prepare-ngf-dependencies-with-plus Install NGF dependencies with Plus on configured kind cluster -prepare-ngf-dependencies Install NGF dependencies on configured kind cluster -reset-go-modules Reset the go modules changes -run-conformance-tests Run conformance tests -undo-manifests-update Undo the changes in the manifest files -uninstall-ngf Uninstall NGF on configured kind cluster and undo manifest changes -update-go-modules Update the gateway-api go modules to latest main version -update-ngf-manifest-with-plus Update the NGF deployment manifest image names and imagePullPolicies including nginx-plus -update-ngf-manifest Update the NGF deployment manifest image names and imagePullPolicies -``` - -**Note:** The following variables are configurable when running the below `make` commands: - -| Variable | Default | Description | -| -------------------- | ------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- | -| CONFORMANCE_TAG | latest | The tag for the conformance test image | -| CONFORMANCE_PREFIX | conformance-test-runner | The prefix for the conformance test image | -| TAG | edge | The tag for the locally built NGF image | -| PREFIX | nginx-gateway-fabric | The prefix for the locally built NGF image | -| GW_API_VERSION | 1.1.0 | Tag for the Gateway API version to check out. Set to `main` to get the latest version | -| KIND_IMAGE | Latest kind image, as defined in the tests/Dockerfile | The kind image to use | -| KIND_KUBE_CONFIG | ~/.kube/kind/config | The location of the kubeconfig | -| GATEWAY_CLASS | nginx | The gateway class that should be used for the tests | -| SUPPORTED_FEATURES | HTTPRoute,HTTPRouteQueryParamMatching, HTTPRouteMethodMatching,HTTPRoutePortRedirect, HTTPRouteSchemeRedirect | The supported features that should be tested by the conformance tests. Ensure the list is comma separated with no spaces. | -| EXEMPT_FEATURES | ReferenceGrant | The features that should not be tested by the conformance tests | -| NGF_MANIFEST | ../deploy/manifests/nginx-gateway.yaml | The location of the NGF manifest | -| SERVICE_MANIFEST | ../deploy/manifests/service/nodeport.yaml | The location of the NGF Service manifest | -| STATIC_MANIFEST | provisioner/static-deployment.yaml | The location of the NGF static deployment manifest | -| PROVISIONER_MANIFEST | provisioner/provisioner.yaml | The location of the NGF provisioner manifest | -| ENABLE_EXPERIMENTAL | false | Enable experimental features. Installs the Gateway APIs from the experimental channel and enables any supported experimental features in NGF. | - -### Step 1 - Create a kind Cluster - -```makefile -make create-kind-cluster -``` - -> Note: The default kind cluster deployed is the latest available version. You can specify a different version by -> defining the kind image to use through the KIND_IMAGE variable, e.g. - -```makefile -make create-kind-cluster KIND_IMAGE=kindest/node:v1.27.3 -``` - -### Step 2 - Install NGINX Gateway Fabric to configured kind cluster - -> Note: If you want to run the latest conformance tests from the Gateway API `main` branch, set the following -> environment variable before deploying NGF: - -```bash - export GW_API_VERSION=main -``` - -> Otherwise, the latest stable version will be used by default. -> Additionally, if you want to run conformance tests with experimental features enabled, set the following -> environment variable before deploying NGF: - -```bash - export ENABLE_EXPERIMENTAL=true -``` - -#### *Option 1* Build and install NGINX Gateway Fabric from local to configured kind cluster - -```makefile -make install-ngf-local-build -``` - -Or, to install NGF with NGINX Plus enabled (NGINX Plus cert and key must exist in the root of the repo): - -```makefile -make install-ngf-local-build-with-plus -``` - -#### *Option 2* Install NGINX Gateway Fabric from local already built image to configured kind cluster - -You can optionally skip the actual *build* step. - -```makefile -make install-ngf-local-no-build -``` - -Or, to install NGF with NGINX Plus enabled: - -```makefile -make install-ngf-no-build-with-plus -``` - -> Note: If choosing this option, the following step *must* be completed manually *before* you build the image: - -```makefile -make update-ngf-manifest PREFIX= TAG= -``` - -Or, if you are building the NGINX Plus image: - -```makefile -make update-ngf-manifest-with-plus PREFIX= TAG= -``` - -#### *Option 3* Install NGINX Gateway Fabric from edge to configured kind cluster - -You can also skip the build NGF image step and prepare the environment to instead use the `edge` image. Note that this -option does not currently support installing with NGINX Plus enabled. - -```makefile -make install-ngf-edge -``` - -### Step 3 - Build conformance test runner image - -> Note: If you want to run the latest conformance tests from the Gateway API `main` branch, run the following -> make command to update the Go modules to `main`: - - ```makefile - make update-go-modules - ``` - -> You can also point to a specific fork/branch by running: - - ```bash - go mod edit -replace=sigs.k8s.io/gateway-api=@ - go mod download - go mod verify - go mod tidy - ``` - -> Otherwise, the latest stable version will be used by default. - -```makefile -make build-test-runner-image -``` - -### Step 4 - Run Gateway conformance tests - -```makefile -make run-conformance-tests -``` - -### Step 5 - Cleanup the conformance test fixtures and uninstall NGINX Gateway Fabric - -```makefile -make cleanup-conformance-tests -``` - -```makefile -make uninstall-ngf -``` - -### Step 6 - Revert changes to Go modules - -**Optional** Not required if you aren't running the `main` Gateway API tests. - -```makefile -make reset-go-modules -``` - -### Step 7 - Delete kind cluster - -```makefile -make delete-kind-cluster -``` diff --git a/conformance/tests/Dockerfile b/conformance/tests/Dockerfile deleted file mode 100644 index 1d64d1624c..0000000000 --- a/conformance/tests/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -# syntax=docker/dockerfile:1.7 -# this is here so we can grab the latest version of kind and have dependabot keep it up to date -FROM kindest/node:v1.29.2 - -FROM golang:1.22 - -WORKDIR /go/src/github.com/nginxinc/nginx-gateway-fabric/conformance/tests/ - -COPY --link go.mod /go/src/github.com/nginxinc/nginx-gateway-fabric/ -COPY --link go.sum /go/src/github.com/nginxinc/nginx-gateway-fabric/ -RUN go mod download - -COPY --link conformance/tests /go/src/github.com/nginxinc/nginx-gateway-fabric/conformance/tests/ diff --git a/docs/developer/quickstart.md b/docs/developer/quickstart.md index 245a285029..3a8dc3dd87 100644 --- a/docs/developer/quickstart.md +++ b/docs/developer/quickstart.md @@ -195,7 +195,7 @@ For more details on testing, see the [testing](/docs/developer/testing.md) docum ## Gateway API Conformance Testing -To run Gateway API conformance tests, please follow the instructions on [this](/conformance/README.md) page. +To run Gateway API conformance tests, please follow the instructions on [this](/tests/README.md#conformance-testing) page. ## Run the Linter diff --git a/docs/developer/release-process.md b/docs/developer/release-process.md index ea961b93b7..c986176e06 100644 --- a/docs/developer/release-process.md +++ b/docs/developer/release-process.md @@ -36,7 +36,7 @@ To create a new release, follow these steps: 6. If a compatibility issue is found, add a note to the release notes explaining that the previous version is not supported. 7. Create a release branch following the `release-X.Y` naming convention. 8. Run the [Release PR](./../../.github/workflows/release-pr.yml) workflow to update the repo files for the release. Then there are a few manual steps to complete: - 1. Update the tag of NGF container images used in the [provisioner manifest](/conformance/provisioner/provisioner.yaml). + 1. Update the tag of NGF container images used in the [provisioner manifest](/tests/conformance/provisioner/provisioner.yaml). 2. Update any installation instructions to ensure that the supported Gateway API is correct. Specifically, helm README and `site/content/includes/installation/install-gateway-api-resources.md`. 3. Update the [README](/README.md) to include information about the release. diff --git a/docs/developer/testing.md b/docs/developer/testing.md index d7f4f50296..205ae8f37b 100644 --- a/docs/developer/testing.md +++ b/docs/developer/testing.md @@ -103,4 +103,4 @@ submitting them for review and integration into the project. ## Gateway API Conformance Testing -To run Gateway API conformance tests, please follow the instructions on [this](/conformance/README.md) page. +To run Gateway API conformance tests, please follow the instructions on [this](/tests/README.md#conformance-testing) page. diff --git a/embedded.go b/embedded.go index 8c24e6db65..0147f76f0e 100644 --- a/embedded.go +++ b/embedded.go @@ -4,8 +4,8 @@ import _ "embed" // StaticModeDeploymentYAML contains the YAML manifest of the Deployment resource for the static mode. // We put this in the root of the repo because goembed doesn't support relative/absolute paths and symlinks, -// and we want to keep the static mode deployment manifest for the provisioner in the conformance/provisioner/ +// and we want to keep the static mode deployment manifest for the provisioner in the config/tests/ // directory. // -//go:embed conformance/provisioner/static-deployment.yaml +//go:embed config/tests/static-deployment.yaml var StaticModeDeploymentYAML []byte diff --git a/go.mod b/go.mod index d1705fd7d8..2f54b0cc62 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,6 @@ require ( sigs.k8s.io/controller-runtime v0.18.2 sigs.k8s.io/controller-tools v0.15.0 sigs.k8s.io/gateway-api v1.1.0 - sigs.k8s.io/yaml v1.4.0 ) require ( @@ -54,7 +53,6 @@ require ( github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/gorilla/websocket v1.5.1 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -63,17 +61,12 @@ require ( github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/miekg/dns v1.1.58 // indirect - github.com/moby/spdystream v0.2.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.6.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect - github.com/stretchr/testify v1.9.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0 // indirect go.opentelemetry.io/otel/metric v1.26.0 // indirect go.opentelemetry.io/otel/sdk v1.26.0 // indirect @@ -103,4 +96,5 @@ require ( k8s.io/utils v0.0.0-20240423183400-0849a56e8f22 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect ) diff --git a/go.sum b/go.sum index ead43772d6..2c96a1d165 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,3 @@ -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= @@ -59,9 +57,6 @@ github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 h1:k7nVchz72niMH6YLQN github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= -github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 h1:/c3QmbOGMGTOumP2iT/rCwB7b0QDGLKzqOmktBjT+Is= github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1/go.mod h1:5SN9VR2LTsRFsrEC6FHgRbTWrTHu6tqPeKxEQv15giM= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= @@ -87,10 +82,6 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1 h1:NicmruxkeqHjDv03SfSxqmaLuisddudfP3h5wdXFbhM= github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1/go.mod h1:eyp4DdUJAKkr9tvxR3jWhw2mDK7CWABMG5r9uyaKC7I= -github.com/miekg/dns v1.1.58 h1:ca2Hdkz+cDg/7eNF6V56jjzuZ4aCAE+DbVkILdQWG/4= -github.com/miekg/dns v1.1.58/go.mod h1:Ypv+3b/KadlvW9vJfXOTf300O4UqaHFzFCuHz+rPkBY= -github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= -github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -98,8 +89,6 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/nginxinc/nginx-plus-go-client v1.2.0 h1:NVfRsHbMJ7lOhkqMG52uvODiDBhQZNp20c0tV2lU3wg= github.com/nginxinc/nginx-plus-go-client v1.2.0/go.mod h1:n8OFLzrJulJ2fur28Cwa1Qp5DZNS2VicLV+Adt30LQ4= github.com/nginxinc/nginx-prometheus-exporter v1.1.0 h1:Uj+eWKGvUionZc8gWFDnrb3jpdkuZAlPKo4ck96cOmE= diff --git a/tests/Dockerfile b/tests/Dockerfile new file mode 100644 index 0000000000..4994cd329f --- /dev/null +++ b/tests/Dockerfile @@ -0,0 +1,3 @@ +# syntax=docker/dockerfile:1.7 +# this is here so we can grab the latest version of kind and have dependabot keep it up to date +FROM kindest/node:v1.29.2 diff --git a/tests/Makefile b/tests/Makefile index 947fe6812e..4e4516d906 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,59 +1,73 @@ -TAG = edge -PREFIX = nginx-gateway-fabric -NGINX_PREFIX = $(PREFIX)/nginx -NGINX_PLUS_PREFIX = $(PREFIX)/nginx-plus -PLUS_ENABLED=false -PULL_POLICY=Never -GW_API_PREV_VERSION ?= 1.0.0 ## Supported Gateway API version from previous NGF release -GW_API_VERSION ?= 1.1.0 ## Supported Gateway API version for NGF under test -K8S_VERSION ?= latest ## Expected format: 1.24 (major.minor) or latest -GW_SERVICE_TYPE=NodePort -GW_SVC_GKE_INTERNAL=false -GINKGO_LABEL= -GINKGO_FLAGS= -NGF_VERSION= CI=false -TELEMETRY_ENDPOINT= -TELEMETRY_ENDPOINT_INSECURE=false +CONFORMANCE_PREFIX = conformance-test-runner## Prefix for the conformance test runner image +CONFORMANCE_TAG = latest## Tag for the conformance test runner image +GATEWAY_CLASS = nginx## Gateway class to use +GINKGO_FLAGS = +GINKGO_LABEL = +GW_API_PREV_VERSION ?= 1.0.0## Supported Gateway API version from previous NGF release +GW_SERVICE_TYPE=NodePort## Service type to use for the gateway +GW_SVC_GKE_INTERNAL=false +K8S_VERSION ?= latest## Kubernetes version to use. Expected format: 1.24 (major.minor) or latest +NGF_VERSION ?= $(shell git describe --tags $(shell git rev-list --tags --max-count=1))## NGF version to be tested (defaults to latest tag) +PULL_POLICY=Never## Pull policy for the images +PROVISIONER_MANIFEST = conformance/provisioner/provisioner.yaml +SUPPORTED_FEATURES = HTTPRouteQueryParamMatching,HTTPRouteMethodMatching,HTTPRoutePortRedirect,HTTPRouteSchemeRedirect,HTTPRouteHostRewrite,HTTPRoutePathRewrite,GatewayPort8080,HTTPRouteResponseHeaderModification,GRPCExactMethodMatching,GRPCRouteListenerHostnameMatching,GRPCRouteHeaderMatching ifneq ($(GINKGO_LABEL),) override GINKGO_FLAGS += --label-filter "$(GINKGO_LABEL)" endif +.PHONY: update-go-modules +update-go-modules: ## Update the gateway-api go modules to latest main version + go get -u sigs.k8s.io/gateway-api@main + go mod tidy + +.PHONY: build-test-runner-image +build-test-runner-image: ## Build conformance test runner image + docker build -t $(CONFORMANCE_PREFIX):$(CONFORMANCE_TAG) -f conformance/Dockerfile . + +.PHONY: run-conformance-tests +run-conformance-tests: ## Run conformance tests + kind load docker-image $(CONFORMANCE_PREFIX):$(CONFORMANCE_TAG) + kubectl apply -f conformance/conformance-rbac.yaml + kubectl run -i conformance \ + --image=$(CONFORMANCE_PREFIX):$(CONFORMANCE_TAG) --image-pull-policy=Never \ + --overrides='{ "spec": { "serviceAccountName": "conformance" } }' \ + --restart=Never -- sh -c "go test -v . -tags conformance,experimental -args --gateway-class=$(GATEWAY_CLASS) \ + --supported-features=$(SUPPORTED_FEATURES) --version=$(NGF_VERSION) \ + --report-output=output.txt; cat output.txt" | tee output.txt + bash scripts/check-pod-exit-code.sh + sed -e '1,/CONFORMANCE PROFILE/d' output.txt > conformance-profile.yaml + rm output.txt + grpc_core_result=`yq '.profiles[0].core.result' conformance-profile.yaml`; \ + http_core_result=`yq '.profiles[1].core.result' conformance-profile.yaml`; \ + http_extended_result=`yq '.profiles[1].extended.result' conformance-profile.yaml`; \ + if [ "$$grpc_core_result" != "failure" ] && [ "$$http_core_result" != "failure" ] && [ "$$http_extended_result" != "failure" ] ; then \ + exit 0; \ + else \ + exit 2; \ + fi + +.PHONY: cleanup-conformance-tests +cleanup-conformance-tests: ## Clean up conformance tests fixtures + kubectl delete pod conformance + kubectl delete -f tests/conformance-rbac.yaml + +.PHONY: build +build: generate-static-deployment + +.PHONY: reset-go-modules +reset-go-modules: ## Reset the go modules changes + git checkout -- ../go.mod ../go.sum + +-include ../Makefile + # Check if PLUS_ENABLED is true ifeq ($(PLUS_ENABLED),true) # If true, set NGINX_PREFIX to $NGINX_PLUS_PREFIX NGINX_PREFIX := $(NGINX_PLUS_PREFIX) endif -.PHONY: help -help: Makefile ## Display this help - @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "; printf "Usage:\n\n make \033[36m\033[0m\n\nTargets:\n\n"}; {printf " \033[36m%-30s\033[0m %s\n", $$1, $$2}' - -.PHONY: create-kind-cluster -create-kind-cluster: ## Create a kind cluster - cd .. && make create-kind-cluster - -.PHONY: delete-kind-cluster -delete-kind-cluster: ## Delete kind cluster - kind delete cluster - -.PHONY: build-images -build-images: ## Build NGF and NGINX images - cd .. && make PREFIX=$(PREFIX) TAG=$(TAG) TELEMETRY_ENDPOINT=$(TELEMETRY_ENDPOINT) TELEMETRY_ENDPOINT_INSECURE=$(TELEMETRY_ENDPOINT_INSECURE) build-images - -.PHONY: build-images-with-plus -build-images-with-plus: ## Build NGF and NGINX Plus images - cd .. && make PREFIX=$(PREFIX) TAG=$(TAG) TELEMETRY_ENDPOINT=$(TELEMETRY_ENDPOINT) TELEMETRY_ENDPOINT_INSECURE=$(TELEMETRY_ENDPOINT_INSECURE) build-images-with-plus - -.PHONY: load-images -load-images: ## Load NGF and NGINX images on configured kind cluster - cd .. && make PREFIX=$(PREFIX) TAG=$(TAG) load-images - -.PHONY: load-images-with-plus -load-images-with-plus: ## Load NGF and NGINX Plus images on configured kind cluster - cd .. && make PREFIX=$(PREFIX) TAG=$(TAG) load-images-with-plus - .PHONY: setup-gcp-and-run-tests setup-gcp-and-run-tests: create-gke-router create-and-setup-vm run-tests-on-vm ## Create and setup a GKE router and GCP VM for tests and run the functional tests @@ -85,12 +99,12 @@ nfr-test: ## Run the NFR tests on a GCP VM NFR=true bash scripts/run-tests-gcp-vm.sh .PHONY: start-longevity-test -start-longevity-test: ## Start the longevity test to run for 4 days in GKE - START_LONGEVITY=true $(MAKE) nfr-test +start-longevity-test: START_LONGEVITY=true +start-longevity-test: nfr-test ## Start the longevity test to run for 4 days in GKE .PHONY: stop-longevity-test -stop-longevity-test: ## Stops the longevity test and collects results - STOP_LONGEVITY=true $(MAKE) nfr-test +stop-longevity-test: STOP_LONGEVITY=true +stop-longevity-test: nfr-test ## Stop the longevity test and collects results .PHONY: .vm-nfr-test .vm-nfr-test: ## Runs the NFR tests on the GCP VM (called by `nfr-test`) @@ -112,8 +126,8 @@ test: ## Runs the functional tests on your default k8s cluster --is-gke-internal-lb=$(GW_SVC_GKE_INTERNAL) .PHONY: test-with-plus -test-with-plus: ## Runs the functional tests for NGF with NGINX Plus on your default k8s cluster - make test PLUS_ENABLED=true +test-with-plus: PLUS_ENABLED=true +test-with-plus: test ## Runs the functional tests for NGF with NGINX Plus on your default k8s cluster .PHONY: cleanup-gcp cleanup-gcp: cleanup-router cleanup-vm delete-gke-cluster ## Cleanup all GCP resources @@ -133,3 +147,39 @@ delete-gke-cluster: ## Delete the GKE cluster .PHONY: add-local-ip-to-cluster add-local-ip-to-cluster: ## Add local IP to the GKE cluster master-authorized-networks bash scripts/add-local-ip-auth-networks.sh + +HELM_PARAMETERS += --set nameOverride=nginx-gateway --set nginxGateway.kind=skip --set service.create=false + +.PHONY: deploy-updated-provisioner +deploy-updated-provisioner: ## Update provisioner manifest and deploy to the configured kind cluster + yq '(select(di != 3))' $(PROVISIONER_MANIFEST) | kubectl apply -f - + yq '(select(.spec.template.spec.containers[].image) | .spec.template.spec.containers[].image="$(PREFIX):$(TAG)" | .spec.template.spec.containers[].imagePullPolicy = "Never")' $(PROVISIONER_MANIFEST) | kubectl apply -f - + +.PHONY: generate-static-deployment +generate-static-deployment: + helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) --set metrics.enable=false --set nginxGateway.productTelemetry.enable=false -n nginx-gateway -s templates/deployment.yaml --set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginxGateway.image.pullPolicy=Never --set nginx.image.repository=$(NGINX_PREFIX) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never --set nginxGateway.gwAPIExperimentalFeatures.enable=$(ENABLE_EXPERIMENTAL) --set nginx.plus=$(PLUS_ENABLED) > $(SELF_DIR)config/tests/static-deployment.yaml + +.PHONY: install-ngf-local-build +install-ngf-local-build: deploy-updated-provisioner + +.PHONY: install-ngf-local-no-build +install-ngf-local-no-build: load-images helm-install-local deploy-updated-provisioner ## Install NGF from local build with provisioner on configured kind cluster but do not build the NGF image + +.PHONY: install-ngf-local-build-with-plus +install-ngf-local-build-with-plus: deploy-updated-provisioner + +.PHONY: install-ngf-local-no-build-with-plus +install-ngf-local-no-build-with-plus: load-images-with-plus helm-install-local-with-plus deploy-updated-provisioner ## Install NGF with Plus from local build with provisioner on configured kind cluster but do not build the NGF image + +.PHONY: install-ngf-edge +install-ngf-edge: load-images helm-install-local ## Install NGF with provisioner from edge on configured kind cluster + kubectl apply -f $(PROVISIONER_MANIFEST) + +.PHONY: uninstall-ngf +uninstall-ngf: ## Uninstall NGF on configured kind cluster and undo manifest changes + -helm uninstall nginx-gateway -n nginx-gateway + -make uninstall-gateway-crds + -kubectl delete clusterrole nginx-gateway-provisioner + -kubectl delete clusterrolebinding nginx-gateway-provisioner + -kubectl delete namespace nginx-gateway + -kubectl kustomize ../config/crd | kubectl delete -f - diff --git a/tests/README.md b/tests/README.md index c94d8a3a85..541b898914 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,31 +1,46 @@ -# System Testing - -The tests in this directory are meant to be run on a live Kubernetes environment to verify a real system. These -are similar to the existing [conformance tests](../conformance/README.md), but will verify things such as: - -- NGF-specific functionality -- Non-Functional requirements (NFR) testing (such as performance, scale, etc.) - -When running locally, the tests create a port-forward from your NGF Pod to localhost using a port chosen by the -test framework. Traffic is sent over this port. If running on a GCP VM targeting a GKE cluster, the tests will create an -internal LoadBalancer service which will receive the test traffic. - -**Important**: NFR tests can only be run on a GKE cluster. - -Directory structure is as follows: - -- `framework`: contains utility functions for running the tests -- `results`: contains the results files for the NFR tests -- `scripts`: contain scripts used to set up the environment and run the tests -- `suite`: contains the test files - -> Note: Existing NFR tests will be migrated into this testing `suite` and results stored in the `results` directory. +# NGINX Gateway Fabric Testing + +## Overview + +This directory contains the tests for NGINX Gateway Fabric. The tests are divided into two categories: + +1. Conformance Testing. This is to ensure that the NGINX Gateway Fabric conforms to the Gateway API specification. +2. System Testing. This is to ensure that the NGINX Gateway Fabric works as expected in a real system. + +## Table of Contents + +- [Prerequisites](#prerequisites) +- [Common steps for all tests](#common-steps-for-all-tests) + - [Step 1 - Create a Kubernetes cluster](#step-1---create-a-kubernetes-cluster) + - [Step 2 - Build and Load Images](#step-2---build-and-load-images) +- [Conformance Testing](#conformance-testing) + - [Step 1 - Install NGINX Gateway Fabric to configured kind cluster](#step-1---install-nginx-gateway-fabric-to-configured-kind-cluster) + - [_Option 1_ Build and install NGINX Gateway Fabric from local to configured kind cluster](#option-1-build-and-install-nginx-gateway-fabric-from-local-to-configured-kind-cluster) + - [_Option 2_ Install NGINX Gateway Fabric from local already built image to configured kind cluster](#option-2-install-nginx-gateway-fabric-from-local-already-built-image-to-configured-kind-cluster) + - [_Option 3_ Install NGINX Gateway Fabric from edge to configured kind cluster](#option-3-install-nginx-gateway-fabric-from-edge-to-configured-kind-cluster) + - [Step 2 - Build conformance test runner image](#step-2---build-conformance-test-runner-image) + - [Step 3 - Run Gateway conformance tests](#step-3---run-gateway-conformance-tests) + - [Step 4 - Cleanup the conformance test fixtures and uninstall NGINX Gateway Fabric](#step-4---cleanup-the-conformance-test-fixtures-and-uninstall-nginx-gateway-fabric) + - [Step 5 - Revert changes to Go modules](#step-5---revert-changes-to-go-modules) + - [Step 6 - Delete kind cluster](#step-6---delete-kind-cluster) +- [System Testing](#system-testing) + - [Step 1 - Run the tests](#step-1---run-the-tests) + - [1a - Run the functional tests locally](#1a---run-the-functional-tests-locally) + - [1b - Run the tests on a GKE cluster from a GCP VM](#1b---run-the-tests-on-a-gke-cluster-from-a-gcp-vm) + - [Functional Tests](#functional-tests) + - [NFR tests](#nfr-tests) + - [Longevity testing](#longevity-testing) + - [Common test amendments](#common-test-amendments) + - [Step 2 - Cleanup](#step-2---cleanup) ## Prerequisites - Kubernetes cluster. +- [kind](https://kind.sigs.k8s.io/). - Docker. - Golang. +- [yq](https://github.com/mikefarah/yq/#install) +- Make. If running NFR tests, or running functional tests in GKE: @@ -33,60 +48,11 @@ If running NFR tests, or running functional tests in GKE: - A GKE cluster (if `master-authorized-networks` is enabled, please set `ADD_VM_IP_AUTH_NETWORKS=true` in your vars.env file) - Access to GCP Service Account with Kubernetes admin permissions -> Note: all commands in steps below are executed from the `tests` directory - -```shell -make -``` - -```text -add-local-ip-to-cluster Add local IP to the GKE cluster master-authorized-networks -build-images-with-plus Build NGF and NGINX Plus images -build-images Build NGF and NGINX images -cleanup-gcp Cleanup all GCP resources -cleanup-router Delete the GKE router -cleanup-vm Delete the test GCP VM and delete the firewall rule -create-and-setup-vm Create and setup a GCP VM for tests -create-gke-cluster Create a GKE cluster -create-gke-router Create a GKE router to allow egress traffic from private nodes (allows for external image pulls) -create-kind-cluster Create a kind cluster -delete-gke-cluster Delete the GKE cluster -delete-kind-cluster Delete kind cluster -help Display this help -load-images-with-plus Load NGF and NGINX Plus images on configured kind cluster -load-images Load NGF and NGINX images on configured kind cluster -nfr-test Run the NFR tests on a GCP VM -run-tests-on-vm Run the functional tests on a GCP VM -setup-gcp-and-run-nfr-tests Create and setup a GKE router and GCP VM for tests and run the NFR tests -setup-gcp-and-run-tests Create and setup a GKE router and GCP VM for tests and run the functional tests -start-longevity-test Start the longevity test to run for 4 days in GKE -stop-longevity-test Stops the longevity test and collects results -sync-files-to-vm Syncs your local NGF files with the NGF repo on the VM -test Runs the functional tests on your default k8s cluster -test-with-plus Runs the functional tests for NGF with NGINX Plus on your default k8s cluster -``` - -**Note:** The following variables are configurable when running the below `make` commands: - -| Variable | Default | Description | -|------------------------------|---------------------------------|---------------------------------------------------------------------| -| TAG | edge | tag for the locally built NGF images | -| PREFIX | nginx-gateway-fabric | prefix for the locally built NGF image | -| NGINX_PREFIX | nginx-gateway-fabric/nginx | prefix for the locally built NGINX image | -| NGINX_PLUS_PREFIX | nginx-gateway-fabric/nginx-plus | prefix for the locally built NGINX Plus image | -| PLUS_ENABLED | false | Flag to indicate if NGINX Plus should be enabled | -| PULL_POLICY | Never | NGF image pull policy | -| GW_API_VERSION | 1.1.0 | version of Gateway API resources to install | -| K8S_VERSION | latest | version of k8s that the tests are run on | -| GW_SERVICE_TYPE | NodePort | type of Service that should be created | -| GW_SVC_GKE_INTERNAL | false | specifies if the LoadBalancer should be a GKE internal service | -| GINKGO_LABEL | "" | name of the ginkgo label that will filter the tests to run | -| GINKGO_FLAGS | "" | other ginkgo flags to pass to the go test command | -| TELEMETRY_ENDPOINT | Set in the main Makefile | The endpoint to which telemetry reports are sent | -| TELEMETRY_ENDPOINT_INSECURE= | Set in the main Makefile | Controls whether TLS should be used when sending telemetry reports. | - - -## Step 1 - Create a Kubernetes cluster +All the commands below are executed from the `tests` directory. You can see all the available commands by running `make help`. + +## Common steps for all tests + +### Step 1 - Create a Kubernetes cluster This can be done in a cloud provider of choice, or locally using `kind`. @@ -125,7 +91,7 @@ make create-gke-cluster make add-local-ip-to-cluster ``` -## Step 2 - Build and Load Images +### Step 2 - Build and Load Images Loading the images only applies to a `kind` cluster. If using a cloud provider, you will need to tag and push your images to a registry that is accessible from that cloud provider. @@ -146,9 +112,152 @@ For the telemetry test, which requires a OTel collector, build an image with the TELEMETRY_ENDPOINT=otel-collector-opentelemetry-collector.collector.svc.cluster.local:4317 TELEMETRY_ENDPOINT_INSECURE=true ``` -## Step 3 - Run the tests +## Conformance Testing + +### Step 1 - Install NGINX Gateway Fabric to configured kind cluster + +> Note: If you want to run the latest conformance tests from the Gateway API `main` branch, set the following +> environment variable before deploying NGF: + +```bash + export GW_API_VERSION=main +``` + +> Otherwise, the latest stable version will be used by default. +> Additionally, if you want to run conformance tests with experimental features enabled, set the following +> environment variable before deploying NGF: + +```bash + export ENABLE_EXPERIMENTAL=true +``` + +#### _Option 1_ Build and install NGINX Gateway Fabric from local to configured kind cluster + +```makefile +make install-ngf-local-build +``` + +Or, to install NGF with NGINX Plus enabled (NGINX Plus cert and key must exist in the root of the repo): + +```makefile +make install-ngf-local-build-with-plus +``` + +#### _Option 2_ Install NGINX Gateway Fabric from local already built image to configured kind cluster + +You can optionally skip the actual _build_ step. + +```makefile +make install-ngf-local-no-build +``` + +Or, to install NGF with NGINX Plus enabled: + +```makefile +make install-ngf-no-build-with-plus +``` + +> Note: If choosing this option, the following step _must_ be completed manually _before_ you build the image: + +```makefile +make update-ngf-manifest PREFIX= TAG= +``` + +Or, if you are building the NGINX Plus image: + +```makefile +make update-ngf-manifest-with-plus PREFIX= TAG= +``` + +#### _Option 3_ Install NGINX Gateway Fabric from edge to configured kind cluster + +You can also skip the build NGF image step and prepare the environment to instead use the `edge` image. Note that this +option does not currently support installing with NGINX Plus enabled. + +```makefile +make install-ngf-edge +``` + +### Step 2 - Build conformance test runner image + +> Note: If you want to run the latest conformance tests from the Gateway API `main` branch, run the following +> make command to update the Go modules to `main`: + +```makefile +make update-go-modules +``` + +> You can also point to a specific fork/branch by running: + +```bash +go mod edit -replace=sigs.k8s.io/gateway-api=@ +go mod download +go mod verify +go mod tidy +``` + +> Otherwise, the latest stable version will be used by default. + +```makefile +make build-test-runner-image +``` + +### Step 3 - Run Gateway conformance tests + +```makefile +make run-conformance-tests +``` + +### Step 4 - Cleanup the conformance test fixtures and uninstall NGINX Gateway Fabric + +```makefile +make cleanup-conformance-tests +``` + +```makefile +make uninstall-ngf +``` + +### Step 5 - Revert changes to Go modules + +**Optional** Not required if you aren't running the `main` Gateway API tests. + +```makefile +make reset-go-modules +``` -### 3a - Run the functional tests locally +### Step 6 - Delete kind cluster + +```makefile +make delete-kind-cluster +``` + +## System Testing + +The system tests are meant to be run on a live Kubernetes environment to verify a real system. These +are similar to the existing conformance tests, but will verify things such as: + +- NGF-specific functionality +- Non-Functional requirements (NFR) testing (such as performance, scale, etc.) + +When running locally, the tests create a port-forward from your NGF Pod to localhost using a port chosen by the +test framework. Traffic is sent over this port. If running on a GCP VM targeting a GKE cluster, the tests will create an +internal LoadBalancer service which will receive the test traffic. + +**Important**: NFR tests can only be run on a GKE cluster. + +Directory structure is as follows: + +- `framework`: contains utility functions for running the tests +- `results`: contains the results files for the NFR tests +- `scripts`: contain scripts used to set up the environment and run the tests +- `suite`: contains the test files + +> Note: Existing NFR tests will be migrated into this testing `suite` and results stored in the `results` directory. + +### Step 1 - Run the tests + +#### 1a - Run the functional tests locally ```makefile make test TAG=$(whoami) @@ -169,7 +278,7 @@ To run the telemetry test: make test TAG=$(whoami) GINKGO_LABEL=telemetry ``` -### 3b - Run the tests on a GKE cluster from a GCP VM +#### 1b - Run the tests on a GKE cluster from a GCP VM This step only applies if you are running the NFR tests, or would like to run the functional tests on a GKE cluster from a GCP based VM. @@ -215,7 +324,6 @@ make run-tests-on-vm To set up the GCP environment with the router and VM and then run the tests, run the following command: - ```makefile make setup-gcp-and-run-nfr-tests ``` @@ -238,9 +346,11 @@ make start-longevity-test ``` + > Note: If you want to change the time period for which the test runs, update the `wrk` commands in `suite/scripts/longevity-wrk.sh` to the time period you want, and run `make sync-files-to-vm`. + > Note: If you want to re-run the longevity test, you need to clear out the `cafe.example.com` entry from the `/etc/hosts` file on your VM. You can verify the test is working by checking nginx logs to see traffic flow, and check that the cronjob is running and redeploying apps. @@ -310,30 +420,30 @@ XIt("runs some test", func(){ For more information of filtering specs, see [the docs here](https://onsi.github.io/ginkgo/#filtering-specs). -## Step 4 - Cleanup +## Step 2 - Cleanup 1. Delete kind cluster, if required - ```makefile - make delete-kind-cluster - ``` + ```makefile + make delete-kind-cluster + ``` 2. Delete the GCP components (GKE cluster, GKE router, VM, and firewall rule), if required - ```makefile - make cleanup-gcp - ``` + ```makefile + make cleanup-gcp + ``` - or + or - ```makefile - make cleanup-router - ``` + ```makefile + make cleanup-router + ``` - ```makefile - make cleanup-vm - ``` + ```makefile + make cleanup-vm + ``` - ```makefile - make delete-gke-cluster - ``` + ```makefile + make delete-gke-cluster + ``` diff --git a/tests/conformance/Dockerfile b/tests/conformance/Dockerfile new file mode 100644 index 0000000000..b1d2c6f921 --- /dev/null +++ b/tests/conformance/Dockerfile @@ -0,0 +1,10 @@ +# syntax=docker/dockerfile:1.7 +FROM golang:1.22 + +WORKDIR /go/src/github.com/nginxinc/nginx-gateway-fabric/tests/conformance + +COPY --link go.mod /go/src/github.com/nginxinc/nginx-gateway-fabric/tests/ +COPY --link go.sum /go/src/github.com/nginxinc/nginx-gateway-fabric/tests/ +RUN go mod download + +COPY --link conformance /go/src/github.com/nginxinc/nginx-gateway-fabric/tests/conformance diff --git a/conformance/tests/conformance-rbac.yaml b/tests/conformance/conformance-rbac.yaml similarity index 100% rename from conformance/tests/conformance-rbac.yaml rename to tests/conformance/conformance-rbac.yaml diff --git a/conformance/tests/conformance_test.go b/tests/conformance/conformance_test.go similarity index 87% rename from conformance/tests/conformance_test.go rename to tests/conformance/conformance_test.go index ad8cfb4310..2bf8739984 100644 --- a/conformance/tests/conformance_test.go +++ b/tests/conformance/conformance_test.go @@ -15,7 +15,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -package tests +package conformance import ( "os" @@ -23,8 +23,8 @@ import ( . "github.com/onsi/gomega" "k8s.io/apimachinery/pkg/util/sets" - conf_v1 "sigs.k8s.io/gateway-api/conformance/apis/v1" "sigs.k8s.io/gateway-api/conformance" + conf_v1 "sigs.k8s.io/gateway-api/conformance/apis/v1" "sigs.k8s.io/gateway-api/conformance/tests" "sigs.k8s.io/gateway-api/conformance/utils/flags" "sigs.k8s.io/gateway-api/conformance/utils/suite" @@ -41,14 +41,14 @@ func TestConformance(t *testing.T) { opts := conformance.DefaultOptions(t) opts.Implementation = conf_v1.Implementation{ - Organization: "nginxinc", - Project: "nginx-gateway-fabric", - URL: "https://github.com/nginxinc/nginx-gateway-fabric", - Version: *flags.ImplementationVersion, - Contact: []string{ - "https://github.com/nginxinc/nginx-gateway-fabric/discussions/new/choose", + Organization: "nginxinc", + Project: "nginx-gateway-fabric", + URL: "https://github.com/nginxinc/nginx-gateway-fabric", + Version: *flags.ImplementationVersion, + Contact: []string{ + "https://github.com/nginxinc/nginx-gateway-fabric/discussions/new/choose", }, - } + } opts.ConformanceProfiles = sets.New(suite.GatewayHTTPConformanceProfileName, suite.GatewayGRPCConformanceProfileName) testSuite, err := suite.NewConformanceTestSuite(opts) diff --git a/conformance/provisioner/README.md b/tests/conformance/provisioner/README.md similarity index 92% rename from conformance/provisioner/README.md rename to tests/conformance/provisioner/README.md index ff1d0dfe25..44f73b5265 100644 --- a/conformance/provisioner/README.md +++ b/tests/conformance/provisioner/README.md @@ -19,7 +19,7 @@ Global Flags: https://github.com/nginxinc/nginx-gateway-fabric/issues/634). However, it can be used in the Gateway API conformance tests, which expect a Gateway API implementation to provision an independent data plane per Gateway. > -> Note: Provisioner uses [this manifest](https://github.com/nginxinc/nginx-gateway-fabric/blob/main/conformance/provisioner/static-deployment.yaml) +> Note: Provisioner uses [this manifest](https://github.com/nginxinc/nginx-gateway-fabric/blob/main/config/tests/static-deployment.yaml) to create an NGF static mode Deployment. > This manifest gets included into the NGF binary during the NGF build. To customize the Deployment, modify the manifest and **re-build** NGF. @@ -31,7 +31,7 @@ How to deploy: 1. Deploy provisioner: ```shell - kubectl apply -f conformance/provisioner/provisioner.yaml + kubectl apply -f provisioner.yaml ``` 1. Confirm the provisioner is running in nginx-gateway namespace: diff --git a/conformance/provisioner/provisioner.yaml b/tests/conformance/provisioner/provisioner.yaml similarity index 100% rename from conformance/provisioner/provisioner.yaml rename to tests/conformance/provisioner/provisioner.yaml diff --git a/tests/go.mod b/tests/go.mod index e04023adfe..7d9f67c87c 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -41,20 +41,24 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/miekg/dns v1.1.58 // indirect github.com/moby/spdystream v0.2.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_golang v1.17.0 // indirect - github.com/prometheus/client_model v0.5.0 // indirect - github.com/prometheus/common v0.45.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_golang v1.16.0 // indirect + github.com/prometheus/client_model v0.4.0 // indirect + github.com/prometheus/common v0.44.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/stretchr/testify v1.9.0 // indirect golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f // indirect + golang.org/x/mod v0.17.0 // indirect golang.org/x/net v0.24.0 // indirect golang.org/x/oauth2 v0.19.0 // indirect golang.org/x/sync v0.7.0 // indirect @@ -64,6 +68,8 @@ require ( golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.20.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de // indirect + google.golang.org/grpc v1.63.2 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/tests/go.sum b/tests/go.sum index 5cbda18b2e..24c2f575b8 100644 --- a/tests/go.sum +++ b/tests/go.sum @@ -35,6 +35,7 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= @@ -69,8 +70,10 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= -github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/miekg/dns v1.1.58 h1:ca2Hdkz+cDg/7eNF6V56jjzuZ4aCAE+DbVkILdQWG/4= +github.com/miekg/dns v1.1.58/go.mod h1:Ypv+3b/KadlvW9vJfXOTf300O4UqaHFzFCuHz+rPkBY= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -90,12 +93,12 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= -github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= -github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= -github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= -github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= -github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= +github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= +github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= +github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= +github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= @@ -128,6 +131,8 @@ golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2 golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -136,6 +141,7 @@ golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/oauth2 v0.19.0 h1:9+E/EZBCbTLNrbN35fHv/a/d/mOBatymz1zbtQrXpIg= golang.org/x/oauth2 v0.19.0/go.mod h1:vYi7skDa1x015PmRRYZ7+s1cWyPgrPiSYRe4rnsexc8= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -170,6 +176,10 @@ gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuB gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca h1:PupagGYwj8+I4ubCxcmcBRk3VlUWtTg5huQpZR9flmE= gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de h1:cZGRis4/ot9uVm639a+rHCUaG0JJHEsdyzSQTMX+suY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:H4O17MA/PE9BsGx3w+a+W2VOLLD1Qf7oJneAoU6WktY= +google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= +google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/conformance/scripts/check-pod-exit-code.sh b/tests/scripts/check-pod-exit-code.sh similarity index 100% rename from conformance/scripts/check-pod-exit-code.sh rename to tests/scripts/check-pod-exit-code.sh diff --git a/conformance/scripts/install-gateway.sh b/tests/scripts/install-gateway.sh similarity index 100% rename from conformance/scripts/install-gateway.sh rename to tests/scripts/install-gateway.sh diff --git a/conformance/scripts/uninstall-gateway.sh b/tests/scripts/uninstall-gateway.sh similarity index 100% rename from conformance/scripts/uninstall-gateway.sh rename to tests/scripts/uninstall-gateway.sh