Skip to content

Commit 789b801

Browse files
sjbermanbjee19
sjberman
and
bjee19
authored
Add test utilities for installing NGF (#1256)
Problem: 1. In order to start running system tests, we need a framework in place. 2. Sometimes when installed, NGF would start before the control plane CRD existed. Solution: 1. Implement a basic framework that installs and uninstalls NGF. 2. Add a polling loop to find the control plane CRD before exiting. --------- Co-authored-by: bjee19 <[email protected]>
1 parent 7de105c commit 789b801

File tree

11 files changed

+360
-30
lines changed

11 files changed

+360
-30
lines changed

‎.github/workflows/conformance.yml

+3-3
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ jobs:
7474
run: |
7575
ngf_prefix=$(echo ${{ steps.ngf-meta.outputs.tags }} | cut -d ":" -f 1)
7676
ngf_tag=$(echo ${{ steps.ngf-meta.outputs.tags }} | cut -d ":" -f 2)
77-
make update-ngf-manifest NGF_PREFIX=${ngf_prefix} NGF_TAG=${ngf_tag}
77+
make update-ngf-manifest PREFIX=${ngf_prefix} TAG=${ngf_tag}
7878
working-directory: ./conformance
7979

8080
- name: Build binary
@@ -146,12 +146,12 @@ jobs:
146146
ngf_tag=$(echo ${{ steps.ngf-meta.outputs.tags }} | cut -d ":" -f 2)
147147
if [ ${{ github.event_name }} == "schedule" ]; then export GW_API_VERSION=main; fi
148148
if [ ${{ startsWith(matrix.k8s-version, '1.23') || startsWith(matrix.k8s-version, '1.24') }} == "true" ]; then export INSTALL_WEBHOOK=true; fi
149-
make install-ngf-local-no-build NGF_PREFIX=${ngf_prefix} NGF_TAG=${ngf_tag}
149+
make install-ngf-local-no-build PREFIX=${ngf_prefix} TAG=${ngf_tag}
150150
working-directory: ./conformance
151151

152152
- name: Run conformance tests
153153
run: |
154-
make run-conformance-tests TAG=${{ github.sha }} VERSION=${{ github.ref_name }}
154+
make run-conformance-tests CONFORMANCE_TAG=${{ github.sha }} VERSION=${{ github.ref_name }}
155155
core_result=$(cat conformance-profile.yaml | yq '.profiles[0].core.result')
156156
extended_result=$(cat conformance-profile.yaml | yq '.profiles[0].extended.result')
157157
if [ "${core_result}" == "failure" ] || [ "${extended_result}" == "failure" ]; then echo "Conformance test failed, see above for details." && exit 2; fi

‎Makefile

+1-1
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,7 @@ lint: ## Run golangci-lint against code
123123

124124
.PHONY: unit-test
125125
unit-test: ## Run unit tests for the go code
126-
go test ./... -race -coverprofile cover.out
126+
go test ./... -tags unit -race -coverprofile cover.out
127127
go tool cover -html=cover.out -o cover.html
128128

129129
.PHONY: njs-unit-test

‎conformance/Makefile

+12-12
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
1-
NGF_TAG = edge
2-
NGF_PREFIX = nginx-gateway-fabric
3-
NGINX_IMAGE_NAME = $(NGF_PREFIX)/nginx
1+
TAG = edge
2+
PREFIX = nginx-gateway-fabric
3+
NGINX_PREFIX = $(PREFIX)/nginx
44
GW_API_VERSION ?= 1.0.0
55
GATEWAY_CLASS = nginx
66
SUPPORTED_FEATURES = HTTPRoute,HTTPRouteQueryParamMatching,HTTPRouteMethodMatching,HTTPRoutePortRedirect,HTTPRouteSchemeRedirect,GatewayClassObservedGenerationBump
77
KIND_IMAGE ?= $(shell grep -m1 'FROM kindest/node' <tests/Dockerfile | awk -F'[ ]' '{print $$2}')
88
KIND_KUBE_CONFIG=$${HOME}/.kube/kind/config
9-
TAG = latest
10-
PREFIX = conformance-test-runner
9+
CONFORMANCE_TAG = latest
10+
CONFORMANCE_PREFIX = conformance-test-runner
1111
NGF_MANIFEST=../deploy/manifests/nginx-gateway.yaml
1212
CRDS=../deploy/manifests/crds/
1313
STATIC_MANIFEST=provisioner/static-deployment.yaml
@@ -27,7 +27,7 @@ update-go-modules: ## Update the gateway-api go modules to latest main version
2727

2828
.PHONY: build-test-runner-image
2929
build-test-runner-image: ## Build conformance test runner image
30-
docker build -t $(PREFIX):$(TAG) -f tests/Dockerfile ..
30+
docker build -t $(CONFORMANCE_PREFIX):$(CONFORMANCE_TAG) -f tests/Dockerfile ..
3131

3232
.PHONY: create-kind-cluster
3333
create-kind-cluster: ## Create a kind cluster
@@ -36,15 +36,15 @@ create-kind-cluster: ## Create a kind cluster
3636

3737
.PHONY: update-ngf-manifest
3838
update-ngf-manifest: ## Update the NGF deployment manifest image names and imagePullPolicies
39-
cd .. && make generate-manifests HELM_TEMPLATE_EXTRA_ARGS_FOR_ALL_MANIFESTS_FILE="--set nginxGateway.kind=skip" HELM_TEMPLATE_COMMON_ARGS="--set nginxGateway.image.repository=$(NGF_PREFIX) --set nginxGateway.image.tag=$(NGF_TAG) --set nginxGateway.image.pullPolicy=Never --set nginx.image.repository=$(NGINX_IMAGE_NAME) --set nginx.image.tag=$(NGF_TAG) --set nginx.image.pullPolicy=Never" && cd -
39+
cd .. && make generate-manifests HELM_TEMPLATE_EXTRA_ARGS_FOR_ALL_MANIFESTS_FILE="--set nginxGateway.kind=skip" HELM_TEMPLATE_COMMON_ARGS="--set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginxGateway.image.pullPolicy=Never --set nginx.image.repository=$(NGINX_PREFIX) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never" && cd -
4040

4141
.PHONY: build-images
4242
build-images: ## Build NGF and nginx images
43-
cd .. && make PREFIX=$(NGF_PREFIX) TAG=$(NGF_TAG) build-images
43+
cd .. && make PREFIX=$(PREFIX) TAG=$(TAG) build-images
4444

4545
.PHONY: load-images
4646
load-images: ## Load NGF and NGINX images on configured kind cluster
47-
kind load docker-image $(NGF_PREFIX):$(NGF_TAG) $(NGINX_IMAGE_NAME):$(NGF_TAG)
47+
kind load docker-image $(PREFIX):$(TAG) $(NGINX_PREFIX):$(TAG)
4848

4949
.PHONY: prepare-ngf-dependencies
5050
prepare-ngf-dependencies: update-ngf-manifest ## Install NGF dependencies on configured kind cluster
@@ -55,7 +55,7 @@ prepare-ngf-dependencies: update-ngf-manifest ## Install NGF dependencies on con
5555
.PHONY: deploy-updated-provisioner
5656
deploy-updated-provisioner: ## Update provisioner manifest and deploy to the configured kind cluster
5757
yq '(select(di != 3))' $(PROVISIONER_MANIFEST) | kubectl apply -f -
58-
yq '(select(.spec.template.spec.containers[].image) | .spec.template.spec.containers[].image="$(NGF_PREFIX):$(NGF_TAG)" | .spec.template.spec.containers[].imagePullPolicy = "Never")' $(PROVISIONER_MANIFEST) | kubectl apply -f -
58+
yq '(select(.spec.template.spec.containers[].image) | .spec.template.spec.containers[].image="$(PREFIX):$(TAG)" | .spec.template.spec.containers[].imagePullPolicy = "Never")' $(PROVISIONER_MANIFEST) | kubectl apply -f -
5959

6060
.PHONY: install-ngf-local-build
6161
install-ngf-local-build: prepare-ngf-dependencies build-images load-images deploy-updated-provisioner ## Install NGF from local build with provisioner on configured kind cluster
@@ -69,10 +69,10 @@ install-ngf-edge: prepare-ngf-dependencies ## Install NGF with provisioner from
6969

7070
.PHONY: run-conformance-tests
7171
run-conformance-tests: ## Run conformance tests
72-
kind load docker-image $(PREFIX):$(TAG)
72+
kind load docker-image $(CONFORMANCE_PREFIX):$(CONFORMANCE_TAG)
7373
kubectl apply -f tests/conformance-rbac.yaml
7474
kubectl run -i conformance \
75-
--image=$(PREFIX):$(TAG) --image-pull-policy=Never \
75+
--image=$(CONFORMANCE_PREFIX):$(CONFORMANCE_TAG) --image-pull-policy=Never \
7676
--overrides='{ "spec": { "serviceAccountName": "conformance" } }' \
7777
--restart=Never -- sh -c "go test -v . -tags conformance,experimental -args --gateway-class=$(GATEWAY_CLASS) \
7878
--supported-features=$(SUPPORTED_FEATURES) --version=$(VERSION) \

‎conformance/README.md

+5-5
Original file line numberDiff line numberDiff line change
@@ -40,10 +40,10 @@ update-ngf-manifest Update the NGF deployment manifest image names an
4040

4141
| Variable | Default | Description |
4242
|----------------------|---------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------|
43-
| TAG | latest | The tag for the conformance test image |
44-
| PREFIX | conformance-test-runner | The prefix for the conformance test image |
45-
| NGF_TAG | edge | The tag for the locally built NGF image |
46-
| NGF_PREFIX | nginx-gateway-fabric | The prefix for the locally built NGF image |
43+
| CONFORMANCE_TAG | latest | The tag for the conformance test image |
44+
| CONFORMANCE_PREFIX | conformance-test-runner | The prefix for the conformance test image |
45+
| TAG | edge | The tag for the locally built NGF image |
46+
| PREFIX | nginx-gateway-fabric | The prefix for the locally built NGF image |
4747
| GW_API_VERSION | 1.0.0 | Tag for the Gateway API version to check out. Set to `main` to get the latest version |
4848
| KIND_IMAGE | Latest kind image, as defined in the tests/Dockerfile | The kind image to use |
4949
| KIND_KUBE_CONFIG | ~/.kube/kind/config | The location of the kubeconfig |
@@ -97,7 +97,7 @@ make install-ngf-local-no-build
9797
> Note: If choosing this option, the following step *must* be completed manually *before* you build the image:
9898
9999
```makefile
100-
make update-ngf-manifest NGF_PREFIX=<ngf_repo_name> NGF_TAG=<ngf_image_tag>
100+
make update-ngf-manifest PREFIX=<ngf_repo_name> TAG=<ngf_image_tag>
101101
```
102102

103103
#### *Option 3* Install NGINX Gateway Fabric from edge to configured kind cluster

‎docs/developer/release-process.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ To create a new release, follow these steps:
3434
6. Prepare and merge a PR into the release branch to update the repo files for the release:
3535
1. Update the Helm [Chart.yaml](/deploy/helm-chart/Chart.yaml): the `appVersion` to `X.Y.Z`, the icon and source
3636
URLs to point at `vX.Y.Z`, and bump the `version`.
37-
2. Adjust the `VERSION` variable in the [Makefile](/Makefile) and the `NGF_TAG` in the
37+
2. Adjust the `VERSION` variable in the [Makefile](/Makefile) and the `TAG` in the
3838
[conformance tests Makefile](/conformance/Makefile) to `X.Y.Z`.
3939
3. Update the tag of NGF container images used in the Helm [values.yaml](/deploy/helm-chart/values.yaml) file, the
4040
[provisioner manifest](/conformance/provisioner/provisioner.yaml), and all docs to `X.Y.Z`.

‎go.mod

+2-2
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ require (
1616
github.com/spf13/cobra v1.8.0
1717
go.uber.org/zap v1.26.0
1818
k8s.io/api v0.28.4
19+
k8s.io/apiextensions-apiserver v0.28.4
1920
k8s.io/apimachinery v0.28.4
2021
k8s.io/client-go v0.28.4
2122
sigs.k8s.io/controller-runtime v0.16.3
@@ -82,8 +83,7 @@ require (
8283
gopkg.in/inf.v0 v0.9.1 // indirect
8384
gopkg.in/yaml.v2 v2.4.0 // indirect
8485
gopkg.in/yaml.v3 v3.0.1 // indirect
85-
k8s.io/apiextensions-apiserver v0.28.3 // indirect
86-
k8s.io/component-base v0.28.3 // indirect
86+
k8s.io/component-base v0.28.4 // indirect
8787
k8s.io/klog/v2 v2.100.1 // indirect
8888
k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
8989
k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect

‎go.sum

+4-4
Original file line numberDiff line numberDiff line change
@@ -255,14 +255,14 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
255255
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
256256
k8s.io/api v0.28.4 h1:8ZBrLjwosLl/NYgv1P7EQLqoO8MGQApnbgH8tu3BMzY=
257257
k8s.io/api v0.28.4/go.mod h1:axWTGrY88s/5YE+JSt4uUi6NMM+gur1en2REMR7IRj0=
258-
k8s.io/apiextensions-apiserver v0.28.3 h1:Od7DEnhXHnHPZG+W9I97/fSQkVpVPQx2diy+2EtmY08=
259-
k8s.io/apiextensions-apiserver v0.28.3/go.mod h1:NE1XJZ4On0hS11aWWJUTNkmVB03j9LM7gJSisbRt8Lc=
258+
k8s.io/apiextensions-apiserver v0.28.4 h1:AZpKY/7wQ8n+ZYDtNHbAJBb+N4AXXJvyZx6ww6yAJvU=
259+
k8s.io/apiextensions-apiserver v0.28.4/go.mod h1:pgQIZ1U8eJSMQcENew/0ShUTlePcSGFq6dxSxf2mwPM=
260260
k8s.io/apimachinery v0.28.4 h1:zOSJe1mc+GxuMnFzD4Z/U1wst50X28ZNsn5bhgIIao8=
261261
k8s.io/apimachinery v0.28.4/go.mod h1:wI37ncBvfAoswfq626yPTe6Bz1c22L7uaJ8dho83mgg=
262262
k8s.io/client-go v0.28.4 h1:Np5ocjlZcTrkyRJ3+T3PkXDpe4UpatQxj85+xjaD2wY=
263263
k8s.io/client-go v0.28.4/go.mod h1:0VDZFpgoZfelyP5Wqu0/r/TRYcLYuJ2U1KEeoaPa1N4=
264-
k8s.io/component-base v0.28.3 h1:rDy68eHKxq/80RiMb2Ld/tbH8uAE75JdCqJyi6lXMzI=
265-
k8s.io/component-base v0.28.3/go.mod h1:fDJ6vpVNSk6cRo5wmDa6eKIG7UlIQkaFmZN2fYgIUD8=
264+
k8s.io/component-base v0.28.4 h1:c/iQLWPdUgI90O+T9TeECg8o7N3YJTiuz2sKxILYcYo=
265+
k8s.io/component-base v0.28.4/go.mod h1:m9hR0uvqXDybiGL2nf/3Lf0MerAfQXzkfWhUY58JUbU=
266266
k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg=
267267
k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
268268
k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780=

‎internal/mode/static/manager.go

+18-2
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,12 @@ import (
99
"github.com/prometheus/client_golang/prometheus"
1010
apiv1 "k8s.io/api/core/v1"
1111
discoveryV1 "k8s.io/api/discovery/v1"
12+
apierrors "k8s.io/apimachinery/pkg/api/errors"
1213
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1314
"k8s.io/apimachinery/pkg/runtime"
1415
"k8s.io/apimachinery/pkg/types"
1516
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
17+
"k8s.io/apimachinery/pkg/util/wait"
1618
"k8s.io/client-go/tools/record"
1719
ctlr "sigs.k8s.io/controller-runtime"
1820
"sigs.k8s.io/controller-runtime/pkg/client"
@@ -376,8 +378,22 @@ func setInitialConfig(
376378
defer cancel()
377379

378380
var config ngfAPI.NginxGateway
379-
if err := reader.Get(ctx, configName, &config); err != nil {
380-
return err
381+
// Polling to wait for CRD to exist if the Deployment is created first.
382+
if err := wait.PollUntilContextCancel(
383+
ctx,
384+
500*time.Millisecond,
385+
true, /* poll immediately */
386+
func(ctx context.Context) (bool, error) {
387+
if err := reader.Get(ctx, configName, &config); err != nil {
388+
if !apierrors.IsNotFound(err) {
389+
return false, err
390+
}
391+
return false, nil
392+
}
393+
return true, nil
394+
},
395+
); err != nil {
396+
return fmt.Errorf("NginxGateway %s not found: %w", configName, err)
381397
}
382398

383399
// status is not updated until the status updater's cache is started and the

‎tests/Makefile

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
TAG = edge
2+
PREFIX = nginx-gateway-fabric
3+
NGINX_PREFIX = $(PREFIX)/nginx
4+
PULL_POLICY=Never
5+
GW_API_VERSION ?= 1.0.0
6+
K8S_VERSION ?= latest ## Expected format: 1.24 (major.minor) or latest
7+
8+
.PHONY: help
9+
help: Makefile ## Display this help
10+
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "; printf "Usage:\n\n make \033[36m<target>\033[0m\n\nTargets:\n\n"}; {printf " \033[36m%-30s\033[0m %s\n", $$1, $$2}'
11+
12+
.PHONY: create-kind-cluster
13+
create-kind-cluster: ## Create a kind cluster
14+
cd .. && make create-kind-cluster
15+
16+
.PHONY: build-images
17+
build-images: ## Build NGF and NGINX images
18+
cd .. && make PREFIX=$(PREFIX) TAG=$(TAG) build-images
19+
20+
.PHONY: load-images
21+
load-images: ## Load NGF and NGINX images on configured kind cluster
22+
kind load docker-image $(PREFIX):$(TAG) $(NGINX_PREFIX):$(TAG)
23+
24+
test: ## Run the system tests against your default k8s cluster
25+
go test -v . -args --gateway-api-version=$(GW_API_VERSION) --image-tag=$(TAG) \
26+
--ngf-image-repo=$(PREFIX) --nginx-image-repo=$(NGINX_PREFIX) --pull-policy=$(PULL_POLICY) \
27+
--k8s-version=$(K8S_VERSION)
28+
29+
.PHONY: delete-kind-cluster
30+
delete-kind-cluster: ## Delete kind cluster
31+
kind delete cluster

0 commit comments

Comments
 (0)