Skip to content

Commit

Permalink
Generate manifests using Helm (#911)
Browse files Browse the repository at this point in the history
* Generate deployment manifests using Helm
  • Loading branch information
ciarams87 authored Aug 2, 2023
1 parent e00d64c commit 36daa1d
Show file tree
Hide file tree
Showing 27 changed files with 738 additions and 336 deletions.
5 changes: 4 additions & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -52,12 +52,15 @@ jobs:
- name: Check if go.mod and go.sum are up to date
run: go mod tidy && git diff --exit-code -- go.mod go.sum

- name: Check if generated files are up to date
- name: Check if generated go files are up to date
run: make generate && git diff --exit-code

- name: Check if njs-modules yaml is up to date
run: make generate-njs-yaml && git diff --exit-code

- name: Check if generated manifests are up to date
run: make generate-manifests && git diff --exit-code

unit-tests:
name: Unit Tests
runs-on: ubuntu-22.04
Expand Down
10 changes: 10 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ TARGET ?= local## The target of the build. Possible values: local and container
KIND_KUBE_CONFIG=$${HOME}/.kube/kind/config## The location of the kind kubeconfig
OUT_DIR ?= $(shell pwd)/build/out## The folder where the binary will be stored
ARCH ?= amd64## The architecture of the image and/or binary. For example: amd64 or arm64
override HELM_TEMPLATE_COMMON_ARGS += --set creator=template --set nameOverride=nginx-gateway## The common options for the Helm template command.
override HELM_TEMPLATE_EXTRA_ARGS_FOR_ALL_MANIFESTS_FILE += --set service.create=false## The options to be passed to the full Helm templating command only.
override DOCKER_BUILD_OPTIONS += --build-arg VERSION=$(VERSION) --build-arg GIT_COMMIT=$(GIT_COMMIT) --build-arg DATE=$(DATE)## The options for the docker build command. For example, --pull

.DEFAULT_GOAL := help
Expand Down Expand Up @@ -117,5 +119,13 @@ debug-build: build ## Build binary with debug info, symbols, and no optimization
.PHONY: debug-container
debug-container: debug-build container ## Build container with debug binary

.PHONY: generate-manifests
generate-manifests: ## Generate manifests using Helm.
helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) $(HELM_TEMPLATE_EXTRA_ARGS_FOR_ALL_MANIFESTS_FILE) -n nginx-gateway | cat $(strip $(MANIFEST_DIR))/namespace.yaml - > $(strip $(MANIFEST_DIR))/nginx-gateway.yaml
helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) -n nginx-gateway -s templates/deployment.yaml > conformance/provisioner/static-deployment.yaml
helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) -n nginx-gateway -s templates/service.yaml > $(strip $(MANIFEST_DIR))/service/loadbalancer.yaml
helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) --set service.annotations.'service\.beta\.kubernetes\.io\/aws-load-balancer-type'="nlb" -n nginx-gateway -s templates/service.yaml > $(strip $(MANIFEST_DIR))/service/loadbalancer-aws-nlb.yaml
helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) --set service.type=NodePort --set service.externalTrafficPolicy="" -n nginx-gateway -s templates/service.yaml > $(strip $(MANIFEST_DIR))/service/nodeport.yaml

.PHONY: dev-all
dev-all: deps fmt njs-fmt vet lint unit-test njs-unit-test ## Run all the development checks
48 changes: 24 additions & 24 deletions conformance/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,11 @@ SUPPORTED_FEATURES = HTTPRoute,HTTPRouteQueryParamMatching,HTTPRouteMethodMatchi
KIND_KUBE_CONFIG=$${HOME}/.kube/kind/config
TAG = latest
PREFIX = conformance-test-runner
NKG_DEPLOYMENT_MANIFEST=../deploy/manifests/deployment.yaml
NGINX_IMAGE=$(shell yq '.spec.template.spec.containers[1].image as $$nginx_ver | $$nginx_ver' $(NKG_DEPLOYMENT_MANIFEST))
NKG_MANIFEST=../deploy/manifests/nginx-gateway.yaml
SERVICE_MANIFEST=../deploy/manifests/service/nodeport.yaml
STATIC_MANIFEST=provisioner/static-deployment.yaml
PROVISIONER_MANIFEST=provisioner/provisioner.yaml
NGINX_IMAGE=$(shell yq '.spec.template.spec.containers[1].image as $$nginx_ver | $$nginx_ver' $(STATIC_MANIFEST))
.DEFAULT_GOAL := help

.PHONY: help
Expand Down Expand Up @@ -36,42 +39,37 @@ preload-nginx-container: ## Preload NGINX container on configured kind cluster

.PHONY: update-nkg-manifest
update-nkg-manifest: ## Update the NKG deployment manifest image name and imagePullPolicy
yq -i 'with(.spec.template.spec.containers[0]; .image = "$(NKG_PREFIX):$(NKG_TAG)" | .imagePullPolicy = "Never")' $(NKG_DEPLOYMENT_MANIFEST)
cd .. && make generate-manifests HELM_TEMPLATE_EXTRA_ARGS_FOR_ALL_MANIFESTS_FILE="--set nginxGateway.kind=skip" HELM_TEMPLATE_COMMON_ARGS="--set nginxGateway.image.repository=$(NKG_PREFIX) --set nginxGateway.image.tag=$(NKG_TAG) --set nginxGateway.image.pullPolicy=Never" && cd -

.PHONY: build-nkg-image
build-nkg-image: update-nkg-manifest ## Build NKG container and load it and NGINX container on configured kind cluster
build-nkg-image: ## Build NKG container and load it and NGINX container on configured kind cluster
cd .. && make PREFIX=$(NKG_PREFIX) TAG=$(NKG_TAG) container

.PHONY: load-images
load-images: preload-nginx-container ## Load NKG and NGINX containers on configured kind cluster
kind load docker-image $(NKG_PREFIX):$(NKG_TAG)

.PHONY: prepare-nkg-dependencies
prepare-nkg-dependencies: ## Install NKG dependencies on configured kind cluster
prepare-nkg-dependencies: update-nkg-manifest ## Install NKG dependencies on configured kind cluster
./scripts/install-gateway.sh $(GW_API_VERSION)
kubectl wait --for=condition=available --timeout=60s deployment gateway-api-admission-server -n gateway-system
kubectl apply -f ../deploy/manifests/namespace.yaml
cd .. && make generate-njs-yaml && cd -
kubectl apply -f ../deploy/manifests/njs-modules.yaml -n nginx-gateway
kubectl apply -f ../deploy/manifests/nginx-conf.yaml
kubectl apply -f ../deploy/manifests/rbac.yaml
kubectl apply -f ../deploy/manifests/gatewayclass.yaml
kubectl apply -f ../deploy/manifests/service/nodeport.yaml
kubectl apply -f $(NKG_MANIFEST)
kubectl apply -f $(SERVICE_MANIFEST)

.PHONY: deploy-updated-provisioner
deploy-updated-provisioner: ## Update provisioner manifest and deploy to the configured kind cluster
yq '(select(di != 3))' provisioner/provisioner.yaml | kubectl apply -f -
yq '(select(.spec.template.spec.containers[].image) | .spec.template.spec.containers[].image="$(NKG_PREFIX):$(NKG_TAG)" | .spec.template.spec.containers[].imagePullPolicy = "Never")' provisioner/provisioner.yaml | kubectl apply -f -
yq '(select(di != 3))' $(PROVISIONER_MANIFEST) | kubectl apply -f -
yq '(select(.spec.template.spec.containers[].image) | .spec.template.spec.containers[].image="$(NKG_PREFIX):$(NKG_TAG)" | .spec.template.spec.containers[].imagePullPolicy = "Never")' $(PROVISIONER_MANIFEST) | kubectl apply -f -

.PHONY: install-nkg-local-build
install-nkg-local-build: build-nkg-image load-images prepare-nkg-dependencies deploy-updated-provisioner ## Install NKG from local build with provisioner on configured kind cluster
install-nkg-local-build: prepare-nkg-dependencies build-nkg-image load-images deploy-updated-provisioner ## Install NKG from local build with provisioner on configured kind cluster

.PHONY: install-nkg-local-build
install-nkg-local-no-build: load-images prepare-nkg-dependencies deploy-updated-provisioner ## Install NKG from local build with provisioner on configured kind cluster but do not build the NKG image
.PHONY: install-nkg-local-no-build
install-nkg-local-no-build: prepare-nkg-dependencies load-images deploy-updated-provisioner ## Install NKG from local build with provisioner on configured kind cluster but do not build the NKG image

.PHONY: install-nkg-edge
install-nkg-edge: preload-nginx-container prepare-nkg-dependencies ## Install NKG with provisioner from edge on configured kind cluster
kubectl apply -f provisioner/provisioner.yaml
kubectl apply -f $(PROVISIONER_MANIFEST)

.PHONY: run-conformance-tests
run-conformance-tests: ## Run conformance tests
Expand All @@ -89,16 +87,18 @@ cleanup-conformance-tests: ## Clean up conformance tests fixtures
kubectl delete -f tests/conformance-rbac.yaml

.PHONY: uninstall-nkg
uninstall-nkg: ## Uninstall NKG on configured kind cluster
uninstall-nkg: uninstall-k8s-components undo-manifests-update ## Uninstall NKG on configured kind cluster and undo manifest changes

.PHONY: uninstall-k8s-components
uninstall-k8s-components: ## Uninstall installed components on configured kind cluster
kubectl delete -f $(NKG_MANIFEST)
./scripts/uninstall-gateway.sh $(GW_API_VERSION)
kubectl delete -f ../deploy/manifests/rbac.yaml
kubectl delete -f ../deploy/manifests/namespace.yaml
kubectl delete clusterrole nginx-gateway-provisioner
kubectl delete clusterrolebinding nginx-gateway-provisioner

.PHONY: undo-image-update
undo-image-update: ## Undo the NKG image name and tag in deployment manifest
git checkout -- $(NKG_DEPLOYMENT_MANIFEST)
.PHONY: undo-manifests-update
undo-manifests-update: ## Undo the changes in the manifest files
cd .. && make generate-manifests && cd -

.PHONY: reset-go-modules
reset-go-modules: ## Reset the go modules changes
Expand Down
43 changes: 14 additions & 29 deletions conformance/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@ preload-nginx-container Preload NGINX container on configured kind cluste
prepare-nkg-dependencies Install NKG dependencies on configured kind cluster
reset-go-modules Reset the go modules changes
run-conformance-tests Run conformance tests
undo-image-update Undo the NKG image name and tag in deployment manifest
uninstall-nkg Uninstall NKG on configured kind cluster
undo-manifests-update Undo the changes in the manifest files
uninstall-nkg Uninstall NKG on configured kind cluster and undo manifest changes
update-go-modules Update the gateway-api go modules to latest main version
update-nkg-manifest Update the NKG deployment manifest image name and imagePullPolicy
```
Expand All @@ -50,8 +50,11 @@ update-nkg-manifest Update the NKG deployment manifest image name and
| GATEWAY_CLASS | nginx | The gateway class that should be used for the tests |
| SUPPORTED_FEATURES | HTTPRoute,HTTPRouteQueryParamMatching, HTTPRouteMethodMatching,HTTPRoutePortRedirect, HTTPRouteSchemeRedirect | The supported features that should be tested by the conformance tests. Ensure the list is comma separated with no spaces. |
| EXEMPT_FEATURES | ReferenceGrant | The features that should not be tested by the conformance tests |
| NGINX_IMAGE | as defined in the ../deploy/manifests/deployment.yaml file | The NGINX image for the NKG deployments |
| NKG_DEPLOYMENT_MANIFEST | ../deploy/manifests/deployment.yaml | The location of the NKG deployment manifest |
| NGINX_IMAGE | as defined in the provisioner/static-deployment.yaml file | The NGINX image for the NKG deployments |
| NKG_MANIFEST | ../deploy/manifests/nginx-gateway.yaml | The location of the NKG manifest |
| SERVICE_MANIFEST | ../deploy/manifests/service/nodeport.yaml | The location of the NKG Service manifest |
| STATIC_MANIFEST | provisioner/static-deployment.yaml | The location of the NKG static deployment manifest |
| PROVISIONER_MANIFEST | provisioner/provisioner.yaml | The location of the NKG provisioner manifest |

### Step 1 - Create a kind Cluster

Expand All @@ -77,27 +80,17 @@ make install-nkg-local-build
```

#### *Option 2* Install Nginx Kubernetes Gateway from local already built image to configured kind cluster
You can optionally skip the actual *build* step.

```makefile
make install-nkg-local-no-build
```

**Note:** You can optionally skip the actual *build* step. However, if choosing
this option, the following step *must* be completed manually *before* the build step:

- Set NKG_PREFIX=<nkg_repo_name> NKG_TAG=<nkg_image_tag> to preferred values.
- Navigate to `deploy/manifests` and update values in `deployment.yaml` as specified in below code-block.
- Save the changes.

```text
.
..
containers:
- image: <nkg_repo_name>:<nkg_image_tag>
imagePullPolicy: Never
..
.
```
> Note: If choosing this option, the following step *must* be completed manually *before* you build the image:
```makefile
make update-nkg-manifest NKG_PREFIX=<nkg_repo_name> NKG_TAG=<nkg_image_tag>
```

#### *Option 3* Install Nginx Kubernetes Gateway from edge to configured kind cluster
You can also skip the build NKG image step and prepare the environment to instead use the `edge` image
Expand Down Expand Up @@ -153,15 +146,7 @@ make uninstall-nkg
make reset-go-modules
```

### Step 7 - Revert changes to the NKG deployment manifest
**Optional** Not required if using `edge` image
**Warning**: `make undo-image-update` will hard reset changes to the deploy/manifests/deployment.yaml file!

```makefile
make undo-image-update
```

### Step 8 - Delete kind cluster
### Step 7 - Delete kind cluster

```makefile
make delete-kind-cluster
Expand Down
99 changes: 99 additions & 0 deletions conformance/provisioner/static-deployment.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
---
# Source: nginx-kubernetes-gateway/templates/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx-gateway
namespace: nginx-gateway
labels:
app.kubernetes.io/name: nginx-gateway
app.kubernetes.io/instance: nginx-gateway
app.kubernetes.io/version: "edge"
spec:
# We only support a single replica for now
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: nginx-gateway
app.kubernetes.io/instance: nginx-gateway
template:
metadata:
labels:
app.kubernetes.io/name: nginx-gateway
app.kubernetes.io/instance: nginx-gateway
spec:
containers:
- args:
- static-mode
- --gateway-ctlr-name=k8s-gateway.nginx.org/nginx-gateway-controller
- --gatewayclass=nginx
env:
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
image: ghcr.io/nginxinc/nginx-kubernetes-gateway:edge
imagePullPolicy: Always
name: nginx-gateway
securityContext:
capabilities:
add:
- KILL
drop:
- ALL
runAsUser: 1001
volumeMounts:
- mountPath: /etc/nginx
name: nginx
- image: nginx:1.25
imagePullPolicy: Always
name: nginx
ports:
- containerPort: 80
name: http
- containerPort: 443
name: https
securityContext:
capabilities:
add:
- CHOWN
- NET_BIND_SERVICE
- SETGID
- SETUID
- DAC_OVERRIDE
drop:
- ALL
volumeMounts:
- mountPath: /etc/nginx
name: nginx
- mountPath: /etc/nginx/nginx.conf
name: nginx-conf
subPath: nginx.conf
- mountPath: /var/lib/nginx
name: var-lib-nginx
- mountPath: /usr/lib/nginx/modules/njs
name: njs-modules
initContainers:
- command:
- sh
- -c
- rm -r /etc/nginx/conf.d /etc/nginx/secrets; mkdir /etc/nginx/conf.d /etc/nginx/secrets
&& chown 1001:0 /etc/nginx/conf.d /etc/nginx/secrets
image: busybox:1.36
name: set-permissions
volumeMounts:
- mountPath: /etc/nginx
name: nginx
serviceAccountName: nginx-gateway
shareProcessNamespace: true
volumes:
- emptyDir: {}
name: nginx
- configMap:
name: nginx-gateway-conf
name: nginx-conf
- emptyDir: {}
name: var-lib-nginx
- configMap:
name: nginx-gateway-njs-modules
name: njs-modules
2 changes: 1 addition & 1 deletion deploy/helm-chart/Chart.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ name: nginx-kubernetes-gateway
description: NGINX Kubernetes Gateway
type: application
version: 0.1.0
appVersion: "0.4.0"
appVersion: "edge"
home: https://github.com/nginxinc/nginx-kubernetes-gateway
icon: https://raw.githubusercontent.com/nginxinc/nginx-kubernetes-gateway/tree/main/deploy/helm-chart/chart-icon.png
sources:
Expand Down
7 changes: 3 additions & 4 deletions deploy/helm-chart/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,6 @@ This chart deploys the NGINX Kubernetes Gateway in your Kubernetes cluster.
- [Helm 3.0+](https://helm.sh/docs/intro/install/)
- [kubectl](https://kubernetes.io/docs/tasks/tools/)

> Note: NGINX Kubernetes Gateway can only run in the `nginx-gateway` namespace. This limitation will be addressed in
the future releases.

### Installing the Gateway API resources

> Note: The Gateway API resources from the standard channel (the CRDs and the validating webhook) must be installed
Expand Down Expand Up @@ -87,9 +84,10 @@ To uninstall/delete the release `my-release`:

```shell
helm uninstall my-release -n nginx-gateway
kubectl delete ns nginx-gateway
```

The command removes all the Kubernetes components associated with the release and deletes the release.
These commands remove all the Kubernetes components associated with the release and deletes the release.

### Uninstalling the Gateway Resources

Expand All @@ -114,6 +112,7 @@ The following tables lists the configurable parameters of the NGINX Kubernetes G
|`nginxGateway.image.pullPolicy` | The `imagePullPolicy` for the NGINX Kubernetes Gateway image. | Always |
|`nginxGateway.gatewayClassName` | The name of the GatewayClass for the NGINX Kubernetes Gateway deployment. | nginx |
|`nginxGateway.gatewayControllerName` | The name of the Gateway controller. The controller name must be of the form: DOMAIN/PATH. The controller's domain is k8s-gateway.nginx.org. | k8s-gateway.nginx.org/nginx-gateway-controller |
|`nginxGateway.kind` | The kind of the NGINX Kubernetes Gateway installation - currently, only Deployment is supported. | deployment |
|`nginx.image.repository` | The repository for the NGINX image. | nginx |
|`nginx.image.tag` | The tag for the NGINX image. | 1.25 |
|`nginx.image.pullPolicy` | The `imagePullPolicy` for the NGINX image. | Always |
Expand Down
4 changes: 3 additions & 1 deletion deploy/helm-chart/templates/_helpers.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,14 @@ Create chart name and version as used by the chart label.
Common labels
*/}}
{{- define "nginx-gateway.labels" -}}
helm.sh/chart: {{ include "nginx-gateway.chart" . }}
{{ include "nginx-gateway.selectorLabels" . }}
{{- if .Chart.AppVersion }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
{{- if eq (default "helm" .Values.creator) "helm" }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
helm.sh/chart: {{ include "nginx-gateway.chart" . }}
{{- end -}}
{{- end }}

{{/*
Expand Down
Loading

0 comments on commit 36daa1d

Please sign in to comment.