From 1e1458901296caff35a7f6213078655e3a11c672 Mon Sep 17 00:00:00 2001 From: Luca Comellini Date: Thu, 2 May 2024 14:34:23 -0700 Subject: [PATCH] Add Gateway API CRDs to the repo with kustomize --- .github/workflows/ci.yml | 2 +- Makefile | 7 +++-- charts/nginx-gateway-fabric/README.md | 6 ++--- .../experimental/kustomization.yaml | 4 +++ .../gateway-api/standard/kustomization.yaml | 4 +++ config/crd/kustomization.yaml | 2 ++ docs/developer/quickstart.md | 4 +-- docs/developer/release-process.md | 21 +++++++-------- .../install-gateway-api-resources.md | 6 +++-- .../uninstall-gateway-api-resources.md | 18 ++++++------- .../installation/installing-ngf/helm.md | 6 ++--- .../installation/installing-ngf/manifests.md | 4 +-- tests/Makefile | 8 ++++++ tests/reconfig/setup.md | 27 ++++++++++--------- tests/scripts/install-gateway.sh | 24 ----------------- tests/scripts/uninstall-gateway.sh | 25 ----------------- 16 files changed, 68 insertions(+), 100 deletions(-) create mode 100644 config/crd/gateway-api/experimental/kustomization.yaml create mode 100644 config/crd/gateway-api/standard/kustomization.yaml delete mode 100755 tests/scripts/install-gateway.sh delete mode 100755 tests/scripts/uninstall-gateway.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 92cd4ae3a..a29dfb2e3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -309,7 +309,7 @@ jobs: make create-kind-cluster KIND_KUBE_CONFIG=${kube_config} echo "KUBECONFIG=${kube_config}" >> "$GITHUB_ENV" kind load docker-image ghcr.io/nginxinc/nginx-gateway-fabric:${{ steps.ngf-meta.outputs.version }} ghcr.io/nginxinc/nginx-gateway-fabric/nginx:${{ steps.nginx-meta.outputs.version }} - kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.1.0/standard-install.yaml + kubectl kustomize config/crd/gateway-api/standard | kubectl apply -f - - name: Install Chart run: > diff --git a/Makefile b/Makefile index b9c042c4d..a99c59068 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,6 @@ TELEMETRY_REPORT_PERIOD = 24h TELEMETRY_ENDPOINT=# if empty, NGF will report telemetry in its logs at debug level. TELEMETRY_ENDPOINT_INSECURE = false -GW_API_VERSION = 1.1.0 ENABLE_EXPERIMENTAL ?= false NODE_VERSION = $(shell cat .nvmrc) @@ -112,15 +111,15 @@ generate-crds: ## Generate CRDs and Go types using kubebuilder .PHONY: install-crds install-crds: ## Install CRDs - kubectl kustomize config/crd | kubectl apply -f - + kubectl kustomize $(SELF_DIR)config/crd | kubectl apply -f - .PHONY: install-gateway-crds install-gateway-crds: ## Install Gateway API CRDs - $(SELF_DIR)tests/scripts/install-gateway.sh $(GW_API_VERSION) $(ENABLE_EXPERIMENTAL) + kubectl kustomize $(SELF_DIR)config/crd/gateway-api/$(if $(filter true,$(ENABLE_EXPERIMENTAL)),experimental,standard) | kubectl apply -f - .PHONY: uninstall-gateway-crds uninstall-gateway-crds: ## Uninstall Gateway API CRDs - $(SELF_DIR)tests/scripts/uninstall-gateway.sh $(GW_API_VERSION) $(ENABLE_EXPERIMENTAL) + kubectl kustomize $(SELF_DIR)config/crd/gateway-api/$(if $(filter true,$(ENABLE_EXPERIMENTAL)),experimental,standard) | kubectl delete -f - .PHONY: generate-manifests generate-manifests: ## Generate manifests using Helm. diff --git a/charts/nginx-gateway-fabric/README.md b/charts/nginx-gateway-fabric/README.md index c6714495d..72784ad08 100644 --- a/charts/nginx-gateway-fabric/README.md +++ b/charts/nginx-gateway-fabric/README.md @@ -40,7 +40,7 @@ This chart deploys the NGINX Gateway Fabric in your Kubernetes cluster. > [see the Technical Specifications](https://github.com/nginxinc/nginx-gateway-fabric/blob/main/README.md#technical-specifications). ```shell -kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.1.0/standard-install.yaml +kubectl kustomize https://github.com/nginxinc/nginx-gateway-fabric/config/crd/gateway-api/standard | kubectl apply -f - ``` ## Installing the Chart @@ -128,7 +128,7 @@ Gateway Fabric - [see the Technical Specifications](../../README.md#technical-sp To upgrade the Gateway CRDs from [the Gateway API repo](https://github.com/kubernetes-sigs/gateway-api), run: ```shell -kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.1.0/standard-install.yaml +kubectl kustomize https://github.com/nginxinc/nginx-gateway-fabric/config/crd/gateway-api/standard | kubectl apply -f - ``` ### Upgrading the CRDs @@ -238,7 +238,7 @@ These commands remove all the Kubernetes components associated with the release To delete the Gateway API CRDs from [the Gateway API repo](https://github.com/kubernetes-sigs/gateway-api), run: ```shell -kubectl delete -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.1.0/standard-install.yaml +kubectl kustomize https://github.com/nginxinc/nginx-gateway-fabric/config/crd/gateway-api/standard | kubectl delete -f - ``` ## Configuration diff --git a/config/crd/gateway-api/experimental/kustomization.yaml b/config/crd/gateway-api/experimental/kustomization.yaml new file mode 100644 index 000000000..0eaa2d293 --- /dev/null +++ b/config/crd/gateway-api/experimental/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - https://github.com/kubernetes-sigs/gateway-api/config/crd/experimental?timeout=120&ref=v1.1.0 diff --git a/config/crd/gateway-api/standard/kustomization.yaml b/config/crd/gateway-api/standard/kustomization.yaml new file mode 100644 index 000000000..4f2165d71 --- /dev/null +++ b/config/crd/gateway-api/standard/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- https://github.com/kubernetes-sigs/gateway-api/config/crd?timeout=120&ref=v1.1.0 diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 873746dd8..0128a189c 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -1,3 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization resources: - bases/gateway.nginx.org_clientsettingspolicies.yaml - bases/gateway.nginx.org_nginxgateways.yaml diff --git a/docs/developer/quickstart.md b/docs/developer/quickstart.md index 3d2985803..83b889371 100644 --- a/docs/developer/quickstart.md +++ b/docs/developer/quickstart.md @@ -133,13 +133,13 @@ This will build the docker images `nginx-gateway-fabric:` and `nginx- 3. Install Gateway API CRDs: ```shell - kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.1.0/standard-install.yaml + kubectl kustomize config/crd/gateway-api/standard | kubectl apply -f - ``` If you're implementing experimental Gateway API features, install Gateway API CRDs from the experimental channel: ```shell - kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.1.0/experimental-install.yaml + kubectl kustomize config/crd/gateway-api/experimental | kubectl apply -f - ``` 4. Install NGF using your custom image and expose NGF with a NodePort Service: diff --git a/docs/developer/release-process.md b/docs/developer/release-process.md index c986176e0..81b74b418 100644 --- a/docs/developer/release-process.md +++ b/docs/developer/release-process.md @@ -37,10 +37,8 @@ To create a new release, follow these steps: 7. Create a release branch following the `release-X.Y` naming convention. 8. Run the [Release PR](./../../.github/workflows/release-pr.yml) workflow to update the repo files for the release. Then there are a few manual steps to complete: 1. Update the tag of NGF container images used in the [provisioner manifest](/tests/conformance/provisioner/provisioner.yaml). - 2. Update any installation instructions to ensure that the supported Gateway API is correct. - Specifically, helm README and `site/content/includes/installation/install-gateway-api-resources.md`. - 3. Update the [README](/README.md) to include information about the release. - 4. Update the [changelog](/CHANGELOG.md). There is going to be a new blank section generated by the automation that needs to be adjusted accordingly. + 2. Update the [README](/README.md) to include information about the release. + 3. Update the [changelog](/CHANGELOG.md). There is going to be a new blank section generated by the automation that needs to be adjusted accordingly. The changelog includes only important (from the user perspective) changes to NGF. This is in contrast with the autogenerated full changelog, which is created in the next step. As a starting point, copy the important features, bug fixes, and dependencies from the autogenerated @@ -51,14 +49,13 @@ To create a new release, follow these steps: - Build NGF, NGINX and NGINX Plus container images with the release tag `X.Y.Z` and push them to the registries. - Package and publish the Helm chart to the registry. - Create a GitHub release with an autogenerated changelog and attached release artifacts. -10. Prepare and merge a PR into the main branch to update with similar information that you did in the release -branch docs PR. Specifically: - 1. [README](/README.md) to include the information about the latest release. - 2. [changelog](/CHANGELOG.md). - 3. Helm chart `version` field. - 4. `VERSION` in top-level Makefile. - 5. Any references in the docs to the previous release. - 6. Any installation instructions to ensure that the supported Gateway API and NGF versions are correct.Specifically, helm README and `site/content/includes/installation/install-gateway-api-resources.md`. +10. Prepare and merge a PR into the main branch to update with similar information that you did in the release branch docs PR. Specifically: + 1. [README](/README.md) to include the information about the latest release. + 2. [changelog](/CHANGELOG.md). + 3. Helm chart `version` field. + 4. `VERSION` in top-level Makefile. + 5. Any references in the docs to the previous release. + 6. Any installation instructions to ensure that the supported Gateway API and NGF versions are correct. Specifically, helm README. 11. Ask the docs team to update the production branch for NGF in Netlify to our latest release branch and run the deployment pipeline. 12. Close the issue created in Step 1. 13. Ensure that the [associated milestone](https://github.com/nginxinc/nginx-gateway-fabric/milestones) is closed. diff --git a/site/content/includes/installation/install-gateway-api-resources.md b/site/content/includes/installation/install-gateway-api-resources.md index 459ea709a..a0ba68a58 100644 --- a/site/content/includes/installation/install-gateway-api-resources.md +++ b/site/content/includes/installation/install-gateway-api-resources.md @@ -7,12 +7,14 @@ docs: "DOCS-1438" To install the Gateway API resources, run the following: ```shell -kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.1.0/standard-install.yaml +kubectl kustomize "https://github.com/nginxinc/nginx-gateway-fabric/config/crd/gateway-api/standard?ref=v1.2.0" | kubectl apply -f - ``` +{{}}If you plan to use the `edge` version of NGINX Gateway Fabric, you can replace the vesion in `ref` with `main`, for example `ref=main`.{{}} + Alternatively, you can install the Gateway API resources from the experimental channel. We support a subset of the additional features provided by the experimental channel. To install from the experimental channel, run the following: ```shell -kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.1.0/experimental-install.yaml +kubectl kustomize "https://github.com/nginxinc/nginx-gateway-fabric/config/crd/gateway-api/experimental?ref=v1.2.0" | kubectl apply -f - ``` diff --git a/site/content/includes/installation/uninstall-gateway-api-resources.md b/site/content/includes/installation/uninstall-gateway-api-resources.md index 9fa16ace2..eda543598 100644 --- a/site/content/includes/installation/uninstall-gateway-api-resources.md +++ b/site/content/includes/installation/uninstall-gateway-api-resources.md @@ -2,16 +2,16 @@ docs: "DOCS-1436" --- - {{}}This will remove all corresponding custom resources in your entire cluster, across all namespaces. Double-check to make sure you don't have any custom resources you need to keep, and confirm that there are no other Gateway API implementations active in your cluster.{{}} +{{}}This will remove all corresponding custom resources in your entire cluster, across all namespaces. Double-check to make sure you don't have any custom resources you need to keep, and confirm that there are no other Gateway API implementations active in your cluster.{{}} - To uninstall the Gateway API resources, run the following: +To uninstall the Gateway API resources, run the following: - ```shell - kubectl delete -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.1.0/standard-install.yaml - ``` +```shell +kubectl kustomize "https://github.com/nginxinc/nginx-gateway-fabric/config/crd/gateway-api/experimental?ref=v1.2.0" | kubectl delete -f - +``` - Alternatively, if you installed the Gateway APIs from the experimental channel, run the following: +Alternatively, if you installed the Gateway APIs from the experimental channel, run the following: - ```shell - kubectl delete -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.1.0/experimental-install.yaml - ``` +```shell +kubectl kustomize "https://github.com/nginxinc/nginx-gateway-fabric/config/crd/gateway-api/experimental?ref=v1.2.0" | kubectl delete -f - +``` diff --git a/site/content/installation/installing-ngf/helm.md b/site/content/installation/installing-ngf/helm.md index 658425f6e..49f19084a 100644 --- a/site/content/installation/installing-ngf/helm.md +++ b/site/content/installation/installing-ngf/helm.md @@ -126,17 +126,17 @@ To upgrade NGINX Gateway Fabric and get the latest features and improvements, ta To upgrade your Gateway API resources, take the following steps: - Verify the Gateway API resources are compatible with your NGINX Gateway Fabric version. Refer to the [Technical Specifications]({{< relref "reference/technical-specifications.md" >}}) for details. -- Review the [release notes](https://github.com/kubernetes-sigs/gateway-api/releases/tag/v1.1.0) for any important upgrade-specific information. +- Review the [release notes](https://github.com/kubernetes-sigs/gateway-api/releases) for any important upgrade-specific information. - To upgrade the Gateway API resources, run: ```shell - kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.1.0/standard-install.yaml + kubectl kustomize "https://github.com/nginxinc/nginx-gateway-fabric/config/crd/gateway-api/standard?ref=v1.2.0" | kubectl apply -f - ``` or, if you installed the from the experimental channel: ```shell - kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.1.0/experimental-install.yaml + kubectl kustomize "https://github.com/nginxinc/nginx-gateway-fabric/config/crd/gateway-api/experimental?ref=v1.2.0" | kubectl apply -f - ``` ### Upgrade NGINX Gateway Fabric CRDs diff --git a/site/content/installation/installing-ngf/manifests.md b/site/content/installation/installing-ngf/manifests.md index f6200be88..d842ef4c0 100644 --- a/site/content/installation/installing-ngf/manifests.md +++ b/site/content/installation/installing-ngf/manifests.md @@ -131,13 +131,13 @@ To upgrade NGINX Gateway Fabric and get the latest features and improvements, ta - To upgrade the Gateway API resources, run: ```shell - kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.1.0/standard-install.yaml + kubectl kustomize "https://github.com/nginxinc/nginx-gateway-fabric/config/crd/gateway-api/standard?ref=v1.2.0" | kubectl apply -f - ``` or, if you installed the from the experimental channel: ```shell - kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.1.0/experimental-install.yaml + kubectl kustomize "https://github.com/nginxinc/nginx-gateway-fabric/config/crd/gateway-api/experimental?ref=v1.2.0" | kubectl apply -f - ``` 1. **Upgrade NGINX Gateway Fabric CRDs:** diff --git a/tests/Makefile b/tests/Makefile index 4e4516d90..800708f7a 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -4,6 +4,7 @@ CONFORMANCE_TAG = latest## Tag for the conformance test runner image GATEWAY_CLASS = nginx## Gateway class to use GINKGO_FLAGS = GINKGO_LABEL = +GW_API_VERSION ?= $(shell yq eval '.resources[0] | split("ref=v") | .[1]' ../config/crd/gateway-api/standard/kustomization.yaml)## Supported Gateway API version from current NGF GW_API_PREV_VERSION ?= 1.0.0## Supported Gateway API version from previous NGF release GW_SERVICE_TYPE=NodePort## Service type to use for the gateway GW_SVC_GKE_INTERNAL=false @@ -159,6 +160,13 @@ deploy-updated-provisioner: ## Update provisioner manifest and deploy to the con generate-static-deployment: helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) --set metrics.enable=false --set nginxGateway.productTelemetry.enable=false -n nginx-gateway -s templates/deployment.yaml --set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginxGateway.image.pullPolicy=Never --set nginx.image.repository=$(NGINX_PREFIX) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never --set nginxGateway.gwAPIExperimentalFeatures.enable=$(ENABLE_EXPERIMENTAL) --set nginx.plus=$(PLUS_ENABLED) > $(SELF_DIR)config/tests/static-deployment.yaml +# this target is used to install the gateway-api CRDs from the main branch (only used in the nightly CI job) +# it overrides the target in the main Makefile when the GW_API_VERSION is set to main +ifeq ($(GW_API_VERSION),main) +install-gateway-crds: + kubectl kustomize "https://github.com/kubernetes-sigs/gateway-api/config/crd/$(if $(filter true,$(ENABLE_EXPERIMENTAL)),experimental,)?timeout=120&ref=main" | kubectl apply -f - +endif + .PHONY: install-ngf-local-build install-ngf-local-build: deploy-updated-provisioner diff --git a/tests/reconfig/setup.md b/tests/reconfig/setup.md index 60db6c352..46a9c6f71 100644 --- a/tests/reconfig/setup.md +++ b/tests/reconfig/setup.md @@ -24,7 +24,7 @@ ## Test Environment - The following cluster will be sufficient: +The following cluster will be sufficient: - A Kubernetes cluster with 4 nodes on GKE - Node: e2-medium (2 vCPU, 4GB memory) @@ -32,10 +32,10 @@ ## Setup 1. Create cloud cluster -2. Deploy CRDs: +2. Install Gateway API Resources: ```bash - kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.1.0/standard-install.yaml + kubectl kustomize config/crd/gateway-api/standard | kubectl apply -f - ``` 3. Deploy NGF from edge using Helm install and wait for LoadBalancer Service to be ready @@ -59,16 +59,17 @@ fixtures `scripts/delete-multiple.sh` which takes a number (needs to be the same number as what was used in the create script.) 5. After each individual test: - - Describe the Gateway resource and make sure the status is correct. - - Check the logs of both NGF containers for errors. - - Parse the logs for TimeToReady numbers (see steps 6-7 below). - - Grab metrics. - Note: You can expose metrics by running the below snippet and then navigating to `127.0.0.1:9113/metrics`: - - ```console - GW_POD=$(k get pods -n nginx-gateway | sed -n '2s/^\([^[:space:]]*\).*$/\1/p') - kubectl port-forward $GW_POD -n nginx-gateway 9113:9113 & - ``` + + - Describe the Gateway resource and make sure the status is correct. + - Check the logs of both NGF containers for errors. + - Parse the logs for TimeToReady numbers (see steps 6-7 below). + - Grab metrics. + Note: You can expose metrics by running the below snippet and then navigating to `127.0.0.1:9113/metrics`: + + ```console + GW_POD=$(k get pods -n nginx-gateway | sed -n '2s/^\([^[:space:]]*\).*$/\1/p') + kubectl port-forward $GW_POD -n nginx-gateway 9113:9113 & + ``` 6. Measure NGINX Reloads and Time to Ready Results 1. TimeToReadyTotal as described in each test - NGF logs. diff --git a/tests/scripts/install-gateway.sh b/tests/scripts/install-gateway.sh deleted file mode 100755 index d2c942dba..000000000 --- a/tests/scripts/install-gateway.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -if [ -z $1 ]; then - echo "gateway API version argument not set; exiting" - exit 1 -fi - -if [ $1 == "main" ]; then - temp_dir=$(mktemp -d) - cd ${temp_dir} - curl -s https://codeload.github.com/kubernetes-sigs/gateway-api/tar.gz/main | tar -xz --strip=2 gateway-api-main/config - if [ $2 == "true" ]; then - kubectl apply -f crd/experimental - else - kubectl apply -f crd/standard - fi - rm -rf ${temp_dir} -else - if [ $2 == "true" ]; then - kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v$1/experimental-install.yaml - else - kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v$1/standard-install.yaml - fi -fi diff --git a/tests/scripts/uninstall-gateway.sh b/tests/scripts/uninstall-gateway.sh deleted file mode 100755 index 74f5d6a29..000000000 --- a/tests/scripts/uninstall-gateway.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -if [ -z $1 ]; then - echo "gatway API version argument not set; exiting" - exit 1 -fi - - -if [ $1 == "main" ]; then - temp_dir=$(mktemp -d) - cd ${temp_dir} - curl -s https://codeload.github.com/kubernetes-sigs/gateway-api/tar.gz/main | tar -xz --strip=2 gateway-api-main/config - if [ $2 == "true" ]; then - kubectl delete -f crd/experimental - else - kubectl delete -f crd/standard - fi - rm -rf ${temp_dir} -else - if [ $2 == "true" ]; then - kubectl delete -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v$1/experimental-install.yaml - else - kubectl delete -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v$1/standard-install.yaml - fi -fi