Add readiness endpoint

nginxinc · Jul 9, 2020 · 2f512b7 · 2f512b7
1 parent d634eed
commit 2f512b7
Show file tree

Hide file tree

Showing 13 changed files with 124 additions and 2 deletions.
diff --git a/cmd/nginx-ingress/main.go b/cmd/nginx-ingress/main.go
@@ -154,6 +154,10 @@ var (
 	spireAgentAddress = flag.String("spire-agent-address", "",
 		`Specifies the address of the running Spire agent. For use with NGINX Service Mesh only. If the flag is set,
 			but the Ingress Controller is not able to connect with the Spire Agent, the Ingress Controller will fail to start.`)
+
+	readyStatus = flag.Bool("ready-status", true, "Enables the readiness endpoint '/nginx-ready'. The endpoint returns a success code when NGINX has loaded all the config after the startup")
+
+	readyStatusPort = flag.Int("ready-status-port", 8081, "Set the port where the readiness endpoint is exposed. [1024 - 65535]")
 )
 
 func main() {
@@ -189,6 +193,11 @@ func main() {
 		glog.Fatalf("Invalid value for prometheus-metrics-listen-port: %v", metricsPortValidationError)
 	}
 
+	readyStatusPortValidationError := validatePort(*readyStatusPort)
+	if readyStatusPortValidationError != nil {
+		glog.Fatalf("Invalid value for ready-status-port: %v", readyStatusPortValidationError)
+	}
+
 	allowedCIDRs, err := parseNginxStatusAllowCIDRs(*nginxStatusAllowCIDRs)
 	if err != nil {
 		glog.Fatalf(`Invalid value for nginx-status-allow-cidrs: %v`, err)
@@ -514,11 +523,21 @@ func main() {
 
 	lbc := k8s.NewLoadBalancerController(lbcInput)
 
+	if *readyStatus {
+		go func() {
+			port := fmt.Sprintf(":%v", *readyStatusPort)
+			s := http.NewServeMux()
+			s.HandleFunc("/nginx-ready", ready(lbc))
+			glog.Fatal(http.ListenAndServe(port, s))
+		}()
+	}
+
 	if *appProtect {
 		go handleTerminationWithAppProtect(lbc, nginxManager, nginxDone, aPAgentDone, aPPluginDone)
 	} else {
 		go handleTermination(lbc, nginxManager, nginxDone)
 	}
+
 	lbc.Run()
 
 	for {
@@ -708,3 +727,14 @@ func parseReloadTimeout(appProtectEnabled bool, timeout int) int {
 
 	return defaultTimeout
 }
+
+func ready(lbc *k8s.LoadBalancerController) http.HandlerFunc {
+	return func(w http.ResponseWriter, _ *http.Request) {
+		if !lbc.IsNginxReady() {
+			http.Error(w, http.StatusText(http.StatusServiceUnavailable), http.StatusServiceUnavailable)
+			return
+		}
+		w.WriteHeader(http.StatusOK)
+		fmt.Fprintln(w, "Ready")
+	}
+}
diff --git a/deployments/daemon-set/nginx-ingress.yaml b/deployments/daemon-set/nginx-ingress.yaml
@@ -27,8 +27,15 @@ spec:
         - name: https
           containerPort: 443
           hostPort: 443
+        - name: readiness-port
+          containerPort: 8081
        #- name: prometheus
          #containerPort: 9113
+        readinessProbe:
+         httpGet:
+           path: /nginx-ready
+           port: readiness-port
+         periodSeconds: 1
         securityContext:
           allowPrivilegeEscalation: true
           runAsUser: 101 #nginx

diff --git a/deployments/daemon-set/nginx-plus-ingress.yaml b/deployments/daemon-set/nginx-plus-ingress.yaml
@@ -27,8 +27,15 @@ spec:
         - name: https
           containerPort: 443
           hostPort: 443
+        - name: readiness-port
+          containerPort: 8081
        #- name: prometheus
          #containerPort: 9113
+        readinessProbe:
+          httpGet:
+            path: /nginx-ready
+            port: readiness-port
+          periodSeconds: 1
         securityContext:
           allowPrivilegeEscalation: true
           runAsUser: 101 #nginx

diff --git a/deployments/deployment/nginx-ingress.yaml b/deployments/deployment/nginx-ingress.yaml
@@ -26,8 +26,15 @@ spec:
           containerPort: 80
         - name: https
           containerPort: 443
+        - name: readiness-port
+          containerPort: 8081
        #- name: prometheus
          #containerPort: 9113
+        readinessProbe:
+          httpGet:
+            path: /nginx-ready
+            port: readiness-port
+          periodSeconds: 1
         securityContext:
           allowPrivilegeEscalation: true
           runAsUser: 101 #nginx

diff --git a/deployments/deployment/nginx-plus-ingress.yaml b/deployments/deployment/nginx-plus-ingress.yaml
@@ -26,8 +26,15 @@ spec:
           containerPort: 80
         - name: https
           containerPort: 443
+        - name: readiness-port
+          containerPort: 8081
        #- name: prometheus
          #containerPort: 9113
+        readinessProbe:
+          httpGet:
+            path: /nginx-ready
+            port: readiness-port
+          periodSeconds: 1
         securityContext:
           allowPrivilegeEscalation: true
           runAsUser: 101 #nginx

diff --git a/deployments/helm-chart/README.md b/deployments/helm-chart/README.md
@@ -255,6 +255,8 @@ Parameter | Description | Default
 `controller.reportIngressStatus.annotations` | The annotations of the leader election configmap. | {}
 `controller.pod.annotations` | The annotations of the Ingress Controller pod. | {}
 `controller.appprotect.enable` | Enables the App Protect module in the Ingress Controller. | false
+`controller.readyStatus.enable` | Enables the readiness endpoint `"/nginx-ready"`. The endpoint returns a success code when NGINX has loaded all the config after the startup. This also configures a readiness probe for the Ingress Controller pods that uses the readiness endpoint. | true
+`controller.readyStaus.port` | The HTTP port for the readiness endpoint. | 8081
 `rbac.create` | Configures RBAC. | true
 `prometheus.create` | Expose NGINX or NGINX Plus metrics in the Prometheus format. | false
 `prometheus.port` | Configures the port to scrape the metrics. | 9113

diff --git a/deployments/helm-chart/templates/controller-daemonset.yaml b/deployments/helm-chart/templates/controller-daemonset.yaml
@@ -64,6 +64,15 @@ spec:
 {{- if .Values.prometheus.create }}
         - name: prometheus
           containerPort: {{ .Values.prometheus.port }}
+{{- end }}
+{{- if .Values.controller.readyStatus.enable }}
+        - name: readiness-port
+          containerPort: {{ .Values.controller.readyStatus.port}}
+        readinessProbe:
+          httpGet:
+            path: /nginx-ready
+            port: readiness-port
+          periodSeconds: 1
 {{- end }}
         securityContext:
           allowPrivilegeEscalation: true
@@ -137,4 +146,6 @@ spec:
           - -global-configuration=$(POD_NAMESPACE)/{{ .Release.Name }}
 {{- end }}
 {{- end }}
+          - -ready-status={{ .Values.controller.readyStatus.enable }}
+          - -ready-status-port={{ .Values.controller.readyStatus.port }}
 {{- end }}
diff --git a/deployments/helm-chart/templates/controller-deployment.yaml b/deployments/helm-chart/templates/controller-deployment.yaml
@@ -62,6 +62,15 @@ spec:
 {{- if .Values.prometheus.create }}
         - name: prometheus
           containerPort: {{ .Values.prometheus.port }}
+{{- end }}
+{{- if .Values.controller.readyStatus.enable }}
+        - name: readiness-port
+          containerPort: {{ .Values.controller.readyStatus.port}}
+        readinessProbe:
+          httpGet:
+            path: /nginx-ready
+            port: readiness-port
+          periodSeconds: 1
 {{- end }}
         resources:
 {{ toYaml .Values.controller.resources | indent 10 }}
@@ -135,4 +144,6 @@ spec:
           - -global-configuration=$(POD_NAMESPACE)/{{ .Release.Name }}
 {{- end }}
 {{- end }}
+          - -ready-status={{ .Values.controller.readyStatus.enable }}
+          - -ready-status-port={{ .Values.controller.readyStatus.port }}
 {{- end }}
diff --git a/deployments/helm-chart/values.yaml b/deployments/helm-chart/values.yaml
@@ -260,6 +260,13 @@ controller:
   ## The PriorityClass of the ingress controller pods.
   priorityClassName:
 
+  readyStatus:
+    ## Enables readiness endpoint "/nginx-ready". The endpoint returns a success code when NGINX has loaded all the config after startup.
+    enable: true
+
+    ## Set the port where the readiness endpoint is exposed.
+    port: 8081
+
 rbac:
   ## Configures RBAC.
   create: true

diff --git a/docs-web/configuration/global-configuration/command-line-arguments.md b/docs-web/configuration/global-configuration/command-line-arguments.md
@@ -188,5 +188,15 @@ Below we describe the available command-line arguments:
    Requires :option:`-nginx-plus`
 
 	 - If the argument is set, but `nginx-plus` is set to false, the Ingress Controller will fail to start.
-	 
+
+.. option:: -ready-status
+
+ 	Enables the readiness endpoint "/nginx-ready". The endpoint returns a success code when NGINX has loaded all the config after the startup. (default true)
+
+.. option:: -ready-status-port
+
+	The HTTP port for the readiness endpoint.
+
+	Format: ``[1024 - 65535]`` (default 8081)
+
 ```
diff --git a/docs-web/installation/installation-with-helm.md b/docs-web/installation/installation-with-helm.md
@@ -385,6 +385,12 @@ The following tables lists the configurable parameters of the NGINX Ingress cont
    * - ``controller.pod.annotations``
      - The annotations of the Ingress Controller pod.
      - {}
+   * - ``controller.readyStatus.enable``
+     - Enables the readiness endpoint `"/nginx-ready"`. The endpoint returns a success code when NGINX has loaded all the config after the startup. This also configures a readiness probe for the Ingress Controller pods that uses the readiness endpoint.
+     - true
+   * - ``controller.readyStaus.port``
+     - The HTTP port for the readiness endpoint.
+     - 8081
    * - ``rbac.create``
      - Configures RBAC.
      - true

diff --git a/internal/k8s/controller.go b/internal/k8s/controller.go
@@ -145,6 +145,7 @@ type LoadBalancerController struct {
 	transportServerValidator      *validation.TransportServerValidator
 	spiffeController              *spiffeController
 	syncLock                      sync.Mutex
+	isNginxReady                  bool
 }
 
 var keyFunc = cache.DeletionHandlingMetaNamespaceKeyFunc
@@ -271,7 +272,6 @@ func NewLoadBalancerController(input NewLoadBalancerControllerInput) *LoadBalanc
 	}
 
 	lbc.updateIngressMetrics()
-
 	return lbc
 }
 
@@ -835,6 +835,11 @@ func (lbc *LoadBalancerController) sync(task task) {
 	case appProtectLogConf:
 		lbc.syncAppProtectLogConf(task)
 	}
+
+	if !lbc.isNginxReady && lbc.syncQueue.Len() == 0 {
+		lbc.isNginxReady = true
+		glog.V(3).Infof("NGINX is ready")
+	}
 }
 
 func (lbc *LoadBalancerController) syncPolicy(task task) {
@@ -935,6 +940,7 @@ func (lbc *LoadBalancerController) syncPolicy(task task) {
 			}
 		}
 	}
+
 }
 
 func (lbc *LoadBalancerController) syncTransportServer(task task) {
@@ -3255,3 +3261,8 @@ func (lbc *LoadBalancerController) findIngressesForAppProtectResource(namespace
 	}
 	return apIngs
 }
+
+// IsNginxReady returns ready status of NGINX
+func (lbc *LoadBalancerController) IsNginxReady() bool {
+	return lbc.isNginxReady
+}
diff --git a/internal/k8s/task_queue.go b/internal/k8s/task_queue.go
@@ -65,6 +65,12 @@ func (tq *taskQueue) Requeue(task task, err error) {
 	tq.queue.Add(task)
 }
 
+// Len returns the length of the queue
+func (tq *taskQueue) Len() int {
+	glog.V(3).Infof("The queue has %v element(s)", tq.queue.Len())
+	return tq.queue.Len()
+}
+
 // RequeueAfter adds the task to the queue after the given duration
 func (tq *taskQueue) RequeueAfter(t task, err error, after time.Duration) {
 	glog.Errorf("Requeuing %v after %s, err %v", t.Key, after.String(), err)