Add readiness endpoint

cmd/nginx-ingress/main.go

@@ -154,6 +154,10 @@ var (
 	spireAgentAddress = flag.String("spire-agent-address", "",
 		`Specifies the address of the running Spire agent. For use with NGINX Service Mesh only. If the flag is set,
 			but the Ingress Controller is not able to connect with the Spire Agent, the Ingress Controller will fail to start.`)
+
+	readyStatus = flag.Bool("ready-status", false, "Enables the readiness endpoint '/nginx-ready'. The endpoint returns a success code when NGINX has loaded all the config after the startup")
+
+	readyStatusPort = flag.Int("ready-status-port", 8081, "Set the port where the readiness endpoint is exposed. [1024 - 65535]")
 )
 
 func main() {
@@ -189,6 +193,11 @@ func main() {
 		glog.Fatalf("Invalid value for prometheus-metrics-listen-port: %v", metricsPortValidationError)
 	}
 
+	readyStatusPortValidationError := validatePort(*readyStatusPort)
+	if readyStatusPortValidationError != nil {
+		glog.Fatalf("Invalid value for ready-status-port: %v", readyStatusPortValidationError)
+	}
+
 	allowedCIDRs, err := parseNginxStatusAllowCIDRs(*nginxStatusAllowCIDRs)
 	if err != nil {
 		glog.Fatalf(`Invalid value for nginx-status-allow-cidrs: %v`, err)
@@ -510,15 +519,26 @@ func main() {
 		GlobalConfigurationValidator: globalConfigurationValidator,
 		TransportServerValidator:     transportServerValidator,
 		SpireAgentAddress:            *spireAgentAddress,
+		FirstRun:                     *readyStatus,
 	}
 
 	lbc := k8s.NewLoadBalancerController(lbcInput)
 
+	if *readyStatus {
+		go func() {
+			port := fmt.Sprintf(":%v", *readyStatusPort)
+			s := http.NewServeMux()
+			s.HandleFunc("/nginx-ready", ready(lbc))
+			glog.Fatal(http.ListenAndServe(port, s))
+		}()
+	}
+
 	if *appProtect {
 		go handleTerminationWithAppProtect(lbc, nginxManager, nginxDone, aPAgentDone, aPPluginDone)
 	} else {
 		go handleTermination(lbc, nginxManager, nginxDone)
 	}
+
 	lbc.Run()
 
 	for {
@@ -708,3 +728,14 @@ func parseReloadTimeout(appProtectEnabled bool, timeout int) int {
 
 	return defaultTimeout
 }
+
+func ready(lbc *k8s.LoadBalancerController) http.HandlerFunc {
+	return func(w http.ResponseWriter, _ *http.Request) {
+		if !lbc.IsNginxReady() {
+			http.Error(w, http.StatusText(http.StatusServiceUnavailable), http.StatusServiceUnavailable)
+			return
+		}
+		w.WriteHeader(http.StatusOK)
+		fmt.Fprintln(w, "Ready")
+	}
+}

deployments/daemon-set/nginx-ingress.yaml

@@ -29,6 +29,13 @@ spec:
           hostPort: 443
        #- name: prometheus
          #containerPort: 9113
+       #- name: readiness-port
+         #containerPort: 8081
+       #readinessProbe:
+         #httpGet:
+           #path: /nginx-ready
+           #port: readiness-port
+         #periodSeconds: 1
         securityContext:
           allowPrivilegeEscalation: true
           runAsUser: 101 #nginx
@@ -50,6 +57,7 @@ spec:
           - -nginx-configmaps=$(POD_NAMESPACE)/nginx-config
           - -default-server-tls-secret=$(POD_NAMESPACE)/default-server-secret
          #- -v=3 # Enables extensive logging. Useful for troubleshooting.
+         #- -ready-status
          #- -report-ingress-status
          #- -external-service=nginx-ingress
          #- -enable-prometheus-metrics

deployments/daemon-set/nginx-plus-ingress.yaml

@@ -29,6 +29,13 @@ spec:
           hostPort: 443
        #- name: prometheus
          #containerPort: 9113
+       #- name: readiness-port
+         #containerPort: 8081
+       #readinessProbe:
+         #httpGet:
+           #path: /nginx-ready
+           #port: readiness-port
+         #periodSeconds: 1
         securityContext:
           allowPrivilegeEscalation: true
           runAsUser: 101 #nginx
@@ -52,6 +59,7 @@ spec:
           - -default-server-tls-secret=$(POD_NAMESPACE)/default-server-secret
          #- -enable-app-protect        
          #- -v=3 # Enables extensive logging. Useful for troubleshooting.
+         #- -ready-status
          #- -report-ingress-status
          #- -external-service=nginx-ingress
          #- -enable-prometheus-metrics

deployments/deployment/nginx-ingress.yaml

@@ -28,6 +28,13 @@ spec:
           containerPort: 443
        #- name: prometheus
          #containerPort: 9113
+       #- name: readiness-port
+         #containerPort: 8081
+       #readinessProbe:
+         #httpGet:
+           #path: /nginx-ready
+           #port: readiness-port
+         #periodSeconds: 1
         securityContext:
           allowPrivilegeEscalation: true
           runAsUser: 101 #nginx
@@ -49,6 +56,7 @@ spec:
           - -nginx-configmaps=$(POD_NAMESPACE)/nginx-config
           - -default-server-tls-secret=$(POD_NAMESPACE)/default-server-secret
          #- -v=3 # Enables extensive logging. Useful for troubleshooting.
+         #- -ready-status
          #- -report-ingress-status
          #- -external-service=nginx-ingress
          #- -enable-prometheus-metrics

deployments/deployment/nginx-plus-ingress.yaml

@@ -28,6 +28,13 @@ spec:
           containerPort: 443
        #- name: prometheus
          #containerPort: 9113
+       #- name: readiness-port
+         #containerPort: 8081
+       #readinessProbe:
+         #httpGet:
+           #path: /nginx-ready
+           #port: readiness-port
+         #periodSeconds: 1
         securityContext:
           allowPrivilegeEscalation: true
           runAsUser: 101 #nginx
@@ -51,6 +58,7 @@ spec:
           - -default-server-tls-secret=$(POD_NAMESPACE)/default-server-secret
          #- -enable-app-protect
          #- -v=3 # Enables extensive logging. Useful for troubleshooting.
+         #- -ready-status
          #- -report-ingress-status
          #- -external-service=nginx-ingress
          #- -enable-prometheus-metrics

deployments/helm-chart/README.md

@@ -255,6 +255,8 @@ Parameter | Description | Default
 `controller.reportIngressStatus.annotations` | The annotations of the leader election configmap. | {}
 `controller.pod.annotations` | The annotations of the Ingress Controller pod. | {}
 `controller.appprotect.enable` | Enables the App Protect module in the Ingress Controller. | false
+`controller.readyStatus.enable` | Enables the readiness endpoint `"/nginx-ready"`. The endpoint returns a success code when NGINX has loaded all the config after the startup. This also configures a readiness probe for the Ingress Controller pods that uses the readiness endpoint. | false
+ `controller.readyStaus.port` | The HTTP port for the readiness endpoint. | 8081
 `rbac.create` | Configures RBAC. | true
 `prometheus.create` | Expose NGINX or NGINX Plus metrics in the Prometheus format. | false
 `prometheus.port` | Configures the port to scrape the metrics. | 9113

deployments/helm-chart/templates/controller-daemonset.yaml

@@ -64,6 +64,15 @@ spec:
 {{- if .Values.prometheus.create }}
         - name: prometheus
           containerPort: {{ .Values.prometheus.port }}
+{{- end }}
+{{- if .Values.controller.readyStatus.enable }}
+        - name: readiness-port
+          containerPort: {{ .Values.controller.readyStatus.port}}
+        readinessProbe:
+          httpGet:
+            path: /nginx-ready
+            port: readiness-port
+          periodSeconds: 1
 {{- end }}
         securityContext:
           allowPrivilegeEscalation: true
@@ -137,4 +146,8 @@ spec:
           - -global-configuration=$(POD_NAMESPACE)/{{ .Release.Name }}
 {{- end }}
 {{- end }}
+{{- if .Values.controller.readyStatus.enable }}
+          - -ready-status
+          - -ready-status-port={{ .Values.controller.readyStaus.port }}
+{{- end }}
 {{- end }}

deployments/helm-chart/templates/controller-deployment.yaml

@@ -62,6 +62,15 @@ spec:
 {{- if .Values.prometheus.create }}
         - name: prometheus
           containerPort: {{ .Values.prometheus.port }}
+{{- end }}
+{{- if .Values.controller.readyStatus.enable }}
+        - name: readiness-port
+          containerPort: {{ .Values.controller.readyStatus.port}}
+        readinessProbe:
+          httpGet:
+            path: /nginx-ready
+            port: readiness-port
+          periodSeconds: 1
 {{- end }}
         resources:
 {{ toYaml .Values.controller.resources | indent 10 }}
@@ -135,4 +144,8 @@ spec:
           - -global-configuration=$(POD_NAMESPACE)/{{ .Release.Name }}
 {{- end }}
 {{- end }}
+{{- if .Values.controller.readyStatus.enable }}
+          - -ready-status
+          - -ready-status-port={{ .Values.controller.readyStaus.port }}
+{{- end }}
 {{- end }}

deployments/helm-chart/values.yaml

@@ -260,6 +260,13 @@ controller:
   ## The PriorityClass of the ingress controller pods.
   priorityClassName:
 
+  readyStatus:
+    ## Enables readiness endpoint "/nginx-ready". The endpoint returns a success code when NGINX has loaded all the config after startup.
+    enable: false
+
+    ## Set the port where the readiness endpoint is exposed.
+    port: 8081
+
 rbac:
   ## Configures RBAC.
   create: true

docs-web/configuration/global-configuration/command-line-arguments.md

@@ -188,5 +188,15 @@ Below we describe the available command-line arguments:
    Requires :option:`-nginx-plus`
 
 	 - If the argument is set, but `nginx-plus` is set to false, the Ingress Controller will fail to start.
-
+
+.. option:: -ready-status
+
+	Enables readiness endpoint "/nginx-ready". The endpoint returns a success code when NGINX has loaded all the config after startup.
+
+.. option:: -ready-status-port
+
+	The HTTP port for the readiness endpoint.
+
+	Format: ``[1024 - 65535]`` (default 8081)
+
 ```

internal/k8s/controller.go

@@ -145,6 +145,8 @@ type LoadBalancerController struct {
 	transportServerValidator      *validation.TransportServerValidator
 	spiffeController              *spiffeController
 	syncLock                      sync.Mutex
+	firstRun                      bool
+	isNginxReady                  bool
 }
 
 var keyFunc = cache.DeletionHandlingMetaNamespaceKeyFunc
@@ -175,6 +177,7 @@ type NewLoadBalancerControllerInput struct {
 	GlobalConfigurationValidator *validation.GlobalConfigurationValidator
 	TransportServerValidator     *validation.TransportServerValidator
 	SpireAgentAddress            string
+	FirstRun                     bool
 }
 
 // NewLoadBalancerController creates a controller
@@ -200,6 +203,7 @@ func NewLoadBalancerController(input NewLoadBalancerControllerInput) *LoadBalanc
 		metricsCollector:             input.MetricsCollector,
 		globalConfigurationValidator: input.GlobalConfigurationValidator,
 		transportServerValidator:     input.TransportServerValidator,
+		firstRun:                     input.FirstRun,
 	}
 
 	eventBroadcaster := record.NewBroadcaster()
@@ -271,7 +275,6 @@ func NewLoadBalancerController(input NewLoadBalancerControllerInput) *LoadBalanc
 	}
 
 	lbc.updateIngressMetrics()
-
 	return lbc
 }
 
@@ -835,6 +838,12 @@ func (lbc *LoadBalancerController) sync(task task) {
 	case appProtectLogConf:
 		lbc.syncAppProtectLogConf(task)
 	}
+
+	if lbc.firstRun && lbc.syncQueue.Len() == 0 {
+		lbc.firstRun = false
+		lbc.isNginxReady = true
+		glog.V(3).Infof("NGINX is ready")
+	}
 }
 
 func (lbc *LoadBalancerController) syncPolicy(task task) {
@@ -935,6 +944,7 @@ func (lbc *LoadBalancerController) syncPolicy(task task) {
 			}
 		}
 	}
+
 }
 
 func (lbc *LoadBalancerController) syncTransportServer(task task) {
@@ -3255,3 +3265,8 @@ func (lbc *LoadBalancerController) findIngressesForAppProtectResource(namespace
 	}
 	return apIngs
 }
+
+// IsNginxReady returns ready status of NGINX
+func (lbc *LoadBalancerController) IsNginxReady() bool {
+	return lbc.isNginxReady
+}

internal/k8s/task_queue.go

@@ -65,6 +65,12 @@ func (tq *taskQueue) Requeue(task task, err error) {
 	tq.queue.Add(task)
 }
 
+// Len returns the length of the queue
+func (tq *taskQueue) Len() int {
+	glog.V(3).Infof("The queue has %v element(s)", tq.queue.Len())
+	return tq.queue.Len()
+}
+
 // RequeueAfter adds the task to the queue after the given duration
 func (tq *taskQueue) RequeueAfter(t task, err error, after time.Duration) {
 	glog.Errorf("Requeuing %v after %s, err %v", t.Key, after.String(), err)