Merge branch 'healthcheck_support'

switch to internal health endpoint

Dockerfile

@@ -36,6 +36,8 @@ RUN CGO_ENABLED=0 go build \
         -X main.BuildDate=$(date -u ""+%Y%m%d-%H:%M:%S"")" \
     -o /app .
 
+# Second stage: get statically linked curl
+FROM wmark/curl:latest as curl
 
 # Final stage: the running container.
 FROM scratch AS final
@@ -52,6 +54,9 @@ COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 # Import the compiled executable from the first stage.
 COPY --from=builder /app /app
 
+# Get curl from image
+COPY --from=curl /usr/bin/curl /usr/bin/curl
+
 # Declare the port on which the webserver will be exposed.
 # As we're going to run the executable as an unprivileged user, we can't bind
 # to ports below 1024.
@@ -60,5 +65,7 @@ EXPOSE 9419
 # Perform any further action as an unprivileged user.
 USER nobody:nobody
 
+# Check if exporter is alive; 10 retries gives prometheus some time to retrieve bad data (5 minutes)
+HEALTHCHECK --retries=10 CMD "/usr/bin/curl -s -f http://localhost:9419/health || exit 1"]
 # Run the compiled binary.
-ENTRYPOINT ["/app"]
+ENTRYPOINT ["/app"]

VERSION

@@ -1 +1 @@
-master
+master

exporter.go

@@ -42,12 +42,14 @@ type exporter struct {
 	exporter                     map[string]Exporter
 	overviewExporter             *exporterOverview
 	self                         string
+	lastScrapeOK                 bool
 }
 
 //Exporter interface for prometheus metrics. Collect is fetching the data and therefore can return an error
 type Exporter interface {
 	Collect(ctx context.Context, ch chan<- prometheus.Metric) error
 	Describe(ch chan<- *prometheus.Desc)
+	LastScrapeOK() bool
 }
 
 func newExporter() *exporter {
@@ -64,9 +66,16 @@ func newExporter() *exporter {
 		endpointScrapeDurationMetric: newGaugeVec("module_scrape_duration_seconds", "Duration of the last scrape in seconds", []string{"cluster", "node", "module"}),
 		exporter:                     enabledExporter,
 		overviewExporter:             newExporterOverview(),
+		lastScrapeOK:                 true, //return true after start. Value will be updated with each scraping
 	}
 }
 
+func (e *exporter) LastScrapeOK() bool {
+	e.mutex.Lock() // To protect metrics from concurrent collects.
+	defer e.mutex.Unlock()
+	return e.lastScrapeOK
+}
+
 func (e *exporter) Describe(ch chan<- *prometheus.Desc) {
 
 	e.overviewExporter.Describe(ch)
@@ -106,6 +115,7 @@ func (e *exporter) Collect(ch chan<- prometheus.Metric) {
 	} else {
 		e.upMetric.WithLabelValues(e.overviewExporter.NodeInfo().ClusterName, e.overviewExporter.NodeInfo().Node).Set(0)
 	}
+	e.lastScrapeOK = allUp
 	e.upMetric.Collect(ch)
 	e.endpointUpMetric.Collect(ch)
 	e.endpointScrapeDurationMetric.Collect(ch)

main.go

@@ -88,6 +88,13 @@ func main() {
              </body>
              </html>`))
 	})
+	handler.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
+		if exporter.LastScrapeOK() {
+			w.WriteHeader(http.StatusOK)
+		} else {
+			w.WriteHeader(http.StatusGatewayTimeout)
+		}
+	})
 
 	server := &http.Server{Addr: config.PublishAddr + ":" + config.PublishPort, Handler: handler}