health: expand health check functionality with health package

cmd/geth/config.go

@@ -196,6 +196,10 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) {
 	// Configure log filter RPC API.
 	filterSystem := utils.RegisterFilterAPI(stack, backend, &cfg.Eth)
 
+	// Configure the healthcheck API if requested.
+	if ctx.IsSet(utils.HTTPHealthEnabledFlag.Name) {
+		utils.RegisterHealthService(stack, &cfg.Node)
+	}
 	// Configure GraphQL if requested.
 	if ctx.IsSet(utils.GraphQLEnabledFlag.Name) {
 		utils.RegisterGraphQLService(stack, backend, filterSystem, &cfg.Node)

cmd/geth/main.go

@@ -153,6 +153,7 @@ var (
 		utils.HTTPListenAddrFlag,
 		utils.HTTPPortFlag,
 		utils.HTTPCORSDomainFlag,
+		utils.HTTPHealthEnabledFlag,
 		utils.AuthListenFlag,
 		utils.AuthPortFlag,
 		utils.AuthVirtualHostsFlag,

cmd/utils/flags.go

@@ -55,6 +55,7 @@ import (
 	"github.com/ethereum/go-ethereum/ethdb/remotedb"
 	"github.com/ethereum/go-ethereum/ethstats"
 	"github.com/ethereum/go-ethereum/graphql"
+	"github.com/ethereum/go-ethereum/health"
 	"github.com/ethereum/go-ethereum/internal/ethapi"
 	"github.com/ethereum/go-ethereum/internal/flags"
 	"github.com/ethereum/go-ethereum/log"
@@ -613,6 +614,11 @@ var (
 		Value:    "",
 		Category: flags.APICategory,
 	}
+	HTTPHealthEnabledFlag = &cli.BoolFlag{
+		Name:     "http.health",
+		Usage:    "Enable the HTTP healthcheck API at path '/health'.",
+		Category: flags.APICategory,
+	}
 	GraphQLEnabledFlag = &cli.BoolFlag{
 		Name:     "graphql",
 		Usage:    "Enable GraphQL on the HTTP-RPC server. Note that GraphQL can only be started if an HTTP server is started as well.",
@@ -1883,6 +1889,14 @@ func RegisterGraphQLService(stack *node.Node, backend ethapi.Backend, filterSyst
 	}
 }
 
+// RegisterHealthService adds the Health API to the node.
+func RegisterHealthService(stack *node.Node, cfg *node.Config) {
+	err := health.New(stack, cfg.HTTPCors, cfg.HTTPVirtualHosts)
+	if err != nil {
+		Fatalf("Failed to register the health service: %v", err)
+	}
+}
+
 // RegisterFilterAPI adds the eth log filtering RPC API to the node.
 func RegisterFilterAPI(stack *node.Node, backend ethapi.Backend, ethcfg *ethconfig.Config) *filters.FilterSystem {
 	filterSystem := filters.NewFilterSystem(backend, filters.Config{

health/check_block.go

@@ -0,0 +1,15 @@
+package health
+
+import (
+	"context"
+	"math/big"
+)
+
+// checkBlockNumber confirms this node is aware of a specific block.
+func checkBlockNumber(ec ethClient, blockNumber *big.Int) error {
+	_, err := ec.BlockByNumber(context.TODO(), blockNumber)
+	if err != nil {
+		return err
+	}
+	return nil
+}

health/check_peers.go

@@ -0,0 +1,23 @@
+package health
+
+import (
+	"context"
+	"errors"
+	"fmt"
+)
+
+var (
+	errNotEnoughPeers = errors.New("not enough peers")
+)
+
+// checkMinPeers returns 'errNotEnoughPeers' if the current peer count its lower than 'minPeerCount'
+func checkMinPeers(ec ethClient, minPeerCount uint64) error {
+	peerCount, err := ec.PeerCount(context.TODO())
+	if err != nil {
+		return err
+	}
+	if peerCount < minPeerCount {
+		return fmt.Errorf("%w: %d (minimum %d)", errNotEnoughPeers, peerCount, minPeerCount)
+	}
+	return nil
+}

health/check_synced.go

@@ -0,0 +1,27 @@
+package health
+
+import (
+	"context"
+	"errors"
+	"net/http"
+
+	"github.com/ethereum/go-ethereum/log"
+)
+
+var (
+	errNotSynced = errors.New("not synced")
+)
+
+// checkSynced returns 'errNotSynced' if the node is in the syncing state.
+func checkSynced(ec ethClient, r *http.Request) error {
+	i, err := ec.SyncProgress(context.TODO())
+	if err != nil {
+		log.Root().Warn("Unable to check sync status for healthcheck", "err", err.Error())
+		return err
+	}
+	if i == nil {
+		return nil
+	}
+
+	return errNotSynced
+}

health/check_time.go

@@ -0,0 +1,30 @@
+package health
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+)
+
+var (
+	errTimestampTooOld = errors.New("timestamp too old")
+)
+
+// checkTime fetches the timestamp of the most recent block and returns an error if it is earlier than 'minTimestamp'.
+func checkTime(
+	ec ethClient,
+	r *http.Request,
+	minTimestamp int,
+) error {
+	i, err := ec.BlockByNumber(context.TODO(), nil)
+	if err != nil {
+		return err
+	}
+	timestamp := i.Time()
+	if timestamp < uint64(minTimestamp) {
+		return fmt.Errorf("%w: got ts: %d, need: %d", errTimestampTooOld, timestamp, minTimestamp)
+	}
+
+	return nil
+}

health/health.go

@@ -0,0 +1,211 @@
+package health
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"math/big"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/ethereum/go-ethereum/log"
+)
+
+const (
+	healthHeader     = "X-GETH-HEALTHCHECK"
+	query            = "query"
+	synced           = "synced"
+	minPeerCount     = "min_peer_count"
+	checkBlock       = "check_block"
+	maxSecondsBehind = "max_seconds_behind"
+)
+
+var (
+	errCheckDisabled = errors.New("error check disabled")
+	errInvalidValue  = errors.New("invalid value provided")
+)
+
+type requestBody struct {
+	Synced           *bool    `json:"synced"`
+	MinPeerCount     *uint    `json:"min_peer_count"`
+	CheckBlock       *big.Int `json:"check_block"`
+	MaxSecondsBehind *int     `json:"max_seconds_behind"`
+}
+
+// processFromHeaders handles requests when 'X-GETH-HEALTHCHECK' header labels are present.
+func processFromHeaders(ec ethClient, headers []string, w http.ResponseWriter, r *http.Request) {
+	var (
+		errCheckSynced  = errCheckDisabled
+		errCheckPeer    = errCheckDisabled
+		errCheckBlock   = errCheckDisabled
+		errCheckSeconds = errCheckDisabled
+	)
+
+	for _, header := range headers {
+		lHeader := strings.ToLower(header)
+		switch {
+		case lHeader == synced:
+			errCheckSynced = checkSynced(ec, r)
+		case strings.HasPrefix(lHeader, minPeerCount):
+			peers, err := strconv.Atoi(strings.TrimPrefix(lHeader, minPeerCount))
+			if err != nil {
+				errCheckPeer = err
+				break
+			}
+			errCheckPeer = checkMinPeers(ec, uint64(peers))
+		case strings.HasPrefix(lHeader, checkBlock):
+			block, err := strconv.Atoi(strings.TrimPrefix(lHeader, checkBlock))
+			if err != nil {
+				errCheckBlock = err
+				break
+			}
+			errCheckBlock = checkBlockNumber(ec, big.NewInt(int64(block)))
+		case strings.HasPrefix(lHeader, maxSecondsBehind):
+			seconds, err := strconv.Atoi(strings.TrimPrefix(lHeader, maxSecondsBehind))
+			if err != nil {
+				errCheckSeconds = err
+				break
+			}
+			if seconds < 0 {
+				errCheckSeconds = errInvalidValue
+				break
+			}
+			now := time.Now().Unix()
+			errCheckSeconds = checkTime(ec, r, int(now)-seconds)
+		}
+	}
+
+	reportHealth(nil, errCheckSynced, errCheckPeer, errCheckBlock, errCheckSeconds, w)
+}
+
+// processFromBody handles requests when 'X-GETH-HEALTHCHECK' headers are not present.
+func processFromBody(ec ethClient, w http.ResponseWriter, r *http.Request) {
+	body, errParse := parseHealthCheckBody(r.Body)
+	defer r.Body.Close()
+
+	var (
+		errCheckSynced  = errCheckDisabled
+		errCheckPeer    = errCheckDisabled
+		errCheckBlock   = errCheckDisabled
+		errCheckSeconds = errCheckDisabled
+	)
+
+	if errParse != nil {
+		log.Root().Warn("Unable to process healthcheck request", "err", errParse)
+	} else {
+		if body.Synced != nil {
+			errCheckSynced = checkSynced(ec, r)
+		}
+
+		if body.MinPeerCount != nil {
+			errCheckPeer = checkMinPeers(ec, uint64(*body.MinPeerCount))
+		}
+
+		if body.CheckBlock != nil {
+			errCheckBlock = checkBlockNumber(ec, body.CheckBlock)
+		}
+
+		if body.MaxSecondsBehind != nil {
+			seconds := *body.MaxSecondsBehind
+			if seconds < 0 {
+				errCheckSeconds = errInvalidValue
+			} else {
+				now := time.Now().Unix()
+				errCheckSeconds = checkTime(ec, r, int(now)-seconds)
+			}
+		}
+	}
+
+	err := reportHealth(errParse, errCheckSynced, errCheckPeer, errCheckBlock, errCheckSeconds, w)
+	if err != nil {
+		log.Root().Warn("Unable to process healthcheck request", "err", err)
+	}
+}
+
+// reportHealth builds the response body, sets the status code and calls for it to be written.
+func reportHealth(errParse, errCheckSynced, errCheckPeer, errCheckBlock, errCheckSeconds error, w http.ResponseWriter) error {
+	statusCode := http.StatusOK
+	errs := make(map[string]string)
+
+	if shouldChangeStatusCode(errParse) {
+		statusCode = http.StatusInternalServerError
+	}
+	errs[query] = errorStringOrOK(errParse)
+
+	if shouldChangeStatusCode(errCheckSynced) {
+		statusCode = http.StatusInternalServerError
+	}
+	errs[synced] = errorStringOrOK(errCheckSynced)
+
+	if shouldChangeStatusCode(errCheckPeer) {
+		statusCode = http.StatusInternalServerError
+	}
+	errs[minPeerCount] = errorStringOrOK(errCheckPeer)
+
+	if shouldChangeStatusCode(errCheckBlock) {
+		statusCode = http.StatusInternalServerError
+	}
+	errs[checkBlock] = errorStringOrOK(errCheckBlock)
+
+	if shouldChangeStatusCode(errCheckSeconds) {
+		statusCode = http.StatusInternalServerError
+	}
+	errs[maxSecondsBehind] = errorStringOrOK(errCheckSeconds)
+
+	return writeResponse(w, errs, statusCode)
+}
+
+// parseHealthCheckBody parses and type checks the request body when 'X-GETH-HEALTHCHECK' headers are not present.
+func parseHealthCheckBody(reader io.Reader) (requestBody, error) {
+	var body requestBody
+
+	bodyBytes, err := io.ReadAll(reader)
+	if err != nil {
+		return body, err
+	}
+
+	err = json.Unmarshal(bodyBytes, &body)
+	if err != nil {
+		return body, err
+	}
+
+	return body, nil
+}
+
+// writeResponse delivers the status and body to the response writer.
+func writeResponse(w http.ResponseWriter, errs map[string]string, statusCode int) error {
+	w.WriteHeader(statusCode)
+
+	bodyJson, err := json.Marshal(errs)
+	if err != nil {
+		return err
+	}
+
+	_, err = w.Write(bodyJson)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// shouldChangeStatusCode returns 'true' if an error exists and is not 'errCheckDisabled'.
+func shouldChangeStatusCode(err error) bool {
+	return err != nil && !errors.Is(err, errCheckDisabled)
+}
+
+// errorStringOrOK returns "OK", "DISABLED" or the error message based on the output of the check.
+func errorStringOrOK(err error) string {
+	if err == nil {
+		return "OK"
+	}
+
+	if errors.Is(err, errCheckDisabled) {
+		return "DISABLED"
+	}
+
+	return fmt.Sprintf("ERROR: %v", err)
+}