Liveness agent state

changelog/fragments/1757502748-liveness-include-agent-aggregated-status.yaml

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: bug-fix
+
+# Change summary; a 80ish characters long description of the change.
+summary: Include aggregated agent status in HTTP liveness checks
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: elastic-agent
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/elastic-agent/pull/9673
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+issue: https://github.com/elastic/elastic-agent/issues/9576

internal/pkg/agent/application/monitoring/liveness.go

@@ -14,6 +14,7 @@ import (
 	"github.com/elastic/elastic-agent-client/v7/pkg/client"
 
 	"github.com/elastic/elastic-agent/internal/pkg/otel/otelhelpers"
+	agentclient "github.com/elastic/elastic-agent/pkg/control/v2/client"
 )
 
 const formValueKey = "failon"
@@ -44,7 +45,7 @@ func handleFormValues(req *http.Request) (LivenessFailConfig, error) {
 	case "failed":
 		return LivenessFailConfig{Degraded: false, Failed: true, Heartbeat: true}, nil
 	case "degraded":
-		return LivenessFailConfig{Failed: true, Degraded: true, Heartbeat: true}, nil
+		return LivenessFailConfig{Degraded: true, Failed: true, Heartbeat: true}, nil
 	case "heartbeat", "":
 		return defaultUserCfg, nil
 	default:
@@ -76,12 +77,10 @@ func livenessHandler(coord CoordinatorState) func(http.ResponseWriter, *http.Req
 			return fmt.Errorf("error handling form values: %w", err)
 		}
 
-		// if user has requested `coordinator` mode, just revert to that, skip everything else
-		if !failConfig.Degraded && !failConfig.Failed && failConfig.Heartbeat {
-			if !isUp {
-				w.WriteHeader(http.StatusServiceUnavailable)
-				return nil
-			}
+		unhealthyState := (failConfig.Failed && state.State == agentclient.Failed) || (failConfig.Degraded && state.State == agentclient.Degraded)
+		if unhealthyState {
+			w.WriteHeader(http.StatusInternalServerError)
+			return nil
 		}
 
 		unhealthyComponent := false

internal/pkg/agent/application/monitoring/liveness_test.go

@@ -13,14 +13,14 @@ import (
 	"time"
 
 	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/status"
-	"go.opentelemetry.io/collector/component/componentstatus"
-
 	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/collector/component/componentstatus"
 
 	"github.com/elastic/elastic-agent-client/v7/pkg/client"
 	"github.com/elastic/elastic-agent/internal/pkg/agent/application/coordinator"
 	"github.com/elastic/elastic-agent/pkg/component"
 	"github.com/elastic/elastic-agent/pkg/component/runtime"
+	agentclient "github.com/elastic/elastic-agent/pkg/control/v2/client"
 )
 
 type mockCoordinator struct {
@@ -41,15 +41,19 @@ func TestLivenessProcessHTTPHandler(t *testing.T) {
 	defer cancel()
 	testCases := []struct {
 		name         string
-		coord        mockCoordinator
+		coord        CoordinatorState
 		expectedCode int
-		liveness     bool
 		failon       string
 	}{
 		{
 			name:         "healthy-nocoord",
+			coord:        nil,
+			expectedCode: 200,
+			failon:       "heartbeat",
+		},
+		{
+			name:         "healthy",
 			expectedCode: 200,
-			liveness:     true,
 			failon:       "heartbeat",
 		},
 		{
@@ -72,7 +76,6 @@ func TestLivenessProcessHTTPHandler(t *testing.T) {
 				},
 			},
 			expectedCode: 200,
-			liveness:     true,
 			failon:       "heartbeat",
 		},
 		{
@@ -95,7 +98,6 @@ func TestLivenessProcessHTTPHandler(t *testing.T) {
 				},
 			},
 			expectedCode: 200,
-			liveness:     true,
 			failon:       "heartbeat",
 		},
 		{
@@ -118,7 +120,6 @@ func TestLivenessProcessHTTPHandler(t *testing.T) {
 				},
 			},
 			expectedCode: 500,
-			liveness:     true,
 			failon:       "degraded",
 		},
 		{
@@ -141,34 +142,10 @@ func TestLivenessProcessHTTPHandler(t *testing.T) {
 				},
 			},
 			expectedCode: 200,
-			liveness:     true,
 			failon:       "failed",
 		},
 		{
-			name: "degraded-liveness-off",
-			coord: mockCoordinator{
-				isUp: true,
-				state: coordinator.State{
-					Components: []runtime.ComponentComponentState{
-						{
-							LegacyPID: "2",
-							State:     runtime.ComponentState{State: client.UnitStateDegraded},
-							Component: component.Component{
-								ID: "test-component",
-								InputSpec: &component.InputRuntimeSpec{
-									BinaryName: "testbeat",
-								},
-							},
-						},
-					},
-				},
-			},
-			expectedCode: 200,
-			liveness:     false,
-			failon:       "degraded",
-		},
-		{
-			name: "healthy",
+			name: "healthy-components",
 			coord: mockCoordinator{
 				isUp: true,
 				state: coordinator.State{
@@ -187,7 +164,6 @@ func TestLivenessProcessHTTPHandler(t *testing.T) {
 				},
 			},
 			expectedCode: 200,
-			liveness:     true,
 			failon:       "degraded",
 		},
 		{
@@ -210,7 +186,6 @@ func TestLivenessProcessHTTPHandler(t *testing.T) {
 				},
 			},
 			expectedCode: 503,
-			liveness:     true,
 			failon:       "heartbeat",
 		},
 		{
@@ -233,7 +208,6 @@ func TestLivenessProcessHTTPHandler(t *testing.T) {
 				},
 			},
 			expectedCode: 503,
-			liveness:     true,
 			failon:       "heartbeat",
 		},
 		{
@@ -256,7 +230,6 @@ func TestLivenessProcessHTTPHandler(t *testing.T) {
 				},
 			},
 			expectedCode: 503,
-			liveness:     true,
 			failon:       "degraded",
 		},
 		{
@@ -279,7 +252,6 @@ func TestLivenessProcessHTTPHandler(t *testing.T) {
 				},
 			},
 			expectedCode: 503,
-			liveness:     true,
 			failon:       "degraded",
 		},
 		{
@@ -302,30 +274,6 @@ func TestLivenessProcessHTTPHandler(t *testing.T) {
 				},
 			},
 			expectedCode: 503,
-			liveness:     true,
-			failon:       "degraded",
-		},
-		{
-			name: "healthy-liveness-off",
-			coord: mockCoordinator{
-				isUp: true,
-				state: coordinator.State{
-					Components: []runtime.ComponentComponentState{
-						{
-							LegacyPID: "5",
-							State:     runtime.ComponentState{State: client.UnitStateHealthy},
-							Component: component.Component{
-								ID: "test-component3",
-								InputSpec: &component.InputRuntimeSpec{
-									BinaryName: "testbeat",
-								},
-							},
-						},
-					},
-				},
-			},
-			expectedCode: 200,
-			liveness:     false,
 			failon:       "degraded",
 		},
 		{
@@ -358,11 +306,10 @@ func TestLivenessProcessHTTPHandler(t *testing.T) {
 				},
 			},
 			expectedCode: 500,
-			liveness:     true,
 			failon:       "degraded",
 		},
 		{
-			name: "healthy-liveness-off-otel",
+			name: "healthy-otel",
 			coord: mockCoordinator{
 				isUp: true,
 				state: coordinator.State{
@@ -380,7 +327,6 @@ func TestLivenessProcessHTTPHandler(t *testing.T) {
 				},
 			},
 			expectedCode: 200,
-			liveness:     false,
 			failon:       "degraded",
 		},
 		{
@@ -402,7 +348,39 @@ func TestLivenessProcessHTTPHandler(t *testing.T) {
 				},
 			},
 			expectedCode: 500,
-			liveness:     true,
+			failon:       "degraded",
+		},
+		{
+			name: "unhealthy-failed-coordinator",
+			coord: mockCoordinator{
+				isUp: true,
+				state: coordinator.State{
+					State: agentclient.Failed,
+				},
+			},
+			expectedCode: 500,
+			failon:       "failed",
+		},
+		{
+			name: "unhealthy-degraded-coordinator",
+			coord: mockCoordinator{
+				isUp: true,
+				state: coordinator.State{
+					State: agentclient.Degraded,
+				},
+			},
+			expectedCode: 500,
+			failon:       "degraded",
+		},
+		{
+			name: "healthy-degraded-coordinator",
+			coord: mockCoordinator{
+				isUp: true,
+				state: coordinator.State{
+					State: agentclient.Failed,
+				},
+			},
+			expectedCode: 500,
 			failon:       "degraded",
 		},
 	}
@@ -418,8 +396,9 @@ func TestLivenessProcessHTTPHandler(t *testing.T) {
 			require.NoError(t, err)
 			res, err := http.DefaultClient.Do(req)
 			require.NoError(t, err)
-			res.Body.Close()
+			defer res.Body.Close()
 
+			require.Equal(t, test.expectedCode, res.StatusCode)
 		})
 	}
 

internal/pkg/agent/application/monitoring/readiness_test.go

@@ -19,24 +19,27 @@ func TestReadinessProcessHTTPHandler(t *testing.T) {
 	defer cancel()
 	testCases := []struct {
 		name         string
-		coord        mockCoordinator
+		coord        CoordinatorState
 		expectedCode int
-		liveness     bool
 	}{
 		{
 			name:         "healthy-nocoord",
+			coord:        nil,
 			expectedCode: 200,
-			liveness:     true,
 		},
 		{
-			name:         "healthy",
+			name: "healthy",
+			coord: mockCoordinator{
+				isUp: true,
+			},
 			expectedCode: 200,
-			liveness:     true,
 		},
 		{
-			name:         "unhealthy",
+			name: "unhealthy",
+			coord: mockCoordinator{
+				isUp: false,
+			},
 			expectedCode: 503,
-			liveness:     false,
 		},
 	}
 
@@ -50,7 +53,9 @@ func TestReadinessProcessHTTPHandler(t *testing.T) {
 			require.NoError(t, err)
 			res, err := http.DefaultClient.Do(req)
 			require.NoError(t, err)
-			res.Body.Close()
+			defer res.Body.Close()
+
+			require.Equal(t, test.expectedCode, res.StatusCode)
 		})
 	}