Support Envoy's MaxEjectionPercent and BaseEjectionTime config entries for passive health checks

.changelog/15979.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+envoy: add `MaxEjectionPercent` and `BaseEjectionTime` to passive health check configs.
+```

GNUmakefile

@@ -405,6 +405,7 @@ ui-build-image:
 	@echo "Building UI build container"
 	@docker build $(NOCACHE) $(QUIET) -t $(UI_BUILD_TAG) - < build-support/docker/Build-UI.dockerfile
 
+# Builds consul in a docker container and then dumps executable into ./pkg/bin/...
 consul-docker: go-build-image
 	@$(SHELL) $(CURDIR)/build-support/scripts/build-docker.sh consul
 
@@ -538,6 +539,11 @@ envoy-regen:
 	@find "command/connect/envoy/testdata" -name '*.golden' -delete
 	@go test -tags '$(GOTAGS)' ./command/connect/envoy -update
 
+# Point your web browser to http://localhost:3000/consul to live render docs from ./website/
+.PHONY: docs
+docs:
+	make -C website
+
 .PHONY: help
 help:
 	$(info available make targets)

agent/consul/config_endpoint_test.go

@@ -1484,6 +1484,8 @@ func TestConfigEntry_ResolveServiceConfig_Upstreams(t *testing.T) {
 								Interval:                10,
 								MaxFailures:             2,
 								EnforcingConsecutive5xx: uintPointer(60),
+								MaxEjectionPercent:      uintPointer(61),
+								BaseEjectionTime:        durationPointer(62 * time.Second),
 							},
 						},
 						Overrides: []*structs.UpstreamConfig{
@@ -1518,6 +1520,8 @@ func TestConfigEntry_ResolveServiceConfig_Upstreams(t *testing.T) {
 								"Interval":                int64(10),
 								"MaxFailures":             int64(2),
 								"EnforcingConsecutive5xx": int64(60),
+								"MaxEjectionPercent":      int64(61),
+								"BaseEjectionTime":        uint64(62 * time.Second),
 							},
 							"mesh_gateway": map[string]interface{}{
 								"Mode": "none",
@@ -1532,6 +1536,8 @@ func TestConfigEntry_ResolveServiceConfig_Upstreams(t *testing.T) {
 								"Interval":                int64(10),
 								"MaxFailures":             int64(2),
 								"EnforcingConsecutive5xx": int64(60),
+								"MaxEjectionPercent":      int64(61),
+								"BaseEjectionTime":        uint64(62 * time.Second),
 							},
 							"mesh_gateway": map[string]interface{}{
 								"Mode": "local",
@@ -2639,3 +2645,7 @@ func Test_gateWriteToSecondary_AllowedKinds(t *testing.T) {
 func uintPointer(v uint32) *uint32 {
 	return &v
 }
+
+func durationPointer(d time.Duration) *time.Duration {
+	return &d
+}

agent/proxycfg/proxycfg.deepcopy.go

@@ -7,6 +7,7 @@ import (
 	"github.com/hashicorp/consul/agent/structs"
 	"github.com/hashicorp/consul/proto/private/pbpeering"
 	"github.com/hashicorp/consul/types"
+	"time"
 )
 
 // DeepCopy generates a deep copy of *ConfigSnapshot
@@ -452,6 +453,14 @@ func (o *configSnapshotIngressGateway) DeepCopy() *configSnapshotIngressGateway
 			cp.Defaults.PassiveHealthCheck.EnforcingConsecutive5xx = new(uint32)
 			*cp.Defaults.PassiveHealthCheck.EnforcingConsecutive5xx = *o.Defaults.PassiveHealthCheck.EnforcingConsecutive5xx
 		}
+		if o.Defaults.PassiveHealthCheck.MaxEjectionPercent != nil {
+			cp.Defaults.PassiveHealthCheck.MaxEjectionPercent = new(uint32)
+			*cp.Defaults.PassiveHealthCheck.MaxEjectionPercent = *o.Defaults.PassiveHealthCheck.MaxEjectionPercent
+		}
+		if o.Defaults.PassiveHealthCheck.BaseEjectionTime != nil {
+			cp.Defaults.PassiveHealthCheck.BaseEjectionTime = new(time.Duration)
+			*cp.Defaults.PassiveHealthCheck.BaseEjectionTime = *o.Defaults.PassiveHealthCheck.BaseEjectionTime
+		}
 	}
 	return &cp
 }

agent/structs/config_entry.go

@@ -1101,6 +1101,16 @@ type PassiveHealthCheck struct {
 	// when an outlier status is detected through consecutive 5xx.
 	// This setting can be used to disable ejection or to ramp it up slowly. Defaults to 100.
 	EnforcingConsecutive5xx *uint32 `json:",omitempty" alias:"enforcing_consecutive_5xx"`
+
+	// The maximum % of an upstream cluster that can be ejected due to outlier detection.
+	// Defaults to 10% but will eject at least one host regardless of the value.
+	// TODO: remove me
+	MaxEjectionPercent *uint32 `json:",omitempty" alias:"max_ejection_percent"`
+
+	// The base time that a host is ejected for. The real time is equal to the base time
+	// multiplied by the number of times the host has been ejected and is capped by
+	// max_ejection_time (Default 300s). Defaults to 30000ms or 30s.
+	BaseEjectionTime *time.Duration `json:",omitempty" alias:"base_ejection_time"`
 }
 
 func (chk *PassiveHealthCheck) Clone() *PassiveHealthCheck {
@@ -1120,6 +1130,15 @@ func (chk PassiveHealthCheck) Validate() error {
 	if chk.Interval < 0*time.Second {
 		return fmt.Errorf("passive health check interval cannot be negative")
 	}
+	if chk.EnforcingConsecutive5xx != nil && *chk.EnforcingConsecutive5xx > 100 {
+		return fmt.Errorf("passive health check enforcing_consecutive_5xx must be a percentage between 0 and 100")
+	}
+	if chk.MaxEjectionPercent != nil && *chk.MaxEjectionPercent > 100 {
+		return fmt.Errorf("passive health check max_ejection_percent must be a percentage between 0 and 100")
+	}
+	if chk.BaseEjectionTime != nil && *chk.BaseEjectionTime < 0*time.Second {
+		return fmt.Errorf("passive health check base_ejection_time cannot be negative")
+	}
 	return nil
 }
 

agent/structs/config_entry_test.go

@@ -416,6 +416,9 @@ func TestDecodeConfigEntry(t *testing.T) {
 							passive_health_check {
 								interval = "2s"
 								max_failures = 3
+								enforcing_consecutive_5xx = 4
+								max_ejection_percent = 5
+								base_ejection_time = "6s"
 							}
 						},
 						{
@@ -460,6 +463,9 @@ func TestDecodeConfigEntry(t *testing.T) {
 							PassiveHealthCheck {
 								MaxFailures = 3
 								Interval = "2s"
+								EnforcingConsecutive5xx = 4
+								MaxEjectionPercent = 5
+								BaseEjectionTime = "6s"
 							}
 						},
 						{
@@ -502,8 +508,11 @@ func TestDecodeConfigEntry(t *testing.T) {
 						{
 							Name: "redis",
 							PassiveHealthCheck: &PassiveHealthCheck{
-								MaxFailures: 3,
-								Interval:    2 * time.Second,
+								MaxFailures:             3,
+								Interval:                2 * time.Second,
+								EnforcingConsecutive5xx: uintPointer(4),
+								MaxEjectionPercent:      uintPointer(5),
+								BaseEjectionTime:        durationPointer(6 * time.Second),
 							},
 						},
 						{
@@ -2351,6 +2360,39 @@ func TestPassiveHealthCheck_Validate(t *testing.T) {
 			wantErr: true,
 			wantMsg: "cannot be negative",
 		},
+		{
+			name:    "valid enforcing_consecutive_5xx",
+			input:   PassiveHealthCheck{EnforcingConsecutive5xx: uintPointer(100)},
+			wantErr: false,
+		},
+		{
+			name:    "invalid enforcing_consecutive_5xx",
+			input:   PassiveHealthCheck{EnforcingConsecutive5xx: uintPointer(101)},
+			wantErr: true,
+			wantMsg: "must be a percentage",
+		},
+		{
+			name:    "valid max_ejection_percent",
+			input:   PassiveHealthCheck{MaxEjectionPercent: uintPointer(100)},
+			wantErr: false,
+		},
+		{
+			name:    "invalid max_ejection_percent",
+			input:   PassiveHealthCheck{MaxEjectionPercent: uintPointer(101)},
+			wantErr: true,
+			wantMsg: "must be a percentage",
+		},
+		{
+			name:    "valid base_ejection_time",
+			input:   PassiveHealthCheck{BaseEjectionTime: durationPointer(0 * time.Second)},
+			wantErr: false,
+		},
+		{
+			name:    "negative base_ejection_time",
+			input:   PassiveHealthCheck{BaseEjectionTime: durationPointer(-1 * time.Second)},
+			wantErr: true,
+			wantMsg: "cannot be negative",
+		},
 	}
 
 	for _, tc := range tt {
@@ -2890,8 +2932,11 @@ func TestUpstreamConfig_MergeInto(t *testing.T) {
 					MaxConcurrentRequests: intPointer(5),
 				},
 				PassiveHealthCheck: &PassiveHealthCheck{
-					MaxFailures: 3,
-					Interval:    2 * time.Second,
+					Interval:                2 * time.Second,
+					MaxFailures:             3,
+					EnforcingConsecutive5xx: uintPointer(4),
+					MaxEjectionPercent:      uintPointer(5),
+					BaseEjectionTime:        durationPointer(6 * time.Second),
 				},
 				MeshGateway: MeshGatewayConfig{Mode: MeshGatewayModeRemote},
 			},
@@ -2908,8 +2953,11 @@ func TestUpstreamConfig_MergeInto(t *testing.T) {
 					MaxConcurrentRequests: intPointer(5),
 				},
 				"passive_health_check": &PassiveHealthCheck{
-					MaxFailures: 3,
-					Interval:    2 * time.Second,
+					Interval:                2 * time.Second,
+					MaxFailures:             3,
+					EnforcingConsecutive5xx: uintPointer(4),
+					MaxEjectionPercent:      uintPointer(5),
+					BaseEjectionTime:        durationPointer(6 * time.Second),
 				},
 				"mesh_gateway": MeshGatewayConfig{Mode: MeshGatewayModeRemote},
 			},
@@ -2928,8 +2976,11 @@ func TestUpstreamConfig_MergeInto(t *testing.T) {
 					MaxConcurrentRequests: intPointer(5),
 				},
 				PassiveHealthCheck: &PassiveHealthCheck{
-					MaxFailures: 3,
-					Interval:    2 * time.Second,
+					Interval:                2 * time.Second,
+					MaxFailures:             3,
+					EnforcingConsecutive5xx: uintPointer(4),
+					MaxEjectionPercent:      uintPointer(5),
+					BaseEjectionTime:        durationPointer(6 * time.Second),
 				},
 				MeshGateway: MeshGatewayConfig{Mode: MeshGatewayModeRemote},
 			},
@@ -2945,8 +2996,11 @@ func TestUpstreamConfig_MergeInto(t *testing.T) {
 					MaxConcurrentRequests: intPointer(12),
 				},
 				"passive_health_check": &PassiveHealthCheck{
-					MaxFailures: 13,
-					Interval:    14 * time.Second,
+					MaxFailures:             13,
+					Interval:                14 * time.Second,
+					EnforcingConsecutive5xx: uintPointer(15),
+					MaxEjectionPercent:      uintPointer(16),
+					BaseEjectionTime:        durationPointer(17 * time.Second),
 				},
 				"mesh_gateway": MeshGatewayConfig{Mode: MeshGatewayModeLocal},
 			},
@@ -2962,8 +3016,11 @@ func TestUpstreamConfig_MergeInto(t *testing.T) {
 					MaxConcurrentRequests: intPointer(5),
 				},
 				"passive_health_check": &PassiveHealthCheck{
-					MaxFailures: 3,
-					Interval:    2 * time.Second,
+					Interval:                2 * time.Second,
+					MaxFailures:             3,
+					EnforcingConsecutive5xx: uintPointer(4),
+					MaxEjectionPercent:      uintPointer(5),
+					BaseEjectionTime:        durationPointer(6 * time.Second),
 				},
 				"mesh_gateway": MeshGatewayConfig{Mode: MeshGatewayModeRemote},
 			},
@@ -2985,7 +3042,9 @@ func TestUpstreamConfig_MergeInto(t *testing.T) {
 				"passive_health_check": &PassiveHealthCheck{
 					MaxFailures:             13,
 					Interval:                14 * time.Second,
-					EnforcingConsecutive5xx: uintPointer(80),
+					EnforcingConsecutive5xx: uintPointer(15),
+					MaxEjectionPercent:      uintPointer(16),
+					BaseEjectionTime:        durationPointer(17 * time.Second),
 				},
 				"mesh_gateway": MeshGatewayConfig{Mode: MeshGatewayModeLocal},
 			},
@@ -3003,7 +3062,9 @@ func TestUpstreamConfig_MergeInto(t *testing.T) {
 				"passive_health_check": &PassiveHealthCheck{
 					MaxFailures:             13,
 					Interval:                14 * time.Second,
-					EnforcingConsecutive5xx: uintPointer(80),
+					EnforcingConsecutive5xx: uintPointer(15),
+					MaxEjectionPercent:      uintPointer(16),
+					BaseEjectionTime:        durationPointer(17 * time.Second),
 				},
 				"mesh_gateway": MeshGatewayConfig{Mode: MeshGatewayModeLocal},
 			},
@@ -3138,15 +3199,21 @@ func TestParseUpstreamConfig(t *testing.T) {
 			name: "passive health check map",
 			input: map[string]interface{}{
 				"passive_health_check": map[string]interface{}{
-					"interval":     "22s",
-					"max_failures": 7,
+					"interval":                  "22s",
+					"max_failures":              7,
+					"enforcing_consecutive_5xx": 8,
+					"max_ejection_percent":      9,
+					"base_ejection_time":        "10s",
 				},
 			},
 			want: UpstreamConfig{
 				ConnectTimeoutMs: 5000,
 				PassiveHealthCheck: &PassiveHealthCheck{
-					Interval:    22 * time.Second,
-					MaxFailures: 7,
+					Interval:                22 * time.Second,
+					MaxFailures:             7,
+					EnforcingConsecutive5xx: uintPointer(8),
+					MaxEjectionPercent:      uintPointer(9),
+					BaseEjectionTime:        durationPointer(10 * time.Second),
 				},
 				Protocol: "tcp",
 			},
@@ -3273,10 +3340,6 @@ func requireContainsLower(t *testing.T, haystack, needle string) {
 	require.Contains(t, strings.ToLower(haystack), strings.ToLower(needle))
 }
 
-func intPointer(i int) *int {
-	return &i
-}
-
 func TestConfigEntryQuery_CacheInfoKey(t *testing.T) {
 	assertCacheInfoKeyIsComplete(t, &ConfigEntryQuery{})
 }
@@ -3371,6 +3434,14 @@ func testConfigEntryNormalizeAndValidate(t *testing.T, cases map[string]configEn
 	}
 }
 
+func intPointer(i int) *int {
+	return &i
+}
+
 func uintPointer(v uint32) *uint32 {
 	return &v
 }
+
+func durationPointer(d time.Duration) *time.Duration {
+	return &d
+}

agent/structs/structs.deepcopy.go

@@ -529,6 +529,14 @@ func (o *IngressListener) DeepCopy() *IngressListener {
 					cp.Services[i2].PassiveHealthCheck.EnforcingConsecutive5xx = new(uint32)
 					*cp.Services[i2].PassiveHealthCheck.EnforcingConsecutive5xx = *o.Services[i2].PassiveHealthCheck.EnforcingConsecutive5xx
 				}
+				if o.Services[i2].PassiveHealthCheck.MaxEjectionPercent != nil {
+					cp.Services[i2].PassiveHealthCheck.MaxEjectionPercent = new(uint32)
+					*cp.Services[i2].PassiveHealthCheck.MaxEjectionPercent = *o.Services[i2].PassiveHealthCheck.MaxEjectionPercent
+				}
+				if o.Services[i2].PassiveHealthCheck.BaseEjectionTime != nil {
+					cp.Services[i2].PassiveHealthCheck.BaseEjectionTime = new(time.Duration)
+					*cp.Services[i2].PassiveHealthCheck.BaseEjectionTime = *o.Services[i2].PassiveHealthCheck.BaseEjectionTime
+				}
 			}
 			if o.Services[i2].Meta != nil {
 				cp.Services[i2].Meta = make(map[string]string, len(o.Services[i2].Meta))
@@ -1134,6 +1142,14 @@ func (o *UpstreamConfiguration) DeepCopy() *UpstreamConfiguration {
 						cp.Overrides[i2].PassiveHealthCheck.EnforcingConsecutive5xx = new(uint32)
 						*cp.Overrides[i2].PassiveHealthCheck.EnforcingConsecutive5xx = *o.Overrides[i2].PassiveHealthCheck.EnforcingConsecutive5xx
 					}
+					if o.Overrides[i2].PassiveHealthCheck.MaxEjectionPercent != nil {
+						cp.Overrides[i2].PassiveHealthCheck.MaxEjectionPercent = new(uint32)
+						*cp.Overrides[i2].PassiveHealthCheck.MaxEjectionPercent = *o.Overrides[i2].PassiveHealthCheck.MaxEjectionPercent
+					}
+					if o.Overrides[i2].PassiveHealthCheck.BaseEjectionTime != nil {
+						cp.Overrides[i2].PassiveHealthCheck.BaseEjectionTime = new(time.Duration)
+						*cp.Overrides[i2].PassiveHealthCheck.BaseEjectionTime = *o.Overrides[i2].PassiveHealthCheck.BaseEjectionTime
+					}
 				}
 			}
 		}
@@ -1164,6 +1180,14 @@ func (o *UpstreamConfiguration) DeepCopy() *UpstreamConfiguration {
 				cp.Defaults.PassiveHealthCheck.EnforcingConsecutive5xx = new(uint32)
 				*cp.Defaults.PassiveHealthCheck.EnforcingConsecutive5xx = *o.Defaults.PassiveHealthCheck.EnforcingConsecutive5xx
 			}
+			if o.Defaults.PassiveHealthCheck.MaxEjectionPercent != nil {
+				cp.Defaults.PassiveHealthCheck.MaxEjectionPercent = new(uint32)
+				*cp.Defaults.PassiveHealthCheck.MaxEjectionPercent = *o.Defaults.PassiveHealthCheck.MaxEjectionPercent
+			}
+			if o.Defaults.PassiveHealthCheck.BaseEjectionTime != nil {
+				cp.Defaults.PassiveHealthCheck.BaseEjectionTime = new(time.Duration)
+				*cp.Defaults.PassiveHealthCheck.BaseEjectionTime = *o.Defaults.PassiveHealthCheck.BaseEjectionTime
+			}
 		}
 	}
 	return &cp