Add native sidecar support (#11465)

* Add native sidecar support Kubernetes will be providing beta support for native sidecar containers in version 1.29. This feature improves network proxy sidecar compatibility for jobs and initContainers. Introduce a new annotation config.alpha.linkerd.io/proxy-enable-native-sidecar and configuration option Proxy.NativeSidecar that causes the proxy container to run as an init-container. Fixes: #11461 Signed-off-by: TJ Miller <[email protected]>
linkerd · Nov 22, 2023 · 1b37e19 · 1b37e19
1 parent e849b1a
commit 1b37e19
Show file tree

Hide file tree

Showing 40 changed files with 458 additions and 27 deletions.
diff --git a/charts/linkerd-control-plane/README.md b/charts/linkerd-control-plane/README.md
@@ -239,6 +239,7 @@ Kubernetes: `>=1.21.0-0`
 | proxy.inboundDiscoveryCacheUnusedTimeout | string | `"90s"` | Maximum time allowed before an unused inbound discovery result is evicted from the cache |
 | proxy.logFormat | string | `"plain"` | Log format (`plain` or `json`) for the proxy |
 | proxy.logLevel | string | `"warn,linkerd=info,trust_dns=error"` | Log level for the proxy |
+| proxy.nativeSidecar | bool | `false` | Enable KEP-753 native sidecars This is an experimental feature. It requires Kubernetes >= 1.29. If enabled, .proxy.waitBeforeExitSeconds should not be used. |
 | proxy.opaquePorts | string | `"25,587,3306,4444,5432,6379,9300,11211"` | Default set of opaque ports - SMTP (25,587) server-first - MYSQL (3306) server-first - Galera (4444) server-first - PostgreSQL (5432) server-first - Redis (6379) server-first - ElasticSearch (9300) server-first - Memcached (11211) clients do not issue any preamble, which breaks detection |
 | proxy.outboundConnectTimeout | string | `"1000ms"` | Maximum time allowed for the proxy to establish an outbound TCP connection |
 | proxy.outboundDiscoveryCacheUnusedTimeout | string | `"5s"` | Maximum time allowed before an unused outbound discovery result is evicted from the cache |
@@ -254,6 +255,7 @@ Kubernetes: `>=1.21.0-0`
 | proxy.resources.memory.limit | string | `""` | Maximum amount of memory that the proxy can use |
 | proxy.resources.memory.request | string | `""` | Maximum amount of memory that the proxy requests |
 | proxy.shutdownGracePeriod | string | `""` | Grace period for graceful proxy shutdowns. If this timeout elapses before all open connections have completed, the proxy will terminate forcefully, closing any remaining connections. |
+| proxy.startupProbe | object | `{"failureThreshold":120,"initialDelaySeconds":0,"periodSeconds":1}` | Native sidecar proxy startup probe parameters. |
 | proxy.uid | int | `2102` | User id under which the proxy runs |
 | proxy.waitBeforeExitSeconds | int | `0` | If set the injected proxy sidecars in the data plane will stay alive for at least the given period before receiving the SIGTERM signal from Kubernetes but no longer than the pod's `terminationGracePeriodSeconds`. See [Lifecycle hooks](https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks) for more info on container lifecycle hooks. |
 | proxyInit.closeWaitTimeoutSecs | int | `0` |  |

diff --git a/charts/linkerd-control-plane/templates/destination.yaml b/charts/linkerd-control-plane/templates/destination.yaml
@@ -190,7 +190,9 @@ spec:
       */}}
       {{- $_ := set $tree.Values.proxy "defaultInboundPolicy" "all-unauthenticated" }}
       {{- $_ := set $tree.Values.proxy "capabilities" (dict "drop" (list "ALL")) }}
+      {{- if not $tree.Values.proxy.nativeSidecar }}
       - {{- include "partials.proxy" $tree | indent 8 | trimPrefix (repeat 7 " ") }}
+      {{- end }}
       - args:
         - destination
         - -addr=:8086
@@ -341,6 +343,12 @@ spec:
       {{- $_ := set $tree.Values.proxyInit "ignoreOutboundPorts" .Values.proxyInit.kubeAPIServerPorts -}}
       - {{- include "partials.proxy-init" $tree | indent 8 | trimPrefix (repeat 7 " ") }}
       {{ end -}}
+      {{- if $tree.Values.proxy.nativeSidecar }}
+        {{- $_ := set $tree.Values.proxy "startupProbeInitialDelaySeconds" 35 }}
+        {{- $_ := set $tree.Values.proxy "startupProbePeriodSeconds" 5 }}
+        {{- $_ := set $tree.Values.proxy "startupProbeFailureThreshold" 20 }}
+      - {{- include "partials.proxy" $tree | indent 8 | trimPrefix (repeat 7 " ") }}
+      {{ end -}}
       {{- if .Values.priorityClassName -}}
       priorityClassName: {{ .Values.priorityClassName }}
       {{ end -}}

diff --git a/charts/linkerd-control-plane/templates/identity.yaml b/charts/linkerd-control-plane/templates/identity.yaml
@@ -206,6 +206,7 @@ spec:
       {{- $_ := set $tree.Values.proxy "await" false }}
       {{- $_ := set $tree.Values.proxy "loadTrustBundleFromConfigMap" true }}
       {{- $_ := set $tree.Values.proxy "podInboundPorts" "8080,9990" }}
+      {{- $_ := set $tree.Values.proxy "nativeSidecar" false }}
       {{- /*
         The identity controller cannot discover policies, so we configure it with defaults that
         enforce TLS on the identity service.

diff --git a/charts/linkerd-control-plane/templates/proxy-injector.yaml b/charts/linkerd-control-plane/templates/proxy-injector.yaml
@@ -70,7 +70,9 @@ spec:
       {{- $_ := set $tree.Values.proxy "capabilities" (dict "drop" (list "ALL")) }}
       {{- $_ := set $tree.Values.proxy "outboundDiscoveryCacheUnusedTimeout" "5s" }}
       {{- $_ := set $tree.Values.proxy "inboundDiscoveryCacheUnusedTimeout" "90s" }}
+      {{- if not $tree.Values.proxy.nativeSidecar }}
       - {{- include "partials.proxy" $tree | indent 8 | trimPrefix (repeat 7 " ") }}
+      {{- end }}
       - args:
         - proxy-injector
         - -log-level={{.Values.controllerLogLevel}}
@@ -127,6 +129,12 @@ spec:
       {{- $_ := set $tree.Values.proxyInit "ignoreOutboundPorts" .Values.proxyInit.kubeAPIServerPorts -}}
       - {{- include "partials.proxy-init" $tree | indent 8 | trimPrefix (repeat 7 " ") }}
       {{ end -}}
+      {{- if $tree.Values.proxy.nativeSidecar }}
+        {{- $_ := set $tree.Values.proxy "startupProbeInitialDelaySeconds" 35 }}
+        {{- $_ := set $tree.Values.proxy "startupProbePeriodSeconds" 5 }}
+        {{- $_ := set $tree.Values.proxy "startupProbeFailureThreshold" 20 }}
+      - {{- include "partials.proxy" $tree | indent 8 | trimPrefix (repeat 7 " ") }}
+      {{ end -}}
       {{- if .Values.priorityClassName -}}
       priorityClassName: {{ .Values.priorityClassName }}
       {{ end -}}

diff --git a/charts/linkerd-control-plane/values.yaml b/charts/linkerd-control-plane/values.yaml
@@ -191,6 +191,15 @@ proxy:
   # "all-unauthenticated", "cluster-authenticated", "cluster-unauthenticated", "deny"
   # @default -- "all-unauthenticated"
   defaultInboundPolicy: "all-unauthenticated"
+  # -- Enable KEP-753 native sidecars
+  # This is an experimental feature. It requires Kubernetes >= 1.29.
+  # If enabled, .proxy.waitBeforeExitSeconds should not be used.
+  nativeSidecar: false
+  # -- Native sidecar proxy startup probe parameters.
+  startupProbe:
+    initialDelaySeconds: 0
+    periodSeconds: 1
+    failureThreshold: 120
 
 # proxy-init configuration
 proxyInit:

diff --git a/charts/partials/templates/_proxy.tpl b/charts/partials/templates/_proxy.tpl
@@ -1,4 +1,7 @@
 {{ define "partials.proxy" -}}
+{{ if and .Values.proxy.nativeSidecar .Values.proxy.waitBeforeExitSeconds }}
+{{ fail "proxy.nativeSidecar and waitBeforeExitSeconds cannot be used simultaneously" }}
+{{- end }}
 {{- $trustDomain := (.Values.identityTrustDomain | default .Values.clusterDomain) -}}
 env:
 - name: _pod_name
@@ -168,6 +171,15 @@ readinessProbe:
     path: /ready
     port: {{.Values.proxy.ports.admin}}
   initialDelaySeconds: 2
+{{- if and .Values.proxy.nativeSidecar .Values.proxy.await }}
+startupProbe:
+  httpGet:
+    path: /ready
+    port: {{.Values.proxy.ports.admin}}
+  initialDelaySeconds: {{.Values.proxy.startupProbe.initialDelaySeconds}}
+  periodSeconds: {{.Values.proxy.startupProbe.periodSeconds}}
+  failureThreshold: {{.Values.proxy.startupProbe.failureThreshold}}
+{{- end }}
 {{- if .Values.proxy.resources }}
 {{ include "partials.resources" .Values.proxy.resources }}
 {{- end }}
@@ -182,7 +194,7 @@ securityContext:
   seccompProfile:
     type: RuntimeDefault
 terminationMessagePolicy: FallbackToLogsOnError
-{{- if or (.Values.proxy.await) (.Values.proxy.waitBeforeExitSeconds) }}
+{{- if and (not .Values.proxy.nativeSidecar) (or .Values.proxy.await .Values.proxy.waitBeforeExitSeconds) }}
 lifecycle:
 {{- if .Values.proxy.await }}
   postStart:
@@ -212,4 +224,7 @@ volumeMounts:
   name: {{.Values.proxy.saMountPath.name}}
   readOnly: {{.Values.proxy.saMountPath.readOnly}}
 {{- end -}}
+{{- if .Values.proxy.nativeSidecar }}
+restartPolicy: Always
+{{- end -}}
 {{- end }}
diff --git a/charts/patch/templates/patch.json b/charts/patch/templates/patch.json
@@ -1,4 +1,14 @@
 {{ $prefix := .Values.pathPrefix -}}
+{{/*
+$initIndex represents the patch insertion index of the next initContainer when
+proxy.nativeSidecar is true. If enabled, the proxy-init or network-validator
+should run first, immediately followed by the proxy. This ordering allows us
+to proxy traffic in subsequent initContainers.
+
+Note: dig is not used directly on .Values because it rejects chartutil.Values
+structs.
+*/}}
+{{- $initIndex := ternary "0" "-" (.Values.proxy | default (dict) | dig "nativeSidecar" false) -}}
 [
   {{- if .Values.addRootMetadata }}
   {
@@ -62,14 +72,14 @@
   },
   {
     "op": "add",
-    "path": "{{$prefix}}/spec/initContainers/-",
+    "path": "{{$prefix}}/spec/initContainers/{{$initIndex}}{{$initIndex = add1 $initIndex}}",
     "value":
       {{- include "partials.proxy-init" . | fromYaml | toPrettyJson | nindent 6 }}
   },
   {{- else if and .Values.proxy .Values.cniEnabled }}
   {
     "op": "add",
-    "path": "{{$prefix}}/spec/initContainers/-",
+    "path": "{{$prefix}}/spec/initContainers/{{$initIndex}}{{$initIndex = add1 $initIndex}}",
     "value":
       {{- include "partials.network-validator" . | fromYaml | toPrettyJson | nindent 6 }}
   },
@@ -103,7 +113,9 @@
   {{- end }}
   {
     "op": "add",
-  {{- if .Values.proxy.await }}
+  {{- if .Values.proxy.nativeSidecar }}
+    "path": "{{$prefix}}/spec/initContainers/{{$initIndex}}",
+  {{- else if .Values.proxy.await }}
     "path": "{{$prefix}}/spec/containers/0",
   {{- else }}
     "path": "{{$prefix}}/spec/containers/-",

diff --git a/cli/cmd/doc.go b/cli/cmd/doc.go
@@ -280,5 +280,9 @@ func generateAnnotationsDocs() []annotationDoc {
 			Name:        k8s.ProxyShutdownGracePeriodAnnotation,
 			Description: "Grace period for graceful proxy shutdowns. If this timeout elapses before all open connections have completed, the proxy will terminate forcefully, closing any remaining connections.",
 		},
+		{
+			Name:        k8s.ProxyEnableNativeSidecarAnnotation,
+			Description: "Enable KEP-753 native sidecars. This is an experimental feature. It requires Kubernetes >= 1.29. If enabled, .proxy.waitBeforeExitSeconds should not be used.",
+		},
 	}
 }
diff --git a/cli/cmd/inject.go b/cli/cmd/inject.go
@@ -491,6 +491,10 @@ func getOverrideAnnotations(values *linkerd2.Values, base *linkerd2.Values) map[
 		overrideAnnotations[k8s.ProxyShutdownGracePeriodAnnotation] = proxy.ShutdownGracePeriod
 	}
 
+	if proxy.NativeSidecar != baseProxy.NativeSidecar {
+		overrideAnnotations[k8s.ProxyEnableNativeSidecarAnnotation] = strconv.FormatBool(proxy.NativeSidecar)
+	}
+
 	return overrideAnnotations
 }
 

diff --git a/cli/cmd/inject_test.go b/cli/cmd/inject_test.go
@@ -344,6 +344,17 @@ func TestUninjectAndInject(t *testing.T) {
 				return values
 			}(),
 		},
+		{
+			inputFileName:  "inject_emojivoto_deployment.input.yml",
+			goldenFileName: "inject_emojivoto_deployment_native_sidecar.golden.yml",
+			reportFileName: "inject_emojivoto_deployment.report",
+			injectProxy:    true,
+			testInjectConfig: func() *linkerd2.Values {
+				values := defaultConfig()
+				values.Proxy.NativeSidecar = true
+				return values
+			}(),
+		},
 	}
 
 	for i, tc := range testCases {
@@ -678,6 +689,7 @@ func TestProxyConfigurationAnnotations(t *testing.T) {
 	values.Proxy.Await = false
 	values.Proxy.AccessLog = "apache"
 	values.Proxy.ShutdownGracePeriod = "60s"
+	values.Proxy.NativeSidecar = true
 
 	expectedOverrides := map[string]string{
 		k8s.ProxyIgnoreInboundPortsAnnotation:  "8500-8505",
@@ -699,6 +711,7 @@ func TestProxyConfigurationAnnotations(t *testing.T) {
 		k8s.ProxyAwait:                            "disabled",
 		k8s.ProxyAccessLogAnnotation:              "apache",
 		k8s.ProxyShutdownGracePeriodAnnotation:    "60s",
+		k8s.ProxyEnableNativeSidecarAnnotation:    "true",
 	}
 
 	overrides := getOverrideAnnotations(values, baseValues)

diff --git a/cli/cmd/options.go b/cli/cmd/options.go
@@ -441,6 +441,12 @@ func makeInjectFlags(defaults *l5dcharts.Values) ([]flag.Flag, *pflag.FlagSet) {
 	injectFlags := pflag.NewFlagSet("inject", pflag.ExitOnError)
 
 	flags := []flag.Flag{
+		flag.NewBoolFlag(injectFlags, "native-sidecar", false, "Enable native sidecar",
+			func(values *l5dcharts.Values, value bool) error {
+				values.Proxy.NativeSidecar = value
+				return nil
+			}),
+
 		flag.NewInt64Flag(injectFlags, "wait-before-exit-seconds", int64(defaults.Proxy.WaitBeforeExitSeconds),
 			"The period during which the proxy sidecar must stay alive while its pod is terminating. "+
 				"Must be smaller than terminationGracePeriodSeconds for the pod (default 0)",