Switch to JSON for access logs

monzo · May 8, 2024 · 5e28e72 · 5e28e72
1 parent 3bc9d49
commit 5e28e72
Show file tree

Hide file tree

Showing 20 changed files with 2,178 additions and 27 deletions.
diff --git a/Makefile b/Makefile
@@ -1,7 +1,7 @@
 # Image URL to use all building/pushing image targets
 IMG ?= ${ACC}.dkr.ecr.eu-west-1.amazonaws.com/monzo/egress-operator:manager-$(shell git rev-parse --short head)
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
-CRD_OPTIONS ?= "crd:trivialVersions=false"
+CRD_OPTIONS ?= "crd:trivialVersions=true"
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))

diff --git a/controllers/configmap.go b/controllers/configmap.go
@@ -3,6 +3,8 @@ package controllers
 import (
 	"context"
 	"fmt"
+	//structpb "github.com/golang/protobuf/ptypes/struct"
+	"google.golang.org/protobuf/types/known/anypb"
 	"google.golang.org/protobuf/types/known/durationpb"
 	"hash/fnv"
 	"strconv"
@@ -13,16 +15,16 @@ import (
 	envoycorev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 	envoyendpoint "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
 	envoylistener "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
-	filev3 "github.com/envoyproxy/go-control-plane/envoy/extensions/access_loggers/file/v3"
+	streamv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/access_loggers/stream/v3"
 	aggregatev3 "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/aggregate/v3"
 	tcpproxyv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/tcp_proxy/v3"
 	udpproxyv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/udp/udp_proxy/v3"
-	"github.com/golang/protobuf/ptypes/wrappers"
-	"google.golang.org/protobuf/types/known/wrapperspb"
-
 	"github.com/golang/protobuf/jsonpb"
 	"github.com/golang/protobuf/ptypes"
 	"github.com/golang/protobuf/ptypes/duration"
+	"github.com/golang/protobuf/ptypes/wrappers"
+	"google.golang.org/protobuf/types/known/structpb"
+	"google.golang.org/protobuf/types/known/wrapperspb"
 	corev1 "k8s.io/api/core/v1"
 	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -35,6 +37,34 @@ import (
 
 // +kubebuilder:rbac:namespace=egress-operator-system,groups=core,resources=configmaps,verbs=get;list;watch;create;patch
 
+var (
+	logFields = &structpb.Struct{
+		Fields: map[string]*structpb.Value{
+			"authority":                         {Kind: &structpb.Value_StringValue{StringValue: "%REQ(:AUTHORITY)%"}},
+			"bytes_received":                    {Kind: &structpb.Value_StringValue{StringValue: "%BYTES_RECEIVED%"}},
+			"bytes_sent":                        {Kind: &structpb.Value_StringValue{StringValue: "%BYTES_SENT%"}},
+			"connection_termination_details":    {Kind: &structpb.Value_StringValue{StringValue: "%CONNECTION_TERMINATION_DETAILS%"}},
+			"downstream_local_address":          {Kind: &structpb.Value_StringValue{StringValue: "%DOWNSTREAM_LOCAL_ADDRESS%"}},
+			"downstream_remote_address":         {Kind: &structpb.Value_StringValue{StringValue: "%DOWNSTREAM_REMOTE_ADDRESS%"}},
+			"duration":                          {Kind: &structpb.Value_StringValue{StringValue: "%DURATION%"}},
+			"method":                            {Kind: &structpb.Value_StringValue{StringValue: "%REQ(:METHOD)%"}},
+			"path":                              {Kind: &structpb.Value_StringValue{StringValue: "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%"}},
+			"protocol":                          {Kind: &structpb.Value_StringValue{StringValue: "%PROTOCOL%"}},
+			"requested_server_name":             {Kind: &structpb.Value_StringValue{StringValue: "%REQUESTED_SERVER_NAME%"}},
+			"response_code":                     {Kind: &structpb.Value_StringValue{StringValue: "%RESPONSE_CODE%"}},
+			"response_code_details":             {Kind: &structpb.Value_StringValue{StringValue: "%RESPONSE_CODE_DETAILS%"}},
+			"response_flags":                    {Kind: &structpb.Value_StringValue{StringValue: "%RESPONSE_FLAGS%"}},
+			"start_time":                        {Kind: &structpb.Value_StringValue{StringValue: "%START_TIME%"}},
+			"upstream_cluster":                  {Kind: &structpb.Value_StringValue{StringValue: "%UPSTREAM_CLUSTER%"}},
+			"upstream_host":                     {Kind: &structpb.Value_StringValue{StringValue: "%UPSTREAM_HOST%"}},
+			"upstream_local_address":            {Kind: &structpb.Value_StringValue{StringValue: "%UPSTREAM_LOCAL_ADDRESS%"}},
+			"upstream_service_time":             {Kind: &structpb.Value_StringValue{StringValue: "%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%"}},
+			"upstream_transport_failure_reason": {Kind: &structpb.Value_StringValue{StringValue: "%UPSTREAM_TRANSPORT_FAILURE_REASON%"}},
+			"user_agent":                        {Kind: &structpb.Value_StringValue{StringValue: "%REQ(USER-AGENT)%"}},
+		},
+	}
+)
+
 func (r *ExternalServiceReconciler) reconcileConfigMap(ctx context.Context, req ctrl.Request, es *egressv1.ExternalService, desired *corev1.ConfigMap) error {
 	if err := ctrl.SetControllerReference(es, desired, r.Scheme); err != nil {
 		return err
@@ -86,9 +116,6 @@ func adminPort(es *egressv1.ExternalService) int32 {
 	panic("couldn't find a port for admin listener")
 }
 
-const accessLogFormat = `[%START_TIME%] %BYTES_RECEIVED% %BYTES_SENT% %DURATION% "%DOWNSTREAM_REMOTE_ADDRESS%" "%UPSTREAM_HOST%" "%UPSTREAM_CLUSTER%"
-`
-
 func envoyConfig(es *egressv1.ExternalService) (string, error) {
 	config := bootstrap.Bootstrap{
 		Node: &envoycorev3.Node{
@@ -197,16 +224,21 @@ func envoyConfig(es *egressv1.ExternalService) (string, error) {
 		var listener *envoylistener.Listener
 		switch protocol {
 		case envoycorev3.SocketAddress_TCP:
-			accessConfig, err := ptypes.MarshalAny(&filev3.FileAccessLog{
-				AccessLogFormat: &filev3.FileAccessLog_Format{
-					Format: accessLogFormat,
+			accessConfig, err := anypb.New(&streamv3.StdoutAccessLog{
+				AccessLogFormat: &streamv3.StdoutAccessLog_LogFormat{
+					LogFormat: &envoycorev3.SubstitutionFormatString{
+						Format: &envoycorev3.SubstitutionFormatString_JsonFormat{
+							JsonFormat: logFields,
+						},
+						OmitEmptyValues:   true,
+						ContentType:       "application/json; charset=UTF-8",
+						JsonFormatOptions: nil,
+					},
 				},
-				Path: "/dev/stdout",
 			})
-
-			filterConfig, err := ptypes.MarshalAny(&tcpproxyv3.TcpProxy{
+			filterConfig, err := anypb.New(&tcpproxyv3.TcpProxy{
 				AccessLog: []*accesslogfilterv3.AccessLog{{
-					Name:       "envoy.file_access_log",
+					Name:       "envoy.stdout_access_log",
 					ConfigType: &accesslogfilterv3.AccessLog_TypedConfig{TypedConfig: accessConfig},
 				}},
 				StatPrefix: "tcp_proxy",
@@ -236,7 +268,7 @@ func envoyConfig(es *egressv1.ExternalService) (string, error) {
 						}}}}},
 			}
 		case envoycorev3.SocketAddress_UDP:
-			filterConfig, err := ptypes.MarshalAny(&udpproxyv3.UdpProxyConfig{
+			filterConfig, err := anypb.New(&udpproxyv3.UdpProxyConfig{
 				StatPrefix: "udp_proxy",
 				RouteSpecifier: &udpproxyv3.UdpProxyConfig_Cluster{
 					Cluster: name,

diff --git a/controllers/configmap_test.go b/controllers/configmap_test.go
@@ -1,9 +1,11 @@
 package controllers
 
 import (
+	bootstrap "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v3"
+	"github.com/google/go-cmp/cmp"
+	"sigs.k8s.io/yaml"
 	"testing"
 
-	"github.com/google/go-cmp/cmp"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
@@ -97,12 +99,34 @@ staticResources:
         typedConfig:
           '@type': type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy
           accessLog:
-          - name: envoy.file_access_log
+          - name: envoy.stdout_access_log
             typedConfig:
-              '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog
-              format: |
-                [%START_TIME%] %BYTES_RECEIVED% %BYTES_SENT% %DURATION% "%DOWNSTREAM_REMOTE_ADDRESS%" "%UPSTREAM_HOST%" "%UPSTREAM_CLUSTER%"
-              path: /dev/stdout
+              '@type': type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StdoutAccessLog
+              logFormat:
+                contentType: application/json; charset=UTF-8
+                jsonFormat:
+                  authority: '%REQ(:AUTHORITY)%'
+                  bytes_received: '%BYTES_RECEIVED%'
+                  bytes_sent: '%BYTES_SENT%'
+                  connection_termination_details: '%CONNECTION_TERMINATION_DETAILS%'
+                  downstream_local_address: '%DOWNSTREAM_LOCAL_ADDRESS%'
+                  downstream_remote_address: '%DOWNSTREAM_REMOTE_ADDRESS%'
+                  duration: '%DURATION%'
+                  method: '%REQ(:METHOD)%'
+                  path: '%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%'
+                  protocol: '%PROTOCOL%'
+                  requested_server_name: '%REQUESTED_SERVER_NAME%'
+                  response_code: '%RESPONSE_CODE%'
+                  response_code_details: '%RESPONSE_CODE_DETAILS%'
+                  response_flags: '%RESPONSE_FLAGS%'
+                  start_time: '%START_TIME%'
+                  upstream_cluster: '%UPSTREAM_CLUSTER%'
+                  upstream_host: '%UPSTREAM_HOST%'
+                  upstream_local_address: '%UPSTREAM_LOCAL_ADDRESS%'
+                  upstream_service_time: '%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%'
+                  upstream_transport_failure_reason: '%UPSTREAM_TRANSPORT_FAILURE_REASON%'
+                  user_agent: '%REQ(USER-AGENT)%'
+                omitEmptyValues: true
           cluster: foo_TCP_101
           statPrefix: tcp_proxy
     name: foo_TCP_101
@@ -209,12 +233,34 @@ staticResources:
         typedConfig:
           '@type': type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy
           accessLog:
-          - name: envoy.file_access_log
+          - name: envoy.stdout_access_log
             typedConfig:
-              '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog
-              format: |
-                [%START_TIME%] %BYTES_RECEIVED% %BYTES_SENT% %DURATION% "%DOWNSTREAM_REMOTE_ADDRESS%" "%UPSTREAM_HOST%" "%UPSTREAM_CLUSTER%"
-              path: /dev/stdout
+              '@type': type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StdoutAccessLog
+              logFormat:
+                contentType: application/json; charset=UTF-8
+                jsonFormat:
+                  authority: '%REQ(:AUTHORITY)%'
+                  bytes_received: '%BYTES_RECEIVED%'
+                  bytes_sent: '%BYTES_SENT%'
+                  connection_termination_details: '%CONNECTION_TERMINATION_DETAILS%'
+                  downstream_local_address: '%DOWNSTREAM_LOCAL_ADDRESS%'
+                  downstream_remote_address: '%DOWNSTREAM_REMOTE_ADDRESS%'
+                  duration: '%DURATION%'
+                  method: '%REQ(:METHOD)%'
+                  path: '%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%'
+                  protocol: '%PROTOCOL%'
+                  requested_server_name: '%REQUESTED_SERVER_NAME%'
+                  response_code: '%RESPONSE_CODE%'
+                  response_code_details: '%RESPONSE_CODE_DETAILS%'
+                  response_flags: '%RESPONSE_FLAGS%'
+                  start_time: '%START_TIME%'
+                  upstream_cluster: '%UPSTREAM_CLUSTER%'
+                  upstream_host: '%UPSTREAM_HOST%'
+                  upstream_local_address: '%UPSTREAM_LOCAL_ADDRESS%'
+                  upstream_service_time: '%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%'
+                  upstream_transport_failure_reason: '%UPSTREAM_TRANSPORT_FAILURE_REASON%'
+                  user_agent: '%REQ(USER-AGENT)%'
+                omitEmptyValues: true
           cluster: foo_TCP_101
           statPrefix: tcp_proxy
     name: foo_TCP_101
@@ -246,7 +292,15 @@ staticResources:
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			tt.args.es.Spec.EnvoyClusterMaxConnections = tt.args.maxConns
+			got, _ := envoyConfig(tt.args.es)
+
+			var x bootstrap.Bootstrap
+			err := yaml.Unmarshal([]byte(got), &x)
+			if err != nil {
+				t.Error()
+			}
 			if got, _ := envoyConfig(tt.args.es); got != tt.want {
+
 				t.Errorf("envoyConfig() = %v, want %v", got, tt.want)
 				t.Error(cmp.Diff(got, tt.want))
 			}

diff --git a/controllers/testdata/expected_udp_tcp.yaml b/controllers/testdata/expected_udp_tcp.yaml
@@ -0,0 +1,102 @@
+admin:
+  accessLogPath: /dev/stdout
+  address:
+    socketAddress:
+      address: 0.0.0.0
+      portValue: 11000
+node:
+  cluster: foo
+staticResources:
+  clusters:
+    - connectTimeout: 1s
+      dnsLookupFamily: V4_ONLY
+      loadAssignment:
+        clusterName: foo_UDP_100
+        endpoints:
+          - lbEndpoints:
+              - endpoint:
+                  address:
+                    socketAddress:
+                      address: google.com
+                      portValue: 100
+                      protocol: UDP
+      name: foo_UDP_100
+      type: LOGICAL_DNS
+      upstreamConnectionOptions:
+        tcpKeepalive:
+          keepaliveInterval: 5
+          keepaliveProbes: 3
+          keepaliveTime: 30
+    - connectTimeout: 1s
+      dnsLookupFamily: V4_ONLY
+      loadAssignment:
+        clusterName: foo_TCP_101
+        endpoints:
+          - lbEndpoints:
+              - endpoint:
+                  address:
+                    socketAddress:
+                      address: google.com
+                      portValue: 101
+      name: foo_TCP_101
+      type: LOGICAL_DNS
+      upstreamConnectionOptions:
+        tcpKeepalive:
+          keepaliveInterval: 5
+          keepaliveProbes: 3
+          keepaliveTime: 30
+  listeners:
+    - address:
+        socketAddress:
+          address: 0.0.0.0
+          portValue: 100
+          protocol: UDP
+      filterChains:
+        - filters:
+            - name: envoy.filters.udp_listener.udp_proxy
+              typedConfig:
+                "@type": type.googleapis.com/envoy.extensions.filters.udp.udp_proxy.v3.UdpProxyConfig
+                cluster: foo_UDP_100
+                statPrefix: udp_proxy
+      name: foo_UDP_100
+    - address:
+        socketAddress:
+          address: 0.0.0.0
+          portValue: 101
+      filterChains:
+        - filters:
+            - name: envoy.tcp_proxy
+              typedConfig:
+                '@type': type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy
+                accessLog:
+                  - name: envoy.stdout_access_log
+                    typedConfig:
+                      '@type': type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StdoutAccessLog
+                      logFormat:
+                      contentType: "application/json; charset=UTF-8"
+                      jsonFormat:
+                        authority: '%REQ(:AUTHORITY)%'
+                        bytes_received: '%BYTES_RECEIVED%'
+                        bytes_sent: '%BYTES_SENT%'
+                        connection_termination_details: '%CONNECTION_TERMINATION_DETAILS%'
+                        downstream_local_address: '%DOWNSTREAM_LOCAL_ADDRESS%'
+                        downstream_remote_address: '%DOWNSTREAM_REMOTE_ADDRESS%'
+                        duration: '%DURATION%'
+                        method: '%REQ(:METHOD)%'
+                        path: '%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%'
+                        protocol: '%PROTOCOL%'
+                        requested_server_name: '%REQUESTED_SERVER_NAME%'
+                        response_code: '%RESPONSE_CODE%'
+                        response_code_details: '%RESPONSE_CODE_DETAILS%'
+                        response_flags: '%RESPONSE_FLAGS%'
+                        start_time: '%START__TIME%'
+                        upstream_cluster: '%UPSTREAM_CLUSTER%'
+                        upstream_host: '%UPSTREAM_HOST%'
+                        upstream_local_address: '%UPSTREAM_LOCAL_ADDRESS%'
+                        upstream_service_time: '%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%'
+                        upstream_transport_failure_reason: '%UPSTREAM_TRANSPORT_FAILURE_REASON%'
+                        user_agent: '%REQ(USER-AGENT)%'
+                      omitEmptyValues: True
+                        cluster: "foo_TCP_101"
+                        statPrefix: "tcp_proxy"
+                      name: foo_TCP_101