Antrea Prometheus integration (antrea-io#236)

Integrate with Prometheus monitoring solution. Integration of the Prometheus client into Antrea controller and agent allows the exposure of various metrics to Prometheus server. In addition to Antrea's own set of metrics, Prometheus client will also expose metrics which are defined by various components which are part of the Antrea ecosystem, e.g golang, Prometheus itself etc.
ksamoray · Apr 22, 2020 · 0ab5d9e · 0ab5d9e
1 parent 24442bb
commit 0ab5d9e
Show file tree

Hide file tree

Showing 18 changed files with 299 additions and 77 deletions.
diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml
@@ -332,6 +332,9 @@ data:
     # Note that if it's set to another value, the `containerPort` of the `api` port of the
     # `antrea-agent` container must be set to the same value.
     #apiPort: 10350
+
+    # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener
+    #enablePrometheusMetrics: false
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -354,12 +357,15 @@ data:
     # Note that if it's set to another value, the `containerPort` of the `api` port of the
     # `antrea-controller` container must be set to the same value.
     #apiPort: 10349
+
+    # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener.
+    #enablePrometheusMetrics: false
 kind: ConfigMap
 metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-t4m46b8f6h
+  name: antrea-config-td846cf4bm
   namespace: kube-system
 ---
 apiVersion: v1
@@ -459,7 +465,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-t4m46b8f6h
+          name: antrea-config-td846cf4bm
         name: antrea-config
       - hostPath:
           path: /var/log/antrea
@@ -555,7 +561,6 @@ spec:
         name: antrea-agent
         ports:
         - containerPort: 10350
-          hostPort: 10350
           name: api
           protocol: TCP
         readinessProbe:
@@ -658,7 +663,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-t4m46b8f6h
+          name: antrea-config-td846cf4bm
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml
@@ -332,6 +332,9 @@ data:
     # Note that if it's set to another value, the `containerPort` of the `api` port of the
     # `antrea-agent` container must be set to the same value.
     #apiPort: 10350
+
+    # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener
+    #enablePrometheusMetrics: false
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -354,12 +357,15 @@ data:
     # Note that if it's set to another value, the `containerPort` of the `api` port of the
     # `antrea-controller` container must be set to the same value.
     #apiPort: 10349
+
+    # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener.
+    #enablePrometheusMetrics: false
 kind: ConfigMap
 metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-5754dg84hf
+  name: antrea-config-h45gtb8dbg
   namespace: kube-system
 ---
 apiVersion: v1
@@ -459,7 +465,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-5754dg84hf
+          name: antrea-config-h45gtb8dbg
         name: antrea-config
       - hostPath:
           path: /var/log/antrea
@@ -555,7 +561,6 @@ spec:
         name: antrea-agent
         ports:
         - containerPort: 10350
-          hostPort: 10350
           name: api
           protocol: TCP
         readinessProbe:
@@ -658,7 +663,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-5754dg84hf
+          name: antrea-config-h45gtb8dbg
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml
@@ -332,6 +332,9 @@ data:
     # Note that if it's set to another value, the `containerPort` of the `api` port of the
     # `antrea-agent` container must be set to the same value.
     #apiPort: 10350
+
+    # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener
+    #enablePrometheusMetrics: false
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -354,12 +357,15 @@ data:
     # Note that if it's set to another value, the `containerPort` of the `api` port of the
     # `antrea-controller` container must be set to the same value.
     #apiPort: 10349
+
+    # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener.
+    #enablePrometheusMetrics: false
 kind: ConfigMap
 metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-c7579447k2
+  name: antrea-config-d7m57h87ck
   namespace: kube-system
 ---
 apiVersion: v1
@@ -468,7 +474,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-c7579447k2
+          name: antrea-config-d7m57h87ck
         name: antrea-config
       - hostPath:
           path: /var/log/antrea
@@ -596,7 +602,6 @@ spec:
         name: antrea-agent
         ports:
         - containerPort: 10350
-          hostPort: 10350
           name: api
           protocol: TCP
         readinessProbe:
@@ -699,7 +704,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-c7579447k2
+          name: antrea-config-d7m57h87ck
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml
@@ -332,6 +332,9 @@ data:
     # Note that if it's set to another value, the `containerPort` of the `api` port of the
     # `antrea-agent` container must be set to the same value.
     #apiPort: 10350
+
+    # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener
+    #enablePrometheusMetrics: false
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -354,12 +357,15 @@ data:
     # Note that if it's set to another value, the `containerPort` of the `api` port of the
     # `antrea-controller` container must be set to the same value.
     #apiPort: 10349
+
+    # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener.
+    #enablePrometheusMetrics: false
 kind: ConfigMap
 metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-428d4tg64g
+  name: antrea-config-f95kf94mk9
   namespace: kube-system
 ---
 apiVersion: v1
@@ -459,7 +465,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-428d4tg64g
+          name: antrea-config-f95kf94mk9
         name: antrea-config
       - hostPath:
           path: /var/log/antrea
@@ -555,7 +561,6 @@ spec:
         name: antrea-agent
         ports:
         - containerPort: 10350
-          hostPort: 10350
           name: api
           protocol: TCP
         readinessProbe:
@@ -658,7 +663,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-428d4tg64g
+          name: antrea-config-f95kf94mk9
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/base/agent.yml b/build/yamls/base/agent.yml
@@ -77,7 +77,6 @@ spec:
                   fieldPath: spec.nodeName
           ports:
             - containerPort: 10350
-              hostPort: 10350
               name: api
               protocol: TCP
           livenessProbe:

diff --git a/build/yamls/base/conf/antrea-agent.conf b/build/yamls/base/conf/antrea-agent.conf
@@ -46,3 +46,6 @@
 # Note that if it's set to another value, the `containerPort` of the `api` port of the
 # `antrea-agent` container must be set to the same value.
 #apiPort: 10350
+
+# Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener
+#enablePrometheusMetrics: false
diff --git a/build/yamls/base/conf/antrea-controller.conf b/build/yamls/base/conf/antrea-controller.conf
@@ -2,3 +2,6 @@
 # Note that if it's set to another value, the `containerPort` of the `api` port of the
 # `antrea-controller` container must be set to the same value.
 #apiPort: 10349
+
+# Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener.
+#enablePrometheusMetrics: false
diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go
@@ -30,6 +30,7 @@ import (
 	"github.com/vmware-tanzu/antrea/pkg/agent/controller/networkpolicy"
 	"github.com/vmware-tanzu/antrea/pkg/agent/controller/noderoute"
 	"github.com/vmware-tanzu/antrea/pkg/agent/interfacestore"
+	"github.com/vmware-tanzu/antrea/pkg/agent/metrics"
 	"github.com/vmware-tanzu/antrea/pkg/agent/openflow"
 	"github.com/vmware-tanzu/antrea/pkg/agent/querier"
 	"github.com/vmware-tanzu/antrea/pkg/agent/route"
@@ -161,11 +162,19 @@ func run(o *Options) error {
 		ovsBridgeClient,
 		networkPolicyController)
 
+	if o.config.EnablePrometheusMetrics {
+		go metrics.InitializePrometheusMetrics(o.config.OVSBridge, ifaceStore, ofClient)
+	}
+
 	agentMonitor := monitor.NewAgentMonitor(crdClient, agentQuerier)
 
 	go agentMonitor.Run(stopCh)
 
-	apiServer, err := apiserver.New(agentQuerier, networkPolicyController, o.config.APIPort)
+	apiServer, err := apiserver.New(
+		agentQuerier,
+		networkPolicyController,
+		o.config.APIPort,
+		o.config.EnablePrometheusMetrics)
 	if err != nil {
 		return fmt.Errorf("error when creating agent API server: %v", err)
 	}

diff --git a/cmd/antrea-agent/config.go b/cmd/antrea-agent/config.go
@@ -75,4 +75,7 @@ type AgentConfig struct {
 	// APIPort is the port for the antrea-agent APIServer to serve on.
 	// Defaults to 10350.
 	APIPort int `yaml:"apiPort,omitempty"`
+	// Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener
+	// Defaults to false.
+	EnablePrometheusMetrics bool `yaml:"enablePrometheusMetrics,omitempty"`
 }
diff --git a/cmd/antrea-controller/config.go b/cmd/antrea-controller/config.go
@@ -25,4 +25,7 @@ type ControllerConfig struct {
 	// APIPort is the port for the antrea-controller APIServer to serve on.
 	// Defaults to 10349.
 	APIPort int `yaml:"apiPort,omitempty"`
+	// Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener
+	// Defaults to false.
+	EnablePrometheusMetrics bool `yaml:"enablePrometheusMetrics,omitempty"`
 }
diff --git a/cmd/antrea-controller/controller.go b/cmd/antrea-controller/controller.go
@@ -19,6 +19,7 @@ import (
 	"net"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
 	genericopenapi "k8s.io/apiserver/pkg/endpoints/openapi"
 	genericapiserver "k8s.io/apiserver/pkg/server"
 	genericoptions "k8s.io/apiserver/pkg/server/options"
@@ -34,6 +35,7 @@ import (
 	"github.com/vmware-tanzu/antrea/pkg/k8s"
 	"github.com/vmware-tanzu/antrea/pkg/monitor"
 	"github.com/vmware-tanzu/antrea/pkg/signals"
+	"github.com/vmware-tanzu/antrea/pkg/util/env"
 	"github.com/vmware-tanzu/antrea/pkg/version"
 )
 
@@ -78,7 +80,8 @@ func run(o *Options) error {
 		addressGroupStore,
 		appliedToGroupStore,
 		networkPolicyStore,
-		controllerQuerier)
+		controllerQuerier,
+		o.config.EnablePrometheusMetrics)
 	if err != nil {
 		return fmt.Errorf("error creating API server config: %v", err)
 	}
@@ -100,23 +103,50 @@ func run(o *Options) error {
 
 	go apiServer.GenericAPIServer.PrepareRun().Run(stopCh)
 
+	if o.config.EnablePrometheusMetrics {
+		go initializePrometheusMetrics()
+	}
+
 	<-stopCh
 	klog.Info("Stopping Antrea controller")
 	return nil
 }
 
+// Initialize Prometheus metrics collection.
+func initializePrometheusMetrics() {
+
+	nodeName, err := env.GetNodeName()
+	if err != nil {
+		klog.Errorf("Failed to retrieve controller K8S node name, %v", err)
+	}
+
+	klog.Info("Initializing prometheus")
+	gaugeHost := prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "antrea_controller_node",
+		Help: "Antrea controller node name (as a label), typically used in grouping/aggregating stats; " +
+			"The value of the gauge is always set to 1.",
+		ConstLabels: prometheus.Labels{"k8s_nodename": nodeName},
+	})
+	gaugeHost.Set(1)
+	prometheus.MustRegister(gaugeHost)
+}
+
 func createAPIServerConfig(kubeconfig string,
 	bindPort int,
 	addressGroupStore storage.Interface,
 	appliedToGroupStore storage.Interface,
 	networkPolicyStore storage.Interface,
-	controllerQuerier querier.ControllerQuerier) (*apiserver.Config, error) {
+	controllerQuerier querier.ControllerQuerier,
+	authorizeMetrics bool) (*apiserver.Config, error) {
 	// TODO:
 	// 1. Support user-provided certificate.
 	secureServing := genericoptions.NewSecureServingOptions().WithLoopback()
 	authentication := genericoptions.NewDelegatingAuthenticationOptions()
 	authorization := genericoptions.NewDelegatingAuthorizationOptions()
 
+	if authorizeMetrics {
+		authorization.WithAlwaysAllowPaths("/metrics")
+	}
 	// Set the PairName but leave certificate directory blank to generate in-memory by default
 	secureServing.ServerCert.CertDirectory = ""
 	secureServing.ServerCert.PairName = "antrea-apiserver"

diff --git a/go.mod b/go.mod
@@ -25,6 +25,7 @@ require (
 	github.com/imdario/mergo v0.3.7 // indirect
 	github.com/j-keck/arping v1.0.0
 	github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd
+	github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829
 	github.com/satori/go.uuid v1.2.0
 	github.com/sirupsen/logrus v1.4.1
 	github.com/spf13/cobra v0.0.5