Antrea Prometheus integration (antrea-io#236)

Integrate with Prometheus monitoring solution. Integration of the Prometheus client into Antrea controller and agent allows the exposure of various metrics to Prometheus server. In addition to Antrea's own set of metrics, Prometheus client will also expose metrics which are defined by various components which are part of the Antrea ecosystem, e.g golang, Prometheus itself etc.
ksamoray · Mar 31, 2020 · a674dce · a674dce
1 parent 775ce88
commit a674dce
Show file tree

Hide file tree

Showing 13 changed files with 306 additions and 14 deletions.
diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml
@@ -286,6 +286,15 @@ data:
     # networkPolicyOnly: Antrea enforces NetworkPolicy only, and utilizes CNI chaining and delegates Pod IPAM and connectivity to primary CNI.
     #
     trafficEncapMode: networkPolicyOnly
+
+    # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener
+    #enablePrometheusMetrics: false
+
+    # Enable golang metrics exposure via Prometheus.
+    #enablePrometheusGoMetrics: false
+
+    # Enable process metrics exposure via Prometheus.
+    #enablePrometheusProcessMetrics: false
   antrea-cni.conf: |
     {
         "cniVersion":"0.3.0",
@@ -295,18 +304,30 @@ data:
             "type": "host-local"
         }
     }
-  antrea-controller.conf: ""
+  antrea-controller.conf: |
+    # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener.
+    #enablePrometheusMetrics: false
+
+    # Enable golang metrics exposure via Prometheus.
+    #enablePrometheusGoMetrics: false
+
+    # Enable process metrics exposure via Prometheus.
+    #enablePrometheusProcessMetrics: false
 kind: ConfigMap
 metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-57hfkfg8kd
+  name: antrea-config-b4d69452ff
   namespace: kube-system
 ---
 apiVersion: v1
 kind: Service
 metadata:
+  annotations:
+    prometheus.io/port: "443"
+    prometheus.io/scheme: https
+    prometheus.io/scrape: "true"
   labels:
     app: antrea
   name: antrea
@@ -394,7 +415,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-57hfkfg8kd
+          name: antrea-config-b4d69452ff
         name: antrea-config
 ---
 apiVersion: apiregistration.k8s.io/v1
@@ -444,6 +465,10 @@ spec:
       component: antrea-agent
   template:
     metadata:
+      annotations:
+        prometheus.io/port: "10443"
+        prometheus.io/scheme: https
+        prometheus.io/scrape: "true"
       labels:
         app: antrea
         component: antrea-agent
@@ -578,7 +603,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-57hfkfg8kd
+          name: antrea-config-b4d69452ff
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml
@@ -286,6 +286,15 @@ data:
     # networkPolicyOnly: Antrea enforces NetworkPolicy only, and utilizes CNI chaining and delegates Pod IPAM and connectivity to primary CNI.
     #
     #trafficEncapMode: encap
+
+    # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener
+    #enablePrometheusMetrics: false
+
+    # Enable golang metrics exposure via Prometheus.
+    #enablePrometheusGoMetrics: false
+
+    # Enable process metrics exposure via Prometheus.
+    #enablePrometheusProcessMetrics: false
   antrea-cni.conf: |
     {
         "cniVersion":"0.3.0",
@@ -295,13 +304,21 @@ data:
             "type": "host-local"
         }
     }
-  antrea-controller.conf: ""
+  antrea-controller.conf: |
+    # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener.
+    #enablePrometheusMetrics: false
+
+    # Enable golang metrics exposure via Prometheus.
+    #enablePrometheusGoMetrics: false
+
+    # Enable process metrics exposure via Prometheus.
+    #enablePrometheusProcessMetrics: false
 kind: ConfigMap
 metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-f59cfh8thg
+  name: antrea-config-fggd7g4h2k
   namespace: kube-system
 ---
 apiVersion: v1
@@ -316,6 +333,10 @@ type: Opaque
 apiVersion: v1
 kind: Service
 metadata:
+  annotations:
+    prometheus.io/port: "443"
+    prometheus.io/scheme: https
+    prometheus.io/scrape: "true"
   labels:
     app: antrea
   name: antrea
@@ -403,7 +424,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-f59cfh8thg
+          name: antrea-config-fggd7g4h2k
         name: antrea-config
 ---
 apiVersion: apiregistration.k8s.io/v1
@@ -453,6 +474,10 @@ spec:
       component: antrea-agent
   template:
     metadata:
+      annotations:
+        prometheus.io/port: "10443"
+        prometheus.io/scheme: https
+        prometheus.io/scrape: "true"
       labels:
         app: antrea
         component: antrea-agent
@@ -619,7 +644,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-f59cfh8thg
+          name: antrea-config-fggd7g4h2k
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml
@@ -286,6 +286,15 @@ data:
     # networkPolicyOnly: Antrea enforces NetworkPolicy only, and utilizes CNI chaining and delegates Pod IPAM and connectivity to primary CNI.
     #
     #trafficEncapMode: encap
+
+    # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener
+    #enablePrometheusMetrics: false
+
+    # Enable golang metrics exposure via Prometheus.
+    #enablePrometheusGoMetrics: false
+
+    # Enable process metrics exposure via Prometheus.
+    #enablePrometheusProcessMetrics: false
   antrea-cni.conf: |
     {
         "cniVersion":"0.3.0",
@@ -295,18 +304,30 @@ data:
             "type": "host-local"
         }
     }
-  antrea-controller.conf: ""
+  antrea-controller.conf: |
+    # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener.
+    #enablePrometheusMetrics: false
+
+    # Enable golang metrics exposure via Prometheus.
+    #enablePrometheusGoMetrics: false
+
+    # Enable process metrics exposure via Prometheus.
+    #enablePrometheusProcessMetrics: false
 kind: ConfigMap
 metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-b2b5bdkh8t
+  name: antrea-config-656thg244c
   namespace: kube-system
 ---
 apiVersion: v1
 kind: Service
 metadata:
+  annotations:
+    prometheus.io/port: "443"
+    prometheus.io/scheme: https
+    prometheus.io/scrape: "true"
   labels:
     app: antrea
   name: antrea
@@ -394,7 +415,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-b2b5bdkh8t
+          name: antrea-config-656thg244c
         name: antrea-config
 ---
 apiVersion: apiregistration.k8s.io/v1
@@ -444,6 +465,10 @@ spec:
       component: antrea-agent
   template:
     metadata:
+      annotations:
+        prometheus.io/port: "10443"
+        prometheus.io/scheme: https
+        prometheus.io/scrape: "true"
       labels:
         app: antrea
         component: antrea-agent
@@ -578,7 +603,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-b2b5bdkh8t
+          name: antrea-config-656thg244c
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/base/agent.yml b/build/yamls/base/agent.yml
@@ -13,6 +13,10 @@ spec:
     type: RollingUpdate
   template:
     metadata:
+      annotations:
+        prometheus.io/port: "10443"
+        prometheus.io/scrape: "true"
+        prometheus.io/scheme: "https"
       labels:
         component: antrea-agent
     spec:

diff --git a/build/yamls/base/conf/antrea-agent.conf b/build/yamls/base/conf/antrea-agent.conf
@@ -41,3 +41,12 @@
 # networkPolicyOnly: Antrea enforces NetworkPolicy only, and utilizes CNI chaining and delegates Pod IPAM and connectivity to primary CNI.
 #
 #trafficEncapMode: encap
+
+# Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener
+#enablePrometheusMetrics: false
+
+# Enable golang metrics exposure via Prometheus.
+#enablePrometheusGoMetrics: false
+
+# Enable process metrics exposure via Prometheus.
+#enablePrometheusProcessMetrics: false
diff --git a/build/yamls/base/conf/antrea-controller.conf b/build/yamls/base/conf/antrea-controller.conf
@@ -0,0 +1,8 @@
+# Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener.
+#enablePrometheusMetrics: false
+
+# Enable golang metrics exposure via Prometheus.
+#enablePrometheusGoMetrics: false
+
+# Enable process metrics exposure via Prometheus.
+#enablePrometheusProcessMetrics: false
diff --git a/build/yamls/base/controller.yml b/build/yamls/base/controller.yml
@@ -2,6 +2,10 @@
 apiVersion: v1
 kind: Service
 metadata:
+  annotations:
+    prometheus.io/port: "443"
+    prometheus.io/scrape: "true"
+    prometheus.io/scheme: "https"
   name: antrea
 spec:
   ports:

diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go
@@ -30,6 +30,7 @@ import (
 	"github.com/vmware-tanzu/antrea/pkg/agent/controller/networkpolicy"
 	"github.com/vmware-tanzu/antrea/pkg/agent/controller/noderoute"
 	"github.com/vmware-tanzu/antrea/pkg/agent/interfacestore"
+	"github.com/vmware-tanzu/antrea/pkg/agent/metrics"
 	"github.com/vmware-tanzu/antrea/pkg/agent/openflow"
 	"github.com/vmware-tanzu/antrea/pkg/agent/route"
 	"github.com/vmware-tanzu/antrea/pkg/apis/networking/v1beta1"
@@ -151,6 +152,13 @@ func run(o *Options) error {
 
 	go networkPolicyController.Run(stopCh)
 
+	if o.config.EnablePrometheusMetrics {
+		go metrics.InitializePrometheusMetrics(
+			o.config.EnablePrometheusGoMetrics,
+			o.config.EnablePrometheusProcessMetrics,
+			o.config.OVSBridge, ifaceStore, ofClient)
+	}
+
 	agentMonitor := monitor.NewAgentMonitor(
 		crdClient,
 		o.config.OVSBridge,

diff --git a/cmd/antrea-agent/config.go b/cmd/antrea-agent/config.go
@@ -72,4 +72,13 @@ type AgentConfig struct {
 	// Hybrid: noEncap if worker Nodes on same subnet, otherwise encap.
 	// NetworkPolicyOnly: Antrea enforces NetworkPolicy only, and utilizes CNI chaining and delegates Pod IPAM and connectivity to primary CNI.
 	TrafficEncapMode string `yaml:"trafficEncapMode,omitempty"`
+	// Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener
+	// Defaults to false.
+	EnablePrometheusMetrics bool `yaml:"enablePrometheusMetrics,omitempty"`
+	// Enable golang metrics exposure via Prometheus
+	// Defaults to false.
+	EnablePrometheusGoMetrics bool `yaml:"enablePrometheusGoMetrics,omitempty"`
+	// Enable process metrics exposure via Prometheus
+	// Defaults to false.
+	EnablePrometheusProcessMetrics bool `yaml:"enablePrometheusProcessMetrics,omitempty"`
 }
diff --git a/cmd/antrea-controller/config.go b/cmd/antrea-controller/config.go
@@ -22,4 +22,13 @@ type ControllerConfig struct {
 	// clientConnection specifies the kubeconfig file and client connection settings for the agent
 	// to communicate with the apiserver.
 	ClientConnection componentbaseconfig.ClientConnectionConfiguration `yaml:"clientConnection"`
+	// Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener
+	// Defaults to false.
+	EnablePrometheusMetrics bool `yaml:"enablePrometheusMetrics,omitempty"`
+	// Enable golang metrics exposure via Prometheus
+	// Defaults to false.
+	EnablePrometheusGoMetrics bool `yaml:"enablePrometheusGoMetrics,omitempty"`
+	// Enable process metrics exposure via Prometheus
+	// Defaults to false.
+	EnablePrometheusProcessMetrics bool `yaml:"enablePrometheusProcessMetrics,omitempty"`
 }