Antrea Prometheus integration (antrea-io#321)

Integrate with Prometheus monitoring solution. Integration of the Prometheus client into Antrea controller and agent allows the exposure of various metrics to Prometheus server. In addition to Antrea's own set of metrics, Prometheus client will also expose metrics which are defined by various components which are part of the Antrea ecosystem, e.g golang, Prometheus itself etc. See antrea-io#236
Dyanngg · Apr 29, 2020 · 6e3be6c · 6e3be6c
1 parent b9a5bd0
commit 6e3be6c
Show file tree

Hide file tree

Showing 18 changed files with 353 additions and 72 deletions.
diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml
@@ -326,6 +326,9 @@ data:
     # Note that if it's set to another value, the `containerPort` of the `api` port of the
     # `antrea-agent` container must be set to the same value.
     #apiPort: 10350
+
+    # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener.
+    #enablePrometheusMetrics: false
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -348,12 +351,15 @@ data:
     # Note that if it's set to another value, the `containerPort` of the `api` port of the
     # `antrea-controller` container must be set to the same value.
     #apiPort: 10349
+
+    # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener.
+    #enablePrometheusMetrics: false
 kind: ConfigMap
 metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-t4m46b8f6h
+  name: antrea-config-522bfh947f
   namespace: kube-system
 ---
 apiVersion: v1
@@ -453,7 +459,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-t4m46b8f6h
+          name: antrea-config-522bfh947f
         name: antrea-config
       - hostPath:
           path: /var/log/antrea
@@ -651,7 +657,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-t4m46b8f6h
+          name: antrea-config-522bfh947f
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml
@@ -326,6 +326,9 @@ data:
     # Note that if it's set to another value, the `containerPort` of the `api` port of the
     # `antrea-agent` container must be set to the same value.
     #apiPort: 10350
+
+    # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener.
+    #enablePrometheusMetrics: false
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -348,12 +351,15 @@ data:
     # Note that if it's set to another value, the `containerPort` of the `api` port of the
     # `antrea-controller` container must be set to the same value.
     #apiPort: 10349
+
+    # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener.
+    #enablePrometheusMetrics: false
 kind: ConfigMap
 metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-5754dg84hf
+  name: antrea-config-t5f26b6m9f
   namespace: kube-system
 ---
 apiVersion: v1
@@ -453,7 +459,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-5754dg84hf
+          name: antrea-config-t5f26b6m9f
         name: antrea-config
       - hostPath:
           path: /var/log/antrea
@@ -651,7 +657,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-5754dg84hf
+          name: antrea-config-t5f26b6m9f
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml
@@ -326,6 +326,9 @@ data:
     # Note that if it's set to another value, the `containerPort` of the `api` port of the
     # `antrea-agent` container must be set to the same value.
     #apiPort: 10350
+
+    # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener.
+    #enablePrometheusMetrics: false
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -348,12 +351,15 @@ data:
     # Note that if it's set to another value, the `containerPort` of the `api` port of the
     # `antrea-controller` container must be set to the same value.
     #apiPort: 10349
+
+    # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener.
+    #enablePrometheusMetrics: false
 kind: ConfigMap
 metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-c7579447k2
+  name: antrea-config-hdhc998mb8
   namespace: kube-system
 ---
 apiVersion: v1
@@ -462,7 +468,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-c7579447k2
+          name: antrea-config-hdhc998mb8
         name: antrea-config
       - hostPath:
           path: /var/log/antrea
@@ -692,7 +698,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-c7579447k2
+          name: antrea-config-hdhc998mb8
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml
@@ -326,6 +326,9 @@ data:
     # Note that if it's set to another value, the `containerPort` of the `api` port of the
     # `antrea-agent` container must be set to the same value.
     #apiPort: 10350
+
+    # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener.
+    #enablePrometheusMetrics: false
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -348,12 +351,15 @@ data:
     # Note that if it's set to another value, the `containerPort` of the `api` port of the
     # `antrea-controller` container must be set to the same value.
     #apiPort: 10349
+
+    # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener.
+    #enablePrometheusMetrics: false
 kind: ConfigMap
 metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-428d4tg64g
+  name: antrea-config-m8cb9g82tf
   namespace: kube-system
 ---
 apiVersion: v1
@@ -453,7 +459,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-428d4tg64g
+          name: antrea-config-m8cb9g82tf
         name: antrea-config
       - hostPath:
           path: /var/log/antrea
@@ -651,7 +657,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-428d4tg64g
+          name: antrea-config-m8cb9g82tf
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/base/conf/antrea-agent.conf b/build/yamls/base/conf/antrea-agent.conf
@@ -46,3 +46,6 @@
 # Note that if it's set to another value, the `containerPort` of the `api` port of the
 # `antrea-agent` container must be set to the same value.
 #apiPort: 10350
+
+# Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener.
+#enablePrometheusMetrics: false
diff --git a/build/yamls/base/conf/antrea-controller.conf b/build/yamls/base/conf/antrea-controller.conf
@@ -2,3 +2,6 @@
 # Note that if it's set to another value, the `containerPort` of the `api` port of the
 # `antrea-controller` container must be set to the same value.
 #apiPort: 10349
+
+# Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener.
+#enablePrometheusMetrics: false
diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go
@@ -30,6 +30,7 @@ import (
 	"github.com/vmware-tanzu/antrea/pkg/agent/controller/networkpolicy"
 	"github.com/vmware-tanzu/antrea/pkg/agent/controller/noderoute"
 	"github.com/vmware-tanzu/antrea/pkg/agent/interfacestore"
+	"github.com/vmware-tanzu/antrea/pkg/agent/metrics"
 	"github.com/vmware-tanzu/antrea/pkg/agent/openflow"
 	"github.com/vmware-tanzu/antrea/pkg/agent/querier"
 	"github.com/vmware-tanzu/antrea/pkg/agent/route"
@@ -161,11 +162,19 @@ func run(o *Options) error {
 		ovsBridgeClient,
 		networkPolicyController)
 
+	if o.config.EnablePrometheusMetrics {
+		metrics.InitializePrometheusMetrics(o.config.OVSBridge, ifaceStore, ofClient)
+	}
+
 	agentMonitor := monitor.NewAgentMonitor(crdClient, agentQuerier)
 
 	go agentMonitor.Run(stopCh)
 
-	apiServer, err := apiserver.New(agentQuerier, networkPolicyController, o.config.APIPort)
+	apiServer, err := apiserver.New(
+		agentQuerier,
+		networkPolicyController,
+		o.config.APIPort,
+		o.config.EnablePrometheusMetrics)
 	if err != nil {
 		return fmt.Errorf("error when creating agent API server: %v", err)
 	}

diff --git a/cmd/antrea-agent/config.go b/cmd/antrea-agent/config.go
@@ -75,4 +75,7 @@ type AgentConfig struct {
 	// APIPort is the port for the antrea-agent APIServer to serve on.
 	// Defaults to 10350.
 	APIPort int `yaml:"apiPort,omitempty"`
+	// Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener
+	// Defaults to false.
+	EnablePrometheusMetrics bool `yaml:"enablePrometheusMetrics,omitempty"`
 }
diff --git a/cmd/antrea-controller/config.go b/cmd/antrea-controller/config.go
@@ -25,4 +25,7 @@ type ControllerConfig struct {
 	// APIPort is the port for the antrea-controller APIServer to serve on.
 	// Defaults to 10349.
 	APIPort int `yaml:"apiPort,omitempty"`
+	// Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener
+	// Defaults to false.
+	EnablePrometheusMetrics bool `yaml:"enablePrometheusMetrics,omitempty"`
 }
diff --git a/cmd/antrea-controller/controller.go b/cmd/antrea-controller/controller.go
@@ -31,6 +31,7 @@ import (
 	"github.com/vmware-tanzu/antrea/pkg/apiserver"
 	"github.com/vmware-tanzu/antrea/pkg/apiserver/openapi"
 	"github.com/vmware-tanzu/antrea/pkg/apiserver/storage"
+	"github.com/vmware-tanzu/antrea/pkg/controller/metrics"
 	"github.com/vmware-tanzu/antrea/pkg/controller/networkpolicy"
 	"github.com/vmware-tanzu/antrea/pkg/controller/networkpolicy/store"
 	"github.com/vmware-tanzu/antrea/pkg/controller/querier"
@@ -81,7 +82,8 @@ func run(o *Options) error {
 		addressGroupStore,
 		appliedToGroupStore,
 		networkPolicyStore,
-		controllerQuerier)
+		controllerQuerier,
+		o.config.EnablePrometheusMetrics)
 	if err != nil {
 		return fmt.Errorf("error creating API server config: %v", err)
 	}
@@ -103,6 +105,10 @@ func run(o *Options) error {
 
 	go apiServer.GenericAPIServer.PrepareRun().Run(stopCh)
 
+	if o.config.EnablePrometheusMetrics {
+		metrics.InitializePrometheusMetrics()
+	}
+
 	<-stopCh
 	klog.Info("Stopping Antrea controller")
 	return nil
@@ -113,7 +119,8 @@ func createAPIServerConfig(kubeconfig string,
 	addressGroupStore storage.Interface,
 	appliedToGroupStore storage.Interface,
 	networkPolicyStore storage.Interface,
-	controllerQuerier querier.ControllerQuerier) (*apiserver.Config, error) {
+	controllerQuerier querier.ControllerQuerier,
+	enableMetrics bool) (*apiserver.Config, error) {
 	// TODO:
 	// 1. Support user-provided certificate.
 	secureServing := genericoptions.NewSecureServingOptions().WithLoopback()
@@ -155,6 +162,7 @@ func createAPIServerConfig(kubeconfig string,
 		openapi.GetOpenAPIDefinitions,
 		genericopenapi.NewDefinitionNamer(apiserver.Scheme))
 	serverConfig.OpenAPIConfig.Info.Title = "Antrea"
+	serverConfig.EnableMetrics = enableMetrics
 
 	return apiserver.NewConfig(
 		serverConfig,

diff --git a/go.mod b/go.mod
@@ -25,6 +25,7 @@ require (
 	github.com/imdario/mergo v0.3.7 // indirect
 	github.com/j-keck/arping v1.0.0
 	github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd
+	github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829
 	github.com/satori/go.uuid v1.2.0
 	github.com/sirupsen/logrus v1.4.1
 	github.com/spf13/cobra v0.0.5

diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go
@@ -37,12 +37,11 @@ import (
 	"github.com/vmware-tanzu/antrea/pkg/agent/types"
 	"github.com/vmware-tanzu/antrea/pkg/agent/util"
 	"github.com/vmware-tanzu/antrea/pkg/ovs/ovsconfig"
+	"github.com/vmware-tanzu/antrea/pkg/util/env"
 )
 
 const (
 	maxRetryForHostLink = 5
-	// nodeNameEnvKey is environment variable.
-	nodeNameEnvKey = "NODE_NAME"
 	// ipsecPSKEnvKey is environment variable.
 	ipsecPSKEnvKey          = "ANTREA_IPSEC_PSK"
 	roundNumKey             = "roundNum" // round number key in externalIDs.
@@ -436,7 +435,7 @@ func (i *Initializer) setupDefaultTunnelInterface(tunnelPortName string) error {
 // initNodeLocalConfig retrieves node's subnet CIDR from node.spec.PodCIDR, which is used for IPAM and setup
 // host gateway interface.
 func (i *Initializer) initNodeLocalConfig() error {
-	nodeName, err := getNodeName()
+	nodeName, err := env.GetNodeName()
 	if err != nil {
 		return err
 	}
@@ -476,24 +475,6 @@ func (i *Initializer) initNodeLocalConfig() error {
 	return nil
 }
 
-// getNodeName returns the node's name used in Kubernetes, based on the priority:
-// - Environment variable NODE_NAME, which should be set by Downward API
-// - OS's hostname
-func getNodeName() (string, error) {
-	nodeName := os.Getenv(nodeNameEnvKey)
-	if nodeName != "" {
-		return nodeName, nil
-	}
-	klog.Infof("Environment variable %s not found, using hostname instead", nodeNameEnvKey)
-	var err error
-	nodeName, err = os.Hostname()
-	if err != nil {
-		klog.Errorf("Failed to get local hostname: %v", err)
-		return "", err
-	}
-	return nodeName, nil
-}
-
 // readIPSecPSK reads the IPSec PSK value from environment variable
 // ANTREA_IPSEC_PSK, when enableIPSecTunnel is set to true.
 func (i *Initializer) readIPSecPSK() error {

diff --git a/pkg/agent/agent_test.go b/pkg/agent/agent_test.go
@@ -17,7 +17,6 @@ package agent
 import (
 	"fmt"
 	"net"
-	"os"
 	"testing"
 
 	mock "github.com/golang/mock/gomock"
@@ -30,38 +29,6 @@ import (
 	ovsconfigtest "github.com/vmware-tanzu/antrea/pkg/ovs/ovsconfig/testing"
 )
 
-func TestGetNodeName(t *testing.T) {
-	hostName, err := os.Hostname()
-	if err != nil {
-		t.Fatalf("Failed to retrieve hostname, %v", err)
-	}
-	testTable := map[string]string{
-		"node1":     "node1",
-		"node_12":   "node_12",
-		"":          hostName,
-		"node-1234": "node-1234",
-	}
-
-	for k, v := range testTable {
-		compareNodeName(k, v, t)
-	}
-}
-
-func compareNodeName(k, v string, t *testing.T) {
-	if k != "" {
-		_ = os.Setenv(nodeNameEnvKey, k)
-		defer os.Unsetenv(nodeNameEnvKey)
-	}
-	nodeName, err := getNodeName()
-	if err != nil {
-		t.Errorf("Failure with expected name %s, %v", k, err)
-		return
-	}
-	if nodeName != v {
-		t.Errorf("Failed to retrieve nodename, want: %s, get: %s", v, nodeName)
-	}
-}
-
 func newAgentInitializer(ovsBridgeClient ovsconfig.OVSBridgeClient, ifaceStore interfacestore.InterfaceStore) *Initializer {
 	return &Initializer{ovsBridgeClient: ovsBridgeClient, ifaceStore: ifaceStore, hostGateway: "gw0"}
 }