Add the toggle option

Signed-off-by: Weiqiang TANG <[email protected]>

build/yamls/antrea-eks.yml

@@ -377,6 +377,9 @@ data:
 
     # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener.
     #enablePrometheusMetrics: false
+
+    # Enable antrea proxy.
+    #enableAntreaProxy: false
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -414,7 +417,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-hmd2mdhg89
+  name: antrea-config-657g4h2t9f
   namespace: kube-system
 ---
 apiVersion: v1
@@ -519,7 +522,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-hmd2mdhg89
+          name: antrea-config-657g4h2t9f
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -733,7 +736,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-hmd2mdhg89
+          name: antrea-config-657g4h2t9f
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

build/yamls/antrea-gke.yml

@@ -377,6 +377,9 @@ data:
 
     # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener.
     #enablePrometheusMetrics: false
+
+    # Enable antrea proxy.
+    #enableAntreaProxy: false
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -414,7 +417,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-ff5ff2btgc
+  name: antrea-config-g98724bmdh
   namespace: kube-system
 ---
 apiVersion: v1
@@ -519,7 +522,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-ff5ff2btgc
+          name: antrea-config-g98724bmdh
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -731,7 +734,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-ff5ff2btgc
+          name: antrea-config-g98724bmdh
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

build/yamls/antrea-ipsec.yml

@@ -377,6 +377,9 @@ data:
 
     # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener.
     #enablePrometheusMetrics: false
+
+    # Enable antrea proxy.
+    #enableAntreaProxy: false
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -414,7 +417,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-fggkd66d2h
+  name: antrea-config-5f8fch594g
   namespace: kube-system
 ---
 apiVersion: v1
@@ -528,7 +531,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-fggkd66d2h
+          name: antrea-config-5f8fch594g
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -775,7 +778,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-fggkd66d2h
+          name: antrea-config-5f8fch594g
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

build/yamls/antrea.yml

@@ -377,6 +377,9 @@ data:
 
     # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener.
     #enablePrometheusMetrics: false
+
+    # Enable antrea proxy.
+    #enableAntreaProxy: false
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -414,7 +417,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-mf4t8c67c8
+  name: antrea-config-44th2t2fgh
   namespace: kube-system
 ---
 apiVersion: v1
@@ -519,7 +522,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-mf4t8c67c8
+          name: antrea-config-44th2t2fgh
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -731,7 +734,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-mf4t8c67c8
+          name: antrea-config-44th2t2fgh
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

build/yamls/base/conf/antrea-agent.conf

@@ -49,3 +49,6 @@
 
 # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener.
 #enablePrometheusMetrics: false
+
+# Enable antrea proxy.
+#enableAntreaProxy: false

cmd/antrea-agent/agent.go

@@ -81,7 +81,7 @@ func run(o *Options) error {
 
 	ovsBridgeClient := ovsconfig.NewOVSBridge(o.config.OVSBridge, o.config.OVSDatapathType, ovsdbConnection)
 	ovsBridgeMgmtAddr := ofconfig.GetMgmtAddress(o.config.OVSRunDir, o.config.OVSBridge)
-	ofClient := openflow.NewClient(o.config.OVSBridge, ovsBridgeMgmtAddr)
+	ofClient := openflow.NewClient(o.config.OVSBridge, ovsBridgeMgmtAddr, o.config.EnableAntreaProxy)
 
 	_, serviceCIDRNet, _ := net.ParseCIDR(o.config.ServiceCIDR)
 	_, encapMode := config.GetTrafficEncapModeFromStr(o.config.TrafficEncapMode)
@@ -106,7 +106,8 @@ func run(o *Options) error {
 		o.config.HostGateway,
 		o.config.DefaultMTU,
 		serviceCIDRNet,
-		networkConfig)
+		networkConfig,
+		o.config.EnableAntreaProxy)
 	err = agentInitializer.Initialize()
 	if err != nil {
 		return fmt.Errorf("error initializing agent: %v", err)
@@ -133,10 +134,14 @@ func run(o *Options) error {
 		isChaining = true
 	}
 
-	proxyInstance, err := proxy.New(nodeConfig.Name, informerFactory)
-	if err != nil {
-		return err
+	var proxyInstance proxy.Instance
+	if o.config.EnableAntreaProxy {
+		proxyInstance, err = proxy.New(nodeConfig.Name, informerFactory)
+		if err != nil {
+			return err
+		}
 	}
+
 	cniServer := cniserver.New(
 		o.config.CNISocket,
 		o.config.HostProcPathPrefix,
@@ -188,7 +193,9 @@ func run(o *Options) error {
 
 	go agentMonitor.Run(stopCh)
 
-	go proxyInstance.Run(agentQuerier, ofClient, stopCh)
+	if o.config.EnableAntreaProxy {
+		go proxyInstance.Run(agentQuerier, ofClient, stopCh)
+	}
 
 	apiServer, err := apiserver.New(
 		agentQuerier,

cmd/antrea-agent/config.go

@@ -83,4 +83,7 @@ type AgentConfig struct {
 	// Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener
 	// Defaults to false.
 	EnablePrometheusMetrics bool `yaml:"enablePrometheusMetrics,omitempty"`
+	// Enable antrea proxy.
+	// Defaults to false.
+	EnableAntreaProxy bool `yaml:"enableAntreaProxy,omitempty"`
 }

hack/update-codegen.sh

@@ -24,6 +24,8 @@ IMAGE_NAME="antrea/codegen:kubernetes-1.17.6"
 function docker_run() {
   docker pull ${IMAGE_NAME}
   docker run --rm \
+	  -e GOPROXY=$(go env GOPROXY) \
+		-e GOSUMDB=off \
 		-w /go/src/github.com/vmware-tanzu/antrea \
 		-v ${ANTREA_ROOT}:/go/src/github.com/vmware-tanzu/antrea \
 		"${IMAGE_NAME}" "$@"

pkg/agent/agent.go

@@ -62,6 +62,7 @@ type Initializer struct {
 	serviceCIDR     *net.IPNet // K8s Service ClusterIP CIDR
 	networkConfig   *config.NetworkConfig
 	nodeConfig      *config.NodeConfig
+	enableProxy     bool
 }
 
 func NewInitializer(
@@ -74,7 +75,8 @@ func NewInitializer(
 	hostGateway string,
 	mtu int,
 	serviceCIDR *net.IPNet,
-	networkConfig *config.NetworkConfig) *Initializer {
+	networkConfig *config.NetworkConfig,
+	enableProxy bool) *Initializer {
 	return &Initializer{
 		ovsBridgeClient: ovsBridgeClient,
 		client:          k8sClient,
@@ -86,6 +88,7 @@ func NewInitializer(
 		mtu:             mtu,
 		serviceCIDR:     serviceCIDR,
 		networkConfig:   networkConfig,
+		enableProxy:     enableProxy,
 	}
 }
 
@@ -279,12 +282,24 @@ func (i *Initializer) initOpenFlowPipeline() error {
 		}
 	}
 
-	// Setup flow entries to enable service connectivity. The agent proxy handles
-	// the clusterIP service while the upstream kube-proxy is leveraged to handle
-	// any other kinds of services.
-	if err := i.ofClient.InstallClusterServiceFlows(); err != nil {
-		klog.Errorf("Failed to setup openflow entries for Cluster Service: %v", err)
-		return err
+	if !i.enableProxy {
+		// Setup flow entries to enable service connectivity. Upstream kube-proxy is leveraged to
+		// provide load-balancing, and the flows installed by this method ensure that traffic sent
+		// from local Pods to any Service address can be forwarded to the host gateway interface
+		// correctly. Otherwise packets might be dropped by egress rules before they are DNATed to
+		// backend Pods.
+		if err := i.ofClient.InstallClusterServiceCIDRFlows(i.serviceCIDR, gateway.MAC, gatewayOFPort); err != nil {
+			klog.Errorf("Failed to setup openflow entries for Cluster Service CIDR %s: %v", i.serviceCIDR, err)
+			return err
+		}
+	} else {
+		// Setup flow entries to enable service connectivity. The agent proxy handles
+		// the clusterIP service while the upstream kube-proxy is leveraged to handle
+		// any other kinds of services.
+		if err := i.ofClient.InstallClusterServiceFlows(); err != nil {
+			klog.Errorf("Failed to setup openflow entries for Cluster Service: %v", err)
+			return err
+		}
 	}
 
 	go func() {

pkg/agent/openflow/client.go

@@ -45,6 +45,11 @@ type Client interface {
 	// host networking. These flows are only needed on windows platform.
 	InstallBridgeUplinkFlows(uplinkPort uint32, bridgeLocalPort uint32) error
 
+	// InstallClusterServiceCIDRFlows sets up the appropriate flows so that traffic can reach
+	// the different Services running in the Cluster. This method needs to be invoked once with
+	// the Cluster Service CIDR as a parameter.
+	InstallClusterServiceCIDRFlows(serviceNet *net.IPNet, gatewayMAC net.HardwareAddr, gatewayOFPort uint32) error
+
 	// InstallClusterServiceFlows sets up the appropriate flows so that traffic can reach
 	// the different Services running in the Cluster. This method needs to be invoked once with
 	// the Cluster Service CIDR as a parameter.
@@ -374,10 +379,9 @@ func (c *client) UninstallServiceFlows(svcIP net.IP, svcPort uint16, protocol bi
 }
 
 func (c *client) InstallClusterServiceFlows() error {
-	// TODO: illustrate here
 	flows := []binding.Flow{
 		c.serviceSelfAccessResponseDNATFlow(cookie.Default),
-		c.serviceCIDRDNATFlow(cookie.Default),
+		c.serviceNeedLBFlow(cookie.Default),
 		c.sessionAffinityReselectFlow(cookie.Default),
 	}
 	if err := c.ofEntryOperations.AddAll(flows); err != nil {
@@ -387,16 +391,27 @@ func (c *client) InstallClusterServiceFlows() error {
 	return nil
 }
 
+func (c *client) InstallClusterServiceCIDRFlows(serviceNet *net.IPNet, gatewayMAC net.HardwareAddr, gatewayOFPort uint32) error {
+	flow := c.serviceCIDRDNATFlow(serviceNet, gatewayMAC, gatewayOFPort, cookie.Service)
+	if err := c.ofEntryOperations.Add(flow); err != nil {
+		return err
+	}
+	c.defaultServiceFlows = []binding.Flow{flow}
+	return nil
+}
+
 func (c *client) InstallGatewayFlows(gatewayAddr net.IP, gatewayMAC net.HardwareAddr, gatewayOFPort uint32) error {
 	flows := []binding.Flow{
 		c.gatewayClassifierFlow(gatewayOFPort, cookie.Default),
 		c.gatewayIPSpoofGuardFlow(gatewayOFPort, cookie.Default),
 		c.gatewayARPSpoofGuardFlow(gatewayOFPort, gatewayAddr, gatewayMAC, cookie.Default),
 		c.ctRewriteDstMACFlow(gatewayMAC, cookie.Default),
-		c.selfAccessDNATFlow(cookie.Default),
 		c.l2ForwardCalcFlow(gatewayMAC, gatewayOFPort, cookie.Default),
 		c.localProbeFlow(gatewayAddr, cookie.Default),
 	}
+	if c.enableProxy {
+		flows = append(flows, c.selfAccessDNATFlow(cookie.Default))
+	}
 
 	// In NoEncap , no traffic from tunnel port
 	if c.encapMode.SupportsEncap() {
@@ -449,8 +464,10 @@ func (c *client) initialize() error {
 	if err := c.ofEntryOperations.AddAll(c.establishedConnectionFlows(cookie.Default)); err != nil {
 		return fmt.Errorf("failed to install flows to skip established connections: %v", err)
 	}
-	if err := c.ofEntryOperations.Add(c.l2ForwardOutputServiceSelfAccessFlow()); err != nil {
-		return fmt.Errorf("failed to install L2 forward flow for service self-access: %v", err)
+	if c.enableProxy {
+		if err := c.ofEntryOperations.Add(c.l2ForwardOutputServiceSelfAccessFlow()); err != nil {
+			return fmt.Errorf("failed to install L2 forward flow for service self-access: %v", err)
+		}
 	}
 
 	if c.encapMode.SupportsNoEncap() {

pkg/agent/openflow/client_test.go

@@ -88,7 +88,7 @@ func TestIdempotentFlowInstallation(t *testing.T) {
 			ctrl := gomock.NewController(t)
 			defer ctrl.Finish()
 			m := oftest.NewMockOFEntryOperations(ctrl)
-			ofClient := NewClient(bridgeName, bridgeMgmtAddr)
+			ofClient := NewClient(bridgeName, bridgeMgmtAddr, true)
 			client := ofClient.(*client)
 			client.cookieAllocator = cookie.NewAllocator(0)
 			client.nodeConfig = &config.NodeConfig{}
@@ -116,7 +116,7 @@ func TestIdempotentFlowInstallation(t *testing.T) {
 			ctrl := gomock.NewController(t)
 			defer ctrl.Finish()
 			m := oftest.NewMockOFEntryOperations(ctrl)
-			ofClient := NewClient(bridgeName, bridgeMgmtAddr)
+			ofClient := NewClient(bridgeName, bridgeMgmtAddr, true)
 			client := ofClient.(*client)
 			client.cookieAllocator = cookie.NewAllocator(0)
 			client.nodeConfig = &config.NodeConfig{}
@@ -157,7 +157,7 @@ func TestFlowInstallationFailed(t *testing.T) {
 			ctrl := gomock.NewController(t)
 			defer ctrl.Finish()
 			m := oftest.NewMockOFEntryOperations(ctrl)
-			ofClient := NewClient(bridgeName, bridgeMgmtAddr)
+			ofClient := NewClient(bridgeName, bridgeMgmtAddr, true)
 			client := ofClient.(*client)
 			client.cookieAllocator = cookie.NewAllocator(0)
 			client.nodeConfig = &config.NodeConfig{}
@@ -191,7 +191,7 @@ func TestConcurrentFlowInstallation(t *testing.T) {
 			ctrl := gomock.NewController(t)
 			defer ctrl.Finish()
 			m := oftest.NewMockOFEntryOperations(ctrl)
-			ofClient := NewClient(bridgeName, bridgeMgmtAddr)
+			ofClient := NewClient(bridgeName, bridgeMgmtAddr, true)
 			client := ofClient.(*client)
 			client.cookieAllocator = cookie.NewAllocator(0)
 			client.nodeConfig = &config.NodeConfig{}