Add external interfaces support for custom networks, update example

Signed-off-by: Rastislav Szabo <[email protected]>

k8s/examples/custom-network/README.md

@@ -0,0 +1,85 @@
+# Custom Networks in Contiv-VPP
+
+This example showcases usage of custom networks for multi-interface pods in Contiv-VPP.
+The main use-case for this feature are CNF deployments, where it can be combined with
+[service function chaining](../sfc).
+
+For more information on multi-interface pods in Contiv-VPP, look at the 
+[custom pod interfaces documentation](../../../docs/operation/CUSTOM_POD_INTERFACES.md).
+
+## Demo Topology
+This demo deploys 2 CNF pods, each with one additional TAP interface in a layer 2 
+custom network defined as a CRD. Additionally, an external DPDK interface is defined 
+with another CRD, and is placed into the same custom network as well. This means that the 
+CNF interfaces and the external DPDK interface will be interconnected in a bridge 
+domain on VPP:
+
+![CNF - Custom Network](custom-network.png)
+
+Note that this demo refers to a single-node k8s cluster deployment. If the CNFs are deployed
+on different k8s nodes, they will be connected to different bridge domains, but still
+interconnected on L2 layer using a VXLAN tunnel between the bridge domains 
+(actually, this is not working now, but will be implemented soon).
+
+This folder contains three yaml files:
+  - [custom-network.yaml](custom-network.yaml) defines a L2 custom network with the name `l2net`
+  - [cnfs-linux.yaml](cnfs-linux.yaml) defines CNF pods with additional interfaces in the `l2net`
+  network (using the annotation `contivpp.io/custom-if: tap1/tap/l2net`)
+  - [external-interface.yaml](external-interface.yaml) defines an external DPDK sub-interface
+  in the `l2net` network.
+
+### Setup
+ Before deploying, [external interface deployment yaml](external-interface.yaml) needs to be modified 
+ to match your setup:
+
+ - change `node` identifier to match your hostname:
+ - change `vppInterfaceName` identifier to match a DPDK interface on the particular node:
+
+```yaml
+  nodes:
+    - node: k8s-master
+      vppInterfaceName: GigabitEthernet0/a/0
+```
+
+Don't forget to [modify your VPP startup config file](../../../docs/setup/VPP_CONFIG.md) 
+with the PCI address of the external interface.
+
+## Demo
+Start by deploying the yaml files:
+ ```bash
+kubectl apply -f custom-network.yaml
+kubectl apply -f cnfs-linux.yaml
+kubectl apply -f external-interface.yaml
+```
+
+Verify that `linux-cnf` pods are running:
+```bash
+$ kubectl get pods -o wide
+NAME         READY   STATUS    RESTARTS   AGE   IP         NODE      NOMINATED NODE   READINESS GATES
+linux-cnf1   1/1     Running   0          56m   10.1.1.4   lubuntu   <none>           <none>
+linux-cnf2   1/1     Running   0          56m   10.1.1.5   lubuntu   <none>           <none>
+
+```
+
+Verify that CNF taps and DPDK sub-interface are all connected into the same bridge domain on the
+vswitch VPP:
+```bash
+$ sudo vppctl
+    _______    _        _   _____  ___ 
+ __/ __/ _ \  (_)__    | | / / _ \/ _ \
+ _/ _// // / / / _ \   | |/ / ___/ ___/
+ /_/ /____(_)_/\___/   |___/_/  /_/    
+
+vpp# sh inter addr
+...
+GigabitEthernet0/a/0.200 (up):
+  L2 bridge bd-id 2 idx 2 shg 1  
+...
+tap5 (up):
+  L2 bridge bd-id 2 idx 2 shg 1  
+tap6 (up):
+  L2 bridge bd-id 2 idx 2 shg 1  
+```
+
+Try sending some L2 broadcast traffic (e.g. ARP) from one of the CNFs or external interface.
+You should see it coming to all other interfaces in the bridge domain.

k8s/examples/custom-network/external-interface.yaml

@@ -0,0 +1,12 @@
+---
+apiVersion: contivpp.io/v1
+kind: ExternalInterface
+metadata:
+  name: vlan-200
+spec:
+  type: L2
+  network: l2net
+  nodes:
+    - node: k8s-master
+      vppInterfaceName: GigabitEthernet0/a/0
+      VLAN: 200

plugins/crd/handler/externalinterface/external_interface_handler.go

@@ -90,7 +90,8 @@ func (h *Handler) PublishCrdStatus(obj interface{}, opRetval error) error {
 
 func (h *Handler) externalInterfaceToProto(externalIf *v1.ExternalInterface) *model.ExternalInterface {
 	protoVal := &model.ExternalInterface{
-		Name: externalIf.Name,
+		Name:    externalIf.Name,
+		Network: externalIf.Spec.Network,
 	}
 	switch externalIf.Spec.Type {
 	case "L2":

plugins/crd/handler/externalinterface/model/externalinterface.proto

@@ -30,12 +30,16 @@ message ExternalInterface {
     }
     Type type = 2;
 
+    // Custom network to which this interface belongs.
+    // "" or "default" means no specific custom network.
+    string network = 3;
+
     // list of physical interfaces on individual nodes
     message NodeInterface {
         string node = 1;
         string vpp_interface_name = 2;
         string ip = 3;
         uint32 vlan = 4;
     }
-    repeated NodeInterface nodes = 3;
+    repeated NodeInterface nodes = 4;
 }

plugins/crd/pkg/apis/contivppio/v1/types.go

@@ -74,9 +74,10 @@ type ExternalInterface struct {
 
 // ExternalInterfaceSpec is the spec for external interface configuration resource
 type ExternalInterfaceSpec struct {
-	Name  string          `json:"name"`
-	Type  string          `json:"type"`
-	Nodes []NodeInterface `json:"nodes"`
+	Name    string          `json:"name"`
+	Type    string          `json:"type"`
+	Network string          `json:"network"`
+	Nodes   []NodeInterface `json:"nodes"`
 }
 
 // NodeInterface describe config for an interface referenced by logical name on a node

plugins/ipnet/node.go

@@ -194,12 +194,13 @@ func (n *IPNet) partialNodeConnectivityConfig(node *nodesync.Node) (config contr
 }
 
 // externalInterfaceConfig returns configuration of an external interface of the vswitch VPP.
-func (n *IPNet) externalInterfaceConfig(extIf *extifmodel.ExternalInterface) (config controller.KeyValuePairs, err error) {
+func (n *IPNet) externalInterfaceConfig(extIf *extifmodel.ExternalInterface, eventType configEventType) (
+	config controller.KeyValuePairs, updateConfig controller.KeyValuePairs, err error) {
+
 	config = make(controller.KeyValuePairs)
+	updateConfig = make(controller.KeyValuePairs)
 	myNodeName := n.ServiceLabel.GetAgentLabel()
 
-	// TODO: interface in non-default network
-
 	for _, nodeIf := range extIf.Nodes {
 		if nodeIf.Node == myNodeName {
 			// parse IP address
@@ -212,7 +213,7 @@ func (n *IPNet) externalInterfaceConfig(extIf *extifmodel.ExternalInterface) (co
 					ip = contivconf.IPsWithNetworks{&contivconf.IPWithNetwork{Address: ipAddr, Network: ipNet}}
 				}
 			}
-
+			vppIfName := nodeIf.VppInterfaceName
 			if nodeIf.Vlan == 0 {
 				// standard interface config
 				key, iface := n.physicalInterface(nodeIf.VppInterfaceName, ip)
@@ -223,11 +224,16 @@ func (n *IPNet) externalInterfaceConfig(extIf *extifmodel.ExternalInterface) (co
 				config[key] = iface
 				key, iface = n.subInterface(nodeIf.VppInterfaceName, nodeIf.Vlan, ip)
 				config[key] = iface
+				vppIfName = iface.Name
+			}
+			if !n.isDefaultPodNetwork(extIf.Network) && !n.isStubNetwork(extIf.Network) {
+				// configure interface in custom network
+				nwConfig := n.interfaceCustomNwConfig(vppIfName, extIf.Network, eventType != configDelete)
+				mergeConfiguration(updateConfig, nwConfig)
 			}
 		}
 	}
-
-	return config, nil
+	return
 }
 
 /*********************************** DHCP *************************************/

plugins/ipnet/resync_events.go

@@ -63,9 +63,10 @@ func (n *IPNet) Resync(event controller.Event, kubeStateData controller.KubeStat
 	// external interfaces
 	for _, extIfProto := range kubeStateData[extifmodel.Keyword] {
 		extIf := extIfProto.(*extifmodel.ExternalInterface)
-		config, err := n.externalInterfaceConfig(extIf)
+		config, updateConfig, err := n.externalInterfaceConfig(extIf, configResync)
 		if err == nil {
 			controller.PutAll(txn, config)
+			controller.PutAll(txn, updateConfig)
 		} else {
 			wasErr = err
 			n.Log.Error(err)

plugins/ipnet/update_events.go

@@ -343,7 +343,7 @@ func (n *IPNet) updatePodCustomIfs(podID podmodel.ID, txn controller.UpdateOpera
 func (n *IPNet) updateExternalIf(extIf *extifmodel.ExternalInterface, txn controller.UpdateOperations,
 	eventType configEventType) (change string, err error) {
 
-	config, err := n.externalInterfaceConfig(extIf)
+	config, updateConfig, err := n.externalInterfaceConfig(extIf, eventType)
 	if err != nil {
 		return "", err
 	}
@@ -355,9 +355,11 @@ func (n *IPNet) updateExternalIf(extIf *extifmodel.ExternalInterface, txn contro
 
 	if eventType != configDelete {
 		controller.PutAll(txn, config)
+		controller.PutAll(txn, updateConfig)
 		return "configure external interfaces", nil
 	}
 	controller.DeleteAll(txn, config)
+	controller.PutAll(txn, updateConfig)
 	return "un-configure external interfaces", nil
 }