diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5b3d11a7a4..777e73a0e0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -483,6 +483,7 @@ jobs: - {"target": "network-segmentation", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv6", "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones"} - {"target": "bgp", "ha": "noHA", "gateway-mode": "local", "ipfamily": "dualstack", "disable-snat-multiple-gws": "snatGW", "second-bridge": "1br", "ic": "ic-single-node-zones", "routeadvertisements": "advertise-default", "network-segmentation": "enable-network-segmentation", "dns-name-resolver": "enable-dns-name-resolver"} - {"target": "bgp", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "dualstack", "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones", "routeadvertisements": "advertise-default", "network-segmentation": "enable-network-segmentation", "dns-name-resolver": "enable-dns-name-resolver"} + - {"target": "bgp", "ha": "noHA", "gateway-mode": "local", "ipfamily": "ipv6", "disable-snat-multiple-gws": "snatGW", "second-bridge": "1br", "ic": "ic-single-node-zones", "routeadvertisements": "advertise-default", "network-segmentation": "enable-network-segmentation"} - {"target": "bgp-loose-isolation", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "dualstack", "disable-snat-multiple-gws": "snatGW", "second-bridge": "1br", "ic": "ic-single-node-zones", "routeadvertisements": "advertise-default", "network-segmentation": "enable-network-segmentation", "advertised-udn-isolation-mode": "loose"} - {"target": "traffic-flow-test-only","ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv4", "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones", "traffic-flow-tests": "1-24", "network-segmentation": "enable-network-segmentation"} - {"target": "tools", "ha": "noHA", "gateway-mode": "local", "ipfamily": "dualstack", "disable-snat-multiple-gws": "SnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones", "network-segmentation": "enable-network-segmentation"} diff --git a/go-controller/pkg/node/bridgeconfig/bridgeconfig_testutil.go b/go-controller/pkg/node/bridgeconfig/bridgeconfig_testutil.go index 8395baf06d..d13568a529 100644 --- a/go-controller/pkg/node/bridgeconfig/bridgeconfig_testutil.go +++ b/go-controller/pkg/node/bridgeconfig/bridgeconfig_testutil.go @@ -47,10 +47,10 @@ func CheckUDNSvcIsolationOVSFlows(flows []string, netConfig *BridgeUDNConfigurat var protoPrefix string if net2.IsIPv4CIDR(svcCIDR) { mgmtMasqIP = netConfig.V4MasqIPs.ManagementPort.IP.String() - protoPrefix = "ip" + protoPrefix = protoPrefixV4 } else { mgmtMasqIP = netConfig.V6MasqIPs.ManagementPort.IP.String() - protoPrefix = "ip6" + protoPrefix = protoPrefixV6 } var nFlows int @@ -78,11 +78,11 @@ func CheckAdvertisedUDNSvcIsolationOVSFlows(flows []string, netConfig *BridgeUDN if net2.IsIPv4CIDR(svcCIDR) { matchingIPFamilySubnet, err = util.MatchFirstIPNetFamily(false, udnAdvertisedSubnets) Expect(err).ToNot(HaveOccurred()) - protoPrefix = "ip" + protoPrefix = protoPrefixV4 } else { matchingIPFamilySubnet, err = util.MatchFirstIPNetFamily(true, udnAdvertisedSubnets) Expect(err).ToNot(HaveOccurred()) - protoPrefix = "ip6" + protoPrefix = protoPrefixV6 } var nFlows int @@ -107,11 +107,11 @@ func CheckDefaultSvcIsolationOVSFlows(flows []string, defaultConfig *BridgeUDNCo var masqSubnet string var protoPrefix string if net2.IsIPv4CIDR(svcCIDR) { - protoPrefix = "ip" + protoPrefix = protoPrefixV4 masqIP = config.Gateway.MasqueradeIPs.V4HostMasqueradeIP.String() masqSubnet = config.Gateway.V4MasqueradeSubnet } else { - protoPrefix = "ip6" + protoPrefix = protoPrefixV6 masqIP = config.Gateway.MasqueradeIPs.V6HostMasqueradeIP.String() masqSubnet = config.Gateway.V6MasqueradeSubnet } diff --git a/go-controller/pkg/node/bridgeconfig/bridgeflows.go b/go-controller/pkg/node/bridgeconfig/bridgeflows.go index 4d2ee1240f..78cf8a2c42 100644 --- a/go-controller/pkg/node/bridgeconfig/bridgeflows.go +++ b/go-controller/pkg/node/bridgeconfig/bridgeflows.go @@ -14,6 +14,11 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" ) +const ( + protoPrefixV4 = "ip" + protoPrefixV6 = "ipv6" +) + func (b *BridgeConfiguration) DefaultBridgeFlows(hostSubnets []*net.IPNet, extraIPs []net.IP) ([]string, error) { b.mutex.Lock() defer b.mutex.Unlock() @@ -95,9 +100,10 @@ func (b *BridgeConfiguration) flowsForDefaultBridge(extraIPs []net.IP) ([]string for _, netConfig := range b.patchedNetConfigs() { // table 0, SVC Hairpin from OVN destined to local host, DNAT and go to table 4 dftFlows = append(dftFlows, - fmt.Sprintf("cookie=%s, priority=500, in_port=%s, ip, ip_dst=%s, ip_src=%s,"+ + fmt.Sprintf("cookie=%s, priority=500, in_port=%s, %s, %s_dst=%s, %s_src=%s,"+ "actions=ct(commit,zone=%d,nat(dst=%s),table=4)", - nodetypes.DefaultOpenFlowCookie, netConfig.OfPortPatch, config.Gateway.MasqueradeIPs.V4HostMasqueradeIP.String(), physicalIP.IP, + nodetypes.DefaultOpenFlowCookie, netConfig.OfPortPatch, protoPrefixV4, protoPrefixV4, + config.Gateway.MasqueradeIPs.V4HostMasqueradeIP.String(), protoPrefixV4, physicalIP.IP, config.Default.HostMasqConntrackZone, physicalIP.IP)) } @@ -118,18 +124,20 @@ func (b *BridgeConfiguration) flowsForDefaultBridge(extraIPs []net.IP) ([]string for _, netConfig := range b.patchedNetConfigs() { dftFlows = append(dftFlows, - fmt.Sprintf("cookie=%s, priority=500, in_port=%s, ip, ip_dst=%s, ip_src=%s,"+ + fmt.Sprintf("cookie=%s, priority=500, in_port=%s, %s, %s_dst=%s, %s_src=%s,"+ "actions=ct(commit,zone=%d,table=4)", - nodetypes.DefaultOpenFlowCookie, netConfig.OfPortPatch, ip.String(), physicalIP.IP, + nodetypes.DefaultOpenFlowCookie, netConfig.OfPortPatch, protoPrefixV4, + protoPrefixV4, ip.String(), protoPrefixV4, physicalIP.IP, config.Default.HostMasqConntrackZone)) } } // table 0, Reply SVC traffic from Host -> OVN, unSNAT and goto table 5 dftFlows = append(dftFlows, - fmt.Sprintf("cookie=%s, priority=500, in_port=%s, ip, ip_dst=%s,"+ + fmt.Sprintf("cookie=%s, priority=500, in_port=%s, %s, %s_dst=%s,"+ "actions=ct(zone=%d,nat,table=5)", - nodetypes.DefaultOpenFlowCookie, ofPortHost, config.Gateway.MasqueradeIPs.V4OVNMasqueradeIP.String(), config.Default.OVNMasqConntrackZone)) + nodetypes.DefaultOpenFlowCookie, ofPortHost, protoPrefixV4, protoPrefixV4, + config.Gateway.MasqueradeIPs.V4OVNMasqueradeIP.String(), config.Default.OVNMasqConntrackZone)) } if config.IPv6Mode { if ofPortPhys != "" { @@ -157,9 +165,10 @@ func (b *BridgeConfiguration) flowsForDefaultBridge(extraIPs []net.IP) ([]string // table 0, SVC Hairpin from OVN destined to local host, DNAT to host, send to table 4 for _, netConfig := range b.patchedNetConfigs() { dftFlows = append(dftFlows, - fmt.Sprintf("cookie=%s, priority=500, in_port=%s, ipv6, ipv6_dst=%s, ipv6_src=%s,"+ + fmt.Sprintf("cookie=%s, priority=500, in_port=%s, %s, %s_dst=%s, %s_src=%s,"+ "actions=ct(commit,zone=%d,nat(dst=%s),table=4)", - nodetypes.DefaultOpenFlowCookie, netConfig.OfPortPatch, config.Gateway.MasqueradeIPs.V6HostMasqueradeIP.String(), physicalIP.IP, + nodetypes.DefaultOpenFlowCookie, netConfig.OfPortPatch, protoPrefixV6, protoPrefixV6, + config.Gateway.MasqueradeIPs.V6HostMasqueradeIP.String(), protoPrefixV6, physicalIP.IP, config.Default.HostMasqConntrackZone, physicalIP.IP)) } @@ -180,18 +189,20 @@ func (b *BridgeConfiguration) flowsForDefaultBridge(extraIPs []net.IP) ([]string for _, netConfig := range b.patchedNetConfigs() { dftFlows = append(dftFlows, - fmt.Sprintf("cookie=%s, priority=500, in_port=%s, ipv6, ipv6_dst=%s, ipv6_src=%s,"+ + fmt.Sprintf("cookie=%s, priority=500, in_port=%s, %s, %s_dst=%s, %s_src=%s,"+ "actions=ct(commit,zone=%d,table=4)", - nodetypes.DefaultOpenFlowCookie, netConfig.OfPortPatch, ip.String(), physicalIP.IP, + nodetypes.DefaultOpenFlowCookie, netConfig.OfPortPatch, protoPrefixV6, protoPrefixV6, + ip.String(), protoPrefixV6, physicalIP.IP, config.Default.HostMasqConntrackZone)) } } // table 0, Reply SVC traffic from Host -> OVN, unSNAT and goto table 5 dftFlows = append(dftFlows, - fmt.Sprintf("cookie=%s, priority=500, in_port=%s, ipv6, ipv6_dst=%s,"+ + fmt.Sprintf("cookie=%s, priority=500, in_port=%s, %s, %s_dst=%s,"+ "actions=ct(zone=%d,nat,table=5)", - nodetypes.DefaultOpenFlowCookie, ofPortHost, config.Gateway.MasqueradeIPs.V6OVNMasqueradeIP.String(), config.Default.OVNMasqConntrackZone)) + nodetypes.DefaultOpenFlowCookie, ofPortHost, protoPrefixV6, protoPrefixV6, + config.Gateway.MasqueradeIPs.V6OVNMasqueradeIP.String(), config.Default.OVNMasqConntrackZone)) } var protoPrefix, masqIP, masqSubnet string @@ -199,11 +210,11 @@ func (b *BridgeConfiguration) flowsForDefaultBridge(extraIPs []net.IP) ([]string // table 0, packets coming from Host -> Service for _, svcCIDR := range config.Kubernetes.ServiceCIDRs { if utilnet.IsIPv4CIDR(svcCIDR) { - protoPrefix = "ip" + protoPrefix = protoPrefixV4 masqIP = config.Gateway.MasqueradeIPs.V4HostMasqueradeIP.String() masqSubnet = config.Gateway.V4MasqueradeSubnet } else { - protoPrefix = "ipv6" + protoPrefix = protoPrefixV6 masqIP = config.Gateway.MasqueradeIPs.V6HostMasqueradeIP.String() masqSubnet = config.Gateway.V6MasqueradeSubnet } @@ -300,50 +311,50 @@ func (b *BridgeConfiguration) flowsForDefaultBridge(extraIPs []net.IP) ([]string if config.IPv4Mode { // table 1, established and related connections in zone 64000 with ct_mark CtMarkOVN go to OVN dftFlows = append(dftFlows, - fmt.Sprintf("cookie=%s, priority=100, table=1, ip, ct_state=+trk+est, ct_mark=%s, "+ - "actions=%s", nodetypes.DefaultOpenFlowCookie, netConfig.MasqCTMark, actions)) + fmt.Sprintf("cookie=%s, priority=100, table=1, %s, ct_state=+trk+est, ct_mark=%s, "+ + "actions=%s", nodetypes.DefaultOpenFlowCookie, protoPrefixV4, netConfig.MasqCTMark, actions)) dftFlows = append(dftFlows, - fmt.Sprintf("cookie=%s, priority=100, table=1, ip, ct_state=+trk+rel, ct_mark=%s, "+ - "actions=%s", nodetypes.DefaultOpenFlowCookie, netConfig.MasqCTMark, actions)) + fmt.Sprintf("cookie=%s, priority=100, table=1, %s, ct_state=+trk+rel, ct_mark=%s, "+ + "actions=%s", nodetypes.DefaultOpenFlowCookie, protoPrefixV4, netConfig.MasqCTMark, actions)) } if config.IPv6Mode { // table 1, established and related connections in zone 64000 with ct_mark CtMarkOVN go to OVN dftFlows = append(dftFlows, - fmt.Sprintf("cookie=%s, priority=100, table=1, ipv6, ct_state=+trk+est, ct_mark=%s, "+ - "actions=%s", nodetypes.DefaultOpenFlowCookie, netConfig.MasqCTMark, actions)) + fmt.Sprintf("cookie=%s, priority=100, table=1, %s, ct_state=+trk+est, ct_mark=%s, "+ + "actions=%s", nodetypes.DefaultOpenFlowCookie, protoPrefixV6, netConfig.MasqCTMark, actions)) dftFlows = append(dftFlows, - fmt.Sprintf("cookie=%s, priority=100, table=1, ipv6, ct_state=+trk+rel, ct_mark=%s, "+ - "actions=%s", nodetypes.DefaultOpenFlowCookie, netConfig.MasqCTMark, actions)) + fmt.Sprintf("cookie=%s, priority=100, table=1, %s, ct_state=+trk+rel, ct_mark=%s, "+ + "actions=%s", nodetypes.DefaultOpenFlowCookie, protoPrefixV6, netConfig.MasqCTMark, actions)) } } if config.IPv4Mode { // table 1, established and related connections in zone 64000 with ct_mark CtMarkHost go to host dftFlows = append(dftFlows, - fmt.Sprintf("cookie=%s, priority=100, table=1, %s ip, ct_state=+trk+est, ct_mark=%s, "+ + fmt.Sprintf("cookie=%s, priority=100, table=1, %s %s, ct_state=+trk+est, ct_mark=%s, "+ "actions=%soutput:%s", - nodetypes.DefaultOpenFlowCookie, match_vlan, nodetypes.CtMarkHost, strip_vlan, ofPortHost)) + nodetypes.DefaultOpenFlowCookie, match_vlan, protoPrefixV4, nodetypes.CtMarkHost, strip_vlan, ofPortHost)) dftFlows = append(dftFlows, - fmt.Sprintf("cookie=%s, priority=100, table=1, %s ip, ct_state=+trk+rel, ct_mark=%s, "+ + fmt.Sprintf("cookie=%s, priority=100, table=1, %s %s, ct_state=+trk+rel, ct_mark=%s, "+ "actions=%soutput:%s", - nodetypes.DefaultOpenFlowCookie, match_vlan, nodetypes.CtMarkHost, strip_vlan, ofPortHost)) + nodetypes.DefaultOpenFlowCookie, match_vlan, protoPrefixV4, nodetypes.CtMarkHost, strip_vlan, ofPortHost)) } if config.IPv6Mode { // table 1, established and related connections in zone 64000 with ct_mark CtMarkHost go to host dftFlows = append(dftFlows, - fmt.Sprintf("cookie=%s, priority=100, table=1, %s ip6, ct_state=+trk+est, ct_mark=%s, "+ + fmt.Sprintf("cookie=%s, priority=100, table=1, %s %s, ct_state=+trk+est, ct_mark=%s, "+ "actions=%soutput:%s", - nodetypes.DefaultOpenFlowCookie, match_vlan, nodetypes.CtMarkHost, strip_vlan, ofPortHost)) + nodetypes.DefaultOpenFlowCookie, match_vlan, protoPrefixV6, nodetypes.CtMarkHost, strip_vlan, ofPortHost)) dftFlows = append(dftFlows, - fmt.Sprintf("cookie=%s, priority=100, table=1, %s ip6, ct_state=+trk+rel, ct_mark=%s, "+ + fmt.Sprintf("cookie=%s, priority=100, table=1, %s %s, ct_state=+trk+rel, ct_mark=%s, "+ "actions=%soutput:%s", - nodetypes.DefaultOpenFlowCookie, match_vlan, nodetypes.CtMarkHost, strip_vlan, ofPortHost)) + nodetypes.DefaultOpenFlowCookie, match_vlan, protoPrefixV6, nodetypes.CtMarkHost, strip_vlan, ofPortHost)) } @@ -385,22 +396,23 @@ func (b *BridgeConfiguration) flowsForDefaultBridge(extraIPs []net.IP) ([]string // the correct patch port of it's own network where it's a deadend if the clusterIP is not part of // that UDN network and works if it is part of the UDN network. dftFlows = append(dftFlows, - fmt.Sprintf("cookie=%s, priority=200, table=2, ip, ip_src=%s, "+ + fmt.Sprintf("cookie=%s, priority=200, table=2, %s, %s_src=%s, "+ "actions=drop", - nodetypes.DefaultOpenFlowCookie, matchingIPFamilySubnet.String())) + nodetypes.DefaultOpenFlowCookie, protoPrefixV4, protoPrefixV4, matchingIPFamilySubnet.String())) } // Drop traffic coming from the masquerade IP or the UDN subnet(for advertised UDNs) to ensure that // isolation between networks is enforced. This handles the case where a pod on the UDN subnet is sending traffic to // a service in another UDN. dftFlows = append(dftFlows, - fmt.Sprintf("cookie=%s, priority=200, table=2, ip, ip_src=%s, "+ + fmt.Sprintf("cookie=%s, priority=200, table=2, %s, %s_src=%s, "+ "actions=drop", - nodetypes.DefaultOpenFlowCookie, netConfig.V4MasqIPs.ManagementPort.IP.String())) + nodetypes.DefaultOpenFlowCookie, protoPrefixV4, protoPrefixV4, + netConfig.V4MasqIPs.ManagementPort.IP.String())) dftFlows = append(dftFlows, - fmt.Sprintf("cookie=%s, priority=250, table=2, ip, pkt_mark=%s, "+ + fmt.Sprintf("cookie=%s, priority=250, table=2, %s, pkt_mark=%s, "+ "actions=set_field:%s->eth_dst,%soutput:%s", - nodetypes.DefaultOpenFlowCookie, netConfig.PktMark, + nodetypes.DefaultOpenFlowCookie, protoPrefixV4, netConfig.PktMark, bridgeMacAddress, mod_vlan_id, netConfig.OfPortPatch)) } } @@ -424,18 +436,20 @@ func (b *BridgeConfiguration) flowsForDefaultBridge(extraIPs []net.IP) ([]string } dftFlows = append(dftFlows, - fmt.Sprintf("cookie=%s, priority=200, table=2, ip6, ipv6_src=%s, "+ + fmt.Sprintf("cookie=%s, priority=200, table=2, %s, %s_src=%s, "+ "actions=drop", - nodetypes.DefaultOpenFlowCookie, matchingIPFamilySubnet.String())) + nodetypes.DefaultOpenFlowCookie, protoPrefixV6, protoPrefixV6, + matchingIPFamilySubnet.String())) } dftFlows = append(dftFlows, - fmt.Sprintf("cookie=%s, priority=200, table=2, ip6, ipv6_src=%s, "+ + fmt.Sprintf("cookie=%s, priority=200, table=2, %s, %s_src=%s, "+ "actions=drop", - nodetypes.DefaultOpenFlowCookie, netConfig.V6MasqIPs.ManagementPort.IP.String())) + nodetypes.DefaultOpenFlowCookie, protoPrefixV6, protoPrefixV6, + netConfig.V6MasqIPs.ManagementPort.IP.String())) dftFlows = append(dftFlows, - fmt.Sprintf("cookie=%s, priority=250, table=2, ip6, pkt_mark=%s, "+ + fmt.Sprintf("cookie=%s, priority=250, table=2, %s, pkt_mark=%s, "+ "actions=set_field:%s->eth_dst,%soutput:%s", - nodetypes.DefaultOpenFlowCookie, netConfig.PktMark, + nodetypes.DefaultOpenFlowCookie, protoPrefixV6, netConfig.PktMark, bridgeMacAddress, mod_vlan_id, netConfig.OfPortPatch)) } } @@ -450,28 +464,28 @@ func (b *BridgeConfiguration) flowsForDefaultBridge(extraIPs []net.IP) ([]string // We need to SNAT and masquerade OVN GR IP, send to table 3 for dispatch to Host if config.IPv4Mode { dftFlows = append(dftFlows, - fmt.Sprintf("cookie=%s, table=4,ip,"+ + fmt.Sprintf("cookie=%s, table=4,%s,"+ "actions=ct(commit,zone=%d,nat(src=%s),table=3)", - nodetypes.DefaultOpenFlowCookie, config.Default.OVNMasqConntrackZone, config.Gateway.MasqueradeIPs.V4OVNMasqueradeIP.String())) + nodetypes.DefaultOpenFlowCookie, protoPrefixV4, config.Default.OVNMasqConntrackZone, config.Gateway.MasqueradeIPs.V4OVNMasqueradeIP.String())) } if config.IPv6Mode { dftFlows = append(dftFlows, - fmt.Sprintf("cookie=%s, table=4,ipv6, "+ + fmt.Sprintf("cookie=%s, table=4,%s, "+ "actions=ct(commit,zone=%d,nat(src=%s),table=3)", - nodetypes.DefaultOpenFlowCookie, config.Default.OVNMasqConntrackZone, config.Gateway.MasqueradeIPs.V6OVNMasqueradeIP.String())) + nodetypes.DefaultOpenFlowCookie, protoPrefixV6, config.Default.OVNMasqConntrackZone, config.Gateway.MasqueradeIPs.V6OVNMasqueradeIP.String())) } // table 5, Host Reply traffic to hairpinned svc, need to unDNAT, send to table 2 if config.IPv4Mode { dftFlows = append(dftFlows, - fmt.Sprintf("cookie=%s, table=5, ip, "+ + fmt.Sprintf("cookie=%s, table=5, %s, "+ "actions=ct(commit,zone=%d,nat,table=2)", - nodetypes.DefaultOpenFlowCookie, config.Default.HostMasqConntrackZone)) + nodetypes.DefaultOpenFlowCookie, protoPrefixV4, config.Default.HostMasqConntrackZone)) } if config.IPv6Mode { dftFlows = append(dftFlows, - fmt.Sprintf("cookie=%s, table=5, ipv6, "+ + fmt.Sprintf("cookie=%s, table=5, %s, "+ "actions=ct(commit,zone=%d,nat,table=2)", - nodetypes.DefaultOpenFlowCookie, config.Default.HostMasqConntrackZone)) + nodetypes.DefaultOpenFlowCookie, protoPrefixV6, config.Default.HostMasqConntrackZone)) } return dftFlows, nil } @@ -491,18 +505,20 @@ func generateIPFragmentReassemblyFlow(ofPortPhys string) []string { flows := make([]string, 0, 2) if config.IPv4Mode { flows = append(flows, - fmt.Sprintf("cookie=%s, priority=110, table=0, in_port=%s, ip, nw_frag=yes, actions=ct(table=0,zone=%d)", + fmt.Sprintf("cookie=%s, priority=110, table=0, in_port=%s, %s, nw_frag=yes, actions=ct(table=0,zone=%d)", nodetypes.DefaultOpenFlowCookie, ofPortPhys, + protoPrefixV4, config.Default.ReassemblyConntrackZone, ), ) } if config.IPv6Mode { flows = append(flows, - fmt.Sprintf("cookie=%s, priority=110, table=0, in_port=%s, ipv6, nw_frag=yes, actions=ct(table=0,zone=%d)", + fmt.Sprintf("cookie=%s, priority=110, table=0, in_port=%s, %s, nw_frag=yes, actions=ct(table=0,zone=%d)", nodetypes.DefaultOpenFlowCookie, ofPortPhys, + protoPrefixV6, config.Default.ReassemblyConntrackZone, ), ) @@ -588,10 +604,10 @@ func (b *BridgeConfiguration) commonFlows(hostSubnets []*net.IPNet) ([]string, e // egressService pods will also undergo this SNAT to nodeIP since these features are tied // together at the OVN policy level on the distributed router. dftFlows = append(dftFlows, - fmt.Sprintf("cookie=%s, priority=105, in_port=%s, dl_src=%s, ip, pkt_mark=%s "+ + fmt.Sprintf("cookie=%s, priority=105, in_port=%s, dl_src=%s, %s, pkt_mark=%s "+ "actions=ct(commit, zone=%d, nat(src=%s), exec(set_field:%s->ct_mark)),output:%s", - nodetypes.DefaultOpenFlowCookie, netConfig.OfPortPatch, bridgeMacAddress, nodetypes.OvnKubeNodeSNATMark, - config.Default.ConntrackZone, physicalIP.IP, netConfig.MasqCTMark, ofPortPhys)) + nodetypes.DefaultOpenFlowCookie, netConfig.OfPortPatch, bridgeMacAddress, protoPrefixV4, + nodetypes.OvnKubeNodeSNATMark, config.Default.ConntrackZone, physicalIP.IP, netConfig.MasqCTMark, ofPortPhys)) // table 0, packets coming from egressIP pods only from user defined networks. If an egressIP is assigned to // this node, then all networks get a flow even if no pods on that network were selected for by this egressIP. @@ -600,9 +616,9 @@ func (b *BridgeConfiguration) commonFlows(hostSubnets []*net.IPNet) ([]string, e if netConfig.MasqCTMark != nodetypes.CtMarkOVN { for mark, eip := range b.eipMarkIPs.GetIPv4() { dftFlows = append(dftFlows, - fmt.Sprintf("cookie=%s, priority=105, in_port=%s, dl_src=%s, ip, pkt_mark=%d, "+ + fmt.Sprintf("cookie=%s, priority=105, in_port=%s, dl_src=%s, %s, pkt_mark=%d, "+ "actions=ct(commit, zone=%d, nat(src=%s), exec(set_field:%s->ct_mark)), output:%s", - nodetypes.DefaultOpenFlowCookie, netConfig.OfPortPatch, bridgeMacAddress, mark, + nodetypes.DefaultOpenFlowCookie, netConfig.OfPortPatch, bridgeMacAddress, protoPrefixV4, mark, config.Default.ConntrackZone, eip, netConfig.MasqCTMark, ofPortPhys)) } } @@ -612,10 +628,10 @@ func (b *BridgeConfiguration) commonFlows(hostSubnets []*net.IPNet) ([]string, e // so that reverse direction goes back to the pods. if netConfig.IsDefaultNetwork() { dftFlows = append(dftFlows, - fmt.Sprintf("cookie=%s, priority=100, in_port=%s, dl_src=%s, ip, "+ + fmt.Sprintf("cookie=%s, priority=100, in_port=%s, dl_src=%s, %s, "+ "actions=ct(commit, zone=%d, exec(set_field:%s->ct_mark)), output:%s", - nodetypes.DefaultOpenFlowCookie, netConfig.OfPortPatch, bridgeMacAddress, config.Default.ConntrackZone, - netConfig.MasqCTMark, ofPortPhys)) + nodetypes.DefaultOpenFlowCookie, netConfig.OfPortPatch, bridgeMacAddress, protoPrefixV4, + config.Default.ConntrackZone, netConfig.MasqCTMark, ofPortPhys)) // Allow (a) OVN->host traffic on the same node // (b) host->host traffic on the same node @@ -625,9 +641,10 @@ func (b *BridgeConfiguration) commonFlows(hostSubnets []*net.IPNet) ([]string, e } else { // for UDN we additionally SNAT the packet from masquerade IP -> node IP dftFlows = append(dftFlows, - fmt.Sprintf("cookie=%s, priority=100, in_port=%s, dl_src=%s, ip, ip_src=%s, "+ + fmt.Sprintf("cookie=%s, priority=100, in_port=%s, dl_src=%s, %s, %s_src=%s, "+ "actions=ct(commit, zone=%d, nat(src=%s), exec(set_field:%s->ct_mark)), output:%s", - nodetypes.DefaultOpenFlowCookie, netConfig.OfPortPatch, bridgeMacAddress, netConfig.V4MasqIPs.GatewayRouter.IP, config.Default.ConntrackZone, + nodetypes.DefaultOpenFlowCookie, netConfig.OfPortPatch, bridgeMacAddress, protoPrefixV4, protoPrefixV4, + netConfig.V4MasqIPs.GatewayRouter.IP, config.Default.ConntrackZone, physicalIP.IP, netConfig.MasqCTMark, ofPortPhys)) } } @@ -635,9 +652,10 @@ func (b *BridgeConfiguration) commonFlows(hostSubnets []*net.IPNet) ([]string, e // table 0, packets coming from host Commit connections with ct_mark CtMarkHost // so that reverse direction goes back to the host. dftFlows = append(dftFlows, - fmt.Sprintf("cookie=%s, priority=100, in_port=%s, ip, "+ + fmt.Sprintf("cookie=%s, priority=100, in_port=%s, %s, "+ "actions=ct(commit, zone=%d, exec(set_field:%s->ct_mark)), %soutput:%s", - nodetypes.DefaultOpenFlowCookie, ofPortHost, config.Default.ConntrackZone, nodetypes.CtMarkHost, mod_vlan_id, ofPortPhys)) + nodetypes.DefaultOpenFlowCookie, ofPortHost, protoPrefixV4, config.Default.ConntrackZone, + nodetypes.CtMarkHost, mod_vlan_id, ofPortPhys)) } if config.Gateway.Mode == config.GatewayModeLocal { for _, netConfig := range b.patchedNetConfigs() { @@ -669,8 +687,8 @@ func (b *BridgeConfiguration) commonFlows(hostSubnets []*net.IPNet) ([]string, e // Send it through conntrack and resubmit to table 1 to know the state and mark of the connection. // Note, there are higher priority rules that take care of traffic coming from LOCAL and OVN ports. dftFlows = append(dftFlows, - fmt.Sprintf("cookie=%s, priority=50, ip, dl_dst=%s, actions=ct(zone=%d, nat, table=1)", - nodetypes.DefaultOpenFlowCookie, bridgeMacAddress, config.Default.ConntrackZone)) + fmt.Sprintf("cookie=%s, priority=50, %s, dl_dst=%s, actions=ct(zone=%d, nat, table=1)", + nodetypes.DefaultOpenFlowCookie, protoPrefixV4, bridgeMacAddress, config.Default.ConntrackZone)) } } @@ -687,9 +705,9 @@ func (b *BridgeConfiguration) commonFlows(hostSubnets []*net.IPNet) ([]string, e // egressService pods will also undergo this SNAT to nodeIP since these features are tied // together at the OVN policy level on the distributed router. dftFlows = append(dftFlows, - fmt.Sprintf("cookie=%s, priority=105, in_port=%s, dl_src=%s, ipv6, pkt_mark=%s "+ + fmt.Sprintf("cookie=%s, priority=105, in_port=%s, dl_src=%s, %s, pkt_mark=%s "+ "actions=ct(commit, zone=%d, nat(src=%s), exec(set_field:%s->ct_mark)),output:%s", - nodetypes.DefaultOpenFlowCookie, netConfig.OfPortPatch, bridgeMacAddress, nodetypes.OvnKubeNodeSNATMark, + nodetypes.DefaultOpenFlowCookie, netConfig.OfPortPatch, bridgeMacAddress, protoPrefixV6, nodetypes.OvnKubeNodeSNATMark, config.Default.ConntrackZone, physicalIP.IP, netConfig.MasqCTMark, ofPortPhys)) // table 0, packets coming from egressIP pods only from user defined networks. If an egressIP is assigned to @@ -699,9 +717,9 @@ func (b *BridgeConfiguration) commonFlows(hostSubnets []*net.IPNet) ([]string, e if netConfig.MasqCTMark != nodetypes.CtMarkOVN { for mark, eip := range b.eipMarkIPs.GetIPv6() { dftFlows = append(dftFlows, - fmt.Sprintf("cookie=%s, priority=105, in_port=%s, dl_src=%s, ipv6, pkt_mark=%d, "+ + fmt.Sprintf("cookie=%s, priority=105, in_port=%s, dl_src=%s, %s, pkt_mark=%d, "+ "actions=ct(commit, zone=%d, nat(src=%s), exec(set_field:%s->ct_mark)), output:%s", - nodetypes.DefaultOpenFlowCookie, netConfig.OfPortPatch, bridgeMacAddress, mark, + nodetypes.DefaultOpenFlowCookie, netConfig.OfPortPatch, bridgeMacAddress, protoPrefixV6, mark, config.Default.ConntrackZone, eip, netConfig.MasqCTMark, ofPortPhys)) } } @@ -711,9 +729,10 @@ func (b *BridgeConfiguration) commonFlows(hostSubnets []*net.IPNet) ([]string, e // so that reverse direction goes back to the pods. if netConfig.IsDefaultNetwork() { dftFlows = append(dftFlows, - fmt.Sprintf("cookie=%s, priority=100, in_port=%s, dl_src=%s, ipv6, "+ + fmt.Sprintf("cookie=%s, priority=100, in_port=%s, dl_src=%s, %s, "+ "actions=ct(commit, zone=%d, exec(set_field:%s->ct_mark)), output:%s", - nodetypes.DefaultOpenFlowCookie, netConfig.OfPortPatch, bridgeMacAddress, config.Default.ConntrackZone, netConfig.MasqCTMark, ofPortPhys)) + nodetypes.DefaultOpenFlowCookie, netConfig.OfPortPatch, bridgeMacAddress, protoPrefixV6, + config.Default.ConntrackZone, netConfig.MasqCTMark, ofPortPhys)) // Allow (a) OVN->host traffic on the same node // (b) host->host traffic on the same node @@ -723,9 +742,10 @@ func (b *BridgeConfiguration) commonFlows(hostSubnets []*net.IPNet) ([]string, e } else { // for UDN we additionally SNAT the packet from masquerade IP -> node IP dftFlows = append(dftFlows, - fmt.Sprintf("cookie=%s, priority=100, in_port=%s, dl_src=%s, ipv6, ipv6_src=%s, "+ + fmt.Sprintf("cookie=%s, priority=100, in_port=%s, dl_src=%s, %s, %s_src=%s, "+ "actions=ct(commit, zone=%d, nat(src=%s), exec(set_field:%s->ct_mark)), output:%s", - nodetypes.DefaultOpenFlowCookie, netConfig.OfPortPatch, bridgeMacAddress, netConfig.V6MasqIPs.GatewayRouter.IP, config.Default.ConntrackZone, + nodetypes.DefaultOpenFlowCookie, netConfig.OfPortPatch, bridgeMacAddress, protoPrefixV6, protoPrefixV6, + netConfig.V6MasqIPs.GatewayRouter.IP, config.Default.ConntrackZone, physicalIP.IP, netConfig.MasqCTMark, ofPortPhys)) } } @@ -733,9 +753,10 @@ func (b *BridgeConfiguration) commonFlows(hostSubnets []*net.IPNet) ([]string, e // table 0, packets coming from host. Commit connections with ct_mark CtMarkHost // so that reverse direction goes back to the host. dftFlows = append(dftFlows, - fmt.Sprintf("cookie=%s, priority=100, in_port=%s, ipv6, "+ + fmt.Sprintf("cookie=%s, priority=100, in_port=%s, %s, "+ "actions=ct(commit, zone=%d, exec(set_field:%s->ct_mark)), %soutput:%s", - nodetypes.DefaultOpenFlowCookie, ofPortHost, config.Default.ConntrackZone, nodetypes.CtMarkHost, mod_vlan_id, ofPortPhys)) + nodetypes.DefaultOpenFlowCookie, ofPortHost, protoPrefixV6, + config.Default.ConntrackZone, nodetypes.CtMarkHost, mod_vlan_id, ofPortPhys)) } if config.Gateway.Mode == config.GatewayModeLocal { @@ -743,17 +764,17 @@ func (b *BridgeConfiguration) commonFlows(hostSubnets []*net.IPNet) ([]string, e // table 0, any packet coming from OVN send to host in LGW mode, host will take care of sending it outside if needed. // exceptions are traffic for egressIP and egressGW features and ICMP related traffic which will hit the priority 100 flow instead of this. dftFlows = append(dftFlows, - fmt.Sprintf("cookie=%s, priority=175, in_port=%s, tcp6, ipv6_src=%s, "+ + fmt.Sprintf("cookie=%s, priority=175, in_port=%s, tcp6, %s_src=%s, "+ "actions=ct(table=4,zone=%d)", - nodetypes.DefaultOpenFlowCookie, netConfig.OfPortPatch, physicalIP.IP, config.Default.HostMasqConntrackZone)) + nodetypes.DefaultOpenFlowCookie, netConfig.OfPortPatch, protoPrefixV6, physicalIP.IP, config.Default.HostMasqConntrackZone)) dftFlows = append(dftFlows, - fmt.Sprintf("cookie=%s, priority=175, in_port=%s, udp6, ipv6_src=%s, "+ + fmt.Sprintf("cookie=%s, priority=175, in_port=%s, udp6, %s_src=%s, "+ "actions=ct(table=4,zone=%d)", - nodetypes.DefaultOpenFlowCookie, netConfig.OfPortPatch, physicalIP.IP, config.Default.HostMasqConntrackZone)) + nodetypes.DefaultOpenFlowCookie, netConfig.OfPortPatch, protoPrefixV6, physicalIP.IP, config.Default.HostMasqConntrackZone)) dftFlows = append(dftFlows, - fmt.Sprintf("cookie=%s, priority=175, in_port=%s, sctp6, ipv6_src=%s, "+ + fmt.Sprintf("cookie=%s, priority=175, in_port=%s, sctp6, %s_src=%s, "+ "actions=ct(table=4,zone=%d)", - nodetypes.DefaultOpenFlowCookie, netConfig.OfPortPatch, physicalIP.IP, config.Default.HostMasqConntrackZone)) + nodetypes.DefaultOpenFlowCookie, netConfig.OfPortPatch, protoPrefixV6, physicalIP.IP, config.Default.HostMasqConntrackZone)) if ofPortPhys != "" { // We send BFD traffic coming from OVN to outside directly using a higher priority flow dftFlows = append(dftFlows, @@ -766,8 +787,8 @@ func (b *BridgeConfiguration) commonFlows(hostSubnets []*net.IPNet) ([]string, e // table 0, packets coming from external. Send it through conntrack and // resubmit to table 1 to know the state and mark of the connection. dftFlows = append(dftFlows, - fmt.Sprintf("cookie=%s, priority=50, ipv6, dl_dst=%s, actions=ct(zone=%d, nat, table=1)", - nodetypes.DefaultOpenFlowCookie, bridgeMacAddress, config.Default.ConntrackZone)) + fmt.Sprintf("cookie=%s, priority=50, %s, dl_dst=%s, actions=ct(zone=%d, nat, table=1)", + nodetypes.DefaultOpenFlowCookie, protoPrefixV6, bridgeMacAddress, config.Default.ConntrackZone)) } } if ofPortPhys != "" { @@ -993,9 +1014,9 @@ func (b *BridgeConfiguration) allowNodeIPGARPFlows(nodeIPs []net.IP) []string { } func getIPv(ipnet *net.IPNet) string { - prefix := "ip" + prefix := protoPrefixV4 if utilnet.IsIPv6CIDR(ipnet) { - prefix = "ipv6" + prefix = protoPrefixV6 } return prefix } @@ -1011,10 +1032,10 @@ func hostNetworkNormalActionFlows(netConfig *BridgeUDNConfiguration, srcMAC stri var ipFamily, ipFamilyDest string if isV6 { - ipFamily = "ipv6" - ipFamilyDest = "ipv6_dst" + ipFamily = protoPrefixV6 + ipFamilyDest = protoPrefixV6 + "_dst" } else { - ipFamily = "ip" + ipFamily = protoPrefixV4 ipFamilyDest = "nw_dst" } diff --git a/go-controller/pkg/node/gateway_shared_intf.go b/go-controller/pkg/node/gateway_shared_intf.go index 51f747fdfb..85141b1ff5 100644 --- a/go-controller/pkg/node/gateway_shared_intf.go +++ b/go-controller/pkg/node/gateway_shared_intf.go @@ -41,6 +41,8 @@ import ( ) const ( + protoPrefixV4 = "ip" + protoPrefixV6 = "ipv6" // etpSvcOpenFlowCookie identifies constant open flow rules added to the host OVS // bridge to move packets between host and external for etp=local traffic. // The hex number 0xe745ecf105, represents etp(e74)-service(5ec)-flows which makes it easier for debugging. @@ -400,36 +402,42 @@ func (npw *nodePortWatcher) updateServiceFlowCache(service *corev1.Service, netI return utilerrors.Join(errors...) } - ipPrefix := "ip" - if !utilnet.IsIPv4String(service.Spec.ClusterIP) { - ipPrefix = "ipv6" - } - // table 2, user-defined network host -> OVN towards default cluster network services defaultNetConfig := npw.ofm.defaultBridge.GetActiveNetworkBridgeConfigCopy(types.DefaultNetworkName) + var flows []string + clusterIPs := util.GetClusterIPs(service) outputActions := fmt.Sprintf("output:%s", defaultNetConfig.OfPortPatch) if config.Gateway.VLANID != 0 { outputActions = fmt.Sprintf("mod_vlan_vid:%d,%s", config.Gateway.VLANID, outputActions) } - // sample flow: cookie=0xdeff105, duration=2319.685s, table=2, n_packets=496, n_bytes=67111, priority=300, - // ip,nw_dst=10.96.0.1 actions=mod_dl_dst:02:42:ac:12:00:03,output:"patch-breth0_ov" - // This flow is used for UDNs and advertised UDNs to be able to reach kapi and dns services alone on default network - flows := []string{fmt.Sprintf("cookie=%s, priority=300, table=2, %s, %s_dst=%s, "+ - "actions=set_field:%s->eth_dst,%s", - nodetypes.DefaultOpenFlowCookie, ipPrefix, ipPrefix, service.Spec.ClusterIP, - npw.ofm.getDefaultBridgeMAC().String(), outputActions)} - if util.IsRouteAdvertisementsEnabled() { - // if the network is advertised, then for the reply from kapi and dns services to go back - // into the UDN's VRF we need flows that statically send this to the local port - // sample flow: cookie=0xdeff105, duration=264.196s, table=0, n_packets=0, n_bytes=0, priority=490,ip, - // in_port="patch-breth0_ov",nw_src=10.96.0.10,actions=ct(table=3,zone=64001,nat) - // this flow is meant to match all advertised UDNs and then the ip rules on the host will take - // this packet into the corresponding UDNs - // NOTE: We chose priority 490 to differentiate this flow from the flow at priority 500 added for the - // non-advertised UDNs reponse for debugging purposes: - // sample flow for non-advertised UDNs: cookie=0xdeff105, duration=684.087s, table=0, n_packets=0, n_bytes=0, - // idle_age=684, priority=500,ip,in_port=2,nw_src=10.96.0.0/16,nw_dst=169.254.0.0/17 actions=ct(table=3,zone=64001,nat) - flows = append(flows, fmt.Sprintf("cookie=%s, priority=490, in_port=%s, ip, ip_src=%s,actions=ct(zone=%d,nat,table=3)", - nodetypes.DefaultOpenFlowCookie, defaultNetConfig.OfPortPatch, service.Spec.ClusterIP, config.Default.HostMasqConntrackZone)) + + for _, clusterIP := range clusterIPs { + ipPrefix := protoPrefixV4 + if utilnet.IsIPv6String(clusterIP) { + ipPrefix = protoPrefixV6 + } + // table 2, user-defined network host -> OVN towards default cluster network services + // sample flow: cookie=0xdeff105, duration=2319.685s, table=2, n_packets=496, n_bytes=67111, priority=300, + // ip,nw_dst=10.96.0.1 actions=mod_dl_dst:02:42:ac:12:00:03,output:"patch-breth0_ov" + // This flow is used for UDNs and advertised UDNs to be able to reach kapi and dns services alone on default network + flows = append(flows, fmt.Sprintf("cookie=%s, priority=300, table=2, %s, %s_dst=%s, "+ + "actions=set_field:%s->eth_dst,%s", + nodetypes.DefaultOpenFlowCookie, ipPrefix, ipPrefix, clusterIP, + npw.ofm.getDefaultBridgeMAC().String(), outputActions)) + + if util.IsRouteAdvertisementsEnabled() { + // if the network is advertised, then for the reply from kapi and dns services to go back + // into the UDN's VRF we need flows that statically send this to the local port + // sample flow: cookie=0xdeff105, duration=264.196s, table=0, n_packets=0, n_bytes=0, priority=490,ip, + // in_port="patch-breth0_ov",nw_src=10.96.0.10,actions=ct(table=3,zone=64001,nat) + // this flow is meant to match all advertised UDNs and then the ip rules on the host will take + // this packet into the corresponding UDNs + // NOTE: We chose priority 490 to differentiate this flow from the flow at priority 500 added for the + // non-advertised UDNs reponse for debugging purposes: + // sample flow for non-advertised UDNs: cookie=0xdeff105, duration=684.087s, table=0, n_packets=0, n_bytes=0, + // idle_age=684, priority=500,ip,in_port=2,nw_src=10.96.0.0/16,nw_dst=169.254.0.0/17 actions=ct(table=3,zone=64001,nat) + flows = append(flows, fmt.Sprintf("cookie=%s, priority=490, in_port=%s, %s, %s_src=%s,actions=ct(zone=%d,nat,table=3)", + nodetypes.DefaultOpenFlowCookie, defaultNetConfig.OfPortPatch, ipPrefix, ipPrefix, clusterIP, config.Default.HostMasqConntrackZone)) + } } npw.ofm.updateFlowCacheEntry(key, flows) } diff --git a/go-controller/pkg/node/gateway_udn_test.go b/go-controller/pkg/node/gateway_udn_test.go index 4611ea97ed..984a666195 100644 --- a/go-controller/pkg/node/gateway_udn_test.go +++ b/go-controller/pkg/node/gateway_udn_test.go @@ -101,6 +101,12 @@ func setManagementPortFakeCommands(fexec *ovntest.FakeExec, nodeName string) { Cmd: "ip route replace table 7 172.16.1.0/24 via 100.128.0.1 dev ovn-k8s-mp0", Output: "0", }) + if config.IPv6Mode { + fexec.AddFakeCmd(&ovntest.ExpectedCmd{ + Cmd: "ip route replace table 7 fd02::/112 via ae70::1 dev ovn-k8s-mp0", + Output: "0", + }) + } fexec.AddFakeCmd(&ovntest.ExpectedCmd{ Cmd: "ip -4 rule", Output: "0", @@ -276,6 +282,10 @@ var _ = Describe("UserDefinedNetworkGateway", func() { // Restore global default values before each testcase err := config.PrepareTestConfig() Expect(err).NotTo(HaveOccurred()) + + // Set dual-stack service CIDRs directly after PrepareTestConfig + config.Kubernetes.ServiceCIDRs = ovntest.MustParseIPNets("172.16.1.0/24", "fd02::/112") + config.OVNKubernetesFeature.EnableMultiNetwork = true config.OVNKubernetesFeature.EnableNetworkSegmentation = true // Use a larger masq subnet to allow OF manager to allocate IPs for UDNs. @@ -653,7 +663,7 @@ var _ = Describe("UserDefinedNetworkGateway", func() { &kubeMock, vrf, ipRulesManager, localGw) Expect(err).NotTo(HaveOccurred()) flowMap := udnGateway.gateway.openflowManager.flowCache - Expect(flowMap["DEFAULT"]).To(HaveLen(46)) + Expect(flowMap["DEFAULT"]).To(HaveLen(50)) Expect(udnGateway.masqCTMark).To(Equal(udnGateway.masqCTMark)) var udnFlows int @@ -671,7 +681,7 @@ var _ = Describe("UserDefinedNetworkGateway", func() { Expect(udnGateway.AddNetwork()).To(Succeed()) flowMap = udnGateway.gateway.openflowManager.flowCache - Expect(flowMap["DEFAULT"]).To(HaveLen(64)) // 18 UDN Flows are added by default + Expect(flowMap["DEFAULT"]).To(HaveLen(70)) // 18 UDN Flows are added by default Expect(udnGateway.openflowManager.defaultBridge.GetNetConfigLen()).To(Equal(2)) // default network + UDN network defaultUdnConfig := udnGateway.openflowManager.defaultBridge.GetNetworkConfig("default") bridgeUdnConfig := udnGateway.openflowManager.defaultBridge.GetNetworkConfig("bluenet") @@ -687,7 +697,7 @@ var _ = Describe("UserDefinedNetworkGateway", func() { } } } - Expect(udnFlows).To(Equal(14)) + Expect(udnFlows).To(Equal(16)) openflowManagerCheckPorts(udnGateway.openflowManager) for _, svcCIDR := range config.Kubernetes.ServiceCIDRs { @@ -707,7 +717,7 @@ var _ = Describe("UserDefinedNetworkGateway", func() { kubeMock.On("UpdateNodeStatus", cnode).Return(nil) // check if network key gets deleted from annotation Expect(udnGateway.DelNetwork()).To(Succeed()) flowMap = udnGateway.gateway.openflowManager.flowCache - Expect(flowMap["DEFAULT"]).To(HaveLen(46)) // only default network flows are present + Expect(flowMap["DEFAULT"]).To(HaveLen(50)) // only default network flows are present Expect(udnGateway.openflowManager.defaultBridge.GetNetConfigLen()).To(Equal(1)) // default network only udnFlows = 0 for _, flows := range flowMap { @@ -885,7 +895,7 @@ var _ = Describe("UserDefinedNetworkGateway", func() { &kubeMock, vrf, ipRulesManager, localGw) Expect(err).NotTo(HaveOccurred()) flowMap := udnGateway.gateway.openflowManager.flowCache - Expect(flowMap["DEFAULT"]).To(HaveLen(46)) + Expect(flowMap["DEFAULT"]).To(HaveLen(50)) Expect(udnGateway.masqCTMark).To(Equal(udnGateway.masqCTMark)) var udnFlows int for _, flows := range flowMap { @@ -902,7 +912,7 @@ var _ = Describe("UserDefinedNetworkGateway", func() { Expect(udnGateway.AddNetwork()).To(Succeed()) flowMap = udnGateway.gateway.openflowManager.flowCache - Expect(flowMap["DEFAULT"]).To(HaveLen(64)) // 18 UDN Flows are added by default + Expect(flowMap["DEFAULT"]).To(HaveLen(70)) // 18 UDN Flows are added by default Expect(udnGateway.openflowManager.defaultBridge.GetNetConfigLen()).To(Equal(2)) // default network + UDN network defaultUdnConfig := udnGateway.openflowManager.defaultBridge.GetNetworkConfig("default") bridgeUdnConfig := udnGateway.openflowManager.defaultBridge.GetNetworkConfig("bluenet") @@ -918,7 +928,7 @@ var _ = Describe("UserDefinedNetworkGateway", func() { } } } - Expect(udnFlows).To(Equal(14)) + Expect(udnFlows).To(Equal(16)) openflowManagerCheckPorts(udnGateway.openflowManager) for _, svcCIDR := range config.Kubernetes.ServiceCIDRs { @@ -938,7 +948,7 @@ var _ = Describe("UserDefinedNetworkGateway", func() { kubeMock.On("UpdateNodeStatus", cnode).Return(nil) // check if network key gets deleted from annotation Expect(udnGateway.DelNetwork()).To(Succeed()) flowMap = udnGateway.gateway.openflowManager.flowCache - Expect(flowMap["DEFAULT"]).To(HaveLen(46)) // only default network flows are present + Expect(flowMap["DEFAULT"]).To(HaveLen(50)) // only default network flows are present Expect(udnGateway.openflowManager.defaultBridge.GetNetConfigLen()).To(Equal(1)) // default network only udnFlows = 0 for _, flows := range flowMap { @@ -1125,7 +1135,7 @@ var _ = Describe("UserDefinedNetworkGateway", func() { &kubeMock, vrf, ipRulesManager, localGw) Expect(err).NotTo(HaveOccurred()) flowMap := udnGateway.gateway.openflowManager.flowCache - Expect(flowMap["DEFAULT"]).To(HaveLen(46)) + Expect(flowMap["DEFAULT"]).To(HaveLen(50)) Expect(udnGateway.masqCTMark).To(Equal(udnGateway.masqCTMark)) var udnFlows int @@ -1143,7 +1153,7 @@ var _ = Describe("UserDefinedNetworkGateway", func() { Expect(udnGateway.AddNetwork()).To(Succeed()) flowMap = udnGateway.gateway.openflowManager.flowCache - Expect(flowMap["DEFAULT"]).To(HaveLen(73)) // 18 UDN Flows, 5 advertisedUDN flows, and 2 packet mark flows (IPv4+IPv6) are added by default + Expect(flowMap["DEFAULT"]).To(HaveLen(80)) // 18 UDN Flows, 5 advertisedUDN flows, and 2 packet mark flows (IPv4+IPv6) are added by default Expect(udnGateway.openflowManager.defaultBridge.GetNetConfigLen()).To(Equal(2)) // default network + UDN network defaultUdnConfig := udnGateway.openflowManager.defaultBridge.GetNetworkConfig("default") bridgeUdnConfig := udnGateway.openflowManager.defaultBridge.GetNetworkConfig("bluenet") @@ -1159,7 +1169,7 @@ var _ = Describe("UserDefinedNetworkGateway", func() { } } } - Expect(udnFlows).To(Equal(16)) + Expect(udnFlows).To(Equal(18)) openflowManagerCheckPorts(udnGateway.openflowManager) for _, svcCIDR := range config.Kubernetes.ServiceCIDRs { @@ -1181,7 +1191,7 @@ var _ = Describe("UserDefinedNetworkGateway", func() { kubeMock.On("UpdateNodeStatus", cnode).Return(nil) // check if network key gets deleted from annotation Expect(udnGateway.DelNetwork()).To(Succeed()) flowMap = udnGateway.gateway.openflowManager.flowCache - Expect(flowMap["DEFAULT"]).To(HaveLen(46)) // only default network flows are present + Expect(flowMap["DEFAULT"]).To(HaveLen(50)) // only default network flows are present Expect(udnGateway.openflowManager.defaultBridge.GetNetConfigLen()).To(Equal(1)) // default network only udnFlows = 0 for _, flows := range flowMap { @@ -1237,39 +1247,44 @@ var _ = Describe("UserDefinedNetworkGateway", func() { routes, err := udnGateway.computeRoutesForUDN(mplink) Expect(err).NotTo(HaveOccurred()) - Expect(routes).To(HaveLen(9)) - Expect(err).NotTo(HaveOccurred()) + Expect(routes).To(HaveLen(10)) + Expect(*routes[0].Dst).To(Equal(*ovntest.MustParseIPNet("172.16.1.0/24"))) // default service subnet Expect(routes[0].LinkIndex).To(Equal(bridgelink.Attrs().Index)) Expect(routes[0].Gw).To(Equal(config.Gateway.MasqueradeIPs.V4DummyNextHopMasqueradeIP)) + + Expect(*routes[1].Dst).To(Equal(*ovntest.MustParseIPNet("fd02::/112"))) // default service subnet + Expect(routes[1].LinkIndex).To(Equal(bridgelink.Attrs().Index)) + Expect(routes[1].Gw).To(Equal(config.Gateway.MasqueradeIPs.V6DummyNextHopMasqueradeIP)) + cidr, err := util.GetIPNetFullMask("169.254.0.16") Expect(err).NotTo(HaveOccurred()) - Expect(*routes[1].Dst).To(Equal(*cidr)) - Expect(routes[1].LinkIndex).To(Equal(mplink.Attrs().Index)) - cidr, err = util.GetIPNetFullMask("fd69::10") - Expect(err).NotTo(HaveOccurred()) Expect(*routes[2].Dst).To(Equal(*cidr)) Expect(routes[2].LinkIndex).To(Equal(mplink.Attrs().Index)) - - // IPv4 ETP=Local service masquerade IP route - Expect(*routes[3].Dst).To(Equal(*ovntest.MustParseIPNet("169.254.169.3/32"))) // ETP=Local svc masq IP + cidr, err = util.GetIPNetFullMask("fd69::10") + Expect(err).NotTo(HaveOccurred()) + Expect(*routes[3].Dst).To(Equal(*cidr)) Expect(routes[3].LinkIndex).To(Equal(mplink.Attrs().Index)) - Expect(routes[3].Gw.Equal(ovntest.MustParseIP("100.128.0.1"))).To(BeTrue()) - // IPv4 cluster subnet route - Expect(*routes[4].Dst).To(Equal(*ovntest.MustParseIPNet("100.128.0.0/16"))) // cluster subnet route + // IPv4 ETP=Local service masquerade IP route + Expect(*routes[4].Dst).To(Equal(*ovntest.MustParseIPNet("169.254.169.3/32"))) // ETP=Local svc masq IP Expect(routes[4].LinkIndex).To(Equal(mplink.Attrs().Index)) Expect(routes[4].Gw.Equal(ovntest.MustParseIP("100.128.0.1"))).To(BeTrue()) - // IPv6 ETP=Local service masquerade IP route - Expect(*routes[5].Dst).To(Equal(*ovntest.MustParseIPNet("fd69::3/128"))) // ETP=Local svc masq IP + // IPv4 cluster subnet route + Expect(*routes[5].Dst).To(Equal(*ovntest.MustParseIPNet("100.128.0.0/16"))) // cluster subnet route Expect(routes[5].LinkIndex).To(Equal(mplink.Attrs().Index)) - Expect(routes[5].Gw.Equal(ovntest.MustParseIP("ae70::1"))).To(BeTrue()) + Expect(routes[5].Gw.Equal(ovntest.MustParseIP("100.128.0.1"))).To(BeTrue()) - // IPv6 cluster subnet route - Expect(*routes[6].Dst).To(Equal(*ovntest.MustParseIPNet("ae70::/60"))) // cluster subnet route + // IPv6 ETP=Local service masquerade IP route + Expect(*routes[6].Dst).To(Equal(*ovntest.MustParseIPNet("fd69::3/128"))) // ETP=Local svc masq IP Expect(routes[6].LinkIndex).To(Equal(mplink.Attrs().Index)) Expect(routes[6].Gw.Equal(ovntest.MustParseIP("ae70::1"))).To(BeTrue()) + + // IPv6 cluster subnet route + Expect(*routes[7].Dst).To(Equal(*ovntest.MustParseIPNet("ae70::/60"))) // cluster subnet route + Expect(routes[7].LinkIndex).To(Equal(mplink.Attrs().Index)) + Expect(routes[7].Gw.Equal(ovntest.MustParseIP("ae70::1"))).To(BeTrue()) return nil }) Expect(err).NotTo(HaveOccurred()) @@ -1394,11 +1409,11 @@ var _ = Describe("UserDefinedNetworkGateway", func() { routes, err := udnGateway.computeRoutesForUDN(mplink) Expect(err).NotTo(HaveOccurred()) - Expect(routes).To(HaveLen(10)) + Expect(routes).To(HaveLen(11)) Expect(err).NotTo(HaveOccurred()) - Expect(*routes[1].Dst).To(Equal(*ovntest.MustParseIPNet("0.0.0.0/0"))) - Expect(routes[1].LinkIndex).To(Equal(bridgelink.Attrs().Index)) - Expect(routes[1].Gw.Equal(ovntest.MustParseIP(config.Gateway.NextHop))).To(BeTrue()) + Expect(*routes[2].Dst).To(Equal(*ovntest.MustParseIPNet("0.0.0.0/0"))) + Expect(routes[2].LinkIndex).To(Equal(bridgelink.Attrs().Index)) + Expect(routes[2].Gw.Equal(ovntest.MustParseIP(config.Gateway.NextHop))).To(BeTrue()) return nil }) Expect(err).NotTo(HaveOccurred()) @@ -1437,7 +1452,7 @@ var _ = Describe("UserDefinedNetworkGateway", func() { routes, err := udnGateway.computeRoutesForUDN(mplink) Expect(err).NotTo(HaveOccurred()) - Expect(routes).To(HaveLen(9)) + Expect(routes).To(HaveLen(10)) Expect(err).NotTo(HaveOccurred()) Expect(*routes[1].Dst).To(Not(Equal(*ovntest.MustParseIPNet("0.0.0.0/0")))) Expect(routes[1].Gw.Equal(ovntest.MustParseIP(config.Gateway.NextHop))).To(BeFalse()) diff --git a/go-controller/pkg/node/user_defined_node_network_controller_test.go b/go-controller/pkg/node/user_defined_node_network_controller_test.go index 3c79c227fd..af6a7a9018 100644 --- a/go-controller/pkg/node/user_defined_node_network_controller_test.go +++ b/go-controller/pkg/node/user_defined_node_network_controller_test.go @@ -166,7 +166,7 @@ var _ = Describe("UserDefinedNodeNetworkController: UserDefinedPrimaryNetwork Ga routeManager *routemanager.Controller ipRulesManager *iprulemanager.Controller v4NodeSubnet = "100.128.0.0/24" - v6NodeSubnet = "ae70::66/112" + v6NodeSubnet = "ae70::/112" mgtPort = fmt.Sprintf("%s%d", types.K8sMgmtIntfNamePrefix, netID) gatewayInterface = "eth0" gatewayBridge = "breth0" @@ -270,6 +270,7 @@ var _ = Describe("UserDefinedNodeNetworkController: UserDefinedPrimaryNetwork Ga config.IPv6Mode = true config.IPv4Mode = true config.Gateway.NodeportEnable = true + config.Kubernetes.ServiceCIDRs = ovntest.MustParseIPNets("172.16.1.0/24", "fd02::/112") ifAddrs := ovntest.MustParseIPNets(v4NodeIP, v6NodeIP) By("creating necessary mocks") diff --git a/test/e2e/route_advertisements.go b/test/e2e/route_advertisements.go index 5f9e53a8cc..95dfc5e7c3 100644 --- a/test/e2e/route_advertisements.go +++ b/test/e2e/route_advertisements.go @@ -125,17 +125,19 @@ var _ = ginkgo.Describe("BGP: Pod to external server when default podNetwork is framework.ExpectNoError(err, "must get bgpnet subnets") framework.Logf("the network cidrs to be imported are v4=%s and v6=%s", externalServerV4CIDR, externalServerV6CIDR) for _, node := range nodes.Items { - ipVer := "" - bgpRouteCommand := strings.Split(fmt.Sprintf("ip%s route show %s", ipVer, externalServerV4CIDR), " ") - framework.Logf("Checking for server's route in node %s", node.Name) - gomega.Eventually(func() bool { - routes, err := infraprovider.Get().ExecK8NodeCommand(node.GetName(), bgpRouteCommand) - framework.ExpectNoError(err, "failed to get BGP routes from node") - framework.Logf("Routes in node %s", routes) - return strings.Contains(routes, frrContainerIPv4) - }, 30*time.Second).Should(gomega.BeTrue()) - if isDualStackCluster(nodes) { - ipVer = " -6" + if isIPv4Supported(f.ClientSet) { + ipVer := "" + bgpRouteCommand := strings.Split(fmt.Sprintf("ip%s route show %s", ipVer, externalServerV4CIDR), " ") + framework.Logf("Checking for server's route in node %s", node.Name) + gomega.Eventually(func() bool { + routes, err := infraprovider.Get().ExecK8NodeCommand(node.GetName(), bgpRouteCommand) + framework.ExpectNoError(err, "failed to get BGP routes from node") + framework.Logf("Routes in node %s", routes) + return strings.Contains(routes, frrContainerIPv4) + }, 30*time.Second).Should(gomega.BeTrue()) + } + if isIPv6Supported(f.ClientSet) { + ipVer := " -6" nodeIPv6LLA, err := GetNodeIPv6LinkLocalAddressForEth0(routerContainerName) gomega.Expect(err).NotTo(gomega.HaveOccurred()) bgpRouteCommand := strings.Split(fmt.Sprintf("ip%s route show %s", ipVer, externalServerV6CIDR), " ") @@ -352,17 +354,19 @@ var _ = ginkgo.Describe("BGP: Pod to external server when CUDN network is advert gomega.Expect(err).NotTo(gomega.HaveOccurred()) } for _, node := range nodes.Items { - ipVer := "" - bgpRouteCommand := strings.Split(fmt.Sprintf("ip%s route show %s", ipVer, externalServerV4CIDR), " ") - framework.Logf("Checking for server's route in node %s", node.Name) - gomega.Eventually(func() bool { - routes, err := infraprovider.Get().ExecK8NodeCommand(node.GetName(), bgpRouteCommand) - framework.ExpectNoError(err, "failed to get BGP routes from node") - framework.Logf("Routes in node %s", routes) - return strings.Contains(routes, frrContainerIPv4) - }, 30*time.Second).Should(gomega.BeTrue()) - if isDualStackCluster(nodes) { - ipVer = " -6" + if isIPv4Supported(f.ClientSet) { + ipVer := "" + bgpRouteCommand := strings.Split(fmt.Sprintf("ip%s route show %s", ipVer, externalServerV4CIDR), " ") + framework.Logf("Checking for server's route in node %s", node.Name) + gomega.Eventually(func() bool { + routes, err := infraprovider.Get().ExecK8NodeCommand(node.GetName(), bgpRouteCommand) + framework.ExpectNoError(err, "failed to get BGP routes from node") + framework.Logf("Routes in node %s", routes) + return strings.Contains(routes, frrContainerIPv4) + }, 30*time.Second).Should(gomega.BeTrue()) + } + if isIPv6Supported(f.ClientSet) { + ipVer := " -6" bgpRouteCommand := strings.Split(fmt.Sprintf("ip%s route show %s", ipVer, externalServerV6CIDR), " ") framework.Logf("Checking for server's route in node %s", node.Name) gomega.Eventually(func() bool { @@ -395,7 +399,14 @@ var _ = ginkgo.Describe("BGP: Pod to external server when CUDN network is advert 60*time.Second) framework.ExpectNoError(err, fmt.Sprintf("Testing pod to external traffic failed: %v", err)) if isIPv6Supported(f.ClientSet) && utilnet.IsIPv6String(serverContainerIP) { - podIP, err = getPodAnnotationIPsForAttachmentByIndex(f.ClientSet, f.Namespace.Name, clientPod.Name, namespacedName(f.Namespace.Name, cUDN.Name), 1) + if isIPv4Supported(f.ClientSet) && isIPv6Supported(f.ClientSet) { + // for dualstack we need to fetch the IP at index1 + // if singlestack IPV6 the original podIP at index0 is the correct one + // FIXME: This util call assumes the first index will always be the IPv4 address + // and second index will always be the IPv6 address + // which is not always the case. + podIP, err = getPodAnnotationIPsForAttachmentByIndex(f.ClientSet, f.Namespace.Name, clientPod.Name, namespacedName(f.Namespace.Name, cUDN.Name), 1) + } // For IPv6 addresses, need to handle the brackets in the output outputIP := strings.TrimPrefix(strings.Split(stdout, "]:")[0], "[") gomega.Expect(outputIP).To(gomega.Equal(podIP), @@ -497,10 +508,6 @@ var _ = ginkgo.Describe("BGP: Pod to external server when CUDN network is advert var _ = ginkgo.DescribeTableSubtree("BGP: isolation between advertised networks", feature.RouteAdvertisements, func(cudnATemplate, cudnBTemplate *udnv1.ClusterUserDefinedNetwork) { const curlConnectionTimeoutCode = "28" - const ( - ipFamilyV4 = iota - ipFamilyV6 - ) f := wrappedTestFramework("bgp-network-isolation") f.SkipNamespaceCreation = true @@ -752,7 +759,7 @@ var _ = ginkgo.DescribeTableSubtree("BGP: isolation between advertised networks" }) ginkgo.DescribeTable("connectivity between networks", - func(connInfo func(ipFamilyIndex int) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool)) { + func(connInfo func(ipFamily utilnet.IPFamily) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool)) { // checkConnectivity performs a curl command from a specified client (pod or node) // to targetAddress. If clientNamespace is empty the function assumes clientName is a node that will be used as the // client. @@ -783,28 +790,16 @@ var _ = ginkgo.DescribeTableSubtree("BGP: isolation between advertised networks" framework.Logf("Connectivity check successful:'%s' -> %s", client, targetAddress) return out, nil } - clientName, clientNamespace, dst, expectedOutput, expectErr := connInfo(ipFamilyV4) - - asyncAssertion := gomega.Eventually - timeout := time.Second * 30 - if expectErr { - // When the connectivity check is expected to fail it should be failing consistently - asyncAssertion = gomega.Consistently - timeout = time.Second * 15 - } - asyncAssertion(func() error { - out, err := checkConnectivity(clientName, clientNamespace, dst) - if expectErr != (err != nil) { - return fmt.Errorf("expected connectivity check to return error(%t), got %v, output %v", expectErr, err, out) - } - if expectedOutput != "" { - if !strings.Contains(out, expectedOutput) { - return fmt.Errorf("expected connectivity check to contain %q, got %q", expectedOutput, out) - } + for _, ipFamily := range getSupportedIPFamiliesSlice(f.ClientSet) { + clientName, clientNamespace, dst, expectedOutput, expectErr := connInfo(ipFamily) + asyncAssertion := gomega.Eventually + timeout := time.Second * 30 + if expectErr { + // When the connectivity check is expected to fail it should be failing consistently + asyncAssertion = gomega.Consistently + timeout = time.Second * 15 } - if isIPv6Supported(f.ClientSet) && isIPv4Supported(f.ClientSet) { - // use ipFamilyIndex of 1 to pick the IPv6 addresses - clientName, clientNamespace, dst, expectedOutput, expectErr := connInfo(ipFamilyV6) + asyncAssertion(func() error { out, err := checkConnectivity(clientName, clientNamespace, dst) if expectErr != (err != nil) { return fmt.Errorf("expected connectivity check to return error(%t), got %v, output %v", expectErr, err, out) @@ -814,12 +809,12 @@ var _ = ginkgo.DescribeTableSubtree("BGP: isolation between advertised networks" return fmt.Errorf("expected connectivity check to contain %q, got %q", expectedOutput, out) } } - } - return nil - }, timeout).Should(gomega.BeNil()) + return nil + }, timeout).Should(gomega.BeNil()) + } }, ginkgo.Entry("pod to pod on the same network and same node should work", - func(ipFamilyIndex int) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { + func(ipFamily utilnet.IPFamily) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { // podsNetA[0] and podsNetA[1] are on the same node clientPod := podsNetA[0] srvPod := podsNetA[1] @@ -828,10 +823,11 @@ var _ = ginkgo.DescribeTableSubtree("BGP: isolation between advertised networks" framework.ExpectNoError(err) srvPodStatus, err := getPodAnnotationForAttachment(srvPod, namespacedName(srvPod.Namespace, cudnATemplate.Name)) framework.ExpectNoError(err) - return clientPod.Name, clientPod.Namespace, net.JoinHostPort(srvPodStatus.IPs[ipFamilyIndex].IP.String(), "8080") + "/clientip", clientPodStatus.IPs[ipFamilyIndex].IP.String(), false + return clientPod.Name, clientPod.Namespace, net.JoinHostPort(getFirstCIDROfFamily(ipFamily, srvPodStatus.IPs).IP.String(), "8080") + "/clientip", + getFirstCIDROfFamily(ipFamily, clientPodStatus.IPs).IP.String(), false }), ginkgo.Entry("pod to pod on the same network and different nodes should work", - func(ipFamilyIndex int) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { + func(ipFamily utilnet.IPFamily) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { // podsNetA[0] and podsNetA[2] are on different nodes clientPod := podsNetA[0] srvPod := podsNetA[2] @@ -840,10 +836,11 @@ var _ = ginkgo.DescribeTableSubtree("BGP: isolation between advertised networks" framework.ExpectNoError(err) srvPodStatus, err := getPodAnnotationForAttachment(srvPod, namespacedName(srvPod.Namespace, cudnATemplate.Name)) framework.ExpectNoError(err) - return clientPod.Name, clientPod.Namespace, net.JoinHostPort(srvPodStatus.IPs[ipFamilyIndex].IP.String(), "8080") + "/clientip", clientPodStatus.IPs[ipFamilyIndex].IP.String(), false + return clientPod.Name, clientPod.Namespace, net.JoinHostPort(getFirstCIDROfFamily(ipFamily, srvPodStatus.IPs).IP.String(), "8080") + "/clientip", + getFirstCIDROfFamily(ipFamily, clientPodStatus.IPs).IP.String(), false }), ginkgo.Entry("pod to pod connectivity on different networks and same node", - func(ipFamilyIndex int) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { + func(ipFamily utilnet.IPFamily) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { // podsNetA[2] and podNetB are on the same node clientPod := podsNetA[2] srvPod := podNetB @@ -862,18 +859,18 @@ var _ = ginkgo.DescribeTableSubtree("BGP: isolation between advertised networks" framework.ExpectNoError(err) // With the above underlay routing configuration client pod can reach server pod. - curlOutput = clientPodStatus.IPs[ipFamilyIndex].IP.String() + curlOutput = getFirstCIDROfFamily(ipFamily, clientPodStatus.IPs).IP.String() curlErr = false } else { curlOutput = curlConnectionTimeoutCode curlErr = true } - return clientPod.Name, clientPod.Namespace, net.JoinHostPort(srvPodStatus.IPs[ipFamilyIndex].IP.String(), "8080") + "/clientip", + return clientPod.Name, clientPod.Namespace, net.JoinHostPort(getFirstCIDROfFamily(ipFamily, srvPodStatus.IPs).IP.String(), "8080") + "/clientip", curlOutput, curlErr }), ginkgo.Entry("pod to pod connectivity on different networks and different nodes", - func(ipFamilyIndex int) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { + func(ipFamily utilnet.IPFamily) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { // podsNetA[0] and podNetB are on different nodes clientPod := podsNetA[0] srvPod := podNetB @@ -888,44 +885,48 @@ var _ = ginkgo.DescribeTableSubtree("BGP: isolation between advertised networks" clientPodStatus, err := getPodAnnotationForAttachment(clientPod, namespacedName(clientPod.Namespace, cudnATemplate.Name)) framework.ExpectNoError(err) - curlOutput = clientPodStatus.IPs[ipFamilyIndex].IP.String() + curlOutput = getFirstCIDROfFamily(ipFamily, clientPodStatus.IPs).IP.String() curlErr = false } else { curlOutput = curlConnectionTimeoutCode curlErr = true } - return clientPod.Name, clientPod.Namespace, net.JoinHostPort(srvPodStatus.IPs[ipFamilyIndex].IP.String(), "8080") + "/clientip", curlOutput, curlErr + return clientPod.Name, clientPod.Namespace, net.JoinHostPort(getFirstCIDROfFamily(ipFamily, srvPodStatus.IPs).IP.String(), "8080") + "/clientip", + curlOutput, curlErr }), ginkgo.Entry("pod in the default network should not be able to access an advertised UDN pod on the same node", - func(ipFamilyIndex int) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { + func(ipFamily utilnet.IPFamily) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { // podNetDefault and podNetB are on the same node clientPod := podNetDefault srvPod := podNetB srvPodStatus, err := getPodAnnotationForAttachment(srvPod, namespacedName(srvPod.Namespace, cudnBTemplate.Name)) framework.ExpectNoError(err) - return clientPod.Name, clientPod.Namespace, net.JoinHostPort(srvPodStatus.IPs[ipFamilyIndex].IP.String(), "8080") + "/clientip", curlConnectionTimeoutCode, true + return clientPod.Name, clientPod.Namespace, net.JoinHostPort(getFirstCIDROfFamily(ipFamily, srvPodStatus.IPs).IP.String(), "8080") + "/clientip", + curlConnectionTimeoutCode, true }), ginkgo.Entry("pod in the default network should not be able to access an advertised UDN pod on a different node", - func(ipFamilyIndex int) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { + func(ipFamily utilnet.IPFamily) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { // podNetDefault and podsNetA[0] are on different nodes clientPod := podNetDefault srvPod := podsNetA[0] srvPodStatus, err := getPodAnnotationForAttachment(srvPod, namespacedName(srvPod.Namespace, cudnATemplate.Name)) framework.ExpectNoError(err) - return clientPod.Name, clientPod.Namespace, net.JoinHostPort(srvPodStatus.IPs[ipFamilyIndex].IP.String(), "8080") + "/clientip", curlConnectionTimeoutCode, true + return clientPod.Name, clientPod.Namespace, net.JoinHostPort(getFirstCIDROfFamily(ipFamily, srvPodStatus.IPs).IP.String(), "8080") + "/clientip", + curlConnectionTimeoutCode, true }), ginkgo.Entry("pod in the default network should not be able to access a UDN service", - func(ipFamilyIndex int) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { - return podNetDefault.Name, podNetDefault.Namespace, net.JoinHostPort(svcNetA.Spec.ClusterIPs[ipFamilyIndex], "8080") + "/clientip", curlConnectionTimeoutCode, true + func(ipFamily utilnet.IPFamily) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { + return podNetDefault.Name, podNetDefault.Namespace, net.JoinHostPort(getFirstIPStringOfFamily(ipFamily, svcNetA.Spec.ClusterIPs), "8080") + "/clientip", + curlConnectionTimeoutCode, true }), ginkgo.Entry("pod in the UDN should be able to access a service in the same network", - func(ipFamilyIndex int) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { - return podsNetA[0].Name, podsNetA[0].Namespace, net.JoinHostPort(svcNetA.Spec.ClusterIPs[ipFamilyIndex], "8080") + "/clientip", "", false + func(ipFamily utilnet.IPFamily) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { + return podsNetA[0].Name, podsNetA[0].Namespace, net.JoinHostPort(getFirstIPStringOfFamily(ipFamily, svcNetA.Spec.ClusterIPs), "8080") + "/clientip", "", false }), ginkgo.Entry("pod in the UDN should not be able to access a default network service", - func(ipFamilyIndex int) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { + func(ipFamily utilnet.IPFamily) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { err := true out := curlConnectionTimeoutCode if cudnATemplate.Spec.Network.Topology == udnv1.NetworkTopologyLayer2 { @@ -935,41 +936,66 @@ var _ = ginkgo.DescribeTableSubtree("BGP: isolation between advertised networks" // this causes curl timeout with code 7 host unreachable instead of code 28 out = "" } - return podsNetA[0].Name, podsNetA[0].Namespace, net.JoinHostPort(svcNetDefault.Spec.ClusterIPs[ipFamilyIndex], "8080") + "/clientip", out, err + return podsNetA[0].Name, podsNetA[0].Namespace, net.JoinHostPort(getFirstIPStringOfFamily(ipFamily, svcNetDefault.Spec.ClusterIPs), "8080") + "/clientip", out, err }), ginkgo.Entry("pod in the UDN should be able to access kapi in default network service", - func(ipFamilyIndex int) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { + func(ipFamily utilnet.IPFamily) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { return podsNetA[0].Name, podsNetA[0].Namespace, "https://kubernetes.default/healthz", "", false }), + ginkgo.Entry("pod in the UDN should be able to access kapi service cluster IP directly", + func(ipFamily utilnet.IPFamily) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { + // Get kubernetes service from default namespace + kubernetesService, err := f.ClientSet.CoreV1().Services("default").Get(context.TODO(), "kubernetes", metav1.GetOptions{}) + framework.ExpectNoError(err, "should be able to get kubernetes service") + + // NOTE: See https://github.com/kubernetes/enhancements/tree/master/keps/sig-network/2438-dual-stack-apiserver + // Today the kubernetes.default service is single-stack and cannot be dual-stack. + if isDualStackCluster(nodes) && ipFamily == utilnet.IPv6 { + e2eskipper.Skipf("Dual stack kubernetes.default service is not supported in kubernetes") + } + // Get the cluster IP for the specified IP family + clusterIP := getFirstIPStringOfFamily(ipFamily, kubernetesService.Spec.ClusterIPs) + gomega.Expect(clusterIP).NotTo(gomega.BeEmpty(), fmt.Sprintf("no cluster IP available for IP family %v", ipFamily)) + + // Access the kubernetes API at the cluster IP directly on port 443 + return podsNetA[0].Name, podsNetA[0].Namespace, fmt.Sprintf("https://%s/healthz", net.JoinHostPort(clusterIP, "443")), "", false + }), ginkgo.Entry("pod in the UDN should not be able to access a service in a different UDN", - func(ipFamilyIndex int) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { - return podsNetA[0].Name, podsNetA[0].Namespace, net.JoinHostPort(svcNetB.Spec.ClusterIPs[ipFamilyIndex], "8080") + "/clientip", curlConnectionTimeoutCode, true + func(ipFamily utilnet.IPFamily) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { + return podsNetA[0].Name, podsNetA[0].Namespace, net.JoinHostPort(getFirstIPStringOfFamily(ipFamily, svcNetB.Spec.ClusterIPs), "8080") + "/clientip", + curlConnectionTimeoutCode, true }), ginkgo.Entry("host to a local UDN pod should not work", - func(ipFamilyIndex int) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { + func(ipFamily utilnet.IPFamily) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { clientNode := podsNetA[0].Spec.NodeName srvPod := podsNetA[0] srvPodStatus, err := getPodAnnotationForAttachment(srvPod, namespacedName(srvPod.Namespace, cudnATemplate.Name)) framework.ExpectNoError(err) - return clientNode, "", net.JoinHostPort(srvPodStatus.IPs[ipFamilyIndex].IP.String(), "8080") + "/clientip", curlConnectionTimeoutCode, true + return clientNode, "", net.JoinHostPort(getFirstCIDROfFamily(ipFamily, srvPodStatus.IPs).IP.String(), "8080") + "/clientip", + curlConnectionTimeoutCode, true }), ginkgo.Entry("host to a different node UDN pod should not work", - func(ipFamilyIndex int) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { + func(ipFamily utilnet.IPFamily) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { // podsNetA[0] and podsNetA[2] are on different nodes clientNode := podsNetA[2].Spec.NodeName srvPod := podsNetA[0] srvPodStatus, err := getPodAnnotationForAttachment(srvPod, namespacedName(srvPod.Namespace, cudnATemplate.Name)) framework.ExpectNoError(err) - return clientNode, "", net.JoinHostPort(srvPodStatus.IPs[ipFamilyIndex].IP.String(), "8080") + "/clientip", curlConnectionTimeoutCode, true + return clientNode, "", net.JoinHostPort(getFirstCIDROfFamily(ipFamily, srvPodStatus.IPs).IP.String(), "8080") + "/clientip", + curlConnectionTimeoutCode, true }), ginkgo.Entry("UDN pod to local node should not work", - func(ipFamilyIndex int) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { + func(ipFamily utilnet.IPFamily) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { clientPod := podsNetA[0] node, err := f.ClientSet.CoreV1().Nodes().Get(context.TODO(), clientPod.Spec.NodeName, metav1.GetOptions{}) framework.ExpectNoError(err) - nodeIP := node.Status.Addresses[ipFamilyIndex].Address + nodeIPv4, nodeIPv6 := getNodeAddresses(node) + nodeIP := nodeIPv4 + if ipFamily == utilnet.IPv6 { + nodeIP = nodeIPv6 + } // FIXME: add the host process socket to the VRF for this test to work. // This scenario is something that is not supported yet. So the test will continue to fail. // This works the same on both normal UDNs and advertised UDNs. @@ -991,52 +1017,72 @@ var _ = ginkgo.DescribeTableSubtree("BGP: isolation between advertised networks" return clientPod.Name, clientPod.Namespace, net.JoinHostPort(nodeIP, fmt.Sprint(hostNetworkPort)) + "/hostname", "", true }), ginkgo.Entry("UDN pod to a different node should work", - func(ipFamilyIndex int) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { + func(ipFamily utilnet.IPFamily) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { clientPod := podsNetA[0] // podsNetA[0] and podsNetA[2] are on different nodes so we can pick the node of podsNetA[2] as the different node destination node, err := f.ClientSet.CoreV1().Nodes().Get(context.TODO(), podsNetA[2].Spec.NodeName, metav1.GetOptions{}) framework.ExpectNoError(err) - nodeIP := node.Status.Addresses[ipFamilyIndex].Address + nodeIPv4, nodeIPv6 := getNodeAddresses(node) + nodeIP := nodeIPv4 + if ipFamily == utilnet.IPv6 { + nodeIP = nodeIPv6 + } clientNode, err := f.ClientSet.CoreV1().Nodes().Get(context.TODO(), clientPod.Spec.NodeName, metav1.GetOptions{}) framework.ExpectNoError(err) - clientNodeIP := clientNode.Status.Addresses[ipFamilyIndex].Address + clientNodeIPv4, clientNodeIPv6 := getNodeAddresses(clientNode) + clientNodeIP := clientNodeIPv4 + if ipFamily == utilnet.IPv6 { + clientNodeIP = clientNodeIPv6 + } // pod -> node traffic should use the node's IP as the source for advertised UDNs. return clientPod.Name, clientPod.Namespace, net.JoinHostPort(nodeIP, fmt.Sprint(hostNetworkPort)) + "/clientip", clientNodeIP, false }), ginkgo.Entry("UDN pod to the same node nodeport service in default network should not work", // FIXME: https://github.com/ovn-kubernetes/ovn-kubernetes/issues/5410 - func(ipFamilyIndex int) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { + func(ipFamily utilnet.IPFamily) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { clientPod := podsNetA[0] // podsNetA[0] is on nodes[0]. We need the same node. Let's hit the nodeport on nodes[0]. node, err := f.ClientSet.CoreV1().Nodes().Get(context.TODO(), nodes.Items[0].Name, metav1.GetOptions{}) framework.ExpectNoError(err) - nodeIP := node.Status.Addresses[ipFamilyIndex].Address + nodeIPv4, nodeIPv6 := getNodeAddresses(node) + nodeIP := nodeIPv4 + if ipFamily == utilnet.IPv6 { + nodeIP = nodeIPv6 + } nodePort := svcNetDefault.Spec.Ports[0].NodePort return clientPod.Name, clientPod.Namespace, net.JoinHostPort(nodeIP, fmt.Sprint(nodePort)) + "/hostname", curlConnectionTimeoutCode, true }), ginkgo.Entry("UDN pod to a different node nodeport service in default network should work", - func(ipFamilyIndex int) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { + func(ipFamily utilnet.IPFamily) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { clientPod := podsNetA[0] // podsNetA[0] is on nodes[0]. We need a different node. podNetDefault is on nodes[1]. // The service is backed by podNetDefault. Let's hit the nodeport on nodes[2]. node, err := f.ClientSet.CoreV1().Nodes().Get(context.TODO(), nodes.Items[2].Name, metav1.GetOptions{}) framework.ExpectNoError(err) - nodeIP := node.Status.Addresses[ipFamilyIndex].Address + nodeIPv4, nodeIPv6 := getNodeAddresses(node) + nodeIP := nodeIPv4 + if ipFamily == utilnet.IPv6 { + nodeIP = nodeIPv6 + } nodePort := svcNetDefault.Spec.Ports[0].NodePort return clientPod.Name, clientPod.Namespace, net.JoinHostPort(nodeIP, fmt.Sprint(nodePort)) + "/hostname", "", false }), ginkgo.Entry("UDN pod to the same node nodeport service in same UDN network should work", - func(ipFamilyIndex int) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { + func(ipFamily utilnet.IPFamily) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { clientPod := podsNetA[0] // The service is backed by pods in podsNetA. // We want to hit the nodeport on the same node. // client is on nodes[0]. Let's hit nodeport on nodes[0]. node, err := f.ClientSet.CoreV1().Nodes().Get(context.TODO(), nodes.Items[0].Name, metav1.GetOptions{}) framework.ExpectNoError(err) - nodeIP := node.Status.Addresses[ipFamilyIndex].Address + nodeIPv4, nodeIPv6 := getNodeAddresses(node) + nodeIP := nodeIPv4 + if ipFamily == utilnet.IPv6 { + nodeIP = nodeIPv6 + } nodePort := svcNetA.Spec.Ports[0].NodePort // The service can be backed by any of the pods in podsNetA, so we can't reliably check the output hostname. @@ -1044,14 +1090,18 @@ var _ = ginkgo.DescribeTableSubtree("BGP: isolation between advertised networks" return clientPod.Name, clientPod.Namespace, net.JoinHostPort(nodeIP, fmt.Sprint(nodePort)) + "/hostname", "", false }), ginkgo.Entry("UDN pod to a different node nodeport service in same UDN network should work", - func(ipFamilyIndex int) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { + func(ipFamily utilnet.IPFamily) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { clientPod := podsNetA[0] // The service is backed by pods in podsNetA. // We want to hit the nodeport on a different node. // client is on nodes[0]. Let's hit nodeport on nodes[2]. node, err := f.ClientSet.CoreV1().Nodes().Get(context.TODO(), nodes.Items[2].Name, metav1.GetOptions{}) framework.ExpectNoError(err) - nodeIP := node.Status.Addresses[ipFamilyIndex].Address + nodeIPv4, nodeIPv6 := getNodeAddresses(node) + nodeIP := nodeIPv4 + if ipFamily == utilnet.IPv6 { + nodeIP = nodeIPv6 + } nodePort := svcNetA.Spec.Ports[0].NodePort // sourceIP will be joinSubnetIP for nodeports, so only using hostname endpoint @@ -1070,15 +1120,19 @@ var _ = ginkgo.DescribeTableSubtree("BGP: isolation between advertised networks" // fails as it doesn't know how to reach this masqueradeIP. // There is also inconsistency in behaviour within Layer2 networks for how IPv4 works and how IPv6 works where the traffic // works on ipv6 because of the flows described below. - func(ipFamilyIndex int) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { + func(ipFamily utilnet.IPFamily) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { clientPod := podsNetA[0] node, err := f.ClientSet.CoreV1().Nodes().Get(context.TODO(), nodes.Items[0].Name, metav1.GetOptions{}) framework.ExpectNoError(err) - nodeIP := node.Status.Addresses[ipFamilyIndex].Address + nodeIPv4, nodeIPv6 := getNodeAddresses(node) + nodeIP := nodeIPv4 + if ipFamily == utilnet.IPv6 { + nodeIP = nodeIPv6 + } nodePort := svcNetB.Spec.Ports[0].NodePort out := curlConnectionTimeoutCode errBool := true - if ipFamilyIndex == ipFamilyV6 && cudnATemplate.Spec.Network.Topology == udnv1.NetworkTopologyLayer2 { + if ipFamily == utilnet.IPv6 && cudnATemplate.Spec.Network.Topology == udnv1.NetworkTopologyLayer2 { // For Layer2 networks, we have these flows we add on breth0: // cookie=0xdeff105, duration=173.245s, table=1, n_packets=0, n_bytes=0, idle_age=173, priority=14,icmp6,icmp_type=134 actions=FLOOD // cookie=0xdeff105, duration=173.245s, table=1, n_packets=8, n_bytes=640, idle_age=4, priority=14,icmp6,icmp_type=136 actions=FLOOD @@ -1095,14 +1149,18 @@ var _ = ginkgo.DescribeTableSubtree("BGP: isolation between advertised networks" return clientPod.Name, clientPod.Namespace, net.JoinHostPort(nodeIP, fmt.Sprint(nodePort)) + "/hostname", out, errBool }), ginkgo.Entry("UDN pod to a different node nodeport service in different UDN network should work", - func(ipFamilyIndex int) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { + func(ipFamily utilnet.IPFamily) (clientName string, clientNamespace string, dst string, expectedOutput string, expectErr bool) { clientPod := podsNetA[0] // The service is backed by podNetB. // We want to hit the nodeport on a different node from the client. // client is on nodes[0]. Let's hit nodeport on nodes[2]. node, err := f.ClientSet.CoreV1().Nodes().Get(context.TODO(), nodes.Items[2].Name, metav1.GetOptions{}) framework.ExpectNoError(err) - nodeIP := node.Status.Addresses[ipFamilyIndex].Address + nodeIPv4, nodeIPv6 := getNodeAddresses(node) + nodeIP := nodeIPv4 + if ipFamily == utilnet.IPv6 { + nodeIP = nodeIPv6 + } nodePort := svcNetB.Spec.Ports[0].NodePort // sourceIP will be joinSubnetIP for nodeports, so only using hostname endpoint @@ -2366,8 +2424,8 @@ func checkL3NodePodRoute(node corev1.Node, serverContainerIP, routerContainerNam if isIPv6 { podCIDR = podv6CIDR } - gomega.Expect(podCIDR).NotTo(gomega.BeEmpty(), - "pod CIDR for family (isIPv6=%t) missing for node %s on network %s", isIPv6, node.Name, netName) + gomega.Expect(podCIDR).NotTo(gomega.BeEmpty(), + "pod CIDR for family (isIPv6=%t) missing for node %s on network %s", isIPv6, node.Name, netName) checkRouteInFRR(node, podCIDR, routerContainerName, isIPv6) } diff --git a/test/scripts/e2e-cp.sh b/test/scripts/e2e-cp.sh index 096debe8a6..1ab06622f6 100755 --- a/test/scripts/e2e-cp.sh +++ b/test/scripts/e2e-cp.sh @@ -160,7 +160,14 @@ else # TODO: perhaps the secondary network attached pods should not be attached to default network skip "Multi Homing A single pod with an OVN-K secondary network attached to a localnet network mapped to external primary interface bridge can be reached by a client pod in the default network on the same node" skip "Multi Homing A single pod with an OVN-K secondary network attached to a localnet network mapped to external primary interface bridge can be reached by a client pod in the default network on a different node" - + if [ "$PLATFORM_IPV6_SUPPORT" == true ] && [ "$PLATFORM_IPV4_SUPPORT" == false ]; then + # Skip all Multi Homing tests in BGP IPv6 only mode + # TODO: The tests are doing weird static ipv4, ipv6, dualstack specific ginkgo entries instead of relying on + # cluster family type to make a dynamic determination. These tests need to be refactored to be family-friendly + # instead of assuming only single stack v4 or dualstack lanes exist. + # https://github.com/ovn-kubernetes/ovn-kubernetes/issues/5569 + skip "Multi Homing" + fi # these tests require metallb but the configuration we do for it is not compatible with the configuration we do to advertise the default network # TODO: consolidate configuration skip "Load Balancer Service Tests with MetalLB"