From 11b7e2f1c2bc4afc2bf69a26caf46547ff56af6a Mon Sep 17 00:00:00 2001 From: Nadia Pinaeva Date: Mon, 13 Mar 2023 17:33:18 +0100 Subject: [PATCH 01/40] update all fields on CreateOrUpdateACLsOps, especially for nil values, which mean reset, not ignore. When acls are updated on initial sync (by handling add events) if namespace log levels were reset, acls should also be updated with nil Severity. Signed-off-by: Nadia Pinaeva --- go-controller/pkg/libovsdbops/acl.go | 9 +- go-controller/pkg/libovsdbops/acl_test.go | 138 ++++++++++++++++++++++ 2 files changed, 145 insertions(+), 2 deletions(-) create mode 100644 go-controller/pkg/libovsdbops/acl_test.go diff --git a/go-controller/pkg/libovsdbops/acl.go b/go-controller/pkg/libovsdbops/acl.go index efe17dbf89..39158c5b37 100644 --- a/go-controller/pkg/libovsdbops/acl.go +++ b/go-controller/pkg/libovsdbops/acl.go @@ -21,6 +21,11 @@ func GetACLName(acl *nbdb.ACL) string { return "" } +func getACLMutableFields(acl *nbdb.ACL) []interface{} { + return []interface{}{&acl.Action, &acl.Direction, &acl.ExternalIDs, &acl.Log, &acl.Match, &acl.Meter, + &acl.Name, &acl.Options, &acl.Priority, &acl.Severity} +} + // isEquivalentACL if it has same uuid, or if it has same name // and external ids, or if it has same priority, direction, match // and action. @@ -137,7 +142,7 @@ func CreateOrUpdateACLsOps(nbClient libovsdbclient.Client, ops []libovsdb.Operat opModel := operationModel{ Model: acl, ModelPredicate: func(item *nbdb.ACL) bool { return isEquivalentACL(item, acl) }, - OnModelUpdates: onModelUpdatesAllNonDefault(), + OnModelUpdates: getACLMutableFields(acl), ErrNotFound: false, BulkOp: false, } @@ -156,7 +161,7 @@ func UpdateACLsOps(nbClient libovsdbclient.Client, ops []libovsdb.Operation, acl opModel := operationModel{ Model: acl, ModelPredicate: func(item *nbdb.ACL) bool { return isEquivalentACL(item, acl) }, - OnModelUpdates: onModelUpdatesAllNonDefault(), + OnModelUpdates: getACLMutableFields(acl), ErrNotFound: true, BulkOp: false, } diff --git a/go-controller/pkg/libovsdbops/acl_test.go b/go-controller/pkg/libovsdbops/acl_test.go new file mode 100644 index 0000000000..ae4ef21fcc --- /dev/null +++ b/go-controller/pkg/libovsdbops/acl_test.go @@ -0,0 +1,138 @@ +package libovsdbops + +import ( + "fmt" + "testing" + + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" + libovsdbtest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/libovsdb" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" +) + +func TestCreateOrUpdateACL(t *testing.T) { + aclName := "acl1" + aclSev := nbdb.ACLSeverityInfo + aclMeter := types.OvnACLLoggingMeter + initialACL := &nbdb.ACL{ + UUID: buildNamedUUID(), + Action: nbdb.ACLActionAllow, + Direction: nbdb.ACLDirectionToLport, + ExternalIDs: map[string]string{"key": "value"}, + Log: true, + Match: "match", + Meter: &aclMeter, + Name: &aclName, + Options: map[string]string{"key": "value"}, + Priority: 1, + Severity: &aclSev, + } + + tests := []struct { + desc string + initialACL *nbdb.ACL + finalACL *nbdb.ACL + }{ + { + desc: "updates Severity to empty", + initialACL: initialACL, + finalACL: &nbdb.ACL{ + Action: nbdb.ACLActionAllow, + Direction: nbdb.ACLDirectionToLport, + ExternalIDs: map[string]string{"key": "value"}, + Log: true, + Match: "match", + Meter: &aclMeter, + Name: &aclName, + Options: map[string]string{"key": "value"}, + Priority: 1, + Severity: nil, + }, + }, + { + desc: "updates Name to empty", + initialACL: initialACL, + finalACL: &nbdb.ACL{ + Action: nbdb.ACLActionAllow, + Direction: nbdb.ACLDirectionToLport, + ExternalIDs: map[string]string{"key": "value"}, + Log: true, + Match: "match", + Meter: &aclMeter, + Name: nil, + Options: map[string]string{"key": "value"}, + Priority: 1, + Severity: &aclSev, + }, + }, + { + desc: "updates Options to empty", + initialACL: initialACL, + finalACL: &nbdb.ACL{ + Action: nbdb.ACLActionAllow, + Direction: nbdb.ACLDirectionToLport, + ExternalIDs: map[string]string{"key": "value"}, + Log: true, + Match: "match", + Meter: &aclMeter, + Name: &aclName, + Options: nil, + Priority: 1, + Severity: &aclSev, + }, + }, + { + desc: "updates ExternalIDs to empty", + initialACL: initialACL, + finalACL: &nbdb.ACL{ + Action: nbdb.ACLActionAllow, + Direction: nbdb.ACLDirectionToLport, + ExternalIDs: nil, + Log: true, + Match: "match", + Meter: &aclMeter, + Name: &aclName, + Options: map[string]string{"key": "value"}, + Priority: 1, + Severity: &aclSev, + }, + }, + } + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + nbClient, cleanup, err := libovsdbtest.NewNBTestHarness(libovsdbtest.TestSetup{ + NBData: []libovsdbtest.TestData{ + tt.initialACL, + }, + }, nil) + if err != nil { + t.Fatalf("test: \"%s\" failed to set up test harness: %v", tt.desc, err) + } + t.Cleanup(cleanup.Cleanup) + + // test update with UUID set + initialACLs, err := FindACLs(nbClient, []*nbdb.ACL{{ + ExternalIDs: tt.initialACL.ExternalIDs, + }}) + if err != nil { + t.Fatalf("test: \"%s\" failed to find initial ACL: %v", tt.desc, err) + } + if len(initialACLs) != 1 { + t.Fatalf("test: \"%s\" found %d intitial ACls, expected 1", tt.desc, len(initialACLs)) + } + tt.finalACL.UUID = initialACLs[0].UUID + + err = CreateOrUpdateACLs(nbClient, tt.finalACL) + if err != nil { + t.Fatalf("test: \"%s\" failed to set up test harness: %v", tt.desc, err) + } + matcher := libovsdbtest.HaveData([]libovsdbtest.TestData{tt.finalACL}) + success, err := matcher.Match(nbClient) + if !success { + t.Fatal(fmt.Errorf("test: \"%s\" didn't match expected with actual, err: %v", tt.desc, matcher.FailureMessage(nbClient))) + } + if err != nil { + t.Fatal(fmt.Errorf("test: \"%s\" encountered error: %v", tt.desc, err)) + } + }) + } +} From 18dcd72ad8b757a716ec93fc441d90d25c7c20a2 Mon Sep 17 00:00:00 2001 From: Nadia Pinaeva Date: Wed, 29 Mar 2023 13:02:51 +0200 Subject: [PATCH 02/40] rename aclType to aclPipelineType, add aclDirection type to separate intention from implementation. Signed-off-by: Nadia Pinaeva --- go-controller/pkg/ovn/acl.go | 54 +++++++++++++++++++++++++++++------- 1 file changed, 44 insertions(+), 10 deletions(-) diff --git a/go-controller/pkg/ovn/acl.go b/go-controller/pkg/ovn/acl.go index a0c03c2575..0563f73178 100644 --- a/go-controller/pkg/ovn/acl.go +++ b/go-controller/pkg/ovn/acl.go @@ -17,36 +17,54 @@ func joinACLName(substrings ...string) string { return strings.Join(substrings, "_") } -// aclType defines when ACLs will be applied (direction and pipeline stage). +// aclPipelineType defines when ACLs will be applied (direction and pipeline stage). // All acls of the same type will be sorted by priority, priorities for different types are independent. -type aclType string +type aclPipelineType string const ( // lportIngress will be converted to direction="to-lport" ACL - lportIngress aclType = "to-lport" + lportIngress aclPipelineType = "to-lport" // lportEgressAfterLB will be converted to direction="from-lport", options={"apply-after-lb": "true"} ACL - lportEgressAfterLB aclType = "from-lport-after-lb" + lportEgressAfterLB aclPipelineType = "from-lport-after-lb" ) -func aclTypeToPolicyType(aclT aclType) knet.PolicyType { +func aclTypeToPolicyType(aclT aclPipelineType) knet.PolicyType { switch aclT { case lportEgressAfterLB: return knet.PolicyTypeEgress case lportIngress: return knet.PolicyTypeIngress default: - panic(fmt.Sprintf("Failed to convert aclType to PolicyType: unknown acl type %s", aclT)) + panic(fmt.Sprintf("Failed to convert aclPipelineType to PolicyType: unknown acl type %s", aclT)) } } -func policyTypeToAclType(policyType knet.PolicyType) aclType { +func policyTypeToAclPipeline(policyType knet.PolicyType) aclPipelineType { switch policyType { case knet.PolicyTypeEgress: return lportEgressAfterLB case knet.PolicyTypeIngress: return lportIngress default: - panic(fmt.Sprintf("Failed to convert PolicyType to aclType: unknown policyType type %s", policyType)) + panic(fmt.Sprintf("Failed to convert PolicyType to aclPipelineType: unknown policyType type %s", policyType)) + } +} + +type aclDirection string + +const ( + aclEgress aclDirection = "Egress" + aclIngress aclDirection = "Ingress" +) + +func aclDirectionToACLPipeline(aclDir aclDirection) aclPipelineType { + switch aclDir { + case aclEgress: + return lportEgressAfterLB + case aclIngress: + return lportIngress + default: + panic(fmt.Sprintf("Failed to convert aclDirection to aclPipelineType: unknown aclDirection type %s", aclDir)) } } @@ -58,7 +76,7 @@ func hashedPortGroup(s string) string { // BuildACL should be used to build ACL instead of directly calling libovsdbops.BuildACL. // It can properly set and reset log settings for ACL based on ACLLoggingLevels func BuildACL(aclName string, priority int, match, action string, - logLevels *ACLLoggingLevels, aclT aclType, externalIDs map[string]string) *nbdb.ACL { + logLevels *ACLLoggingLevels, aclT aclPipelineType, externalIDs map[string]string) *nbdb.ACL { var options map[string]string var direction string switch aclT { @@ -88,7 +106,7 @@ func BuildACL(aclName string, priority int, match, action string, return ACL } -func getACLMatch(portGroupName, match string, aclT aclType) string { +func getACLMatch(portGroupName, match string, aclT aclPipelineType) string { var aclMatch string switch aclT { case lportIngress: @@ -104,6 +122,22 @@ func getACLMatch(portGroupName, match string, aclT aclType) string { return aclMatch } +func getACLMatchFromACLDir(portGroupName, match string, aclDir aclDirection) string { + var aclMatch string + switch aclDir { + case aclIngress: + aclMatch = "outport == @" + portGroupName + case aclEgress: + aclMatch = "inport == @" + portGroupName + default: + panic(fmt.Sprintf("Unknown acl direction %s", aclDir)) + } + if match != "" { + aclMatch += " && " + match + } + return aclMatch +} + // GetNamespaceACLLogging retrieves ACLLoggingLevels for the Namespace. // nsInfo will be locked (and unlocked at the end) for given namespace if it exists. func (oc *DefaultNetworkController) GetNamespaceACLLogging(ns string) *ACLLoggingLevels { From f89913e09b3acaad5bbda9eb54bf3636c6085265 Mon Sep 17 00:00:00 2001 From: Nadia Pinaeva Date: Fri, 24 Mar 2023 17:31:19 +0100 Subject: [PATCH 03/40] Start using new ExternalIDs for ACLs. Multicast. Add new BuildACLFromDbIDs function, that should be the only one used in the end of acls indexes update. Add 2 owner Types for global and namespaced multicast resources. Generate ACL name based on dbIDs, don't include owner information to save some symbols. Add (objectIDs *DbObjectIDs) StringNoOwner() for this purpose. Add cleanup function for disabled multicast support and namespaced multicast cleanup. Move noneMatch to the gress_policy.go, where it is used. Add tests for multicast acl sync and cleanup. Rename address_set_syncer package to external_ids_syncer, add acl syncer sub-package. Signed-off-by: Nadia Pinaeva --- .../pkg/libovsdbops/db_object_ids.go | 40 +- .../pkg/libovsdbops/db_object_types.go | 32 +- go-controller/pkg/ovn/acl.go | 47 ++ .../address_set_syncer_suite_test.go | 13 - .../pkg/ovn/default_network_controller.go | 13 +- .../external_ids_syncer/acl/acl_suite_test.go | 13 + .../ovn/external_ids_syncer/acl/acl_sync.go | 138 +++++ .../external_ids_syncer/acl/acl_sync_test.go | 169 ++++++ .../address_set/address_set_suite_test.go | 13 + .../address_set}/address_set_sync.go | 2 +- .../address_set}/address_set_sync_test.go | 7 +- go-controller/pkg/ovn/gress_policy.go | 6 +- go-controller/pkg/ovn/master.go | 4 + go-controller/pkg/ovn/multicast.go | 184 ++++-- go-controller/pkg/ovn/multicast_test.go | 574 ++++++++++++++---- go-controller/pkg/ovn/namespace.go | 7 + go-controller/pkg/ovn/policy.go | 8 +- 17 files changed, 1037 insertions(+), 233 deletions(-) delete mode 100644 go-controller/pkg/ovn/address_set_syncer/address_set_syncer_suite_test.go create mode 100644 go-controller/pkg/ovn/external_ids_syncer/acl/acl_suite_test.go create mode 100644 go-controller/pkg/ovn/external_ids_syncer/acl/acl_sync.go create mode 100644 go-controller/pkg/ovn/external_ids_syncer/acl/acl_sync_test.go create mode 100644 go-controller/pkg/ovn/external_ids_syncer/address_set/address_set_suite_test.go rename go-controller/pkg/ovn/{address_set_syncer => external_ids_syncer/address_set}/address_set_sync.go (99%) rename go-controller/pkg/ovn/{address_set_syncer => external_ids_syncer/address_set}/address_set_sync_test.go (99%) diff --git a/go-controller/pkg/libovsdbops/db_object_ids.go b/go-controller/pkg/libovsdbops/db_object_ids.go index 417a996d5c..2828fb0a08 100644 --- a/go-controller/pkg/libovsdbops/db_object_ids.go +++ b/go-controller/pkg/libovsdbops/db_object_ids.go @@ -89,25 +89,27 @@ func newObjectIDsType(dbTable dbObjType, ownerObjectType ownerType, keys []Exter // - objectIDs provide values for keys that are used by given idsType. To create a new object, all fields should be set. // For predicate search, only some values that need to be matched may be set. // -// dbIndex := NewDbObjectIDs(AddressSetEgressFirewallDNS, "DefaultController", -// map[ExternalIDKey]string{ -// ObjectNameKey: "dns.name", -// AddressSetIPFamilyKey: "ipv4" -// }) -// uses AddressSetEgressFirewallDNS = newObjectIDsType(addressSet, EgressFirewallDNSOwnerType, []ExternalIDKey{ -// // dnsName -// ObjectNameKey, -// AddressSetIPFamilyKey, -// }) +// dbIndex := NewDbObjectIDs(AddressSetEgressFirewallDNS, "DefaultController", +// map[ExternalIDKey]string{ +// ObjectNameKey: "dns.name", +// AddressSetIPFamilyKey: "ipv4" +// }) +// +// uses AddressSetEgressFirewallDNS = newObjectIDsType(addressSet, EgressFirewallDNSOwnerType, []ExternalIDKey{ +// // dnsName +// ObjectNameKey, +// AddressSetIPFamilyKey, +// }) // // its dbIndex will be mapped to the following ExternalIDs -// { -// "k8s.ovn.org/owner-controller": "DefaultController" -// "k8s.ovn.org/owner-type": "EgressFirewallDNS" (value of EgressFirewallDNSOwnerType) -// "k8s.ovn.org/name": "dns.name" -// "k8s.ovn.org/ipFamily": "ipv4" -// "k8s.ovn.org/id": "DefaultController:EgressFirewallDNS:dns.name:ipv4" -// } +// +// { +// "k8s.ovn.org/owner-controller": "DefaultController" +// "k8s.ovn.org/owner-type": "EgressFirewallDNS" (value of EgressFirewallDNSOwnerType) +// "k8s.ovn.org/name": "dns.name" +// "k8s.ovn.org/ipFamily": "ipv4" +// "k8s.ovn.org/id": "DefaultController:EgressFirewallDNS:dns.name:ipv4" +// } type DbObjectIDs struct { idsType *ObjectIDsType // ownerControllerName specifies which controller owns the object. @@ -229,6 +231,10 @@ func (objectIDs *DbObjectIDs) String() string { return id } +func (objectIDs *DbObjectIDs) GetIDsType() *ObjectIDsType { + return objectIDs.idsType +} + // getUniqueID returns primary id that is build based on objectIDs values. // If at least one required key is missing, an error will be returned. func (objectIDs *DbObjectIDs) getUniqueID() (string, error) { diff --git a/go-controller/pkg/libovsdbops/db_object_types.go b/go-controller/pkg/libovsdbops/db_object_types.go index e7ce45a140..2ba8f936fa 100644 --- a/go-controller/pkg/libovsdbops/db_object_types.go +++ b/go-controller/pkg/libovsdbops/db_object_types.go @@ -10,19 +10,22 @@ const ( EgressFirewallDNSOwnerType ownerType = "EgressFirewallDNS" EgressQoSOwnerType ownerType = "EgressQoS" // only used for cleanup now, as the stale owner of network policy address sets - NetworkPolicyOwnerType ownerType = "NetworkPolicy" - NetpolDefaultOwnerType ownerType = "NetpolDefault" - PodSelectorOwnerType ownerType = "PodSelector" - NamespaceOwnerType ownerType = "Namespace" - HybridNodeRouteOwnerType ownerType = "HybridNodeRoute" - EgressIPOwnerType ownerType = "EgressIP" - EgressServiceOwnerType ownerType = "EgressService" + NetworkPolicyOwnerType ownerType = "NetworkPolicy" + NetpolDefaultOwnerType ownerType = "NetpolDefault" + PodSelectorOwnerType ownerType = "PodSelector" + NamespaceOwnerType ownerType = "Namespace" + HybridNodeRouteOwnerType ownerType = "HybridNodeRoute" + EgressIPOwnerType ownerType = "EgressIP" + EgressServiceOwnerType ownerType = "EgressService" + MulticastNamespaceOwnerType ownerType = "MulticastNS" + MulticastClusterOwnerType ownerType = "MulticastCluster" // owner extra IDs, make sure to define only 1 ExternalIDKey for every string value PriorityKey ExternalIDKey = "priority" PolicyDirectionKey ExternalIDKey = "direction" GressIdxKey ExternalIDKey = "gress-index" AddressSetIPFamilyKey ExternalIDKey = "ip-family" + TypeKey ExternalIDKey = "type" ) // ObjectIDsTypes should only be created here @@ -88,3 +91,18 @@ var ACLNetpolDefault = newObjectIDsType(acl, NetpolDefaultOwnerType, []ExternalI // egress or ingress PolicyDirectionKey, }) + +var ACLMulticastNamespace = newObjectIDsType(acl, MulticastNamespaceOwnerType, []ExternalIDKey{ + // namespace + ObjectNameKey, + // egress or ingress + PolicyDirectionKey, +}) + +var ACLMulticastCluster = newObjectIDsType(acl, MulticastClusterOwnerType, []ExternalIDKey{ + // cluster-scoped multicast acls + // there are 2 possible TypeKey values for cluster default multicast acl: DefaultDeny and AllowInterNode + TypeKey, + // egress or ingress + PolicyDirectionKey, +}) diff --git a/go-controller/pkg/ovn/acl.go b/go-controller/pkg/ovn/acl.go index 0563f73178..3cba2a25cd 100644 --- a/go-controller/pkg/ovn/acl.go +++ b/go-controller/pkg/ovn/acl.go @@ -73,6 +73,17 @@ func hashedPortGroup(s string) string { return util.HashForOVN(s) } +// acl.Name is cropped to 64 symbols and is used for logging. +// currently only egress firewall, gress network policy and default deny network policy ACLs are logged. +// Other ACLs don't need a name. +// Just a namespace name may be 63 symbols long, therefore some information may be cropped. +// Therefore, "feature" as "EF" for EgressFirewall and "NP" for network policy goes first, then namespace, +// then acl-related info. +func getACLName(dbIDs *libovsdbops.DbObjectIDs) string { + aclName := "" + return fmt.Sprintf("%.63s", aclName) +} + // BuildACL should be used to build ACL instead of directly calling libovsdbops.BuildACL. // It can properly set and reset log settings for ACL based on ACLLoggingLevels func BuildACL(aclName string, priority int, match, action string, @@ -106,6 +117,42 @@ func BuildACL(aclName string, priority int, match, action string, return ACL } +// BuildACLFromDbIDs should be used to build ACL instead of directly calling libovsdbops.BuildACL. +// It can properly set and reset log settings for ACL based on ACLLoggingLevels, and +// set acl.Name and acl.ExternalIDs based on given DbIDs +func BuildACLFromDbIDs(dbIDs *libovsdbops.DbObjectIDs, priority int, match, action string, logLevels *ACLLoggingLevels, + aclT aclPipelineType) *nbdb.ACL { + var options map[string]string + var direction string + switch aclT { + case lportEgressAfterLB: + direction = nbdb.ACLDirectionFromLport + options = map[string]string{ + "apply-after-lb": "true", + } + case lportIngress: + direction = nbdb.ACLDirectionToLport + default: + panic(fmt.Sprintf("Failed to build ACL: unknown acl type %s", aclT)) + } + externalIDs := dbIDs.GetExternalIDs() + aclName := getACLName(dbIDs) + log, logSeverity := getLogSeverity(action, logLevels) + ACL := libovsdbops.BuildACL( + aclName, + direction, + priority, + match, + action, + types.OvnACLLoggingMeter, + logSeverity, + log, + externalIDs, + options, + ) + return ACL +} + func getACLMatch(portGroupName, match string, aclT aclPipelineType) string { var aclMatch string switch aclT { diff --git a/go-controller/pkg/ovn/address_set_syncer/address_set_syncer_suite_test.go b/go-controller/pkg/ovn/address_set_syncer/address_set_syncer_suite_test.go deleted file mode 100644 index f66fbdac37..0000000000 --- a/go-controller/pkg/ovn/address_set_syncer/address_set_syncer_suite_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package address_set_syncer - -import ( - "testing" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -func TestAddressSetSyncer(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Address Set Syncer Operations Suite") -} diff --git a/go-controller/pkg/ovn/default_network_controller.go b/go-controller/pkg/ovn/default_network_controller.go index f253b03ad3..0674bed87b 100644 --- a/go-controller/pkg/ovn/default_network_controller.go +++ b/go-controller/pkg/ovn/default_network_controller.go @@ -17,10 +17,11 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" addressset "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/address_set" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/address_set_syncer" egresssvc "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/controller/egress_services" svccontroller "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/controller/services" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/controller/unidling" + aclsyncer "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/external_ids_syncer/acl" + addrsetsyncer "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/external_ids_syncer/address_set" lsm "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/logical_switch_manager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/retry" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/syncmap" @@ -275,12 +276,18 @@ func (oc *DefaultNetworkController) Start(ctx context.Context) error { // sync address sets, only required for DefaultNetworkController, since any old objects in the db without // Owner set are owned by the default network controller. - syncer := address_set_syncer.NewAddressSetSyncer(oc.nbClient, DefaultNetworkControllerName) - err := syncer.SyncAddressSets() + addrSetSyncer := addrsetsyncer.NewAddressSetSyncer(oc.nbClient, DefaultNetworkControllerName) + err := addrSetSyncer.SyncAddressSets() if err != nil { return fmt.Errorf("failed to sync address sets on controller init: %v", err) } + aclSyncer := aclsyncer.NewACLSyncer(oc.nbClient, DefaultNetworkControllerName) + err = aclSyncer.SyncACLs() + if err != nil { + return fmt.Errorf("failed to sync acls on controller init: %v", err) + } + // sync shared resources // pod selector address sets err = oc.cleanupPodSelectorAddressSets() diff --git a/go-controller/pkg/ovn/external_ids_syncer/acl/acl_suite_test.go b/go-controller/pkg/ovn/external_ids_syncer/acl/acl_suite_test.go new file mode 100644 index 0000000000..b699917aed --- /dev/null +++ b/go-controller/pkg/ovn/external_ids_syncer/acl/acl_suite_test.go @@ -0,0 +1,13 @@ +package acl_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestAcl(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Acl Suite") +} diff --git a/go-controller/pkg/ovn/external_ids_syncer/acl/acl_sync.go b/go-controller/pkg/ovn/external_ids_syncer/acl/acl_sync.go new file mode 100644 index 0000000000..f706de03ea --- /dev/null +++ b/go-controller/pkg/ovn/external_ids_syncer/acl/acl_sync.go @@ -0,0 +1,138 @@ +package acl + +import ( + "fmt" + "strings" + + libovsdbclient "github.com/ovn-org/libovsdb/client" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdbops" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util/batching" + + knet "k8s.io/api/networking/v1" + "k8s.io/klog/v2" +) + +const ( + defaultDenyPolicyTypeACLExtIdKey = "default-deny-policy-type" + mcastDefaultDenyID = "DefaultDeny" + mcastAllowInterNodeID = "AllowInterNode" +) + +type aclSyncer struct { + nbClient libovsdbclient.Client + controllerName string + // txnBatchSize is used to control how many acls will be updated with 1 db transaction. + txnBatchSize int +} + +// controllerName is the name of the new controller that should own all acls without controller +func NewACLSyncer(nbClient libovsdbclient.Client, controllerName string) *aclSyncer { + return &aclSyncer{ + nbClient: nbClient, + controllerName: controllerName, + // create time (which is the upper bound of how much time an update can take) for 20K ACLs + // (gress ACL were used for testing as the ones that have the biggest number of ExternalIDs) + // is ~4 sec, which is safe enough to not exceed 10 sec transaction timeout. + txnBatchSize: 20000, + } +} + +func (syncer *aclSyncer) SyncACLs() error { + // stale acls don't have controller ID + legacyAclPred := libovsdbops.GetNoOwnerPredicate[*nbdb.ACL]() + legacyACLs, err := libovsdbops.FindACLsWithPredicate(syncer.nbClient, legacyAclPred) + if err != nil { + return fmt.Errorf("unable to find stale ACLs, cannot update stale data: %v", err) + } + if len(legacyACLs) == 0 { + return nil + } + + updatedACLs := syncer.updateStaleMulticastACLsDbIDs(legacyACLs) + klog.Infof("Found %d stale multicast ACLs", len(updatedACLs)) + + err = batching.Batch[*nbdb.ACL](syncer.txnBatchSize, updatedACLs, func(batchACLs []*nbdb.ACL) error { + return libovsdbops.CreateOrUpdateACLs(syncer.nbClient, batchACLs...) + }) + if err != nil { + return fmt.Errorf("cannot update stale ACLs: %v", err) + } + return nil +} + +func (syncer *aclSyncer) getDefaultMcastACLDbIDs(mcastType, policyDirection string) *libovsdbops.DbObjectIDs { + // there are 2 types of default multicast ACLs in every direction (Ingress/Egress) + // DefaultDeny = deny multicast by default + // AllowInterNode = allow inter-node multicast + return libovsdbops.NewDbObjectIDs(libovsdbops.ACLMulticastCluster, syncer.controllerName, + map[libovsdbops.ExternalIDKey]string{ + libovsdbops.TypeKey: mcastType, + libovsdbops.PolicyDirectionKey: policyDirection, + }) + +} + +func (syncer *aclSyncer) getNamespaceMcastACLDbIDs(ns, policyDirection string) *libovsdbops.DbObjectIDs { + // namespaces ACL + return libovsdbops.NewDbObjectIDs(libovsdbops.ACLMulticastNamespace, syncer.controllerName, + map[libovsdbops.ExternalIDKey]string{ + libovsdbops.ObjectNameKey: ns, + libovsdbops.PolicyDirectionKey: policyDirection, + }) +} + +// updateStaleMulticastACLsDbIDs updates multicast ACLs that don't have new ExternalIDs set. +// Must be run before WatchNamespace, since namespaceSync function uses syncNsMulticast, which relies on the new IDs. +func (syncer *aclSyncer) updateStaleMulticastACLsDbIDs(legacyACLs []*nbdb.ACL) []*nbdb.ACL { + updatedACLs := []*nbdb.ACL{} + for _, acl := range legacyACLs { + var dbIDs *libovsdbops.DbObjectIDs + if acl.Priority == types.DefaultMcastDenyPriority { + // there is only 1 type acl type with this priority: default deny + dbIDs = syncer.getDefaultMcastACLDbIDs(mcastDefaultDenyID, acl.ExternalIDs[defaultDenyPolicyTypeACLExtIdKey]) + } else if acl.Priority == types.DefaultMcastAllowPriority { + // there are 4 multicast allow types + if acl.ExternalIDs[defaultDenyPolicyTypeACLExtIdKey] == string(knet.PolicyTypeIngress) { + // ingress allow acl + // either default of namespaced + if strings.Contains(acl.Match, types.ClusterRtrPortGroupName) { + // default allow ingress + dbIDs = syncer.getDefaultMcastACLDbIDs(mcastAllowInterNodeID, string(knet.PolicyTypeIngress)) + } else { + // namespace allow ingress + // acl Name can be truncated (max length 64), but k8s namespace is limited to 63 symbols, + // therefore it is safe to extract it from the name + ns := strings.Split(libovsdbops.GetACLName(acl), "_")[0] + dbIDs = syncer.getNamespaceMcastACLDbIDs(ns, string(knet.PolicyTypeIngress)) + } + } else if acl.ExternalIDs[defaultDenyPolicyTypeACLExtIdKey] == string(knet.PolicyTypeEgress) { + // egress allow acl + // either default of namespaced + if strings.Contains(acl.Match, types.ClusterRtrPortGroupName) { + // default allow egress + dbIDs = syncer.getDefaultMcastACLDbIDs(mcastAllowInterNodeID, string(knet.PolicyTypeEgress)) + } else { + // namespace allow egress + // acl Name can be truncated (max length 64), but k8s namespace is limited to 63 symbols, + // therefore it is safe to extract it from the name + ns := strings.Split(libovsdbops.GetACLName(acl), "_")[0] + dbIDs = syncer.getNamespaceMcastACLDbIDs(ns, string(knet.PolicyTypeEgress)) + } + } else { + // unexpected, acl with multicast priority should have ExternalIDs[defaultDenyPolicyTypeACLExtIdKey] set + klog.Warningf("Found stale ACL with multicast priority %d, but without expected ExternalID[%s]: %+v", + acl.Priority, defaultDenyPolicyTypeACLExtIdKey, acl) + continue + } + } else { + //non-multicast acl + continue + } + // update externalIDs + acl.ExternalIDs = dbIDs.GetExternalIDs() + updatedACLs = append(updatedACLs, acl) + } + return updatedACLs +} diff --git a/go-controller/pkg/ovn/external_ids_syncer/acl/acl_sync_test.go b/go-controller/pkg/ovn/external_ids_syncer/acl/acl_sync_test.go new file mode 100644 index 0000000000..a9333b349e --- /dev/null +++ b/go-controller/pkg/ovn/external_ids_syncer/acl/acl_sync_test.go @@ -0,0 +1,169 @@ +package acl + +import ( + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdbops" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" + libovsdbtest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/libovsdb" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" + "strings" +) + +type aclSync struct { + before *nbdb.ACL + after *libovsdbops.DbObjectIDs +} + +func testSyncerWithData(data []aclSync, controllerName string) { + // create initial db setup + dbSetup := libovsdbtest.TestSetup{NBData: []libovsdbtest.TestData{}} + for _, asSync := range data { + asSync.before.UUID = *asSync.before.Name + "-UUID" + dbSetup.NBData = append(dbSetup.NBData, asSync.before) + } + libovsdbOvnNBClient, _, libovsdbCleanup, err := libovsdbtest.NewNBSBTestHarness(dbSetup) + defer libovsdbCleanup.Cleanup() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + // create expected data using addressSetFactory + expectedDbState := []libovsdbtest.TestData{} + for _, aclSync := range data { + acl := aclSync.before + acl.ExternalIDs = aclSync.after.GetExternalIDs() + expectedDbState = append(expectedDbState, acl) + } + // run sync + syncer := NewACLSyncer(libovsdbOvnNBClient, controllerName) + err = syncer.SyncACLs() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + // check results + gomega.Eventually(libovsdbOvnNBClient).Should(libovsdbtest.HaveData(expectedDbState)) +} + +func joinACLName(substrings ...string) string { + return strings.Join(substrings, "_") +} + +var _ = ginkgo.Describe("OVN ACL Syncer", func() { + const ( + controllerName = "fake-controller" + namespace1 = "namespace1" + ) + var syncerToBuildData = aclSyncer{ + controllerName: controllerName, + } + + ginkgo.It("updates multicast acls", func() { + testData := []aclSync{ + // defaultDenyEgressACL + { + before: libovsdbops.BuildACL( + joinACLName(types.ClusterPortGroupName, "DefaultDenyMulticastEgress"), + nbdb.ACLDirectionFromLport, + types.DefaultMcastDenyPriority, + "(ip4.mcast || mldv1 || mldv2 || (ip6.dst[120..127] == 0xff && ip6.dst[116] == 1))", + nbdb.ACLActionDrop, + types.OvnACLLoggingMeter, + "", + false, + map[string]string{ + defaultDenyPolicyTypeACLExtIdKey: "Egress", + }, + nil, + ), + after: syncerToBuildData.getDefaultMcastACLDbIDs(mcastDefaultDenyID, "Egress"), + }, + // defaultDenyIngressACL + { + before: libovsdbops.BuildACL( + joinACLName(types.ClusterPortGroupName, "DefaultDenyMulticastIngress"), + nbdb.ACLDirectionToLport, + types.DefaultMcastDenyPriority, + "(ip4.mcast || mldv1 || mldv2 || (ip6.dst[120..127] == 0xff && ip6.dst[116] == 1))", + nbdb.ACLActionDrop, + types.OvnACLLoggingMeter, + "", + false, + map[string]string{ + defaultDenyPolicyTypeACLExtIdKey: "Ingress", + }, + nil, + ), + after: syncerToBuildData.getDefaultMcastACLDbIDs(mcastDefaultDenyID, "Ingress"), + }, + // defaultAllowEgressACL + { + before: libovsdbops.BuildACL( + joinACLName(types.ClusterRtrPortGroupName, "DefaultAllowMulticastEgress"), + nbdb.ACLDirectionFromLport, + types.DefaultMcastAllowPriority, + "inport == @clusterRtrPortGroup && (ip4.mcast || mldv1 || mldv2 || (ip6.dst[120..127] == 0xff && ip6.dst[116] == 1))", + nbdb.ACLActionAllow, + types.OvnACLLoggingMeter, + "", + false, + map[string]string{ + defaultDenyPolicyTypeACLExtIdKey: "Egress", + }, + nil, + ), + after: syncerToBuildData.getDefaultMcastACLDbIDs(mcastAllowInterNodeID, "Egress"), + }, + // defaultAllowIngressACL + { + before: libovsdbops.BuildACL( + joinACLName(types.ClusterRtrPortGroupName, "DefaultAllowMulticastIngress"), + nbdb.ACLDirectionToLport, + types.DefaultMcastAllowPriority, + "outport == @clusterRtrPortGroup && (ip4.mcast || mldv1 || mldv2 || (ip6.dst[120..127] == 0xff && ip6.dst[116] == 1))", + nbdb.ACLActionAllow, + types.OvnACLLoggingMeter, + "", + false, + map[string]string{ + defaultDenyPolicyTypeACLExtIdKey: "Ingress", + }, + nil, + ), + after: syncerToBuildData.getDefaultMcastACLDbIDs(mcastAllowInterNodeID, "Ingress"), + }, + // nsAllowEgressACL + { + before: libovsdbops.BuildACL( + joinACLName(namespace1, "MulticastAllowEgress"), + nbdb.ACLDirectionFromLport, + types.DefaultMcastAllowPriority, + "inport == @a16982411286042166782 && ip4.mcast", + nbdb.ACLActionAllow, + types.OvnACLLoggingMeter, + "", + false, + map[string]string{ + defaultDenyPolicyTypeACLExtIdKey: "Egress", + }, + nil, + ), + after: syncerToBuildData.getNamespaceMcastACLDbIDs(namespace1, "Egress"), + }, + // nsAllowIngressACL + { + before: libovsdbops.BuildACL( + joinACLName(namespace1, "MulticastAllowIngress"), + nbdb.ACLDirectionToLport, + types.DefaultMcastAllowPriority, + "outport == @a16982411286042166782 && (igmp || (ip4.src == $a4322231855293774466 && ip4.mcast))", + nbdb.ACLActionAllow, + types.OvnACLLoggingMeter, + "", + false, + map[string]string{ + defaultDenyPolicyTypeACLExtIdKey: "Ingress", + }, + nil, + ), + after: syncerToBuildData.getNamespaceMcastACLDbIDs(namespace1, "Ingress"), + }, + } + testSyncerWithData(testData, controllerName) + }) +}) diff --git a/go-controller/pkg/ovn/external_ids_syncer/address_set/address_set_suite_test.go b/go-controller/pkg/ovn/external_ids_syncer/address_set/address_set_suite_test.go new file mode 100644 index 0000000000..5159a898fc --- /dev/null +++ b/go-controller/pkg/ovn/external_ids_syncer/address_set/address_set_suite_test.go @@ -0,0 +1,13 @@ +package address_set_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestAddressSet(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "AddressSet Suite") +} diff --git a/go-controller/pkg/ovn/address_set_syncer/address_set_sync.go b/go-controller/pkg/ovn/external_ids_syncer/address_set/address_set_sync.go similarity index 99% rename from go-controller/pkg/ovn/address_set_syncer/address_set_sync.go rename to go-controller/pkg/ovn/external_ids_syncer/address_set/address_set_sync.go index d755624499..a03bbff797 100644 --- a/go-controller/pkg/ovn/address_set_syncer/address_set_sync.go +++ b/go-controller/pkg/ovn/external_ids_syncer/address_set/address_set_sync.go @@ -1,4 +1,4 @@ -package address_set_syncer +package address_set import ( "fmt" diff --git a/go-controller/pkg/ovn/address_set_syncer/address_set_sync_test.go b/go-controller/pkg/ovn/external_ids_syncer/address_set/address_set_sync_test.go similarity index 99% rename from go-controller/pkg/ovn/address_set_syncer/address_set_sync_test.go rename to go-controller/pkg/ovn/external_ids_syncer/address_set/address_set_sync_test.go index f9f50ffa03..18d0ed7cac 100644 --- a/go-controller/pkg/ovn/address_set_syncer/address_set_sync_test.go +++ b/go-controller/pkg/ovn/external_ids_syncer/address_set/address_set_sync_test.go @@ -1,4 +1,4 @@ -package address_set_syncer +package address_set import ( "fmt" @@ -6,7 +6,6 @@ import ( "github.com/onsi/gomega" "strings" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdbops" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" libovsdbtest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/libovsdb" @@ -34,10 +33,6 @@ func testSyncerWithData(data []asSync, initialDbState, finalDbState []libovsdbte for _, asSync := range data { dbSetup.NBData = append(dbSetup.NBData, asSync.before) } - // Restore global default values before each testcase - err := config.PrepareTestConfig() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - //config.IPv6Mode = true libovsdbOvnNBClient, _, libovsdbCleanup, err := libovsdbtest.NewNBSBTestHarness(dbSetup) defer libovsdbCleanup.Cleanup() gomega.Expect(err).NotTo(gomega.HaveOccurred()) diff --git a/go-controller/pkg/ovn/gress_policy.go b/go-controller/pkg/ovn/gress_policy.go index c9b0d3ab8d..7ffa8bb1d3 100644 --- a/go-controller/pkg/ovn/gress_policy.go +++ b/go-controller/pkg/ovn/gress_policy.go @@ -16,6 +16,10 @@ import ( utilnet "k8s.io/utils/net" ) +const ( + noneMatch = "None" +) + type gressPolicy struct { controllerName string policyNamespace string @@ -315,7 +319,7 @@ func getGressPolicyACLName(ns, policyName string, idx int) string { // buildACLAllow builds an allow-related ACL for a given match func (gp *gressPolicy) buildACLAllow(match, l4Match string, ipBlockCIDR int, aclLogging *ACLLoggingLevels) *nbdb.ACL { - aclT := policyTypeToAclType(gp.policyType) + aclT := policyTypeToAclPipeline(gp.policyType) priority := types.DefaultAllowPriority action := nbdb.ACLActionAllowRelated if gp.isNetPolStateless { diff --git a/go-controller/pkg/ovn/master.go b/go-controller/pkg/ovn/master.go index 8bb2eacff9..fd6950d87d 100644 --- a/go-controller/pkg/ovn/master.go +++ b/go-controller/pkg/ovn/master.go @@ -173,6 +173,10 @@ func (oc *DefaultNetworkController) SetupMaster(existingNodeNames []string) erro klog.Errorf("Failed to create default deny multicast policy, error: %v", err) return err } + } else { + if err = oc.disableMulticast(); err != nil { + return fmt.Errorf("failed to delete default multicast policy, error: %v", err) + } } // Create OVNJoinSwitch that will be used to connect gateway routers to the distributed router. diff --git a/go-controller/pkg/ovn/multicast.go b/go-controller/pkg/ovn/multicast.go index cb358ed920..63f8545eca 100644 --- a/go-controller/pkg/ovn/multicast.go +++ b/go-controller/pkg/ovn/multicast.go @@ -7,17 +7,25 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdbops" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" + + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog/v2" ) +type defaultMcastACLTypeID string + const ( - noneMatch = "None" // IPv6 multicast traffic destined to dynamic groups must have the "T" bit // set to 1: https://tools.ietf.org/html/rfc3307#section-4.3 - ipv6DynamicMulticastMatch = "(ip6.dst[120..127] == 0xff && ip6.dst[116] == 1)" + ipv6DynamicMulticastMatch = "(ip6.dst[120..127] == 0xff && ip6.dst[116] == 1)" + mcastDefaultDenyID defaultMcastACLTypeID = "DefaultDeny" + mcastAllowInterNodeID defaultMcastACLTypeID = "AllowInterNode" +) + +// Legacy const, should only be used in sync and tests +const ( // Legacy multicastDefaultDeny port group removed by commit 40a90f0 legacyMulticastDefaultDenyPortGroup = "mcastPortGroupDeny" ) @@ -76,8 +84,29 @@ func getMulticastACLEgrMatch() string { return getACLMatchAF(ipv4Match, ipv6Match) } -func getMcastACLName(nsORpg, mcastSuffix string) string { - return joinACLName(nsORpg, mcastSuffix) +func getDefaultMcastACLDbIDs(mcastType defaultMcastACLTypeID, aclDir aclDirection, controller string) *libovsdbops.DbObjectIDs { + // there are 2 types of default multicast ACLs in every direction (Ingress/Egress) + // DefaultDeny = deny multicast by default + // AllowInterNode = allow inter-node multicast + return libovsdbops.NewDbObjectIDs(libovsdbops.ACLMulticastCluster, controller, + map[libovsdbops.ExternalIDKey]string{ + libovsdbops.TypeKey: string(mcastType), + libovsdbops.PolicyDirectionKey: string(aclDir), + }) + +} + +func getNamespaceMcastACLDbIDs(ns string, aclDir aclDirection, controller string) *libovsdbops.DbObjectIDs { + // namespaces ACL + return libovsdbops.NewDbObjectIDs(libovsdbops.ACLMulticastNamespace, controller, + map[libovsdbops.ExternalIDKey]string{ + libovsdbops.ObjectNameKey: ns, + libovsdbops.PolicyDirectionKey: string(aclDir), + }) +} + +func getMulticastPortGroupName(namespace string) string { + return hashedPortGroup(namespace) } // Creates a policy to allow multicast traffic within 'ns': @@ -88,19 +117,19 @@ func getMcastACLName(nsORpg, mcastSuffix string) string { // This matches only traffic originated by pods in 'ns' (based on the // namespace address set). func (oc *DefaultNetworkController) createMulticastAllowPolicy(ns string, nsInfo *namespaceInfo) error { - portGroupName := hashedPortGroup(ns) + portGroupName := getMulticastPortGroupName(ns) - aclT := lportEgressAfterLB - egressMatch := getACLMatch(portGroupName, getMulticastACLEgrMatch(), aclT) - egressACL := BuildACL(getMcastACLName(ns, "MulticastAllowEgress"), - types.DefaultMcastAllowPriority, egressMatch, nbdb.ACLActionAllow, nil, aclT, - getDefaultDenyPolicyExternalIDs(aclT)) + aclDir := aclEgress + egressMatch := getACLMatchFromACLDir(portGroupName, getMulticastACLEgrMatch(), aclDir) + dbIDs := getNamespaceMcastACLDbIDs(ns, aclDir, oc.controllerName) + aclPipeline := aclDirectionToACLPipeline(aclDir) + egressACL := BuildACLFromDbIDs(dbIDs, types.DefaultMcastAllowPriority, egressMatch, nbdb.ACLActionAllow, nil, aclPipeline) - aclT = lportIngress - ingressMatch := getACLMatch(portGroupName, getMulticastACLIgrMatch(nsInfo), aclT) - ingressACL := BuildACL(getMcastACLName(ns, "MulticastAllowIngress"), - types.DefaultMcastAllowPriority, ingressMatch, nbdb.ACLActionAllow, nil, aclT, - getDefaultDenyPolicyExternalIDs(aclT)) + aclDir = aclIngress + ingressMatch := getACLMatchFromACLDir(portGroupName, getMulticastACLIgrMatch(nsInfo), aclDir) + dbIDs = getNamespaceMcastACLDbIDs(ns, aclDir, oc.controllerName) + aclPipeline = aclDirectionToACLPipeline(aclDir) + ingressACL := BuildACLFromDbIDs(dbIDs, types.DefaultMcastAllowPriority, ingressMatch, nbdb.ACLActionAllow, nil, aclPipeline) acls := []*nbdb.ACL{egressACL, ingressACL} ops, err := libovsdbops.CreateOrUpdateACLsOps(oc.nbClient, nil, acls...) @@ -140,7 +169,7 @@ func (oc *DefaultNetworkController) createMulticastAllowPolicy(ns string, nsInfo } func deleteMulticastAllowPolicy(nbClient libovsdbclient.Client, ns string) error { - portGroupName := hashedPortGroup(ns) + portGroupName := getMulticastPortGroupName(ns) // ACLs referenced by the port group wil be deleted by db if there are no other references err := libovsdbops.DeletePortGroups(nbClient, portGroupName) if err != nil { @@ -158,28 +187,24 @@ func deleteMulticastAllowPolicy(nbClient libovsdbclient.Client, ns string) error // // Caller must hold the namespace's namespaceInfo object lock. func (oc *DefaultNetworkController) createDefaultDenyMulticastPolicy() error { - match := getMulticastACLMatch() - // By default deny any egress multicast traffic from any pod. This drops // IP multicast membership reports therefore denying any multicast traffic // to be forwarded to pods. - aclT := lportEgressAfterLB - egressACL := BuildACL(getMcastACLName(types.ClusterPortGroupName, "DefaultDenyMulticastEgress"), - types.DefaultMcastDenyPriority, match, nbdb.ACLActionDrop, nil, - aclT, getDefaultDenyPolicyExternalIDs(aclT)) - // By default deny any ingress multicast traffic to any pod. - aclT = lportIngress - ingressACL := BuildACL(getMcastACLName(types.ClusterPortGroupName, "DefaultDenyMulticastIngress"), - types.DefaultMcastDenyPriority, match, nbdb.ACLActionDrop, nil, - aclT, getDefaultDenyPolicyExternalIDs(aclT)) - - ops, err := libovsdbops.CreateOrUpdateACLsOps(oc.nbClient, nil, egressACL, ingressACL) + match := getMulticastACLMatch() + acls := make([]*nbdb.ACL, 0, 2) + for _, aclDir := range []aclDirection{aclEgress, aclIngress} { + dbIDs := getDefaultMcastACLDbIDs(mcastDefaultDenyID, aclDir, oc.controllerName) + aclPipeline := aclDirectionToACLPipeline(aclDir) + acl := BuildACLFromDbIDs(dbIDs, types.DefaultMcastDenyPriority, match, nbdb.ACLActionDrop, nil, aclPipeline) + acls = append(acls, acl) + } + ops, err := libovsdbops.CreateOrUpdateACLsOps(oc.nbClient, nil, acls...) if err != nil { return err } - ops, err = libovsdbops.AddACLsToPortGroupOps(oc.nbClient, ops, types.ClusterPortGroupName, egressACL, ingressACL) + ops, err = libovsdbops.AddACLsToPortGroupOps(oc.nbClient, ops, types.ClusterPortGroupName, acls...) if err != nil { return err } @@ -205,25 +230,21 @@ func (oc *DefaultNetworkController) createDefaultDenyMulticastPolicy() error { // Caller must hold the namespace's namespaceInfo object lock. func (oc *DefaultNetworkController) createDefaultAllowMulticastPolicy() error { mcastMatch := getMulticastACLMatch() + acls := make([]*nbdb.ACL, 0, 2) + for _, aclDir := range []aclDirection{aclEgress, aclIngress} { + match := getACLMatchFromACLDir(types.ClusterRtrPortGroupName, mcastMatch, aclDir) + dbIDs := getDefaultMcastACLDbIDs(mcastAllowInterNodeID, aclDir, oc.controllerName) + aclPipeline := aclDirectionToACLPipeline(aclDir) + acl := BuildACLFromDbIDs(dbIDs, types.DefaultMcastAllowPriority, match, nbdb.ACLActionAllow, nil, aclPipeline) + acls = append(acls, acl) + } - aclT := lportEgressAfterLB - egressMatch := getACLMatch(types.ClusterRtrPortGroupName, mcastMatch, aclT) - egressACL := BuildACL(getMcastACLName(types.ClusterRtrPortGroupName, "DefaultAllowMulticastEgress"), - types.DefaultMcastAllowPriority, egressMatch, nbdb.ACLActionAllow, nil, - aclT, getDefaultDenyPolicyExternalIDs(aclT)) - - aclT = lportIngress - ingressMatch := getACLMatch(types.ClusterRtrPortGroupName, mcastMatch, aclT) - ingressACL := BuildACL(getMcastACLName(types.ClusterRtrPortGroupName, "DefaultAllowMulticastIngress"), - types.DefaultMcastAllowPriority, ingressMatch, nbdb.ACLActionAllow, nil, - aclT, getDefaultDenyPolicyExternalIDs(aclT)) - - ops, err := libovsdbops.CreateOrUpdateACLsOps(oc.nbClient, nil, egressACL, ingressACL) + ops, err := libovsdbops.CreateOrUpdateACLsOps(oc.nbClient, nil, acls...) if err != nil { return err } - ops, err = libovsdbops.AddACLsToPortGroupOps(oc.nbClient, ops, types.ClusterRtrPortGroupName, egressACL, ingressACL) + ops, err = libovsdbops.AddACLsToPortGroupOps(oc.nbClient, ops, types.ClusterRtrPortGroupName, acls...) if err != nil { return err } @@ -236,16 +257,85 @@ func (oc *DefaultNetworkController) createDefaultAllowMulticastPolicy() error { return nil } +func (oc *DefaultNetworkController) disableMulticast() error { + // default mcast acls have ACLMulticastCluster type + predicateIDs := libovsdbops.NewDbObjectIDs(libovsdbops.ACLMulticastCluster, oc.controllerName, nil) + mcastAclPred := libovsdbops.GetPredicate[*nbdb.ACL](predicateIDs, nil) + mcastACLs, err := libovsdbops.FindACLsWithPredicate(oc.nbClient, mcastAclPred) + if err != nil { + return fmt.Errorf("unable to find default multicast ACLs: %v", err) + } + err = libovsdbops.DeleteACLsFromPortGroups(oc.nbClient, []string{types.ClusterRtrPortGroupName, types.ClusterPortGroupName}, + mcastACLs...) + if err != nil { + return fmt.Errorf("unable to delete default multicast acls: %v", err) + } + // run sync for empty namespaces list, this should delete namespaces objects + err = oc.syncNsMulticast(map[string]bool{}) + if err != nil { + return fmt.Errorf("unable to delete namespaced multicast objects: %v", err) + } + return nil +} + // podAddAllowMulticastPolicy adds the pod's logical switch port to the namespace's // multicast port group. Caller must hold the namespace's namespaceInfo object // lock. func podAddAllowMulticastPolicy(nbClient libovsdbclient.Client, ns string, portInfo *lpInfo) error { - return libovsdbops.AddPortsToPortGroup(nbClient, hashedPortGroup(ns), portInfo.uuid) + return libovsdbops.AddPortsToPortGroup(nbClient, getMulticastPortGroupName(ns), portInfo.uuid) } // podDeleteAllowMulticastPolicy removes the pod's logical switch port from the // namespace's multicast port group. Caller must hold the namespace's // namespaceInfo object lock. func podDeleteAllowMulticastPolicy(nbClient libovsdbclient.Client, ns string, portUUID string) error { - return libovsdbops.DeletePortsFromPortGroup(nbClient, hashedPortGroup(ns), portUUID) + return libovsdbops.DeletePortsFromPortGroup(nbClient, getMulticastPortGroupName(ns), portUUID) +} + +// syncNsMulticast finds and deletes stale multicast db entries for namespaces that don't exist anymore +func (oc *DefaultNetworkController) syncNsMulticast(k8sNamespaces map[string]bool) error { + // to find namespaces that have multicast enabled, we need to find corresponding port groups. + // since we can't filter multicast port groups specifically, find multicast ACLs, and then find + // port groups they are referenced from. + predicateIDs := libovsdbops.NewDbObjectIDs(libovsdbops.ACLMulticastNamespace, oc.controllerName, nil) + mcastAclPred := libovsdbops.GetPredicate[*nbdb.ACL](predicateIDs, nil) + mcastACLs, err := libovsdbops.FindACLsWithPredicate(oc.nbClient, mcastAclPred) + if err != nil { + return fmt.Errorf("unable to find multicast ACLs for namespaces: %v", err) + } + if len(mcastACLs) == 0 { + return nil + } + + mcastAclUUIDs := sets.Set[string]{} + for _, acl := range mcastACLs { + mcastAclUUIDs.Insert(acl.UUID) + } + staleNamespaces := []string{} + + // pg.ExternalIDs["name"] contains namespace (and pg.Name has hashed namespace) + pgPred := func(item *nbdb.PortGroup) bool { + for _, aclUUID := range item.ACLs { + if mcastAclUUIDs.Has(aclUUID) { + // add namespace to the stale list + if !k8sNamespaces[item.ExternalIDs["name"]] { + staleNamespaces = append(staleNamespaces, item.ExternalIDs["name"]) + } + } + } + return false + } + _, err = libovsdbops.FindPortGroupsWithPredicate(oc.nbClient, pgPred) + if err != nil { + return fmt.Errorf("unable to find multicast port groups: %v", err) + } + + for _, staleNs := range staleNamespaces { + if err = deleteMulticastAllowPolicy(oc.nbClient, staleNs); err != nil { + return fmt.Errorf("unable to delete multicast allow policy for stale ns %s: %v", staleNs, err) + } + } + klog.Infof("Sync multicast removed ACLs for %d stale namespaces", len(staleNamespaces)) + + return nil } diff --git a/go-controller/pkg/ovn/multicast_test.go b/go-controller/pkg/ovn/multicast_test.go index a9b3c053ba..c13dbba35c 100644 --- a/go-controller/pkg/ovn/multicast_test.go +++ b/go-controller/pkg/ovn/multicast_test.go @@ -2,11 +2,11 @@ package ovn import ( "context" - "fmt" "github.com/onsi/ginkgo" "github.com/onsi/gomega" "github.com/onsi/gomega/format" + "github.com/urfave/cli/v2" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdbops" @@ -15,10 +15,7 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" - "github.com/urfave/cli/v2" - v1 "k8s.io/api/core/v1" - knet "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -39,7 +36,15 @@ func getIpModes() []ipMode { } func ipModeStr(m ipMode) string { - return fmt.Sprintf("(IPv4 %t IPv6 %t)", m.IPv4Mode, m.IPv6Mode) + if m.IPv4Mode && m.IPv6Mode { + return "dualstack" + } else if m.IPv4Mode { + return "ipv4" + } else if m.IPv6Mode { + return "ipv6" + } else { + return "no IP mode set" + } } func setIpMode(m ipMode) { @@ -47,18 +52,151 @@ func setIpMode(m ipMode) { config.IPv6Mode = m.IPv6Mode } -type multicastPolicy struct{} +func getMulticastDefaultExpectedData(clusterPortGroup, clusterRtrPortGroup *nbdb.PortGroup) []libovsdb.TestData { + match := getMulticastACLMatch() + aclIDs := getDefaultMcastACLDbIDs(mcastDefaultDenyID, aclEgress, DefaultNetworkControllerName) + aclName := getACLName(aclIDs) + defaultDenyEgressACL := libovsdbops.BuildACL( + aclName, + nbdb.ACLDirectionFromLport, + types.DefaultMcastDenyPriority, + match, + nbdb.ACLActionDrop, + types.OvnACLLoggingMeter, + "", + false, + aclIDs.GetExternalIDs(), + map[string]string{ + "apply-after-lb": "true", + }, + ) + defaultDenyEgressACL.UUID = "defaultDenyEgressACL_UUID" + + aclIDs = getDefaultMcastACLDbIDs(mcastDefaultDenyID, aclIngress, DefaultNetworkControllerName) + aclName = getACLName(aclIDs) + defaultDenyIngressACL := libovsdbops.BuildACL( + aclName, + nbdb.ACLDirectionToLport, + types.DefaultMcastDenyPriority, + match, + nbdb.ACLActionDrop, + types.OvnACLLoggingMeter, + "", + false, + aclIDs.GetExternalIDs(), + nil, + ) + defaultDenyIngressACL.UUID = "defaultDenyIngressACL_UUID" + clusterPortGroup.ACLs = []string{defaultDenyEgressACL.UUID, defaultDenyIngressACL.UUID} + + aclIDs = getDefaultMcastACLDbIDs(mcastAllowInterNodeID, aclEgress, DefaultNetworkControllerName) + aclName = getACLName(aclIDs) + egressMatch := getACLMatchFromACLDir(types.ClusterRtrPortGroupName, match, aclEgress) + defaultAllowEgressACL := libovsdbops.BuildACL( + aclName, + nbdb.ACLDirectionFromLport, + types.DefaultMcastAllowPriority, + egressMatch, + nbdb.ACLActionAllow, + types.OvnACLLoggingMeter, + "", + false, + aclIDs.GetExternalIDs(), + map[string]string{ + "apply-after-lb": "true", + }, + ) + defaultAllowEgressACL.UUID = "defaultAllowEgressACL_UUID" + + aclIDs = getDefaultMcastACLDbIDs(mcastAllowInterNodeID, aclIngress, DefaultNetworkControllerName) + aclName = getACLName(aclIDs) + ingressMatch := getACLMatchFromACLDir(types.ClusterRtrPortGroupName, match, aclIngress) + defaultAllowIngressACL := libovsdbops.BuildACL( + aclName, + nbdb.ACLDirectionToLport, + types.DefaultMcastAllowPriority, + ingressMatch, + nbdb.ACLActionAllow, + types.OvnACLLoggingMeter, + "", + false, + aclIDs.GetExternalIDs(), + nil, + ) + defaultAllowIngressACL.UUID = "defaultAllowIngressACL_UUID" + clusterRtrPortGroup.ACLs = []string{defaultAllowEgressACL.UUID, defaultAllowIngressACL.UUID} + return []libovsdb.TestData{ + defaultDenyIngressACL, + defaultDenyEgressACL, + defaultAllowEgressACL, + defaultAllowIngressACL, + clusterPortGroup, + clusterRtrPortGroup, + } +} + +func getMulticastDefaultStaleData(clusterPortGroup, clusterRtrPortGroup *nbdb.PortGroup) []libovsdb.TestData { + testData := getMulticastDefaultExpectedData(clusterPortGroup, clusterRtrPortGroup) + defaultDenyIngressACL := testData[0].(*nbdb.ACL) + newName := joinACLName(types.ClusterPortGroupName, "DefaultDenyMulticastIngress") + defaultDenyIngressACL.Name = &newName + defaultDenyIngressACL.Options = nil + + defaultDenyEgressACL := testData[1].(*nbdb.ACL) + newName1 := joinACLName(types.ClusterPortGroupName, "DefaultDenyMulticastEgress") + defaultDenyEgressACL.Name = &newName1 + defaultDenyEgressACL.Options = nil -func (p multicastPolicy) getMulticastPolicyExpectedData(ns string, ports []string) []libovsdb.TestData { - pg_hash := hashedPortGroup(ns) - egressMatch := getACLMatch(pg_hash, getMulticastACLEgrMatch(), lportEgressAfterLB) + defaultAllowEgressACL := testData[2].(*nbdb.ACL) + newName2 := joinACLName(types.ClusterRtrPortGroupName, "DefaultAllowMulticastEgress") + defaultAllowEgressACL.Name = &newName2 + defaultAllowEgressACL.Options = nil + + defaultAllowIngressACL := testData[3].(*nbdb.ACL) + newName3 := joinACLName(types.ClusterRtrPortGroupName, "DefaultAllowMulticastIngress") + defaultAllowIngressACL.Name = &newName3 + defaultAllowIngressACL.Options = nil + + return []libovsdb.TestData{ + defaultDenyIngressACL, + defaultDenyEgressACL, + defaultAllowEgressACL, + defaultAllowIngressACL, + testData[4], + testData[5], + } +} + +func getDefaultPortGroups() (clusterPortGroup, clusterRtrPortGroup *nbdb.PortGroup) { + clusterPortGroup = &nbdb.PortGroup{ + UUID: types.ClusterPortGroupName + "-UUID", + Name: types.ClusterPortGroupName, + ExternalIDs: map[string]string{ + "name": types.ClusterPortGroupName, + }, + } + clusterRtrPortGroup = &nbdb.PortGroup{ + UUID: types.ClusterRtrPortGroupName + "-UUID", + Name: types.ClusterRtrPortGroupName, + ExternalIDs: map[string]string{ + "name": types.ClusterRtrPortGroupName, + }, + } + return +} + +func getMulticastPolicyExpectedData(ns string, ports []string) []libovsdb.TestData { + pg_hash := getMulticastPortGroupName(ns) + egressMatch := getACLMatchFromACLDir(pg_hash, getMulticastACLEgrMatch(), aclEgress) ip4AddressSet, ip6AddressSet := getNsAddrSetHashNames(ns) mcastMatch := getACLMatchAF(getMulticastACLIgrMatchV4(ip4AddressSet), getMulticastACLIgrMatchV6(ip6AddressSet)) - ingressMatch := getACLMatch(pg_hash, mcastMatch, lportIngress) + ingressMatch := getACLMatchFromACLDir(pg_hash, mcastMatch, aclIngress) + aclIDs := getNamespaceMcastACLDbIDs(ns, aclEgress, DefaultNetworkControllerName) + aclName := getACLName(aclIDs) egressACL := libovsdbops.BuildACL( - getMcastACLName(ns, "MulticastAllowEgress"), + aclName, nbdb.ACLDirectionFromLport, types.DefaultMcastAllowPriority, egressMatch, @@ -66,17 +204,17 @@ func (p multicastPolicy) getMulticastPolicyExpectedData(ns string, ports []strin types.OvnACLLoggingMeter, "", false, - map[string]string{ - defaultDenyPolicyTypeACLExtIdKey: string(knet.PolicyTypeEgress), - }, + aclIDs.GetExternalIDs(), map[string]string{ "apply-after-lb": "true", }, ) - egressACL.UUID = *egressACL.Name + "-UUID" + egressACL.UUID = ns + "mc-egress-UUID" + aclIDs = getNamespaceMcastACLDbIDs(ns, aclIngress, DefaultNetworkControllerName) + aclName = getACLName(aclIDs) ingressACL := libovsdbops.BuildACL( - getMcastACLName(ns, "MulticastAllowIngress"), + aclName, nbdb.ACLDirectionToLport, types.DefaultMcastAllowPriority, ingressMatch, @@ -84,12 +222,10 @@ func (p multicastPolicy) getMulticastPolicyExpectedData(ns string, ports []strin types.OvnACLLoggingMeter, "", false, - map[string]string{ - defaultDenyPolicyTypeACLExtIdKey: string(knet.PolicyTypeIngress), - }, + aclIDs.GetExternalIDs(), nil, ) - ingressACL.UUID = *ingressACL.Name + "-UUID" + ingressACL.UUID = ns + "mc-ingress-UUID" lsps := []*nbdb.LogicalSwitchPort{} for _, uuid := range ports { @@ -111,15 +247,88 @@ func (p multicastPolicy) getMulticastPolicyExpectedData(ns string, ports []strin } } -var _ = ginkgo.Describe("OVN NetworkPolicy Operations with IP Address Family", func() { +func getMulticastPolicyStaleData(ns string, ports []string) []libovsdb.TestData { + testData := getMulticastPolicyExpectedData(ns, ports) + + egressACL := testData[0].(*nbdb.ACL) + newName := joinACLName(ns, "MulticastAllowEgress") + egressACL.Name = &newName + egressACL.Options = nil + + ingressACL := testData[1].(*nbdb.ACL) + newName1 := joinACLName(ns, "MulticastAllowIngress") + ingressACL.Name = &newName1 + ingressACL.Options = nil + + return []libovsdb.TestData{ + egressACL, + ingressACL, + testData[2], + } +} + +func getNodeSwitch(nodeName string) []libovsdb.TestData { + return []libovsdb.TestData{ + &nbdb.LogicalSwitch{ + UUID: nodeName + "_UUID", + Name: nodeName, + }, + } +} + +func createTestPods(nodeName, namespace string, ipM ipMode) (pods []v1.Pod, tPods []testPod, tPodIPs []string) { + nPodTestV4 := newTPod( + nodeName, + "10.128.1.0/24", + "10.128.1.2", + "10.128.1.1", + "myPod1", + "10.128.1.3", + "0a:58:0a:80:01:03", + namespace, + ) + nPodTestV6 := newTPod( + nodeName, + "fd00:10:244::/64", + "fd00:10:244::2", + "fd00:10:244::1", + "myPod2", + "fd00:10:244::3", + "0a:58:dd:33:05:d8", + namespace, + ) + if ipM.IPv4Mode { + tPods = append(tPods, nPodTestV4) + tPodIPs = append(tPodIPs, nPodTestV4.podIP) + } + if ipM.IPv6Mode { + tPods = append(tPods, nPodTestV6) + tPodIPs = append(tPodIPs, nPodTestV6.podIP) + } + for _, tPod := range tPods { + pods = append(pods, *newPod(tPod.namespace, tPod.podName, tPod.nodeName, tPod.podIP)) + } + return +} + +func updateMulticast(fakeOvn *FakeOVN, ns *v1.Namespace, enable bool) { + if enable { + ns.Annotations[util.NsMulticastAnnotation] = "true" + } else { + ns.Annotations[util.NsMulticastAnnotation] = "false" + } + _, err := fakeOvn.fakeClient.KubeClient.CoreV1().Namespaces().Update(context.TODO(), ns, metav1.UpdateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) +} + +var _ = ginkgo.Describe("OVN Multicast with IP Address Family", func() { const ( namespaceName1 = "namespace1" - namespaceName2 = "namespace2" + nodeName = "node1" ) var ( app *cli.App fakeOvn *FakeOVN - initialDB libovsdb.TestSetup gomegaFormatMaxLength int ) @@ -136,13 +345,6 @@ var _ = ginkgo.Describe("OVN NetworkPolicy Operations with IP Address Family", f fakeOvn = NewFakeOVN() gomegaFormatMaxLength = format.MaxLength format.MaxLength = 0 - initialDB = libovsdb.TestSetup{ - NBData: []libovsdb.TestData{ - &nbdb.LogicalSwitch{ - Name: "node1", - }, - }, - } }) ginkgo.AfterEach(func() { @@ -150,6 +352,183 @@ var _ = ginkgo.Describe("OVN NetworkPolicy Operations with IP Address Family", f format.MaxLength = gomegaFormatMaxLength }) + ginkgo.Context("on startup", func() { + ginkgo.It("creates default Multicast ACLs", func() { + app.Action = func(ctx *cli.Context) error { + clusterPortGroup, clusterRtrPortGroup := getDefaultPortGroups() + fakeOvn.startWithDBSetup(libovsdb.TestSetup{ + NBData: []libovsdb.TestData{ + clusterPortGroup, + clusterRtrPortGroup, + }, + }) + // this is "if oc.multicastSupport" part of SetupMaster + err := fakeOvn.controller.createDefaultDenyMulticastPolicy() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.createDefaultAllowMulticastPolicy() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + gomega.Eventually(fakeOvn.nbClient).Should(libovsdb.HaveData( + getMulticastDefaultExpectedData(clusterPortGroup, clusterRtrPortGroup))) + return nil + } + err := app.Run([]string{app.Name}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + ginkgo.It("updates stale default Multicast ACLs", func() { + app.Action = func(ctx *cli.Context) error { + // start with stale ACLs + clusterPortGroup, clusterRtrPortGroup := getDefaultPortGroups() + fakeOvn.startWithDBSetup(libovsdb.TestSetup{ + NBData: getMulticastDefaultStaleData(clusterPortGroup, clusterRtrPortGroup), + }) + // this is "if oc.multicastSupport" part of SetupMaster + err := fakeOvn.controller.createDefaultDenyMulticastPolicy() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOvn.controller.createDefaultAllowMulticastPolicy() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + // check acls are updated + gomega.Eventually(fakeOvn.nbClient).Should(libovsdb.HaveData( + getMulticastDefaultExpectedData(clusterPortGroup, clusterRtrPortGroup))) + return nil + } + err := app.Run([]string{app.Name}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + ginkgo.It("cleans up Multicast resources when multicast is disabled", func() { + app.Action = func(ctx *cli.Context) error { + clusterPortGroup, clusterRtrPortGroup := getDefaultPortGroups() + initialData := getMulticastDefaultExpectedData(clusterPortGroup, clusterRtrPortGroup) + // test server doesn't delete de-referenced acls, so they will stay + expectedData := initialData[:len(initialData)-2] + clusterPortGroup, clusterRtrPortGroup = getDefaultPortGroups() + + nsData := getMulticastPolicyExpectedData(namespaceName1, nil) + initialData = append(initialData, nsData...) + expectedData = append(expectedData, nsData[:len(nsData)-1]...) + + // namespace is still present, but multicast support is disabled + namespace1 := *newNamespace(namespaceName1) + fakeOvn.startWithDBSetup(libovsdb.TestSetup{NBData: initialData}, + &v1.NamespaceList{ + Items: []v1.Namespace{ + namespace1, + }, + }, + ) + // this "if !oc.multicastSupport" part of SetupMaster + err := fakeOvn.controller.disableMulticast() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + // check acls are deleted when multicast is disabled + + expectedData = append(expectedData, clusterPortGroup, clusterRtrPortGroup) + gomega.Eventually(fakeOvn.nbClient).Should(libovsdb.HaveData( + expectedData)) + return nil + } + err := app.Run([]string{app.Name}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + ginkgo.It("creates namespace Multicast ACLs", func() { + app.Action = func(ctx *cli.Context) error { + clusterPortGroup, clusterRtrPortGroup := getDefaultPortGroups() + expectedData := getMulticastDefaultExpectedData(clusterPortGroup, clusterRtrPortGroup) + // namespace exists, but multicast acls do not + namespace1 := *newNamespace(namespaceName1) + namespace1.Annotations[util.NsMulticastAnnotation] = "true" + fakeOvn.startWithDBSetup(libovsdb.TestSetup{NBData: expectedData}, + &v1.NamespaceList{ + Items: []v1.Namespace{ + namespace1, + }, + }, + ) + err := fakeOvn.controller.WatchNamespaces() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + expectedData = append(expectedData, getMulticastPolicyExpectedData(namespaceName1, nil)...) + gomega.Eventually(fakeOvn.nbClient).Should(libovsdb.HaveData(expectedData)) + return nil + } + err := app.Run([]string{app.Name}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + ginkgo.It("updates stale namespace Multicast ACLs", func() { + app.Action = func(ctx *cli.Context) error { + // start with stale ACLs for existing namespace + clusterPortGroup, clusterRtrPortGroup := getDefaultPortGroups() + expectedData := getMulticastDefaultExpectedData(clusterPortGroup, clusterRtrPortGroup) + expectedData = append(expectedData, getMulticastPolicyStaleData(namespaceName1, nil)...) + namespace1 := *newNamespace(namespaceName1) + namespace1.Annotations[util.NsMulticastAnnotation] = "true" + fakeOvn.startWithDBSetup(libovsdb.TestSetup{NBData: expectedData}, + &v1.NamespaceList{ + Items: []v1.Namespace{ + namespace1, + }, + }, + ) + + err := fakeOvn.controller.WatchNamespaces() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + expectedData = getMulticastDefaultExpectedData(clusterPortGroup, clusterRtrPortGroup) + expectedData = append(expectedData, getMulticastPolicyExpectedData(namespaceName1, nil)...) + gomega.Eventually(fakeOvn.nbClient).Should(libovsdb.HaveData(expectedData)) + return nil + } + + err := app.Run([]string{app.Name}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + ginkgo.It("cleans up namespace Multicast ACLs when namespace doesn't exist", func() { + app.Action = func(ctx *cli.Context) error { + // start with stale ACLs + clusterPortGroup, clusterRtrPortGroup := getDefaultPortGroups() + initialData := getMulticastDefaultExpectedData(clusterPortGroup, clusterRtrPortGroup) + initialData = append(initialData, getMulticastPolicyExpectedData(namespaceName1, nil)...) + // namespace was deleted + fakeOvn.startWithDBSetup(libovsdb.TestSetup{NBData: initialData}) + + err := fakeOvn.controller.WatchNamespaces() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + // ACLs and port group should be deleted + // test server doesn't delete de-referenced acls, so they will stay + expectedData := initialData[:len(initialData)-1] + gomega.Eventually(fakeOvn.nbClient).Should(libovsdb.HaveData(expectedData)) + return nil + } + err := app.Run([]string{app.Name}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + ginkgo.It("cleans up namespace Multicast ACLs when multicast is disabled for namespace", func() { + app.Action = func(ctx *cli.Context) error { + // start with stale ACLs + clusterPortGroup, clusterRtrPortGroup := getDefaultPortGroups() + initialData := getMulticastDefaultExpectedData(clusterPortGroup, clusterRtrPortGroup) + initialData = append(initialData, getMulticastPolicyExpectedData(namespaceName1, nil)...) + namespace1 := *newNamespace(namespaceName1) + + fakeOvn.startWithDBSetup(libovsdb.TestSetup{NBData: initialData}, + &v1.NamespaceList{ + Items: []v1.Namespace{ + namespace1, + }, + }, + ) + + err := fakeOvn.controller.WatchNamespaces() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + // ACLs and port group should be deleted + // test server doesn't delete de-referenced acls, so they will stay + expectedData := initialData[:len(initialData)-1] + gomega.Eventually(fakeOvn.nbClient).Should(libovsdb.HaveData(expectedData)) + return nil + } + err := app.Run([]string{app.Name}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + }) + ginkgo.Context("during execution", func() { for _, m := range getIpModes() { m := m @@ -177,19 +556,14 @@ var _ = ginkgo.Describe("OVN NetworkPolicy Operations with IP Address Family", f gomega.Expect(ok).To(gomega.BeFalse()) // Enable multicast in the namespace. - mcastPolicy := multicastPolicy{} - expectedData := mcastPolicy.getMulticastPolicyExpectedData(namespace1.Name, nil) - ns.Annotations[util.NsMulticastAnnotation] = "true" - _, err = fakeOvn.fakeClient.KubeClient.CoreV1().Namespaces().Update(context.TODO(), ns, metav1.UpdateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + updateMulticast(fakeOvn, ns, true) + expectedData := getMulticastPolicyExpectedData(namespace1.Name, nil) gomega.Eventually(fakeOvn.nbClient).Should(libovsdb.HaveData(expectedData...)) // Disable multicast in the namespace. - ns.Annotations[util.NsMulticastAnnotation] = "false" - _, err = fakeOvn.fakeClient.KubeClient.CoreV1().Namespaces().Update(context.TODO(), ns, metav1.UpdateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - acls := expectedData[:len(expectedData)-1] - gomega.Eventually(fakeOvn.nbClient).Should(libovsdb.HaveData(acls)) + updateMulticast(fakeOvn, ns, false) + // test server doesn't delete de-referenced acls, so they will stay + gomega.Eventually(fakeOvn.nbClient).Should(libovsdb.HaveData(expectedData[:len(expectedData)-1]...)) return nil } @@ -200,43 +574,9 @@ var _ = ginkgo.Describe("OVN NetworkPolicy Operations with IP Address Family", f ginkgo.It("tests enabling multicast in a namespace with a pod "+ipModeStr(m), func() { app.Action = func(ctx *cli.Context) error { namespace1 := *newNamespace(namespaceName1) - nPodTestV4 := newTPod( - "node1", - "10.128.1.0/24", - "10.128.1.2", - "10.128.1.1", - "myPod1", - "10.128.1.3", - "0a:58:0a:80:01:03", - namespace1.Name, - ) - nPodTestV6 := newTPod( - "node1", - "fd00:10:244::/64", - "fd00:10:244::2", - "fd00:10:244::1", - "myPod2", - "fd00:10:244::3", - "0a:58:dd:33:05:d8", - namespace1.Name, - ) - var tPods []testPod - var tPodIPs []string - if m.IPv4Mode { - tPods = append(tPods, nPodTestV4) - tPodIPs = append(tPodIPs, nPodTestV4.podIP) - } - if m.IPv6Mode { - tPods = append(tPods, nPodTestV6) - tPodIPs = append(tPodIPs, nPodTestV6.podIP) - } - - var pods []v1.Pod - for _, tPod := range tPods { - pods = append(pods, *newPod(tPod.namespace, tPod.podName, tPod.nodeName, tPod.podIP)) - } + pods, tPods, tPodIPs := createTestPods(nodeName, namespaceName1, m) - fakeOvn.startWithDBSetup(initialDB, + fakeOvn.startWithDBSetup(libovsdb.TestSetup{NBData: getNodeSwitch(nodeName)}, &v1.NamespaceList{ Items: []v1.Namespace{ namespace1, @@ -249,28 +589,26 @@ var _ = ginkgo.Describe("OVN NetworkPolicy Operations with IP Address Family", f setIpMode(m) for _, tPod := range tPods { - tPod.populateLogicalSwitchCache(fakeOvn, getLogicalSwitchUUID(fakeOvn.controller.nbClient, "node1")) + tPod.populateLogicalSwitchCache(fakeOvn, getLogicalSwitchUUID(fakeOvn.controller.nbClient, nodeName)) } err := fakeOvn.controller.WatchNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - fakeOvn.controller.WatchPods() + err = fakeOvn.controller.WatchPods() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) ns, err := fakeOvn.fakeClient.KubeClient.CoreV1().Namespaces().Get(context.TODO(), namespace1.Name, metav1.GetOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) gomega.Expect(ns).NotTo(gomega.BeNil()) // Enable multicast in the namespace - mcastPolicy := multicastPolicy{} + updateMulticast(fakeOvn, ns, true) + // calculate expected data ports := []string{} for _, tPod := range tPods { ports = append(ports, tPod.portUUID) } - - expectedData := mcastPolicy.getMulticastPolicyExpectedData(namespace1.Name, ports) - expectedData = append(expectedData, getExpectedDataPodsAndSwitches(tPods, []string{"node1"})...) - ns.Annotations[util.NsMulticastAnnotation] = "true" - _, err = fakeOvn.fakeClient.KubeClient.CoreV1().Namespaces().Update(context.TODO(), ns, metav1.UpdateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + expectedData := getMulticastPolicyExpectedData(namespace1.Name, ports) + expectedData = append(expectedData, getExpectedDataPodsAndSwitches(tPods, []string{nodeName})...) gomega.Eventually(fakeOvn.nbClient).Should(libovsdb.HaveData(expectedData...)) fakeOvn.asf.ExpectAddressSetWithIPs(namespace1.Name, tPodIPs) return nil @@ -283,42 +621,14 @@ var _ = ginkgo.Describe("OVN NetworkPolicy Operations with IP Address Family", f ginkgo.It("tests adding a pod to a multicast enabled namespace "+ipModeStr(m), func() { app.Action = func(ctx *cli.Context) error { namespace1 := *newNamespace(namespaceName1) - nPodTestV4 := newTPod( - "node1", - "10.128.1.0/24", - "10.128.1.2", - "10.128.1.1", - "myPod1", - "10.128.1.3", - "0a:58:0a:80:01:03", - namespace1.Name, - ) - nPodTestV6 := newTPod( - "node1", - "fd00:10:244::/64", - "fd00:10:244::2", - "fd00:10:244::1", - "myPod2", - "fd00:10:244::3", - "0a:58:dd:33:05:d8", - namespace1.Name, - ) - var tPods []testPod - var tPodIPs []string + _, tPods, tPodIPs := createTestPods(nodeName, namespaceName1, m) + ports := []string{} - if m.IPv4Mode { - tPods = append(tPods, nPodTestV4) - tPodIPs = append(tPodIPs, nPodTestV4.podIP) - } - if m.IPv6Mode { - tPods = append(tPods, nPodTestV6) - tPodIPs = append(tPodIPs, nPodTestV6.podIP) - } for _, pod := range tPods { ports = append(ports, pod.portUUID) } - fakeOvn.startWithDBSetup(initialDB, + fakeOvn.startWithDBSetup(libovsdb.TestSetup{NBData: getNodeSwitch(nodeName)}, &v1.NamespaceList{ Items: []v1.Namespace{ namespace1, @@ -329,44 +639,40 @@ var _ = ginkgo.Describe("OVN NetworkPolicy Operations with IP Address Family", f err := fakeOvn.controller.WatchNamespaces() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - fakeOvn.controller.WatchPods() + err = fakeOvn.controller.WatchPods() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) ns, err := fakeOvn.fakeClient.KubeClient.CoreV1().Namespaces().Get(context.TODO(), namespace1.Name, metav1.GetOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) gomega.Expect(ns).NotTo(gomega.BeNil()) // Enable multicast in the namespace. - mcastPolicy := multicastPolicy{} - expectedData := mcastPolicy.getMulticastPolicyExpectedData(namespace1.Name, nil) - ns.Annotations[util.NsMulticastAnnotation] = "true" - _, err = fakeOvn.fakeClient.KubeClient.CoreV1().Namespaces().Update(context.TODO(), ns, metav1.UpdateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Eventually(fakeOvn.nbClient).Should(libovsdb.HaveData(append(expectedData, &nbdb.LogicalSwitch{ - UUID: "node1-UUID", - Name: "node1", - })...)) + updateMulticast(fakeOvn, ns, true) + // Check expected data without pods + expectedDataWithoutPods := getMulticastPolicyExpectedData(namespace1.Name, nil) + expectedDataWithoutPods = append(expectedDataWithoutPods, getNodeSwitch(nodeName)...) + gomega.Eventually(fakeOvn.nbClient).Should(libovsdb.HaveData(expectedDataWithoutPods)) + // Create pods for _, tPod := range tPods { - tPod.populateLogicalSwitchCache(fakeOvn, getLogicalSwitchUUID(fakeOvn.controller.nbClient, "node1")) + tPod.populateLogicalSwitchCache(fakeOvn, getLogicalSwitchUUID(fakeOvn.controller.nbClient, nodeName)) _, err = fakeOvn.fakeClient.KubeClient.CoreV1().Pods(tPod.namespace).Create(context.TODO(), newPod( tPod.namespace, tPod.podName, tPod.nodeName, tPod.podIP), metav1.CreateOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } + // Check pods were added fakeOvn.asf.EventuallyExpectAddressSetWithIPs(namespace1.Name, tPodIPs) - expectedDataWithPods := mcastPolicy.getMulticastPolicyExpectedData(namespace1.Name, ports) - expectedDataWithPods = append(expectedDataWithPods, getExpectedDataPodsAndSwitches(tPods, []string{"node1"})...) + expectedDataWithPods := getMulticastPolicyExpectedData(namespace1.Name, ports) + expectedDataWithPods = append(expectedDataWithPods, getExpectedDataPodsAndSwitches(tPods, []string{nodeName})...) gomega.Eventually(fakeOvn.nbClient).Should(libovsdb.HaveData(expectedDataWithPods...)) + // Delete the pod from the namespace. for _, tPod := range tPods { - // Delete the pod from the namespace. err = fakeOvn.fakeClient.KubeClient.CoreV1().Pods(tPod.namespace).Delete(context.TODO(), tPod.podName, *metav1.NewDeleteOptions(0)) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } fakeOvn.asf.EventuallyExpectEmptyAddressSetExist(namespace1.Name) - gomega.Eventually(fakeOvn.nbClient).Should(libovsdb.HaveData(append(expectedData, &nbdb.LogicalSwitch{ - UUID: "node1-UUID", - Name: "node1", - })...)) + gomega.Eventually(fakeOvn.nbClient).Should(libovsdb.HaveData(expectedDataWithoutPods)) return nil } diff --git a/go-controller/pkg/ovn/namespace.go b/go-controller/pkg/ovn/namespace.go index f0518f7443..df7b1d4d18 100644 --- a/go-controller/pkg/ovn/namespace.go +++ b/go-controller/pkg/ovn/namespace.go @@ -67,12 +67,16 @@ func getNamespaceAddrSetDbIDs(namespaceName, controller string) *libovsdbops.DbO // Upon failure, it may be invoked multiple times in order to avoid a pod restart. func (oc *DefaultNetworkController) syncNamespaces(namespaces []interface{}) error { expectedNs := make(map[string]bool) + nsWithMulticast := make(map[string]bool) for _, nsInterface := range namespaces { ns, ok := nsInterface.(*kapi.Namespace) if !ok { return fmt.Errorf("spurious object in syncNamespaces: %v", nsInterface) } expectedNs[ns.Name] = true + if isNamespaceMulticastEnabled(ns.Annotations) { + nsWithMulticast[ns.Name] = true + } } err := oc.addressSetFactory.ProcessEachAddressSet(oc.controllerName, libovsdbops.AddressSetNamespace, @@ -89,6 +93,9 @@ func (oc *DefaultNetworkController) syncNamespaces(namespaces []interface{}) err if err != nil { return fmt.Errorf("error in syncing namespaces: %v", err) } + if err = oc.syncNsMulticast(nsWithMulticast); err != nil { + return fmt.Errorf("error in syncing multicast for namespaces: %v", err) + } return nil } diff --git a/go-controller/pkg/ovn/policy.go b/go-controller/pkg/ovn/policy.go index 62f00ed3ab..3ea5b6019f 100644 --- a/go-controller/pkg/ovn/policy.go +++ b/go-controller/pkg/ovn/policy.go @@ -246,7 +246,7 @@ func (oc *DefaultNetworkController) updateStaleDefaultDenyACLNames(npType knet.P } // loop through the cleanUp map and per namespace update the first ACL's name and delete the rest for namespace, aclList := range cleanUpDefaultDeny { - var aclT aclType + var aclT aclPipelineType if aclList[0].Direction == nbdb.ACLDirectionToLport { aclT = lportIngress } else { @@ -459,7 +459,7 @@ func addAllowACLFromNode(nodeName string, mgmtPortIP net.IP, nbClient libovsdbcl return nil } -func getDefaultDenyPolicyACLName(ns string, aclT aclType) string { +func getDefaultDenyPolicyACLName(ns string, aclT aclPipelineType) string { var defaultDenySuffix string switch aclT { case lportIngress: @@ -472,7 +472,7 @@ func getDefaultDenyPolicyACLName(ns string, aclT aclType) string { return joinACLName(ns, defaultDenySuffix) } -func getDefaultDenyPolicyExternalIDs(aclT aclType) map[string]string { +func getDefaultDenyPolicyExternalIDs(aclT aclPipelineType) map[string]string { return map[string]string{defaultDenyPolicyTypeACLExtIdKey: string(aclTypeToPolicyType(aclT))} } @@ -484,7 +484,7 @@ func defaultDenyPortGroupName(namespace, gressSuffix string) string { return hashedPortGroup(namespace) + "_" + gressSuffix } -func buildDenyACLs(namespace, pg string, aclLogging *ACLLoggingLevels, aclT aclType) (denyACL, allowACL *nbdb.ACL) { +func buildDenyACLs(namespace, pg string, aclLogging *ACLLoggingLevels, aclT aclPipelineType) (denyACL, allowACL *nbdb.ACL) { denyMatch := getACLMatch(pg, "", aclT) allowMatch := getACLMatch(pg, arpAllowPolicyMatch, aclT) denyACL = BuildACL(getDefaultDenyPolicyACLName(namespace, aclT), types.DefaultDenyPriority, denyMatch, From cb4c4b3d6394b6f8ac8af3b4b539732da2dfd120 Mon Sep 17 00:00:00 2001 From: Nadia Pinaeva Date: Wed, 29 Mar 2023 13:38:48 +0200 Subject: [PATCH 04/40] Allow from node ACL: 1. Use dbIDs with new NetpolNodeOwnerType 2. Add aclSync for allow from node ACL 3. Rework AllowFromNode ACL unit tests to run everything for ipv4 and ipv6 Signed-off-by: Nadia Pinaeva --- .../pkg/libovsdbops/db_object_types.go | 9 + .../pkg/ovn/default_network_controller.go | 6 +- .../ovn/external_ids_syncer/acl/acl_sync.go | 66 ++++- .../external_ids_syncer/acl/acl_sync_test.go | 63 ++++- go-controller/pkg/ovn/gateway_test.go | 2 +- go-controller/pkg/ovn/master.go | 2 +- go-controller/pkg/ovn/policy.go | 25 +- go-controller/pkg/ovn/policy_test.go | 256 ++++++++++-------- 8 files changed, 300 insertions(+), 129 deletions(-) diff --git a/go-controller/pkg/libovsdbops/db_object_types.go b/go-controller/pkg/libovsdbops/db_object_types.go index 2ba8f936fa..938ae5fa71 100644 --- a/go-controller/pkg/libovsdbops/db_object_types.go +++ b/go-controller/pkg/libovsdbops/db_object_types.go @@ -19,6 +19,7 @@ const ( EgressServiceOwnerType ownerType = "EgressService" MulticastNamespaceOwnerType ownerType = "MulticastNS" MulticastClusterOwnerType ownerType = "MulticastCluster" + NetpolNodeOwnerType ownerType = "NetpolNode" // owner extra IDs, make sure to define only 1 ExternalIDKey for every string value PriorityKey ExternalIDKey = "priority" @@ -26,6 +27,7 @@ const ( GressIdxKey ExternalIDKey = "gress-index" AddressSetIPFamilyKey ExternalIDKey = "ip-family" TypeKey ExternalIDKey = "type" + IpKey ExternalIDKey = "ip" ) // ObjectIDsTypes should only be created here @@ -106,3 +108,10 @@ var ACLMulticastCluster = newObjectIDsType(acl, MulticastClusterOwnerType, []Ext // egress or ingress PolicyDirectionKey, }) + +var ACLNetpolNode = newObjectIDsType(acl, NetpolNodeOwnerType, []ExternalIDKey{ + // node name + ObjectNameKey, + // exact ip for management port, every node may have more than 1 management ip + IpKey, +}) diff --git a/go-controller/pkg/ovn/default_network_controller.go b/go-controller/pkg/ovn/default_network_controller.go index 0674bed87b..57efc6ca86 100644 --- a/go-controller/pkg/ovn/default_network_controller.go +++ b/go-controller/pkg/ovn/default_network_controller.go @@ -282,8 +282,12 @@ func (oc *DefaultNetworkController) Start(ctx context.Context) error { return fmt.Errorf("failed to sync address sets on controller init: %v", err) } + existingNodes, err := oc.kube.GetNodes() + if err != nil { + return fmt.Errorf("failed to get existing nodes: %w", err) + } aclSyncer := aclsyncer.NewACLSyncer(oc.nbClient, DefaultNetworkControllerName) - err = aclSyncer.SyncACLs() + err = aclSyncer.SyncACLs(existingNodes) if err != nil { return fmt.Errorf("failed to sync acls on controller init: %v", err) } diff --git a/go-controller/pkg/ovn/external_ids_syncer/acl/acl_sync.go b/go-controller/pkg/ovn/external_ids_syncer/acl/acl_sync.go index f706de03ea..ecc7e7ad29 100644 --- a/go-controller/pkg/ovn/external_ids_syncer/acl/acl_sync.go +++ b/go-controller/pkg/ovn/external_ids_syncer/acl/acl_sync.go @@ -8,10 +8,13 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdbops" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util/batching" + v1 "k8s.io/api/core/v1" knet "k8s.io/api/networking/v1" "k8s.io/klog/v2" + utilnet "k8s.io/utils/net" ) const ( @@ -39,7 +42,7 @@ func NewACLSyncer(nbClient libovsdbclient.Client, controllerName string) *aclSyn } } -func (syncer *aclSyncer) SyncACLs() error { +func (syncer *aclSyncer) SyncACLs(existingNodes *v1.NodeList) error { // stale acls don't have controller ID legacyAclPred := libovsdbops.GetNoOwnerPredicate[*nbdb.ACL]() legacyACLs, err := libovsdbops.FindACLsWithPredicate(syncer.nbClient, legacyAclPred) @@ -50,8 +53,14 @@ func (syncer *aclSyncer) SyncACLs() error { return nil } - updatedACLs := syncer.updateStaleMulticastACLsDbIDs(legacyACLs) - klog.Infof("Found %d stale multicast ACLs", len(updatedACLs)) + var updatedACLs []*nbdb.ACL + multicastACLs := syncer.updateStaleMulticastACLsDbIDs(legacyACLs) + klog.Infof("Found %d stale multicast ACLs", len(multicastACLs)) + updatedACLs = append(updatedACLs, multicastACLs...) + + allowFromNodeACLs := syncer.updateStaleNetpolNodeACLs(legacyACLs, existingNodes.Items) + klog.Infof("Found %d stale allow from node ACLs", len(allowFromNodeACLs)) + updatedACLs = append(updatedACLs, allowFromNodeACLs...) err = batching.Batch[*nbdb.ACL](syncer.txnBatchSize, updatedACLs, func(batchACLs []*nbdb.ACL) error { return libovsdbops.CreateOrUpdateACLs(syncer.nbClient, batchACLs...) @@ -136,3 +145,54 @@ func (syncer *aclSyncer) updateStaleMulticastACLsDbIDs(legacyACLs []*nbdb.ACL) [ } return updatedACLs } + +func (syncer *aclSyncer) getAllowFromNodeACLDbIDs(nodeName, mgmtPortIP string) *libovsdbops.DbObjectIDs { + return libovsdbops.NewDbObjectIDs(libovsdbops.ACLNetpolNode, syncer.controllerName, + map[libovsdbops.ExternalIDKey]string{ + libovsdbops.ObjectNameKey: nodeName, + libovsdbops.IpKey: mgmtPortIP, + }) +} + +// updateStaleNetpolNodeACLs updates allow from node ACLs, that don't have new ExternalIDs based +// on DbObjectIDs set. Allow from node acls are applied on the node switch, therefore the cleanup for deleted is not needed, +// since acl will be deleted toegther with the node switch. +func (syncer *aclSyncer) updateStaleNetpolNodeACLs(legacyACLs []*nbdb.ACL, existingNodes []v1.Node) []*nbdb.ACL { + // ACL to allow traffic from host via management port has no name or ExternalIDs + // The only way to find it is by exact match + type aclInfo struct { + nodeName string + ip string + } + matchToNode := map[string]aclInfo{} + for _, node := range existingNodes { + hostSubnets, err := util.ParseNodeHostSubnetAnnotation(&node, types.DefaultNetworkName) + if err != nil { + klog.Warningf("Couldn't parse hostSubnet annotation for node %s: %v", node.Name, err) + continue + } + for _, hostSubnet := range hostSubnets { + mgmtIfAddr := util.GetNodeManagementIfAddr(hostSubnet) + ipFamily := "ip4" + if utilnet.IsIPv6(mgmtIfAddr.IP) { + ipFamily = "ip6" + } + match := fmt.Sprintf("%s.src==%s", ipFamily, mgmtIfAddr.IP.String()) + matchToNode[match] = aclInfo{ + nodeName: node.Name, + ip: mgmtIfAddr.IP.String(), + } + } + } + updatedACLs := []*nbdb.ACL{} + for _, acl := range legacyACLs { + if _, ok := matchToNode[acl.Match]; ok { + aclInfo := matchToNode[acl.Match] + dbIDs := syncer.getAllowFromNodeACLDbIDs(aclInfo.nodeName, aclInfo.ip) + // Update ExternalIDs and Name based on new dbIndex + acl.ExternalIDs = dbIDs.GetExternalIDs() + updatedACLs = append(updatedACLs, acl) + } + } + return updatedACLs +} diff --git a/go-controller/pkg/ovn/external_ids_syncer/acl/acl_sync_test.go b/go-controller/pkg/ovn/external_ids_syncer/acl/acl_sync_test.go index a9333b349e..c2368de036 100644 --- a/go-controller/pkg/ovn/external_ids_syncer/acl/acl_sync_test.go +++ b/go-controller/pkg/ovn/external_ids_syncer/acl/acl_sync_test.go @@ -1,12 +1,15 @@ package acl import ( + "encoding/json" "github.com/onsi/ginkgo" "github.com/onsi/gomega" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdbops" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" libovsdbtest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/libovsdb" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "strings" ) @@ -15,18 +18,19 @@ type aclSync struct { after *libovsdbops.DbObjectIDs } -func testSyncerWithData(data []aclSync, controllerName string) { +func testSyncerWithData(data []aclSync, controllerName string, initialDbState []libovsdbtest.TestData, + existingNodes []v1.Node) { // create initial db setup - dbSetup := libovsdbtest.TestSetup{NBData: []libovsdbtest.TestData{}} + dbSetup := libovsdbtest.TestSetup{NBData: initialDbState} for _, asSync := range data { - asSync.before.UUID = *asSync.before.Name + "-UUID" + asSync.before.UUID = asSync.after.String() + "-UUID" dbSetup.NBData = append(dbSetup.NBData, asSync.before) } libovsdbOvnNBClient, _, libovsdbCleanup, err := libovsdbtest.NewNBSBTestHarness(dbSetup) defer libovsdbCleanup.Cleanup() gomega.Expect(err).NotTo(gomega.HaveOccurred()) // create expected data using addressSetFactory - expectedDbState := []libovsdbtest.TestData{} + expectedDbState := initialDbState for _, aclSync := range data { acl := aclSync.before acl.ExternalIDs = aclSync.after.GetExternalIDs() @@ -34,7 +38,7 @@ func testSyncerWithData(data []aclSync, controllerName string) { } // run sync syncer := NewACLSyncer(libovsdbOvnNBClient, controllerName) - err = syncer.SyncACLs() + err = syncer.SyncACLs(&v1.NodeList{Items: existingNodes}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) // check results gomega.Eventually(libovsdbOvnNBClient).Should(libovsdbtest.HaveData(expectedDbState)) @@ -164,6 +168,53 @@ var _ = ginkgo.Describe("OVN ACL Syncer", func() { after: syncerToBuildData.getNamespaceMcastACLDbIDs(namespace1, "Ingress"), }, } - testSyncerWithData(testData, controllerName) + testSyncerWithData(testData, controllerName, []libovsdbtest.TestData{}, nil) + }) + ginkgo.It("updates allow from node acls", func() { + nodeName := "node1" + ipv4MgmtIP := "10.244.0.2" + ipv6MgmtIP := "fd02:0:0:2::2" + + testData := []aclSync{ + // ipv4 acl + { + before: libovsdbops.BuildACL( + "", + nbdb.ACLDirectionToLport, + types.DefaultAllowPriority, + "ip4.src=="+ipv4MgmtIP, + nbdb.ACLActionAllowRelated, + types.OvnACLLoggingMeter, + "", + false, + nil, + nil), + after: syncerToBuildData.getAllowFromNodeACLDbIDs(nodeName, ipv4MgmtIP), + }, + // ipv6 acl + { + before: libovsdbops.BuildACL( + "", + nbdb.ACLDirectionToLport, + types.DefaultAllowPriority, + "ip6.src=="+ipv6MgmtIP, + nbdb.ACLActionAllowRelated, + types.OvnACLLoggingMeter, + "", + false, + nil, + nil), + after: syncerToBuildData.getAllowFromNodeACLDbIDs(nodeName, ipv6MgmtIP), + }, + } + hostSubnets := map[string][]string{types.DefaultNetworkName: {"10.244.0.0/24", "fd02:0:0:2::2895/64"}} + bytes, err := json.Marshal(hostSubnets) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + existingNodes := []v1.Node{{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + Annotations: map[string]string{"k8s.ovn.org/node-subnets": string(bytes)}, + }}} + testSyncerWithData(testData, controllerName, []libovsdbtest.TestData{}, existingNodes) }) }) diff --git a/go-controller/pkg/ovn/gateway_test.go b/go-controller/pkg/ovn/gateway_test.go index 43de59d01c..7b7fa5486d 100644 --- a/go-controller/pkg/ovn/gateway_test.go +++ b/go-controller/pkg/ovn/gateway_test.go @@ -267,7 +267,7 @@ func generateGatewayInitExpectedNB(testData []libovsdb.TestData, expectedOVNClus testData = append(testData, expectedOVNClusterRouter) if len(nodeMgmtPortIP) != 0 { - _, nodeACL := generateAllowFromNodeData(nodeName, nodeMgmtPortIP) + nodeACL := getAllowFromNodeExpectedACL(nodeName, nodeMgmtPortIP, nil, DefaultNetworkControllerName) testData = append(testData, nodeACL) expectedNodeSwitch.ACLs = append(expectedNodeSwitch.ACLs, nodeACL.UUID) diff --git a/go-controller/pkg/ovn/master.go b/go-controller/pkg/ovn/master.go index fd6950d87d..4f0e66f2f9 100644 --- a/go-controller/pkg/ovn/master.go +++ b/go-controller/pkg/ovn/master.go @@ -262,7 +262,7 @@ func (oc *DefaultNetworkController) syncNodeManagementPort(node *kapi.Node, host mgmtIfAddr := util.GetNodeManagementIfAddr(hostSubnet) addresses += " " + mgmtIfAddr.IP.String() - if err := addAllowACLFromNode(node.Name, mgmtIfAddr.IP, oc.nbClient); err != nil { + if err := oc.addAllowACLFromNode(node.Name, mgmtIfAddr.IP); err != nil { return err } diff --git a/go-controller/pkg/ovn/policy.go b/go-controller/pkg/ovn/policy.go index 3ea5b6019f..5ebc67b1e7 100644 --- a/go-controller/pkg/ovn/policy.go +++ b/go-controller/pkg/ovn/policy.go @@ -427,31 +427,38 @@ func (oc *DefaultNetworkController) syncNetworkPolicies(networkPolicies []interf return nil } -func getAllowFromNodeACLName() string { - return "" +func getAllowFromNodeACLDbIDs(nodeName, mgmtPortIP, controller string) *libovsdbops.DbObjectIDs { + return libovsdbops.NewDbObjectIDs(libovsdbops.ACLNetpolNode, controller, + map[libovsdbops.ExternalIDKey]string{ + libovsdbops.ObjectNameKey: nodeName, + libovsdbops.IpKey: mgmtPortIP, + }) } -func addAllowACLFromNode(nodeName string, mgmtPortIP net.IP, nbClient libovsdbclient.Client) error { +// There is no delete function for this ACL type, because the ACL is applied on a node switch. +// When the node is deleted, switch will be deleted by the node sync, and the dependent ACLs will be +// garbage-collected. +func (oc *DefaultNetworkController) addAllowACLFromNode(nodeName string, mgmtPortIP net.IP) error { ipFamily := "ip4" if utilnet.IsIPv6(mgmtPortIP) { ipFamily = "ip6" } match := fmt.Sprintf("%s.src==%s", ipFamily, mgmtPortIP.String()) + dbIDs := getAllowFromNodeACLDbIDs(nodeName, mgmtPortIP.String(), oc.controllerName) + nodeACL := BuildACLFromDbIDs(dbIDs, types.DefaultAllowPriority, match, + nbdb.ACLActionAllowRelated, nil, lportIngress) - nodeACL := BuildACL(getAllowFromNodeACLName(), types.DefaultAllowPriority, match, - nbdb.ACLActionAllowRelated, nil, lportIngress, nil) - - ops, err := libovsdbops.CreateOrUpdateACLsOps(nbClient, nil, nodeACL) + ops, err := libovsdbops.CreateOrUpdateACLsOps(oc.nbClient, nil, nodeACL) if err != nil { return fmt.Errorf("failed to create or update ACL %v: %v", nodeACL, err) } - ops, err = libovsdbops.AddACLsToLogicalSwitchOps(nbClient, ops, nodeName, nodeACL) + ops, err = libovsdbops.AddACLsToLogicalSwitchOps(oc.nbClient, ops, nodeName, nodeACL) if err != nil { return fmt.Errorf("failed to add ACL %v to switch %s: %v", nodeACL, nodeName, err) } - _, err = libovsdbops.TransactAndCheck(nbClient, ops) + _, err = libovsdbops.TransactAndCheck(oc.nbClient, ops) if err != nil { return err } diff --git a/go-controller/pkg/ovn/policy_test.go b/go-controller/pkg/ovn/policy_test.go index c3f7181717..b79477b58e 100644 --- a/go-controller/pkg/ovn/policy_test.go +++ b/go-controller/pkg/ovn/policy_test.go @@ -3,6 +3,7 @@ package ovn import ( "context" "fmt" + "net" "sort" "strconv" "time" @@ -2221,20 +2222,68 @@ func asMatch(hashedAddressSets []string) string { return match } -var _ = ginkgo.Describe("OVN NetworkPolicy Low-Level Operations", func() { +func getAllowFromNodeExpectedACL(nodeName, mgmtIP string, logicalSwitch *nbdb.LogicalSwitch, controllerName string) *nbdb.ACL { + var ipFamily = "ip4" + if utilnet.IsIPv6(ovntest.MustParseIP(mgmtIP)) { + ipFamily = "ip6" + } + match := fmt.Sprintf("%s.src==%s", ipFamily, mgmtIP) + + dbIDs := getAllowFromNodeACLDbIDs(nodeName, mgmtIP, controllerName) + nodeACL := libovsdbops.BuildACL( + getACLName(dbIDs), + nbdb.ACLDirectionToLport, + types.DefaultAllowPriority, + match, + nbdb.ACLActionAllowRelated, + types.OvnACLLoggingMeter, + "", + false, + dbIDs.GetExternalIDs(), + nil) + nodeACL.UUID = dbIDs.String() + "-UUID" + if logicalSwitch != nil { + logicalSwitch.ACLs = []string{nodeACL.UUID} + } + return nodeACL +} + +func getAllowFromNodeStaleACL(nodeName, mgmtIP string, logicalSwitch *nbdb.LogicalSwitch, controllerName string) *nbdb.ACL { + acl := getAllowFromNodeExpectedACL(nodeName, mgmtIP, logicalSwitch, controllerName) + newName := "" + acl.Name = &newName + + return acl +} + +// here only low-level operation are tested (directly calling updateStaleNetpolNodeACLs) +var _ = ginkgo.Describe("OVN AllowFromNode ACL low-level operations", func() { var ( - asFactory *addressset.FakeAddressSetFactory - nbCleanup *libovsdbtest.Cleanup + nbCleanup *libovsdbtest.Cleanup + logicalSwitch *nbdb.LogicalSwitch ) const ( - nodeName = "node1" - ipv4MgmtIP = "192.168.10.10" - ipv6MgmtIP = "fd01::1234" + nodeName = "node1" + ipv4MgmtIP = "192.168.10.10" + ipv6MgmtIP = "fd01::1234" + controllerName = DefaultNetworkControllerName ) + getFakeController := func(nbClient libovsdbclient.Client) *DefaultNetworkController { + controller := &DefaultNetworkController{ + BaseNetworkController: BaseNetworkController{controllerName: controllerName}, + } + controller.nbClient = nbClient + return controller + } + ginkgo.BeforeEach(func() { nbCleanup = nil + logicalSwitch = &nbdb.LogicalSwitch{ + Name: nodeName, + UUID: nodeName + "_UUID", + } }) ginkgo.AfterEach(func() { @@ -2243,6 +2292,99 @@ var _ = ginkgo.Describe("OVN NetworkPolicy Low-Level Operations", func() { } }) + for _, ipMode := range []string{"ipv4", "ipv6"} { + var mgmtIP string + if ipMode == "ipv4" { + mgmtIP = ipv4MgmtIP + } else { + mgmtIP = ipv6MgmtIP + } + ginkgo.It(fmt.Sprintf("sync existing ACLs on startup, %s mode", ipMode), func() { + // mock existing management port + mgmtPortMAC := "0a:58:0a:01:01:02" + mgmtPort := &nbdb.LogicalSwitchPort{ + Name: types.K8sPrefix + nodeName, + UUID: types.K8sPrefix + nodeName + "-UUID", + Type: "", + Options: nil, + Addresses: []string{mgmtPortMAC + " " + mgmtIP}, + } + logicalSwitch.Ports = []string{mgmtPort.UUID} + initialNbdb := libovsdbtest.TestSetup{ + NBData: []libovsdbtest.TestData{ + getAllowFromNodeStaleACL(nodeName, mgmtIP, logicalSwitch, controllerName), + logicalSwitch, + mgmtPort, + }, + } + var err error + var nbClient libovsdbclient.Client + nbClient, nbCleanup, err = libovsdbtest.NewNBTestHarness(initialNbdb, nil) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + fakeController := getFakeController(nbClient) + err = fakeController.addAllowACLFromNode(nodeName, net.ParseIP(mgmtIP)) + + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + expectedData := []libovsdb.TestData{ + logicalSwitch, + mgmtPort, + getAllowFromNodeExpectedACL(nodeName, mgmtIP, logicalSwitch, controllerName), + } + gomega.Expect(nbClient).Should(libovsdb.HaveData(expectedData...)) + }) + ginkgo.It(fmt.Sprintf("adding an existing ACL to the node switch, %s mode", ipMode), func() { + initialNbdb := libovsdbtest.TestSetup{ + NBData: []libovsdbtest.TestData{ + logicalSwitch, + getAllowFromNodeExpectedACL(nodeName, mgmtIP, nil, controllerName), + }, + } + var err error + var nbClient libovsdbclient.Client + nbClient, nbCleanup, err = libovsdbtest.NewNBTestHarness(initialNbdb, nil) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + fakeController := getFakeController(nbClient) + err = fakeController.addAllowACLFromNode(nodeName, ovntest.MustParseIP(mgmtIP)) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + expectedData := []libovsdb.TestData{ + logicalSwitch, + getAllowFromNodeExpectedACL(nodeName, mgmtIP, logicalSwitch, controllerName), + } + gomega.Expect(nbClient).Should(libovsdb.HaveData(expectedData...)) + }) + + ginkgo.It(fmt.Sprintf("creating new ACL and adding it to node switch, %s mode", ipMode), func() { + initialNbdb := libovsdbtest.TestSetup{ + NBData: []libovsdbtest.TestData{ + logicalSwitch, + }, + } + + var err error + var nbClient libovsdbclient.Client + nbClient, nbCleanup, err = libovsdbtest.NewNBTestHarness(initialNbdb, nil) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + fakeController := getFakeController(nbClient) + err = fakeController.addAllowACLFromNode(nodeName, ovntest.MustParseIP(mgmtIP)) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + expectedData := []libovsdb.TestData{ + logicalSwitch, + getAllowFromNodeExpectedACL(nodeName, mgmtIP, logicalSwitch, controllerName), + } + gomega.Expect(nbClient).Should(libovsdb.HaveData(expectedData...)) + }) + } +}) + +var _ = ginkgo.Describe("OVN NetworkPolicy Low-Level Operations", func() { + var asFactory *addressset.FakeAddressSetFactory + buildExpectedIngressPeerNSv4ACL := func(gp *gressPolicy, pgName string, asDbIDses []*libovsdbops.DbObjectIDs, aclLogging *ACLLoggingLevels) *nbdb.ACL { name := getGressPolicyACLName(gp.policyNamespace, gp.policyName, gp.idx) @@ -2385,106 +2527,4 @@ var _ = ginkgo.Describe("OVN NetworkPolicy Low-Level Operations", func() { // deleting again is no-op gomega.Expect(gp.delNamespaceAddressSet(four.GetObjectID(libovsdbops.ObjectNameKey))).To(gomega.BeFalse()) }) - - ginkgo.It("Tests AddAllowACLFromNode", func() { - ginkgo.By("adding an existing ACL to the node switch", func() { - initialNbdb := libovsdbtest.TestSetup{ - NBData: []libovsdbtest.TestData{ - &nbdb.LogicalSwitch{ - Name: nodeName, - }, - &nbdb.ACL{ - Match: "ip4.src==" + ipv4MgmtIP, - Priority: types.DefaultAllowPriority, - Action: "allow-related", - Direction: nbdb.ACLDirectionToLport, - }, - }, - } - - var err error - var nbClient libovsdbclient.Client - nbClient, nbCleanup, err = libovsdbtest.NewNBTestHarness(initialNbdb, nil) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - err = addAllowACLFromNode(nodeName, ovntest.MustParseIP(ipv4MgmtIP), nbClient) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - testNode, nodeACL := generateAllowFromNodeData(nodeName, ipv4MgmtIP) - expectedData := []libovsdb.TestData{ - testNode, - nodeACL, - } - gomega.Expect(nbClient).Should(libovsdb.HaveData(expectedData...)) - }) - - ginkgo.By("creating an ipv4 ACL and adding it to node switch", func() { - initialNbdb := libovsdbtest.TestSetup{ - NBData: []libovsdbtest.TestData{ - &nbdb.LogicalSwitch{ - Name: nodeName, - }, - }, - } - - var err error - var nbClient libovsdbclient.Client - nbClient, nbCleanup, err = libovsdbtest.NewNBTestHarness(initialNbdb, nil) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - err = addAllowACLFromNode(nodeName, ovntest.MustParseIP(ipv4MgmtIP), nbClient) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - testNode, nodeACL := generateAllowFromNodeData(nodeName, ipv4MgmtIP) - expectedData := []libovsdb.TestData{ - testNode, - nodeACL, - } - gomega.Expect(nbClient).Should(libovsdb.HaveData(expectedData...)) - }) - ginkgo.By("creating an ipv6 ACL and adding it to node switch", func() { - initialNbdb := libovsdbtest.TestSetup{ - NBData: []libovsdbtest.TestData{ - &nbdb.LogicalSwitch{ - Name: nodeName, - }, - }, - } - - var err error - var nbClient libovsdbclient.Client - nbClient, nbCleanup, err = libovsdbtest.NewNBTestHarness(initialNbdb, nil) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - err = addAllowACLFromNode(nodeName, ovntest.MustParseIP(ipv6MgmtIP), nbClient) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - testNode, nodeACL := generateAllowFromNodeData(nodeName, ipv6MgmtIP) - expectedData := []libovsdb.TestData{ - testNode, - nodeACL, - } - gomega.Expect(nbClient).Should(libovsdb.HaveData(expectedData...)) - }) - }) }) - -func generateAllowFromNodeData(nodeName, mgmtIP string) (nodeSwitch *nbdb.LogicalSwitch, acl *nbdb.ACL) { - var ipFamily = "ip4" - if utilnet.IsIPv6(ovntest.MustParseIP(mgmtIP)) { - ipFamily = "ip6" - } - - match := fmt.Sprintf("%s.src==%s", ipFamily, mgmtIP) - - nodeACL := libovsdbops.BuildACL(getAllowFromNodeACLName(), nbdb.ACLDirectionToLport, types.DefaultAllowPriority, match, "allow-related", types.OvnACLLoggingMeter, "", false, nil, nil) - nodeACL.UUID = "nodeACL-UUID" - - testNode := &nbdb.LogicalSwitch{ - UUID: nodeName + "-UUID", - Name: nodeName, - ACLs: []string{nodeACL.UUID}, - } - - return testNode, nodeACL -} From 2aab19eef998d10addddfaca80d1c98ee4e3e457 Mon Sep 17 00:00:00 2001 From: Nadia Pinaeva Date: Wed, 29 Mar 2023 14:59:25 +0200 Subject: [PATCH 05/40] Update gress policy ACLs to use new dbIndex. Move gress-related constants from policy.go to gress_policy.go. Use old externalIDs for syncs only. policy test "correctly retries recreating a network policy with the same name" last expected data was updated, since newly generated ACLs Signed-off-by: Nadia Pinaeva --- .../pkg/libovsdbops/db_object_types.go | 25 ++- go-controller/pkg/ovn/acl.go | 6 + .../ovn/external_ids_syncer/acl/acl_sync.go | 99 ++++++++++ .../external_ids_syncer/acl/acl_sync_test.go | 124 ++++++++++++ go-controller/pkg/ovn/gress_policy.go | 182 +++++++++--------- go-controller/pkg/ovn/policy.go | 78 ++++---- go-controller/pkg/ovn/policy_test.go | 126 ++++-------- 7 files changed, 425 insertions(+), 215 deletions(-) diff --git a/go-controller/pkg/libovsdbops/db_object_types.go b/go-controller/pkg/libovsdbops/db_object_types.go index 938ae5fa71..1b3acbe216 100644 --- a/go-controller/pkg/libovsdbops/db_object_types.go +++ b/go-controller/pkg/libovsdbops/db_object_types.go @@ -9,7 +9,8 @@ const ( // owner types EgressFirewallDNSOwnerType ownerType = "EgressFirewallDNS" EgressQoSOwnerType ownerType = "EgressQoS" - // only used for cleanup now, as the stale owner of network policy address sets + // NetworkPolicyOwnerType is deprecated for address sets, should only be used for sync. + // New owner of network policy address sets, is PodSelectorOwnerType. NetworkPolicyOwnerType ownerType = "NetworkPolicy" NetpolDefaultOwnerType ownerType = "NetpolDefault" PodSelectorOwnerType ownerType = "PodSelector" @@ -28,6 +29,8 @@ const ( AddressSetIPFamilyKey ExternalIDKey = "ip-family" TypeKey ExternalIDKey = "type" IpKey ExternalIDKey = "ip" + PortPolicyIndexKey ExternalIDKey = "port-policy-index" + IpBlockIndexKey ExternalIDKey = "ip-block-index" ) // ObjectIDsTypes should only be created here @@ -115,3 +118,23 @@ var ACLNetpolNode = newObjectIDsType(acl, NetpolNodeOwnerType, []ExternalIDKey{ // exact ip for management port, every node may have more than 1 management ip IpKey, }) + +// ACLNetworkPolicy define a unique index for every network policy ACL. +// ingress/egress + NetworkPolicy[In/E]gressRule idx - defines given gressPolicy. +// ACLs are created for every gp.portPolicies: +// - for empty policy (no selectors and no ip blocks) - empty ACL (see allIPsMatch) +// OR +// - all selector-based peers ACL +// - for every IPBlock +1 ACL +// Therefore unique id for a given gressPolicy is portPolicy idx + IPBlock idx +// (empty policy and all selector-based peers ACLs will have idx=-1) +var ACLNetworkPolicy = newObjectIDsType(acl, NetworkPolicyOwnerType, []ExternalIDKey{ + // policy namespace+name + ObjectNameKey, + // egress or ingress + PolicyDirectionKey, + // gress rule index + GressIdxKey, + PortPolicyIndexKey, + IpBlockIndexKey, +}) diff --git a/go-controller/pkg/ovn/acl.go b/go-controller/pkg/ovn/acl.go index 3cba2a25cd..9cb49cd3d2 100644 --- a/go-controller/pkg/ovn/acl.go +++ b/go-controller/pkg/ovn/acl.go @@ -80,7 +80,13 @@ func hashedPortGroup(s string) string { // Therefore, "feature" as "EF" for EgressFirewall and "NP" for network policy goes first, then namespace, // then acl-related info. func getACLName(dbIDs *libovsdbops.DbObjectIDs) string { + t := dbIDs.GetIDsType() aclName := "" + switch { + case t.IsSameType(libovsdbops.ACLNetworkPolicy): + aclName = "NP:" + dbIDs.GetObjectID(libovsdbops.ObjectNameKey) + ":" + dbIDs.GetObjectID(libovsdbops.PolicyDirectionKey) + + ":" + dbIDs.GetObjectID(libovsdbops.GressIdxKey) + } return fmt.Sprintf("%.63s", aclName) } diff --git a/go-controller/pkg/ovn/external_ids_syncer/acl/acl_sync.go b/go-controller/pkg/ovn/external_ids_syncer/acl/acl_sync.go index ecc7e7ad29..226908d545 100644 --- a/go-controller/pkg/ovn/external_ids_syncer/acl/acl_sync.go +++ b/go-controller/pkg/ovn/external_ids_syncer/acl/acl_sync.go @@ -2,6 +2,7 @@ package acl import ( "fmt" + "strconv" "strings" libovsdbclient "github.com/ovn-org/libovsdb/client" @@ -21,6 +22,14 @@ const ( defaultDenyPolicyTypeACLExtIdKey = "default-deny-policy-type" mcastDefaultDenyID = "DefaultDeny" mcastAllowInterNodeID = "AllowInterNode" + l4MatchACLExtIdKey = "l4Match" + ipBlockCIDRACLExtIdKey = "ipblock_cidr" + namespaceACLExtIdKey = "namespace" + policyACLExtIdKey = "policy" + policyTypeACLExtIdKey = "policy_type" + policyTypeNumACLExtIdKey = "%s_num" + noneMatch = "None" + emptyIdx = -1 ) type aclSyncer struct { @@ -62,6 +71,13 @@ func (syncer *aclSyncer) SyncACLs(existingNodes *v1.NodeList) error { klog.Infof("Found %d stale allow from node ACLs", len(allowFromNodeACLs)) updatedACLs = append(updatedACLs, allowFromNodeACLs...) + gressPolicyACLs, err := syncer.updateStaleGressPolicies(legacyACLs) + if err != nil { + return fmt.Errorf("failed to update gress policy ACLs: %w", err) + } + klog.Infof("Found %d stale gress ACLs", len(gressPolicyACLs)) + updatedACLs = append(updatedACLs, gressPolicyACLs...) + err = batching.Batch[*nbdb.ACL](syncer.txnBatchSize, updatedACLs, func(batchACLs []*nbdb.ACL) error { return libovsdbops.CreateOrUpdateACLs(syncer.nbClient, batchACLs...) }) @@ -196,3 +212,86 @@ func (syncer *aclSyncer) updateStaleNetpolNodeACLs(legacyACLs []*nbdb.ACL, exist } return updatedACLs } + +func (syncer *aclSyncer) getNetpolGressACLDbIDs(policyNamespace, policyName, policyType string, + gressIdx, portPolicyIdx, ipBlockIdx int) *libovsdbops.DbObjectIDs { + return libovsdbops.NewDbObjectIDs(libovsdbops.ACLNetworkPolicy, syncer.controllerName, + map[libovsdbops.ExternalIDKey]string{ + // policy namespace+name + libovsdbops.ObjectNameKey: policyNamespace + ":" + policyName, + // egress or ingress + libovsdbops.PolicyDirectionKey: policyType, + // gress rule index + libovsdbops.GressIdxKey: strconv.Itoa(gressIdx), + // acls are created for every gp.portPolicies: + // - for empty policy (no selectors and no ip blocks) - empty ACL + // OR + // - all selector-based peers ACL + // - for every IPBlock +1 ACL + // Therefore unique id for given gressPolicy is portPolicy idx + IPBlock idx + // (empty policy and all selector-based peers ACLs will have idx=-1) + libovsdbops.PortPolicyIndexKey: strconv.Itoa(portPolicyIdx), + libovsdbops.IpBlockIndexKey: strconv.Itoa(ipBlockIdx), + }) +} + +func (syncer *aclSyncer) updateStaleGressPolicies(legacyACLs []*nbdb.ACL) (updatedACLs []*nbdb.ACL, err error) { + if len(legacyACLs) == 0 { + return + } + // for every gress policy build mapping to count port policies. + // l4MatchACLExtIdKey was previously assigned based on port policy, + // we can just assign idx to every port Policy and make sure there are no equal ACLs. + gressPolicyPortCount := map[string]map[string]int{} + for _, acl := range legacyACLs { + if acl.ExternalIDs[policyTypeACLExtIdKey] == "" { + // not gress ACL + continue + } + policyNamespace := acl.ExternalIDs[namespaceACLExtIdKey] + policyName := acl.ExternalIDs[policyACLExtIdKey] + policyType := acl.ExternalIDs[policyTypeACLExtIdKey] + idxKey := fmt.Sprintf(policyTypeNumACLExtIdKey, policyType) + idx, err := strconv.Atoi(acl.ExternalIDs[idxKey]) + if err != nil { + return nil, fmt.Errorf("unable to parse gress policy idx %s: %v", + acl.ExternalIDs[idxKey], err) + } + var ipBlockIdx int + // ipBlockCIDRACLExtIdKey is "false" for non-ipBlock ACLs. + // Then for the first ipBlock in a given gress policy it is "true", + // and for the rest of them is idx+1 + if acl.ExternalIDs[ipBlockCIDRACLExtIdKey] == "true" { + ipBlockIdx = 0 + } else if acl.ExternalIDs[ipBlockCIDRACLExtIdKey] == "false" { + ipBlockIdx = emptyIdx + } else { + ipBlockIdx, err = strconv.Atoi(acl.ExternalIDs[ipBlockCIDRACLExtIdKey]) + if err != nil { + return nil, fmt.Errorf("unable to parse gress policy ipBlockCIDRACLExtIdKey %s: %v", + acl.ExternalIDs[ipBlockCIDRACLExtIdKey], err) + } + ipBlockIdx -= 1 + } + gressACLID := strings.Join([]string{policyNamespace, policyName, policyType, fmt.Sprintf("%d", idx)}, "_") + if gressPolicyPortCount[gressACLID] == nil { + gressPolicyPortCount[gressACLID] = map[string]int{} + } + var portIdx int + l4Match := acl.ExternalIDs[l4MatchACLExtIdKey] + if l4Match == noneMatch { + portIdx = emptyIdx + } else { + if _, ok := gressPolicyPortCount[gressACLID][l4Match]; !ok { + // this l4MatchACLExtIdKey is new for given gressPolicy, assign the next idx to it + gressPolicyPortCount[gressACLID][l4Match] = len(gressPolicyPortCount[gressACLID]) + } + portIdx = gressPolicyPortCount[gressACLID][l4Match] + } + dbIDs := syncer.getNetpolGressACLDbIDs(policyNamespace, policyName, + policyType, idx, portIdx, ipBlockIdx) + acl.ExternalIDs = dbIDs.GetExternalIDs() + updatedACLs = append(updatedACLs, acl) + } + return +} diff --git a/go-controller/pkg/ovn/external_ids_syncer/acl/acl_sync_test.go b/go-controller/pkg/ovn/external_ids_syncer/acl/acl_sync_test.go index c2368de036..17b972ba1a 100644 --- a/go-controller/pkg/ovn/external_ids_syncer/acl/acl_sync_test.go +++ b/go-controller/pkg/ovn/external_ids_syncer/acl/acl_sync_test.go @@ -2,6 +2,7 @@ package acl import ( "encoding/json" + "fmt" "github.com/onsi/ginkgo" "github.com/onsi/gomega" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdbops" @@ -9,6 +10,7 @@ import ( libovsdbtest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/libovsdb" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" v1 "k8s.io/api/core/v1" + knet "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "strings" ) @@ -217,4 +219,126 @@ var _ = ginkgo.Describe("OVN ACL Syncer", func() { }}} testSyncerWithData(testData, controllerName, []libovsdbtest.TestData{}, existingNodes) }) + ginkgo.It("updates gress policy acls", func() { + policyNamespace := "policyNamespace" + policyName := "policyName" + testData := []aclSync{ + { + before: libovsdbops.BuildACL( + policyNamespace+"_"+policyName+"_0", + nbdb.ACLDirectionToLport, + types.DefaultAllowPriority, + "ip4.dst == 10.244.1.5/32 && inport == @a2653181086423119552", + nbdb.ACLActionAllowRelated, + types.OvnACLLoggingMeter, + "", + false, + map[string]string{ + l4MatchACLExtIdKey: "tcp && 6380<=tcp.dst<=7000", + ipBlockCIDRACLExtIdKey: "true", + namespaceACLExtIdKey: policyNamespace, + policyACLExtIdKey: policyName, + policyTypeACLExtIdKey: string(knet.PolicyTypeEgress), + fmt.Sprintf(policyTypeNumACLExtIdKey, knet.PolicyTypeEgress): "0", + }, + nil, + ), + after: syncerToBuildData.getNetpolGressACLDbIDs(policyNamespace, policyName, string(knet.PolicyTypeEgress), + 0, 0, 0), + }, + { + before: libovsdbops.BuildACL( + policyNamespace+"_"+policyName+"_0", + nbdb.ACLDirectionToLport, + types.DefaultAllowPriority, + "ip4.dst == 10.244.1.5/32 && inport == @a2653181086423119552", + nbdb.ACLActionAllowRelated, + types.OvnACLLoggingMeter, + "", + false, + map[string]string{ + l4MatchACLExtIdKey: "tcp && 6380<=tcp.dst<=7000", + ipBlockCIDRACLExtIdKey: "2", + namespaceACLExtIdKey: policyNamespace, + policyACLExtIdKey: policyName, + policyTypeACLExtIdKey: string(knet.PolicyTypeEgress), + fmt.Sprintf(policyTypeNumACLExtIdKey, knet.PolicyTypeEgress): "0", + }, + nil, + ), + after: syncerToBuildData.getNetpolGressACLDbIDs(policyNamespace, policyName, string(knet.PolicyTypeEgress), + 0, 0, 1), + }, + { + before: libovsdbops.BuildACL( + policyNamespace+"_"+policyName+"_0", + nbdb.ACLDirectionToLport, + types.DefaultAllowPriority, + "ip4.dst == 10.244.1.5/32 && inport == @a2653181086423119552", + nbdb.ACLActionAllowRelated, + types.OvnACLLoggingMeter, + "", + false, + map[string]string{ + l4MatchACLExtIdKey: "tcp && 1<=tcp.dst<=3", + ipBlockCIDRACLExtIdKey: "true", + namespaceACLExtIdKey: policyNamespace, + policyACLExtIdKey: policyName, + policyTypeACLExtIdKey: string(knet.PolicyTypeEgress), + fmt.Sprintf(policyTypeNumACLExtIdKey, knet.PolicyTypeEgress): "0", + }, + nil, + ), + after: syncerToBuildData.getNetpolGressACLDbIDs(policyNamespace, policyName, string(knet.PolicyTypeEgress), + 0, 1, 0), + }, + { + before: libovsdbops.BuildACL( + policyNamespace+"_"+policyName+"_0", + nbdb.ACLDirectionToLport, + types.DefaultAllowPriority, + "ip4.dst == 10.244.1.5/32 && inport == @a2653181086423119552", + nbdb.ACLActionAllowRelated, + types.OvnACLLoggingMeter, + "", + false, + map[string]string{ + l4MatchACLExtIdKey: "tcp && 1<=tcp.dst<=3", + ipBlockCIDRACLExtIdKey: "2", + namespaceACLExtIdKey: policyNamespace, + policyACLExtIdKey: policyName, + policyTypeACLExtIdKey: string(knet.PolicyTypeEgress), + fmt.Sprintf(policyTypeNumACLExtIdKey, knet.PolicyTypeEgress): "0", + }, + nil, + ), + after: syncerToBuildData.getNetpolGressACLDbIDs(policyNamespace, policyName, string(knet.PolicyTypeEgress), + 0, 1, 1), + }, + { + before: libovsdbops.BuildACL( + policyNamespace+"_"+policyName+"_0", + nbdb.ACLDirectionFromLport, + types.DefaultAllowPriority, + "(ip4.src == {$a3733136965153973077} || (ip4.src == 169.254.169.5 && ip4.dst == {$a3733136965153973077})) && outport == @a2653181086423119552", + nbdb.ACLActionAllowRelated, + types.OvnACLLoggingMeter, + "", + false, + map[string]string{ + l4MatchACLExtIdKey: noneMatch, + ipBlockCIDRACLExtIdKey: "false", + namespaceACLExtIdKey: policyNamespace, + policyACLExtIdKey: policyName, + policyTypeACLExtIdKey: string(knet.PolicyTypeIngress), + fmt.Sprintf(policyTypeNumACLExtIdKey, knet.PolicyTypeIngress): "0", + }, + nil, + ), + after: syncerToBuildData.getNetpolGressACLDbIDs(policyNamespace, policyName, string(knet.PolicyTypeIngress), + 0, emptyIdx, emptyIdx), + }, + } + testSyncerWithData(testData, controllerName, []libovsdbtest.TestData{}, nil) + }) }) diff --git a/go-controller/pkg/ovn/gress_policy.go b/go-controller/pkg/ovn/gress_policy.go index 7ffa8bb1d3..55c61b415a 100644 --- a/go-controller/pkg/ovn/gress_policy.go +++ b/go-controller/pkg/ovn/gress_policy.go @@ -8,6 +8,7 @@ import ( "sync" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdbops" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" addressset "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/address_set" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" @@ -18,6 +19,8 @@ import ( const ( noneMatch = "None" + // emptyIdx is used to create ACL for gressPolicy that doesn't have ports or ipBlocks + emptyIdx = -1 ) type gressPolicy struct { @@ -25,6 +28,7 @@ type gressPolicy struct { policyNamespace string policyName string policyType knet.PolicyType + aclPipeline aclPipelineType idx int // peerVxAddressSets include PodSelectorAddressSet names, and address sets for selected namespaces @@ -42,7 +46,7 @@ type gressPolicy struct { // the rule in question. portPolicies []*portPolicy - ipBlock []*knet.IPBlock + ipBlocks []*knet.IPBlock // set to true for stateless network policies (stateless acls), otherwise set to false isNetPolStateless bool @@ -81,6 +85,7 @@ func newGressPolicy(policyType knet.PolicyType, idx int, namespace, name, contro policyNamespace: namespace, policyName: name, policyType: policyType, + aclPipeline: policyTypeToAclPipeline(policyType), idx: idx, peerV4AddressSets: &sync.Map{}, peerV6AddressSets: &sync.Map{}, @@ -130,7 +135,7 @@ func (gp *gressPolicy) addPortPolicy(portJSON *knet.NetworkPolicyPort) { } func (gp *gressPolicy) addIPBlock(ipblockJSON *knet.IPBlock) { - gp.ipBlock = append(gp.ipBlock, ipblockJSON) + gp.ipBlocks = append(gp.ipBlocks, ipblockJSON) } // getL3MatchFromAddressSet may return empty string, which means that there are no address sets selected for giver @@ -168,7 +173,7 @@ func (gp *gressPolicy) getL3MatchFromAddressSet() string { return match } -func constructEmptyMatchString() string { +func allIPsMatch() string { switch { case config.IPv4Mode && config.IPv6Mode: return "(ip4 || ip6)" @@ -180,13 +185,34 @@ func constructEmptyMatchString() string { } func (gp *gressPolicy) getMatchFromIPBlock(lportMatch, l4Match string) []string { - var ipBlockMatches []string + var direction string if gp.policyType == knet.PolicyTypeIngress { - ipBlockMatches = constructIPBlockStringsForACL("src", gp.ipBlock, lportMatch, l4Match) + direction = "src" } else { - ipBlockMatches = constructIPBlockStringsForACL("dst", gp.ipBlock, lportMatch, l4Match) + direction = "dst" + } + var matchStrings []string + var matchStr, ipVersion string + for _, ipBlock := range gp.ipBlocks { + if utilnet.IsIPv6CIDRString(ipBlock.CIDR) { + ipVersion = "ip6" + } else { + ipVersion = "ip4" + } + if len(ipBlock.Except) == 0 { + matchStr = fmt.Sprintf("%s.%s == %s", ipVersion, direction, ipBlock.CIDR) + } else { + matchStr = fmt.Sprintf("%s.%s == %s && %s.%s != {%s}", ipVersion, direction, ipBlock.CIDR, + ipVersion, direction, strings.Join(ipBlock.Except, ", ")) + } + if l4Match == noneMatch { + matchStr = fmt.Sprintf("%s && %s", matchStr, lportMatch) + } else { + matchStr = fmt.Sprintf("%s && %s && %s", matchStr, l4Match, lportMatch) + } + matchStrings = append(matchStrings, matchStr) } - return ipBlockMatches + return matchStrings } // addNamespaceAddressSet adds a namespace address set to the gress policy. @@ -246,7 +272,7 @@ func (gp *gressPolicy) delNamespaceAddressSet(name string) bool { func (gp *gressPolicy) isEmpty() bool { // empty gress allows all, it is empty if there are no ipBlocks and no peerSelectors - return !gp.hasPeerSelector && len(gp.ipBlock) == 0 + return !gp.hasPeerSelector && len(gp.ipBlocks) == 0 } // buildLocalPodACLs builds the ACLs that implement the gress policy's rules to the @@ -256,50 +282,66 @@ func (gp *gressPolicy) isEmpty() bool { // since creation, or are safe for concurrent use like peerVXAddressSets func (gp *gressPolicy) buildLocalPodACLs(portGroupName string, aclLogging *ACLLoggingLevels) (createdACLs []*nbdb.ACL, skippedACLs []*nbdb.ACL) { - var lportMatch, match, l3Match string + var lportMatch string if gp.policyType == knet.PolicyTypeIngress { lportMatch = fmt.Sprintf("outport == @%s", portGroupName) } else { lportMatch = fmt.Sprintf("inport == @%s", portGroupName) } - - var l4Matches []string + var portPolicyIdxs []int if len(gp.portPolicies) == 0 { - l4Matches = append(l4Matches, noneMatch) + portPolicyIdxs = []int{emptyIdx} } else { - l4Matches = make([]string, 0, len(gp.portPolicies)) - for _, port := range gp.portPolicies { - l4Match, err := port.getL4Match() + portPolicyIdxs = make([]int, 0, len(gp.portPolicies)) + for i := 0; i < len(gp.portPolicies); i++ { + portPolicyIdxs = append(portPolicyIdxs, i) + } + } + var l4Match string + var err error + action := nbdb.ACLActionAllowRelated + if gp.isNetPolStateless { + action = nbdb.ACLActionAllowStateless + } + for _, portPolIdx := range portPolicyIdxs { + if portPolIdx != emptyIdx { + portPol := gp.portPolicies[portPolIdx] + l4Match, err = portPol.getL4Match() if err != nil { continue } - l4Matches = append(l4Matches, l4Match) + } else { + l4Match = noneMatch } - } - for _, l4Match := range l4Matches { - if len(gp.ipBlock) > 0 { + + if len(gp.ipBlocks) > 0 { // Add ACL allow rule for IPBlock CIDR - cidrMatches := gp.getMatchFromIPBlock(lportMatch, l4Match) - for i, cidrMatch := range cidrMatches { - acl := gp.buildACLAllow(cidrMatch, l4Match, i+1, aclLogging) + ipBlockMatches := gp.getMatchFromIPBlock(lportMatch, l4Match) + for ipBlockIdx, ipBlockMatch := range ipBlockMatches { + aclIDs := gp.getNetpolACLDbIDs(portPolIdx, ipBlockIdx) + acl := BuildACLFromDbIDs(aclIDs, types.DefaultAllowPriority, ipBlockMatch, action, + aclLogging, gp.aclPipeline) createdACLs = append(createdACLs, acl) } } // if there are pod/namespace selector, then allow packets from/to that address_set or // if the NetworkPolicyPeer is empty, then allow from all sources or to all destinations. if gp.hasPeerSelector || gp.isEmpty() { + var l3Match, addrSetMatch string if gp.isEmpty() { - l3Match = constructEmptyMatchString() + l3Match = allIPsMatch() } else { l3Match = gp.getL3MatchFromAddressSet() } if l4Match == noneMatch { - match = fmt.Sprintf("%s && %s", l3Match, lportMatch) + addrSetMatch = fmt.Sprintf("%s && %s", l3Match, lportMatch) } else { - match = fmt.Sprintf("%s && %s && %s", l3Match, l4Match, lportMatch) + addrSetMatch = fmt.Sprintf("%s && %s && %s", l3Match, l4Match, lportMatch) } - acl := gp.buildACLAllow(match, l4Match, 0, aclLogging) + aclIDs := gp.getNetpolACLDbIDs(portPolIdx, emptyIdx) + acl := BuildACLFromDbIDs(aclIDs, types.DefaultAllowPriority, addrSetMatch, action, + aclLogging, gp.aclPipeline) if l3Match == "" { // if l3Match is empty, then no address sets are selected for a given gressPolicy. // fortunately l3 match is not a part of externalIDs, that means that we can find @@ -313,70 +355,36 @@ func (gp *gressPolicy) buildLocalPodACLs(portGroupName string, aclLogging *ACLLo return } -func getGressPolicyACLName(ns, policyName string, idx int) string { - return joinACLName(ns, policyName, fmt.Sprintf("%v", idx)) +func getACLPolicyKey(policyNamespace, policyName string) string { + return policyNamespace + ":" + policyName } -// buildACLAllow builds an allow-related ACL for a given match -func (gp *gressPolicy) buildACLAllow(match, l4Match string, ipBlockCIDR int, aclLogging *ACLLoggingLevels) *nbdb.ACL { - aclT := policyTypeToAclPipeline(gp.policyType) - priority := types.DefaultAllowPriority - action := nbdb.ACLActionAllowRelated - if gp.isNetPolStateless { - action = nbdb.ACLActionAllowStateless - } - aclName := getGressPolicyACLName(gp.policyNamespace, gp.policyName, gp.idx) - - // For backward compatibility with existing ACLs, we use "ipblock_cidr=false" for - // non-ipblock ACLs and "ipblock_cidr=true" for the first ipblock ACL in a policy, - // but then number them after that. - var ipBlockCIDRString string - switch ipBlockCIDR { - case 0: - ipBlockCIDRString = "false" - case 1: - ipBlockCIDRString = "true" - default: - ipBlockCIDRString = strconv.FormatInt(int64(ipBlockCIDR), 10) - } - - policyTypeNum := fmt.Sprintf(policyTypeNumACLExtIdKey, gp.policyType) - policyTypeIndex := strconv.FormatInt(int64(gp.idx), 10) - - externalIds := map[string]string{ - l4MatchACLExtIdKey: l4Match, - ipBlockCIDRACLExtIdKey: ipBlockCIDRString, - namespaceACLExtIdKey: gp.policyNamespace, - policyACLExtIdKey: gp.policyName, - policyTypeACLExtIdKey: string(gp.policyType), - policyTypeNum: policyTypeIndex, +func parseACLPolicyKey(aclPolicyKey string) (string, string, error) { + s := strings.Split(aclPolicyKey, ":") + if len(s) != 2 { + return "", "", fmt.Errorf("failed to parse network policy acl key %s, "+ + "expected format :", aclPolicyKey) } - acl := BuildACL(aclName, priority, match, action, aclLogging, aclT, externalIds) - return acl + return s[0], s[1], nil } -func constructIPBlockStringsForACL(direction string, ipBlocks []*knet.IPBlock, lportMatch, l4Match string) []string { - var matchStrings []string - var matchStr, ipVersion string - for _, ipBlock := range ipBlocks { - if utilnet.IsIPv6CIDRString(ipBlock.CIDR) { - ipVersion = "ip6" - } else { - ipVersion = "ip4" - } - if len(ipBlock.Except) == 0 { - matchStr = fmt.Sprintf("%s.%s == %s", ipVersion, direction, ipBlock.CIDR) - - } else { - matchStr = fmt.Sprintf("%s.%s == %s && %s.%s != {%s}", ipVersion, direction, ipBlock.CIDR, - ipVersion, direction, strings.Join(ipBlock.Except, ", ")) - } - if l4Match == noneMatch { - matchStr = fmt.Sprintf("%s && %s", matchStr, lportMatch) - } else { - matchStr = fmt.Sprintf("%s && %s && %s", matchStr, l4Match, lportMatch) - } - matchStrings = append(matchStrings, matchStr) - } - return matchStrings +func (gp *gressPolicy) getNetpolACLDbIDs(portPolicyIdx, ipBlockIdx int) *libovsdbops.DbObjectIDs { + return libovsdbops.NewDbObjectIDs(libovsdbops.ACLNetworkPolicy, gp.controllerName, + map[libovsdbops.ExternalIDKey]string{ + // policy namespace+name + libovsdbops.ObjectNameKey: getACLPolicyKey(gp.policyNamespace, gp.policyName), + // egress or ingress + libovsdbops.PolicyDirectionKey: string(gp.policyType), + // gress rule index + libovsdbops.GressIdxKey: strconv.Itoa(gp.idx), + // acls are created for every gp.portPolicies: + // - for empty policy (no selectors and no ip blocks) - empty ACL + // OR + // - all selector-based peers ACL + // - for every IPBlock +1 ACL + // Therefore unique id for given gressPolicy is portPolicy idx + IPBlock idx + // (empty policy and all selector-based peers ACLs will have idx=-1) + libovsdbops.PortPolicyIndexKey: strconv.Itoa(portPolicyIdx), + libovsdbops.IpBlockIndexKey: strconv.Itoa(ipBlockIdx), + }) } diff --git a/go-controller/pkg/ovn/policy.go b/go-controller/pkg/ovn/policy.go index 5ebc67b1e7..b1dae44eae 100644 --- a/go-controller/pkg/ovn/policy.go +++ b/go-controller/pkg/ovn/policy.go @@ -29,18 +29,6 @@ import ( const ( // defaultDenyPolicyTypeACLExtIdKey external ID key for default deny policy type defaultDenyPolicyTypeACLExtIdKey = "default-deny-policy-type" - // l4MatchACLExtIdKey external ID key for L4 Match on 'gress policy ACLs - l4MatchACLExtIdKey = "l4Match" - // ipBlockCIDRACLExtIdKey external ID key for IP block CIDR on 'gress policy ACLs - ipBlockCIDRACLExtIdKey = "ipblock_cidr" - // namespaceACLExtIdKey external ID key for namespace on 'gress policy ACLs - namespaceACLExtIdKey = "namespace" - // policyACLExtIdKey external ID key for policy name on 'gress policy ACLs - policyACLExtIdKey = "policy" - // policyACLExtKey external ID key for policy type on 'gress policy ACLs - policyTypeACLExtIdKey = "policy_type" - // policyTypeNumACLExtIdKey external ID key for policy index by type on 'gress policy ACLs - policyTypeNumACLExtIdKey = "%s_num" // ingressDefaultDenySuffix is the suffix used when creating the ingress port group for a namespace ingressDefaultDenySuffix = "ingressDefaultDeny" // egressDefaultDenySuffix is the suffix used when creating the ingress port group for a namespace @@ -302,36 +290,42 @@ func (oc *DefaultNetworkController) syncNetworkPolicies(networkPolicies []interf } // cleanup port groups based on acl search - p := func(item *nbdb.ACL) bool { - return item.ExternalIDs[policyACLExtIdKey] != "" || item.ExternalIDs[defaultDenyPolicyTypeACLExtIdKey] != "" - } + // netpol-owned acls + predicateIDs := libovsdbops.NewDbObjectIDs(libovsdbops.ACLNetworkPolicy, oc.controllerName, nil) + p := libovsdbops.GetPredicate[*nbdb.ACL](predicateIDs, nil) netpolACLs, err := libovsdbops.FindACLsWithPredicate(oc.nbClient, p) if err != nil { return fmt.Errorf("cannot find NetworkPolicy ACLs: %v", err) } stalePGs := sets.Set[string]{} - if len(netpolACLs) > 0 { - for _, netpolACL := range netpolACLs { - if netpolACL.ExternalIDs[policyACLExtIdKey] != "" { - // policy-owned acl - namespace := netpolACL.ExternalIDs[namespaceACLExtIdKey] - policyName := netpolACL.ExternalIDs[policyACLExtIdKey] - if !expectedPolicies[namespace][policyName] { - // policy doesn't exist on k8s, cleanup - portGroupName, _ := getNetworkPolicyPGName(namespace, policyName) - stalePGs.Insert(portGroupName) - } - } else if netpolACL.ExternalIDs[defaultDenyPolicyTypeACLExtIdKey] != "" { - // default deny acl - // parse the namespace.Name from the ACL name (if ACL name is 63 chars, then it will fully be namespace.Name) - namespace := strings.Split(*netpolACL.Name, "_")[0] - if _, ok := expectedPolicies[namespace]; !ok { - // no policies in that namespace are found, delete default deny port group - stalePGs.Insert(defaultDenyPortGroupName(namespace, ingressDefaultDenySuffix)) - stalePGs.Insert(defaultDenyPortGroupName(namespace, egressDefaultDenySuffix)) - } - } - + for _, netpolACL := range netpolACLs { + // policy-owned acl + namespace, policyName, err := parseACLPolicyKey(netpolACL.ExternalIDs[libovsdbops.ObjectNameKey.String()]) + if err != nil { + return fmt.Errorf("failed to sync stale network policies: acl IDs parsing failed: %w", err) + } + if !expectedPolicies[namespace][policyName] { + // policy doesn't exist on k8s, cleanup + portGroupName, _ := getNetworkPolicyPGName(namespace, policyName) + stalePGs.Insert(portGroupName) + } + } + // default deny acls + p = func(item *nbdb.ACL) bool { + return item.ExternalIDs[defaultDenyPolicyTypeACLExtIdKey] != "" + } + netpolACLs, err = libovsdbops.FindACLsWithPredicate(oc.nbClient, p) + if err != nil { + return fmt.Errorf("cannot find NetworkPolicy ACLs: %v", err) + } + for _, netpolACL := range netpolACLs { + // default deny acl + // parse the namespace.Name from the ACL name (if ACL name is 63 chars, then it will fully be namespace.Name) + namespace := strings.Split(*netpolACL.Name, "_")[0] + if _, ok := expectedPolicies[namespace]; !ok { + // no policies in that namespace are found, delete default deny port group + stalePGs.Insert(defaultDenyPortGroupName(namespace, ingressDefaultDenySuffix)) + stalePGs.Insert(defaultDenyPortGroupName(namespace, egressDefaultDenySuffix)) } } if len(stalePGs) > 0 { @@ -348,8 +342,7 @@ func (oc *DefaultNetworkController) syncNetworkPolicies(networkPolicies []interf // we'd still need to update its apply-after-lb=true option, so that the ACL priorities can apply properly; // If acl's option["apply-after-lb"] is already set to true, then its direction should be also correct. p = func(item *nbdb.ACL) bool { - return (item.ExternalIDs[policyTypeACLExtIdKey] == string(knet.PolicyTypeEgress) || - item.ExternalIDs[defaultDenyPolicyTypeACLExtIdKey] == string(knet.PolicyTypeEgress)) && + return item.ExternalIDs[defaultDenyPolicyTypeACLExtIdKey] == string(knet.PolicyTypeEgress) && item.Options["apply-after-lb"] != "true" } egressACLs, err := libovsdbops.FindACLsWithPredicate(oc.nbClient, p) @@ -601,9 +594,10 @@ func (oc *DefaultNetworkController) updateACLLoggingForPolicy(np *networkPolicy, } // Predicate for given network policy ACLs - p := func(item *nbdb.ACL) bool { - return item.ExternalIDs[namespaceACLExtIdKey] == np.namespace && item.ExternalIDs[policyACLExtIdKey] == np.name - } + predicateIDs := libovsdbops.NewDbObjectIDs(libovsdbops.ACLNetworkPolicy, oc.controllerName, map[libovsdbops.ExternalIDKey]string{ + libovsdbops.ObjectNameKey: getACLPolicyKey(np.namespace, np.name), + }) + p := libovsdbops.GetPredicate[*nbdb.ACL](predicateIDs, nil) return UpdateACLLoggingWithPredicate(oc.nbClient, p, aclLogging) } diff --git a/go-controller/pkg/ovn/policy_test.go b/go-controller/pkg/ovn/policy_test.go index b79477b58e..657414aa69 100644 --- a/go-controller/pkg/ovn/policy_test.go +++ b/go-controller/pkg/ovn/policy_test.go @@ -203,7 +203,6 @@ func getStaleDefaultACL(acls []*nbdb.ACL) []*nbdb.ACL { func getGressACLs(i int, namespace, policyName string, peerNamespaces []string, tcpPeerPorts []int32, peers []knet.NetworkPolicyPeer, logSeverity nbdb.ACLSeverity, policyType knet.PolicyType, stale, statlessNetPol bool) []*nbdb.ACL { - aclName := getGressPolicyACLName(namespace, policyName, i) pgName, _ := getNetworkPolicyPGName(namespace, policyName) shouldBeLogged := logSeverity != "" var options map[string]string @@ -239,6 +238,13 @@ func getGressACLs(i int, namespace, policyName string, peerNamespaces []string, ipBlock = peer.IPBlock.CIDR } } + gp := gressPolicy{ + policyNamespace: namespace, + policyName: policyName, + policyType: policyType, + idx: i, + controllerName: DefaultNetworkControllerName, + } if len(hashedASNames) > 0 { gressAsMatch := asMatch(hashedASNames) match := fmt.Sprintf("ip4.%s == {%s} && %s == @%s", ipDir, gressAsMatch, portDir, pgName) @@ -246,8 +252,9 @@ func getGressACLs(i int, namespace, policyName string, peerNamespaces []string, if statlessNetPol { action = nbdb.ACLActionAllowStateless } + dbIDs := gp.getNetpolACLDbIDs(emptyIdx, emptyIdx) acl := libovsdbops.BuildACL( - aclName, + getACLName(dbIDs), direction, types.DefaultAllowPriority, match, @@ -255,23 +262,17 @@ func getGressACLs(i int, namespace, policyName string, peerNamespaces []string, types.OvnACLLoggingMeter, logSeverity, shouldBeLogged, - map[string]string{ - l4MatchACLExtIdKey: "None", - ipBlockCIDRACLExtIdKey: "false", - namespaceACLExtIdKey: namespace, - policyACLExtIdKey: policyName, - policyTypeACLExtIdKey: string(policyType), - string(policyType) + "_num": strconv.Itoa(i), - }, + dbIDs.GetExternalIDs(), options, ) - acl.UUID = aclName + string(policyType) + "-UUID" + acl.UUID = dbIDs.String() + "-UUID" acls = append(acls, acl) } if ipBlock != "" { match := fmt.Sprintf("ip4.%s == %s && %s == @%s", ipDir, ipBlock, portDir, pgName) + dbIDs := gp.getNetpolACLDbIDs(emptyIdx, 0) acl := libovsdbops.BuildACL( - aclName, + getACLName(dbIDs), direction, types.DefaultAllowPriority, match, @@ -279,22 +280,16 @@ func getGressACLs(i int, namespace, policyName string, peerNamespaces []string, types.OvnACLLoggingMeter, logSeverity, shouldBeLogged, - map[string]string{ - l4MatchACLExtIdKey: "None", - ipBlockCIDRACLExtIdKey: "true", - namespaceACLExtIdKey: namespace, - policyACLExtIdKey: policyName, - policyTypeACLExtIdKey: string(policyType), - string(policyType) + "_num": strconv.Itoa(i), - }, + dbIDs.GetExternalIDs(), options, ) - acl.UUID = aclName + string(policyType) + "-UUID" + acl.UUID = dbIDs.String() + "-UUID" acls = append(acls, acl) } - for _, v := range tcpPeerPorts { + for portPolicyIdx, v := range tcpPeerPorts { + dbIDs := gp.getNetpolACLDbIDs(portPolicyIdx, emptyIdx) acl := libovsdbops.BuildACL( - aclName, + getACLName(dbIDs), direction, types.DefaultAllowPriority, fmt.Sprintf("ip4 && tcp && tcp.dst==%d && %s == @%s", v, portDir, pgName), @@ -302,29 +297,31 @@ func getGressACLs(i int, namespace, policyName string, peerNamespaces []string, types.OvnACLLoggingMeter, logSeverity, shouldBeLogged, - map[string]string{ - l4MatchACLExtIdKey: fmt.Sprintf("tcp && tcp.dst==%d", v), - ipBlockCIDRACLExtIdKey: "false", - namespaceACLExtIdKey: namespace, - policyACLExtIdKey: policyName, - policyTypeACLExtIdKey: string(policyType), - string(policyType) + "_num": strconv.Itoa(i), - }, + dbIDs.GetExternalIDs(), options, ) - acl.UUID = fmt.Sprintf("%s-%s-port_%d-UUID", string(policyType), aclName, v) + acl.UUID = dbIDs.String() + "-UUID" acls = append(acls, acl) } if stale { - return getStalePolicyACL(acls) + return getStalePolicyACL(acls, namespace, policyName) } return acls } -func getStalePolicyACL(acls []*nbdb.ACL) []*nbdb.ACL { - for _, acl := range acls { +// getStalePolicyACL updated ACL to the most ancient version, which includes +// - old name +// - no options for Egress ACLs +// - wrong direction for egress ACLs +// - non-nil Severity when Log is false +func getStalePolicyACL(acls []*nbdb.ACL, policyNamespace, policyName string) []*nbdb.ACL { + for i, acl := range acls { + staleName := policyNamespace + "_" + policyName + "_" + strconv.Itoa(i) + acl.Name = &staleName acl.Options = nil acl.Direction = nbdb.ACLDirectionToLport + sev := nbdb.ACLSeverityInfo + acl.Severity = &sev } return acls } @@ -731,41 +728,15 @@ var _ = ginkgo.Describe("OVN NetworkPolicy Operations", func() { "egress", "0", DefaultNetworkControllerName) localASName, _ := addressset.GetHashNamesForAS(staleAddrSetIDs) peerASName, _ := getNsAddrSetHashNames(namespace2.Name) - pgName, readableName := getNetworkPolicyPGName(networkPolicy.Namespace, networkPolicy.Name) - staleACL := libovsdbops.BuildACL( - "staleACL", - nbdb.ACLDirectionFromLport, - types.DefaultAllowPriority, - fmt.Sprintf("ip4.dst == {$%s, $%s} && inport == @%s", localASName, peerASName, pgName), - nbdb.ACLActionAllowRelated, - types.OvnACLLoggingMeter, - "", - false, - map[string]string{ - l4MatchACLExtIdKey: "None", - ipBlockCIDRACLExtIdKey: "false", - namespaceACLExtIdKey: networkPolicy.Namespace, - policyACLExtIdKey: networkPolicy.Name, - policyTypeACLExtIdKey: "egress", - "egress_num": "0", - }, - map[string]string{ - "apply-after-lb": "true", - }, - ) - staleACL.UUID = "staleACL-UUID" - pg := libovsdbops.BuildPortGroup( - pgName, - readableName, - nil, - []*nbdb.ACL{staleACL}, - ) - pg.UUID = pg.Name + "-UUID" + pgName, _ := getNetworkPolicyPGName(networkPolicy.Namespace, networkPolicy.Name) + initialData := getPolicyData(networkPolicy, nil, []string{namespace2.Name}, nil, + "", false) + staleACL := initialData[0].(*nbdb.ACL) + staleACL.Match = fmt.Sprintf("ip4.dst == {$%s, $%s} && inport == @%s", localASName, peerASName, pgName) defaultDenyInitialData := getDefaultDenyData(networkPolicy, nil, "", true) - initialData := initialDB.NBData initialData = append(initialData, defaultDenyInitialData...) - initialData = append(initialData, staleACL, pg) + initialData = append(initialData, initialDB.NBData...) _, err := fakeOvn.asf.NewAddressSet(staleAddrSetIDs, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -780,8 +751,6 @@ var _ = ginkgo.Describe("OVN NetworkPolicy Operations", func() { "", false, false) defaultDenyExpectedData := getDefaultDenyData(networkPolicy, nil, "", false) expectedData = append(expectedData, defaultDenyExpectedData...) - // stale acl will be de-refenced, but not garbage collected - expectedData = append(expectedData, staleACL) expectedData = append(expectedData, initialDB.NBData...) fakeOvn.asf.ExpectEmptyAddressSet(staleAddrSetIDs) @@ -1154,10 +1123,6 @@ var _ = ginkgo.Describe("OVN NetworkPolicy Operations", func() { defaultDenyExpectedData2 := getDefaultDenyData(networkPolicy2, []string{nPodTest.portUUID}, "", false) expectedData = getUpdatedInitialDB([]testPod{nPodTest}) - // FIXME(trozet): libovsdb server doesn't remove referenced ACLs to PG when deleting the PG - // https://github.com/ovn-org/libovsdb/issues/219 - expectedPolicy1ACLs := gressPolicy1ExpectedData[:len(gressPolicy1ExpectedData)-1] - expectedData = append(expectedData, expectedPolicy1ACLs...) expectedData = append(expectedData, gressPolicy2ExpectedData...) expectedData = append(expectedData, defaultDenyExpectedData2...) gomega.Eventually(fakeOvn.nbClient).Should(libovsdb.HaveData(expectedData...)) @@ -2387,7 +2352,7 @@ var _ = ginkgo.Describe("OVN NetworkPolicy Low-Level Operations", func() { buildExpectedIngressPeerNSv4ACL := func(gp *gressPolicy, pgName string, asDbIDses []*libovsdbops.DbObjectIDs, aclLogging *ACLLoggingLevels) *nbdb.ACL { - name := getGressPolicyACLName(gp.policyNamespace, gp.policyName, gp.idx) + aclIDs := gp.getNetpolACLDbIDs(emptyIdx, emptyIdx) hashedASNames := []string{} for _, dbIdx := range asDbIDses { @@ -2396,17 +2361,8 @@ var _ = ginkgo.Describe("OVN NetworkPolicy Low-Level Operations", func() { } asMatch := asMatch(hashedASNames) match := fmt.Sprintf("ip4.src == {%s} && outport == @%s", asMatch, pgName) - gpDirection := string(knet.PolicyTypeIngress) - externalIds := map[string]string{ - l4MatchACLExtIdKey: "None", - ipBlockCIDRACLExtIdKey: "false", - namespaceACLExtIdKey: gp.policyNamespace, - policyACLExtIdKey: gp.policyName, - policyTypeACLExtIdKey: gpDirection, - gpDirection + "_num": fmt.Sprintf("%d", gp.idx), - } - acl := libovsdbops.BuildACL(name, nbdb.ACLDirectionToLport, types.DefaultAllowPriority, match, - nbdb.ACLActionAllowRelated, types.OvnACLLoggingMeter, aclLogging.Allow, true, externalIds, nil) + acl := libovsdbops.BuildACL(getACLName(aclIDs), nbdb.ACLDirectionToLport, types.DefaultAllowPriority, match, + nbdb.ACLActionAllowRelated, types.OvnACLLoggingMeter, aclLogging.Allow, true, aclIDs.GetExternalIDs(), nil) return acl } From 28396ad5a975b4c90393311d47ba88cd398f2076 Mon Sep 17 00:00:00 2001 From: Nadia Pinaeva Date: Wed, 29 Mar 2023 15:02:58 +0200 Subject: [PATCH 06/40] Update DefaultDeny ACLs to have new dbIDs. Add NetpolNamespaceOwnerType for namespace-wide default deny acls. Add acl_sync PrimaryID check to make sure in case more than 1 ACLs have the same primaryID, only of them will be updated. This also eliminates the need for "deleting a network policy that failed half-way through creation succeeds" test, which is based on a "found multiple results for provided predicate" condition. Replace tests "stale ACLs should be cleaned up or updated at startup via syncNetworkPolicies" and "ACLs with long names and run syncNetworkPolicies" with the new "reconciles an existing networkPolicy updating stale ACLs with long names", and a part that deletes stale ARP ACLs is now updated and tested as a part of acl_sync. Signed-off-by: Nadia Pinaeva --- .../pkg/libovsdbops/db_object_types.go | 10 + go-controller/pkg/ovn/acl.go | 2 + .../ovn/external_ids_syncer/acl/acl_sync.go | 110 +++- .../external_ids_syncer/acl/acl_sync_test.go | 345 +++++++++- go-controller/pkg/ovn/policy.go | 248 ++------ go-controller/pkg/ovn/policy_test.go | 598 ++++-------------- 6 files changed, 620 insertions(+), 693 deletions(-) diff --git a/go-controller/pkg/libovsdbops/db_object_types.go b/go-controller/pkg/libovsdbops/db_object_types.go index 1b3acbe216..94cf24ed03 100644 --- a/go-controller/pkg/libovsdbops/db_object_types.go +++ b/go-controller/pkg/libovsdbops/db_object_types.go @@ -21,6 +21,7 @@ const ( MulticastNamespaceOwnerType ownerType = "MulticastNS" MulticastClusterOwnerType ownerType = "MulticastCluster" NetpolNodeOwnerType ownerType = "NetpolNode" + NetpolNamespaceOwnerType ownerType = "NetpolNamespace" // owner extra IDs, make sure to define only 1 ExternalIDKey for every string value PriorityKey ExternalIDKey = "priority" @@ -138,3 +139,12 @@ var ACLNetworkPolicy = newObjectIDsType(acl, NetworkPolicyOwnerType, []ExternalI PortPolicyIndexKey, IpBlockIndexKey, }) + +var ACLNetpolNamespace = newObjectIDsType(acl, NetpolNamespaceOwnerType, []ExternalIDKey{ + // namespace + ObjectNameKey, + // in the same namespace there can be 2 default deny port groups, egress and ingress + PolicyDirectionKey, + // every port group has default deny and arp allow acl. + TypeKey, +}) diff --git a/go-controller/pkg/ovn/acl.go b/go-controller/pkg/ovn/acl.go index 9cb49cd3d2..c92442c84f 100644 --- a/go-controller/pkg/ovn/acl.go +++ b/go-controller/pkg/ovn/acl.go @@ -86,6 +86,8 @@ func getACLName(dbIDs *libovsdbops.DbObjectIDs) string { case t.IsSameType(libovsdbops.ACLNetworkPolicy): aclName = "NP:" + dbIDs.GetObjectID(libovsdbops.ObjectNameKey) + ":" + dbIDs.GetObjectID(libovsdbops.PolicyDirectionKey) + ":" + dbIDs.GetObjectID(libovsdbops.GressIdxKey) + case t.IsSameType(libovsdbops.ACLNetpolNamespace): + aclName = "NP:" + dbIDs.GetObjectID(libovsdbops.ObjectNameKey) + ":" + dbIDs.GetObjectID(libovsdbops.PolicyDirectionKey) } return fmt.Sprintf("%.63s", aclName) } diff --git a/go-controller/pkg/ovn/external_ids_syncer/acl/acl_sync.go b/go-controller/pkg/ovn/external_ids_syncer/acl/acl_sync.go index 226908d545..1b21550216 100644 --- a/go-controller/pkg/ovn/external_ids_syncer/acl/acl_sync.go +++ b/go-controller/pkg/ovn/external_ids_syncer/acl/acl_sync.go @@ -6,6 +6,7 @@ import ( "strings" libovsdbclient "github.com/ovn-org/libovsdb/client" + libovsdb "github.com/ovn-org/libovsdb/ovsdb" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdbops" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" @@ -14,6 +15,7 @@ import ( v1 "k8s.io/api/core/v1" knet "k8s.io/api/networking/v1" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog/v2" utilnet "k8s.io/utils/net" ) @@ -30,6 +32,12 @@ const ( policyTypeNumACLExtIdKey = "%s_num" noneMatch = "None" emptyIdx = -1 + defaultDenyACL = "defaultDeny" + arpAllowACL = "arpAllow" + // staleArpAllowPolicyMatch "was" the old match used when creating default allow ARP ACLs for a namespace + // NOTE: This is succeed by arpAllowPolicyMatch to allow support for IPV6. This is currently only + // used when removing stale ACLs from the syncNetworkPolicy function and should NOT be used in any main logic. + staleArpAllowPolicyMatch = "arp" ) type aclSyncer struct { @@ -78,7 +86,46 @@ func (syncer *aclSyncer) SyncACLs(existingNodes *v1.NodeList) error { klog.Infof("Found %d stale gress ACLs", len(gressPolicyACLs)) updatedACLs = append(updatedACLs, gressPolicyACLs...) - err = batching.Batch[*nbdb.ACL](syncer.txnBatchSize, updatedACLs, func(batchACLs []*nbdb.ACL) error { + defaultDenyACLs, deleteACLs, err := syncer.updateStaleDefaultDenyNetpolACLs(legacyACLs) + if err != nil { + return fmt.Errorf("failed to update stale default deny netpol ACLs: %w", err) + } + klog.Infof("Found %d stale default deny netpol ACLs", len(gressPolicyACLs)) + updatedACLs = append(updatedACLs, defaultDenyACLs...) + + // delete stale duplicating acls first + _, err = libovsdbops.TransactAndCheck(syncer.nbClient, deleteACLs) + if err != nil { + return fmt.Errorf("faile to trasact db ops: %v", err) + } + + // make sure there is only 1 ACL with any given primary ID + // 1. collect all existing primary IDs via predicate that will update IDs set, but always return false + existingACLPrimaryIDs := sets.Set[string]{} + _, err = libovsdbops.FindACLsWithPredicate(syncer.nbClient, func(acl *nbdb.ACL) bool { + if acl.ExternalIDs[libovsdbops.PrimaryIDKey.String()] != "" { + existingACLPrimaryIDs.Insert(acl.ExternalIDs[libovsdbops.PrimaryIDKey.String()]) + } + return false + }) + if err != nil { + return fmt.Errorf("failed to find exisitng primary ID acls: %w", err) + } + // 2. Check to-be-updated ACLs don't have the same PrimaryID between themselves and with the existingACLPrimaryIDs + uniquePrimaryIDACLs := []*nbdb.ACL{} + for _, acl := range updatedACLs { + primaryID := acl.ExternalIDs[libovsdbops.PrimaryIDKey.String()] + if existingACLPrimaryIDs.Has(primaryID) { + // don't update that acl, otherwise 2 ACLs with the same primary ID will be in the db + klog.Warningf("Skip updating ACL %+v to the new ExternalIDs, since there is another ACL with the same primary ID", acl) + } else { + existingACLPrimaryIDs.Insert(primaryID) + uniquePrimaryIDACLs = append(uniquePrimaryIDACLs, acl) + } + } + + // update acls with new ExternalIDs + err = batching.Batch[*nbdb.ACL](syncer.txnBatchSize, uniquePrimaryIDACLs, func(batchACLs []*nbdb.ACL) error { return libovsdbops.CreateOrUpdateACLs(syncer.nbClient, batchACLs...) }) if err != nil { @@ -295,3 +342,64 @@ func (syncer *aclSyncer) updateStaleGressPolicies(legacyACLs []*nbdb.ACL) (updat } return } + +func (syncer *aclSyncer) getDefaultDenyPolicyACLIDs(ns, policyType, defaultACLType string) *libovsdbops.DbObjectIDs { + return libovsdbops.NewDbObjectIDs(libovsdbops.ACLNetpolNamespace, syncer.controllerName, + map[libovsdbops.ExternalIDKey]string{ + libovsdbops.ObjectNameKey: ns, + // in the same namespace there can be 2 default deny port groups, egress and ingress, + // every port group has default deny and arp allow acl. + libovsdbops.PolicyDirectionKey: policyType, + libovsdbops.TypeKey: defaultACLType, + }) +} + +func (syncer *aclSyncer) updateStaleDefaultDenyNetpolACLs(legacyACLs []*nbdb.ACL) (updatedACLs []*nbdb.ACL, + deleteOps []libovsdb.Operation, err error) { + for _, acl := range legacyACLs { + // sync default Deny policies + if acl.ExternalIDs[defaultDenyPolicyTypeACLExtIdKey] == "" { + // not default deny policy + continue + } + + // remove stale egress and ingress allow arp ACLs that were leftover as a result + // of ACL migration for "ARPallowPolicy" when the match changed from "arp" to "(arp || nd)" + if strings.Contains(acl.Match, " && "+staleArpAllowPolicyMatch) { + pgName := "" + if strings.Contains(acl.Match, "inport") { + // egress default ARP allow policy ("inport == @a16323395479447859119_egressDefaultDeny && arp") + pgName = strings.TrimPrefix(acl.Match, "inport == @") + } else if strings.Contains(acl.Match, "outport") { + // ingress default ARP allow policy ("outport == @a16323395479447859119_ingressDefaultDeny && arp") + pgName = strings.TrimPrefix(acl.Match, "outport == @") + } + pgName = strings.TrimSuffix(pgName, " && "+staleArpAllowPolicyMatch) + deleteOps, err = libovsdbops.DeleteACLsFromPortGroupOps(syncer.nbClient, deleteOps, pgName, acl) + if err != nil { + err = fmt.Errorf("failed getting delete acl ops: %w", err) + return + } + // acl will be deleted, no need to update it + continue + } + + // acl Name can be truncated (max length 64), but k8s namespace is limited to 63 symbols, + // therefore it is safe to extract it from the name. + // works for both older name _ and newer + // _egressDefaultDeny OR _ingressDefaultDeny + ns := strings.Split(libovsdbops.GetACLName(acl), "_")[0] + + // distinguish ARPAllowACL from DefaultDeny + var defaultDenyACLType string + if strings.Contains(acl.Match, "(arp || nd)") { + defaultDenyACLType = arpAllowACL + } else { + defaultDenyACLType = defaultDenyACL + } + dbIDs := syncer.getDefaultDenyPolicyACLIDs(ns, acl.ExternalIDs[defaultDenyPolicyTypeACLExtIdKey], defaultDenyACLType) + acl.ExternalIDs = dbIDs.GetExternalIDs() + updatedACLs = append(updatedACLs, acl) + } + return +} diff --git a/go-controller/pkg/ovn/external_ids_syncer/acl/acl_sync_test.go b/go-controller/pkg/ovn/external_ids_syncer/acl/acl_sync_test.go index 17b972ba1a..7cba578c61 100644 --- a/go-controller/pkg/ovn/external_ids_syncer/acl/acl_sync_test.go +++ b/go-controller/pkg/ovn/external_ids_syncer/acl/acl_sync_test.go @@ -9,33 +9,52 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" libovsdbtest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/libovsdb" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" v1 "k8s.io/api/core/v1" knet "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "strings" ) +const ( + ingressDefaultDenySuffix = "ingressDefaultDeny" + egressDefaultDenySuffix = "egressDefaultDeny" + arpAllowPolicySuffix = "ARPallowPolicy" + arpAllowPolicyMatch = "(arp || nd)" +) + type aclSync struct { before *nbdb.ACL after *libovsdbops.DbObjectIDs } -func testSyncerWithData(data []aclSync, controllerName string, initialDbState []libovsdbtest.TestData, +func testSyncerWithData(data []aclSync, controllerName string, initialDbState, finalDbState []libovsdbtest.TestData, existingNodes []v1.Node) { // create initial db setup dbSetup := libovsdbtest.TestSetup{NBData: initialDbState} for _, asSync := range data { - asSync.before.UUID = asSync.after.String() + "-UUID" + if asSync.after != nil { + asSync.before.UUID = asSync.after.String() + "-UUID" + } else { + asSync.before.UUID = asSync.before.Match + } dbSetup.NBData = append(dbSetup.NBData, asSync.before) } libovsdbOvnNBClient, _, libovsdbCleanup, err := libovsdbtest.NewNBSBTestHarness(dbSetup) defer libovsdbCleanup.Cleanup() gomega.Expect(err).NotTo(gomega.HaveOccurred()) // create expected data using addressSetFactory - expectedDbState := initialDbState + var expectedDbState []libovsdbtest.TestData + if finalDbState != nil { + expectedDbState = finalDbState + } else { + expectedDbState = initialDbState + } for _, aclSync := range data { acl := aclSync.before - acl.ExternalIDs = aclSync.after.GetExternalIDs() + if aclSync.after != nil { + acl.ExternalIDs = aclSync.after.GetExternalIDs() + } expectedDbState = append(expectedDbState, acl) } // run sync @@ -59,6 +78,44 @@ var _ = ginkgo.Describe("OVN ACL Syncer", func() { controllerName: controllerName, } + ginkgo.It("doesn't add 2 acls with the same PrimaryID", func() { + testData := []aclSync{ + { + before: libovsdbops.BuildACL( + joinACLName(types.ClusterPortGroupName, "DefaultDenyMulticastEgress"), + nbdb.ACLDirectionFromLport, + types.DefaultMcastDenyPriority, + "(ip4.mcast || mldv1 || mldv2 || (ip6.dst[120..127] == 0xff && ip6.dst[116] == 1))", + nbdb.ACLActionDrop, + types.OvnACLLoggingMeter, + "", + false, + map[string]string{ + defaultDenyPolicyTypeACLExtIdKey: "Egress", + }, + nil, + ), + after: syncerToBuildData.getDefaultMcastACLDbIDs(mcastDefaultDenyID, "Egress"), + }, + { + before: libovsdbops.BuildACL( + joinACLName(types.ClusterPortGroupName, "DefaultDenyMulticastEgress"), + nbdb.ACLDirectionFromLport, + types.DefaultMcastDenyPriority, + "(ip4.mcast || mldv1 || mldv2 || (ip6.dst[120..127] == 0xff && ip6.dst[116] == 1))", + nbdb.ACLActionDrop, + types.OvnACLLoggingMeter, + "", + false, + map[string]string{ + defaultDenyPolicyTypeACLExtIdKey: "Egress", + }, + nil, + ), + }, + } + testSyncerWithData(testData, controllerName, []libovsdbtest.TestData{}, nil, nil) + }) ginkgo.It("updates multicast acls", func() { testData := []aclSync{ // defaultDenyEgressACL @@ -170,7 +227,7 @@ var _ = ginkgo.Describe("OVN ACL Syncer", func() { after: syncerToBuildData.getNamespaceMcastACLDbIDs(namespace1, "Ingress"), }, } - testSyncerWithData(testData, controllerName, []libovsdbtest.TestData{}, nil) + testSyncerWithData(testData, controllerName, []libovsdbtest.TestData{}, nil, nil) }) ginkgo.It("updates allow from node acls", func() { nodeName := "node1" @@ -217,7 +274,7 @@ var _ = ginkgo.Describe("OVN ACL Syncer", func() { Name: nodeName, Annotations: map[string]string{"k8s.ovn.org/node-subnets": string(bytes)}, }}} - testSyncerWithData(testData, controllerName, []libovsdbtest.TestData{}, existingNodes) + testSyncerWithData(testData, controllerName, []libovsdbtest.TestData{}, nil, existingNodes) }) ginkgo.It("updates gress policy acls", func() { policyNamespace := "policyNamespace" @@ -339,6 +396,280 @@ var _ = ginkgo.Describe("OVN ACL Syncer", func() { 0, emptyIdx, emptyIdx), }, } - testSyncerWithData(testData, controllerName, []libovsdbtest.TestData{}, nil) + testSyncerWithData(testData, controllerName, []libovsdbtest.TestData{}, nil, nil) + }) + ginkgo.It("updates default deny policy acls", func() { + policyNamespace := "policyNamespace" + + defaultDenyPortGroupName := func(namespace, gressSuffix string) string { + return joinACLName(util.HashForOVN(namespace), gressSuffix) + } + getStaleARPAllowACLName := func(ns string) string { + return joinACLName(ns, arpAllowPolicySuffix) + } + egressPGName := defaultDenyPortGroupName(policyNamespace, egressDefaultDenySuffix) + ingressPGName := defaultDenyPortGroupName(policyNamespace, ingressDefaultDenySuffix) + staleARPEgressACL := libovsdbops.BuildACL( + getStaleARPAllowACLName(policyNamespace), + nbdb.ACLDirectionFromLport, + types.DefaultAllowPriority, + "inport == @"+egressPGName+" && "+staleArpAllowPolicyMatch, + nbdb.ACLActionAllow, + types.OvnACLLoggingMeter, + "", + false, + map[string]string{defaultDenyPolicyTypeACLExtIdKey: string(knet.PolicyTypeEgress)}, + nil, + ) + staleARPEgressACL.UUID = "staleARPEgressACL-UUID" + egressDenyPG := libovsdbops.BuildPortGroup( + egressPGName, + egressPGName, + nil, + []*nbdb.ACL{staleARPEgressACL}, + ) + egressDenyPG.UUID = egressDenyPG.Name + "-UUID" + + staleARPIngressACL := libovsdbops.BuildACL( + getStaleARPAllowACLName(policyNamespace), + nbdb.ACLDirectionToLport, + types.DefaultAllowPriority, + "outport == @"+ingressPGName+" && "+staleArpAllowPolicyMatch, + nbdb.ACLActionAllow, + types.OvnACLLoggingMeter, + "", + false, + map[string]string{defaultDenyPolicyTypeACLExtIdKey: string(knet.PolicyTypeIngress)}, + nil, + ) + staleARPIngressACL.UUID = "staleARPIngressACL-UUID" + ingressDenyPG := libovsdbops.BuildPortGroup( + ingressPGName, + ingressPGName, + nil, + []*nbdb.ACL{staleARPIngressACL}, + ) + ingressDenyPG.UUID = ingressDenyPG.Name + "-UUID" + initialDb := []libovsdbtest.TestData{staleARPEgressACL, egressDenyPG, staleARPIngressACL, ingressDenyPG} + finalEgressDenyPG := libovsdbops.BuildPortGroup( + egressPGName, + egressPGName, + nil, + nil, + ) + finalEgressDenyPG.UUID = finalEgressDenyPG.Name + "-UUID" + finalIngressDenyPG := libovsdbops.BuildPortGroup( + ingressPGName, + ingressPGName, + nil, + nil, + ) + finalIngressDenyPG.UUID = finalIngressDenyPG.Name + "-UUID" + // acls will stay since they are no garbage-collected by test server + finalDb := []libovsdbtest.TestData{staleARPEgressACL, finalEgressDenyPG, staleARPIngressACL, finalIngressDenyPG} + testData := []aclSync{ + // egress deny + { + before: libovsdbops.BuildACL( + policyNamespace+"_"+egressDefaultDenySuffix, + nbdb.ACLDirectionFromLport, + types.DefaultDenyPriority, + "inport == @"+egressPGName, + nbdb.ACLActionDrop, + types.OvnACLLoggingMeter, + "", + false, + map[string]string{defaultDenyPolicyTypeACLExtIdKey: string(knet.PolicyTypeEgress)}, + nil, + ), + after: syncerToBuildData.getDefaultDenyPolicyACLIDs(policyNamespace, string(knet.PolicyTypeEgress), defaultDenyACL), + }, + // egress allow ARP + { + before: libovsdbops.BuildACL( + getStaleARPAllowACLName(policyNamespace), + nbdb.ACLDirectionFromLport, + types.DefaultAllowPriority, + "inport == @"+egressPGName+" && "+arpAllowPolicyMatch, + nbdb.ACLActionDrop, + types.OvnACLLoggingMeter, + "", + false, + map[string]string{defaultDenyPolicyTypeACLExtIdKey: string(knet.PolicyTypeEgress)}, + nil, + ), + after: syncerToBuildData.getDefaultDenyPolicyACLIDs(policyNamespace, string(knet.PolicyTypeEgress), arpAllowACL), + }, + // egress deny + { + before: libovsdbops.BuildACL( + policyNamespace+"_"+ingressDefaultDenySuffix, + nbdb.ACLDirectionToLport, + types.DefaultDenyPriority, + "outport == @"+ingressPGName, + nbdb.ACLActionDrop, + types.OvnACLLoggingMeter, + "", + false, + map[string]string{defaultDenyPolicyTypeACLExtIdKey: string(knet.PolicyTypeIngress)}, + nil, + ), + after: syncerToBuildData.getDefaultDenyPolicyACLIDs(policyNamespace, string(knet.PolicyTypeIngress), defaultDenyACL), + }, + // egress allow ARP + { + before: libovsdbops.BuildACL( + getStaleARPAllowACLName(policyNamespace), + nbdb.ACLDirectionToLport, + types.DefaultAllowPriority, + "outport == @"+ingressPGName+" && "+arpAllowPolicyMatch, + nbdb.ACLActionDrop, + types.OvnACLLoggingMeter, + "", + false, + map[string]string{defaultDenyPolicyTypeACLExtIdKey: string(knet.PolicyTypeIngress)}, + nil, + ), + after: syncerToBuildData.getDefaultDenyPolicyACLIDs(policyNamespace, string(knet.PolicyTypeIngress), arpAllowACL), + }, + } + testSyncerWithData(testData, controllerName, initialDb, finalDb, nil) + }) + ginkgo.It("updates default deny policy acl with long names", func() { + policyNamespace := "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijk" // longest allowed namespace name + defaultDenyPortGroupName := func(namespace, gressSuffix string) string { + return joinACLName(util.HashForOVN(namespace), gressSuffix) + } + getStaleARPAllowACLName := func(ns string) string { + return joinACLName(ns, arpAllowPolicySuffix) + } + egressPGName := defaultDenyPortGroupName(policyNamespace, egressDefaultDenySuffix) + ingressPGName := defaultDenyPortGroupName(policyNamespace, ingressDefaultDenySuffix) + staleARPEgressACL := libovsdbops.BuildACL( + getStaleARPAllowACLName(policyNamespace), + nbdb.ACLDirectionFromLport, + types.DefaultAllowPriority, + "inport == @"+egressPGName+" && "+staleArpAllowPolicyMatch, + nbdb.ACLActionAllow, + types.OvnACLLoggingMeter, + "", + false, + map[string]string{defaultDenyPolicyTypeACLExtIdKey: string(knet.PolicyTypeEgress)}, + nil, + ) + staleARPEgressACL.UUID = "staleARPEgressACL-UUID" + egressDenyPG := libovsdbops.BuildPortGroup( + egressPGName, + egressPGName, + nil, + []*nbdb.ACL{staleARPEgressACL}, + ) + egressDenyPG.UUID = egressDenyPG.Name + "-UUID" + + staleARPIngressACL := libovsdbops.BuildACL( + getStaleARPAllowACLName(policyNamespace), + nbdb.ACLDirectionToLport, + types.DefaultAllowPriority, + "outport == @"+ingressPGName+" && "+staleArpAllowPolicyMatch, + nbdb.ACLActionAllow, + types.OvnACLLoggingMeter, + "", + false, + map[string]string{defaultDenyPolicyTypeACLExtIdKey: string(knet.PolicyTypeIngress)}, + nil, + ) + staleARPIngressACL.UUID = "staleARPIngressACL-UUID" + ingressDenyPG := libovsdbops.BuildPortGroup( + ingressPGName, + ingressPGName, + nil, + []*nbdb.ACL{staleARPIngressACL}, + ) + ingressDenyPG.UUID = ingressDenyPG.Name + "-UUID" + initialDb := []libovsdbtest.TestData{staleARPEgressACL, egressDenyPG, staleARPIngressACL, ingressDenyPG} + finalEgressDenyPG := libovsdbops.BuildPortGroup( + egressPGName, + egressPGName, + nil, + nil, + ) + finalEgressDenyPG.UUID = finalEgressDenyPG.Name + "-UUID" + finalIngressDenyPG := libovsdbops.BuildPortGroup( + ingressPGName, + ingressPGName, + nil, + nil, + ) + finalIngressDenyPG.UUID = finalIngressDenyPG.Name + "-UUID" + // acls will stay since they are no garbage-collected by test server + finalDb := []libovsdbtest.TestData{staleARPEgressACL, finalEgressDenyPG, staleARPIngressACL, finalIngressDenyPG} + + testData := []aclSync{ + // egress deny + { + before: libovsdbops.BuildACL( + policyNamespace, + nbdb.ACLDirectionFromLport, + types.DefaultDenyPriority, + "inport == @"+egressPGName, + nbdb.ACLActionDrop, + types.OvnACLLoggingMeter, + "", + false, + map[string]string{defaultDenyPolicyTypeACLExtIdKey: string(knet.PolicyTypeEgress)}, + nil, + ), + after: syncerToBuildData.getDefaultDenyPolicyACLIDs(policyNamespace, string(knet.PolicyTypeEgress), defaultDenyACL), + }, + // egress allow ARP + { + before: libovsdbops.BuildACL( + getStaleARPAllowACLName(policyNamespace), + nbdb.ACLDirectionFromLport, + types.DefaultAllowPriority, + "inport == @"+egressPGName+" && "+arpAllowPolicyMatch, + nbdb.ACLActionDrop, + types.OvnACLLoggingMeter, + "", + false, + map[string]string{defaultDenyPolicyTypeACLExtIdKey: string(knet.PolicyTypeEgress)}, + nil, + ), + after: syncerToBuildData.getDefaultDenyPolicyACLIDs(policyNamespace, string(knet.PolicyTypeEgress), arpAllowACL), + }, + // egress deny + { + before: libovsdbops.BuildACL( + policyNamespace, + nbdb.ACLDirectionToLport, + types.DefaultDenyPriority, + "outport == @"+ingressPGName, + nbdb.ACLActionDrop, + types.OvnACLLoggingMeter, + "", + false, + map[string]string{defaultDenyPolicyTypeACLExtIdKey: string(knet.PolicyTypeIngress)}, + nil, + ), + after: syncerToBuildData.getDefaultDenyPolicyACLIDs(policyNamespace, string(knet.PolicyTypeIngress), defaultDenyACL), + }, + // egress allow ARP + { + before: libovsdbops.BuildACL( + getStaleARPAllowACLName(policyNamespace), + nbdb.ACLDirectionToLport, + types.DefaultAllowPriority, + "outport == @"+ingressPGName+" && "+arpAllowPolicyMatch, + nbdb.ACLActionDrop, + types.OvnACLLoggingMeter, + "", + false, + map[string]string{defaultDenyPolicyTypeACLExtIdKey: string(knet.PolicyTypeIngress)}, + nil, + ), + after: syncerToBuildData.getDefaultDenyPolicyACLIDs(policyNamespace, string(knet.PolicyTypeIngress), arpAllowACL), + }, + } + testSyncerWithData(testData, controllerName, initialDb, finalDb, nil) }) }) diff --git a/go-controller/pkg/ovn/policy.go b/go-controller/pkg/ovn/policy.go index b1dae44eae..a79f57cbfe 100644 --- a/go-controller/pkg/ovn/policy.go +++ b/go-controller/pkg/ovn/policy.go @@ -8,7 +8,7 @@ import ( "time" libovsdbclient "github.com/ovn-org/libovsdb/client" - ovsdb "github.com/ovn-org/libovsdb/ovsdb" + "github.com/ovn-org/libovsdb/ovsdb" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdbops" @@ -26,22 +26,20 @@ import ( utilnet "k8s.io/utils/net" ) +type netpolDefaultDenyACLType string + const ( - // defaultDenyPolicyTypeACLExtIdKey external ID key for default deny policy type - defaultDenyPolicyTypeACLExtIdKey = "default-deny-policy-type" + // netpolDefaultDenyACLType is used to distinguish default deny and arp allow acls create for the same port group + defaultDenyACL netpolDefaultDenyACLType = "defaultDeny" + arpAllowACL netpolDefaultDenyACLType = "arpAllow" + // port groups suffixes // ingressDefaultDenySuffix is the suffix used when creating the ingress port group for a namespace ingressDefaultDenySuffix = "ingressDefaultDeny" // egressDefaultDenySuffix is the suffix used when creating the ingress port group for a namespace egressDefaultDenySuffix = "egressDefaultDeny" - // arpAllowPolicySuffix is the suffix used when creating default ACLs for a namespace - arpAllowPolicySuffix = "ARPallowPolicy" // arpAllowPolicyMatch is the match used when creating default allow ARP ACLs for a namespace - arpAllowPolicyMatch = "(arp || nd)" - // staleArpAllowPolicyMatch "was" the old match used when creating default allow ARP ACLs for a namespace - // NOTE: This is succeed by arpAllowPolicyMatch to allow support for IPV6. This is currently only - // used when removing stale ACLs from the syncNetworkPolicy function and should NOT be used in any main logic. - staleArpAllowPolicyMatch = "arp" - allowHairpinningACLID = "allow-hairpinning" + arpAllowPolicyMatch = "(arp || nd)" + allowHairpinningACLID = "allow-hairpinning" // ovnStatelessNetPolAnnotationName is an annotation on K8s Network Policy resource to specify that all // the resulting OVN ACLs must be created as stateless ovnStatelessNetPolAnnotationName = "k8s.ovn.org/acl-stateless" @@ -203,71 +201,6 @@ func NewNetworkPolicy(policy *knet.NetworkPolicy) *networkPolicy { return np } -// updateStaleDefaultDenyACLNames updates the naming of the default ingress and egress deny ACLs per namespace -// oldName: _ (lucky winner will be first policy created in the namespace) -// newName: _egressDefaultDeny OR _ingressDefaultDeny -func (oc *DefaultNetworkController) updateStaleDefaultDenyACLNames(npType knet.PolicyType, gressSuffix string) error { - cleanUpDefaultDeny := make(map[string][]*nbdb.ACL) - p := func(item *nbdb.ACL) bool { - if item.Name != nil { // we don't care about node ACLs - aclNameSuffix := strings.Split(*item.Name, "_") - if len(aclNameSuffix) == 1 { - // doesn't have suffix; no update required; append the actual suffix since this ACL can be skipped - aclNameSuffix = append(aclNameSuffix, gressSuffix) - } - return item.ExternalIDs[defaultDenyPolicyTypeACLExtIdKey] == string(npType) && // default-deny-policy-type:Egress or default-deny-policy-type:Ingress - strings.Contains(item.Match, gressSuffix) && // Match:inport == @ablah80448_egressDefaultDeny or Match:inport == @ablah80448_ingressDefaultDeny - !strings.Contains(item.Match, arpAllowPolicyMatch) && // != match: (arp || nd) - !strings.HasPrefix(gressSuffix, aclNameSuffix[1]) // filter out already converted ACLs or ones that are a no-op - } - return false - } - gressACLs, err := libovsdbops.FindACLsWithPredicate(oc.nbClient, p) - if err != nil { - return fmt.Errorf("cannot find NetworkPolicy default deny ACLs: %v", err) - } - for _, acl := range gressACLs { - acl := acl - // parse the namespace.Name from the ACL name (if ACL name is 63 chars, then it will fully be namespace.Name) - namespace := strings.Split(*acl.Name, "_")[0] - cleanUpDefaultDeny[namespace] = append(cleanUpDefaultDeny[namespace], acl) - } - // loop through the cleanUp map and per namespace update the first ACL's name and delete the rest - for namespace, aclList := range cleanUpDefaultDeny { - var aclT aclPipelineType - if aclList[0].Direction == nbdb.ACLDirectionToLport { - aclT = lportIngress - } else { - aclT = lportEgressAfterLB - } - newName := getDefaultDenyPolicyACLName(namespace, aclT) - if len(aclList) > 1 { - // this should never be the case but delete everything except 1st ACL - ingressPGName := defaultDenyPortGroupName(namespace, ingressDefaultDenySuffix) - egressPGName := defaultDenyPortGroupName(namespace, egressDefaultDenySuffix) - err := libovsdbops.DeleteACLsFromPortGroups(oc.nbClient, []string{ingressPGName, egressPGName}, aclList[1:]...) - if err != nil { - return err - } - } - newACL := BuildACL( - newName, // this is the only thing we need to change, keep the rest same - aclList[0].Priority, - aclList[0].Match, - aclList[0].Action, - oc.GetNamespaceACLLogging(namespace), - aclT, - aclList[0].ExternalIDs, - ) - newACL.UUID = aclList[0].UUID // for performance - err := libovsdbops.CreateOrUpdateACLs(oc.nbClient, newACL) - if err != nil { - return fmt.Errorf("cannot update old NetworkPolicy ACLs for namespace %s: %v", namespace, err) - } - } - return nil -} - func (oc *DefaultNetworkController) syncNetworkPolicies(networkPolicies []interface{}) error { // find network policies that don't exist in k8s anymore, but still present in the dbs, and cleanup. // Peer address sets and network policy's port groups (together with acls) will be cleaned up. @@ -279,7 +212,6 @@ func (oc *DefaultNetworkController) syncNetworkPolicies(networkPolicies []interf if !ok { return fmt.Errorf("spurious object in syncNetworkPolicies: %v", npInterface) } - if nsMap, ok := expectedPolicies[policy.Namespace]; ok { nsMap[policy.Name] = true } else { @@ -290,7 +222,7 @@ func (oc *DefaultNetworkController) syncNetworkPolicies(networkPolicies []interf } // cleanup port groups based on acl search - // netpol-owned acls + // netpol-owned port groups first predicateIDs := libovsdbops.NewDbObjectIDs(libovsdbops.ACLNetworkPolicy, oc.controllerName, nil) p := libovsdbops.GetPredicate[*nbdb.ACL](predicateIDs, nil) netpolACLs, err := libovsdbops.FindACLsWithPredicate(oc.nbClient, p) @@ -310,18 +242,16 @@ func (oc *DefaultNetworkController) syncNetworkPolicies(networkPolicies []interf stalePGs.Insert(portGroupName) } } - // default deny acls - p = func(item *nbdb.ACL) bool { - return item.ExternalIDs[defaultDenyPolicyTypeACLExtIdKey] != "" - } + // default deny port groups + predicateIDs = libovsdbops.NewDbObjectIDs(libovsdbops.ACLNetpolNamespace, oc.controllerName, nil) + p = libovsdbops.GetPredicate[*nbdb.ACL](predicateIDs, nil) netpolACLs, err = libovsdbops.FindACLsWithPredicate(oc.nbClient, p) if err != nil { - return fmt.Errorf("cannot find NetworkPolicy ACLs: %v", err) + return fmt.Errorf("cannot find default deny NetworkPolicy ACLs: %v", err) } for _, netpolACL := range netpolACLs { // default deny acl - // parse the namespace.Name from the ACL name (if ACL name is 63 chars, then it will fully be namespace.Name) - namespace := strings.Split(*netpolACL.Name, "_")[0] + namespace := netpolACL.ExternalIDs[libovsdbops.ObjectNameKey.String()] if _, ok := expectedPolicies[namespace]; !ok { // no policies in that namespace are found, delete default deny port group stalePGs.Insert(defaultDenyPortGroupName(namespace, ingressDefaultDenySuffix)) @@ -337,80 +267,6 @@ func (oc *DefaultNetworkController) syncNetworkPolicies(networkPolicies []interf klog.Infof("Network policy sync cleaned up %d stale port groups", len(stalePGs)) } - // Update existing egress network policies to use the updated ACLs - // Note that the default multicast egress acls were created with the correct direction, but - // we'd still need to update its apply-after-lb=true option, so that the ACL priorities can apply properly; - // If acl's option["apply-after-lb"] is already set to true, then its direction should be also correct. - p = func(item *nbdb.ACL) bool { - return item.ExternalIDs[defaultDenyPolicyTypeACLExtIdKey] == string(knet.PolicyTypeEgress) && - item.Options["apply-after-lb"] != "true" - } - egressACLs, err := libovsdbops.FindACLsWithPredicate(oc.nbClient, p) - if err != nil { - return fmt.Errorf("cannot find NetworkPolicy Egress ACLs: %v", err) - } - - if len(egressACLs) > 0 { - for _, acl := range egressACLs { - acl.Direction = nbdb.ACLDirectionFromLport - if acl.Options == nil { - acl.Options = map[string]string{"apply-after-lb": "true"} - } else { - acl.Options["apply-after-lb"] = "true" - } - } - ops, err := libovsdbops.CreateOrUpdateACLsOps(oc.nbClient, nil, egressACLs...) - if err != nil { - return fmt.Errorf("cannot create ops to update old Egress NetworkPolicy ACLs: %v", err) - } - _, err = libovsdbops.TransactAndCheck(oc.nbClient, ops) - if err != nil { - return fmt.Errorf("cannot update old Egress NetworkPolicy ACLs: %v", err) - } - } - - // remove stale egress and ingress allow arp ACLs that were leftover as a result - // of ACL migration for "ARPallowPolicy" when the match changed from "arp" to "(arp || nd)" - p = func(item *nbdb.ACL) bool { - return strings.Contains(item.Match, " && "+staleArpAllowPolicyMatch) && - // default-deny-policy-type:Egress or default-deny-policy-type:Ingress - (item.ExternalIDs[defaultDenyPolicyTypeACLExtIdKey] == string(knet.PolicyTypeEgress) || - item.ExternalIDs[defaultDenyPolicyTypeACLExtIdKey] == string(knet.PolicyTypeIngress)) - } - gressACLs, err := libovsdbops.FindACLsWithPredicate(oc.nbClient, p) - if err != nil { - return fmt.Errorf("cannot find stale arp allow ACLs: %v", err) - } - // Remove these stale ACLs from port groups and then delete them - var ops []ovsdb.Operation - for _, gressACL := range gressACLs { - gressACL := gressACL - pgName := "" - if strings.Contains(gressACL.Match, "inport") { - // egress default ARP allow policy ("inport == @a16323395479447859119_egressDefaultDeny && arp") - pgName = strings.TrimPrefix(gressACL.Match, "inport == @") - } else if strings.Contains(gressACL.Match, "outport") { - // ingress default ARP allow policy ("outport == @a16323395479447859119_ingressDefaultDeny && arp") - pgName = strings.TrimPrefix(gressACL.Match, "outport == @") - } - pgName = strings.TrimSuffix(pgName, " && "+staleArpAllowPolicyMatch) - ops, err = libovsdbops.DeleteACLsFromPortGroupOps(oc.nbClient, ops, pgName, gressACL) - if err != nil { - return fmt.Errorf("failed getting delete acl ops: %v", err) - } - } - _, err = libovsdbops.TransactAndCheck(oc.nbClient, ops) - if err != nil { - return fmt.Errorf("cannot delete stale arp allow ACLs: %v", err) - } - - if err := oc.updateStaleDefaultDenyACLNames(knet.PolicyTypeEgress, egressDefaultDenySuffix); err != nil { - return fmt.Errorf("cannot clean up egress default deny ACL name: %v", err) - } - if err := oc.updateStaleDefaultDenyACLNames(knet.PolicyTypeIngress, ingressDefaultDenySuffix); err != nil { - return fmt.Errorf("cannot clean up ingress default deny ACL name: %v", err) - } - // add default hairpin allow acl err = oc.addHairpinAllowACL() if err != nil { @@ -459,38 +315,32 @@ func (oc *DefaultNetworkController) addAllowACLFromNode(nodeName string, mgmtPor return nil } -func getDefaultDenyPolicyACLName(ns string, aclT aclPipelineType) string { - var defaultDenySuffix string - switch aclT { - case lportIngress: - defaultDenySuffix = ingressDefaultDenySuffix - case lportEgressAfterLB: - defaultDenySuffix = egressDefaultDenySuffix - default: - panic(fmt.Sprintf("Unknown acl type %s", aclT)) - } - return joinACLName(ns, defaultDenySuffix) -} - -func getDefaultDenyPolicyExternalIDs(aclT aclPipelineType) map[string]string { - return map[string]string{defaultDenyPolicyTypeACLExtIdKey: string(aclTypeToPolicyType(aclT))} -} - -func getARPAllowACLName(ns string) string { - return joinACLName(ns, arpAllowPolicySuffix) +func (oc *DefaultNetworkController) getDefaultDenyPolicyACLIDs(ns string, aclDir aclDirection, + defaultACLType netpolDefaultDenyACLType) *libovsdbops.DbObjectIDs { + return libovsdbops.NewDbObjectIDs(libovsdbops.ACLNetpolNamespace, oc.controllerName, + map[libovsdbops.ExternalIDKey]string{ + libovsdbops.ObjectNameKey: ns, + // in the same namespace there can be 2 default deny port groups, egress and ingress, + // every port group has default deny and arp allow acl. + libovsdbops.PolicyDirectionKey: string(aclDir), + libovsdbops.TypeKey: string(defaultACLType), + }) } func defaultDenyPortGroupName(namespace, gressSuffix string) string { return hashedPortGroup(namespace) + "_" + gressSuffix } -func buildDenyACLs(namespace, pg string, aclLogging *ACLLoggingLevels, aclT aclPipelineType) (denyACL, allowACL *nbdb.ACL) { - denyMatch := getACLMatch(pg, "", aclT) - allowMatch := getACLMatch(pg, arpAllowPolicyMatch, aclT) - denyACL = BuildACL(getDefaultDenyPolicyACLName(namespace, aclT), types.DefaultDenyPriority, denyMatch, - nbdb.ACLActionDrop, aclLogging, aclT, getDefaultDenyPolicyExternalIDs(aclT)) - allowACL = BuildACL(getARPAllowACLName(namespace), types.DefaultAllowPriority, allowMatch, - nbdb.ACLActionAllow, nil, aclT, getDefaultDenyPolicyExternalIDs(aclT)) +func (oc *DefaultNetworkController) buildDenyACLs(namespace, pg string, aclLogging *ACLLoggingLevels, + aclDir aclDirection) (denyACL, allowACL *nbdb.ACL) { + denyMatch := getACLMatchFromACLDir(pg, "", aclDir) + allowMatch := getACLMatchFromACLDir(pg, arpAllowPolicyMatch, aclDir) + aclPipeline := aclDirectionToACLPipeline(aclDir) + + denyACL = BuildACLFromDbIDs(oc.getDefaultDenyPolicyACLIDs(namespace, aclDir, defaultDenyACL), + types.DefaultDenyPriority, denyMatch, nbdb.ACLActionDrop, aclLogging, aclPipeline) + allowACL = BuildACLFromDbIDs(oc.getDefaultDenyPolicyACLIDs(namespace, aclDir, arpAllowACL), + types.DefaultAllowPriority, allowMatch, nbdb.ACLActionAllow, nil, aclPipeline) return } @@ -523,7 +373,7 @@ func (oc *DefaultNetworkController) delPolicyFromDefaultPortGroups(np *networkPo delete(sharedPGs.policies, np.getKey()) if len(sharedPGs.policies) == 0 { // last policy was deleted, delete port group - err := oc.deleteDefaultDenyPGAndACLs(np.namespace, np.name) + err := oc.deleteDefaultDenyPGAndACLs(np.namespace) if err != nil { return fmt.Errorf("failed to delete defaul deny port group: %v", err) } @@ -537,9 +387,9 @@ func (oc *DefaultNetworkController) delPolicyFromDefaultPortGroups(np *networkPo // must be called with defaultDenyPortGroups lock func (oc *DefaultNetworkController) createDefaultDenyPGAndACLs(namespace, policy string, aclLogging *ACLLoggingLevels) error { ingressPGName := defaultDenyPortGroupName(namespace, ingressDefaultDenySuffix) - ingressDenyACL, ingressAllowACL := buildDenyACLs(namespace, ingressPGName, aclLogging, lportIngress) + ingressDenyACL, ingressAllowACL := oc.buildDenyACLs(namespace, ingressPGName, aclLogging, aclIngress) egressPGName := defaultDenyPortGroupName(namespace, egressDefaultDenySuffix) - egressDenyACL, egressAllowACL := buildDenyACLs(namespace, egressPGName, aclLogging, lportEgressAfterLB) + egressDenyACL, egressAllowACL := oc.buildDenyACLs(namespace, egressPGName, aclLogging, aclEgress) ops, err := libovsdbops.CreateOrUpdateACLsOps(oc.nbClient, nil, ingressDenyACL, ingressAllowACL, egressDenyACL, egressAllowACL) if err != nil { return err @@ -568,7 +418,7 @@ func (oc *DefaultNetworkController) createDefaultDenyPGAndACLs(namespace, policy // deleteDefaultDenyPGAndACLs deletes the default port groups and acls for a namespace // must be called with defaultDenyPortGroups lock -func (oc *DefaultNetworkController) deleteDefaultDenyPGAndACLs(namespace, policy string) error { +func (oc *DefaultNetworkController) deleteDefaultDenyPGAndACLs(namespace string) error { ingressPGName := defaultDenyPortGroupName(namespace, ingressDefaultDenySuffix) egressPGName := defaultDenyPortGroupName(namespace, egressDefaultDenySuffix) @@ -608,11 +458,17 @@ func (oc *DefaultNetworkController) updateACLLoggingForDefaultACLs(ns string, ns // shared port group doesn't exist, nothing to update return nil } - denyEgressACL, _ := buildDenyACLs(ns, defaultDenyPortGroupName(ns, egressDefaultDenySuffix), - &nsInfo.aclLogging, lportEgressAfterLB) - denyIngressACL, _ := buildDenyACLs(ns, defaultDenyPortGroupName(ns, ingressDefaultDenySuffix), - &nsInfo.aclLogging, lportIngress) - if err := UpdateACLLogging(oc.nbClient, []*nbdb.ACL{denyIngressACL, denyEgressACL}, &nsInfo.aclLogging); err != nil { + predicateIDs := libovsdbops.NewDbObjectIDs(libovsdbops.ACLNetpolNamespace, oc.controllerName, + map[libovsdbops.ExternalIDKey]string{ + libovsdbops.ObjectNameKey: ns, + libovsdbops.TypeKey: string(defaultDenyACL), + }) + p := libovsdbops.GetPredicate[*nbdb.ACL](predicateIDs, nil) + defaultDenyACLs, err := libovsdbops.FindACLsWithPredicate(oc.nbClient, p) + if err != nil { + return fmt.Errorf("failed to find netpol default deny acls for namespace %s: %v", ns, err) + } + if err := UpdateACLLogging(oc.nbClient, defaultDenyACLs, &nsInfo.aclLogging); err != nil { return fmt.Errorf("unable to update ACL logging for namespace %s: %w", ns, err) } return nil @@ -1326,7 +1182,7 @@ func (oc *DefaultNetworkController) addNetworkPolicy(policy *knet.NetworkPolicy) // network policy only reacts to namespace update ACL log level. // Run handleNetPolNamespaceUpdate sequence, but only for 1 newly added policy. if nsInfo.aclLogging.Deny != aclLogging.Deny { - if err := oc.updateACLLoggingForDefaultACLs(policy.Namespace, nsInfo); err != nil { + if err = oc.updateACLLoggingForDefaultACLs(policy.Namespace, nsInfo); err != nil { return fmt.Errorf("network policy %s failed to be created: update default deny ACLs failed: %v", npKey, err) } else { klog.Infof("Policy %s: ACL logging setting updated to deny=%s allow=%s", @@ -1334,7 +1190,7 @@ func (oc *DefaultNetworkController) addNetworkPolicy(policy *knet.NetworkPolicy) } } if nsInfo.aclLogging.Allow != aclLogging.Allow { - if err := oc.updateACLLoggingForPolicy(np, &nsInfo.aclLogging); err != nil { + if err = oc.updateACLLoggingForPolicy(np, &nsInfo.aclLogging); err != nil { return fmt.Errorf("network policy %s failed to be created: update policy ACLs failed: %v", npKey, err) } else { klog.Infof("Policy %s: ACL logging setting updated to deny=%s allow=%s", diff --git a/go-controller/pkg/ovn/policy_test.go b/go-controller/pkg/ovn/policy_test.go index 657414aa69..28d74f5560 100644 --- a/go-controller/pkg/ovn/policy_test.go +++ b/go-controller/pkg/ovn/policy_test.go @@ -32,7 +32,12 @@ import ( apimachinerytypes "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" utilnet "k8s.io/utils/net" - utilpointer "k8s.io/utils/pointer" +) + +// Legacy const, should only be used in sync and tests +const ( + // arpAllowPolicySuffix is the suffix used when creating default ACLs for a namespace + arpAllowPolicySuffix = "ARPallowPolicy" ) func getFakeController(controllerName string) *DefaultNetworkController { @@ -76,9 +81,10 @@ func getDefaultDenyData(networkPolicy *knet.NetworkPolicy, ports []string, egressPGName := defaultDenyPortGroupName(networkPolicy.Namespace, egressDefaultDenySuffix) policyTypeIngress, policyTypeEgress := getPolicyType(networkPolicy) shouldBeLogged := denyLogSeverity != "" - aclName := getDefaultDenyPolicyACLName(networkPolicy.Namespace, lportEgressAfterLB) + fakeController := getFakeController(DefaultNetworkControllerName) + aclIDs := fakeController.getDefaultDenyPolicyACLIDs(networkPolicy.Namespace, aclEgress, defaultDenyACL) egressDenyACL := libovsdbops.BuildACL( - aclName, + getACLName(aclIDs), nbdb.ACLDirectionFromLport, types.DefaultDenyPriority, "inport == @"+egressPGName, @@ -86,18 +92,16 @@ func getDefaultDenyData(networkPolicy *knet.NetworkPolicy, ports []string, types.OvnACLLoggingMeter, denyLogSeverity, shouldBeLogged, - map[string]string{ - defaultDenyPolicyTypeACLExtIdKey: string(knet.PolicyTypeEgress), - }, + aclIDs.GetExternalIDs(), map[string]string{ "apply-after-lb": "true", }, ) - egressDenyACL.UUID = aclName + "-egressDenyACL-UUID" + egressDenyACL.UUID = *egressDenyACL.Name + "-egressDenyACL-UUID" - aclName = getARPAllowACLName(networkPolicy.Namespace) + aclIDs = fakeController.getDefaultDenyPolicyACLIDs(networkPolicy.Namespace, aclEgress, arpAllowACL) egressAllowACL := libovsdbops.BuildACL( - aclName, + getACLName(aclIDs), nbdb.ACLDirectionFromLport, types.DefaultAllowPriority, "inport == @"+egressPGName+" && "+arpAllowPolicyMatch, @@ -105,19 +109,17 @@ func getDefaultDenyData(networkPolicy *knet.NetworkPolicy, ports []string, types.OvnACLLoggingMeter, "", false, - map[string]string{ - defaultDenyPolicyTypeACLExtIdKey: string(knet.PolicyTypeEgress), - }, + aclIDs.GetExternalIDs(), map[string]string{ "apply-after-lb": "true", }, ) - egressAllowACL.UUID = aclName + "-egressAllowACL-UUID" + egressAllowACL.UUID = *egressAllowACL.Name + "-egressAllowACL-UUID" ingressPGName := defaultDenyPortGroupName(networkPolicy.Namespace, ingressDefaultDenySuffix) - aclName = getDefaultDenyPolicyACLName(networkPolicy.Namespace, lportIngress) + aclIDs = fakeController.getDefaultDenyPolicyACLIDs(networkPolicy.Namespace, aclIngress, defaultDenyACL) ingressDenyACL := libovsdbops.BuildACL( - aclName, + getACLName(aclIDs), nbdb.ACLDirectionToLport, types.DefaultDenyPriority, "outport == @"+ingressPGName, @@ -125,16 +127,14 @@ func getDefaultDenyData(networkPolicy *knet.NetworkPolicy, ports []string, types.OvnACLLoggingMeter, denyLogSeverity, shouldBeLogged, - map[string]string{ - defaultDenyPolicyTypeACLExtIdKey: string(knet.PolicyTypeIngress), - }, + aclIDs.GetExternalIDs(), nil, ) - ingressDenyACL.UUID = aclName + "-ingressDenyACL-UUID" + ingressDenyACL.UUID = *ingressDenyACL.Name + "-ingressDenyACL-UUID" - aclName = getARPAllowACLName(networkPolicy.Namespace) + aclIDs = fakeController.getDefaultDenyPolicyACLIDs(networkPolicy.Namespace, aclIngress, arpAllowACL) ingressAllowACL := libovsdbops.BuildACL( - aclName, + getACLName(aclIDs), nbdb.ACLDirectionToLport, types.DefaultAllowPriority, "outport == @"+ingressPGName+" && "+arpAllowPolicyMatch, @@ -142,16 +142,14 @@ func getDefaultDenyData(networkPolicy *knet.NetworkPolicy, ports []string, types.OvnACLLoggingMeter, "", false, - map[string]string{ - defaultDenyPolicyTypeACLExtIdKey: string(knet.PolicyTypeIngress), - }, + aclIDs.GetExternalIDs(), nil, ) - ingressAllowACL.UUID = aclName + "-ingressAllowACL-UUID" + ingressAllowACL.UUID = *ingressAllowACL.Name + "-ingressAllowACL-UUID" if stale { - getStaleDefaultACL([]*nbdb.ACL{egressDenyACL, egressAllowACL}) - getStaleDefaultACL([]*nbdb.ACL{ingressDenyACL, ingressAllowACL}) + getStaleDefaultACL([]*nbdb.ACL{egressDenyACL, egressAllowACL}, networkPolicy.Namespace, networkPolicy.Name) + getStaleDefaultACL([]*nbdb.ACL{ingressDenyACL, ingressAllowACL}, networkPolicy.Namespace, networkPolicy.Name) } lsps := []*nbdb.LogicalSwitchPort{} @@ -193,8 +191,20 @@ func getDefaultDenyData(networkPolicy *knet.NetworkPolicy, ports []string, } } -func getStaleDefaultACL(acls []*nbdb.ACL) []*nbdb.ACL { +func getStaleARPAllowACLName(ns string) string { + return joinACLName(ns, arpAllowPolicySuffix) +} + +func getStaleDefaultACL(acls []*nbdb.ACL, namespace, policyName string) []*nbdb.ACL { for _, acl := range acls { + var staleName string + switch acl.ExternalIDs[libovsdbops.TypeKey.String()] { + case string(defaultDenyACL): + staleName = namespace + "_" + policyName + case string(arpAllowACL): + staleName = getStaleARPAllowACLName(namespace) + } + acl.Name = &staleName acl.Options = nil acl.Direction = nbdb.ACLDirectionToLport } @@ -717,6 +727,44 @@ var _ = ginkgo.Describe("OVN NetworkPolicy Operations", func() { }) + ginkgo.It("reconciles an existing networkPolicy updating stale ACLs with long names", func() { + app.Action = func(ctx *cli.Context) error { + longNamespaceName63 := "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijk" // longest allowed namespace name + namespace1 := *newNamespace(longNamespaceName63) + namespace2 := *newNamespace(namespaceName2) + networkPolicy := getMatchLabelsNetworkPolicy(netPolicyName1, namespace1.Name, + namespace2.Name, "", true, true) + // start with stale ACLs + gressPolicyInitialData := getPolicyData(networkPolicy, nil, []string{namespace2.Name}, + nil, "", true, false) + defaultDenyInitialData := getDefaultDenyData(networkPolicy, nil, "", true) + initialData := initialDB.NBData + initialData = append(initialData, gressPolicyInitialData...) + initialData = append(initialData, defaultDenyInitialData...) + startOvn(libovsdb.TestSetup{NBData: initialData}, []v1.Namespace{namespace1, namespace2}, + []knet.NetworkPolicy{*networkPolicy}, nil, nil) + + fakeOvn.asf.ExpectEmptyAddressSet(longNamespaceName63) + fakeOvn.asf.ExpectEmptyAddressSet(namespaceName2) + + _, err := fakeOvn.fakeClient.KubeClient.NetworkingV1().NetworkPolicies(networkPolicy.Namespace). + Get(context.TODO(), networkPolicy.Name, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + // make sure stale ACLs were updated + expectedData := getPolicyData(networkPolicy, nil, []string{namespace2.Name}, nil, + "", false, false) + defaultDenyExpectedData := getDefaultDenyData(networkPolicy, nil, "", false) + expectedData = append(expectedData, defaultDenyExpectedData...) + expectedData = append(expectedData, initialDB.NBData...) + gomega.Eventually(fakeOvn.nbClient).Should(libovsdb.HaveData(expectedData...)) + + return nil + } + err := app.Run([]string{app.Name}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + }) + ginkgo.It("reconciles an existing networkPolicy updating stale address sets", func() { app.Action = func(ctx *cli.Context) error { namespace1 := *newNamespace(namespaceName1) @@ -730,7 +778,7 @@ var _ = ginkgo.Describe("OVN NetworkPolicy Operations", func() { peerASName, _ := getNsAddrSetHashNames(namespace2.Name) pgName, _ := getNetworkPolicyPGName(networkPolicy.Namespace, networkPolicy.Name) initialData := getPolicyData(networkPolicy, nil, []string{namespace2.Name}, nil, - "", false) + "", false, false) staleACL := initialData[0].(*nbdb.ACL) staleACL.Match = fmt.Sprintf("ip4.dst == {$%s, $%s} && inport == @%s", localASName, peerASName, pgName) @@ -1136,463 +1184,6 @@ var _ = ginkgo.Describe("OVN NetworkPolicy Operations", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) - ginkgo.It("deleting a network policy that failed half-way through creation succeeds", func() { - app.Action = func(ctx *cli.Context) error { - namespace1 := *newNamespace("abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijk") // create with 63 characters - nPodTest := getTestPod(namespace1.Name, nodeName) - networkPolicy := getPortNetworkPolicy(netPolicyName1, namespace1.Name, labelName, labelVal, portNum) - egressOptions := map[string]string{ - "apply-after-lb": "true", - } - egressPGName := defaultDenyPortGroupName(networkPolicy.Namespace, egressDefaultDenySuffix) - aclName := getARPAllowACLName(networkPolicy.Namespace) - leftOverACLFromUpgrade1 := libovsdbops.BuildACL( - aclName, - nbdb.ACLDirectionFromLport, - types.DefaultAllowPriority, - "inport == @"+egressPGName+" && (arp)", // invalid ACL match; won't be cleaned up - nbdb.ACLActionAllow, - types.OvnACLLoggingMeter, - "", - false, - map[string]string{ - defaultDenyPolicyTypeACLExtIdKey: string(knet.PolicyTypeEgress), - }, - egressOptions, - ) - leftOverACLFromUpgrade1.UUID = *leftOverACLFromUpgrade1.Name + "-egressAllowACL-UUID1" - - aclName = getARPAllowACLName(networkPolicy.Namespace) - leftOverACLFromUpgrade2 := libovsdbops.BuildACL( - aclName, - nbdb.ACLDirectionFromLport, - types.DefaultAllowPriority, - "inport == @"+egressPGName+" && "+arpAllowPolicyMatch, - nbdb.ACLActionAllow, - types.OvnACLLoggingMeter, - "", - false, - map[string]string{ - defaultDenyPolicyTypeACLExtIdKey: string(knet.PolicyTypeEgress), - }, - egressOptions, - ) - leftOverACLFromUpgrade2.UUID = *leftOverACLFromUpgrade2.Name + "-egressAllowACL-UUID2" - - initialDB.NBData = append(initialDB.NBData, leftOverACLFromUpgrade1, leftOverACLFromUpgrade2) - startOvn(initialDB, []v1.Namespace{namespace1}, nil, []testPod{nPodTest}, - map[string]string{labelName: labelVal}) - - ginkgo.By("Creating a network policy that applies to a pod and ensuring creation fails") - - err := fakeOvn.controller.addNetworkPolicy(networkPolicy) - gomega.Expect(err).To(gomega.HaveOccurred()) - gomega.Expect(err.Error()).To(gomega.ContainSubstring("failed to create Network Policy " + - "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijk/networkpolicy1: failed to " + - "create default deny port groups: unexpectedly found multiple results for provided predicate")) - - // ensure the default PGs and ACLs were removed via rollback from add failure - expectedData := getUpdatedInitialDB([]testPod{nPodTest}) - // note stale leftovers from previous upgrades won't be cleanedup - expectedData = append(expectedData, leftOverACLFromUpgrade1, leftOverACLFromUpgrade2) - gomega.Eventually(fakeOvn.nbClient).Should(libovsdb.HaveData(expectedData...)) - - ginkgo.By("Deleting the network policy that failed to create and ensuring we don't panic") - err = fakeOvn.controller.deleteNetworkPolicy(networkPolicy) - // I0623 policy.go:1285] Deleting network policy networkpolicy1 in namespace namespace1, np is nil: true - // W0623 policy.go:1315] Unable to delete network policy: namespace1/networkpolicy1 since its not found in cache - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - return nil - } - err := app.Run([]string{app.Name}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - - ginkgo.It("stale ACLs should be cleaned up or updated at startup via syncNetworkPolicies", func() { - app.Action = func(ctx *cli.Context) error { - namespace1 := *newNamespace(namespaceName1) - nPodTest := getTestPod(namespace1.Name, nodeName) - networkPolicy1 := getPortNetworkPolicy(netPolicyName1, namespace1.Name, labelName, labelVal, portNum) - // This is not yet going to be created - networkPolicy2 := getPortNetworkPolicy(netPolicyName2, namespace1.Name, labelName, labelVal, portNum+1) - // network policy should exist for port group to not be cleaned up - networkPolicy3 := getPortNetworkPolicy(netPolicyName1, "leftover1", labelName, labelVal, portNum) - egressPGName := defaultDenyPortGroupName("leftover1", egressDefaultDenySuffix) - ingressPGName := defaultDenyPortGroupName("leftover1", ingressDefaultDenySuffix) - egressOptions := map[string]string{ - // older versions of ACLs don't have, should be added by syncNetworkPolicies on startup - // "apply-after-lb": "true", - } - // ACL1: leftover arp allow ACL egress with old match (arp) - leftOverACL1FromUpgrade := libovsdbops.BuildACL( - getARPAllowACLName("leftover1"), - nbdb.ACLDirectionFromLport, - types.DefaultAllowPriority, - "inport == @"+egressPGName+" && "+staleArpAllowPolicyMatch, - nbdb.ACLActionAllow, - types.OvnACLLoggingMeter, - nbdb.ACLSeverityInfo, - false, - map[string]string{ - defaultDenyPolicyTypeACLExtIdKey: string(knet.PolicyTypeEgress), - }, - egressOptions, - ) - leftOverACL1FromUpgrade.UUID = *leftOverACL1FromUpgrade.Name + "-egressAllowACL-UUID" - testOnlyEgressDenyPG := libovsdbops.BuildPortGroup( - egressPGName, - egressPGName, - nil, - []*nbdb.ACL{leftOverACL1FromUpgrade}, - ) - testOnlyEgressDenyPG.UUID = testOnlyEgressDenyPG.Name + "-UUID" - // ACL2: leftover arp allow ACL ingress with old match (arp) - leftOverACL2FromUpgrade := libovsdbops.BuildACL( - getARPAllowACLName("leftover1"), - nbdb.ACLDirectionToLport, - types.DefaultAllowPriority, - "outport == @"+ingressPGName+" && "+staleArpAllowPolicyMatch, - nbdb.ACLActionAllow, - types.OvnACLLoggingMeter, - nbdb.ACLSeverityInfo, - false, - map[string]string{ - defaultDenyPolicyTypeACLExtIdKey: string(knet.PolicyTypeIngress), - }, - nil, - ) - leftOverACL2FromUpgrade.UUID = *leftOverACL2FromUpgrade.Name + "-ingressAllowACL-UUID" - testOnlyIngressDenyPG := libovsdbops.BuildPortGroup( - ingressPGName, - ingressPGName, - nil, - []*nbdb.ACL{leftOverACL2FromUpgrade}, - ) - testOnlyIngressDenyPG.UUID = testOnlyIngressDenyPG.Name + "-UUID" - - // ACL3: leftover default deny ACL egress with old name (namespace_policyname) - leftOverACL3FromUpgrade := libovsdbops.BuildACL( - "youknownothingjonsnowyouknownothingjonsnowyouknownothingjonsnow"+"_"+networkPolicy2.Name, - nbdb.ACLDirectionFromLport, - types.DefaultDenyPriority, - "inport == @"+egressPGName, - nbdb.ACLActionDrop, - types.OvnACLLoggingMeter, - nbdb.ACLSeverityInfo, - false, - map[string]string{ - defaultDenyPolicyTypeACLExtIdKey: string(knet.PolicyTypeEgress), - }, - egressOptions, - ) - leftOverACL3FromUpgrade.UUID = *leftOverACL3FromUpgrade.Name + "-egressDenyACL-UUID" - - // ACL4: leftover default deny ACL ingress with old name (namespace_policyname) - leftOverACL4FromUpgrade := libovsdbops.BuildACL( - "shortName"+"_"+networkPolicy2.Name, - nbdb.ACLDirectionToLport, - types.DefaultDenyPriority, - "outport == @"+ingressPGName, - nbdb.ACLActionDrop, - types.OvnACLLoggingMeter, - nbdb.ACLSeverityInfo, - false, - map[string]string{ - defaultDenyPolicyTypeACLExtIdKey: string(knet.PolicyTypeIngress), - }, - nil, - ) - leftOverACL4FromUpgrade.UUID = *leftOverACL4FromUpgrade.Name + "-ingressDenyACL-UUID" - - initialDB.NBData = append( - initialDB.NBData, - leftOverACL1FromUpgrade, - leftOverACL2FromUpgrade, - leftOverACL3FromUpgrade, - leftOverACL4FromUpgrade, - testOnlyIngressDenyPG, - testOnlyEgressDenyPG, - ) - - startOvn(initialDB, []v1.Namespace{namespace1}, []knet.NetworkPolicy{*networkPolicy1, *networkPolicy3}, - []testPod{nPodTest}, map[string]string{labelName: labelVal}) - - _, err := fakeOvn.fakeClient.KubeClient.NetworkingV1().NetworkPolicies(networkPolicy1.Namespace). - Create(context.TODO(), networkPolicy2, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - expectedData := getUpdatedInitialDB([]testPod{nPodTest}) - gressPolicy1ExpectedData := getPolicyData(networkPolicy1, []string{nPodTest.portUUID}, - nil, []int32{portNum}, "", false, false) - defaultDeny1ExpectedData := getDefaultDenyData(networkPolicy1, []string{nPodTest.portUUID}, "", false) - expectedData = append(expectedData, gressPolicy1ExpectedData...) - expectedData = append(expectedData, defaultDeny1ExpectedData...) - gressPolicy2ExpectedData := getPolicyData(networkPolicy2, []string{nPodTest.portUUID}, - nil, []int32{portNum + 1}, "", false, false) - expectedData = append(expectedData, gressPolicy2ExpectedData...) - egressOptions = map[string]string{ - "apply-after-lb": "true", - } - leftOverACL3FromUpgrade.Options = egressOptions - newDefaultDenyEgressACLName := "youknownothingjonsnowyouknownothingjonsnowyouknownothingjonsnow" // trims it according to RFC1123 - newDefaultDenyIngressACLName := getDefaultDenyPolicyACLName("shortName", lportIngress) - leftOverACL3FromUpgrade.Name = &newDefaultDenyEgressACLName - leftOverACL4FromUpgrade.Name = &newDefaultDenyIngressACLName - expectedData = append(expectedData, leftOverACL3FromUpgrade) - expectedData = append(expectedData, leftOverACL4FromUpgrade) - testOnlyIngressDenyPG.ACLs = nil // Sync Function should remove stale ACL from PGs - testOnlyEgressDenyPG.ACLs = nil // Sync Function should remove stale ACL from PGs - expectedData = append(expectedData, testOnlyIngressDenyPG) - // since test server doesn't garbage collect dereferenced acls, they will stay in the test db after they - // were deleted. Even though they are derefenced from the port group at this point, they will be updated - // as all the other ACLs. - // Update deleted leftOverACL1FromUpgrade and leftOverACL2FromUpgrade to match on expected data - // Once our test server can delete such acls, this part should be deleted - // start of db hack - newDefaultDenyLeftoverIngressACLName := getDefaultDenyPolicyACLName("leftover1", lportIngress) - newDefaultDenyLeftoverEgressACLName := getDefaultDenyPolicyACLName("leftover1", lportEgressAfterLB) - leftOverACL2FromUpgrade.Name = &newDefaultDenyLeftoverIngressACLName - leftOverACL1FromUpgrade.Name = &newDefaultDenyLeftoverEgressACLName - leftOverACL1FromUpgrade.Options = egressOptions - expectedData = append(expectedData, leftOverACL2FromUpgrade) - expectedData = append(expectedData, leftOverACL1FromUpgrade) - // end of db hack - expectedData = append(expectedData, testOnlyEgressDenyPG) - - fakeOvn.asf.ExpectAddressSetWithIPs(namespaceName1, []string{nPodTest.podIP}) - gomega.Eventually(fakeOvn.nbClient).Should(libovsdb.HaveData(expectedData...)) - - return nil - } - - err := app.Run([]string{app.Name}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - - ginkgo.It("ACLs with long names and run syncNetworkPolicies", func() { - app.Action = func(ctx *cli.Context) error { - longNameSpaceName := "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijk" // create with 63 characters - longNamespace := *newNamespace(longNameSpaceName) - nPodTest := getTestPod(longNamespace.Name, nodeName) - networkPolicy1 := getPortNetworkPolicy(netPolicyName1, longNamespace.Name, labelName, labelVal, portNum) - networkPolicy2 := getPortNetworkPolicy(netPolicyName2, longNamespace.Name, labelName, labelVal, portNum+1) - - longLeftOverNameSpaceName := "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz" // namespace is >45 characters long - longLeftOverNameSpaceName2 := "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxy1" // namespace is >45 characters long - // network policy should exist for port group to not be cleaned up - networkPolicy3 := getPortNetworkPolicy(netPolicyName1, longLeftOverNameSpaceName, labelName, labelVal, portNum) - egressPGName := defaultDenyPortGroupName(longLeftOverNameSpaceName, egressDefaultDenySuffix) - ingressPGName := defaultDenyPortGroupName(longLeftOverNameSpaceName, ingressDefaultDenySuffix) - // ACL1: leftover arp allow ACL egress with old match (arp) - leftOverACL1FromUpgrade := libovsdbops.BuildACL( - longLeftOverNameSpaceName+"_"+arpAllowPolicySuffix, - nbdb.ACLDirectionFromLport, - types.DefaultAllowPriority, - "inport == @"+egressPGName+" && "+staleArpAllowPolicyMatch, - nbdb.ACLActionAllow, - types.OvnACLLoggingMeter, - nbdb.ACLSeverityInfo, - false, - map[string]string{ - defaultDenyPolicyTypeACLExtIdKey: string(knet.PolicyTypeEgress), - }, - nil, - ) - leftOverACL1FromUpgrade.UUID = *leftOverACL1FromUpgrade.Name + "-egressAllowACL-UUID" - testOnlyEgressDenyPG := libovsdbops.BuildPortGroup( - egressPGName, - egressPGName, - nil, - []*nbdb.ACL{leftOverACL1FromUpgrade}, - ) - testOnlyEgressDenyPG.UUID = testOnlyEgressDenyPG.Name + "-UUID" - // ACL2: leftover arp allow ACL ingress with old match (arp) - leftOverACL2FromUpgrade := libovsdbops.BuildACL( - longLeftOverNameSpaceName+"_"+arpAllowPolicySuffix, - nbdb.ACLDirectionToLport, - types.DefaultAllowPriority, - "outport == @"+ingressPGName+" && "+staleArpAllowPolicyMatch, - nbdb.ACLActionAllow, - types.OvnACLLoggingMeter, - nbdb.ACLSeverityInfo, - false, - map[string]string{ - defaultDenyPolicyTypeACLExtIdKey: string(knet.PolicyTypeIngress), - }, - nil, - ) - leftOverACL2FromUpgrade.UUID = *leftOverACL2FromUpgrade.Name + "-ingressAllowACL-UUID" - - // ACL3: leftover arp allow ACL ingress with new match (arp || nd) - leftOverACL3FromUpgrade := libovsdbops.BuildACL( - longLeftOverNameSpaceName+"blah"+"_"+arpAllowPolicySuffix, - nbdb.ACLDirectionToLport, - types.DefaultAllowPriority, - "outport == @"+ingressPGName+" && "+arpAllowPolicyMatch, // new match! this ACL should be left as is! - nbdb.ACLActionAllow, - types.OvnACLLoggingMeter, - nbdb.ACLSeverityInfo, - false, - map[string]string{ - defaultDenyPolicyTypeACLExtIdKey: string(knet.PolicyTypeIngress), - }, - nil, - ) - leftOverACL3FromUpgrade.UUID = *leftOverACL3FromUpgrade.Name + "-ingressAllowACL-UUID1" - testOnlyIngressDenyPG := libovsdbops.BuildPortGroup( - ingressPGName, - ingressPGName, - nil, - []*nbdb.ACL{leftOverACL2FromUpgrade, leftOverACL3FromUpgrade}, - ) - testOnlyIngressDenyPG.UUID = testOnlyIngressDenyPG.Name + "-UUID" - - // ACL4: leftover default deny ACL egress with old name (namespace_policyname) - leftOverACL4FromUpgrade := libovsdbops.BuildACL( - longLeftOverNameSpaceName2+"_"+networkPolicy2.Name, // we are ok here because test server doesn't impose restrictions - nbdb.ACLDirectionFromLport, - types.DefaultDenyPriority, - "inport == @"+egressPGName, - nbdb.ACLActionDrop, - types.OvnACLLoggingMeter, - nbdb.ACLSeverityInfo, - false, - map[string]string{ - defaultDenyPolicyTypeACLExtIdKey: string(knet.PolicyTypeEgress), - }, - nil, - ) - leftOverACL4FromUpgrade.UUID = *leftOverACL4FromUpgrade.Name + "-egressDenyACL-UUID" - - // ACL5: leftover default deny ACL ingress with old name (namespace_policyname) - leftOverACL5FromUpgrade := libovsdbops.BuildACL( - longLeftOverNameSpaceName2+"_"+networkPolicy2.Name, // we are ok here because test server doesn't impose restrictions - nbdb.ACLDirectionToLport, - types.DefaultDenyPriority, - "outport == @"+ingressPGName, - nbdb.ACLActionDrop, - types.OvnACLLoggingMeter, - nbdb.ACLSeverityInfo, - false, - map[string]string{ - defaultDenyPolicyTypeACLExtIdKey: string(knet.PolicyTypeIngress), - }, - nil, - ) - leftOverACL5FromUpgrade.UUID = *leftOverACL5FromUpgrade.Name + "-ingressDenyACL-UUID" - - longLeftOverNameSpaceName62 := "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghij" - // ACL6: leftover default deny ACL ingress with old name (namespace_policyname) but namespace is 62 characters long - leftOverACL6FromUpgrade := libovsdbops.BuildACL( - longLeftOverNameSpaceName62+"_"+networkPolicy2.Name, - nbdb.ACLDirectionToLport, - types.DefaultDenyPriority, - "outport == @"+ingressPGName, - nbdb.ACLActionDrop, - types.OvnACLLoggingMeter, - nbdb.ACLSeverityInfo, - false, - map[string]string{ - defaultDenyPolicyTypeACLExtIdKey: string(knet.PolicyTypeIngress), - }, - nil, - ) - leftOverACL6FromUpgrade.UUID = *leftOverACL6FromUpgrade.Name + "-ingressDenyACL-UUID" - - initialDB.NBData = append( - initialDB.NBData, - leftOverACL1FromUpgrade, - leftOverACL2FromUpgrade, - leftOverACL3FromUpgrade, - leftOverACL4FromUpgrade, - leftOverACL5FromUpgrade, - leftOverACL6FromUpgrade, - testOnlyIngressDenyPG, - testOnlyEgressDenyPG, - ) - - startOvn(initialDB, []v1.Namespace{longNamespace}, []knet.NetworkPolicy{*networkPolicy1, *networkPolicy3}, - []testPod{nPodTest}, map[string]string{labelName: labelVal}) - - ginkgo.By("Creating a network policy that applies to a pod") - _, err := fakeOvn.fakeClient.KubeClient.NetworkingV1().NetworkPolicies(networkPolicy1.Namespace). - Get(context.TODO(), networkPolicy1.Name, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - fakeOvn.asf.ExpectAddressSetWithIPs(longNamespace.Name, []string{nPodTest.podIP}) - expectedData := getUpdatedInitialDB([]testPod{nPodTest}) - gressPolicy1ExpectedData := getPolicyData(networkPolicy1, []string{nPodTest.portUUID}, - nil, []int32{portNum}, "", false, false) - defaultDeny1ExpectedData := getDefaultDenyData(networkPolicy1, []string{nPodTest.portUUID}, - "", false) - expectedData = append(expectedData, gressPolicy1ExpectedData...) - expectedData = append(expectedData, defaultDeny1ExpectedData...) - - // Create a second NP - ginkgo.By("Creating another policy that references that pod") - _, err = fakeOvn.fakeClient.KubeClient.NetworkingV1().NetworkPolicies(networkPolicy1.Namespace). - Create(context.TODO(), networkPolicy2, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - _, err = fakeOvn.fakeClient.KubeClient.NetworkingV1().NetworkPolicies(networkPolicy1.Namespace). - Get(context.TODO(), networkPolicy2.Name, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gressPolicy2ExpectedData := getPolicyData(networkPolicy2, []string{nPodTest.portUUID}, - nil, []int32{portNum + 1}, "", false, false) - expectedData = append(expectedData, gressPolicy2ExpectedData...) - egressOptions := map[string]string{ - "apply-after-lb": "true", - } - leftOverACL4FromUpgrade.Options = egressOptions - leftOverACL3FromUpgrade.Name = utilpointer.StringPtr(longLeftOverNameSpaceName + "blah_ARPall") // trims it according to RFC1123 - leftOverACL4FromUpgrade.Name = utilpointer.StringPtr(longLeftOverNameSpaceName2 + "_egressDefa") // trims it according to RFC1123 - leftOverACL5FromUpgrade.Name = utilpointer.StringPtr(longLeftOverNameSpaceName2 + "_ingressDef") // trims it according to RFC1123 - leftOverACL6FromUpgrade.Name = utilpointer.StringPtr(longLeftOverNameSpaceName62 + "_") // name stays the same here since its no-op - expectedData = append(expectedData, leftOverACL3FromUpgrade) - expectedData = append(expectedData, leftOverACL4FromUpgrade) - expectedData = append(expectedData, leftOverACL5FromUpgrade) - expectedData = append(expectedData, leftOverACL6FromUpgrade) - testOnlyIngressDenyPG.ACLs = []string{leftOverACL3FromUpgrade.UUID} // Sync Function should remove stale ACL from PGs - testOnlyEgressDenyPG.ACLs = nil // Sync Function should remove stale ACL from PGs - - // since test server doesn't garbage collect dereferenced acls, they will stay in the test db after they - // were deleted. Even though they are derefenced from the port group at this point, they will be updated - // as all the other ACLs. - // Update deleted leftOverACL1FromUpgrade and leftOverACL2FromUpgrade to match on expected data - // Once our test server can delete such acls, this part should be deleted - // start of db hack - longLeftOverIngressName := longLeftOverNameSpaceName + "_ingressDef" - longLeftOverEgressName := longLeftOverNameSpaceName + "_egressDefa" - leftOverACL2FromUpgrade.Name = &longLeftOverIngressName - leftOverACL1FromUpgrade.Name = &longLeftOverEgressName - leftOverACL1FromUpgrade.Options = egressOptions - expectedData = append(expectedData, leftOverACL2FromUpgrade) - expectedData = append(expectedData, leftOverACL1FromUpgrade) - // end of db hack - expectedData = append(expectedData, testOnlyIngressDenyPG) - expectedData = append(expectedData, testOnlyEgressDenyPG) - gomega.Eventually(fakeOvn.nbClient).Should(libovsdb.HaveData(expectedData...)) - - // in stable state, after all stale acls were updated and cleanedup invoke sync again to re-enact a master restart - ginkgo.By("Trigger another syncNetworkPolicies run and ensure nothing has changed in the DB") - fakeOvn.controller.syncNetworkPolicies([]interface{}{networkPolicy1, networkPolicy2, networkPolicy3}) - gomega.Eventually(fakeOvn.nbClient).Should(libovsdb.HaveData(expectedData...)) - - ginkgo.By("Simulate the initial re-add of all network policies during upgrade and ensure we are stable") - fakeOvn.controller.networkPolicies.Delete(getPolicyKey(networkPolicy1)) - fakeOvn.controller.sharedNetpolPortGroups.Delete(networkPolicy1.Namespace) // reset cache so that we simulate the add that happens during upgrades - err = fakeOvn.controller.addNetworkPolicy(networkPolicy1) - // TODO: FIX ME - gomega.Expect(err).To(gomega.HaveOccurred()) - gomega.Expect(err.Error()).To(gomega.ContainSubstring("failed to create Network Policy " + - "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijk/networkpolicy1: " + - "failed to create default deny port groups: unexpectedly found multiple results for provided predicate")) - - return nil - } - err := app.Run([]string{app.Name}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - ginkgo.It("reconciles a deleted namespace referenced by a networkpolicy with a local running pod", func() { app.Action = func(ctx *cli.Context) error { namespace1 := *newNamespace(namespaceName1) @@ -2033,6 +1624,37 @@ var _ = ginkgo.Describe("OVN NetworkPolicy Operations", func() { err := app.Run([]string{app.Name}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) + + ginkgo.It("can reconcile network policy with long name", func() { + app.Action = func(ctx *cli.Context) error { + // this problem can be reproduced by starting ovn with existing db rows for network policy + // from namespace with long name, but WatchNetworkPolicy doesn't return error on initial netpol add, + // it just puts network policy to retry loop. + // To check the error message directly, we can explicitly add network policy, then + // delete NetworkPolicy's resources to pretend controller doesn't know about it. + // Then on the next addNetworkPolicy call the result should be the same as on restart. + // Before ACLs were updated to have new DbIDs, defaultDeny acls (arp and default deny) + // were equivalent, since their names were cropped and only contained namespace name, + // and externalIDs only had defaultDenyPolicyTypeACLExtIdKey: Egress/Ingress. + // Now ExternalIDs will always be different, and ACLs won't be equivalent. + longNameSpaceName := "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijk" // create with 63 characters + longNamespace := *newNamespace(longNameSpaceName) + networkPolicy1 := getPortNetworkPolicy(netPolicyName1, longNamespace.Name, labelName, labelVal, portNum) + + startOvn(initialDB, []v1.Namespace{longNamespace}, []knet.NetworkPolicy{*networkPolicy1}, + nil, map[string]string{labelName: labelVal}) + + ginkgo.By("Simulate the initial re-add of all network policies during upgrade and ensure we are stable") + // pretend controller didn't see this netpol object, all related db rows are still present + fakeOvn.controller.networkPolicies.Delete(getPolicyKey(networkPolicy1)) + fakeOvn.controller.sharedNetpolPortGroups.Delete(networkPolicy1.Namespace) + err := fakeOvn.controller.addNetworkPolicy(networkPolicy1) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return nil + } + err := app.Run([]string{app.Name}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) }) ginkgo.Context("ACL logging for network policies", func() { @@ -2236,9 +1858,7 @@ var _ = ginkgo.Describe("OVN AllowFromNode ACL low-level operations", func() { ) getFakeController := func(nbClient libovsdbclient.Client) *DefaultNetworkController { - controller := &DefaultNetworkController{ - BaseNetworkController: BaseNetworkController{controllerName: controllerName}, - } + controller := getFakeController(controllerName) controller.nbClient = nbClient return controller } From d75cee1a0419833e8e8f0a16caa269d98130aedf Mon Sep 17 00:00:00 2001 From: Nadia Pinaeva Date: Wed, 29 Mar 2023 15:13:46 +0200 Subject: [PATCH 07/40] Update EgressFirewall ACLs to have new dbIndex. Add EgressFirewallOwnerType for egress firewall ACLs. Update syncEgressFirewall to use passed egressFirewalls objects for cleanup, create updateEgressFirewallACLsDbIndex function that updates old formatted ACLs. Update egressfirewall_test.go, move a part of setup to BeforeEach. Signed-off-by: Nadia Pinaeva --- .../pkg/libovsdbops/db_object_types.go | 10 + go-controller/pkg/ovn/acl.go | 2 + go-controller/pkg/ovn/egressfirewall.go | 166 +++--- go-controller/pkg/ovn/egressfirewall_test.go | 494 +++++------------- .../ovn/external_ids_syncer/acl/acl_sync.go | 37 +- .../external_ids_syncer/acl/acl_sync_test.go | 35 ++ 6 files changed, 270 insertions(+), 474 deletions(-) diff --git a/go-controller/pkg/libovsdbops/db_object_types.go b/go-controller/pkg/libovsdbops/db_object_types.go index 94cf24ed03..e6b3e34bcc 100644 --- a/go-controller/pkg/libovsdbops/db_object_types.go +++ b/go-controller/pkg/libovsdbops/db_object_types.go @@ -8,6 +8,7 @@ const ( const ( // owner types EgressFirewallDNSOwnerType ownerType = "EgressFirewallDNS" + EgressFirewallOwnerType ownerType = "EgressFirewall" EgressQoSOwnerType ownerType = "EgressQoS" // NetworkPolicyOwnerType is deprecated for address sets, should only be used for sync. // New owner of network policy address sets, is PodSelectorOwnerType. @@ -32,6 +33,7 @@ const ( IpKey ExternalIDKey = "ip" PortPolicyIndexKey ExternalIDKey = "port-policy-index" IpBlockIndexKey ExternalIDKey = "ip-block-index" + RuleIndex ExternalIDKey = "rule-index" ) // ObjectIDsTypes should only be created here @@ -148,3 +150,11 @@ var ACLNetpolNamespace = newObjectIDsType(acl, NetpolNamespaceOwnerType, []Exter // every port group has default deny and arp allow acl. TypeKey, }) + +var ACLEgressFirewall = newObjectIDsType(acl, EgressFirewallOwnerType, []ExternalIDKey{ + // namespace + ObjectNameKey, + // there can only be 1 egress firewall object in every namespace, named "default" + // The only additional id we need is the index of the EgressFirewall.Spec.Egress rule. + RuleIndex, +}) diff --git a/go-controller/pkg/ovn/acl.go b/go-controller/pkg/ovn/acl.go index c92442c84f..9ca7534e27 100644 --- a/go-controller/pkg/ovn/acl.go +++ b/go-controller/pkg/ovn/acl.go @@ -88,6 +88,8 @@ func getACLName(dbIDs *libovsdbops.DbObjectIDs) string { ":" + dbIDs.GetObjectID(libovsdbops.GressIdxKey) case t.IsSameType(libovsdbops.ACLNetpolNamespace): aclName = "NP:" + dbIDs.GetObjectID(libovsdbops.ObjectNameKey) + ":" + dbIDs.GetObjectID(libovsdbops.PolicyDirectionKey) + case t.IsSameType(libovsdbops.ACLEgressFirewall): + aclName = "EF:" + dbIDs.GetObjectID(libovsdbops.ObjectNameKey) + ":" + dbIDs.GetObjectID(libovsdbops.RuleIndex) } return fmt.Sprintf("%.63s", aclName) } diff --git a/go-controller/pkg/ovn/egressfirewall.go b/go-controller/pkg/ovn/egressfirewall.go index c2ce28d6ab..a34e562365 100644 --- a/go-controller/pkg/ovn/egressfirewall.go +++ b/go-controller/pkg/ovn/egressfirewall.go @@ -3,6 +3,7 @@ package ovn import ( "fmt" "net" + "strconv" "strings" "sync" @@ -27,10 +28,7 @@ import ( const ( egressFirewallAppliedCorrectly = "EgressFirewall Rules applied" egressFirewallAddError = "EgressFirewall Rules not correctly added" - // egressFirewallACLExtIdKey external ID key for egress firewall ACLs - egressFirewallACLExtIdKey = "egressFirewall" - egressFirewallACLPriorityKey = "priority" - aclDeleteBatchSize = 1000 + aclDeleteBatchSize = 1000 ) type egressFirewall struct { @@ -124,7 +122,6 @@ func (oc *DefaultNetworkController) newEgressFirewallRule(rawEgressFirewallRule // The following cleanups are needed from the previous versions: // - Cleanup the old implementation (using LRP) in local GW mode // For this it just deletes all LRP setup done for egress firewall -// And also convert all old ACLs which specifed from-lport to specifying to-lport // - Cleanup the old implementation (using ACLs on the join and node switches) // For this it deletes all the ACLs on the join and node switches, they will be created from scratch later. @@ -132,56 +129,24 @@ func (oc *DefaultNetworkController) newEgressFirewallRule(rawEgressFirewallRule // NOTE: Utilize the fact that we know that all egress firewall related setup must have a priority: types.MinimumReservedEgressFirewallPriority <= priority <= types.EgressFirewallStartPriority // Upon failure, it may be invoked multiple times in order to avoid a pod restart. func (oc *DefaultNetworkController) syncEgressFirewall(egressFirewalls []interface{}) error { - // Lookup all ACLs used for egress Firewalls - aclPred := func(item *nbdb.ACL) bool { - return item.Priority >= types.MinimumReservedEgressFirewallPriority && item.Priority <= types.EgressFirewallStartPriority - } - egressFirewallACLs, err := libovsdbops.FindACLsWithPredicate(oc.nbClient, aclPred) - if err != nil { - return fmt.Errorf("unable to list egress firewall ACLs, cannot cleanup old stale data, err: %v", err) - } - - // Update the direction of each egressFirewallACL if needed. - // Update the egressFirewallACL name if needed. - namespaceToACLs := map[string][]*nbdb.ACL{} - for i := range egressFirewallACLs { - egressFirewallACLs[i].Direction = nbdb.ACLDirectionToLport - if namespace, ok := egressFirewallACLs[i].ExternalIDs[egressFirewallACLExtIdKey]; ok && namespace != "" { - aclName := buildEgressFwAclName(namespace, egressFirewallACLs[i].Priority) - egressFirewallACLs[i].Name = &aclName - meterName := types.OvnACLLoggingMeter - egressFirewallACLs[i].Meter = &meterName - namespaceToACLs[namespace] = append(namespaceToACLs[namespace], egressFirewallACLs[i]) - } else { - klog.Warningf("Could not find namespace for egress firewall ACL during refresh operation: %v", egressFirewallACLs[i]) - } - } - err = libovsdbops.CreateOrUpdateACLs(oc.nbClient, egressFirewallACLs...) - if err != nil { - return fmt.Errorf("unable to update ACL information (direction and logging) during resync operation, err: %v", err) - } - - // Update logging related information if needed. - for namespace, nsACLs := range namespaceToACLs { - // access nsInfo with lock - aclLogging := oc.GetNamespaceACLLogging(namespace) - // Update acl logging based on namespace logLevels - err = UpdateACLLogging(oc.nbClient, nsACLs, aclLogging) - if err != nil { - return fmt.Errorf("failed to update egress firewall ACL logging for namespace %s: %v", namespace, err) - } - } - // In any gateway mode, make sure to delete all LRPs on ovn_cluster_router. p := func(item *nbdb.LogicalRouterPolicy) bool { return item.Priority <= types.EgressFirewallStartPriority && item.Priority >= types.MinimumReservedEgressFirewallPriority } - err = libovsdbops.DeleteLogicalRouterPoliciesWithPredicate(oc.nbClient, types.OVNClusterRouter, p) + err := libovsdbops.DeleteLogicalRouterPoliciesWithPredicate(oc.nbClient, types.OVNClusterRouter, p) if err != nil { return fmt.Errorf("error deleting egress firewall policies on router %s: %v", types.OVNClusterRouter, err) } // delete acls from all switches, they reside on the port group now + // Lookup all ACLs used for egress Firewalls + aclPred := func(item *nbdb.ACL) bool { + return item.Priority >= types.MinimumReservedEgressFirewallPriority && item.Priority <= types.EgressFirewallStartPriority + } + egressFirewallACLs, err := libovsdbops.FindACLsWithPredicate(oc.nbClient, aclPred) + if err != nil { + return fmt.Errorf("unable to list egress firewall ACLs, cannot cleanup old stale data, err: %v", err) + } if len(egressFirewallACLs) != 0 { err = batching.Batch[*nbdb.ACL](aclDeleteBatchSize, egressFirewallACLs, func(batchACLs []*nbdb.ACL) error { // optimize the predicate to exclude switches that don't reference deleting acls. @@ -200,34 +165,33 @@ func (oc *DefaultNetworkController) syncEgressFirewall(egressFirewalls []interfa } // sync the ovn and k8s egressFirewall states - // represents the namespaces that have firewalls according to ovn - ovnEgressFirewalls := make(map[string]struct{}) - - for _, egressFirewallACL := range egressFirewallACLs { - if ns, ok := egressFirewallACL.ExternalIDs[egressFirewallACLExtIdKey]; ok { - // Most egressFirewalls will have more then one ACL but we only need to know if there is one for the namespace - // so a map is fine and we will add an entry every iteration but because it is a map will overwrite the previous - // entry if it already existed - ovnEgressFirewalls[ns] = struct{}{} + existingEFNamespaces := map[string]bool{} + for _, efInterface := range egressFirewalls { + ef, ok := efInterface.(*egressfirewallapi.EgressFirewall) + if !ok { + return fmt.Errorf("spurious object in syncEgressFirewall: %v", efInterface) } + existingEFNamespaces[ef.Namespace] = true } - - // get all the k8s EgressFirewall Objects - egressFirewallList, err := oc.kube.GetEgressFirewalls() + predicateIDs := libovsdbops.NewDbObjectIDs(libovsdbops.ACLEgressFirewall, oc.controllerName, nil) + aclP := libovsdbops.GetPredicate[*nbdb.ACL](predicateIDs, nil) + efACLs, err := libovsdbops.FindACLsWithPredicate(oc.nbClient, aclP) if err != nil { - return fmt.Errorf("cannot reconcile the state of egressfirewalls in ovn database and k8s. err: %v", err) - } - // delete entries from the map that exist in k8s and ovn - for _, egressFirewall := range egressFirewallList.Items { - delete(ovnEgressFirewalls, egressFirewall.Namespace) + return fmt.Errorf("cannot find Egress Firewall ACLs: %v", err) } - // any that are left are spurious and should be cleaned up - for spuriousEF := range ovnEgressFirewalls { - err := oc.deleteEgressFirewallRules(spuriousEF) - if err != nil { - return fmt.Errorf("cannot fully reconcile the state of egressfirewalls ACLs for namespace %s still exist in ovn db: %v", spuriousEF, err) + deleteACLs := []*nbdb.ACL{} + for _, efACL := range efACLs { + if !existingEFNamespaces[efACL.ExternalIDs[libovsdbops.ObjectNameKey.String()]] { + // there is no egress firewall in that namespace, delete + deleteACLs = append(deleteACLs, efACL) } } + + err = libovsdbops.DeleteACLsFromPortGroups(oc.nbClient, []string{types.ClusterPortGroupName}, deleteACLs...) + if err != nil { + return err + } + klog.Infof("Deleted %d stale egress firewall ACLs", len(deleteACLs)) return nil } @@ -274,7 +238,7 @@ func (oc *DefaultNetworkController) addEgressFirewall(egressFirewall *egressfire } ipv4HashedAS, ipv6HashedAS := as.GetASHashNames() aclLoggingLevels := oc.GetNamespaceACLLogging(ef.namespace) - if err := oc.addEgressFirewallRules(ef, ipv4HashedAS, ipv6HashedAS, types.EgressFirewallStartPriority, aclLoggingLevels); err != nil { + if err := oc.addEgressFirewallRules(ef, ipv4HashedAS, ipv6HashedAS, aclLoggingLevels); err != nil { return err } oc.egressFirewalls.Store(egressFirewall.Namespace, ef) @@ -330,7 +294,7 @@ func (oc *DefaultNetworkController) updateEgressFirewallStatusWithRetry(egressfi } func (oc *DefaultNetworkController) addEgressFirewallRules(ef *egressFirewall, hashedAddressSetNameIPv4, - hashedAddressSetNameIPv6 string, efStartPriority int, aclLogging *ACLLoggingLevels, ruleIDs ...int) error { + hashedAddressSetNameIPv6 string, aclLogging *ACLLoggingLevels, ruleIDs ...int) error { for _, rule := range ef.egressRules { // check if only specific rule ids are requested to be added if len(ruleIDs) > 0 { @@ -384,14 +348,14 @@ func (oc *DefaultNetworkController) addEgressFirewallRules(ef *egressFirewall, h if len(matchTargets) == 0 { klog.Warningf("Egress Firewall rule: %#v has no destination...ignoring", *rule) // ensure the ACL is removed from OVN - if err := oc.deleteEgressFirewallRule(buildEgressFwAclName(ef.namespace, efStartPriority-rule.id)); err != nil { + if err := oc.deleteEgressFirewallRule(ef.namespace, rule.id); err != nil { return err } continue } match := generateMatch(hashedAddressSetNameIPv4, hashedAddressSetNameIPv6, matchTargets, rule.ports) - err := oc.createEgressFirewallRules(efStartPriority-rule.id, match, action, ef.namespace, aclLogging) + err := oc.createEgressFirewallRules(rule.id, match, action, ef.namespace, aclLogging) if err != nil { return err } @@ -401,23 +365,17 @@ func (oc *DefaultNetworkController) addEgressFirewallRules(ef *egressFirewall, h // createEgressFirewallRules uses the previously generated elements and creates the // acls for all node switches -func (oc *DefaultNetworkController) createEgressFirewallRules(priority int, match, action, externalID string, aclLogging *ACLLoggingLevels) error { - // a name is needed for logging purposes - the name must be unique, so make it - // egressFirewall__ - aclName := buildEgressFwAclName(externalID, priority) - - egressFirewallACL := BuildACL( - aclName, +func (oc *DefaultNetworkController) createEgressFirewallRules(ruleIdx int, match, action, namespace string, aclLogging *ACLLoggingLevels) error { + aclIDs := oc.getEgressFirewallACLDbIDs(namespace, ruleIdx) + priority := types.EgressFirewallStartPriority - ruleIdx + egressFirewallACL := BuildACLFromDbIDs( + aclIDs, priority, match, action, aclLogging, // since egressFirewall has direction to-lport, set type to ingress lportIngress, - map[string]string{ - egressFirewallACLExtIdKey: externalID, - egressFirewallACLPriorityKey: fmt.Sprintf("%d", priority), - }, ) ops, err := libovsdbops.CreateOrUpdateACLsOps(oc.nbClient, nil, egressFirewallACL) if err != nil { @@ -439,14 +397,10 @@ func (oc *DefaultNetworkController) createEgressFirewallRules(priority int, matc return nil } -func (oc *DefaultNetworkController) deleteEgressFirewallRule(name string) error { +func (oc *DefaultNetworkController) deleteEgressFirewallRule(namespace string, ruleIdx int) error { // Find ACLs for a given egressFirewall - pACL := func(item *nbdb.ACL) bool { - if item.Name != nil { - return *item.Name == name - } - return false - } + aclIDs := oc.getEgressFirewallACLDbIDs(namespace, ruleIdx) + pACL := libovsdbops.GetPredicate[*nbdb.ACL](aclIDs, nil) egressFirewallACLs, err := libovsdbops.FindACLsWithPredicate(oc.nbClient, pACL) if err != nil { return fmt.Errorf("unable to list egress firewall ACLs, cannot cleanup old stale data, err: %v", err) @@ -457,7 +411,7 @@ func (oc *DefaultNetworkController) deleteEgressFirewallRule(name string) error } if len(egressFirewallACLs) > 1 { - klog.Errorf("Duplicate ACL found for egress firewall: %s", name) + klog.Errorf("Duplicate ACL found for egress firewall %s, ruleIdx: %d", namespace, ruleIdx) } err = libovsdbops.DeleteACLsFromPortGroups(oc.nbClient, []string{types.ClusterPortGroupName}, egressFirewallACLs...) @@ -465,18 +419,20 @@ func (oc *DefaultNetworkController) deleteEgressFirewallRule(name string) error } // deleteEgressFirewallRules delete egress firewall Acls -func (oc *DefaultNetworkController) deleteEgressFirewallRules(externalID string) error { +func (oc *DefaultNetworkController) deleteEgressFirewallRules(namespace string) error { // Find ACLs for a given egressFirewall - pACL := func(item *nbdb.ACL) bool { - return item.ExternalIDs[egressFirewallACLExtIdKey] == externalID - } + predicateIDs := libovsdbops.NewDbObjectIDs(libovsdbops.ACLEgressFirewall, oc.controllerName, + map[libovsdbops.ExternalIDKey]string{ + libovsdbops.ObjectNameKey: namespace, + }) + pACL := libovsdbops.GetPredicate[*nbdb.ACL](predicateIDs, nil) egressFirewallACLs, err := libovsdbops.FindACLsWithPredicate(oc.nbClient, pACL) if err != nil { return fmt.Errorf("unable to list egress firewall ACLs, cannot cleanup old stale data, err: %v", err) } if len(egressFirewallACLs) == 0 { - klog.Warningf("No egressFirewall ACLs to delete in ns: %s", externalID) + klog.Warningf("No egressFirewall ACLs to delete in ns: %s", namespace) return nil } @@ -702,17 +658,23 @@ func (oc *DefaultNetworkController) updateACLLoggingForEgressFirewall(egressFire defer ef.Unlock() // Predicate for given egress firewall ACLs - p := func(item *nbdb.ACL) bool { - return item.ExternalIDs[egressFirewallACLExtIdKey] == ef.namespace - } + predicateIDs := libovsdbops.NewDbObjectIDs(libovsdbops.ACLEgressFirewall, oc.controllerName, + map[libovsdbops.ExternalIDKey]string{ + libovsdbops.ObjectNameKey: ef.namespace, + }) + p := libovsdbops.GetPredicate[*nbdb.ACL](predicateIDs, nil) if err := UpdateACLLoggingWithPredicate(oc.nbClient, p, &nsInfo.aclLogging); err != nil { return false, fmt.Errorf("unable to update ACL logging in ns %s, err: %v", ef.namespace, err) } return true, nil } -func buildEgressFwAclName(namespace string, priority int) string { - return fmt.Sprintf("egressFirewall_%s_%d", namespace, priority) +func (oc *DefaultNetworkController) getEgressFirewallACLDbIDs(namespace string, ruleIdx int) *libovsdbops.DbObjectIDs { + return libovsdbops.NewDbObjectIDs(libovsdbops.ACLEgressFirewall, oc.controllerName, + map[libovsdbops.ExternalIDKey]string{ + libovsdbops.ObjectNameKey: namespace, + libovsdbops.RuleIndex: strconv.Itoa(ruleIdx), + }) } func getNodeInternalAddrsToString(node *kapi.Node) []string { @@ -780,7 +742,7 @@ func (oc *DefaultNetworkController) updateEgressFirewallForNode(oldNode, newNode } ipv4HashedAS, ipv6HashedAS := as.GetASHashNames() aclLoggingLevels := oc.GetNamespaceACLLogging(ef.namespace) - if err := oc.addEgressFirewallRules(ef, ipv4HashedAS, ipv6HashedAS, types.EgressFirewallStartPriority, + if err := oc.addEgressFirewallRules(ef, ipv4HashedAS, ipv6HashedAS, aclLoggingLevels, modifiedRuleIDs...); err != nil { efErr = fmt.Errorf("failed to add egress firewall for namespace: %s, error: %w", namespace, err) return false diff --git a/go-controller/pkg/ovn/egressfirewall_test.go b/go-controller/pkg/ovn/egressfirewall_test.go index dddd7ed36a..263639daf6 100644 --- a/go-controller/pkg/ovn/egressfirewall_test.go +++ b/go-controller/pkg/ovn/egressfirewall_test.go @@ -34,11 +34,9 @@ func newObjectMeta(name, namespace string) metav1.ObjectMeta { Name: name, Namespace: namespace, } - } func newEgressFirewallObject(name, namespace string, egressRules []egressfirewallapi.EgressFirewallRule) *egressfirewallapi.EgressFirewall { - return &egressfirewallapi.EgressFirewall{ ObjectMeta: newObjectMeta(name, namespace), Spec: egressfirewallapi.EgressFirewallSpec{ @@ -66,6 +64,21 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { Name: t.OVNClusterRouter, } + startOvn := func(dbSetup libovsdb.TestSetup, namespaces []v1.Namespace, egressFirewalls []egressfirewallapi.EgressFirewall) { + fakeOVN.startWithDBSetup(dbSetup, + &egressfirewallapi.EgressFirewallList{ + Items: egressFirewalls, + }, + &v1.NamespaceList{ + Items: namespaces, + }, + ) + err := fakeOVN.controller.WatchNamespaces() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fakeOVN.controller.WatchEgressFirewall() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + } + ginkgo.BeforeEach(func() { // Restore global default values before each testcase config.PrepareTestConfig() @@ -106,8 +119,11 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { ginkgo.It(fmt.Sprintf("reconciles stale ACLs, gateway mode %s", gwMode), func() { config.Gateway.Mode = gwMode app.Action = func(ctx *cli.Context) error { + // owned by non-existing namespace + fakeController := getFakeController(DefaultNetworkControllerName) + purgeIDs := fakeController.getEgressFirewallACLDbIDs("none", 0) purgeACL := libovsdbops.BuildACL( - buildEgressFwAclName("namespace1", t.EgressFirewallStartPriority), + "purgeACL1", nbdb.ACLDirectionFromLport, t.EgressFirewallStartPriority, "", @@ -115,10 +131,24 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { t.OvnACLLoggingMeter, "", false, - map[string]string{egressFirewallACLExtIdKey: "none"}, + purgeIDs.GetExternalIDs(), nil, ) purgeACL.UUID = "purgeACL-UUID" + // no externalIDs present => dbIDs can't be built + purgeACL2 := libovsdbops.BuildACL( + "purgeACL2", + nbdb.ACLDirectionFromLport, + t.EgressFirewallStartPriority, + "", + nbdb.ACLActionDrop, + t.OvnACLLoggingMeter, + "", + false, + nil, + nil, + ) + purgeACL2.UUID = "purgeACL2-UUID" namespace1 := *newNamespace("namespace1") egressFirewall := newEgressFirewallObject("default", namespace1.Name, []egressfirewallapi.EgressFirewallRule{ @@ -129,7 +159,8 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { }, }, }) - keepACL := libovsdbops.BuildACL( + updateIDs := fakeController.getEgressFirewallACLDbIDs(namespace1.Name, 0) + updateACL := libovsdbops.BuildACL( "", nbdb.ACLDirectionFromLport, t.EgressFirewallStartPriority, @@ -138,14 +169,14 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { t.OvnACLLoggingMeter, nbdb.ACLSeverityInfo, false, - map[string]string{egressFirewallACLExtIdKey: namespace1.Name}, + updateIDs.GetExternalIDs(), nil, ) - keepACL.UUID = "keepACL-UUID" + updateACL.UUID = "updateACL-UUID" // this ACL is not in the egress firewall priority range and should be untouched - otherACL := libovsdbops.BuildACL( - buildEgressFwAclName("namespace1", t.EgressFirewallStartPriority-1), + ignoreACL := libovsdbops.BuildACL( + "ignoreACL", nbdb.ACLDirectionFromLport, t.MinimumReservedEgressFirewallPriority-1, "", @@ -153,19 +184,21 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { t.OvnACLLoggingMeter, "", false, - map[string]string{egressFirewallACLExtIdKey: "default"}, + nil, nil, ) - otherACL.UUID = "otherACL-UUID" + ignoreACL.UUID = "ignoreACL-UUID" - nodeSwitch.ACLs = []string{purgeACL.UUID, keepACL.UUID} - joinSwitch.ACLs = []string{purgeACL.UUID, keepACL.UUID} + nodeSwitch.ACLs = []string{purgeACL.UUID, purgeACL2.UUID, updateACL.UUID, ignoreACL.UUID} + joinSwitch.ACLs = []string{purgeACL.UUID, purgeACL2.UUID, updateACL.UUID, ignoreACL.UUID} + clusterPortGroup.ACLs = []string{purgeACL.UUID, purgeACL2.UUID, updateACL.UUID, ignoreACL.UUID} dbSetup := libovsdbtest.TestSetup{ NBData: []libovsdbtest.TestData{ - otherACL, purgeACL, - keepACL, + purgeACL2, + ignoreACL, + updateACL, nodeSwitch, joinSwitch, clusterRouter, @@ -173,70 +206,30 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { }, } - fakeOVN.startWithDBSetup(dbSetup, - &v1.NodeList{ - Items: []v1.Node{ - { - Status: v1.NodeStatus{ - Phase: v1.NodeRunning, - }, - ObjectMeta: newObjectMeta(node1Name, ""), - }, - }, - }) - - // only create one egressFirewall - _, err := fakeOVN.fakeClient.EgressFirewallClient.K8sV1().EgressFirewalls(namespace1.Name). - Create(context.TODO(), egressFirewall, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - err = fakeOVN.controller.WatchEgressFirewall() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + startOvn(dbSetup, []v1.Namespace{namespace1}, []egressfirewallapi.EgressFirewall{*egressFirewall}) - // Both ACLs will be removed from the join switch - joinSwitch.ACLs = nil - // Both ACLs will be removed from the node switch - nodeSwitch.ACLs = nil - - // keepACL will be re-created with the new external ID - asHash, _ := getNsAddrSetHashNames(namespace1.Name) - newKeepACL := libovsdbops.BuildACL( - buildEgressFwAclName(namespace1.Name, t.EgressFirewallStartPriority), - nbdb.ACLDirectionToLport, - t.EgressFirewallStartPriority, - "(ip4.dst == 1.2.3.4/23) && ip4.src == $"+asHash, - nbdb.ACLActionAllow, - t.OvnACLLoggingMeter, - "", - false, - map[string]string{ - egressFirewallACLExtIdKey: "namespace1", - egressFirewallACLPriorityKey: fmt.Sprintf("%d", t.EgressFirewallStartPriority), - }, - nil, - ) - newKeepACL.UUID = "newKeepACL-UUID" - // keepACL will be added to the clusterPortGroup - clusterPortGroup.ACLs = []string{newKeepACL.UUID} + // All ACLs in the egress firewall priority range will be removed from the switches + joinSwitch.ACLs = []string{ignoreACL.UUID} + nodeSwitch.ACLs = []string{ignoreACL.UUID} + // purgeACL will be deleted as its namespace doesn't exist + clusterPortGroup.ACLs = []string{purgeACL2.UUID, updateACL.UUID, ignoreACL.UUID} + // updateACL will be updated // Direction of both ACLs will be converted to - keepACL.Direction = nbdb.ACLDirectionToLport - newName := buildEgressFwAclName(namespace1.Name, t.EgressFirewallStartPriority) - keepACL.Name = &newName + updateACL.Direction = nbdb.ACLDirectionToLport + newName := getACLName(updateIDs) + updateACL.Name = &newName // check severity was reset from default to nil - keepACL.Severity = nil - // purgeACL ACL will be deleted when test server starts deleting dereferenced ACLs - // for now we need to update its fields, since it is present in the db - purgeACL.Direction = nbdb.ACLDirectionToLport - newName2 := buildEgressFwAclName("none", t.EgressFirewallStartPriority) - purgeACL.Name = &newName2 - purgeACL.Severity = nil + updateACL.Severity = nil + // match shouldn't have cluster exclusion + asHash, _ := getNsAddrSetHashNames(namespace1.Name) + updateACL.Match = "(ip4.dst == 1.2.3.4/23) && ip4.src == $" + asHash expectedDatabaseState := []libovsdb.TestData{ - otherACL, purgeACL, - newKeepACL, - keepACL, + purgeACL2, + ignoreACL, + updateACL, nodeSwitch, joinSwitch, clusterRouter, @@ -264,37 +257,15 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { }, }) - fakeOVN.startWithDBSetup(dbSetup, - &egressfirewallapi.EgressFirewallList{ - Items: []egressfirewallapi.EgressFirewall{ - *egressFirewall, - }, - }, - &v1.NamespaceList{ - Items: []v1.Namespace{ - namespace1, - }, - }, &v1.NodeList{ - Items: []v1.Node{ - { - Status: v1.NodeStatus{ - Phase: v1.NodeRunning, - }, - ObjectMeta: newObjectMeta(node1Name, ""), - }, - }, - }) - - err := fakeOVN.controller.WatchNamespaces() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOVN.controller.WatchEgressFirewall() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + startOvn(dbSetup, []v1.Namespace{namespace1}, []egressfirewallapi.EgressFirewall{*egressFirewall}) - _, err = fakeOVN.fakeClient.EgressFirewallClient.K8sV1().EgressFirewalls(egressFirewall.Namespace).Get(context.TODO(), egressFirewall.Name, metav1.GetOptions{}) + _, err := fakeOVN.fakeClient.EgressFirewallClient.K8sV1().EgressFirewalls(egressFirewall.Namespace). + Get(context.TODO(), egressFirewall.Name, metav1.GetOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) asHash, _ := getNsAddrSetHashNames(namespace1.Name) + dbIDs := fakeOVN.controller.getEgressFirewallACLDbIDs(egressFirewall.Namespace, 0) ipv4ACL := libovsdbops.BuildACL( - buildEgressFwAclName("namespace1", t.EgressFirewallStartPriority), + getACLName(dbIDs), nbdb.ACLDirectionToLport, t.EgressFirewallStartPriority, "(ip4.dst == 1.2.3.4/23) && ip4.src == $"+asHash, @@ -302,10 +273,7 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { t.OvnACLLoggingMeter, "", false, - map[string]string{ - egressFirewallACLExtIdKey: "namespace1", - egressFirewallACLPriorityKey: fmt.Sprintf("%d", t.EgressFirewallStartPriority), - }, + dbIDs.GetExternalIDs(), nil, ) ipv4ACL.UUID = "ipv4ACL-UUID" @@ -335,38 +303,15 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { }, }) - fakeOVN.startWithDBSetup(dbSetup, - &egressfirewallapi.EgressFirewallList{ - Items: []egressfirewallapi.EgressFirewall{ - *egressFirewall, - }, - }, - &v1.NamespaceList{ - Items: []v1.Namespace{ - namespace1, - }, - }, &v1.NodeList{ - Items: []v1.Node{ - { - Status: v1.NodeStatus{ - Phase: v1.NodeRunning, - }, - ObjectMeta: newObjectMeta(node1Name, ""), - }, - }, - }) config.IPv6Mode = true + startOvn(dbSetup, []v1.Namespace{namespace1}, []egressfirewallapi.EgressFirewall{*egressFirewall}) - err := fakeOVN.controller.WatchNamespaces() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOVN.controller.WatchEgressFirewall() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - _, err = fakeOVN.fakeClient.EgressFirewallClient.K8sV1().EgressFirewalls(egressFirewall.Namespace).Get(context.TODO(), egressFirewall.Name, metav1.GetOptions{}) + _, err := fakeOVN.fakeClient.EgressFirewallClient.K8sV1().EgressFirewalls(egressFirewall.Namespace).Get(context.TODO(), egressFirewall.Name, metav1.GetOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) _, asHash6 := getNsAddrSetHashNames(namespace1.Name) + dbIDs := fakeOVN.controller.getEgressFirewallACLDbIDs(egressFirewall.Namespace, 0) ipv6ACL := libovsdbops.BuildACL( - buildEgressFwAclName("namespace1", t.EgressFirewallStartPriority), + getACLName(dbIDs), nbdb.ACLDirectionToLport, t.EgressFirewallStartPriority, "(ip6.dst == 2002::1234:abcd:ffff:c0a8:101/64) && ip6.src == $"+asHash6, @@ -374,10 +319,7 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { t.OvnACLLoggingMeter, "", false, - map[string]string{ - egressFirewallACLExtIdKey: "namespace1", - egressFirewallACLPriorityKey: fmt.Sprintf("%d", t.EgressFirewallStartPriority), - }, + dbIDs.GetExternalIDs(), nil, ) ipv6ACL.UUID = "ipv6ACL-UUID" @@ -414,37 +356,15 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { }, }, }) - fakeOVN.startWithDBSetup(dbSetup, - &egressfirewallapi.EgressFirewallList{ - Items: []egressfirewallapi.EgressFirewall{ - *egressFirewall, - }, - }, - &v1.NamespaceList{ - Items: []v1.Namespace{ - namespace1, - }, - }, - &v1.NodeList{ - Items: []v1.Node{ - { - Status: v1.NodeStatus{ - Phase: v1.NodeRunning, - }, - ObjectMeta: newObjectMeta(node1Name, ""), - }, - }, - }) + startOvn(dbSetup, []v1.Namespace{namespace1}, []egressfirewallapi.EgressFirewall{*egressFirewall}) - err := fakeOVN.controller.WatchNamespaces() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - _, err = fakeOVN.fakeClient.EgressFirewallClient.K8sV1().EgressFirewalls(egressFirewall.Namespace).Get(context.TODO(), egressFirewall.Name, metav1.GetOptions{}) + _, err := fakeOVN.fakeClient.EgressFirewallClient.K8sV1().EgressFirewalls(egressFirewall.Namespace).Get(context.TODO(), egressFirewall.Name, metav1.GetOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - fakeOVN.controller.WatchEgressFirewall() asHash, _ := getNsAddrSetHashNames(namespace1.Name) + dbIDs := fakeOVN.controller.getEgressFirewallACLDbIDs(egressFirewall.Namespace, 0) udpACL := libovsdbops.BuildACL( - buildEgressFwAclName("namespace1", t.EgressFirewallStartPriority), + getACLName(dbIDs), nbdb.ACLDirectionToLport, t.EgressFirewallStartPriority, "(ip4.dst == 1.2.3.4/23) && ip4.src == $"+asHash+" && ((udp && ( udp.dst == 100 )))", @@ -452,10 +372,7 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { t.OvnACLLoggingMeter, "", false, - map[string]string{ - egressFirewallACLExtIdKey: "namespace1", - egressFirewallACLPriorityKey: fmt.Sprintf("%d", t.EgressFirewallStartPriority), - }, + dbIDs.GetExternalIDs(), nil, ) udpACL.UUID = "udpACL-UUID" @@ -489,41 +406,12 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { }, }) - fakeOVN.startWithDBSetup(dbSetup, - &egressfirewallapi.EgressFirewallList{ - Items: []egressfirewallapi.EgressFirewall{ - *egressFirewall, - }, - }, - &v1.NamespaceList{ - Items: []v1.Namespace{ - namespace1, - }, - }, &v1.NodeList{ - Items: []v1.Node{ - { - Status: v1.NodeStatus{ - Phase: v1.NodeRunning, - }, - ObjectMeta: newObjectMeta(node1Name, ""), - }, - { - Status: v1.NodeStatus{ - Phase: v1.NodeRunning, - }, - ObjectMeta: newObjectMeta(node2Name, ""), - }, - }, - }) - - err := fakeOVN.controller.WatchNamespaces() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOVN.controller.WatchEgressFirewall() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + startOvn(dbSetup, []v1.Namespace{namespace1}, []egressfirewallapi.EgressFirewall{*egressFirewall}) asHash, _ := getNsAddrSetHashNames(namespace1.Name) + dbIDs := fakeOVN.controller.getEgressFirewallACLDbIDs(egressFirewall.Namespace, 0) ipv4ACL := libovsdbops.BuildACL( - buildEgressFwAclName("namespace1", t.EgressFirewallStartPriority), + getACLName(dbIDs), nbdb.ACLDirectionToLport, t.EgressFirewallStartPriority, "(ip4.dst == 1.2.3.5/23) && ip4.src == $"+asHash+" && ((tcp && ( tcp.dst == 100 )))", @@ -531,10 +419,7 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { t.OvnACLLoggingMeter, "", false, - map[string]string{ - egressFirewallACLExtIdKey: "namespace1", - egressFirewallACLPriorityKey: fmt.Sprintf("%d", t.EgressFirewallStartPriority), - }, + dbIDs.GetExternalIDs(), nil, ) ipv4ACL.UUID = "ipv4ACL-UUID" @@ -545,7 +430,7 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) - err = fakeOVN.fakeClient.EgressFirewallClient.K8sV1().EgressFirewalls(egressFirewall.Namespace).Delete(context.TODO(), egressFirewall.Name, *metav1.NewDeleteOptions(0)) + err := fakeOVN.fakeClient.EgressFirewallClient.K8sV1().EgressFirewalls(egressFirewall.Namespace).Delete(context.TODO(), egressFirewall.Name, *metav1.NewDeleteOptions(0)) gomega.Expect(err).NotTo(gomega.HaveOccurred()) // ACL should be removed from the port group egfw is deleted @@ -581,36 +466,12 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { }, }) - fakeOVN.startWithDBSetup(dbSetup, - &egressfirewallapi.EgressFirewallList{ - Items: []egressfirewallapi.EgressFirewall{ - *egressFirewall, - }, - }, - &v1.NamespaceList{ - Items: []v1.Namespace{ - namespace1, - }, - }, - &v1.NodeList{ - Items: []v1.Node{ - { - Status: v1.NodeStatus{ - Phase: v1.NodeRunning, - }, - ObjectMeta: newObjectMeta(node1Name, ""), - }, - }, - }) - - err := fakeOVN.controller.WatchNamespaces() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOVN.controller.WatchEgressFirewall() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + startOvn(dbSetup, []v1.Namespace{namespace1}, []egressfirewallapi.EgressFirewall{*egressFirewall}) asHash, _ := getNsAddrSetHashNames(namespace1.Name) + dbIDs := fakeOVN.controller.getEgressFirewallACLDbIDs(egressFirewall.Namespace, 0) ipv4ACL := libovsdbops.BuildACL( - buildEgressFwAclName("namespace1", t.EgressFirewallStartPriority), + getACLName(dbIDs), nbdb.ACLDirectionToLport, t.EgressFirewallStartPriority, "(ip4.dst == 1.2.3.4/23) && ip4.src == $"+asHash, @@ -618,10 +479,7 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { t.OvnACLLoggingMeter, "", false, - map[string]string{ - egressFirewallACLExtIdKey: "namespace1", - egressFirewallACLPriorityKey: fmt.Sprintf("%d", t.EgressFirewallStartPriority), - }, + dbIDs.GetExternalIDs(), nil, ) ipv4ACL.UUID = "ipv4ACL-UUID" @@ -631,7 +489,7 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { expectedDatabaseState := append(initialData, ipv4ACL) gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) - _, err = fakeOVN.fakeClient.EgressFirewallClient.K8sV1().EgressFirewalls(egressFirewall.Namespace).Get(context.TODO(), egressFirewall.Name, metav1.GetOptions{}) + _, err := fakeOVN.fakeClient.EgressFirewallClient.K8sV1().EgressFirewalls(egressFirewall.Namespace).Get(context.TODO(), egressFirewall.Name, metav1.GetOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) _, err = fakeOVN.fakeClient.EgressFirewallClient.K8sV1().EgressFirewalls(egressFirewall1.Namespace).Update(context.TODO(), egressFirewall1, metav1.UpdateOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -725,8 +583,9 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { types.MergePatchType, patchData, metav1.PatchOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) asHash, _ := getNsAddrSetHashNames(namespace1.Name) + dbIDs := fakeOVN.controller.getEgressFirewallACLDbIDs(egressFirewall.Namespace, 0) ipv4ACL := libovsdbops.BuildACL( - buildEgressFwAclName(namespace1.Name, t.EgressFirewallStartPriority), + getACLName(dbIDs), nbdb.ACLDirectionToLport, t.EgressFirewallStartPriority, fmt.Sprintf("(ip4.dst == %s) && ip4.src == $%s", nodeIP, asHash), @@ -734,10 +593,7 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { t.OvnACLLoggingMeter, "", false, - map[string]string{ - egressFirewallACLExtIdKey: namespace1.Name, - egressFirewallACLPriorityKey: fmt.Sprintf("%d", t.EgressFirewallStartPriority), - }, + dbIDs.GetExternalIDs(), nil, ) ipv4ACL.UUID = "ipv4ACL-UUID" @@ -810,8 +666,9 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { fakeOVN.controller.WatchEgressFirewall() asHash, _ := getNsAddrSetHashNames(namespace1.Name) + dbIDs := fakeOVN.controller.getEgressFirewallACLDbIDs(egressFirewall.Namespace, 0) ipv4ACL := libovsdbops.BuildACL( - buildEgressFwAclName("namespace1", t.EgressFirewallStartPriority), + getACLName(dbIDs), nbdb.ACLDirectionToLport, t.EgressFirewallStartPriority, "(ip4.dst == 1.2.3.5/23) && ip4.src == $"+asHash+" && ((tcp && ( tcp.dst == 100 )))", @@ -819,10 +676,7 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { t.OvnACLLoggingMeter, "", false, - map[string]string{ - egressFirewallACLExtIdKey: "namespace1", - egressFirewallACLPriorityKey: fmt.Sprintf("%d", t.EgressFirewallStartPriority), - }, + dbIDs.GetExternalIDs(), nil, ) ipv4ACL.UUID = "ipv4ACL-UUID" @@ -894,36 +748,12 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { }, }) - fakeOVN.startWithDBSetup(dbSetup, - &egressfirewallapi.EgressFirewallList{ - Items: []egressfirewallapi.EgressFirewall{ - *egressFirewall, - }, - }, - &v1.NamespaceList{ - Items: []v1.Namespace{ - namespace1, - }, - }, - &v1.NodeList{ - Items: []v1.Node{ - { - Status: v1.NodeStatus{ - Phase: v1.NodeRunning, - }, - ObjectMeta: newObjectMeta(node1Name, ""), - }, - }, - }) - - err := fakeOVN.controller.WatchNamespaces() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOVN.controller.WatchEgressFirewall() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + startOvn(dbSetup, []v1.Namespace{namespace1}, []egressfirewallapi.EgressFirewall{*egressFirewall}) asHash, _ := getNsAddrSetHashNames(namespace1.Name) + dbIDs := fakeOVN.controller.getEgressFirewallACLDbIDs(egressFirewall.Namespace, 0) ipv4ACL := libovsdbops.BuildACL( - buildEgressFwAclName("namespace1", t.EgressFirewallStartPriority), + getACLName(dbIDs), nbdb.ACLDirectionToLport, t.EgressFirewallStartPriority, "(ip4.dst == 1.2.3.4/23) && ip4.src == $"+asHash, @@ -931,10 +761,7 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { t.OvnACLLoggingMeter, "", false, - map[string]string{ - egressFirewallACLExtIdKey: "namespace1", - egressFirewallACLPriorityKey: fmt.Sprintf("%d", t.EgressFirewallStartPriority), - }, + dbIDs.GetExternalIDs(), nil, ) ipv4ACL.UUID = "ipv4ACL-UUID" @@ -951,7 +778,7 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { return fakeOVN.controller.nbClient.Connected() }).Should(gomega.BeFalse()) - _, err = fakeOVN.fakeClient.EgressFirewallClient.K8sV1().EgressFirewalls(egressFirewall.Namespace).Get(context.TODO(), egressFirewall.Name, metav1.GetOptions{}) + _, err := fakeOVN.fakeClient.EgressFirewallClient.K8sV1().EgressFirewalls(egressFirewall.Namespace).Get(context.TODO(), egressFirewall.Name, metav1.GetOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) _, err = fakeOVN.fakeClient.EgressFirewallClient.K8sV1().EgressFirewalls(egressFirewall1.Namespace).Update(context.TODO(), egressFirewall1, metav1.UpdateOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -1001,36 +828,12 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { }, }) - fakeOVN.startWithDBSetup(dbSetup, - &egressfirewallapi.EgressFirewallList{ - Items: []egressfirewallapi.EgressFirewall{ - *egressFirewall, - }, - }, - &v1.NamespaceList{ - Items: []v1.Namespace{ - namespace1, - }, - }, - &v1.NodeList{ - Items: []v1.Node{ - { - Status: v1.NodeStatus{ - Phase: v1.NodeRunning, - }, - ObjectMeta: newObjectMeta(node1Name, ""), - }, - }, - }) - - err := fakeOVN.controller.WatchNamespaces() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOVN.controller.WatchEgressFirewall() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + startOvn(dbSetup, []v1.Namespace{namespace1}, []egressfirewallapi.EgressFirewall{*egressFirewall}) asHash, _ := getNsAddrSetHashNames(namespace1.Name) + dbIDs := fakeOVN.controller.getEgressFirewallACLDbIDs(egressFirewall.Namespace, 0) ipv4ACL := libovsdbops.BuildACL( - buildEgressFwAclName("namespace1", t.EgressFirewallStartPriority), + getACLName(dbIDs), nbdb.ACLDirectionToLport, t.EgressFirewallStartPriority, "(ip4.dst == 1.2.3.4/23) && ip4.src == $"+asHash, @@ -1038,10 +841,7 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { t.OvnACLLoggingMeter, "", false, - map[string]string{ - egressFirewallACLExtIdKey: "namespace1", - egressFirewallACLPriorityKey: fmt.Sprintf("%d", t.EgressFirewallStartPriority), - }, + dbIDs.GetExternalIDs(), nil, ) ipv4ACL.UUID = "ipv4ACL-UUID" @@ -1141,8 +941,9 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { match = fmt.Sprintf("(ip6.dst == %s) && ip6.src == $%s", nodeIP6, asHashv6) } + dbIDs := fakeOVN.controller.getEgressFirewallACLDbIDs(egressFirewall.Namespace, 0) acl := libovsdbops.BuildACL( - buildEgressFwAclName("namespace1", t.EgressFirewallStartPriority), + getACLName(dbIDs), nbdb.ACLDirectionToLport, t.EgressFirewallStartPriority, match, @@ -1150,10 +951,7 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { t.OvnACLLoggingMeter, "", false, - map[string]string{ - egressFirewallACLExtIdKey: "namespace1", - egressFirewallACLPriorityKey: fmt.Sprintf("%d", t.EgressFirewallStartPriority), - }, + dbIDs.GetExternalIDs(), nil, ) acl.UUID = "ACL-UUID" @@ -1186,38 +984,15 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { }, }, }) - fakeOVN.startWithDBSetup(dbSetup, - &egressfirewallapi.EgressFirewallList{ - Items: []egressfirewallapi.EgressFirewall{ - *egressFirewall, - }, - }, - &v1.NamespaceList{ - Items: []v1.Namespace{ - namespace1, - }, - }, - &v1.NodeList{ - Items: []v1.Node{ - { - Status: v1.NodeStatus{ - Phase: v1.NodeRunning, - }, - ObjectMeta: newObjectMeta(node1Name, ""), - }, - }, - }) + startOvn(dbSetup, []v1.Namespace{namespace1}, []egressfirewallapi.EgressFirewall{*egressFirewall}) - err := fakeOVN.controller.WatchNamespaces() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - _, err = fakeOVN.fakeClient.EgressFirewallClient.K8sV1().EgressFirewalls(egressFirewall.Namespace).Get(context.TODO(), egressFirewall.Name, metav1.GetOptions{}) + _, err := fakeOVN.fakeClient.EgressFirewallClient.K8sV1().EgressFirewalls(egressFirewall.Namespace).Get(context.TODO(), egressFirewall.Name, metav1.GetOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - fakeOVN.controller.WatchEgressFirewall() - asHash, _ := getNsAddrSetHashNames(namespace1.Name) + dbIDs := fakeOVN.controller.getEgressFirewallACLDbIDs(egressFirewall.Namespace, 0) acl := libovsdbops.BuildACL( - buildEgressFwAclName("namespace1", t.EgressFirewallStartPriority), + getACLName(dbIDs), nbdb.ACLDirectionToLport, t.EgressFirewallStartPriority, "(ip4.dst == 0.0.0.0/0 && ip4.dst != "+clusterSubnetStr+") && ip4.src == $"+asHash, @@ -1225,10 +1000,7 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { t.OvnACLLoggingMeter, "", false, - map[string]string{ - egressFirewallACLExtIdKey: "namespace1", - egressFirewallACLPriorityKey: fmt.Sprintf("%d", t.EgressFirewallStartPriority), - }, + dbIDs.GetExternalIDs(), nil, ) acl.UUID = "acl-UUID" @@ -1262,26 +1034,12 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { }, }) - fakeOVN.startWithDBSetup(dbSetup, - &egressfirewallapi.EgressFirewallList{ - Items: []egressfirewallapi.EgressFirewall{ - *egressFirewall, - }, - }, - &v1.NamespaceList{ - Items: []v1.Namespace{ - namespace1, - }, - }) - - err := fakeOVN.controller.WatchNamespaces() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOVN.controller.WatchEgressFirewall() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + startOvn(dbSetup, []v1.Namespace{namespace1}, []egressfirewallapi.EgressFirewall{*egressFirewall}) asHash, _ := getNsAddrSetHashNames(namespace1.Name) + aclIDs1 := fakeOVN.controller.getEgressFirewallACLDbIDs(namespace1.Name, 0) ipv4ACL1 := libovsdbops.BuildACL( - buildEgressFwAclName(namespace1.Name, t.EgressFirewallStartPriority), + getACLName(aclIDs1), nbdb.ACLDirectionToLport, t.EgressFirewallStartPriority, "(ip4.dst == 1.2.3.5/23) && ip4.src == $"+asHash, @@ -1289,15 +1047,14 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { t.OvnACLLoggingMeter, "", false, - map[string]string{ - egressFirewallACLExtIdKey: namespace1.Name, - egressFirewallACLPriorityKey: fmt.Sprintf("%d", t.EgressFirewallStartPriority), - }, + aclIDs1.GetExternalIDs(), nil, ) ipv4ACL1.UUID = "ipv4ACL1-UUID" + + aclIDs2 := fakeOVN.controller.getEgressFirewallACLDbIDs(egressFirewall.Namespace, 1) ipv4ACL2 := libovsdbops.BuildACL( - buildEgressFwAclName(namespace1.Name, t.EgressFirewallStartPriority-1), + getACLName(aclIDs2), nbdb.ACLDirectionToLport, t.EgressFirewallStartPriority-1, "(ip4.dst == 2.2.3.5/23) && ip4.src == $"+asHash, @@ -1305,10 +1062,7 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { t.OvnACLLoggingMeter, "", false, - map[string]string{ - egressFirewallACLExtIdKey: namespace1.Name, - egressFirewallACLPriorityKey: fmt.Sprintf("%d", t.EgressFirewallStartPriority-1), - }, + aclIDs2.GetExternalIDs(), nil, ) ipv4ACL2.UUID = "ipv4ACL2-UUID" @@ -1319,9 +1073,7 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) - // NOTE: syncEgressFirewall is not calling libovsdbops.BuildACL and directly calls CreateOrUpdateACLs - // This part of test ensures syncEgressFirewall code path is tested well and that we truncate the ACL names correctly - err = fakeOVN.controller.syncEgressFirewall([]interface{}{*egressFirewall}) + err := fakeOVN.controller.syncEgressFirewall([]interface{}{egressFirewall}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) diff --git a/go-controller/pkg/ovn/external_ids_syncer/acl/acl_sync.go b/go-controller/pkg/ovn/external_ids_syncer/acl/acl_sync.go index 1b21550216..58db133754 100644 --- a/go-controller/pkg/ovn/external_ids_syncer/acl/acl_sync.go +++ b/go-controller/pkg/ovn/external_ids_syncer/acl/acl_sync.go @@ -37,7 +37,8 @@ const ( // staleArpAllowPolicyMatch "was" the old match used when creating default allow ARP ACLs for a namespace // NOTE: This is succeed by arpAllowPolicyMatch to allow support for IPV6. This is currently only // used when removing stale ACLs from the syncNetworkPolicy function and should NOT be used in any main logic. - staleArpAllowPolicyMatch = "arp" + staleArpAllowPolicyMatch = "arp" + egressFirewallACLExtIdKey = "egressFirewall" ) type aclSyncer struct { @@ -93,6 +94,10 @@ func (syncer *aclSyncer) SyncACLs(existingNodes *v1.NodeList) error { klog.Infof("Found %d stale default deny netpol ACLs", len(gressPolicyACLs)) updatedACLs = append(updatedACLs, defaultDenyACLs...) + egressFirewallACLs := syncer.updateStaleEgressFirewallACLs(legacyACLs) + klog.Infof("Found %d stale egress firewall ACLs", len(gressPolicyACLs)) + updatedACLs = append(updatedACLs, egressFirewallACLs...) + // delete stale duplicating acls first _, err = libovsdbops.TransactAndCheck(syncer.nbClient, deleteACLs) if err != nil { @@ -403,3 +408,33 @@ func (syncer *aclSyncer) updateStaleDefaultDenyNetpolACLs(legacyACLs []*nbdb.ACL } return } + +func (syncer *aclSyncer) getEgressFirewallACLDbIDs(namespace string, ruleIdx int) *libovsdbops.DbObjectIDs { + return libovsdbops.NewDbObjectIDs(libovsdbops.ACLEgressFirewall, syncer.controllerName, + map[libovsdbops.ExternalIDKey]string{ + libovsdbops.ObjectNameKey: namespace, + libovsdbops.RuleIndex: strconv.Itoa(ruleIdx), + }) +} + +func (syncer *aclSyncer) updateStaleEgressFirewallACLs(legacyACLs []*nbdb.ACL) []*nbdb.ACL { + updatedACLs := []*nbdb.ACL{} + for _, acl := range legacyACLs { + if acl.Priority < types.MinimumReservedEgressFirewallPriority || acl.Priority > types.EgressFirewallStartPriority { + // not egress firewall acl + continue + } + namespace, ok := acl.ExternalIDs[egressFirewallACLExtIdKey] + if !ok || namespace == "" { + klog.Errorf("Failed to sync stale egress firewall acl: expected non-empty %s key in ExternalIDs %+v", + egressFirewallACLExtIdKey, acl.ExternalIDs) + continue + } + // egress firewall ACL.priority = types.EgressFirewallStartPriority - rule.idx => + // rule.idx = types.EgressFirewallStartPriority - ACL.priority + dbIDs := syncer.getEgressFirewallACLDbIDs(namespace, types.EgressFirewallStartPriority-acl.Priority) + acl.ExternalIDs = dbIDs.GetExternalIDs() + updatedACLs = append(updatedACLs, acl) + } + return updatedACLs +} diff --git a/go-controller/pkg/ovn/external_ids_syncer/acl/acl_sync_test.go b/go-controller/pkg/ovn/external_ids_syncer/acl/acl_sync_test.go index 7cba578c61..9d5153b9b3 100644 --- a/go-controller/pkg/ovn/external_ids_syncer/acl/acl_sync_test.go +++ b/go-controller/pkg/ovn/external_ids_syncer/acl/acl_sync_test.go @@ -672,4 +672,39 @@ var _ = ginkgo.Describe("OVN ACL Syncer", func() { } testSyncerWithData(testData, controllerName, initialDb, finalDb, nil) }) + ginkgo.It("updates egress firewall acls", func() { + testData := []aclSync{ + { + before: libovsdbops.BuildACL( + "random", + nbdb.ACLDirectionFromLport, + types.EgressFirewallStartPriority, + "any", + nbdb.ACLActionDrop, + types.OvnACLLoggingMeter, + "", + false, + map[string]string{egressFirewallACLExtIdKey: namespace1}, + nil, + ), + after: syncerToBuildData.getEgressFirewallACLDbIDs(namespace1, 0), + }, + { + before: libovsdbops.BuildACL( + "random2", + nbdb.ACLDirectionFromLport, + types.EgressFirewallStartPriority-1, + "any2", + nbdb.ACLActionDrop, + types.OvnACLLoggingMeter, + "", + false, + map[string]string{egressFirewallACLExtIdKey: namespace1}, + nil, + ), + after: syncerToBuildData.getEgressFirewallACLDbIDs(namespace1, 1), + }, + } + testSyncerWithData(testData, controllerName, []libovsdbtest.TestData{}, nil, nil) + }) }) From 447b11bad74e82940676749bb0d2a50d16ebe639 Mon Sep 17 00:00:00 2001 From: Nadia Pinaeva Date: Tue, 3 Jan 2023 16:30:40 +0100 Subject: [PATCH 08/40] Finalize new ACL dbIndex usage: 1. rename BuildACLFromDbIndex to BuildACL, since it is the only method that should be used now. 2. Rename getACLMatchFromACLDir to getACLMatch, remove stale version. 3. update isEquivalentACL to checkACLPrimaryID, that matches on primaryID until we get client indexes. 4. Add acls.md doc explaining all acls that are used by ovn-k and their dependencies with examples 5. Update multicast docs Add missing links to README.md Signed-off-by: Nadia Pinaeva --- README.md | 4 + docs/acls.md | 480 ++++++++++++++++++++++++ docs/multicast.md | 55 +-- go-controller/pkg/libovsdbops/acl.go | 33 +- go-controller/pkg/ovn/acl.go | 64 +--- go-controller/pkg/ovn/egressfirewall.go | 2 +- go-controller/pkg/ovn/gress_policy.go | 4 +- go-controller/pkg/ovn/multicast.go | 14 +- go-controller/pkg/ovn/multicast_test.go | 8 +- go-controller/pkg/ovn/policy.go | 18 +- 10 files changed, 529 insertions(+), 153 deletions(-) create mode 100644 docs/acls.md diff --git a/README.md b/README.md index dc33595bf5..f424a62ffd 100644 --- a/README.md +++ b/README.md @@ -140,6 +140,8 @@ that can be brought up locally and within a few minutes. [Debugging OVN](./docs/debugging.md) +[Exposed metrics](./docs/metrics.md) + The golang based [ovn kubernetes go-controller](./go-controller/README.md) is a reliable way to deploy the OVN SDN using kubernetes clients and watchers based on golang. Contains `ovnkube` and `ovn-k8s-cni-overlay` build and usage instructions. @@ -207,6 +209,8 @@ cluster network. [ovnkube-trace](./docs/ovnkube-trace.md) a tool to trace packet simulations between points in an ovn-kubernetes driven cluster. +[ACLs used by ovn-k and their priorities](./docs/acls.md) + # OVN Kubernetes Basics A good resource to get started with understanding `ovn-kubernetes` is the following recording and slides, which run through the basic architecture and functionality of the system. [slides](https://docs.google.com/presentation/d/1ZtwP3t6uNAU0g4S7IbqSxPg2bmQW-pPGyMW2ZNj9Nrg/edit?usp=sharing) diff --git a/docs/acls.md b/docs/acls.md new file mode 100644 index 0000000000..90802f88b6 --- /dev/null +++ b/docs/acls.md @@ -0,0 +1,480 @@ +# ACLs + +## Introduction + +ovn-k uses ACLs to implement multiple features, this doc is intended to list all of them, explaining their IDs, +priorities, and dependencies. + +OVN has 3 independent sets of ACLs, based on direction and pipeline stage, which are applied in the following order: +1. `direction="from-lport"` +2. `direction="from-lport", options={"apply-after-lb": "true"}` +3. `direction="to-lport"` + +The priorities between these stages are independent! + +OVN will apply the `from-lport` ACLs in two stages. ACLs without `apply-after-lb` set, will be applied before the +load balancer stage, and ACLs with this option set will be applied after the load balancer stage. `to-lport` are always +applied after the load balancer stage. + +For now, ovn-k doesn't use `direction="from-lport"` ACLs, since most of the time we need to apply ACLs after loadbalancing. +Here is the current order in which ACLs for different objects are applied (check the rest of the doc for details) + +### `direction="from-lport", options={"apply-after-lb": "true"}` + +1. egress multicast, allow priority = `1012`, deny priority = `1011` +2. egress network policy, default deny priority = `1000`, allow priority = `1001` + +### `direction="to-lport"` + +1. egress firewall, priorities = `2000`-`10000` (egress firewall is applied on this stage to be independently applied +after egress network policy) +2. ingress multicast, allow priority = `1012`, deny priority = `1011` +3. ingress network policy, default deny priority = `1000`, allow priority = `1001` + +## Egress Firewall + +Egress Firewall creates 1 ACL for every specified rule, with `ExternalIDs["k8s.ovn.org/owner-type"]=EgressFirewall` +e.g. given object: + +```yaml +kind: EgressFirewall +apiVersion: k8s.ovn.org/v1 +metadata: + name: default + namespace: default +spec: + egress: + - type: Allow + to: + dnsName: www.openvswitch.org + - type: Allow + to: + cidrSelector: 1.2.3.0/24 + ports: + - protocol: UDP + port: 55 + - type: Deny + to: + cidrSelector: 0.0.0.0/0 +``` + +will create 3 ACLs: + +``` +action : allow +direction : to-lport +external_ids : { + "k8s.ovn.org/id"="default-network-controller:EgressFirewall:default:10000", + "k8s.ovn.org/name"=default, + "k8s.ovn.org/owner-controller"=default-network-controller, + "k8s.ovn.org/owner-type"=EgressFirewall, + rule-idx="0" +} +label : 0 +log : false +match : "(ip4.dst == $a5272457940446250407) && ip4.src == $a4322231855293774466" +meter : acl-logging +name : "EF:default:10000" +options : {} +priority : 10000 +severity : [] + +action : allow +direction : to-lport +external_ids : { + "k8s.ovn.org/id"="default-network-controller:EgressFirewall:default:9999", + "k8s.ovn.org/name"=default, + "k8s.ovn.org/owner-controller"=default-network-controller, + "k8s.ovn.org/owner-type"=EgressFirewall, + rule-idx="1" +} +label : 0 +log : false +match : "(ip4.dst == 1.2.3.0/24) && ip4.src == $a4322231855293774466 && ((udp && ( udp.dst == 55 )))" +meter : acl-logging +name : "EF:default:9999" +options : {} +priority : 9999 +severity : [] + +action : drop +direction : to-lport +external_ids : { + "k8s.ovn.org/id"="default-network-controller:EgressFirewall:default:9998", + "k8s.ovn.org/name"=default, + "k8s.ovn.org/owner-controller"=default-network-controller, + "k8s.ovn.org/owner-type"=EgressFirewall, + rule-idx="2" +} +label : 0 +log : false +match : "(ip4.dst == 0.0.0.0/0 && ip4.dst != 10.244.0.0/16) && ip4.src == $a4322231855293774466" +meter : acl-logging +name : "EF:default:9998" +options : {} +priority : 9998 +severity : [] +``` + +Egress firewall should be applied after egress network policy independently, to make sure that connection that are +allowed by network policy, but denied by egress firewall will be dropped. + +## Multicast + +For more details about Multicast in ovn see [Multicast docs](./multicast.md) +Multicast creates 2 types of ACLs: global default ACLs with `ExternalIDs["k8s.ovn.org/owner-type"]=MulticastCluster` +and per-namespace ACLs with `ExternalIDs["owner_object_type"]="MulticastNS"`. + +When multicast is enabled for ovn-k, you will find 4 default global ACLs: +- 2 ACLs (ingress/egress) dropping all multicast traffic - on all switches (via clusterPortGroup) +- 2 ACLs (ingress/egress) allowing all multicast traffic - on clusterRouterPortGroup + (that allows multicast between pods that reside on different nodes, see + https://github.com/ovn-org/ovn-kubernetes/commit/3864f2b6463392ae2d80c18d06bd46ec44e639f9 for more details) + +``` +action : allow +direction : from-lport +external_ids : { + direction=Egress, + "k8s.ovn.org/id"="default-network-controller:MulticastCluster:AllowInterNode:Ingress", + "k8s.ovn.org/owner-controller"=default-network-controller, + "k8s.ovn.org/owner-type"=MulticastCluster, + type=AllowInterNode +} +label : 0 +log : false +match : "inport == @clusterRtrPortGroup && (ip4.mcast || mldv1 || mldv2 || (ip6.dst[120..127] == 0xff && ip6.dst[116] == 1))" +meter : acl-logging +name : [] +options : {apply-after-lb="true"} +priority : 1012 +severity : [] + +action : allow +direction : to-lport +external_ids : { + direction=Ingress, + "k8s.ovn.org/id"="default-network-controller:MulticastCluster:AllowInterNode:Ingress", + "k8s.ovn.org/owner-controller"=default-network-controller, + "k8s.ovn.org/owner-type"=MulticastCluster, + type=AllowInterNode +} +label : 0 +log : false +match : "outport == @clusterRtrPortGroup && (ip4.mcast || mldv1 || mldv2 || (ip6.dst[120..127] == 0xff && ip6.dst[116] == 1))" +meter : acl-logging +name : [] +options : {} +priority : 1012 +severity : [] + +action : drop +direction : from-lport +external_ids : { + direction=Egress, + "k8s.ovn.org/id"="default-network-controller:MulticastCluster:DefaultDeny:Egress", + "k8s.ovn.org/owner-controller"=default-network-controller, + "k8s.ovn.org/owner-type"=MulticastCluster, + type=DefaultDeny +} +label : 0 +log : false +match : "(ip4.mcast || mldv1 || mldv2 || (ip6.dst[120..127] == 0xff && ip6.dst[116] == 1))" +meter : acl-logging +name : [] +options : {apply-after-lb="true"} +priority : 1011 +severity : [] + +action : drop +direction : to-lport +external_ids : { + direction=Ingress, + "k8s.ovn.org/id"="default-network-controller:MulticastCluster:DefaultDeny:Egress", + "k8s.ovn.org/owner-controller"=default-network-controller, + "k8s.ovn.org/owner-type"=MulticastCluster, + type=DefaultDeny +} +label : 0 +log : false +match : "(ip4.mcast || mldv1 || mldv2 || (ip6.dst[120..127] == 0xff && ip6.dst[116] == 1))" +meter : acl-logging +name : [] +options : {} +priority : 1011 +severity : [] + +``` + +For every namespace with enabled multicast, there are 2 more ACLs, e.g. for default namespace: +``` +action : allow +direction : to-lport +external_ids : { + "k8s.ovn.org/id"="default-network-controller:MulticastNS:default:Allow_Ingress", + "k8s.ovn.org/name"=default, + "k8s.ovn.org/owner-controller"=default-network-controller, + "k8s.ovn.org/owner-type"=MulticastNS, + direction=Ingress +} +label : 0 +log : false +match : "outport == @a16982411286042166782 && (igmp || (ip4.src == $a4322231855293774466 && ip4.mcast))" +meter : acl-logging +name : [] +options : {} +priority : 1012 +severity : [] + +action : allow +direction : from-lport +external_ids : { + "k8s.ovn.org/id"="default-network-controller:MulticastNS:default:Allow_Egress", + "k8s.ovn.org/name"=default, + "k8s.ovn.org/owner-controller"=default-network-controller, + "k8s.ovn.org/owner-type"=MulticastNS, + tdirection=Egress +} +label : 0 +log : false +match : "inport == @a16982411286042166782 && ip4.mcast" +meter : acl-logging +name : [] +options : {apply-after-lb="true"} +priority : 1012 +severity : [] +``` + +## Network Policy + +Every node has 1 ACL to allow traffic from that node's management port IP with `ExternalIDs["k8s.ovn.org/owner-type"]=NetpolNode`, like + +``` +action : allow-related +direction : to-lport +external_ids : { + ip="10.244.2.2", + "k8s.ovn.org/id"="default-network-controller:NetpolNode:ovn-worker:10.244.2.2", + "k8s.ovn.org/name"=ovn-worker, + "k8s.ovn.org/owner-controller"=default-network-controller, + "k8s.ovn.org/owner-type"=NetpolNode +} +label : 0 +log : false +match : "ip4.src==10.244.2.2" +meter : acl-logging +name : [] +options : {} +priority : 1001 +severity : [] + +``` + +There are also 2 Default Allow ACLs for the hairpinned traffic (pod->svc->same pod) with `ExternalIDs["k8s.ovn.org/owner-type"]=NetpolDefault`, +which match on the special V4OVNServiceHairpinMasqueradeIP and V6OVNServiceHairpinMasqueradeIP addresses. +These IPs are assigned as src IP to the hairpinned packets. + +``` +action : allow-related +direction : to-lport +external_ids : { + direction=Ingress, + "k8s.ovn.org/id"="default-network-controller:NetpolDefault:allow-hairpinning:Ingress", + "k8s.ovn.org/name"=allow-hairpinning, + "k8s.ovn.org/owner-controller"=default-network-controller, + "k8s.ovn.org/owner-type"=NetpolDefault +} +label : 0 +log : false +match : "ip4.src == 169.254.169.5" +meter : acl-logging +name : [] +options : {} +priority : 1001 +severity : [] + +action : allow-related +direction : from-lport +external_ids : { + direction=Egress, + "k8s.ovn.org/id"="default-network-controller:NetpolDefault:allow-hairpinning:Egress", + "k8s.ovn.org/name"=allow-hairpinning, + "k8s.ovn.org/owner-controller"=default-network-controller, + "k8s.ovn.org/owner-type"=NetpolDefault +} +label : 0 +log : false +match : "ip4.src == 169.254.169.5" +meter : acl-logging +name : [] +options : {apply-after-lb="true"} +priority : 1001 +severity : [] +``` + +Network Policy creates default deny ACLs for every namespace that has at least 1 network policy with +`ExternalIDs["k8s.ovn.org/owner-type"]=NetpolNamespace`. +There are 2 ACL types (`defaultDeny` and `arpAllow`) for every policy direction (Ingress/Egress) e.g. for `default` namespace, + +Egress: +``` +action : allow +direction : from-lport +external_ids : { + direction=Egress, + "k8s.ovn.org/id"="default-network-controller:NetpolNamespace:default:Egress:arpAllow", + "k8s.ovn.org/name"=default, + "k8s.ovn.org/owner-controller"=default-network-controller, + "k8s.ovn.org/owner-type"=NetpolNamespace, + type=arpAllow +} +label : 0 +log : false +match : "inport == @a16982411286042166782_egressDefaultDeny && (arp || nd)" +meter : acl-logging +name : "NP:default:Egress" +options : {apply-after-lb="true"} +priority : 1001 +severity : [] + +action : drop +direction : from-lport +external_ids : { + direction=Egress, + "k8s.ovn.org/id"="default-network-controller:NetpolNamespace:default:Egress:defaultDeny", + "k8s.ovn.org/name"=default, + "k8s.ovn.org/owner-controller"=default-network-controller, + "k8s.ovn.org/owner-type"=NetpolNamespace, + type=defaultDeny +} +label : 0 +log : false +match : "inport == @a16982411286042166782_egressDefaultDeny" +meter : acl-logging +name : "NP:default:Egress" +options : {apply-after-lb="true"} +priority : 1000 +severity : [] +``` + +Ingress: +``` +_uuid : a6ffa9d4-e811-4aaf-9505-87cc0b2f442a +action : allow +direction : to-lport +external_ids : { + direction=Ingress, + "k8s.ovn.org/id"="default-network-controller:NetpolNamespace:default:Ingress:arpAllow", + "k8s.ovn.org/name"=default, + "k8s.ovn.org/owner-controller"=default-network-controller, + "k8s.ovn.org/owner-type"=NetpolNamespace, + type=arpAllow +} +label : 0 +log : false +match : "outport == @a16982411286042166782_ingressDefaultDeny && (arp || nd)" +meter : acl-logging +name : "NP:default:Ingress" +options : {} +priority : 1001 +severity : [] + +action : drop +direction : to-lport +external_ids : { + direction=Ingress, + "k8s.ovn.org/id"="default-network-controller:NetpolNamespace:default:Ingress:defaultDeny", + "k8s.ovn.org/name"=default, + "k8s.ovn.org/owner-controller"=default-network-controller, + "k8s.ovn.org/owner-type"=NetpolNamespace, + type=defaultDeny +} +label : 0 +log : false +match : "outport == @a16982411286042166782_ingressDefaultDeny" +meter : acl-logging +name : "NP:default:Ingress" +options : {} +priority : 1000 +severity : [] + +``` + +There are also ACLs owned by every network policy object with `ExternalIDs["k8s.ovn.org/owner-type"]=NetworkPolicy`, e.g. +for the following object + +``` +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: test-policy + namespace: default +spec: + podSelector: {} + policyTypes: + - Ingress + - Egress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: default + podSelector: + matchLabels: + app: demo + egress: + - to: + - ipBlock: + cidr: 10.244.1.5/32 +``` + +2 ACLs will be created: + +``` +action : allow-related +direction : from-lport +external_ids : { + direction=egress, + gress-index="0", + ip-block-index="0", + "k8s.ovn.org/id"="default-network-controller:NetworkPolicy:default:test-policy:egress:0:-1:0", + "k8s.ovn.org/name"="default:test-policy", + "k8s.ovn.org/owner-controller"=default-network-controller, + "k8s.ovn.org/owner-type"=NetworkPolicy, + port-policy-index="-1" +} +label : 0 +log : false +match : "ip4.dst == 10.244.1.5/32 && inport == @a2653181086423119552" +meter : acl-logging +name : "NP:default:test-policy:egress:0:-1:0" +options : {apply-after-lb="true"} +priority : 1001 +severity : [] + +action : allow-related +direction : to-lport +external_ids : { + direction=ingress, + gress-index="0", + ip-block-index="-1", + "k8s.ovn.org/id"="default-network-controller:NetworkPolicy:default:test-policy:ingress:0:-1:-1", + "k8s.ovn.org/name"="default:test-policy", + "k8s.ovn.org/owner-controller"=default-network-controller, + "k8s.ovn.org/owner-type"=NetworkPolicy, + port-policy-index="-1" +} +label : 0 +log : false +match : "(ip4.src == {$a3733136965153973077} || (ip4.src == 169.254.169.5 && ip4.dst == {$a3733136965153973077})) && outport == @a2653181086423119552" +meter : acl-logging +name : "NP:default:test-policy:ingress:0:-1:-1" +options : {} +priority : 1001 +severity : [] + +``` + +`"k8s.ovn.org/name"` is the `:` of network policy object, `gress-index` is the index of gress policy in +the `NetworkPolicy.Spec.[In/E]gress`, check `gress_policy.go:getNetpolACLDbIDs` for more details on the rest of the fields. + diff --git a/docs/multicast.md b/docs/multicast.md index d701d29dd5..bbdc716551 100644 --- a/docs/multicast.md +++ b/docs/multicast.md @@ -37,35 +37,22 @@ virt-launcher-vmi-masquerade-thr9j [map[ip:10.244.1.8]] ``` The implementation of IPv4 multicast for ovn-kubernetes relies on: -- an ACL dropping all egress multicast traffic - on all pods -- an ACL dropping all ingress multicast traffic - on all pods +- 2 ACLs (ingress/egress) dropping all multicast traffic - on all switches (via clusterPortGroup) +- 2 ACLs (ingress/egress) allowing all multicast traffic - on clusterRouterPortGroup +(that allows multicast between pods that reside on different nodes, see +https://github.com/ovn-org/ovn-kubernetes/commit/3864f2b6463392ae2d80c18d06bd46ec44e639f9 for more details) -These ACLs look like: + +These ACLs Matches look like: ``` -# ingress direction -_uuid : d5024a91-12c8-49ea-9f11-fb4bb3878613 -action : drop -direction : to-lport -external_ids : {default-deny-policy-type=Ingress} -log : false -match : "(ip4.mcast || mldv1 || mldv2 || (ip6.dst[120..127] == 0xff && ip6.dst[116] == 1))" -meter : acl-logging -name : "97faee09-ae44-4b4e-9bd3-6df88116b25a_DefaultDenyMulticastIngres" -priority : 1011 -severity : info +# deny all multicast match +"(ip4.mcast || mldv1 || mldv2 || (ip6.dst[120..127] == 0xff && ip6.dst[116] == 1))" -# egress direction -_uuid : 150b3f92-9cbc-482d-b5c6-1037be2ca255 -action : drop -direction : from-lport -external_ids : {default-deny-policy-type=Egress} -log : false -match : "(ip4.mcast || mldv1 || mldv2 || (ip6.dst[120..127] == 0xff && ip6.dst[116] == 1))" -meter : acl-logging -name : "97faee09-ae44-4b4e-9bd3-6df88116b25a_DefaultDenyMulticastEgress" -priority : 1011 -severity : info +# allow clusterPortGroup match ingress +"outport == @clusterRtrPortGroup && (ip4.mcast || mldv1 || mldv2 || (ip6.dst[120..127] == 0xff && ip6.dst[116] == 1))" +# allow clusterPortGroup match egress +"inport == @clusterRtrPortGroup && (ip4.mcast || mldv1 || mldv2 || (ip6.dst[120..127] == 0xff && ip6.dst[116] == 1))" ``` Then, for each annotated(`k8s.ovn.org/multicast-enabled=true`) namespace, two @@ -74,28 +61,10 @@ that apply to the `default` namespace. ``` # egress direction -_uuid : f086c9b7-fa61-4a91-b545-f228f6cf954b -action : allow -direction : from-lport -external_ids : {default-deny-policy-type=Egress} -log : false match : "inport == @a16982411286042166782 && ip4.mcast" -meter : acl-logging -name : default_MulticastAllowEgress -priority : 1012 -severity : info # ingress direction -_uuid : b930b6ea-5b16-4eb1-b962-6b3e9273d0a0 -action : allow -direction : to-lport -external_ids : {default-deny-policy-type=Ingress} -log : false match : "outport == @a16982411286042166782 && (igmp || (ip4.src == $a5154718082306775057 && ip4.mcast))" -meter : acl-logging -name : default_MulticastAllowIngress -priority : 1012 -severity : info ``` As can be seen in the match condition of the ACLs above, the former ACL allows diff --git a/go-controller/pkg/libovsdbops/acl.go b/go-controller/pkg/libovsdbops/acl.go index 39158c5b37..de9effec0f 100644 --- a/go-controller/pkg/libovsdbops/acl.go +++ b/go-controller/pkg/libovsdbops/acl.go @@ -3,8 +3,6 @@ package libovsdbops import ( "context" "fmt" - "reflect" - libovsdbclient "github.com/ovn-org/libovsdb/client" libovsdb "github.com/ovn-org/libovsdb/ovsdb" @@ -26,25 +24,10 @@ func getACLMutableFields(acl *nbdb.ACL) []interface{} { &acl.Name, &acl.Options, &acl.Priority, &acl.Severity} } -// isEquivalentACL if it has same uuid, or if it has same name -// and external ids, or if it has same priority, direction, match -// and action. -func isEquivalentACL(existing *nbdb.ACL, searched *nbdb.ACL) bool { - if searched.UUID != "" && existing.UUID == searched.UUID { - return true - } - - eName := GetACLName(existing) - sName := GetACLName(searched) - // TODO if we want to support adding/removing external ids, - // we need to compare them differently, perhaps just the common subset - if eName != "" && eName == sName && reflect.DeepEqual(existing.ExternalIDs, searched.ExternalIDs) { - return true - } - return existing.Priority == searched.Priority && - existing.Direction == searched.Direction && - existing.Match == searched.Match && - existing.Action == searched.Action +// checkACLPrimaryID is a temporary replacement for client indexes, matches on ExternalIDs[PrimaryIDKey]. +func checkACLPrimaryID(existing *nbdb.ACL, searched *nbdb.ACL) bool { + return searched.ExternalIDs != nil && existing.ExternalIDs != nil && + existing.ExternalIDs[PrimaryIDKey.String()] == searched.ExternalIDs[PrimaryIDKey.String()] } type aclPredicate func(*nbdb.ACL) bool @@ -67,7 +50,7 @@ func FindACLs(nbClient libovsdbclient.Client, acls []*nbdb.ACL) ([]*nbdb.ACL, er found := []*nbdb.ACL{} opModel := operationModel{ Model: acl, - ModelPredicate: func(item *nbdb.ACL) bool { return isEquivalentACL(item, acl) }, + ModelPredicate: func(item *nbdb.ACL) bool { return checkACLPrimaryID(item, acl) }, ExistingResult: &found, ErrNotFound: false, BulkOp: false, @@ -141,7 +124,7 @@ func CreateOrUpdateACLsOps(nbClient libovsdbclient.Client, ops []libovsdb.Operat } opModel := operationModel{ Model: acl, - ModelPredicate: func(item *nbdb.ACL) bool { return isEquivalentACL(item, acl) }, + ModelPredicate: func(item *nbdb.ACL) bool { return checkACLPrimaryID(item, acl) }, OnModelUpdates: getACLMutableFields(acl), ErrNotFound: false, BulkOp: false, @@ -160,7 +143,7 @@ func UpdateACLsOps(nbClient libovsdbclient.Client, ops []libovsdb.Operation, acl acl := acls[i] opModel := operationModel{ Model: acl, - ModelPredicate: func(item *nbdb.ACL) bool { return isEquivalentACL(item, acl) }, + ModelPredicate: func(item *nbdb.ACL) bool { return checkACLPrimaryID(item, acl) }, OnModelUpdates: getACLMutableFields(acl), ErrNotFound: true, BulkOp: false, @@ -192,7 +175,7 @@ func UpdateACLsLoggingOps(nbClient libovsdbclient.Client, ops []libovsdb.Operati acl := acls[i] opModel := operationModel{ Model: acl, - ModelPredicate: func(item *nbdb.ACL) bool { return isEquivalentACL(item, acl) }, + ModelPredicate: func(item *nbdb.ACL) bool { return checkACLPrimaryID(item, acl) }, OnModelUpdates: []interface{}{&acl.Severity, &acl.Log}, ErrNotFound: true, BulkOp: false, diff --git a/go-controller/pkg/ovn/acl.go b/go-controller/pkg/ovn/acl.go index 9ca7534e27..70db625299 100644 --- a/go-controller/pkg/ovn/acl.go +++ b/go-controller/pkg/ovn/acl.go @@ -28,17 +28,6 @@ const ( lportEgressAfterLB aclPipelineType = "from-lport-after-lb" ) -func aclTypeToPolicyType(aclT aclPipelineType) knet.PolicyType { - switch aclT { - case lportEgressAfterLB: - return knet.PolicyTypeEgress - case lportIngress: - return knet.PolicyTypeIngress - default: - panic(fmt.Sprintf("Failed to convert aclPipelineType to PolicyType: unknown acl type %s", aclT)) - } -} - func policyTypeToAclPipeline(policyType knet.PolicyType) aclPipelineType { switch policyType { case knet.PolicyTypeEgress: @@ -95,42 +84,9 @@ func getACLName(dbIDs *libovsdbops.DbObjectIDs) string { } // BuildACL should be used to build ACL instead of directly calling libovsdbops.BuildACL. -// It can properly set and reset log settings for ACL based on ACLLoggingLevels -func BuildACL(aclName string, priority int, match, action string, - logLevels *ACLLoggingLevels, aclT aclPipelineType, externalIDs map[string]string) *nbdb.ACL { - var options map[string]string - var direction string - switch aclT { - case lportEgressAfterLB: - direction = nbdb.ACLDirectionFromLport - options = map[string]string{ - "apply-after-lb": "true", - } - case lportIngress: - direction = nbdb.ACLDirectionToLport - default: - panic(fmt.Sprintf("Failed to build ACL: unknown acl type %s", aclT)) - } - log, logSeverity := getLogSeverity(action, logLevels) - ACL := libovsdbops.BuildACL( - aclName, - direction, - priority, - match, - action, - types.OvnACLLoggingMeter, - logSeverity, - log, - externalIDs, - options, - ) - return ACL -} - -// BuildACLFromDbIDs should be used to build ACL instead of directly calling libovsdbops.BuildACL. // It can properly set and reset log settings for ACL based on ACLLoggingLevels, and // set acl.Name and acl.ExternalIDs based on given DbIDs -func BuildACLFromDbIDs(dbIDs *libovsdbops.DbObjectIDs, priority int, match, action string, logLevels *ACLLoggingLevels, +func BuildACL(dbIDs *libovsdbops.DbObjectIDs, priority int, match, action string, logLevels *ACLLoggingLevels, aclT aclPipelineType) *nbdb.ACL { var options map[string]string var direction string @@ -163,23 +119,7 @@ func BuildACLFromDbIDs(dbIDs *libovsdbops.DbObjectIDs, priority int, match, acti return ACL } -func getACLMatch(portGroupName, match string, aclT aclPipelineType) string { - var aclMatch string - switch aclT { - case lportIngress: - aclMatch = "outport == @" + portGroupName - case lportEgressAfterLB: - aclMatch = "inport == @" + portGroupName - default: - panic(fmt.Sprintf("Unknown acl type %s", aclT)) - } - if match != "" { - aclMatch += " && " + match - } - return aclMatch -} - -func getACLMatchFromACLDir(portGroupName, match string, aclDir aclDirection) string { +func getACLMatch(portGroupName, match string, aclDir aclDirection) string { var aclMatch string switch aclDir { case aclIngress: diff --git a/go-controller/pkg/ovn/egressfirewall.go b/go-controller/pkg/ovn/egressfirewall.go index a34e562365..26c7c3fc4d 100644 --- a/go-controller/pkg/ovn/egressfirewall.go +++ b/go-controller/pkg/ovn/egressfirewall.go @@ -368,7 +368,7 @@ func (oc *DefaultNetworkController) addEgressFirewallRules(ef *egressFirewall, h func (oc *DefaultNetworkController) createEgressFirewallRules(ruleIdx int, match, action, namespace string, aclLogging *ACLLoggingLevels) error { aclIDs := oc.getEgressFirewallACLDbIDs(namespace, ruleIdx) priority := types.EgressFirewallStartPriority - ruleIdx - egressFirewallACL := BuildACLFromDbIDs( + egressFirewallACL := BuildACL( aclIDs, priority, match, diff --git a/go-controller/pkg/ovn/gress_policy.go b/go-controller/pkg/ovn/gress_policy.go index 55c61b415a..3d9369a96b 100644 --- a/go-controller/pkg/ovn/gress_policy.go +++ b/go-controller/pkg/ovn/gress_policy.go @@ -319,7 +319,7 @@ func (gp *gressPolicy) buildLocalPodACLs(portGroupName string, aclLogging *ACLLo ipBlockMatches := gp.getMatchFromIPBlock(lportMatch, l4Match) for ipBlockIdx, ipBlockMatch := range ipBlockMatches { aclIDs := gp.getNetpolACLDbIDs(portPolIdx, ipBlockIdx) - acl := BuildACLFromDbIDs(aclIDs, types.DefaultAllowPriority, ipBlockMatch, action, + acl := BuildACL(aclIDs, types.DefaultAllowPriority, ipBlockMatch, action, aclLogging, gp.aclPipeline) createdACLs = append(createdACLs, acl) } @@ -340,7 +340,7 @@ func (gp *gressPolicy) buildLocalPodACLs(portGroupName string, aclLogging *ACLLo addrSetMatch = fmt.Sprintf("%s && %s && %s", l3Match, l4Match, lportMatch) } aclIDs := gp.getNetpolACLDbIDs(portPolIdx, emptyIdx) - acl := BuildACLFromDbIDs(aclIDs, types.DefaultAllowPriority, addrSetMatch, action, + acl := BuildACL(aclIDs, types.DefaultAllowPriority, addrSetMatch, action, aclLogging, gp.aclPipeline) if l3Match == "" { // if l3Match is empty, then no address sets are selected for a given gressPolicy. diff --git a/go-controller/pkg/ovn/multicast.go b/go-controller/pkg/ovn/multicast.go index 63f8545eca..f7d37f9609 100644 --- a/go-controller/pkg/ovn/multicast.go +++ b/go-controller/pkg/ovn/multicast.go @@ -120,16 +120,16 @@ func (oc *DefaultNetworkController) createMulticastAllowPolicy(ns string, nsInfo portGroupName := getMulticastPortGroupName(ns) aclDir := aclEgress - egressMatch := getACLMatchFromACLDir(portGroupName, getMulticastACLEgrMatch(), aclDir) + egressMatch := getACLMatch(portGroupName, getMulticastACLEgrMatch(), aclDir) dbIDs := getNamespaceMcastACLDbIDs(ns, aclDir, oc.controllerName) aclPipeline := aclDirectionToACLPipeline(aclDir) - egressACL := BuildACLFromDbIDs(dbIDs, types.DefaultMcastAllowPriority, egressMatch, nbdb.ACLActionAllow, nil, aclPipeline) + egressACL := BuildACL(dbIDs, types.DefaultMcastAllowPriority, egressMatch, nbdb.ACLActionAllow, nil, aclPipeline) aclDir = aclIngress - ingressMatch := getACLMatchFromACLDir(portGroupName, getMulticastACLIgrMatch(nsInfo), aclDir) + ingressMatch := getACLMatch(portGroupName, getMulticastACLIgrMatch(nsInfo), aclDir) dbIDs = getNamespaceMcastACLDbIDs(ns, aclDir, oc.controllerName) aclPipeline = aclDirectionToACLPipeline(aclDir) - ingressACL := BuildACLFromDbIDs(dbIDs, types.DefaultMcastAllowPriority, ingressMatch, nbdb.ACLActionAllow, nil, aclPipeline) + ingressACL := BuildACL(dbIDs, types.DefaultMcastAllowPriority, ingressMatch, nbdb.ACLActionAllow, nil, aclPipeline) acls := []*nbdb.ACL{egressACL, ingressACL} ops, err := libovsdbops.CreateOrUpdateACLsOps(oc.nbClient, nil, acls...) @@ -196,7 +196,7 @@ func (oc *DefaultNetworkController) createDefaultDenyMulticastPolicy() error { for _, aclDir := range []aclDirection{aclEgress, aclIngress} { dbIDs := getDefaultMcastACLDbIDs(mcastDefaultDenyID, aclDir, oc.controllerName) aclPipeline := aclDirectionToACLPipeline(aclDir) - acl := BuildACLFromDbIDs(dbIDs, types.DefaultMcastDenyPriority, match, nbdb.ACLActionDrop, nil, aclPipeline) + acl := BuildACL(dbIDs, types.DefaultMcastDenyPriority, match, nbdb.ACLActionDrop, nil, aclPipeline) acls = append(acls, acl) } ops, err := libovsdbops.CreateOrUpdateACLsOps(oc.nbClient, nil, acls...) @@ -232,10 +232,10 @@ func (oc *DefaultNetworkController) createDefaultAllowMulticastPolicy() error { mcastMatch := getMulticastACLMatch() acls := make([]*nbdb.ACL, 0, 2) for _, aclDir := range []aclDirection{aclEgress, aclIngress} { - match := getACLMatchFromACLDir(types.ClusterRtrPortGroupName, mcastMatch, aclDir) + match := getACLMatch(types.ClusterRtrPortGroupName, mcastMatch, aclDir) dbIDs := getDefaultMcastACLDbIDs(mcastAllowInterNodeID, aclDir, oc.controllerName) aclPipeline := aclDirectionToACLPipeline(aclDir) - acl := BuildACLFromDbIDs(dbIDs, types.DefaultMcastAllowPriority, match, nbdb.ACLActionAllow, nil, aclPipeline) + acl := BuildACL(dbIDs, types.DefaultMcastAllowPriority, match, nbdb.ACLActionAllow, nil, aclPipeline) acls = append(acls, acl) } diff --git a/go-controller/pkg/ovn/multicast_test.go b/go-controller/pkg/ovn/multicast_test.go index c13dbba35c..d49f1b63b3 100644 --- a/go-controller/pkg/ovn/multicast_test.go +++ b/go-controller/pkg/ovn/multicast_test.go @@ -91,7 +91,7 @@ func getMulticastDefaultExpectedData(clusterPortGroup, clusterRtrPortGroup *nbdb aclIDs = getDefaultMcastACLDbIDs(mcastAllowInterNodeID, aclEgress, DefaultNetworkControllerName) aclName = getACLName(aclIDs) - egressMatch := getACLMatchFromACLDir(types.ClusterRtrPortGroupName, match, aclEgress) + egressMatch := getACLMatch(types.ClusterRtrPortGroupName, match, aclEgress) defaultAllowEgressACL := libovsdbops.BuildACL( aclName, nbdb.ACLDirectionFromLport, @@ -110,7 +110,7 @@ func getMulticastDefaultExpectedData(clusterPortGroup, clusterRtrPortGroup *nbdb aclIDs = getDefaultMcastACLDbIDs(mcastAllowInterNodeID, aclIngress, DefaultNetworkControllerName) aclName = getACLName(aclIDs) - ingressMatch := getACLMatchFromACLDir(types.ClusterRtrPortGroupName, match, aclIngress) + ingressMatch := getACLMatch(types.ClusterRtrPortGroupName, match, aclIngress) defaultAllowIngressACL := libovsdbops.BuildACL( aclName, nbdb.ACLDirectionToLport, @@ -187,11 +187,11 @@ func getDefaultPortGroups() (clusterPortGroup, clusterRtrPortGroup *nbdb.PortGro func getMulticastPolicyExpectedData(ns string, ports []string) []libovsdb.TestData { pg_hash := getMulticastPortGroupName(ns) - egressMatch := getACLMatchFromACLDir(pg_hash, getMulticastACLEgrMatch(), aclEgress) + egressMatch := getACLMatch(pg_hash, getMulticastACLEgrMatch(), aclEgress) ip4AddressSet, ip6AddressSet := getNsAddrSetHashNames(ns) mcastMatch := getACLMatchAF(getMulticastACLIgrMatchV4(ip4AddressSet), getMulticastACLIgrMatchV6(ip6AddressSet)) - ingressMatch := getACLMatchFromACLDir(pg_hash, mcastMatch, aclIngress) + ingressMatch := getACLMatch(pg_hash, mcastMatch, aclIngress) aclIDs := getNamespaceMcastACLDbIDs(ns, aclEgress, DefaultNetworkControllerName) aclName := getACLName(aclIDs) diff --git a/go-controller/pkg/ovn/policy.go b/go-controller/pkg/ovn/policy.go index a79f57cbfe..341aaef3da 100644 --- a/go-controller/pkg/ovn/policy.go +++ b/go-controller/pkg/ovn/policy.go @@ -294,7 +294,7 @@ func (oc *DefaultNetworkController) addAllowACLFromNode(nodeName string, mgmtPor } match := fmt.Sprintf("%s.src==%s", ipFamily, mgmtPortIP.String()) dbIDs := getAllowFromNodeACLDbIDs(nodeName, mgmtPortIP.String(), oc.controllerName) - nodeACL := BuildACLFromDbIDs(dbIDs, types.DefaultAllowPriority, match, + nodeACL := BuildACL(dbIDs, types.DefaultAllowPriority, match, nbdb.ACLActionAllowRelated, nil, lportIngress) ops, err := libovsdbops.CreateOrUpdateACLsOps(oc.nbClient, nil, nodeACL) @@ -333,13 +333,13 @@ func defaultDenyPortGroupName(namespace, gressSuffix string) string { func (oc *DefaultNetworkController) buildDenyACLs(namespace, pg string, aclLogging *ACLLoggingLevels, aclDir aclDirection) (denyACL, allowACL *nbdb.ACL) { - denyMatch := getACLMatchFromACLDir(pg, "", aclDir) - allowMatch := getACLMatchFromACLDir(pg, arpAllowPolicyMatch, aclDir) + denyMatch := getACLMatch(pg, "", aclDir) + allowMatch := getACLMatch(pg, arpAllowPolicyMatch, aclDir) aclPipeline := aclDirectionToACLPipeline(aclDir) - denyACL = BuildACLFromDbIDs(oc.getDefaultDenyPolicyACLIDs(namespace, aclDir, defaultDenyACL), + denyACL = BuildACL(oc.getDefaultDenyPolicyACLIDs(namespace, aclDir, defaultDenyACL), types.DefaultDenyPriority, denyMatch, nbdb.ACLActionDrop, aclLogging, aclPipeline) - allowACL = BuildACLFromDbIDs(oc.getDefaultDenyPolicyACLIDs(namespace, aclDir, arpAllowACL), + allowACL = BuildACL(oc.getDefaultDenyPolicyACLIDs(namespace, aclDir, arpAllowACL), types.DefaultAllowPriority, allowMatch, nbdb.ACLActionAllow, nil, aclPipeline) return } @@ -1535,12 +1535,12 @@ func (oc *DefaultNetworkController) addHairpinAllowACL() error { } ingressACLIDs := oc.getNetpolDefaultACLDbIDs(string(knet.PolicyTypeIngress)) - ingressACL := BuildACL("", types.DefaultAllowPriority, match, - nbdb.ACLActionAllowRelated, nil, lportIngress, ingressACLIDs.GetExternalIDs()) + ingressACL := BuildACL(ingressACLIDs, types.DefaultAllowPriority, match, + nbdb.ACLActionAllowRelated, nil, lportIngress) egressACLIDs := oc.getNetpolDefaultACLDbIDs(string(knet.PolicyTypeEgress)) - egressACL := BuildACL("", types.DefaultAllowPriority, match, - nbdb.ACLActionAllowRelated, nil, lportEgressAfterLB, egressACLIDs.GetExternalIDs()) + egressACL := BuildACL(egressACLIDs, types.DefaultAllowPriority, match, + nbdb.ACLActionAllowRelated, nil, lportEgressAfterLB) ops, err := libovsdbops.CreateOrUpdateACLsOps(oc.nbClient, nil, ingressACL, egressACL) if err != nil { From 448cfc1f1000105cb7bd1c4b14ee68827fb6491e Mon Sep 17 00:00:00 2001 From: Nadia Pinaeva Date: Mon, 27 Mar 2023 16:34:06 +0200 Subject: [PATCH 09/40] Update e2e tests for acl logging with the new acl names. Now Egress firewall ACLs are named "EF::" default deny netpol ACLs "NP::" gress ACLs "NP::::" Signed-off-by: Nadia Pinaeva --- test/e2e/acl_logging.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/e2e/acl_logging.go b/test/e2e/acl_logging.go index 5e14ec0c49..a2d76d2343 100644 --- a/test/e2e/acl_logging.go +++ b/test/e2e/acl_logging.go @@ -34,7 +34,7 @@ var _ = Describe("ACL Logging for NetworkPolicy", func() { namespacePrefix = "acl-logging-netpol" pokerPodIndex = 0 pokedPodIndex = 1 - egressDefaultDenySuffix = "egressDefaultDeny" + egressDefaultDenySuffix = "Egress" ) fr := wrappedTestFramework(namespacePrefix) @@ -83,7 +83,7 @@ var _ = Describe("ACL Logging for NetworkPolicy", func() { It("the logs have the expected log level", func() { clientPodScheduledPodName := pods[pokerPodIndex].Spec.NodeName // Retry here in the case where OVN acls have not been programmed yet - composedPolicyNameRegex := fmt.Sprintf("%s_%s", nsName, egressDefaultDenySuffix) + composedPolicyNameRegex := fmt.Sprintf("NP:%s:%s", nsName, egressDefaultDenySuffix) Eventually(func() (bool, error) { return assertACLLogs( clientPodScheduledPodName, @@ -119,7 +119,7 @@ var _ = Describe("ACL Logging for NetworkPolicy", func() { It("the ACL logs are updated accordingly", func() { clientPodScheduledPodName := pods[pokerPodIndex].Spec.NodeName - composedPolicyNameRegex := fmt.Sprintf("%s_%s", nsName, egressDefaultDenySuffix) + composedPolicyNameRegex := fmt.Sprintf("NP:%s:%s", nsName, egressDefaultDenySuffix) Eventually(func() (bool, error) { return assertACLLogs( clientPodScheduledPodName, @@ -139,7 +139,7 @@ var _ = Describe("ACL Logging for NetworkPolicy", func() { It("ACL logging is disabled", func() { clientPod := pods[pokerPodIndex] pokedPod := pods[pokedPodIndex] - composedPolicyNameRegex := fmt.Sprintf("%s_%s", nsName, egressDefaultDenySuffix) + composedPolicyNameRegex := fmt.Sprintf("NP:%s:%s", nsName, egressDefaultDenySuffix) Consistently(func() (bool, error) { return isCountUpdatedAfterPokePod(fr, &clientPod, &pokedPod, composedPolicyNameRegex, denyACLVerdict, "") }, maxPokeRetries*pokeInterval, pokeInterval).Should(BeFalse()) @@ -155,7 +155,7 @@ var _ = Describe("ACL Logging for NetworkPolicy", func() { It("ACL logging is disabled", func() { clientPod := pods[pokerPodIndex] pokedPod := pods[pokedPodIndex] - composedPolicyNameRegex := fmt.Sprintf("%s_%s", nsName, egressDefaultDenySuffix) + composedPolicyNameRegex := fmt.Sprintf("NP:%s:%s", nsName, egressDefaultDenySuffix) Consistently(func() (bool, error) { return isCountUpdatedAfterPokePod(fr, &clientPod, &pokedPod, composedPolicyNameRegex, denyACLVerdict, "") }, maxPokeRetries*pokeInterval, pokeInterval).Should(BeFalse()) @@ -702,7 +702,7 @@ func isCountUpdatedAfterPokePod(fr *framework.Framework, clientPod, pokedPod *v1 } func generateEgressFwRegex(nsName string) string { - return fmt.Sprintf("egressFirewall_%s_.*", nsName) + return fmt.Sprintf("EF:%s:.*", nsName) } func pokeExternalHost(fr *framework.Framework, pokePod *v1.Pod, dstIP string, dstPort int) { From ab51224f7f8028d275f21acf8f82bf273ed3e5be Mon Sep 17 00:00:00 2001 From: Yun Zhou Date: Wed, 29 Mar 2023 09:41:24 -0700 Subject: [PATCH 10/40] vendor: bump containernetworking/plugins to v1.2.0 The current util.NextIP() function returns the wrong IP for IPv6 address with leading zeros. We can directly use the NextIP() function in vendor containernetworking/plugins instead. Signed-off-by: Yun Zhou --- go-controller/go.mod | 12 +-- go-controller/go.sum | 25 +++--- .../pkg/controller/node_linux_test.go | 3 +- .../pkg/controller/node_windows.go | 5 +- ...ase_secondary_layer2_network_controller.go | 4 +- go-controller/pkg/util/net.go | 48 ++++++----- go-controller/pkg/util/net_unit_test.go | 10 ++- .../alexflint/go-filemutex/filemutex_flock.go | 38 +++------ .../go-filemutex/filemutex_windows.go | 79 ++++--------------- .../containernetworking/cni/libcni/conf.go | 6 +- .../cni/pkg/invoke/exec.go | 51 +++++++++++- .../containernetworking/cni/pkg/skel/skel.go | 6 +- .../cni/pkg/version/plugin.go | 4 +- .../plugins/pkg/ip/cidr.go | 76 ++++++++++++++---- .../plugins/pkg/ip/ipforward_linux.go | 6 +- .../plugins/pkg/ip/route_linux.go | 7 +- .../plugins/pkg/testutils/cmd.go | 4 +- .../plugins/pkg/testutils/dns.go | 3 +- .../plugins/pkg/utils/sysctl/sysctl_linux.go | 6 +- .../github.com/onsi/gomega/CHANGELOG.md | 36 +++++++++ .../github.com/onsi/gomega/RELEASING.md | 2 +- .../github.com/onsi/gomega/gomega_dsl.go | 18 ++--- .../onsi/gomega/internal/async_assertion.go | 14 +++- .../github.com/onsi/gomega/internal/gomega.go | 37 ++++----- .../vendor/github.com/onsi/gomega/matchers.go | 3 +- .../github.com/onsi/gomega/types/types.go | 8 +- .../github.com/safchain/ethtool/ethtool.go | 27 ++++--- go-controller/vendor/modules.txt | 14 ++-- 28 files changed, 320 insertions(+), 232 deletions(-) diff --git a/go-controller/go.mod b/go-controller/go.mod index d5af1b0cd4..2d0eb611f5 100644 --- a/go-controller/go.mod +++ b/go-controller/go.mod @@ -4,13 +4,13 @@ go 1.18 require ( github.com/Mellanox/sriovnet v1.1.0 - github.com/Microsoft/hcsshim v0.8.20 - github.com/alexflint/go-filemutex v1.1.0 + github.com/Microsoft/hcsshim v0.9.6 + github.com/alexflint/go-filemutex v1.2.0 github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d github.com/bhendo/go-powershell v0.0.0-20190719160123-219e7fb4e41e github.com/cenkalti/backoff/v4 v4.1.3 - github.com/containernetworking/cni v1.0.1 - github.com/containernetworking/plugins v1.1.1 + github.com/containernetworking/cni v1.1.2 + github.com/containernetworking/plugins v1.2.0 github.com/coreos/go-iptables v0.6.0 github.com/google/uuid v1.2.0 github.com/gorilla/mux v1.8.0 @@ -20,14 +20,14 @@ require ( github.com/miekg/dns v1.1.31 github.com/mitchellh/copystructure v1.2.0 github.com/onsi/ginkgo v1.16.5 - github.com/onsi/gomega v1.23.0 + github.com/onsi/gomega v1.24.2 github.com/openshift/api v0.0.0-20230213202419-42edf4f1d905 github.com/openshift/client-go v0.0.0-20230120202327-72f107311084 github.com/ovn-org/libovsdb v0.6.1-0.20230203213244-a6a173993830 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.14.0 github.com/prometheus/client_model v0.3.0 - github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1 + github.com/safchain/ethtool v0.2.0 github.com/spf13/afero v1.6.0 github.com/stretchr/testify v1.8.0 github.com/urfave/cli/v2 v2.2.0 diff --git a/go-controller/go.sum b/go-controller/go.sum index 4350a506af..cb1860573e 100644 --- a/go-controller/go.sum +++ b/go-controller/go.sum @@ -78,8 +78,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= -github.com/alexflint/go-filemutex v1.1.0 h1:IAWuUuRYL2hETx5b8vCgwnD+xSdlsTQY6s2JjBsqLdg= -github.com/alexflint/go-filemutex v1.1.0/go.mod h1:7P4iRhttt/nUvUOrYIhcpMzv2G6CY9UnI16Z+UJqRyk= +github.com/alexflint/go-filemutex v1.2.0 h1:1v0TJPDtlhgpW4nJ+GvxCLSlUDC3+gW0CQQvlmfDR/s= +github.com/alexflint/go-filemutex v1.2.0/go.mod h1:mYyQSWvw9Tx2/H2n9qXPb52tTYfE0pZAWcBq5mK025c= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220418222510-f25a4f6275ed/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= @@ -216,12 +216,12 @@ github.com/containerd/zfs v1.0.0/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNR github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= github.com/containernetworking/cni v0.8.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= -github.com/containernetworking/cni v1.0.1 h1:9OIL/sZmMYDBe+G8svzILAlulUpaDTUjeAbtH/JNLBo= -github.com/containernetworking/cni v1.0.1/go.mod h1:AKuhXbN5EzmD4yTNtfSsX3tPcmtrBI6QcRV0NiNt15Y= +github.com/containernetworking/cni v1.1.2 h1:wtRGZVv7olUHMOqouPpn3cXJWpJgM6+EUl31EQbXALQ= +github.com/containernetworking/cni v1.1.2/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw= github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHVlzhJpcY6TQxn/fUyDDM= github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRDjeJr6FLK6vuiUwoH7P8= -github.com/containernetworking/plugins v1.1.1 h1:+AGfFigZ5TiQH00vhR8qPeSatj53eNGz0C1d3wVYlHE= -github.com/containernetworking/plugins v1.1.1/go.mod h1:Sr5TH/eBsGLXK/h71HeLfX19sZPp3ry5uHSkI4LPxV8= +github.com/containernetworking/plugins v1.2.0 h1:SWgg3dQG1yzUo4d9iD8cwSVh1VqI+bP7mkPDoSfP9VU= +github.com/containernetworking/plugins v1.2.0/go.mod h1:/VjX4uHecW5vVimFa1wkG4s+r/s9qIfPdqlLF4TW8c4= github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc= github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgUV4GP9qXPfu4= github.com/containers/ocicrypt v1.1.1/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY= @@ -596,7 +596,6 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= @@ -604,8 +603,8 @@ github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3 github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0= -github.com/onsi/ginkgo/v2 v2.4.0 h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs= github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= +github.com/onsi/ginkgo/v2 v2.6.1 h1:1xQPCjcqYw/J5LchOcp4/2q/jzJFjiAOc25chhnDw+Q= github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= @@ -616,8 +615,9 @@ github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9 github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= github.com/onsi/gomega v1.21.1/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc= github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM= -github.com/onsi/gomega v1.23.0 h1:/oxKu9c2HVap+F3PfKort2Hw5DEU+HGlW8n+tguWsys= github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= +github.com/onsi/gomega v1.24.2 h1:J/tulyYK6JwBldPViHJReihxxZ+22FHs0piGjQAvoUE= +github.com/onsi/gomega v1.24.2/go.mod h1:gs3J10IS7Z7r7eXRoNJIrNqU4ToQukCJhFtKrWgHWnk= github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0-rc1.0.20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= @@ -705,10 +705,9 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= -github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1 h1:ZFfeKAhIQiiOrQaI3/znw0gOmYpO28Tcu1YaqMa/jtQ= -github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= +github.com/safchain/ethtool v0.2.0 h1:dILxMBqDnQfX192cCAPjZr9v2IgVXeElHPy435Z/IdE= +github.com/safchain/ethtool v0.2.0/go.mod h1:WkKB1DnNtvsMlDmQ50sgwowDJV/hGbJSOvJoEXs1AJQ= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= @@ -995,6 +994,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1045,7 +1045,6 @@ golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/go-controller/hybrid-overlay/pkg/controller/node_linux_test.go b/go-controller/hybrid-overlay/pkg/controller/node_linux_test.go index a2c7c1d313..bc005fe523 100644 --- a/go-controller/hybrid-overlay/pkg/controller/node_linux_test.go +++ b/go-controller/hybrid-overlay/pkg/controller/node_linux_test.go @@ -23,6 +23,7 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util/mocks" + iputils "github.com/containernetworking/plugins/pkg/ip" "github.com/vishvananda/netlink" . "github.com/onsi/ginkgo" @@ -152,7 +153,7 @@ func createPod(namespace, name, node, podIP, podMAC string) *v1.Pod { annotations := map[string]string{} if podIP != "" || podMAC != "" { ipn := ovntest.MustParseIPNet(podIP) - gatewayIP := util.NextIP(ipn.IP) + gatewayIP := iputils.NextIP(ipn.IP) annotations[util.OvnPodAnnotationName] = fmt.Sprintf(`{"default": {"ip_address":"` + podIP + `", "mac_address":"` + podMAC + `", "gateway_ip": "` + gatewayIP.String() + `"}}`) } diff --git a/go-controller/hybrid-overlay/pkg/controller/node_windows.go b/go-controller/hybrid-overlay/pkg/controller/node_windows.go index 61d8d23b7c..d2ab58dd21 100644 --- a/go-controller/hybrid-overlay/pkg/controller/node_windows.go +++ b/go-controller/hybrid-overlay/pkg/controller/node_windows.go @@ -9,13 +9,12 @@ import ( houtil "github.com/ovn-org/ovn-kubernetes/go-controller/hybrid-overlay/pkg/util" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" - kapi "k8s.io/api/core/v1" listers "k8s.io/client-go/listers/core/v1" "k8s.io/klog/v2" "github.com/Microsoft/hcsshim/hcn" + iputils "github.com/containernetworking/plugins/pkg/ip" ) const ( @@ -255,7 +254,7 @@ func (n *NodeController) initSelf(node *kapi.Node, nodeSubnet *net.IPNet) error // is hardcoded here to be the first IP on the subnet. // TODO: could be made configurable as Windows doesn't have any restrictions // as to what this gateway address should be. - gatewayAddress := util.NextIP(nodeSubnet.IP) + gatewayAddress := iputils.NextIP(nodeSubnet.IP) if config.HybridOverlay.VXLANPort > 65535 { return fmt.Errorf("the hybrid overlay VXLAN port cannot be greater than 65535. Current value: %v", config.HybridOverlay.VXLANPort) diff --git a/go-controller/pkg/ovn/base_secondary_layer2_network_controller.go b/go-controller/pkg/ovn/base_secondary_layer2_network_controller.go index 002d773aee..f754430453 100644 --- a/go-controller/pkg/ovn/base_secondary_layer2_network_controller.go +++ b/go-controller/pkg/ovn/base_secondary_layer2_network_controller.go @@ -7,12 +7,12 @@ import ( "strconv" "time" + iputils "github.com/containernetworking/plugins/pkg/ip" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdbops" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/retry" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" "k8s.io/klog/v2" utilnet "k8s.io/utils/net" @@ -239,7 +239,7 @@ func (oc *BaseSecondaryLayer2NetworkController) InitializeLogicalSwitch(switchNa // FIXME: allocate IP ranges when https://github.com/ovn-org/ovn-kubernetes/issues/3369 is fixed for _, excludeSubnet := range excludeSubnets { - for excludeIP := excludeSubnet.IP; excludeSubnet.Contains(excludeIP); excludeIP = util.NextIP(excludeIP) { + for excludeIP := excludeSubnet.IP; excludeSubnet.Contains(excludeIP); excludeIP = iputils.NextIP(excludeIP) { var ipMask net.IPMask if excludeIP.To4() != nil { ipMask = net.CIDRMask(32, 32) diff --git a/go-controller/pkg/util/net.go b/go-controller/pkg/util/net.go index 6669de0012..7770a03378 100644 --- a/go-controller/pkg/util/net.go +++ b/go-controller/pkg/util/net.go @@ -4,37 +4,20 @@ import ( "crypto/sha256" "errors" "fmt" - "math/big" "net" "strconv" "strings" - "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdbops" - + iputils "github.com/containernetworking/plugins/pkg/ip" "github.com/ovn-org/libovsdb/client" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdbops" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" + utilnet "k8s.io/utils/net" ) var NoIPError = errors.New("no IP available") -// NextIP returns IP incremented by 1 -func NextIP(ip net.IP) net.IP { - i := ipToInt(ip) - return intToIP(i.Add(i, big.NewInt(1))) -} - -func ipToInt(ip net.IP) *big.Int { - if v := ip.To4(); v != nil { - return big.NewInt(0).SetBytes(v) - } - return big.NewInt(0).SetBytes(ip.To16()) -} - -func intToIP(i *big.Int) net.IP { - return net.IP(i.Bytes()) -} - // ExtractPortAddresses returns the MAC and IPs of the given logical switch port func ExtractPortAddresses(lsp *nbdb.LogicalSwitchPort) (net.HardwareAddr, []net.IP, error) { var addresses []string @@ -104,23 +87,36 @@ func GetOVSPortMACAddress(portName string) (net.HardwareAddr, error) { } // GetNodeGatewayIfAddr returns the node logical switch gateway address -// (the ".1" address) +// (the ".1" address), return nil if the subnet is invalid func GetNodeGatewayIfAddr(subnet *net.IPNet) *net.IPNet { - return &net.IPNet{IP: NextIP(subnet.IP), Mask: subnet.Mask} + if subnet == nil { + return nil + } + ip := iputils.NextIP(subnet.IP) + if ip == nil { + return nil + } + return &net.IPNet{IP: ip, Mask: subnet.Mask} } // GetNodeManagementIfAddr returns the node logical switch management port address -// (the ".2" address) +// (the ".2" address), return nil if the subnet is invalid func GetNodeManagementIfAddr(subnet *net.IPNet) *net.IPNet { gwIfAddr := GetNodeGatewayIfAddr(subnet) - return &net.IPNet{IP: NextIP(gwIfAddr.IP), Mask: subnet.Mask} + if gwIfAddr == nil { + return nil + } + return &net.IPNet{IP: iputils.NextIP(gwIfAddr.IP), Mask: subnet.Mask} } // GetNodeHybridOverlayIfAddr returns the node logical switch hybrid overlay -// port address (the ".3" address) +// port address (the ".3" address), return nil if the subnet is invalid func GetNodeHybridOverlayIfAddr(subnet *net.IPNet) *net.IPNet { mgmtIfAddr := GetNodeManagementIfAddr(subnet) - return &net.IPNet{IP: NextIP(mgmtIfAddr.IP), Mask: subnet.Mask} + if mgmtIfAddr == nil { + return nil + } + return &net.IPNet{IP: iputils.NextIP(mgmtIfAddr.IP), Mask: subnet.Mask} } // JoinHostPortInt32 is like net.JoinHostPort(), but with an int32 for the port diff --git a/go-controller/pkg/util/net_unit_test.go b/go-controller/pkg/util/net_unit_test.go index 2f45454509..a7992a7384 100644 --- a/go-controller/pkg/util/net_unit_test.go +++ b/go-controller/pkg/util/net_unit_test.go @@ -6,6 +6,7 @@ import ( "net" "testing" + iputils "github.com/containernetworking/plugins/pkg/ip" nbdb "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" ovntest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing" mock_k8s_io_utils_exec "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/mocks/k8s.io/utils/exec" @@ -13,7 +14,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestNextIP(t *testing.T) { +func TestNextSloppyIP(t *testing.T) { tests := []struct { desc string input string @@ -40,6 +41,11 @@ func TestNextIP(t *testing.T) { input: "254.255.255.255", expOutput: "255.0.0.0", }, + { + desc: "IPv6: test increment of leading zeros", + input: "10:100:200:2::", + expOutput: "10:100:200:2::1", + }, { desc: "IPv6: test increment of eight hextet", input: "2001:db8::ffff", @@ -53,7 +59,7 @@ func TestNextIP(t *testing.T) { } for i, tc := range tests { t.Run(fmt.Sprintf("%d:%s", i, tc.desc), func(t *testing.T) { - res := NextIP(ovntest.MustParseIP(tc.input)) + res := iputils.NextIP(ovntest.MustParseIP(tc.input)) t.Log(res.String()) assert.Equal(t, tc.expOutput, res.String()) }) diff --git a/go-controller/vendor/github.com/alexflint/go-filemutex/filemutex_flock.go b/go-controller/vendor/github.com/alexflint/go-filemutex/filemutex_flock.go index e5f7742529..a71fe8d430 100644 --- a/go-controller/vendor/github.com/alexflint/go-filemutex/filemutex_flock.go +++ b/go-controller/vendor/github.com/alexflint/go-filemutex/filemutex_flock.go @@ -2,13 +2,11 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin dragonfly freebsd linux netbsd openbsd +// +build darwin dragonfly freebsd linux netbsd openbsd solaris package filemutex -import ( - "syscall" -) +import "golang.org/x/sys/unix" const ( mkdirPerm = 0750 @@ -21,7 +19,7 @@ type FileMutex struct { } func New(filename string) (*FileMutex, error) { - fd, err := syscall.Open(filename, syscall.O_CREAT|syscall.O_RDONLY, mkdirPerm) + fd, err := unix.Open(filename, unix.O_CREAT|unix.O_RDONLY, mkdirPerm) if err != nil { return nil, err } @@ -29,16 +27,13 @@ func New(filename string) (*FileMutex, error) { } func (m *FileMutex) Lock() error { - if err := syscall.Flock(m.fd, syscall.LOCK_EX); err != nil { - return err - } - return nil + return unix.Flock(m.fd, unix.LOCK_EX) } func (m *FileMutex) TryLock() error { - if err := syscall.Flock(m.fd, syscall.LOCK_EX|syscall.LOCK_NB); err != nil { - if errno, ok := err.(syscall.Errno); ok { - if errno == syscall.EWOULDBLOCK { + if err := unix.Flock(m.fd, unix.LOCK_EX|unix.LOCK_NB); err != nil { + if errno, ok := err.(unix.Errno); ok { + if errno == unix.EWOULDBLOCK { return AlreadyLocked } } @@ -48,30 +43,21 @@ func (m *FileMutex) TryLock() error { } func (m *FileMutex) Unlock() error { - if err := syscall.Flock(m.fd, syscall.LOCK_UN); err != nil { - return err - } - return nil + return unix.Flock(m.fd, unix.LOCK_UN) } func (m *FileMutex) RLock() error { - if err := syscall.Flock(m.fd, syscall.LOCK_SH); err != nil { - return err - } - return nil + return unix.Flock(m.fd, unix.LOCK_SH) } func (m *FileMutex) RUnlock() error { - if err := syscall.Flock(m.fd, syscall.LOCK_UN); err != nil { - return err - } - return nil + return unix.Flock(m.fd, unix.LOCK_UN) } // Close unlocks the lock and closes the underlying file descriptor. func (m *FileMutex) Close() error { - if err := syscall.Flock(m.fd, syscall.LOCK_UN); err != nil { + if err := unix.Flock(m.fd, unix.LOCK_UN); err != nil { return err } - return syscall.Close(m.fd) + return unix.Close(m.fd) } diff --git a/go-controller/vendor/github.com/alexflint/go-filemutex/filemutex_windows.go b/go-controller/vendor/github.com/alexflint/go-filemutex/filemutex_windows.go index 4691d514cc..468b9f0b38 100644 --- a/go-controller/vendor/github.com/alexflint/go-filemutex/filemutex_windows.go +++ b/go-controller/vendor/github.com/alexflint/go-filemutex/filemutex_windows.go @@ -6,53 +6,22 @@ package filemutex import ( "syscall" - "unsafe" -) - -var ( - modkernel32 = syscall.NewLazyDLL("kernel32.dll") - procLockFileEx = modkernel32.NewProc("LockFileEx") - procUnlockFileEx = modkernel32.NewProc("UnlockFileEx") -) -const ( - lockfileFailImmediately = 1 - lockfileExclusiveLock = 2 + "golang.org/x/sys/windows" ) -func lockFileEx(h syscall.Handle, flags, reserved, locklow, lockhigh uint32, ol *syscall.Overlapped) (err error) { - r1, _, e1 := syscall.Syscall6(procLockFileEx.Addr(), 6, uintptr(h), uintptr(flags), uintptr(reserved), uintptr(locklow), uintptr(lockhigh), uintptr(unsafe.Pointer(ol))) - if r1 == 0 { - if e1 != 0 { - err = error(e1) - } else { - err = syscall.EINVAL - } - } - return -} - -func unlockFileEx(h syscall.Handle, reserved, locklow, lockhigh uint32, ol *syscall.Overlapped) (err error) { - r1, _, e1 := syscall.Syscall6(procUnlockFileEx.Addr(), 5, uintptr(h), uintptr(reserved), uintptr(locklow), uintptr(lockhigh), uintptr(unsafe.Pointer(ol)), 0) - if r1 == 0 { - if e1 != 0 { - err = error(e1) - } else { - err = syscall.EINVAL - } - } - return -} +// see https://msdn.microsoft.com/en-us/library/windows/desktop/ms681382(v=vs.85).aspx +var errLockUnlocked syscall.Errno = 0x9E // FileMutex is similar to sync.RWMutex, but also synchronizes across processes. // This implementation is based on flock syscall. type FileMutex struct { - fd syscall.Handle + fd windows.Handle } func New(filename string) (*FileMutex, error) { - fd, err := syscall.CreateFile(&(syscall.StringToUTF16(filename)[0]), syscall.GENERIC_READ|syscall.GENERIC_WRITE, - syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE, nil, syscall.OPEN_ALWAYS, syscall.FILE_ATTRIBUTE_NORMAL, 0) + fd, err := windows.CreateFile(&(windows.StringToUTF16(filename)[0]), windows.GENERIC_READ|windows.GENERIC_WRITE, + windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE, nil, windows.OPEN_ALWAYS, windows.FILE_ATTRIBUTE_NORMAL, 0) if err != nil { return nil, err } @@ -60,10 +29,9 @@ func New(filename string) (*FileMutex, error) { } func (m *FileMutex) TryLock() error { - var ol syscall.Overlapped - if err := lockFileEx(m.fd, lockfileFailImmediately|lockfileExclusiveLock, 0, 1, 0, &ol); err != nil { - if errno, ok := err.(syscall.Errno); ok { - if errno == syscall.Errno(0x21) { + if err := windows.LockFileEx(m.fd, windows.LOCKFILE_FAIL_IMMEDIATELY|windows.LOCKFILE_EXCLUSIVE_LOCK, 0, 1, 0, &windows.Overlapped{}); err != nil { + if errno, ok := err.(windows.Errno); ok { + if errno == windows.ERROR_LOCK_VIOLATION { return AlreadyLocked } } @@ -73,42 +41,25 @@ func (m *FileMutex) TryLock() error { } func (m *FileMutex) Lock() error { - var ol syscall.Overlapped - if err := lockFileEx(m.fd, lockfileExclusiveLock, 0, 1, 0, &ol); err != nil { - return err - } - return nil + return windows.LockFileEx(m.fd, windows.LOCKFILE_EXCLUSIVE_LOCK, 0, 1, 0, &windows.Overlapped{}) } func (m *FileMutex) Unlock() error { - var ol syscall.Overlapped - if err := unlockFileEx(m.fd, 0, 1, 0, &ol); err != nil { - return err - } - return nil + return windows.UnlockFileEx(m.fd, 0, 1, 0, &windows.Overlapped{}) } func (m *FileMutex) RLock() error { - var ol syscall.Overlapped - if err := lockFileEx(m.fd, 0, 0, 1, 0, &ol); err != nil { - return err - } - return nil + return windows.LockFileEx(m.fd, 0, 0, 1, 0, &windows.Overlapped{}) } func (m *FileMutex) RUnlock() error { - var ol syscall.Overlapped - if err := unlockFileEx(m.fd, 0, 1, 0, &ol); err != nil { - return err - } - return nil + return windows.UnlockFileEx(m.fd, 0, 1, 0, &windows.Overlapped{}) } // Close unlocks the lock and closes the underlying file descriptor. func (m *FileMutex) Close() error { - var ol syscall.Overlapped - if err := unlockFileEx(m.fd, 0, 1, 0, &ol); err != nil { + if err := windows.UnlockFileEx(m.fd, 0, 1, 0, &windows.Overlapped{}); err != nil && err != errLockUnlocked { return err } - return syscall.Close(m.fd) + return windows.Close(m.fd) } diff --git a/go-controller/vendor/github.com/containernetworking/cni/libcni/conf.go b/go-controller/vendor/github.com/containernetworking/cni/libcni/conf.go index d28135ff3c..3cd6a59d1c 100644 --- a/go-controller/vendor/github.com/containernetworking/cni/libcni/conf.go +++ b/go-controller/vendor/github.com/containernetworking/cni/libcni/conf.go @@ -21,6 +21,8 @@ import ( "os" "path/filepath" "sort" + + "github.com/containernetworking/cni/pkg/types" ) type NotFoundError struct { @@ -41,8 +43,8 @@ func (e NoConfigsFoundError) Error() string { } func ConfFromBytes(bytes []byte) (*NetworkConfig, error) { - conf := &NetworkConfig{Bytes: bytes} - if err := json.Unmarshal(bytes, &conf.Network); err != nil { + conf := &NetworkConfig{Bytes: bytes, Network: &types.NetConf{}} + if err := json.Unmarshal(bytes, conf.Network); err != nil { return nil, fmt.Errorf("error parsing configuration: %w", err) } if conf.Network.Type == "" { diff --git a/go-controller/vendor/github.com/containernetworking/cni/pkg/invoke/exec.go b/go-controller/vendor/github.com/containernetworking/cni/pkg/invoke/exec.go index e79bffe63e..3ad07aa8f2 100644 --- a/go-controller/vendor/github.com/containernetworking/cni/pkg/invoke/exec.go +++ b/go-controller/vendor/github.com/containernetworking/cni/pkg/invoke/exec.go @@ -16,6 +16,7 @@ package invoke import ( "context" + "encoding/json" "fmt" "os" @@ -33,6 +34,49 @@ type Exec interface { Decode(jsonBytes []byte) (version.PluginInfo, error) } +// Plugin must return result in same version as specified in netconf; but +// for backwards compatibility reasons if the result version is empty use +// config version (rather than technically correct 0.1.0). +// https://github.com/containernetworking/cni/issues/895 +func fixupResultVersion(netconf, result []byte) (string, []byte, error) { + versionDecoder := &version.ConfigDecoder{} + confVersion, err := versionDecoder.Decode(netconf) + if err != nil { + return "", nil, err + } + + var rawResult map[string]interface{} + if err := json.Unmarshal(result, &rawResult); err != nil { + return "", nil, fmt.Errorf("failed to unmarshal raw result: %w", err) + } + + // plugin output of "null" is successfully unmarshalled, but results in a nil + // map which causes a panic when the confVersion is assigned below. + if rawResult == nil { + rawResult = make(map[string]interface{}) + } + + // Manually decode Result version; we need to know whether its cniVersion + // is empty, while built-in decoders (correctly) substitute 0.1.0 for an + // empty version per the CNI spec. + if resultVerRaw, ok := rawResult["cniVersion"]; ok { + resultVer, ok := resultVerRaw.(string) + if ok && resultVer != "" { + return resultVer, result, nil + } + } + + // If the cniVersion is not present or empty, assume the result is + // the same CNI spec version as the config + rawResult["cniVersion"] = confVersion + newBytes, err := json.Marshal(rawResult) + if err != nil { + return "", nil, fmt.Errorf("failed to remarshal fixed result: %w", err) + } + + return confVersion, newBytes, nil +} + // For example, a testcase could pass an instance of the following fakeExec // object to ExecPluginWithResult() to verify the incoming stdin and environment // and provide a tailored response: @@ -84,7 +128,12 @@ func ExecPluginWithResult(ctx context.Context, pluginPath string, netconf []byte return nil, err } - return create.CreateFromBytes(stdoutBytes) + resultVersion, fixedBytes, err := fixupResultVersion(netconf, stdoutBytes) + if err != nil { + return nil, err + } + + return create.Create(resultVersion, fixedBytes) } func ExecPluginWithoutResult(ctx context.Context, pluginPath string, netconf []byte, args CNIArgs, exec Exec) error { diff --git a/go-controller/vendor/github.com/containernetworking/cni/pkg/skel/skel.go b/go-controller/vendor/github.com/containernetworking/cni/pkg/skel/skel.go index da42db5594..cb8781972d 100644 --- a/go-controller/vendor/github.com/containernetworking/cni/pkg/skel/skel.go +++ b/go-controller/vendor/github.com/containernetworking/cni/pkg/skel/skel.go @@ -196,6 +196,7 @@ func (t *dispatcher) pluginMain(cmdAdd, cmdCheck, cmdDel func(_ *CmdArgs) error, // Print the about string to stderr when no command is set if err.Code == types.ErrInvalidEnvironmentVariables && t.Getenv("CNI_COMMAND") == "" && about != "" { _, _ = fmt.Fprintln(t.Stderr, about) + _, _ = fmt.Fprintf(t.Stderr, "CNI protocol versions supported: %s\n", strings.Join(versionInfo.SupportedVersions(), ", ")) return nil } return err @@ -248,10 +249,7 @@ func (t *dispatcher) pluginMain(cmdAdd, cmdCheck, cmdDel func(_ *CmdArgs) error, return types.NewError(types.ErrInvalidEnvironmentVariables, fmt.Sprintf("unknown CNI_COMMAND: %v", cmd), "") } - if err != nil { - return err - } - return nil + return err } // PluginMainWithError is the core "main" for a plugin. It accepts diff --git a/go-controller/vendor/github.com/containernetworking/cni/pkg/version/plugin.go b/go-controller/vendor/github.com/containernetworking/cni/pkg/version/plugin.go index d4bc9d169c..17b22b6b0c 100644 --- a/go-controller/vendor/github.com/containernetworking/cni/pkg/version/plugin.go +++ b/go-controller/vendor/github.com/containernetworking/cni/pkg/version/plugin.go @@ -86,8 +86,8 @@ func (*PluginDecoder) Decode(jsonBytes []byte) (PluginInfo, error) { // minor, and micro numbers or returns an error func ParseVersion(version string) (int, int, int, error) { var major, minor, micro int - if version == "" { - return -1, -1, -1, fmt.Errorf("invalid version %q: the version is empty", version) + if version == "" { // special case: no version declared == v0.1.0 + return 0, 1, 0, nil } parts := strings.Split(version, ".") diff --git a/go-controller/vendor/github.com/containernetworking/plugins/pkg/ip/cidr.go b/go-controller/vendor/github.com/containernetworking/plugins/pkg/ip/cidr.go index 7acc2d47c4..8b380fc74c 100644 --- a/go-controller/vendor/github.com/containernetworking/plugins/pkg/ip/cidr.go +++ b/go-controller/vendor/github.com/containernetworking/plugins/pkg/ip/cidr.go @@ -19,43 +19,87 @@ import ( "net" ) -// NextIP returns IP incremented by 1 +// NextIP returns IP incremented by 1, if IP is invalid, return nil func NextIP(ip net.IP) net.IP { - i := ipToInt(ip) - return intToIP(i.Add(i, big.NewInt(1))) + normalizedIP := normalizeIP(ip) + if normalizedIP == nil { + return nil + } + + i := ipToInt(normalizedIP) + return intToIP(i.Add(i, big.NewInt(1)), len(normalizedIP) == net.IPv6len) } -// PrevIP returns IP decremented by 1 +// PrevIP returns IP decremented by 1, if IP is invalid, return nil func PrevIP(ip net.IP) net.IP { - i := ipToInt(ip) - return intToIP(i.Sub(i, big.NewInt(1))) + normalizedIP := normalizeIP(ip) + if normalizedIP == nil { + return nil + } + + i := ipToInt(normalizedIP) + return intToIP(i.Sub(i, big.NewInt(1)), len(normalizedIP) == net.IPv6len) } // Cmp compares two IPs, returning the usual ordering: // a < b : -1 // a == b : 0 // a > b : 1 +// incomparable : -2 func Cmp(a, b net.IP) int { - aa := ipToInt(a) - bb := ipToInt(b) - return aa.Cmp(bb) + normalizedA := normalizeIP(a) + normalizedB := normalizeIP(b) + + if len(normalizedA) == len(normalizedB) && len(normalizedA) != 0 { + return ipToInt(normalizedA).Cmp(ipToInt(normalizedB)) + } + + return -2 } func ipToInt(ip net.IP) *big.Int { - if v := ip.To4(); v != nil { - return big.NewInt(0).SetBytes(v) + return big.NewInt(0).SetBytes(ip) +} + +func intToIP(i *big.Int, isIPv6 bool) net.IP { + intBytes := i.Bytes() + + if len(intBytes) == net.IPv4len || len(intBytes) == net.IPv6len { + return intBytes + } + + if isIPv6 { + return append(make([]byte, net.IPv6len-len(intBytes)), intBytes...) } - return big.NewInt(0).SetBytes(ip.To16()) + + return append(make([]byte, net.IPv4len-len(intBytes)), intBytes...) } -func intToIP(i *big.Int) net.IP { - return net.IP(i.Bytes()) +// normalizeIP will normalize IP by family, +// IPv4 : 4-byte form +// IPv6 : 16-byte form +// others : nil +func normalizeIP(ip net.IP) net.IP { + if ipTo4 := ip.To4(); ipTo4 != nil { + return ipTo4 + } + return ip.To16() } -// Network masks off the host portion of the IP +// Network masks off the host portion of the IP, if IPNet is invalid, +// return nil func Network(ipn *net.IPNet) *net.IPNet { + if ipn == nil { + return nil + } + + maskedIP := ipn.IP.Mask(ipn.Mask) + if maskedIP == nil { + return nil + } + return &net.IPNet{ - IP: ipn.IP.Mask(ipn.Mask), + IP: maskedIP, Mask: ipn.Mask, } } diff --git a/go-controller/vendor/github.com/containernetworking/plugins/pkg/ip/ipforward_linux.go b/go-controller/vendor/github.com/containernetworking/plugins/pkg/ip/ipforward_linux.go index e52a45ba1a..0e8b6b691b 100644 --- a/go-controller/vendor/github.com/containernetworking/plugins/pkg/ip/ipforward_linux.go +++ b/go-controller/vendor/github.com/containernetworking/plugins/pkg/ip/ipforward_linux.go @@ -16,7 +16,7 @@ package ip import ( "bytes" - "io/ioutil" + "os" current "github.com/containernetworking/cni/pkg/types/100" ) @@ -53,10 +53,10 @@ func EnableForward(ips []*current.IPConfig) error { } func echo1(f string) error { - if content, err := ioutil.ReadFile(f); err == nil { + if content, err := os.ReadFile(f); err == nil { if bytes.Equal(bytes.TrimSpace(content), []byte("1")) { return nil } } - return ioutil.WriteFile(f, []byte("1"), 0644) + return os.WriteFile(f, []byte("1"), 0644) } diff --git a/go-controller/vendor/github.com/containernetworking/plugins/pkg/ip/route_linux.go b/go-controller/vendor/github.com/containernetworking/plugins/pkg/ip/route_linux.go index f5c0d08039..e92b6c53e4 100644 --- a/go-controller/vendor/github.com/containernetworking/plugins/pkg/ip/route_linux.go +++ b/go-controller/vendor/github.com/containernetworking/plugins/pkg/ip/route_linux.go @@ -42,6 +42,11 @@ func AddHostRoute(ipn *net.IPNet, gw net.IP, dev netlink.Link) error { // AddDefaultRoute sets the default route on the given gateway. func AddDefaultRoute(gw net.IP, dev netlink.Link) error { - _, defNet, _ := net.ParseCIDR("0.0.0.0/0") + var defNet *net.IPNet + if gw.To4() != nil { + _, defNet, _ = net.ParseCIDR("0.0.0.0/0") + } else { + _, defNet, _ = net.ParseCIDR("::/0") + } return AddRoute(defNet, gw, dev) } diff --git a/go-controller/vendor/github.com/containernetworking/plugins/pkg/testutils/cmd.go b/go-controller/vendor/github.com/containernetworking/plugins/pkg/testutils/cmd.go index ce600f6838..304a1e9d88 100644 --- a/go-controller/vendor/github.com/containernetworking/plugins/pkg/testutils/cmd.go +++ b/go-controller/vendor/github.com/containernetworking/plugins/pkg/testutils/cmd.go @@ -15,7 +15,7 @@ package testutils import ( - "io/ioutil" + "io" "os" "github.com/containernetworking/cni/pkg/skel" @@ -52,7 +52,7 @@ func CmdAdd(cniNetns, cniContainerID, cniIfname string, conf []byte, f func() er var out []byte if err == nil { - out, err = ioutil.ReadAll(r) + out, err = io.ReadAll(r) } os.Stdout = oldStdout diff --git a/go-controller/vendor/github.com/containernetworking/plugins/pkg/testutils/dns.go b/go-controller/vendor/github.com/containernetworking/plugins/pkg/testutils/dns.go index bd1b69d1db..bd0de0a81d 100644 --- a/go-controller/vendor/github.com/containernetworking/plugins/pkg/testutils/dns.go +++ b/go-controller/vendor/github.com/containernetworking/plugins/pkg/testutils/dns.go @@ -16,7 +16,6 @@ package testutils import ( "fmt" - "io/ioutil" "os" "strings" @@ -28,7 +27,7 @@ import ( // an error if any occurs while creating/writing the file. It is the caller's // responsibility to remove the file. func TmpResolvConf(dnsConf types.DNS) (string, error) { - f, err := ioutil.TempFile("", "cni_test_resolv.conf") + f, err := os.CreateTemp("", "cni_test_resolv.conf") if err != nil { return "", fmt.Errorf("failed to get temp file for CNI test resolv.conf: %v", err) } diff --git a/go-controller/vendor/github.com/containernetworking/plugins/pkg/utils/sysctl/sysctl_linux.go b/go-controller/vendor/github.com/containernetworking/plugins/pkg/utils/sysctl/sysctl_linux.go index bc123406cb..469e9be9ef 100644 --- a/go-controller/vendor/github.com/containernetworking/plugins/pkg/utils/sysctl/sysctl_linux.go +++ b/go-controller/vendor/github.com/containernetworking/plugins/pkg/utils/sysctl/sysctl_linux.go @@ -16,7 +16,7 @@ package sysctl import ( "fmt" - "io/ioutil" + "os" "path/filepath" "strings" ) @@ -36,7 +36,7 @@ func Sysctl(name string, params ...string) (string, error) { func getSysctl(name string) (string, error) { fullName := filepath.Join("/proc/sys", toNormalName(name)) - data, err := ioutil.ReadFile(fullName) + data, err := os.ReadFile(fullName) if err != nil { return "", err } @@ -46,7 +46,7 @@ func getSysctl(name string) (string, error) { func setSysctl(name, value string) (string, error) { fullName := filepath.Join("/proc/sys", toNormalName(name)) - if err := ioutil.WriteFile(fullName, []byte(value), 0644); err != nil { + if err := os.WriteFile(fullName, []byte(value), 0644); err != nil { return "", err } diff --git a/go-controller/vendor/github.com/onsi/gomega/CHANGELOG.md b/go-controller/vendor/github.com/onsi/gomega/CHANGELOG.md index e088dc0789..35dfec0677 100644 --- a/go-controller/vendor/github.com/onsi/gomega/CHANGELOG.md +++ b/go-controller/vendor/github.com/onsi/gomega/CHANGELOG.md @@ -1,3 +1,39 @@ +## 1.24.2 + +### Fixes +- Correctly handle assertion failure panics for eventually/consistnetly "g Gomega"s in a goroutine [78f1660] +- docs:Fix typo "you an" -> "you can" (#607) [3187c1f] +- fixes issue #600 (#606) [808d192] + +### Maintenance +- Bump golang.org/x/net from 0.2.0 to 0.4.0 (#611) [6ebc0bf] +- Bump nokogiri from 1.13.9 to 1.13.10 in /docs (#612) [258cfc8] +- Bump github.com/onsi/ginkgo/v2 from 2.5.0 to 2.5.1 (#609) [e6c3eb9] + +## 1.24.1 + +### Fixes +- maintain backward compatibility for Eventually and Consisntetly's signatures [4c7df5e] +- fix small typo (#601) [ea0ebe6] + +### Maintenance +- Bump golang.org/x/net from 0.1.0 to 0.2.0 (#603) [1ba8372] +- Bump github.com/onsi/ginkgo/v2 from 2.4.0 to 2.5.0 (#602) [f9426cb] +- fix label-filter in test.yml [d795db6] +- stop running flakey tests and rely on external network dependencies in CI [7133290] + +## 1.24.0 + +### Features + +Introducting [gcustom](https://onsi.github.io/gomega/#gcustom-a-convenient-mechanism-for-buildling-custom-matchers) - a convenient mechanism for building custom matchers. + +This is an RC release for `gcustom`. The external API may be tweaked in response to feedback however it is expected to remain mostly stable. + +### Maintenance + +- Update BeComparableTo documentation [756eaa0] + ## 1.23.0 ### Features diff --git a/go-controller/vendor/github.com/onsi/gomega/RELEASING.md b/go-controller/vendor/github.com/onsi/gomega/RELEASING.md index 7153b9b94d..9973fff49e 100644 --- a/go-controller/vendor/github.com/onsi/gomega/RELEASING.md +++ b/go-controller/vendor/github.com/onsi/gomega/RELEASING.md @@ -5,7 +5,7 @@ A Gomega release is a tagged sha and a GitHub release. To cut a release: ```bash LAST_VERSION=$(git tag --sort=version:refname | tail -n1) CHANGES=$(git log --pretty=format:'- %s [%h]' HEAD...$LAST_VERSION) - echo -e "## NEXT\n\n$CHANGES\n\n### Features\n\n## Fixes\n\n## Maintenance\n\n$(cat CHANGELOG.md)" > CHANGELOG.md + echo -e "## NEXT\n\n$CHANGES\n\n### Features\n\n### Fixes\n\n### Maintenance\n\n$(cat CHANGELOG.md)" > CHANGELOG.md ``` to update the changelog - Categorize the changes into diff --git a/go-controller/vendor/github.com/onsi/gomega/gomega_dsl.go b/go-controller/vendor/github.com/onsi/gomega/gomega_dsl.go index e236a40f4d..b65c8be9b0 100644 --- a/go-controller/vendor/github.com/onsi/gomega/gomega_dsl.go +++ b/go-controller/vendor/github.com/onsi/gomega/gomega_dsl.go @@ -22,7 +22,7 @@ import ( "github.com/onsi/gomega/types" ) -const GOMEGA_VERSION = "1.23.0" +const GOMEGA_VERSION = "1.24.2" const nilGomegaPanic = `You are trying to make an assertion, but haven't registered Gomega's fail handler. If you're using Ginkgo then you probably forgot to put your assertion in an It(). @@ -368,9 +368,9 @@ is equivalent to Eventually(...).WithTimeout(time.Second).WithPolling(2*time.Second).WithContext(ctx).Should(...) */ -func Eventually(args ...interface{}) AsyncAssertion { +func Eventually(actualOrCtx interface{}, args ...interface{}) AsyncAssertion { ensureDefaultGomegaIsConfigured() - return Default.Eventually(args...) + return Default.Eventually(actualOrCtx, args...) } // EventuallyWithOffset operates like Eventually but takes an additional @@ -382,9 +382,9 @@ func Eventually(args ...interface{}) AsyncAssertion { // `EventuallyWithOffset` specifying a timeout interval (and an optional polling interval) are // the same as `Eventually(...).WithOffset(...).WithTimeout` or // `Eventually(...).WithOffset(...).WithTimeout(...).WithPolling`. -func EventuallyWithOffset(offset int, args ...interface{}) AsyncAssertion { +func EventuallyWithOffset(offset int, actualOrCtx interface{}, args ...interface{}) AsyncAssertion { ensureDefaultGomegaIsConfigured() - return Default.EventuallyWithOffset(offset, args...) + return Default.EventuallyWithOffset(offset, actualOrCtx, args...) } /* @@ -402,9 +402,9 @@ Consistently is useful in cases where you want to assert that something *does no This will block for 200 milliseconds and repeatedly check the channel and ensure nothing has been received. */ -func Consistently(args ...interface{}) AsyncAssertion { +func Consistently(actualOrCtx interface{}, args ...interface{}) AsyncAssertion { ensureDefaultGomegaIsConfigured() - return Default.Consistently(args...) + return Default.Consistently(actualOrCtx, args...) } // ConsistentlyWithOffset operates like Consistently but takes an additional @@ -413,9 +413,9 @@ func Consistently(args ...interface{}) AsyncAssertion { // // `ConsistentlyWithOffset` is the same as `Consistently(...).WithOffset` and // optional `WithTimeout` and `WithPolling`. -func ConsistentlyWithOffset(offset int, args ...interface{}) AsyncAssertion { +func ConsistentlyWithOffset(offset int, actualOrCtx interface{}, args ...interface{}) AsyncAssertion { ensureDefaultGomegaIsConfigured() - return Default.ConsistentlyWithOffset(offset, args...) + return Default.ConsistentlyWithOffset(offset, actualOrCtx, args...) } /* diff --git a/go-controller/vendor/github.com/onsi/gomega/internal/async_assertion.go b/go-controller/vendor/github.com/onsi/gomega/internal/async_assertion.go index c1e4a99959..cc8615a117 100644 --- a/go-controller/vendor/github.com/onsi/gomega/internal/async_assertion.go +++ b/go-controller/vendor/github.com/onsi/gomega/internal/async_assertion.go @@ -20,6 +20,17 @@ type contextWithAttachProgressReporter interface { AttachProgressReporter(func() string) func() } +type asyncGomegaHaltExecutionError struct{} + +func (a asyncGomegaHaltExecutionError) GinkgoRecoverShouldIgnoreThisPanic() {} +func (a asyncGomegaHaltExecutionError) Error() string { + return `An assertion has failed in a goroutine. You should call + + defer GinkgoRecover() + +at the top of the goroutine that caused this panic. This will allow Ginkgo and Gomega to correctly capture and manage this panic.` +} + type AsyncAssertionType uint const ( @@ -229,7 +240,8 @@ func (assertion *AsyncAssertion) buildActualPoller() (func() (interface{}, error } _, file, line, _ := runtime.Caller(skip + 1) assertionFailure = fmt.Errorf("Assertion in callback at %s:%d failed:\n%s", file, line, message) - panic("stop execution") + // we throw an asyncGomegaHaltExecutionError so that defer GinkgoRecover() can catch this error if the user makes an assertion in a goroutine + panic(asyncGomegaHaltExecutionError{}) }))) } if takesContext { diff --git a/go-controller/vendor/github.com/onsi/gomega/internal/gomega.go b/go-controller/vendor/github.com/onsi/gomega/internal/gomega.go index e75d2626ac..2d92877f3d 100644 --- a/go-controller/vendor/github.com/onsi/gomega/internal/gomega.go +++ b/go-controller/vendor/github.com/onsi/gomega/internal/gomega.go @@ -2,7 +2,6 @@ package internal import ( "context" - "fmt" "time" "github.com/onsi/gomega/types" @@ -53,42 +52,38 @@ func (g *Gomega) ExpectWithOffset(offset int, actual interface{}, extra ...inter return NewAssertion(actual, g, offset, extra...) } -func (g *Gomega) Eventually(args ...interface{}) types.AsyncAssertion { - return g.makeAsyncAssertion(AsyncAssertionTypeEventually, 0, args...) +func (g *Gomega) Eventually(actualOrCtx interface{}, args ...interface{}) types.AsyncAssertion { + return g.makeAsyncAssertion(AsyncAssertionTypeEventually, 0, actualOrCtx, args...) } -func (g *Gomega) EventuallyWithOffset(offset int, args ...interface{}) types.AsyncAssertion { - return g.makeAsyncAssertion(AsyncAssertionTypeEventually, offset, args...) +func (g *Gomega) EventuallyWithOffset(offset int, actualOrCtx interface{}, args ...interface{}) types.AsyncAssertion { + return g.makeAsyncAssertion(AsyncAssertionTypeEventually, offset, actualOrCtx, args...) } -func (g *Gomega) Consistently(args ...interface{}) types.AsyncAssertion { - return g.makeAsyncAssertion(AsyncAssertionTypeConsistently, 0, args...) +func (g *Gomega) Consistently(actualOrCtx interface{}, args ...interface{}) types.AsyncAssertion { + return g.makeAsyncAssertion(AsyncAssertionTypeConsistently, 0, actualOrCtx, args...) } -func (g *Gomega) ConsistentlyWithOffset(offset int, args ...interface{}) types.AsyncAssertion { - return g.makeAsyncAssertion(AsyncAssertionTypeConsistently, offset, args...) +func (g *Gomega) ConsistentlyWithOffset(offset int, actualOrCtx interface{}, args ...interface{}) types.AsyncAssertion { + return g.makeAsyncAssertion(AsyncAssertionTypeConsistently, offset, actualOrCtx, args...) } -func (g *Gomega) makeAsyncAssertion(asyncAssertionType AsyncAssertionType, offset int, args ...interface{}) types.AsyncAssertion { +func (g *Gomega) makeAsyncAssertion(asyncAssertionType AsyncAssertionType, offset int, actualOrCtx interface{}, args ...interface{}) types.AsyncAssertion { baseOffset := 3 timeoutInterval := -time.Duration(1) pollingInterval := -time.Duration(1) intervals := []interface{}{} var ctx context.Context - if len(args) == 0 { - g.Fail(fmt.Sprintf("Call to %s is missing a value or function to poll", asyncAssertionType), offset+baseOffset) - return nil - } - actual := args[0] - startingIndex := 1 - if _, isCtx := args[0].(context.Context); isCtx && len(args) > 1 { + actual := actualOrCtx + startingIndex := 0 + if _, isCtx := actualOrCtx.(context.Context); isCtx && len(args) > 0 { // the first argument is a context, we should accept it as the context _only if_ it is **not** the only argumnent **and** the second argument is not a parseable duration // this is due to an unfortunate ambiguity in early version of Gomega in which multi-type durations are allowed after the actual - if _, err := toDuration(args[1]); err != nil { - ctx = args[0].(context.Context) - actual = args[1] - startingIndex = 2 + if _, err := toDuration(args[0]); err != nil { + ctx = actualOrCtx.(context.Context) + actual = args[0] + startingIndex = 1 } } diff --git a/go-controller/vendor/github.com/onsi/gomega/matchers.go b/go-controller/vendor/github.com/onsi/gomega/matchers.go index f9d9f2aad2..857586a910 100644 --- a/go-controller/vendor/github.com/onsi/gomega/matchers.go +++ b/go-controller/vendor/github.com/onsi/gomega/matchers.go @@ -27,7 +27,8 @@ func BeEquivalentTo(expected interface{}) types.GomegaMatcher { } } -// BeComparableTo uses gocmp.Equal to compare. You can pass cmp.Option as options. +// BeComparableTo uses gocmp.Equal from github.com/google/go-cmp (instead of reflect.DeepEqual) to perform a deep comparison. +// You can pass cmp.Option as options. // It is an error for actual and expected to be nil. Use BeNil() instead. func BeComparableTo(expected interface{}, opts ...cmp.Option) types.GomegaMatcher { return &matchers.BeComparableToMatcher{ diff --git a/go-controller/vendor/github.com/onsi/gomega/types/types.go b/go-controller/vendor/github.com/onsi/gomega/types/types.go index 089505a4b5..125de64971 100644 --- a/go-controller/vendor/github.com/onsi/gomega/types/types.go +++ b/go-controller/vendor/github.com/onsi/gomega/types/types.go @@ -19,11 +19,11 @@ type Gomega interface { Expect(actual interface{}, extra ...interface{}) Assertion ExpectWithOffset(offset int, actual interface{}, extra ...interface{}) Assertion - Eventually(args ...interface{}) AsyncAssertion - EventuallyWithOffset(offset int, args ...interface{}) AsyncAssertion + Eventually(actualOrCtx interface{}, args ...interface{}) AsyncAssertion + EventuallyWithOffset(offset int, actualOrCtx interface{}, args ...interface{}) AsyncAssertion - Consistently(args ...interface{}) AsyncAssertion - ConsistentlyWithOffset(offset int, args ...interface{}) AsyncAssertion + Consistently(actualOrCtx interface{}, args ...interface{}) AsyncAssertion + ConsistentlyWithOffset(offset int, actualOrCtx interface{}, args ...interface{}) AsyncAssertion SetDefaultEventuallyTimeout(time.Duration) SetDefaultEventuallyPollingInterval(time.Duration) diff --git a/go-controller/vendor/github.com/safchain/ethtool/ethtool.go b/go-controller/vendor/github.com/safchain/ethtool/ethtool.go index d1cd4d31ff..2409116780 100644 --- a/go-controller/vendor/github.com/safchain/ethtool/ethtool.go +++ b/go-controller/vendor/github.com/safchain/ethtool/ethtool.go @@ -234,13 +234,22 @@ type Ethtool struct { fd int } +// Convert zero-terminated array of chars (string in C) to a Go string. +func goString(s []byte) string { + strEnd := bytes.IndexByte(s, 0) + if strEnd == -1 { + return string(s[:]) + } + return string(s[:strEnd]) +} + // DriverName returns the driver name of the given interface name. func (e *Ethtool) DriverName(intf string) (string, error) { info, err := e.getDriverInfo(intf) if err != nil { return "", err } - return string(bytes.Trim(info.driver[:], "\x00")), nil + return goString(info.driver[:]), nil } // BusInfo returns the bus information of the given interface name. @@ -249,7 +258,7 @@ func (e *Ethtool) BusInfo(intf string) (string, error) { if err != nil { return "", err } - return string(bytes.Trim(info.bus_info[:], "\x00")), nil + return goString(info.bus_info[:]), nil } // ModuleEeprom returns Eeprom information of the given interface name. @@ -281,12 +290,12 @@ func (e *Ethtool) DriverInfo(intf string) (DrvInfo, error) { drvInfo := DrvInfo{ Cmd: i.cmd, - Driver: string(bytes.Trim(i.driver[:], "\x00")), - Version: string(bytes.Trim(i.version[:], "\x00")), - FwVersion: string(bytes.Trim(i.fw_version[:], "\x00")), - BusInfo: string(bytes.Trim(i.bus_info[:], "\x00")), - EromVersion: string(bytes.Trim(i.erom_version[:], "\x00")), - Reserved2: string(bytes.Trim(i.reserved2[:], "\x00")), + Driver: goString(i.driver[:]), + Version: goString(i.version[:]), + FwVersion: goString(i.fw_version[:]), + BusInfo: goString(i.bus_info[:]), + EromVersion: goString(i.erom_version[:]), + Reserved2: goString(i.reserved2[:]), NPrivFlags: i.n_priv_flags, NStats: i.n_stats, TestInfoLen: i.testinfo_len, @@ -500,7 +509,7 @@ func (e *Ethtool) FeatureNames(intf string) (map[string]uint, error) { var result = make(map[string]uint) for i := 0; i != int(length); i++ { b := gstrings.data[i*ETH_GSTRING_LEN : i*ETH_GSTRING_LEN+ETH_GSTRING_LEN] - key := string(bytes.Trim(b, "\x00")) + key := goString(b) if key != "" { result[key] = uint(i) } diff --git a/go-controller/vendor/modules.txt b/go-controller/vendor/modules.txt index d22237887d..258af6657e 100644 --- a/go-controller/vendor/modules.txt +++ b/go-controller/vendor/modules.txt @@ -9,7 +9,7 @@ github.com/Microsoft/go-winio github.com/Microsoft/go-winio/pkg/guid github.com/Microsoft/go-winio/pkg/security github.com/Microsoft/go-winio/vhd -# github.com/Microsoft/hcsshim v0.8.20 => github.com/Microsoft/hcsshim v0.8.20 +# github.com/Microsoft/hcsshim v0.9.6 => github.com/Microsoft/hcsshim v0.8.20 ## explicit; go 1.13 github.com/Microsoft/hcsshim/computestorage github.com/Microsoft/hcsshim/hcn @@ -28,7 +28,7 @@ github.com/Microsoft/hcsshim/internal/runhcs github.com/Microsoft/hcsshim/internal/timeout github.com/Microsoft/hcsshim/internal/vmcompute github.com/Microsoft/hcsshim/osversion -# github.com/alexflint/go-filemutex v1.1.0 +# github.com/alexflint/go-filemutex v1.2.0 ## explicit; go 1.13 github.com/alexflint/go-filemutex # github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d @@ -58,7 +58,7 @@ github.com/cespare/xxhash/v2 # github.com/containerd/cgroups v1.0.1 ## explicit; go 1.13 github.com/containerd/cgroups/stats/v1 -# github.com/containernetworking/cni v1.0.1 +# github.com/containernetworking/cni v1.1.2 ## explicit; go 1.14 github.com/containernetworking/cni/libcni github.com/containernetworking/cni/pkg/invoke @@ -71,7 +71,7 @@ github.com/containernetworking/cni/pkg/types/create github.com/containernetworking/cni/pkg/types/internal github.com/containernetworking/cni/pkg/utils github.com/containernetworking/cni/pkg/version -# github.com/containernetworking/plugins v1.1.1 +# github.com/containernetworking/plugins v1.2.0 ## explicit; go 1.17 github.com/containernetworking/plugins/pkg/ip github.com/containernetworking/plugins/pkg/ns @@ -248,7 +248,7 @@ github.com/onsi/ginkgo/reporters/stenographer github.com/onsi/ginkgo/reporters/stenographer/support/go-colorable github.com/onsi/ginkgo/reporters/stenographer/support/go-isatty github.com/onsi/ginkgo/types -# github.com/onsi/gomega v1.23.0 +# github.com/onsi/gomega v1.24.2 ## explicit; go 1.18 github.com/onsi/gomega github.com/onsi/gomega/format @@ -316,8 +316,8 @@ github.com/prometheus/procfs/internal/util # github.com/russross/blackfriday/v2 v2.1.0 ## explicit github.com/russross/blackfriday/v2 -# github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1 -## explicit +# github.com/safchain/ethtool v0.2.0 +## explicit; go 1.16 github.com/safchain/ethtool # github.com/sirupsen/logrus v1.9.0 ## explicit; go 1.13 From fc17f7ececd49a1f29357e3765c2bd44f63c81dd Mon Sep 17 00:00:00 2001 From: Surya Seetharaman Date: Tue, 28 Mar 2023 10:55:18 +0200 Subject: [PATCH 11/40] Separate make modelgen from make codegen Signed-off-by: Surya Seetharaman --- docs/developer.md | 32 +++++++++++++++++++++++++++ go-controller/Makefile | 5 ++++- go-controller/hack/update-codegen.sh | 16 -------------- go-controller/hack/update-modelgen.sh | 20 +++++++++++++++++ 4 files changed, 56 insertions(+), 17 deletions(-) create mode 100644 docs/developer.md create mode 100755 go-controller/hack/update-modelgen.sh diff --git a/docs/developer.md b/docs/developer.md new file mode 100644 index 0000000000..2ec4f8a2bc --- /dev/null +++ b/docs/developer.md @@ -0,0 +1,32 @@ +# Developer Documentation + +This file aims to have information that is useful to the people contributing to this repo. + +## Generating ovsdb bindings using modelgen + +In order to generate the latest NBDB and SBDB bindings, we have a tool called `modelgen` +which lives in the libovsdb repo: https://github.com/ovn-org/libovsdb#modelgen. It is a +[code generator](https://go.dev/blog/generate) that uses `pkg/nbdb/gen.go` and `pkg/sbdb/gen.go` +files to auto-generate the models and additional code like deep-copy methods. + +In order to use this tool do the following: +``` +$ cd go-controller/ +$ make modelgen +curl -sSL https://raw.githubusercontent.com/ovn-org/ovn/${OVN_SCHEMA_VERSION}/ovn-nb.ovsschema -o pkg/nbdb/ovn-nb.ovsschema +curl -sSL https://raw.githubusercontent.com/ovn-org/ovn/${OVN_SCHEMA_VERSION}/ovn-sb.ovsschema -o pkg/sbdb/ovn-sb.ovsschema +hack/update-modelgen.sh +``` + +If there are new bindings then you should see the changes being generated in the `pkg/nbdb` and +`pkg/sbdb` parts of the repo. Include them and push a commit! + +NOTE1: You have to pay attention to the version of the commit hash used to download the modelgen +client. While the client doesn't change too often it can also become outdated causing wrong +generations. So keep in mind to re-install modelgen with latest commits and change the hash +value in the `hack/update-modelgen.sh` file if you find it outdated. + +NOTE2: From time to time we always bump our fedora version of OVN used by KIND. But we oftentimes +forget to update the `OVN_SCHEMA_VERSION` in our `Makefile` which is used to download the ovsdb schema. +If that version seems to be outdated, probably best to update that as well and re-generate the schema +bindings. diff --git a/go-controller/Makefile b/go-controller/Makefile index 3c7234f4a3..d11a9f60ca 100644 --- a/go-controller/Makefile +++ b/go-controller/Makefile @@ -56,7 +56,10 @@ else RACE=1 hack/test-go.sh endif -codegen: pkg/nbdb/ovn-nb.ovsschema pkg/sbdb/ovn-sb.ovsschema +modelgen: pkg/nbdb/ovn-nb.ovsschema pkg/sbdb/ovn-sb.ovsschema + hack/update-modelgen.sh + +codegen: hack/update-codegen.sh install: diff --git a/go-controller/hack/update-codegen.sh b/go-controller/hack/update-codegen.sh index 31aed57fe9..9ee8888b2a 100755 --- a/go-controller/hack/update-codegen.sh +++ b/go-controller/hack/update-codegen.sh @@ -4,22 +4,6 @@ set -o errexit set -o nounset set -o pipefail -# generate ovsdb bindings -if ! ( command -v modelgen > /dev/null ); then - echo "modelgen not found, installing github.com/ovn-org/libovsdb/cmd/modelgen" - olddir="${PWD}" - builddir="$(mktemp -d)" - cd "${builddir}" - GO111MODULE=on go install github.com/ovn-org/libovsdb/cmd/modelgen@2cbe2d093e1247d42050306dd5c9a2d6c11f2460 - cd "${olddir}" - if [[ "${builddir}" == /tmp/* ]]; then #paranoia - rm -rf "${builddir}" - fi -fi - -go generate ./pkg/nbdb -go generate ./pkg/sbdb - crds=$(ls pkg/crd 2> /dev/null) if [ -z "${crds}" ]; then exit diff --git a/go-controller/hack/update-modelgen.sh b/go-controller/hack/update-modelgen.sh new file mode 100755 index 0000000000..0ac2000810 --- /dev/null +++ b/go-controller/hack/update-modelgen.sh @@ -0,0 +1,20 @@ +set -o errexit +set -o nounset +set -o pipefail + +# generate ovsdb bindings +if ! ( command -v modelgen > /dev/null ); then + echo "modelgen not found, installing github.com/ovn-org/libovsdb/cmd/modelgen" + olddir="${PWD}" + builddir="$(mktemp -d)" + cd "${builddir}" + # ensure the hash value is not outdated, if wrong bindings are being generated re-install modelgen + GO111MODULE=on go install github.com/ovn-org/libovsdb/cmd/modelgen@a4f2602f585a3c2995a0abea7010b333631341d9 + cd "${olddir}" + if [[ "${builddir}" == /tmp/* ]]; then #paranoia + rm -rf "${builddir}" + fi +fi + +go generate ./pkg/nbdb +go generate ./pkg/sbdb From 9d216fcbc8879f496cdaa6dbecd123d4d836cc5d Mon Sep 17 00:00:00 2001 From: Nadia Pinaeva Date: Thu, 6 Apr 2023 15:47:02 +0200 Subject: [PATCH 12/40] Use PrimaryID as a client index for ACL. Signed-off-by: Nadia Pinaeva --- go-controller/pkg/libovsdb/libovsdb.go | 5 +++++ go-controller/pkg/libovsdbops/acl.go | 10 ---------- go-controller/pkg/libovsdbops/db_object_ids.go | 2 +- go-controller/pkg/libovsdbops/model.go | 3 +++ go-controller/pkg/types/const.go | 4 ++++ 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/go-controller/pkg/libovsdb/libovsdb.go b/go-controller/pkg/libovsdb/libovsdb.go index 292e7561ce..28a49ec62c 100644 --- a/go-controller/pkg/libovsdb/libovsdb.go +++ b/go-controller/pkg/libovsdb/libovsdb.go @@ -148,6 +148,11 @@ func NewNBClientWithConfig(cfg config.OvnAuthConfig, promRegistry prometheus.Reg enableMetricsOption := client.WithMetricsRegistryNamespaceSubsystem(promRegistry, "ovnkube", "master_libovsdb") + // define client indexes for objects that are using dbIDs + dbModel.SetIndexes(map[string][]model.ClientIndex{ + "ACL": {{Columns: []model.ColumnKey{{Column: "external_ids", Key: types.PrimaryIDKey}}}}, + }) + c, err := newClient(cfg, dbModel, stopCh, enableMetricsOption) if err != nil { return nil, err diff --git a/go-controller/pkg/libovsdbops/acl.go b/go-controller/pkg/libovsdbops/acl.go index de9effec0f..249c1ef398 100644 --- a/go-controller/pkg/libovsdbops/acl.go +++ b/go-controller/pkg/libovsdbops/acl.go @@ -24,12 +24,6 @@ func getACLMutableFields(acl *nbdb.ACL) []interface{} { &acl.Name, &acl.Options, &acl.Priority, &acl.Severity} } -// checkACLPrimaryID is a temporary replacement for client indexes, matches on ExternalIDs[PrimaryIDKey]. -func checkACLPrimaryID(existing *nbdb.ACL, searched *nbdb.ACL) bool { - return searched.ExternalIDs != nil && existing.ExternalIDs != nil && - existing.ExternalIDs[PrimaryIDKey.String()] == searched.ExternalIDs[PrimaryIDKey.String()] -} - type aclPredicate func(*nbdb.ACL) bool // FindACLsWithPredicate looks up ACLs from the cache based on a given predicate @@ -50,7 +44,6 @@ func FindACLs(nbClient libovsdbclient.Client, acls []*nbdb.ACL) ([]*nbdb.ACL, er found := []*nbdb.ACL{} opModel := operationModel{ Model: acl, - ModelPredicate: func(item *nbdb.ACL) bool { return checkACLPrimaryID(item, acl) }, ExistingResult: &found, ErrNotFound: false, BulkOp: false, @@ -124,7 +117,6 @@ func CreateOrUpdateACLsOps(nbClient libovsdbclient.Client, ops []libovsdb.Operat } opModel := operationModel{ Model: acl, - ModelPredicate: func(item *nbdb.ACL) bool { return checkACLPrimaryID(item, acl) }, OnModelUpdates: getACLMutableFields(acl), ErrNotFound: false, BulkOp: false, @@ -143,7 +135,6 @@ func UpdateACLsOps(nbClient libovsdbclient.Client, ops []libovsdb.Operation, acl acl := acls[i] opModel := operationModel{ Model: acl, - ModelPredicate: func(item *nbdb.ACL) bool { return checkACLPrimaryID(item, acl) }, OnModelUpdates: getACLMutableFields(acl), ErrNotFound: true, BulkOp: false, @@ -175,7 +166,6 @@ func UpdateACLsLoggingOps(nbClient libovsdbclient.Client, ops []libovsdb.Operati acl := acls[i] opModel := operationModel{ Model: acl, - ModelPredicate: func(item *nbdb.ACL) bool { return checkACLPrimaryID(item, acl) }, OnModelUpdates: []interface{}{&acl.Severity, &acl.Log}, ErrNotFound: true, BulkOp: false, diff --git a/go-controller/pkg/libovsdbops/db_object_ids.go b/go-controller/pkg/libovsdbops/db_object_ids.go index 2828fb0a08..e4d8edf6af 100644 --- a/go-controller/pkg/libovsdbops/db_object_ids.go +++ b/go-controller/pkg/libovsdbops/db_object_ids.go @@ -57,7 +57,7 @@ const ( ObjectNameKey ExternalIDKey = types.OvnK8sPrefix + "/name" // PrimaryIDKey will be used as a primary index, that is unique for every db object, // and can be built based on the combination of all the other ids. - PrimaryIDKey ExternalIDKey = types.OvnK8sPrefix + "/id" + PrimaryIDKey ExternalIDKey = types.PrimaryIDKey ) // dbIDsMap is used to make sure the same ownerType is not defined twice for the same dbObjType to avoid conflicts. diff --git a/go-controller/pkg/libovsdbops/model.go b/go-controller/pkg/libovsdbops/model.go index fbeb62328b..bda31f3e58 100644 --- a/go-controller/pkg/libovsdbops/model.go +++ b/go-controller/pkg/libovsdbops/model.go @@ -131,6 +131,9 @@ func copyIndexes(model model.Model) model.Model { case *nbdb.ACL: return &nbdb.ACL{ UUID: t.UUID, + ExternalIDs: map[string]string{ + types.PrimaryIDKey: t.ExternalIDs[types.PrimaryIDKey], + }, } case *nbdb.AddressSet: return &nbdb.AddressSet{ diff --git a/go-controller/pkg/types/const.go b/go-controller/pkg/types/const.go index 3ac8fedb66..9905881c4f 100644 --- a/go-controller/pkg/types/const.go +++ b/go-controller/pkg/types/const.go @@ -181,4 +181,8 @@ const ( Layer3Topology = "layer3" Layer2Topology = "layer2" LocalnetTopology = "localnet" + + // db index keys + // PrimaryIDKey is used as a primary client index + PrimaryIDKey = OvnK8sPrefix + "/id" ) From b7afb8abf9c740262fbfc3b4973c9a3d8be8151c Mon Sep 17 00:00:00 2001 From: Nadia Pinaeva Date: Mon, 3 Apr 2023 12:10:25 +0200 Subject: [PATCH 13/40] optimize deleteStaleNetpolPeerAddrSets to iterate over all acls only once Signed-off-by: Nadia Pinaeva --- .../pkg/ovn/pod_selector_address_set.go | 43 ++++++++++++------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/go-controller/pkg/ovn/pod_selector_address_set.go b/go-controller/pkg/ovn/pod_selector_address_set.go index 48df3590ec..0b9c4845fb 100644 --- a/go-controller/pkg/ovn/pod_selector_address_set.go +++ b/go-controller/pkg/ovn/pod_selector_address_set.go @@ -677,35 +677,48 @@ func (oc *DefaultNetworkController) cleanupPodSelectorAddressSets() error { } predicateIDs := libovsdbops.NewDbObjectIDs(libovsdbops.AddressSetPodSelector, oc.controllerName, nil) - asPred := libovsdbops.GetPredicate[*nbdb.AddressSet](predicateIDs, nil) - return deleteAddrSetsWithoutACLRef(asPred, oc.nbClient) + return deleteAddrSetsWithoutACLRef(predicateIDs, oc.nbClient) } // network policies will start using new shared address sets after the initial Add events handling. // On the next restart old address sets will be unreferenced and can be safely deleted. func (oc *DefaultNetworkController) deleteStaleNetpolPeerAddrSets() error { predicateIDs := libovsdbops.NewDbObjectIDs(libovsdbops.AddressSetNetworkPolicy, oc.controllerName, nil) - asPred := libovsdbops.GetPredicate[*nbdb.AddressSet](predicateIDs, nil) - return deleteAddrSetsWithoutACLRef(asPred, oc.nbClient) + return deleteAddrSetsWithoutACLRef(predicateIDs, oc.nbClient) } -func deleteAddrSetsWithoutACLRef(predicate func(set *nbdb.AddressSet) bool, +func deleteAddrSetsWithoutACLRef(predicateIDs *libovsdbops.DbObjectIDs, nbClient libovsdbclient.Client) error { - addrSets, err := libovsdbops.FindAddressSetsWithPredicate(nbClient, predicate) + // fill existing address set names + addrSetReferenced := map[string]bool{} + predicate := libovsdbops.GetPredicate[*nbdb.AddressSet](predicateIDs, func(item *nbdb.AddressSet) bool { + addrSetReferenced[item.Name] = false + return false + }) + + _, err := libovsdbops.FindAddressSetsWithPredicate(nbClient, predicate) if err != nil { return fmt.Errorf("failed to find address sets with predicate: %w", err) } - ops := []ovsdb.Operation{} - for _, addrSet := range addrSets { - acls, err := libovsdbops.FindACLsWithPredicate(nbClient, func(item *nbdb.ACL) bool { - return strings.Contains(item.Match, addrSet.Name) - }) - if err != nil { - return fmt.Errorf("cannot find ACLs referencing address set: %v", err) + // set addrSetReferenced[addrSetName] = true if referencing acl exists + _, err = libovsdbops.FindACLsWithPredicate(nbClient, func(item *nbdb.ACL) bool { + for addrSetName := range addrSetReferenced { + if strings.Contains(item.Match, addrSetName) { + addrSetReferenced[addrSetName] = true + } } - if len(acls) == 0 { + return false + }) + if err != nil { + return fmt.Errorf("cannot find ACLs referencing address set: %v", err) + } + ops := []ovsdb.Operation{} + for addrSetName, isReferenced := range addrSetReferenced { + if !isReferenced { // no references for stale address set, delete - ops, err = libovsdbops.DeleteAddressSetsOps(nbClient, ops, addrSet) + ops, err = libovsdbops.DeleteAddressSetsOps(nbClient, ops, &nbdb.AddressSet{ + Name: addrSetName, + }) if err != nil { return fmt.Errorf("failed to get delete address set ops: %w", err) } From ffdcf1bf7a6ab33cdb6493b6aa12635438c746d8 Mon Sep 17 00:00:00 2001 From: Nadia Pinaeva Date: Mon, 3 Apr 2023 12:11:12 +0200 Subject: [PATCH 14/40] delete getNewLocalPolicyPorts log Signed-off-by: Nadia Pinaeva --- go-controller/pkg/ovn/policy.go | 1 - 1 file changed, 1 deletion(-) diff --git a/go-controller/pkg/ovn/policy.go b/go-controller/pkg/ovn/policy.go index 341aaef3da..fd8ca91099 100644 --- a/go-controller/pkg/ovn/policy.go +++ b/go-controller/pkg/ovn/policy.go @@ -523,7 +523,6 @@ func getPolicyType(policy *knet.NetworkPolicy) (bool, bool) { func (oc *DefaultNetworkController) getNewLocalPolicyPorts(np *networkPolicy, objs ...interface{}) (policyPortsToUUIDs map[string]string, policyPortUUIDs []string, errObjs []interface{}) { - klog.Infof("Processing NetworkPolicy %s/%s to have %d local pods...", np.namespace, np.name, len(objs)) policyPortUUIDs = make([]string, 0, len(objs)) policyPortsToUUIDs = map[string]string{} From 2a89deff5f4deed9145ccf6499194d290177b0e2 Mon Sep 17 00:00:00 2001 From: Dumitru Ceara Date: Thu, 6 Apr 2023 17:29:39 +0200 Subject: [PATCH 15/40] services: Don't try to list/cleanup templates when OVN doesn't support that. Spotted in upstream ovn-org/ovn CI when running against ovn versions <=22.09 which don't support component templates: https://github.com/ovn-org/ovn/actions/runs/4628617882 Reported error: failed to sync chassis: error: failed to get template var list: error: Wrong parameter type (*nbdb.ChassisTemplateVar): Model not found in Database Model Fixes: 4b3475abb4a2 ("services: Use OVN template load balancers for NodePort services.") Signed-off-by: Dumitru Ceara --- .../pkg/ovn/controller/services/repair.go | 14 ++++++++++---- .../ovn/controller/services/services_controller.go | 12 ++++++++---- go-controller/pkg/ovn/master.go | 10 +++++++--- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/go-controller/pkg/ovn/controller/services/repair.go b/go-controller/pkg/ovn/controller/services/repair.go index 9687880144..fc2310affa 100644 --- a/go-controller/pkg/ovn/controller/services/repair.go +++ b/go-controller/pkg/ovn/controller/services/repair.go @@ -50,7 +50,7 @@ func newRepair(serviceLister corelisters.ServiceLister, nbClient libovsdbclient. } // runBeforeSync performs some cleanup of stale LBs and other miscellaneous setup. -func (r *repair) runBeforeSync() { +func (r *repair) runBeforeSync(useTemplates bool) { // no need to lock, single-threaded. startTime := time.Now() @@ -68,9 +68,15 @@ func (r *repair) runBeforeSync() { } // Find all templates. - allTemplates, err := listSvcTemplates(r.nbClient) - if err != nil { - klog.Errorf("Unable to get templates for repair: %v", err) + allTemplates := TemplateMap{} + + if useTemplates { + var err error + + allTemplates, err = listSvcTemplates(r.nbClient) + if err != nil { + klog.Errorf("Unable to get templates for repair: %v", err) + } } // Find all load-balancers associated with Services diff --git a/go-controller/pkg/ovn/controller/services/services_controller.go b/go-controller/pkg/ovn/controller/services/services_controller.go index f6d94dafd0..a0873c333b 100644 --- a/go-controller/pkg/ovn/controller/services/services_controller.go +++ b/go-controller/pkg/ovn/controller/services/services_controller.go @@ -197,7 +197,7 @@ func (c *Controller) Run(workers int, stopCh <-chan struct{}, runRepair, useLBGr // Run the repair controller only once // it keeps in sync Kubernetes and OVN // and handles removal of stale data on upgrades - c.repair.runBeforeSync() + c.repair.runBeforeSync(c.useTemplates) } if err := c.initTopLevelCache(); err != nil { @@ -272,9 +272,13 @@ func (c *Controller) initTopLevelCache() error { defer c.alreadyAppliedRWLock.Unlock() // First list all the templates. - allTemplates, err := listSvcTemplates(c.nbClient) - if err != nil { - return fmt.Errorf("failed to load templates: %w", err) + allTemplates := TemplateMap{} + + if c.useTemplates { + allTemplates, err = listSvcTemplates(c.nbClient) + if err != nil { + return fmt.Errorf("failed to load templates: %w", err) + } } // Then list all load balancers and their respective services. diff --git a/go-controller/pkg/ovn/master.go b/go-controller/pkg/ovn/master.go index 2ae270bf4e..065cfac974 100644 --- a/go-controller/pkg/ovn/master.go +++ b/go-controller/pkg/ovn/master.go @@ -662,9 +662,13 @@ func (oc *DefaultNetworkController) syncChassis(nodes []*kapi.Node) error { return fmt.Errorf("failed to get chassis private list: %v", err) } - templateVarList, err := libovsdbops.ListTemplateVar(oc.nbClient) - if err != nil { - return fmt.Errorf("failed to get template var list: error: %v", err) + templateVarList := []*nbdb.ChassisTemplateVar{} + + if oc.svcTemplateSupport { + templateVarList, err = libovsdbops.ListTemplateVar(oc.nbClient) + if err != nil { + return fmt.Errorf("failed to get template var list: error: %w", err) + } } chassisHostNameMap := map[string]*sbdb.Chassis{} From c1cf2db4d39cae5d9ca5a200b79b310dd4dfbeb7 Mon Sep 17 00:00:00 2001 From: Andrea Panattoni Date: Tue, 20 Dec 2022 15:29:08 +0100 Subject: [PATCH 16/40] Make `pokeEndpoint` not failing in case of error Asserting on `netexec` response to not containing errors does not allow using retries, as the test would fail during the first attempt. Signed-off-by: Andrea Panattoni --- test/e2e/util.go | 1 + 1 file changed, 1 insertion(+) diff --git a/test/e2e/util.go b/test/e2e/util.go index a66f549e7d..ccb635aac4 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -308,6 +308,7 @@ func pokeEndpoint(namespace, clientContainer, protocol, targetHost string, targe if err != nil { framework.Logf("FAILED Command was %s", curlCommand) framework.Logf("FAILED Response was %v", res) + return "" } framework.ExpectNoError(err) From d9d618783f373ac98d5203e17fe06915222fb926 Mon Sep 17 00:00:00 2001 From: Surya Seetharaman Date: Tue, 28 Mar 2023 09:05:09 +0200 Subject: [PATCH 17/40] Updating endpoints when using allocateLBNP=0, lgw mode fails For unknown reasons, during the kube rebase, we just decided to flip from sorting the list of endpoints to not sorting it? : 3850450#diff-d3aa58d9b58a0a09264f072df46ab01d0501eb508c4656411ae2dc1ac68fb3c4L579 The allocateLBNodePorts feature depends on the list being sorted, else our logic is skewed and the random probability model we use to generate iptables gets skewed up as well. So during endpoints update and such we will behave badly and have left over DNATs towards pods that don't exist anymore which will lead to black-holing the traffic. Signed-off-by: Surya Seetharaman --- go-controller/pkg/node/gateway_shared_intf.go | 20 +++++++++---------- test/e2e/service.go | 2 ++ 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/go-controller/pkg/node/gateway_shared_intf.go b/go-controller/pkg/node/gateway_shared_intf.go index b3c33dfe3b..5015ce0d3b 100644 --- a/go-controller/pkg/node/gateway_shared_intf.go +++ b/go-controller/pkg/node/gateway_shared_intf.go @@ -576,7 +576,7 @@ func (npw *nodePortWatcher) AddService(service *kapi.Service) error { if exists := npw.addOrSetServiceInfo(name, service, hasLocalHostNetworkEp, localEndpoints); !exists { klog.V(5).Infof("Service Add %s event in namespace %s came before endpoint event setting svcConfig", service.Name, service.Namespace) - if err := addServiceRules(service, localEndpoints.UnsortedList(), hasLocalHostNetworkEp, npw); err != nil { + if err := addServiceRules(service, sets.List(localEndpoints), hasLocalHostNetworkEp, npw); err != nil { return fmt.Errorf("AddService failed for nodePortWatcher: %v", err) } } else { @@ -608,14 +608,14 @@ func (npw *nodePortWatcher) UpdateService(old, new *kapi.Service) error { // Delete old rules if needed, but don't delete svcConfig // so that we don't miss any endpoint update events here klog.V(5).Infof("Deleting old service rules for: %v", old) - if err = delServiceRules(old, svcConfig.localEndpoints.UnsortedList(), npw); err != nil { + if err = delServiceRules(old, sets.List(svcConfig.localEndpoints), npw); err != nil { errors = append(errors, err) } } if util.ServiceTypeHasClusterIP(new) && util.IsClusterIPSet(new) { klog.V(5).Infof("Adding new service rules for: %v", new) - if err = addServiceRules(new, svcConfig.localEndpoints.UnsortedList(), svcConfig.hasLocalHostNetworkEp, npw); err != nil { + if err = addServiceRules(new, sets.List(svcConfig.localEndpoints), svcConfig.hasLocalHostNetworkEp, npw); err != nil { errors = append(errors, err) } } @@ -678,7 +678,7 @@ func (npw *nodePortWatcher) DeleteService(service *kapi.Service) error { klog.V(5).Infof("Deleting service %s in namespace %s", service.Name, service.Namespace) name := ktypes.NamespacedName{Namespace: service.Namespace, Name: service.Name} if svcConfig, exists := npw.getAndDeleteServiceInfo(name); exists { - if err = delServiceRules(svcConfig.service, svcConfig.localEndpoints.UnsortedList(), npw); err != nil { + if err = delServiceRules(svcConfig.service, sets.List(svcConfig.localEndpoints), npw); err != nil { errors = append(errors, err) } } else { @@ -730,7 +730,7 @@ func (npw *nodePortWatcher) SyncServices(services []interface{}) error { } // Add correct iptables rules only for Full mode if !npw.dpuMode { - keepIPTRules = append(keepIPTRules, getGatewayIPTRules(service, localEndPoints.UnsortedList(), hasLocalHostNetworkEp)...) + keepIPTRules = append(keepIPTRules, getGatewayIPTRules(service, sets.List(localEndPoints), hasLocalHostNetworkEp)...) } if !npw.dpuMode && shouldConfigureEgressSVC(service, npw) { @@ -820,16 +820,16 @@ func (npw *nodePortWatcher) AddEndpointSlice(epSlice *discovery.EndpointSlice) e out, exists := npw.getAndSetServiceInfo(namespacedName, svc, hasLocalHostNetworkEp, localEndpoints) if !exists { klog.V(5).Infof("Endpointslice %s ADD event in namespace %s is creating rules", epSlice.Name, epSlice.Namespace) - return addServiceRules(svc, localEndpoints.UnsortedList(), hasLocalHostNetworkEp, npw) + return addServiceRules(svc, sets.List(localEndpoints), hasLocalHostNetworkEp, npw) } if out.hasLocalHostNetworkEp != hasLocalHostNetworkEp || (!util.LoadBalancerServiceHasNodePortAllocation(svc) && !reflect.DeepEqual(out.localEndpoints, localEndpoints)) { klog.V(5).Infof("Endpointslice %s ADD event in namespace %s is updating rules", epSlice.Name, epSlice.Namespace) - if err = delServiceRules(svc, out.localEndpoints.UnsortedList(), npw); err != nil { + if err = delServiceRules(svc, sets.List(out.localEndpoints), npw); err != nil { errors = append(errors, err) } - if err = addServiceRules(svc, localEndpoints.UnsortedList(), hasLocalHostNetworkEp, npw); err != nil { + if err = addServiceRules(svc, sets.List(localEndpoints), hasLocalHostNetworkEp, npw); err != nil { errors = append(errors, err) } return apierrors.NewAggregate(errors) @@ -875,10 +875,10 @@ func (npw *nodePortWatcher) DeleteEndpointSlice(epSlice *discovery.EndpointSlice npw.serviceInfoLock.Lock() defer npw.serviceInfoLock.Unlock() - if err = delServiceRules(svcConfig.service, svcConfig.localEndpoints.UnsortedList(), npw); err != nil { + if err = delServiceRules(svcConfig.service, sets.List(svcConfig.localEndpoints), npw); err != nil { errors = append(errors, err) } - if err = addServiceRules(svcConfig.service, localEndpoints.UnsortedList(), hasLocalHostNetworkEp, npw); err != nil { + if err = addServiceRules(svcConfig.service, sets.List(localEndpoints), hasLocalHostNetworkEp, npw); err != nil { errors = append(errors, err) } return apierrors.NewAggregate(errors) diff --git a/test/e2e/service.go b/test/e2e/service.go index 9a4aa7ee25..8f9e8822a4 100644 --- a/test/e2e/service.go +++ b/test/e2e/service.go @@ -824,6 +824,8 @@ spec: ginkgo.By("Scale down endpoints of service: " + svcName + " to ensure iptable rules are also getting recreated correctly") framework.RunKubectlOrDie("default", "scale", "deployment", backendName, "--replicas=3") + err = framework.WaitForServiceEndpointsNum(f.ClientSet, namespaceName, svcName, 3, time.Second, time.Second*120) + framework.ExpectNoError(err, fmt.Sprintf("service: %s never had an endpoint, err: %v", svcName, err)) time.Sleep(time.Second * 5) // buffer to ensure all rules are created correctly // number of iptable rules should have decreased by 1 From b14eb723dae1823fe1046b2b4b78719d17d01b4b Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Tue, 11 Apr 2023 08:49:21 -0500 Subject: [PATCH 18/40] Small cleanups to log messages and test suite name Fix the formatting of the log messages (%w is only valid when creating new errors, not when logging) and update the master test suite name since it hasn't been only gateway init operations for a long time. Signed-off-by: Dan Williams --- .../pkg/clustermanager/network_cluster_controller.go | 4 ++-- go-controller/pkg/ovn/master_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go-controller/pkg/clustermanager/network_cluster_controller.go b/go-controller/pkg/clustermanager/network_cluster_controller.go index 33411b6e98..19e381df5e 100644 --- a/go-controller/pkg/clustermanager/network_cluster_controller.go +++ b/go-controller/pkg/clustermanager/network_cluster_controller.go @@ -260,7 +260,7 @@ func (ncc *networkClusterController) syncNodes(nodes []interface{}) error { } else if hostSubnet != nil { klog.V(5).Infof("Node %s contains subnets: %v", node.Name, hostSubnet) if err := ncc.hybridOverlaySubnetAllocator.MarkSubnetsAllocated(node.Name, hostSubnet); err != nil { - klog.Errorf("Failed to mark the subnet %v as allocated in the hybrid subnet allocator for node %s: %w", hostSubnet, node.Name, err) + klog.Errorf("Failed to mark the subnet %v as allocated in the hybrid subnet allocator for node %s: %v", hostSubnet, node.Name, err) } } } @@ -269,7 +269,7 @@ func (ncc *networkClusterController) syncNodes(nodes []interface{}) error { if len(hostSubnets) > 0 { klog.V(5).Infof("Node %s contains subnets: %v for network : %s", node.Name, hostSubnets, ncc.networkName) if err := ncc.clusterSubnetAllocator.MarkSubnetsAllocated(node.Name, hostSubnets...); err != nil { - klog.Errorf("Failed to mark the subnet %v as allocated in the cluster subnet allocator for node %s: %w", hostSubnets, node.Name, err) + klog.Errorf("Failed to mark the subnet %v as allocated in the cluster subnet allocator for node %s: %v", hostSubnets, node.Name, err) } } else { klog.V(5).Infof("Node %s contains no subnets for network : %s", node.Name, ncc.networkName) diff --git a/go-controller/pkg/ovn/master_test.go b/go-controller/pkg/ovn/master_test.go index 3e4daaa515..8af83aa61e 100644 --- a/go-controller/pkg/ovn/master_test.go +++ b/go-controller/pkg/ovn/master_test.go @@ -862,7 +862,7 @@ func startFakeController(oc *DefaultNetworkController, wg *sync.WaitGroup) []*ne return clusterSubnets } -var _ = ginkgo.Describe("Gateway Init Operations", func() { +var _ = ginkgo.Describe("Default network controller operations", func() { var ( app *cli.App f *factory.WatchFactory From bff6a065bc17dae5f4b1560229a73099cbfcf65e Mon Sep 17 00:00:00 2001 From: Jacob Tanenbaum Date: Mon, 10 Apr 2023 12:00:48 -0400 Subject: [PATCH 19/40] allow the use of the regular addressset in testing the fakeAddresset code was originally created so that during testing we could directly access and figure out what ip addresses where added to addressSets because we could not access an ovn northbound database. Since the introduction of libovsdb we can use the fake database that is provided for us instread of the the fakeAddressSetFactory and fakeAddressSet Signed-off-by: Jacob Tanenbaum --- go-controller/pkg/ovn/egressfirewall_test.go | 4 ++-- go-controller/pkg/ovn/egressgw_test.go | 2 +- go-controller/pkg/ovn/egressip_test.go | 2 +- go-controller/pkg/ovn/egressqos_test.go | 2 +- go-controller/pkg/ovn/egressservices_test.go | 2 +- go-controller/pkg/ovn/gateway_test.go | 2 +- go-controller/pkg/ovn/multicast_test.go | 2 +- go-controller/pkg/ovn/namespace_test.go | 2 +- go-controller/pkg/ovn/ovn_test.go | 13 ++++++++++--- .../pkg/ovn/pod_selector_address_set_test.go | 2 +- go-controller/pkg/ovn/pods_test.go | 2 +- go-controller/pkg/ovn/policy_test.go | 2 +- 12 files changed, 22 insertions(+), 15 deletions(-) diff --git a/go-controller/pkg/ovn/egressfirewall_test.go b/go-controller/pkg/ovn/egressfirewall_test.go index 263639daf6..4f1d2b3090 100644 --- a/go-controller/pkg/ovn/egressfirewall_test.go +++ b/go-controller/pkg/ovn/egressfirewall_test.go @@ -88,7 +88,7 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { app.Name = "test" app.Flags = config.Flags - fakeOVN = NewFakeOVN() + fakeOVN = NewFakeOVN(true) clusterPortGroup = newClusterPortGroup() nodeSwitch = &nbdb.LogicalSwitch{ UUID: node1Name + "-UUID", @@ -1121,7 +1121,7 @@ var _ = ginkgo.Describe("OVN test basic functions", func() { app.Flags = config.Flags dbSetup := libovsdbtest.TestSetup{} - fakeOVN = NewFakeOVN() + fakeOVN = NewFakeOVN(true) a := newObjectMeta(node1Name, "") a.Labels = nodeLabel node1 := v1.Node{ diff --git a/go-controller/pkg/ovn/egressgw_test.go b/go-controller/pkg/ovn/egressgw_test.go index d55dc4ad29..c843e36ea7 100644 --- a/go-controller/pkg/ovn/egressgw_test.go +++ b/go-controller/pkg/ovn/egressgw_test.go @@ -46,7 +46,7 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { app.Name = "test" app.Flags = config.Flags - fakeOvn = NewFakeOVN() + fakeOvn = NewFakeOVN(true) }) ginkgo.AfterEach(func() { diff --git a/go-controller/pkg/ovn/egressip_test.go b/go-controller/pkg/ovn/egressip_test.go index f078cb3dff..9f52de645a 100644 --- a/go-controller/pkg/ovn/egressip_test.go +++ b/go-controller/pkg/ovn/egressip_test.go @@ -245,7 +245,7 @@ var _ = ginkgo.Describe("OVN master EgressIP Operations", func() { app.Name = "test" app.Flags = config.Flags - fakeOvn = NewFakeOVN() + fakeOvn = NewFakeOVN(true) }) ginkgo.AfterEach(func() { diff --git a/go-controller/pkg/ovn/egressqos_test.go b/go-controller/pkg/ovn/egressqos_test.go index 7aa55f0da6..70be20c364 100644 --- a/go-controller/pkg/ovn/egressqos_test.go +++ b/go-controller/pkg/ovn/egressqos_test.go @@ -63,7 +63,7 @@ var _ = ginkgo.Describe("OVN EgressQoS Operations", func() { app.Name = "test" app.Flags = config.Flags - fakeOVN = NewFakeOVN() + fakeOVN = NewFakeOVN(true) }) ginkgo.AfterEach(func() { diff --git a/go-controller/pkg/ovn/egressservices_test.go b/go-controller/pkg/ovn/egressservices_test.go index b0c383c59f..d3ce7a4532 100644 --- a/go-controller/pkg/ovn/egressservices_test.go +++ b/go-controller/pkg/ovn/egressservices_test.go @@ -59,7 +59,7 @@ var _ = ginkgo.Describe("OVN Egress Service Operations", func() { app.Name = "test" app.Flags = config.Flags - fakeOVN = NewFakeOVN() + fakeOVN = NewFakeOVN(true) }) ginkgo.AfterEach(func() { diff --git a/go-controller/pkg/ovn/gateway_test.go b/go-controller/pkg/ovn/gateway_test.go index ec6d4d9b0b..02f78181b9 100644 --- a/go-controller/pkg/ovn/gateway_test.go +++ b/go-controller/pkg/ovn/gateway_test.go @@ -342,7 +342,7 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { // Restore global default values before each testcase config.PrepareTestConfig() - fakeOvn = NewFakeOVN() + fakeOvn = NewFakeOVN(true) }) ginkgo.AfterEach(func() { diff --git a/go-controller/pkg/ovn/multicast_test.go b/go-controller/pkg/ovn/multicast_test.go index d49f1b63b3..aec9934822 100644 --- a/go-controller/pkg/ovn/multicast_test.go +++ b/go-controller/pkg/ovn/multicast_test.go @@ -342,7 +342,7 @@ var _ = ginkgo.Describe("OVN Multicast with IP Address Family", func() { app.Name = "test" app.Flags = config.Flags - fakeOvn = NewFakeOVN() + fakeOvn = NewFakeOVN(true) gomegaFormatMaxLength = format.MaxLength format.MaxLength = 0 }) diff --git a/go-controller/pkg/ovn/namespace_test.go b/go-controller/pkg/ovn/namespace_test.go index f380611f45..0aeed2940a 100644 --- a/go-controller/pkg/ovn/namespace_test.go +++ b/go-controller/pkg/ovn/namespace_test.go @@ -86,7 +86,7 @@ var _ = ginkgo.Describe("OVN Namespace Operations", func() { err := config.PrepareTestConfig() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - fakeOvn = NewFakeOVN() + fakeOvn = NewFakeOVN(true) wg = &sync.WaitGroup{} }) diff --git a/go-controller/pkg/ovn/ovn_test.go b/go-controller/pkg/ovn/ovn_test.go index 50f99ba3bc..06091ee5d7 100644 --- a/go-controller/pkg/ovn/ovn_test.go +++ b/go-controller/pkg/ovn/ovn_test.go @@ -54,9 +54,14 @@ type FakeOVN struct { egressSVCWg *sync.WaitGroup } -func NewFakeOVN() *FakeOVN { +// NOTE: the FakeAddressSetFactory is no longer needed and should no longer be used. starting to phase out FakeAddressSetFactory +func NewFakeOVN(useFakeAddressSet bool) *FakeOVN { + var asf *addressset.FakeAddressSetFactory + if useFakeAddressSet { + asf = addressset.NewFakeAddressSetFactory(DefaultNetworkControllerName) + } return &FakeOVN{ - asf: addressset.NewFakeAddressSetFactory(DefaultNetworkControllerName), + asf: asf, fakeRecorder: record.NewFakeRecorder(10), egressQoSWg: &sync.WaitGroup{}, egressSVCWg: &sync.WaitGroup{}, @@ -149,7 +154,9 @@ func resetNBClient(ctx context.Context, nbClient libovsdbclient.Client) { func NewOvnController(ovnClient *util.OVNMasterClientset, wf *factory.WatchFactory, stopChan chan struct{}, addressSetFactory addressset.AddressSetFactory, libovsdbOvnNBClient libovsdbclient.Client, libovsdbOvnSBClient libovsdbclient.Client, recorder record.EventRecorder, wg *sync.WaitGroup) (*DefaultNetworkController, error) { - if addressSetFactory == nil { + + fakeAddr, ok := addressSetFactory.(*addressset.FakeAddressSetFactory) + if addressSetFactory == nil || (ok && fakeAddr == nil) { addressSetFactory = addressset.NewOvnAddressSetFactory(libovsdbOvnNBClient) } diff --git a/go-controller/pkg/ovn/pod_selector_address_set_test.go b/go-controller/pkg/ovn/pod_selector_address_set_test.go index c6f2032564..3295a9016c 100644 --- a/go-controller/pkg/ovn/pod_selector_address_set_test.go +++ b/go-controller/pkg/ovn/pod_selector_address_set_test.go @@ -45,7 +45,7 @@ var _ = ginkgo.Describe("OVN PodSelectorAddressSet", func() { ginkgo.BeforeEach(func() { // Restore global default values before each testcase config.PrepareTestConfig() - fakeOvn = NewFakeOVN() + fakeOvn = NewFakeOVN(true) initialData := getHairpinningACLsV4AndPortGroup() initialDB = libovsdbtest.TestSetup{ NBData: initialData, diff --git a/go-controller/pkg/ovn/pods_test.go b/go-controller/pkg/ovn/pods_test.go index d44239a186..f33236ceda 100644 --- a/go-controller/pkg/ovn/pods_test.go +++ b/go-controller/pkg/ovn/pods_test.go @@ -248,7 +248,7 @@ var _ = ginkgo.Describe("OVN Pod Operations", func() { app.Name = "test" app.Flags = config.Flags - fakeOvn = NewFakeOVN() + fakeOvn = NewFakeOVN(true) initialDB = libovsdbtest.TestSetup{ NBData: []libovsdbtest.TestData{ &nbdb.LogicalSwitch{ diff --git a/go-controller/pkg/ovn/policy_test.go b/go-controller/pkg/ovn/policy_test.go index 28d74f5560..a7258e0df9 100644 --- a/go-controller/pkg/ovn/policy_test.go +++ b/go-controller/pkg/ovn/policy_test.go @@ -527,7 +527,7 @@ var _ = ginkgo.Describe("OVN NetworkPolicy Operations", func() { app.Name = "test" app.Flags = config.Flags - fakeOvn = NewFakeOVN() + fakeOvn = NewFakeOVN(true) gomegaFormatMaxLength = format.MaxLength format.MaxLength = 0 From 2d6b87004312c92ce316e5bf68b1649ebb1916f4 Mon Sep 17 00:00:00 2001 From: Peng Liu Date: Wed, 22 Mar 2023 16:08:47 +0800 Subject: [PATCH 20/40] Add compact mode support to kind.sh The compact mode means running ovnkube master and node in the same process. The OVN-K runs in this configuration in Microshift. Signed-off-by: Peng Liu --- contrib/kind.sh | 11 ++++- dist/images/daemonset.sh | 9 ++++ dist/images/ovnkube.sh | 16 ++++++- dist/templates/ovnkube-master.yaml.j2 | 69 +++++++++++++++++++++++++-- dist/templates/ovnkube-node.yaml.j2 | 4 +- 5 files changed, 100 insertions(+), 9 deletions(-) diff --git a/contrib/kind.sh b/contrib/kind.sh index fd54781f4e..f3f13651a8 100755 --- a/contrib/kind.sh +++ b/contrib/kind.sh @@ -110,6 +110,7 @@ usage() { echo " [-cn | --cluster-name |" echo " [-ehp|--egress-ip-healthcheck-port ]" echo " [-is | --ipsec]" + echo " [-cm | --compact-mode]" echo " [--isolated]" echo " [-h]]" echo "" @@ -157,6 +158,7 @@ usage() { echo "-ehp | --egress-ip-healthcheck-port TCP port used for gRPC session by egress IP node check. DEFAULT: 9107 (Use "0" for legacy dial to port 9)." echo "-is | --ipsec Enable IPsec encryption (spawns ovn-ipsec pods)" echo "-sm | --scale-metrics Enable scale metrics" + echo "-cm | --compact-mode Enable compact mode, ovnkube master and node run in the same process." echo "--isolated Deploy with an isolated environment (no default gateway)" echo "--delete Delete current cluster" echo "--deploy Deploy ovn kubernetes without restarting kind" @@ -307,6 +309,8 @@ parse_args() { ;; -sm | --scale-metrics ) OVN_METRICS_SCALE_ENABLE=true ;; + -cm | --compact-mode ) OVN_COMPACT_MODE=true + ;; --isolated ) OVN_ISOLATED=true ;; -mne | --multi-network-enable ) shift @@ -537,6 +541,10 @@ set_default_params() { fi ENABLE_MULTI_NET=${ENABLE_MULTI_NET:-false} OVN_SEPARATE_CLUSTER_MANAGER=${OVN_SEPARATE_CLUSTER_MANAGER:-false} + OVN_COMPACT_MODE=${OVN_COMPACT_MODE:-false} + if [ "$OVN_COMPACT_MODE" == true ]; then + KIND_NUM_WORKER=0 + fi } detect_apiserver_url() { @@ -791,7 +799,8 @@ create_ovn_kube_manifests() { --v6-join-subnet="${JOIN_SUBNET_IPV6}" \ --ex-gw-network-interface="${OVN_EX_GW_NETWORK_INTERFACE}" \ --multi-network-enable="${ENABLE_MULTI_NET}" \ - --ovnkube-metrics-scale-enable="${OVN_METRICS_SCALE_ENABLE}" + --ovnkube-metrics-scale-enable="${OVN_METRICS_SCALE_ENABLE}" \ + --compact-mode="${OVN_COMPACT_MODE}" popd } diff --git a/dist/images/daemonset.sh b/dist/images/daemonset.sh index 555dbd50aa..d78b48d953 100755 --- a/dist/images/daemonset.sh +++ b/dist/images/daemonset.sh @@ -284,6 +284,9 @@ while [ "$1" != "" ]; do --stateless-netpol-enable) OVN_STATELESS_NETPOL_ENABLE=$VALUE ;; + --compact-mode) + COMPACT_MODE=$VALUE + ;; *) echo "WARNING: unknown parameter \"$PARAM\"" @@ -432,8 +435,11 @@ ovnkube_metrics_scale_enable=${OVNKUBE_METRICS_SCALE_ENABLE} echo "ovnkube_metrics_scale_enable: ${ovnkube_metrics_scale_enable}" ovn_stateless_netpol_enable=${OVN_STATELESS_NETPOL_ENABLE} echo "ovn_stateless_netpol_enable: ${ovn_stateless_netpol_enable}" +ovnkube_compact_mode_enable=${COMPACT_MODE:-"false"} +echo "ovnkube_compact_mode_enable: ${ovnkube_compact_mode_enable}" ovn_image=${ovnkube_image} \ + ovnkube_compact_mode_enable=${ovnkube_compact_mode_enable} \ ovn_image_pull_policy=${image_pull_policy} \ ovn_unprivileged_mode=${ovn_unprivileged_mode} \ ovn_gateway_mode=${ovn_gateway_mode} \ @@ -476,6 +482,7 @@ ovn_image=${ovnkube_image} \ # ovnkube node for dpu-host daemonset # TODO: we probably dont need all of these when running on dpu host ovn_image=${image} \ + ovnkube_compact_mode_enable=${ovnkube_compact_mode_enable} \ ovn_image_pull_policy=${image_pull_policy} \ kind=${KIND} \ ovn_unprivileged_mode=${ovn_unprivileged_mode} \ @@ -536,6 +543,8 @@ ovn_image=${ovnkube_image} \ ovn_gateway_mode=${ovn_gateway_mode} \ ovn_ex_gw_networking_interface=${ovn_ex_gw_networking_interface} \ ovn_stateless_netpol_enable=${ovn_netpol_acl_enable} \ + ovnkube_compact_mode_enable=${ovnkube_compact_mode_enable} \ + ovn_unprivileged_mode=${ovn_unprivileged_mode} \ j2 ../templates/ovnkube-master.yaml.j2 -o ${output_dir}/ovnkube-master.yaml ovn_image=${image} \ diff --git a/dist/images/ovnkube.sh b/dist/images/ovnkube.sh index 2efb290cb7..3005720132 100755 --- a/dist/images/ovnkube.sh +++ b/dist/images/ovnkube.sh @@ -249,6 +249,8 @@ ovnkube_metrics_scale_enable=${OVNKUBE_METRICS_SCALE_ENABLE:-false} ovn_encap_ip=${OVN_ENCAP_IP:-} ovn_ex_gw_network_interface=${OVN_EX_GW_NETWORK_INTERFACE:-} +# OVNKUBE_COMPACT_MODE_ENABLE indicate if ovnkube run master and node in one process +ovnkube_compact_mode_enable=${OVNKUBE_COMPACT_MODE_ENABLE:-false} # Determine the ovn rundir. if [[ -f /usr/bin/ovn-appctl ]]; then @@ -1020,9 +1022,18 @@ ovn-master() { fi echo "ovn_stateless_netpol_enable_flag: ${ovn_stateless_netpol_enable_flag}" - echo "=============== ovn-master ========== MASTER ONLY" + init_node_flags= + if [[ ${ovnkube_compact_mode_enable} == "true" ]]; then + init_node_flags="--init-node ${K8S_NODE} --nodeport" + echo "init_node_flags: ${init_node_flags}" + echo "=============== ovn-master ========== MASTER and NODE" + else + echo "=============== ovn-master ========== MASTER ONLY" + fi + /usr/bin/ovnkube \ --init-master ${K8S_NODE} \ + ${init_node_flags} \ --cluster-subnets ${net_cidr} --k8s-service-cidr=${svc_cidr} \ --nb-address=${ovn_nbdb} --sb-address=${ovn_sbdb} \ --gateway-mode=${ovn_gateway_mode} \ @@ -1055,6 +1066,9 @@ ovn-master() { echo "=============== ovn-master ========== running" wait_for_event attempts=3 process_ready ovnkube-master + if [[ ${ovnkube_compact_mode_enable} == "true" ]] && [[ ${ovnkube_node_mode} != "dpu" ]]; then + setup_cni + fi process_healthy ovnkube-master exit 9 diff --git a/dist/templates/ovnkube-master.yaml.j2 b/dist/templates/ovnkube-master.yaml.j2 index c99e5fa470..dea09ac1c7 100644 --- a/dist/templates/ovnkube-master.yaml.j2 +++ b/dist/templates/ovnkube-master.yaml.j2 @@ -37,7 +37,9 @@ spec: serviceAccountName: ovn hostNetwork: true dnsPolicy: Default - + {% if ovnkube_compact_mode_enable=="true" and ovn_unprivileged_mode=="no" %} + hostPID: true + {% endif %} # required to be scheduled on a linux node with node-role.kubernetes.io/control-plane label and # only one instance of ovnkube-master pod per node affinity: @@ -125,12 +127,22 @@ spec: - name: ovnkube-master image: "{{ ovn_image | default('docker.io/ovnkube/ovn-daemonset:latest') }}" imagePullPolicy: "{{ ovn_image_pull_policy | default('IfNotPresent') }}" - + {% if ovnkube_compact_mode_enable=="true" %} command: ["/root/ovnkube.sh", "ovn-master"] - securityContext: runAsUser: 0 - + {% if ovn_unprivileged_mode=="no" -%} + privileged: true + {% else %} + capabilities: + add: + - NET_ADMIN + {% endif %} + {% else %} + command: ["/root/ovnkube.sh", "ovn-master"] + securityContext: + runAsUser: 0 + {% endif %} terminationMessagePolicy: FallbackToLogsOnError volumeMounts: # Run directories where we need to be able to access sockets @@ -146,7 +158,29 @@ spec: - mountPath: /ovn-cert name: host-ovn-cert readOnly: true - + {% if ovnkube_compact_mode_enable=="true" %} + # Common mounts + # for the iptables wrapper + - mountPath: /host + name: host-slash + readOnly: true + # CNI related mounts which we take over + - mountPath: /opt/cni/bin + name: host-opt-cni-bin + - mountPath: /etc/cni/net.d + name: host-etc-cni-netd + - mountPath: /var/run/netns + name: host-netns + mountPropagation: HostToContainer + - mountPath: /etc/openvswitch/ + name: host-var-lib-ovs + readOnly: true + - mountPath: /etc/ovn/ + name: host-var-lib-ovs + readOnly: true + - mountPath: /var/run/ovn-kubernetes + name: host-var-run-ovn-kubernetes + {% endif %} resources: requests: cpu: 100m @@ -166,6 +200,8 @@ spec: value: "{{ ovnkube_config_duration_enable }}" - name: OVNKUBE_METRICS_SCALE_ENABLE value: "{{ ovnkube_metrics_scale_enable }}" + - name: OVNKUBE_COMPACT_MODE_ENABLE + value: "{{ ovnkube_compact_mode_enable }}" - name: OVN_NET_CIDR valueFrom: configMapKeyRef: @@ -252,5 +288,28 @@ spec: hostPath: path: /etc/ovn type: DirectoryOrCreate + {% if ovnkube_compact_mode_enable=="true" %} + - name: host-slash + hostPath: + path: / + - name: host-opt-cni-bin + hostPath: + path: /opt/cni/bin + - name: host-etc-cni-netd + hostPath: + path: /etc/cni/net.d + - name: host-netns + hostPath: + path: /var/run/netns + - name: host-run-ovs + hostPath: + path: /run/openvswitch + - name: host-var-lib-ovs + hostPath: + path: /var/lib/openvswitch + - name: host-var-run-ovn-kubernetes + hostPath: + path: /var/run/ovn-kubernetes + {% endif %} tolerations: - operator: "Exists" diff --git a/dist/templates/ovnkube-node.yaml.j2 b/dist/templates/ovnkube-node.yaml.j2 index 212b5343bb..34b7baae47 100644 --- a/dist/templates/ovnkube-node.yaml.j2 +++ b/dist/templates/ovnkube-node.yaml.j2 @@ -35,7 +35,7 @@ spec: dnsPolicy: Default {{ "hostPID: true" if ovn_unprivileged_mode=="no" }} containers: - + {% if ovnkube_compact_mode_enable=="false" %} - name: ovnkube-node image: "{{ ovn_image | default('docker.io/ovnkube/ovn-daemonset:latest') }}" imagePullPolicy: "{{ ovn_image_pull_policy | default('IfNotPresent') }}" @@ -228,7 +228,7 @@ spec: initialDelaySeconds: 30 timeoutSeconds: 30 periodSeconds: 60 - + {% endif %} {% if ovnkube_app_name=="ovnkube-node" -%} - name: ovn-controller image: "{{ ovn_image | default('docker.io/ovnkube/ovn-daemonset:latest') }}" From fd69b62e831b1410aa04a1ce6d0de01a2e51d861 Mon Sep 17 00:00:00 2001 From: Peng Liu Date: Tue, 28 Mar 2023 22:36:45 -0400 Subject: [PATCH 21/40] Add compact-mode test to CI Use a single node cluster with compact-mode enabled to mimic the ovnkube setup in Microshift. Signed-off-by: Peng Liu --- .github/workflows/test.yml | 4 ++++ test/Makefile | 1 + test/scripts/e2e-kind.sh | 3 +++ 3 files changed, 8 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6f0302702e..cb3f5bc440 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -379,6 +379,7 @@ jobs: - {"target": "multi-homing", "ha": "noHA", "gateway-mode": "local", "ipfamily": "ipv4", "disable-snat-multiple-gws": "SnatGW", "second-bridge": "1br", "separate-cluster-manager": "false"} - {"target": "node-ip-migration", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv6", "disable-snat-multiple-gws": "SnatGW", "second-bridge": "1br", "separate-cluster-manager": "true"} - {"target": "node-ip-migration", "ha": "noHA", "gateway-mode": "local", "ipfamily": "ipv4", "disable-snat-multiple-gws": "SnatGW", "second-bridge": "1br", "separate-cluster-manager": "false"} + - {"target": "compact-mode", "ha": "noHA", "gateway-mode": "local", "ipfamily": "ipv4", "disable-snat-multiple-gws": "snatGW", "second-bridge": "1br", "separate-cluster-manager": "false"} needs: [ build-pr ] env: JOB_NAME: "${{ matrix.target }}-${{ matrix.ha }}-${{ matrix.gateway-mode }}-${{ matrix.ipfamily }}-${{ matrix.disable-snat-multiple-gws }}-${{ matrix.second-bridge }}" @@ -394,6 +395,7 @@ jobs: KIND_IPV6_SUPPORT: "${{ matrix.ipfamily == 'IPv6' || matrix.ipfamily == 'dualstack' }}" ENABLE_MULTI_NET: "${{ matrix.target == 'multi-homing' }}" OVN_SEPARATE_CLUSTER_MANAGER: "${{ matrix.separate-cluster-manager == 'true' }}" + OVN_COMPACT_MODE: "${{ matrix.target == 'compact-mode' }}" steps: - name: Free up disk space @@ -448,6 +450,8 @@ jobs: make -C test control-plane WHAT="Multi Homing" elif [ "${{ matrix.target }}" == "node-ip-migration" ]; then make -C test control-plane WHAT="Node IP address migration" + elif [ "${{ matrix.target }}" == "compact-mode" ]; then + SINGLE_NODE_CLUSTER="true" make -C test shard-network else make -C test ${{ matrix.target }} fi diff --git a/test/Makefile b/test/Makefile index c1c8cf86d2..68e01130f4 100644 --- a/test/Makefile +++ b/test/Makefile @@ -24,6 +24,7 @@ shard-%: KIND_IPV4_SUPPORT=$(KIND_IPV4_SUPPORT) \ KIND_IPV6_SUPPORT=$(KIND_IPV6_SUPPORT) \ DUALSTACK_CONVERSION=$(DUALSTACK_CONVERSION) \ + SINGLE_NODE_CLUSTER=$(SINGLE_NODE_CLUSTER) \ ./scripts/e2e-kind.sh $@ $(WHAT) .PHONY: control-plane diff --git a/test/scripts/e2e-kind.sh b/test/scripts/e2e-kind.sh index 7deba56a16..1fedd9f362 100755 --- a/test/scripts/e2e-kind.sh +++ b/test/scripts/e2e-kind.sh @@ -163,6 +163,9 @@ export FLAKE_ATTEMPTS=5 export NUM_NODES=10 # number of parallel (ginkgo) test nodes to run # Kind clusters are three node clusters export NUM_WORKER_NODES=3 +if [ "$SINGLE_NODE_CLUSTER" == true ]; then + export NUM_WORKER_NODES=1 +fi ginkgo --nodes=${NUM_NODES} \ --focus=${FOCUS} \ --skip=${SKIPPED_TESTS} \ From e069c2b2b87c97e749d7c53af413a5f106aae1c1 Mon Sep 17 00:00:00 2001 From: Peng Liu Date: Fri, 7 Apr 2023 12:01:23 +0800 Subject: [PATCH 22/40] Remove the local gw node-ip migration lane from CI Signed-off-by: Peng Liu --- .github/workflows/test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cb3f5bc440..0d2d1f227a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -378,7 +378,6 @@ jobs: - {"target": "control-plane", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv4", "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "2br", "separate-cluster-manager": "false"} - {"target": "multi-homing", "ha": "noHA", "gateway-mode": "local", "ipfamily": "ipv4", "disable-snat-multiple-gws": "SnatGW", "second-bridge": "1br", "separate-cluster-manager": "false"} - {"target": "node-ip-migration", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv6", "disable-snat-multiple-gws": "SnatGW", "second-bridge": "1br", "separate-cluster-manager": "true"} - - {"target": "node-ip-migration", "ha": "noHA", "gateway-mode": "local", "ipfamily": "ipv4", "disable-snat-multiple-gws": "SnatGW", "second-bridge": "1br", "separate-cluster-manager": "false"} - {"target": "compact-mode", "ha": "noHA", "gateway-mode": "local", "ipfamily": "ipv4", "disable-snat-multiple-gws": "snatGW", "second-bridge": "1br", "separate-cluster-manager": "false"} needs: [ build-pr ] env: From 71232c9a92f39f1dce0ae62c426d143cc87d05ba Mon Sep 17 00:00:00 2001 From: Peng Liu Date: Tue, 11 Apr 2023 23:02:24 +0800 Subject: [PATCH 23/40] Add -scm to the output of kind.sh --help Signed-off-by: Peng Liu --- contrib/kind.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/kind.sh b/contrib/kind.sh index f3f13651a8..8d720741de 100755 --- a/contrib/kind.sh +++ b/contrib/kind.sh @@ -119,6 +119,7 @@ usage() { echo "-kt | --keep-taint Do not remove taint components." echo " DEFAULT: Remove taint components." echo "-ha | --ha-enabled Enable high availability. DEFAULT: HA Disabled." + echo "-scm | --separate-cluster-manager Separate cluster manager from ovnkube-master and run as a separate container within ovnkube-master deployment." echo "-ho | --hybrid-enabled Enable hybrid overlay. DEFAULT: Disabled." echo "-ds | --disable-snat-multiple-gws Disable SNAT for multiple gws. DEFAULT: Disabled." echo "-dp | --disable-pkt-mtu-check Disable checking packet size greater than MTU. Default: Disabled" From 0c79228bd73f4040927927935dfb40b7c5cb8d8e Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Mon, 10 Apr 2023 11:36:51 -0500 Subject: [PATCH 24/40] clustermanager: update node annotations on dual->single stack converstion Remove the no-longer-used node host subnet annotation to allow the default network controller's node code to proceed. Also redo how the default network controller counts host subnets to make sure we actually have the right number of each IP family, rather than just counting numbers. Because the allocation handling was split out into the ClusterManager, the network controller's addNode() no longer has the guarantees of a correct set of annotations before it runs, as it and the cluster manager run in parallel. Signed-off-by: Dan Williams --- .../network_cluster_controller.go | 30 +-- .../network_cluster_controller_test.go | 203 ++++++++++++++++++ go-controller/pkg/ovn/master.go | 21 +- go-controller/pkg/ovn/master_test.go | 59 +++++ 4 files changed, 291 insertions(+), 22 deletions(-) create mode 100644 go-controller/pkg/clustermanager/network_cluster_controller_test.go diff --git a/go-controller/pkg/clustermanager/network_cluster_controller.go b/go-controller/pkg/clustermanager/network_cluster_controller.go index 19e381df5e..d2c1a37938 100644 --- a/go-controller/pkg/clustermanager/network_cluster_controller.go +++ b/go-controller/pkg/clustermanager/network_cluster_controller.go @@ -203,32 +203,36 @@ func (ncc *networkClusterController) syncNodeClusterSubnet(node *corev1.Node) er klog.Warningf("Failed to get node %s host subnets annotations for network %s : %v", node.Name, ncc.networkName, err) } + // On return validExistingSubnets will contain any valid subnets that + // were already assigned to the node. allocatedSubnets will contain + // any newly allocated subnets required to ensure that the node has one subnet + // from each enabled IP family. ipv4Mode, ipv6Mode := ncc.IPMode() - hostSubnets, allocatedSubnets, err := ncc.clusterSubnetAllocator.AllocateNodeSubnets(node.Name, existingSubnets, ipv4Mode, ipv6Mode) + validExistingSubnets, allocatedSubnets, err := ncc.clusterSubnetAllocator.AllocateNodeSubnets(node.Name, existingSubnets, ipv4Mode, ipv6Mode) if err != nil { return err } - if len(allocatedSubnets) == 0 { - return nil - } - - // Release the allocation on error - defer func() { + // If the existing subnets weren't OK, or new ones were allocated, update the node annotation. + // This happens in a couple cases: + // 1) new node: no existing subnets and one or more new subnets were allocated + // 2) dual-stack to single-stack conversion: two existing subnets but only one will be valid, and no allocated subnets + // 3) bad subnet annotation: one more existing subnets will be invalid and might have allocated a correct one + if len(existingSubnets) != len(validExistingSubnets) || len(allocatedSubnets) > 0 { + updatedSubnetsMap := map[string][]*net.IPNet{ncc.networkName: validExistingSubnets} + err = ncc.updateNodeSubnetAnnotationWithRetry(node.Name, updatedSubnetsMap) if err != nil { if errR := ncc.clusterSubnetAllocator.ReleaseNodeSubnets(node.Name, allocatedSubnets...); errR != nil { klog.Warningf("Error releasing node %s subnets: %v", node.Name, errR) } + return err } - }() - - hostSubnetsMap := map[string][]*net.IPNet{ncc.networkName: hostSubnets} + } - err = ncc.updateNodeSubnetAnnotationWithRetry(node.Name, hostSubnetsMap) - return err + return nil } -// handleAddUpdateNodeEvent handles the delete node event +// handleDeleteNode handles the delete node event func (ncc *networkClusterController) handleDeleteNode(node *corev1.Node) error { if ncc.enableHybridOverlaySubnetAllocator { ncc.releaseHybridOverlayNodeSubnet(node.Name) diff --git a/go-controller/pkg/clustermanager/network_cluster_controller_test.go b/go-controller/pkg/clustermanager/network_cluster_controller_test.go new file mode 100644 index 0000000000..5cc77ab408 --- /dev/null +++ b/go-controller/pkg/clustermanager/network_cluster_controller_test.go @@ -0,0 +1,203 @@ +package clustermanager + +import ( + "context" + "net" + "sync" + + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" + "github.com/urfave/cli/v2" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" + ovntest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing" + ovntypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" +) + +var _ = ginkgo.Describe("Network Cluster Controller", func() { + var ( + app *cli.App + f *factory.WatchFactory + stopChan chan struct{} + wg *sync.WaitGroup + ) + + ginkgo.BeforeEach(func() { + // Restore global default values before each testcase + gomega.Expect(config.PrepareTestConfig()).To(gomega.Succeed()) + + app = cli.NewApp() + app.Name = "test" + app.Flags = config.Flags + stopChan = make(chan struct{}) + wg = &sync.WaitGroup{} + }) + + ginkgo.AfterEach(func() { + close(stopChan) + if f != nil { + f.Shutdown() + } + wg.Wait() + }) + + ginkgo.Context("Host Subnets", func() { + ginkgo.It("removes an unused dual-stack allocation from single-stack cluster", func() { + app.Action = func(ctx *cli.Context) error { + nodes := []v1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Annotations: map[string]string{ + "k8s.ovn.org/node-subnets": "{\"default\":[\"10.128.0.0/24\", \"fd02:0:0:2::2895/64\"]}", + }, + }, + }, + } + kubeFakeClient := fake.NewSimpleClientset(&v1.NodeList{ + Items: nodes, + }) + fakeClient := &util.OVNClusterManagerClientset{ + KubeClient: kubeFakeClient, + } + + _, err := config.InitConfig(ctx, nil, nil) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + config.Kubernetes.HostNetworkNamespace = "" + + f, err = factory.NewClusterManagerWatchFactory(fakeClient) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = f.Start() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ncc := newNetworkClusterController(ovntypes.DefaultNetworkName, config.Default.ClusterSubnets, + fakeClient, f, false, &util.DefaultNetInfo{}, &util.DefaultNetConfInfo{}) + ncc.Start(ctx.Context) + defer ncc.Stop() + + // Check that the default network controller removes the unused v6 node subnet annotation + gomega.Eventually(func() ([]*net.IPNet, error) { + updatedNode, err := fakeClient.KubeClient.CoreV1().Nodes().Get(context.TODO(), nodes[0].Name, metav1.GetOptions{}) + if err != nil { + return nil, err + } + + return util.ParseNodeHostSubnetAnnotation(updatedNode, ovntypes.DefaultNetworkName) + }, 2).Should(gomega.Equal(ovntest.MustParseIPNets("10.128.0.0/24"))) + + return nil + } + + err := app.Run([]string{ + app.Name, + }) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + + ginkgo.It("allocates a subnet for a new node", func() { + app.Action = func(ctx *cli.Context) error { + nodes := []v1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + }, + } + kubeFakeClient := fake.NewSimpleClientset(&v1.NodeList{ + Items: nodes, + }) + fakeClient := &util.OVNClusterManagerClientset{ + KubeClient: kubeFakeClient, + } + + _, err := config.InitConfig(ctx, nil, nil) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + config.Kubernetes.HostNetworkNamespace = "" + + f, err = factory.NewClusterManagerWatchFactory(fakeClient) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = f.Start() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ncc := newNetworkClusterController(ovntypes.DefaultNetworkName, config.Default.ClusterSubnets, + fakeClient, f, false, &util.DefaultNetInfo{}, &util.DefaultNetConfInfo{}) + ncc.Start(ctx.Context) + defer ncc.Stop() + + // Check that the default network controller adds a subnet for the new node + gomega.Eventually(func() ([]*net.IPNet, error) { + updatedNode, err := fakeClient.KubeClient.CoreV1().Nodes().Get(context.TODO(), nodes[0].Name, metav1.GetOptions{}) + if err != nil { + return nil, err + } + + return util.ParseNodeHostSubnetAnnotation(updatedNode, ovntypes.DefaultNetworkName) + }, 2).Should(gomega.Equal(ovntest.MustParseIPNets("10.128.0.0/23"))) + + return nil + } + + err := app.Run([]string{ + app.Name, + }) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + + ginkgo.It("removes an invalid single-stack annotation", func() { + app.Action = func(ctx *cli.Context) error { + nodes := []v1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Annotations: map[string]string{ + "k8s.ovn.org/node-subnets": "{\"default\":[\"10.128.0.0/24\", \"1.2.3.0/24\"]}", + }, + }, + }, + } + kubeFakeClient := fake.NewSimpleClientset(&v1.NodeList{ + Items: nodes, + }) + fakeClient := &util.OVNClusterManagerClientset{ + KubeClient: kubeFakeClient, + } + + _, err := config.InitConfig(ctx, nil, nil) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + config.Kubernetes.HostNetworkNamespace = "" + + f, err = factory.NewClusterManagerWatchFactory(fakeClient) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = f.Start() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ncc := newNetworkClusterController(ovntypes.DefaultNetworkName, config.Default.ClusterSubnets, + fakeClient, f, false, &util.DefaultNetInfo{}, &util.DefaultNetConfInfo{}) + ncc.Start(ctx.Context) + defer ncc.Stop() + + // Check that the default network controller removes the unused v6 node subnet annotation + gomega.Eventually(func() ([]*net.IPNet, error) { + updatedNode, err := fakeClient.KubeClient.CoreV1().Nodes().Get(context.TODO(), nodes[0].Name, metav1.GetOptions{}) + if err != nil { + return nil, err + } + + return util.ParseNodeHostSubnetAnnotation(updatedNode, ovntypes.DefaultNetworkName) + }, 2).Should(gomega.Equal(ovntest.MustParseIPNets("10.128.0.0/24"))) + + return nil + } + + err := app.Run([]string{ + app.Name, + }) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + }) +}) diff --git a/go-controller/pkg/ovn/master.go b/go-controller/pkg/ovn/master.go index 065cfac974..1428eda56e 100644 --- a/go-controller/pkg/ovn/master.go +++ b/go-controller/pkg/ovn/master.go @@ -402,16 +402,19 @@ func (oc *DefaultNetworkController) addNode(node *kapi.Node) ([]*net.IPNet, erro return nil, err } - // OVN can work in single-stack or dual-stack only. - currentHostSubnets := len(hostSubnets) - expectedHostSubnets := 1 - // if dual-stack mode we expect one subnet per each IP family - if config.IPv4Mode && config.IPv6Mode { - expectedHostSubnets = 2 + // We expect one subnet per configured ClusterNetwork IP family. + var haveV4, haveV6 bool + for _, net := range hostSubnets { + if !haveV4 { + haveV4 = net.IP.To4() != nil + } + if !haveV6 { + haveV6 = net.IP.To4() == nil + } } - - if expectedHostSubnets != currentHostSubnets { - return nil, fmt.Errorf("failed to get expected host subnets for node %s, expected %d subnet(s) but current number of subnet(s) is %d", node.Name, expectedHostSubnets, currentHostSubnets) + if haveV4 != config.IPv4Mode || haveV6 != config.IPv6Mode { + return nil, fmt.Errorf("failed to get expected host subnets for node %s; expected v4 %v have %v, expected v6 %v have %v", + node.Name, config.IPv4Mode, haveV4, config.IPv6Mode, haveV6) } gwLRPIPs, err := oc.joinSwIPManager.EnsureJoinLRPIPs(node.Name) diff --git a/go-controller/pkg/ovn/master_test.go b/go-controller/pkg/ovn/master_test.go index 8af83aa61e..883c115a21 100644 --- a/go-controller/pkg/ovn/master_test.go +++ b/go-controller/pkg/ovn/master_test.go @@ -1464,6 +1464,65 @@ var _ = ginkgo.Describe("Default network controller operations", func() { }) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) + + ginkgo.It("reconciles node host subnets after dual-stack to single-stack downgrade", func() { + app.Action = func(ctx *cli.Context) error { + _, err := config.InitConfig(ctx, nil, nil) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + newNodeSubnet := "10.1.1.0/24" + newNode := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "newNode", + Annotations: map[string]string{ + "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\":[\"%s\", \"fd02:0:0:2::2895/64\"]}", newNodeSubnet), + }, + }, + } + + _, err = kubeFakeClient.CoreV1().Nodes().Create(context.TODO(), newNode, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + startFakeController(oc, wg) + + // check that a node event complaining about the mismatch between + // node subnets and cluster subnets was posted + gomega.Eventually(func() []string { + eventsLock.Lock() + defer eventsLock.Unlock() + eventsCopy := make([]string, 0, len(events)) + for _, e := range events { + eventsCopy = append(eventsCopy, e) + } + return eventsCopy + }, 10).Should(gomega.ContainElement(gomega.ContainSubstring("failed to get expected host subnets for node newNode; expected v4 true have true, expected v6 false have true"))) + + // Simulate the ClusterManager reconciling the node annotations to single-stack + newNode, err = kubeFakeClient.CoreV1().Nodes().Get(context.TODO(), newNode.Name, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + newNode.Annotations["k8s.ovn.org/node-subnets"] = fmt.Sprintf("{\"default\":[\"%s\"]}", newNodeSubnet) + _, err = kubeFakeClient.CoreV1().Nodes().Update(context.TODO(), newNode, metav1.UpdateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + // Ensure that the node's switch is eventually created once the annotations + // are reconiled by the network cluster controller + newNodeLS := &nbdb.LogicalSwitch{Name: newNode.Name} + gomega.Eventually(func() error { + _, err := libovsdbops.GetLogicalSwitch(nbClient, newNodeLS) + return err + }, 10).ShouldNot(gomega.HaveOccurred()) + + return nil + } + + err := app.Run([]string{ + app.Name, + "-cluster-subnets=" + clusterCIDR, + "--init-gateways", + "--nodeport", + }) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) }) func nodeNoHostSubnetAnnotation() map[string]string { From bf3da15efc3be84a9975d8c7ff54a754fe05b658 Mon Sep 17 00:00:00 2001 From: Alin-Gabriel Serdean Date: Wed, 8 Feb 2023 00:12:51 +0100 Subject: [PATCH 25/40] Ensure VF representor is not managed by OVS before renaming. If the representor is managed by OVS when renaming, it will cause an inconsistency inside OVS between the userspace and datapath, resulting in a bad situation where the interface has to be renamed to the original value before the port can be properly removed from the userspace and datapath. This patch bypasses the issue by deleting the port inside OVS if it exists. Signed-off-by: Alin-Gabriel Serdean --- go-controller/pkg/cni/helper_linux.go | 10 ++- go-controller/pkg/cni/helper_linux_test.go | 86 +++++++++++++++++++--- 2 files changed, 85 insertions(+), 11 deletions(-) diff --git a/go-controller/pkg/cni/helper_linux.go b/go-controller/pkg/cni/helper_linux.go index f3ac2ebf33..478cc19c0d 100644 --- a/go-controller/pkg/cni/helper_linux.go +++ b/go-controller/pkg/cni/helper_linux.go @@ -324,7 +324,13 @@ func setupSriovInterface(netns ns.NetNS, containerID, ifName string, ifInfo *Pod } oldHostRepName := rep - // 5. rename the host VF representor + // 5. make sure it's not a port managed by OVS to avoid conflicts when renaming the VF representor + _, err = ovsExec("--if-exists", "del-port", oldHostRepName) + if err != nil { + return nil, nil, err + } + + // 6. rename the host VF representor hostIface.Name = containerID[:(15-len(ifnameSuffix))] + ifnameSuffix if err = renameLink(oldHostRepName, hostIface.Name); err != nil { return nil, nil, fmt.Errorf("failed to rename %s to %s: %v", oldHostRepName, hostIface.Name, err) @@ -336,7 +342,7 @@ func setupSriovInterface(netns ns.NetNS, containerID, ifName string, ifInfo *Pod } hostIface.Mac = link.Attrs().HardwareAddr.String() - // 6. set MTU on VF representor + // 7. set MTU on VF representor if err = util.GetNetLinkOps().LinkSetMTU(link, ifInfo.MTU); err != nil { return nil, nil, fmt.Errorf("failed to set MTU on %s: %v", hostIface.Name, err) } diff --git a/go-controller/pkg/cni/helper_linux_test.go b/go-controller/pkg/cni/helper_linux_test.go index 347f43a040..a2887906bd 100644 --- a/go-controller/pkg/cni/helper_linux_test.go +++ b/go-controller/pkg/cni/helper_linux_test.go @@ -16,11 +16,13 @@ import ( cni_type_mocks "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/mocks/github.com/containernetworking/cni/pkg/types" cni_ns_mocks "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/mocks/github.com/containernetworking/plugins/pkg/ns" netlink_mocks "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/mocks/github.com/vishvananda/netlink" + mock_k8s_io_utils_exec "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/mocks/k8s.io/utils/exec" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" util_mocks "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util/mocks" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/vishvananda/netlink" + kexec "k8s.io/utils/exec" ) func TestRenameLink(t *testing.T) { @@ -424,6 +426,8 @@ func TestSetupSriovInterface(t *testing.T) { mockSriovnetOps := new(util_mocks.SriovnetOps) mockNS := new(cni_ns_mocks.NetNS) mockLink := new(netlink_mocks.Link) + mockKexecIface := new(mock_k8s_io_utils_exec.Interface) + mockCmd := new(mock_k8s_io_utils_exec.Cmd) // below sets the `netLinkOps` in util/net_linux.go to a mock instance for purpose of unit tests execution util.SetNetLinkOpMockInst(mockNetLinkOps) // `cniPluginLibOps` is defined in helper_linux.go @@ -462,6 +466,9 @@ func TestSetupSriovInterface(t *testing.T) { netLinkOpsMockHelper []ovntest.TestifyMockHelper sriovOpsMockHelper []ovntest.TestifyMockHelper linkMockHelper []ovntest.TestifyMockHelper + onRetArgsKexecIface []ovntest.TestifyMockHelper + onRetArgsCmdList []ovntest.TestifyMockHelper + runnerInstance kexec.Interface }{ { desc: "test code path when moveIfToNetns() returns error", @@ -473,12 +480,15 @@ func TestSetupSriovInterface(t *testing.T) { MTU: 1500, VfNetdevName: "en01", }, - inpPCIAddrs: "0000:03:00.1", - errExp: true, - sriovOpsMockHelper: []ovntest.TestifyMockHelper{}, + inpPCIAddrs: "0000:03:00.1", + errExp: true, + sriovOpsMockHelper: []ovntest.TestifyMockHelper{}, + onRetArgsKexecIface: []ovntest.TestifyMockHelper{{OnCallMethodName: "Command", OnCallMethodArgType: []string{"string", "string"}, RetArgList: []interface{}{mockCmd}}}, + onRetArgsCmdList: []ovntest.TestifyMockHelper{{OnCallMethodName: "CombinedOutput", OnCallMethodArgType: []string{}, RetArgList: []interface{}{nil, fmt.Errorf("failed to run 'ovs-vsctl")}}}, netLinkOpsMockHelper: []ovntest.TestifyMockHelper{ {OnCallMethodName: "LinkByName", OnCallMethodArgType: []string{"string"}, RetArgList: []interface{}{nil, fmt.Errorf("mock error")}}, }, + runnerInstance: mockKexecIface, }, { desc: "test code path when Do() returns error", @@ -490,9 +500,11 @@ func TestSetupSriovInterface(t *testing.T) { MTU: 1500, VfNetdevName: "en01", }, - inpPCIAddrs: "0000:03:00.1", - errExp: true, - sriovOpsMockHelper: []ovntest.TestifyMockHelper{}, + inpPCIAddrs: "0000:03:00.1", + errExp: true, + sriovOpsMockHelper: []ovntest.TestifyMockHelper{}, + onRetArgsKexecIface: []ovntest.TestifyMockHelper{{OnCallMethodName: "Command", OnCallMethodArgType: []string{"string", "string"}, RetArgList: []interface{}{mockCmd}}}, + onRetArgsCmdList: []ovntest.TestifyMockHelper{{OnCallMethodName: "CombinedOutput", OnCallMethodArgType: []string{}, RetArgList: []interface{}{nil, nil}}}, netLinkOpsMockHelper: []ovntest.TestifyMockHelper{ {OnCallMethodName: "LinkByName", OnCallMethodArgType: []string{"string"}, RetArgList: []interface{}{mockLink, nil}}, {OnCallMethodName: "LinkSetNsFd", OnCallMethodArgType: []string{"*mocks.Link", "int"}, RetArgList: []interface{}{nil}}, @@ -503,6 +515,7 @@ func TestSetupSriovInterface(t *testing.T) { // The below mock call is for the netns.Do() invocation {OnCallMethodName: "Do", OnCallMethodArgType: []string{"func(ns.NetNS) error"}, RetArgList: []interface{}{fmt.Errorf("mock error")}}, }, + runnerInstance: mockKexecIface, }, { desc: "test code path when GetUplinkRepresentor() returns error", @@ -514,8 +527,11 @@ func TestSetupSriovInterface(t *testing.T) { MTU: 1500, VfNetdevName: "en01", }, - inpPCIAddrs: "0000:03:00.1", - errExp: true, + inpPCIAddrs: "0000:03:00.1", + errExp: true, + onRetArgsKexecIface: []ovntest.TestifyMockHelper{{OnCallMethodName: "Command", OnCallMethodArgType: []string{"string", "string"}, RetArgList: []interface{}{mockCmd}}}, + onRetArgsCmdList: []ovntest.TestifyMockHelper{{OnCallMethodName: "CombinedOutput", OnCallMethodArgType: []string{}, RetArgList: []interface{}{nil, fmt.Errorf("failed to run 'ovs-vsctl")}}}, + runnerInstance: mockKexecIface, sriovOpsMockHelper: []ovntest.TestifyMockHelper{ {OnCallMethodName: "GetUplinkRepresentor", OnCallMethodArgType: []string{"string"}, RetArgList: []interface{}{"", fmt.Errorf("mock error")}}, }, @@ -542,6 +558,11 @@ func TestSetupSriovInterface(t *testing.T) { }, inpPCIAddrs: "0000:03:00.1", errExp: true, + onRetArgsKexecIface: []ovntest.TestifyMockHelper{ + {OnCallMethodName: "Command", OnCallMethodArgType: []string{"string", "string", "string", "string", "string"}, RetArgList: []interface{}{mockCmd}}, + }, + onRetArgsCmdList: []ovntest.TestifyMockHelper{{OnCallMethodName: "CombinedOutput", OnCallMethodArgType: []string{}, RetArgList: []interface{}{nil, fmt.Errorf("failed to run 'ovs-vsctl")}}}, + runnerInstance: mockKexecIface, sriovOpsMockHelper: []ovntest.TestifyMockHelper{ {OnCallMethodName: "GetUplinkRepresentor", OnCallMethodArgType: []string{"string"}, RetArgList: []interface{}{"testlinkrepresentor", nil}}, {OnCallMethodName: "GetVfIndexByPciAddress", OnCallMethodArgType: []string{"string"}, RetArgList: []interface{}{-1, fmt.Errorf("mock error")}}, @@ -569,6 +590,11 @@ func TestSetupSriovInterface(t *testing.T) { }, inpPCIAddrs: "0000:03:00.1", errExp: true, + onRetArgsKexecIface: []ovntest.TestifyMockHelper{ + {OnCallMethodName: "Command", OnCallMethodArgType: []string{"string", "string"}, RetArgList: []interface{}{mockCmd}}, + }, + onRetArgsCmdList: []ovntest.TestifyMockHelper{{OnCallMethodName: "CombinedOutput", OnCallMethodArgType: []string{}, RetArgList: []interface{}{nil, nil}}}, + runnerInstance: mockKexecIface, sriovOpsMockHelper: []ovntest.TestifyMockHelper{ {OnCallMethodName: "GetUplinkRepresentor", OnCallMethodArgType: []string{"string"}, RetArgList: []interface{}{"testlinkrepresentor", nil}}, {OnCallMethodName: "GetVfIndexByPciAddress", OnCallMethodArgType: []string{"string"}, RetArgList: []interface{}{0, nil}}, @@ -597,6 +623,15 @@ func TestSetupSriovInterface(t *testing.T) { }, inpPCIAddrs: "0000:03:00.1", errMatch: fmt.Errorf("failed to rename"), + onRetArgsKexecIface: []ovntest.TestifyMockHelper{ + {OnCallMethodName: "Command", OnCallMethodArgType: []string{"string", "string", "string", "string", "string"}, RetArgList: []interface{}{mockCmd}}, + {OnCallMethodName: "Command", OnCallMethodArgType: []string{"string", "string", "string", "string", "string"}, RetArgList: []interface{}{mockCmd}}, + }, + onRetArgsCmdList: []ovntest.TestifyMockHelper{ + {OnCallMethodName: "CombinedOutput", OnCallMethodArgType: []string{}, RetArgList: []interface{}{nil, nil}}, + {OnCallMethodName: "CombinedOutput", OnCallMethodArgType: []string{}, RetArgList: []interface{}{nil, nil}}, + }, + runnerInstance: mockKexecIface, sriovOpsMockHelper: []ovntest.TestifyMockHelper{ {OnCallMethodName: "GetUplinkRepresentor", OnCallMethodArgType: []string{"string"}, RetArgList: []interface{}{"testlinkrepresentor", nil}}, {OnCallMethodName: "GetVfIndexByPciAddress", OnCallMethodArgType: []string{"string"}, RetArgList: []interface{}{0, nil}}, @@ -628,6 +663,15 @@ func TestSetupSriovInterface(t *testing.T) { }, inpPCIAddrs: "0000:03:00.1", errExp: true, + onRetArgsKexecIface: []ovntest.TestifyMockHelper{ + {OnCallMethodName: "Command", OnCallMethodArgType: []string{"string", "string", "string", "string", "string"}, RetArgList: []interface{}{mockCmd}}, + {OnCallMethodName: "Command", OnCallMethodArgType: []string{"string", "string", "string", "string", "string"}, RetArgList: []interface{}{mockCmd}}, + }, + onRetArgsCmdList: []ovntest.TestifyMockHelper{ + {OnCallMethodName: "CombinedOutput", OnCallMethodArgType: []string{}, RetArgList: []interface{}{nil, nil}}, + {OnCallMethodName: "CombinedOutput", OnCallMethodArgType: []string{}, RetArgList: []interface{}{nil, nil}}, + }, + runnerInstance: mockKexecIface, sriovOpsMockHelper: []ovntest.TestifyMockHelper{ {OnCallMethodName: "GetUplinkRepresentor", OnCallMethodArgType: []string{"string"}, RetArgList: []interface{}{"testlinkrepresentor", nil}}, {OnCallMethodName: "GetVfIndexByPciAddress", OnCallMethodArgType: []string{"string"}, RetArgList: []interface{}{0, nil}}, @@ -664,6 +708,15 @@ func TestSetupSriovInterface(t *testing.T) { }, inpPCIAddrs: "0000:03:00.1", errMatch: fmt.Errorf("failed to set MTU on"), + onRetArgsKexecIface: []ovntest.TestifyMockHelper{ + {OnCallMethodName: "Command", OnCallMethodArgType: []string{"string", "string", "string", "string", "string"}, RetArgList: []interface{}{mockCmd}}, + {OnCallMethodName: "Command", OnCallMethodArgType: []string{"string", "string", "string", "string", "string"}, RetArgList: []interface{}{mockCmd}}, + }, + onRetArgsCmdList: []ovntest.TestifyMockHelper{ + {OnCallMethodName: "CombinedOutput", OnCallMethodArgType: []string{}, RetArgList: []interface{}{nil, nil}}, + {OnCallMethodName: "CombinedOutput", OnCallMethodArgType: []string{}, RetArgList: []interface{}{nil, nil}}, + }, + runnerInstance: mockKexecIface, sriovOpsMockHelper: []ovntest.TestifyMockHelper{ {OnCallMethodName: "GetUplinkRepresentor", OnCallMethodArgType: []string{"string"}, RetArgList: []interface{}{"testlinkrepresentor", nil}}, {OnCallMethodName: "GetVfIndexByPciAddress", OnCallMethodArgType: []string{"string"}, RetArgList: []interface{}{0, nil}}, @@ -708,6 +761,13 @@ func TestSetupSriovInterface(t *testing.T) { inpPCIAddrs: "0000:03:00.1", errExp: false, sriovOpsMockHelper: []ovntest.TestifyMockHelper{}, + onRetArgsKexecIface: []ovntest.TestifyMockHelper{ + {OnCallMethodName: "Command", OnCallMethodArgType: []string{"string", "string", "string", "string", "string"}, RetArgList: []interface{}{mockCmd}}, + }, + onRetArgsCmdList: []ovntest.TestifyMockHelper{ + {OnCallMethodName: "CombinedOutput", OnCallMethodArgType: []string{}, RetArgList: []interface{}{nil, nil}}, + }, + runnerInstance: mockKexecIface, netLinkOpsMockHelper: []ovntest.TestifyMockHelper{ // The below two mock calls are needed for the moveIfToNetns() call that internally invokes them {OnCallMethodName: "LinkByName", OnCallMethodArgType: []string{"string"}, RetArgList: []interface{}{mockLink, nil}}, @@ -1062,8 +1122,14 @@ func TestSetupSriovInterface(t *testing.T) { ovntest.ProcessMockFnList(&mockNS.Mock, tc.nsMockHelper) ovntest.ProcessMockFnList(&mockSriovnetOps.Mock, tc.sriovOpsMockHelper) ovntest.ProcessMockFnList(&mockLink.Mock, tc.linkMockHelper) - netNsDoError = nil + ovntest.ProcessMockFnList(&mockKexecIface.Mock, tc.onRetArgsKexecIface) + ovntest.ProcessMockFnList(&mockCmd.Mock, tc.onRetArgsCmdList) + + runner = tc.runnerInstance + ovsExec() + + netNsDoError = nil hostIface, contIface, err := setupSriovInterface(tc.inpNetNS, tc.inpContID, tc.inpIfaceName, tc.inpPodIfaceInfo, tc.inpPCIAddrs) t.Log(hostIface, contIface, err) if err == nil { @@ -1081,6 +1147,8 @@ func TestSetupSriovInterface(t *testing.T) { mockNS.AssertExpectations(t) mockSriovnetOps.AssertExpectations(t) mockLink.AssertExpectations(t) + mockCmd.AssertExpectations(t) + mockKexecIface.AssertExpectations(t) }) } } From 357e79bf4a145a9faa9b59e2ee6ed703c9a20675 Mon Sep 17 00:00:00 2001 From: Jacob Tanenbaum Date: Mon, 10 Apr 2023 12:03:39 -0400 Subject: [PATCH 26/40] remove the fakeaddressset code from egress_firewall_test.go the fakeAddressSetFactory and by extension FakeAddressSets are no longer needed and can be removed Signed-off-by: Jacob Tanenbaum --- go-controller/pkg/ovn/egressfirewall_test.go | 41 ++++++++++++++++---- go-controller/pkg/ovn/namespace_test.go | 10 +++++ 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/go-controller/pkg/ovn/egressfirewall_test.go b/go-controller/pkg/ovn/egressfirewall_test.go index 4f1d2b3090..905e6af59e 100644 --- a/go-controller/pkg/ovn/egressfirewall_test.go +++ b/go-controller/pkg/ovn/egressfirewall_test.go @@ -77,6 +77,16 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fakeOVN.controller.WatchEgressFirewall() gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + for _, namespace := range namespaces { + namespaceASip4, namespaceASip6 := buildNamespaceAddressSets(namespace.Name, []net.IP{}) + if config.IPv4Mode { + initialData = append(initialData, namespaceASip4) + } + if config.IPv6Mode { + initialData = append(initialData, namespaceASip6) + } + } } ginkgo.BeforeEach(func() { @@ -88,7 +98,7 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { app.Name = "test" app.Flags = config.Flags - fakeOVN = NewFakeOVN(true) + fakeOVN = NewFakeOVN(false) clusterPortGroup = newClusterPortGroup() nodeSwitch = &nbdb.LogicalSwitch{ UUID: node1Name + "-UUID", @@ -151,6 +161,9 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { purgeACL2.UUID = "purgeACL2-UUID" namespace1 := *newNamespace("namespace1") + + namespace1ASip4, _ := buildNamespaceAddressSets(namespace1.Name, []net.IP{}) + egressFirewall := newEgressFirewallObject("default", namespace1.Name, []egressfirewallapi.EgressFirewallRule{ { Type: "Allow", @@ -203,6 +216,7 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { joinSwitch, clusterRouter, clusterPortGroup, + namespace1ASip4, }, } @@ -234,6 +248,7 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { joinSwitch, clusterRouter, clusterPortGroup, + namespace1ASip4, } gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) @@ -513,10 +528,14 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { app.Action = func(ctx *cli.Context) error { expectedOVNClusterRouter := newOVNClusterRouter() expectedClusterPortGroup := newClusterPortGroup() + namespace1 := *newNamespace("namespace1") + + namespace1ASip4, _ := buildNamespaceAddressSets(namespace1.Name, []net.IP{}) initialTestData := []libovsdbtest.TestData{ expectedOVNClusterRouter, expectedClusterPortGroup, + namespace1ASip4, } dbSetup := libovsdbtest.TestSetup{ NBData: initialTestData, @@ -525,7 +544,6 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { labelKey := "name" labelValue := "test" selector := metav1.LabelSelector{MatchLabels: map[string]string{labelKey: labelValue}} - namespace1 := *newNamespace("namespace1") egressFirewall := newEgressFirewallObject("default", namespace1.Name, []egressfirewallapi.EgressFirewallRule{ { Type: "Allow", @@ -600,7 +618,7 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { // new ACL will be added to the port group expectedClusterPortGroup.ACLs = []string{ipv4ACL.UUID} - expectedDatabaseState := []libovsdb.TestData{expectedClusterPortGroup, ipv4ACL, expectedOVNClusterRouter} + expectedDatabaseState := []libovsdb.TestData{expectedClusterPortGroup, ipv4ACL, expectedOVNClusterRouter, namespace1ASip4} gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) @@ -625,6 +643,8 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { config.Gateway.Mode = gwMode app.Action = func(ctx *cli.Context) error { namespace1 := *newNamespace("namespace1") + namespace1ASip4, _ := buildNamespaceAddressSets(namespace1.Name, []net.IP{}) + egressFirewall := newEgressFirewallObject("default", namespace1.Name, []egressfirewallapi.EgressFirewallRule{ { Type: "Allow", @@ -639,7 +659,6 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { }, }, }) - fakeOVN.startWithDBSetup(dbSetup, &egressfirewallapi.EgressFirewallList{ Items: []egressfirewallapi.EgressFirewall{ @@ -683,7 +702,7 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { // new ACL will be added to the port group clusterPortGroup.ACLs = []string{ipv4ACL.UUID} - expectedDatabaseState := append(initialData, ipv4ACL) + expectedDatabaseState := append(initialData, ipv4ACL, namespace1ASip4) gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) ginkgo.By("Bringing down NBDB") @@ -716,7 +735,7 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { // ACL should be removed from the port group after egfw is deleted clusterPortGroup.ACLs = []string{} // this ACL will be deleted when test server starts deleting dereferenced ACLs - expectedDatabaseState = append(initialData, ipv4ACL) + expectedDatabaseState = append(initialData, ipv4ACL, namespace1ASip4) gomega.Eventually(fakeOVN.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) // check the cache no longer has the entry @@ -894,6 +913,14 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { labelValue := "test" selector := metav1.LabelSelector{MatchLabels: map[string]string{labelKey: labelValue}} namespace1 := *newNamespace("namespace1") + namespaceASip4, namespaceASip6 := buildNamespaceAddressSets(namespace1.Name, []net.IP{}) + if config.IPv4Mode { + initialData = append(initialData, namespaceASip4) + } + if config.IPv6Mode { + initialData = append(initialData, namespaceASip6) + } + egressFirewall := newEgressFirewallObject("default", namespace1.Name, []egressfirewallapi.EgressFirewallRule{ { Type: "Allow", @@ -1121,7 +1148,7 @@ var _ = ginkgo.Describe("OVN test basic functions", func() { app.Flags = config.Flags dbSetup := libovsdbtest.TestSetup{} - fakeOVN = NewFakeOVN(true) + fakeOVN = NewFakeOVN(false) a := newObjectMeta(node1Name, "") a.Labels = nodeLabel node1 := v1.Node{ diff --git a/go-controller/pkg/ovn/namespace_test.go b/go-controller/pkg/ovn/namespace_test.go index 0aeed2940a..e3ccc35d33 100644 --- a/go-controller/pkg/ovn/namespace_test.go +++ b/go-controller/pkg/ovn/namespace_test.go @@ -2,10 +2,12 @@ package ovn import ( "context" + "fmt" "net" "sync" "time" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" addressset "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/address_set" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" @@ -69,6 +71,14 @@ func getNsAddrSetHashNames(ns string) (string, string) { return addressset.GetHashNamesForAS(getNamespaceAddrSetDbIDs(ns, DefaultNetworkControllerName)) } +func buildNamespaceAddressSets(namespace string, ips []net.IP) (*nbdb.AddressSet, *nbdb.AddressSet) { + v4set, v6set := addressset.GetDbObjsForAS(getNamespaceAddrSetDbIDs(namespace, "default-network-controller"), ips) + + v4set.UUID = fmt.Sprintf("%s-ipv4-addrSet", namespace) + v6set.UUID = fmt.Sprintf("%s-ipv6-addrSet", namespace) + return v4set, v6set +} + var _ = ginkgo.Describe("OVN Namespace Operations", func() { const ( namespaceName = "namespace1" From 030a4eefdce465128646e54d54a640f0ca29ae4a Mon Sep 17 00:00:00 2001 From: Surya Seetharaman Date: Mon, 3 Apr 2023 14:10:21 +0200 Subject: [PATCH 27/40] Add info about -me flag to kind helper This commit adds the information about how to enable the multicast feature in KIND. This is pretty hidden and not intuitive. Folks have to figure out only by looking at the code. Signed-off-by: Surya Seetharaman --- contrib/kind.sh | 3 ++- docs/kind.md | 3 ++- docs/multicast.md | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/contrib/kind.sh b/contrib/kind.sh index 8d720741de..04849a6be6 100755 --- a/contrib/kind.sh +++ b/contrib/kind.sh @@ -101,7 +101,7 @@ usage() { echo " [-nl |--node-loglevel ] [-ml|--master-loglevel ]" echo " [-dbl|--dbchecker-loglevel ] [-ndl|--ovn-loglevel-northd ]" echo " [-nbl|--ovn-loglevel-nb ] [-sbl|--ovn-loglevel-sb ]" - echo " [-cl |--ovn-loglevel-controller ]" + echo " [-cl |--ovn-loglevel-controller ] [-me|--multicast-enabled]" echo " [-ep |--experimental-provider ] |" echo " [-eb |--egress-gw-separate-bridge] |" echo " [-lr |--local-kind-registry |" @@ -120,6 +120,7 @@ usage() { echo " DEFAULT: Remove taint components." echo "-ha | --ha-enabled Enable high availability. DEFAULT: HA Disabled." echo "-scm | --separate-cluster-manager Separate cluster manager from ovnkube-master and run as a separate container within ovnkube-master deployment." + echo "-me | --multicast-enabled Enable multicast. DEFAULT: Disabled." echo "-ho | --hybrid-enabled Enable hybrid overlay. DEFAULT: Disabled." echo "-ds | --disable-snat-multiple-gws Disable SNAT for multiple gws. DEFAULT: Disabled." echo "-dp | --disable-pkt-mtu-check Disable checking packet size greater than MTU. Default: Disabled" diff --git a/docs/kind.md b/docs/kind.md index 413fed6993..7e3666c7f8 100644 --- a/docs/kind.md +++ b/docs/kind.md @@ -93,7 +93,7 @@ usage: kind.sh [[[-cf |--config-file ] [-kt|keep-taint] [-ha|--ha-enabled] [-nl |--node-loglevel ] [-ml|--master-loglevel ] [-dbl|--dbchecker-loglevel ] [-ndl|--ovn-loglevel-northd ] [-nbl|--ovn-loglevel-nb ] [-sbl|--ovn-loglevel-sb ] - [-cl |--ovn-loglevel-controller ] + [-cl |--ovn-loglevel-controller ] [-me|--multicast-enabled] [-ep |--experimental-provider ] | [-eb |--egress-gw-separate-bridge] [-h]] @@ -103,6 +103,7 @@ usage: kind.sh [[[-cf |--config-file ] [-kt|keep-taint] [-ha|--ha-enabled] -kt | --keep-taint Do not remove taint components. DEFAULT: Remove taint components. -ha | --ha-enabled Enable high availability. DEFAULT: HA Disabled. +-me | --multicast-enabled Enable multicast. DEFAULT: Disabled. -scm | --separate-cluster-manager Separate cluster manager from ovnkube-master and run as a separate container within ovnkube-master deployment. -ho | --hybrid-enabled Enable hybrid overlay. DEFAULT: Disabled. -ds | --disable-snat-multiple-gws Disable SNAT for multiple gws. DEFAULT: Disabled. diff --git a/docs/multicast.md b/docs/multicast.md index bbdc716551..559ce2ced7 100644 --- a/docs/multicast.md +++ b/docs/multicast.md @@ -13,6 +13,8 @@ The multicast group membership is implemented with IGMP. For details, check RFCs and [2236](https://datatracker.ietf.org/doc/html/rfc2236). ## Configuring multicast on the cluster +The feature is gated by config flag. In order to create a KIND cluster with +multicast feature enabled, use the `--multicast-enabled` option with KIND. ### Enabling multicast per namespace The multicast traffic between pods in the cluster is blocked by default; it can From 3395e83e274f908637df84f5666724a719edcad2 Mon Sep 17 00:00:00 2001 From: Surya Seetharaman Date: Tue, 7 Mar 2023 13:05:13 +0100 Subject: [PATCH 28/40] Add unit test to check multicast+long ns name work In multicast feature we have 3 types of ACLs: 1) global allow (one ACL allowing multicast traffic from cluster router ports, one ACL allowing multicast traffic to cluster router ports) 2) global deny (one ACL dropping egress multicast traffic from all pods, one ACL dropping ingress multicast traffic to all pods) 3) per ns allow(one "from-lport" ACL allowing egress multicast traffic from the pods in 'ns', one "to-lport" ACL allowing ingress multicast traffic to pods in 'ns') This PR doesn't change anything functionally, its just adding a unit test to ensure things work correctly when namespaces with long namespace names are created for multicast. Signed-off-by: Surya Seetharaman --- go-controller/pkg/ovn/multicast_test.go | 51 +++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/go-controller/pkg/ovn/multicast_test.go b/go-controller/pkg/ovn/multicast_test.go index d49f1b63b3..f275a97497 100644 --- a/go-controller/pkg/ovn/multicast_test.go +++ b/go-controller/pkg/ovn/multicast_test.go @@ -618,6 +618,57 @@ var _ = ginkgo.Describe("OVN Multicast with IP Address Family", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) + ginkgo.It("tests enabling multicast in multiple namespaces with a long name > 42 characters "+ipModeStr(m), func() { + app.Action = func(ctx *cli.Context) error { + longNameSpace1Name := "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijk" // create with 63 characters + namespace1 := *newNamespace(longNameSpace1Name) + longNameSpace2Name := "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijl" // create with 63 characters + namespace2 := *newNamespace(longNameSpace2Name) + + fakeOvn.startWithDBSetup(libovsdb.TestSetup{NBData: getNodeSwitch(nodeName)}, + &v1.NamespaceList{ + Items: []v1.Namespace{ + namespace1, + namespace2, + }, + }, + ) + setIpMode(m) + + err := fakeOvn.controller.WatchNamespaces() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + fakeOvn.controller.WatchPods() + ns1, err := fakeOvn.fakeClient.KubeClient.CoreV1().Namespaces().Get(context.TODO(), namespace1.Name, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(ns1).NotTo(gomega.BeNil()) + ns2, err := fakeOvn.fakeClient.KubeClient.CoreV1().Namespaces().Get(context.TODO(), namespace2.Name, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(ns2).NotTo(gomega.BeNil()) + + portsns1 := []string{} + expectedData := getMulticastPolicyExpectedData(longNameSpace1Name, portsns1) + acl := expectedData[0].(*nbdb.ACL) + // Post ACL indexing work, multicast ACL's don't have names + // We use externalIDs instead; so we can check if the expected IDs exist for the long namespace so that + // isEquivalent logic will be correct + gomega.Expect(acl.Name).To(gomega.BeNil()) + gomega.Expect(acl.ExternalIDs[libovsdbops.ObjectNameKey.String()]).To(gomega.Equal(longNameSpace1Name)) + expectedData = append(expectedData, getMulticastPolicyExpectedData(longNameSpace2Name, nil)...) + acl = expectedData[3].(*nbdb.ACL) + gomega.Expect(acl.Name).To(gomega.BeNil()) + gomega.Expect(acl.ExternalIDs[libovsdbops.ObjectNameKey.String()]).To(gomega.Equal(longNameSpace2Name)) + expectedData = append(expectedData, getExpectedDataPodsAndSwitches([]testPod{}, []string{"node1"})...) + // Enable multicast in the namespace. + updateMulticast(fakeOvn, ns1, true) + updateMulticast(fakeOvn, ns2, true) + gomega.Eventually(fakeOvn.nbClient).Should(libovsdb.HaveData(expectedData...)) + return nil + } + + err := app.Run([]string{app.Name}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + ginkgo.It("tests adding a pod to a multicast enabled namespace "+ipModeStr(m), func() { app.Action = func(ctx *cli.Context) error { namespace1 := *newNamespace(namespaceName1) From f7469b5ef5c084c4c4193ef0a6898d518b7e9e87 Mon Sep 17 00:00:00 2001 From: Surya Seetharaman Date: Tue, 4 Apr 2023 11:52:33 +0200 Subject: [PATCH 29/40] Automatically copy yamls as j2 files This commit adds the logic to automatically copy the yamls created by update-codegen into dist/templates folder as j2 files which are ultimately used when creating the KIND cluster. Until now this step was manual and hidden. Signed-off-by: Surya Seetharaman --- docs/developer.md | 8 ++++++++ go-controller/hack/update-codegen.sh | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/docs/developer.md b/docs/developer.md index 2ec4f8a2bc..d67bd62a9d 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -30,3 +30,11 @@ NOTE2: From time to time we always bump our fedora version of OVN used by KIND. forget to update the `OVN_SCHEMA_VERSION` in our `Makefile` which is used to download the ovsdb schema. If that version seems to be outdated, probably best to update that as well and re-generate the schema bindings. + +## Generating CRD yamls using codegen + +In order to generate the latest yaml files for a given CRD or to add a new CRD, once +the `types.go` has been created according to sig-apimachinery docs, the developer can run +`make codegen` to be able to generate all the clientgen, listers and informers for the new +CRD along with the deep-copy methods and actual yaml files which get created in `_output/crd` +folder and are copied over to `dist/templates` to then be used when creating a KIND cluster. diff --git a/go-controller/hack/update-codegen.sh b/go-controller/hack/update-codegen.sh index 9ee8888b2a..476af3c535 100755 --- a/go-controller/hack/update-codegen.sh +++ b/go-controller/hack/update-codegen.sh @@ -73,3 +73,11 @@ echo "Editing EgressQoS CRD" ## We desire that only EgressQoS with the name "default" are accepted by the apiserver. sed -i -e':begin;$!N;s/.*metadata:\n.*type: object/&\n properties:\n name:\n type: string\n pattern: ^default$/;P;D' \ _output/crds/k8s.ovn.org_egressqoses.yaml + +echo "Copying the CRDs to dist/templates as j2 files... Add them to your commit..." +echo "Copying egressFirewall CRD" +cp _output/crds/k8s.ovn.org_egressfirewalls.yaml ../dist/templates/k8s.ovn.org_egressfirewalls.yaml.j2 +echo "Copying egressIP CRD" +cp _output/crds/k8s.ovn.org_egressips.yaml ../dist/templates/k8s.ovn.org_egressips.yaml.j2 +echo "Copying egressQoS CRD" +cp _output/crds/k8s.ovn.org_egressqoses.yaml ../dist/templates/k8s.ovn.org_egressqoses.yaml.j2 From 978e4d9a4635c7c9a04cf5e534f9b2c6fd5140e8 Mon Sep 17 00:00:00 2001 From: Peng Liu Date: Fri, 14 Apr 2023 22:38:06 +0800 Subject: [PATCH 30/40] Make --gateway-nexthop support dual-stack Users can specify IPV4 and IPv6 gateway nexthop for dual-stack cluster with '--gateway-nexthop=,' Signed-off-by: Peng Liu --- go-controller/pkg/node/gateway_init.go | 46 ++++++++++---- .../pkg/node/gateway_init_linux_test.go | 60 +++++++++++++++++++ go-controller/pkg/node/helper_linux.go | 8 +-- 3 files changed, 98 insertions(+), 16 deletions(-) diff --git a/go-controller/pkg/node/gateway_init.go b/go-controller/pkg/node/gateway_init.go index f94918aee4..f97d9bdaad 100644 --- a/go-controller/pkg/node/gateway_init.go +++ b/go-controller/pkg/node/gateway_init.go @@ -140,20 +140,42 @@ func getGatewayNextHops() ([]net.IP, string, error) { needIPv6NextHop = true } - // FIXME DUAL-STACK: config.Gateway.NextHop should be a slice of nexthops if config.Gateway.NextHop != "" { - // Parse NextHop to make sure it is valid before using. Return error if not valid. - nextHop := net.ParseIP(config.Gateway.NextHop) - if nextHop == nil { - return nil, "", fmt.Errorf("failed to parse configured next-hop: %s", config.Gateway.NextHop) + nextHopsRaw := strings.Split(config.Gateway.NextHop, ",") + if len(nextHopsRaw) > 2 { + return nil, "", fmt.Errorf("unexpected next-hops are provided, more than 2 next-hops is not allowed: %s", config.Gateway.NextHop) } - if config.IPv4Mode && !utilnet.IsIPv6(nextHop) { - gatewayNextHops = append(gatewayNextHops, nextHop) - needIPv4NextHop = false - } - if config.IPv6Mode && utilnet.IsIPv6(nextHop) { - gatewayNextHops = append(gatewayNextHops, nextHop) - needIPv6NextHop = false + for _, nh := range nextHopsRaw { + // Parse NextHop to make sure it is valid before using. Return error if not valid. + nextHop := net.ParseIP(nh) + if nextHop == nil { + return nil, "", fmt.Errorf("failed to parse configured next-hop: %s", config.Gateway.NextHop) + } + if config.IPv4Mode { + if needIPv4NextHop { + if !utilnet.IsIPv6(nextHop) { + gatewayNextHops = append(gatewayNextHops, nextHop) + needIPv4NextHop = false + } + } else { + if !utilnet.IsIPv6(nextHop) { + return nil, "", fmt.Errorf("only one IPv4 next-hop is allowed: %s", config.Gateway.NextHop) + } + } + } + + if config.IPv6Mode { + if needIPv6NextHop { + if utilnet.IsIPv6(nextHop) { + gatewayNextHops = append(gatewayNextHops, nextHop) + needIPv6NextHop = false + } + } else { + if utilnet.IsIPv6(nextHop) { + return nil, "", fmt.Errorf("only one IPv6 next-hop is allowed: %s", config.Gateway.NextHop) + } + } + } } } gatewayIntf := config.Gateway.Interface diff --git a/go-controller/pkg/node/gateway_init_linux_test.go b/go-controller/pkg/node/gateway_init_linux_test.go index 7b5016daba..789ce8e4f9 100644 --- a/go-controller/pkg/node/gateway_init_linux_test.go +++ b/go-controller/pkg/node/gateway_init_linux_test.go @@ -8,6 +8,7 @@ import ( "context" "fmt" "net" + "strings" "sync" "syscall" @@ -1604,4 +1605,63 @@ var _ = Describe("Gateway unit tests", func() { Expect(err).To(HaveOccurred()) }) }) + + Context("getGatewayNextHops", func() { + + It("Finds correct gateway interface and nexthops without configuration", func() { + _, ipnet, err := net.ParseCIDR("0.0.0.0/0") + Expect(err).ToNot(HaveOccurred()) + config.Kubernetes.ServiceCIDRs = []*net.IPNet{ipnet} + gwIPs := []net.IP{net.ParseIP("10.0.0.11")} + lnk := &linkMock.Link{} + lnkAttr := &netlink.LinkAttrs{ + Name: "ens1f0", + Index: 5, + } + defaultRoute := &netlink.Route{ + Dst: ipnet, + LinkIndex: 5, + Scope: netlink.SCOPE_UNIVERSE, + Gw: gwIPs[0], + MTU: config.Default.MTU, + } + lnk.On("Attrs").Return(lnkAttr) + netlinkMock.On("LinkByName", mock.Anything).Return(lnk, nil) + netlinkMock.On("LinkByIndex", mock.Anything).Return(lnk, nil) + netlinkMock.On("RouteListFiltered", mock.Anything, mock.Anything, mock.Anything).Return([]netlink.Route{*defaultRoute}, nil) + gatewayNextHops, gatewayIntf, err := getGatewayNextHops() + Expect(err).NotTo(HaveOccurred()) + Expect(gatewayIntf).To(Equal(lnkAttr.Name)) + Expect(gatewayNextHops[0]).To(Equal(gwIPs[0])) + }) + + It("Finds correct gateway interface and nexthops with single stack configuration", func() { + ifName := "enf1f0" + nextHopCfg := "10.0.0.11" + gwIPs := []net.IP{net.ParseIP(nextHopCfg)} + config.Gateway.Interface = ifName + config.Gateway.NextHop = nextHopCfg + + gatewayNextHops, gatewayIntf, err := getGatewayNextHops() + Expect(err).NotTo(HaveOccurred()) + Expect(gatewayIntf).To(Equal(ifName)) + Expect(gatewayNextHops[0]).To(Equal(gwIPs[0])) + }) + + It("Finds correct gateway interface and nexthops with dual stack configuration", func() { + ifName := "enf1f0" + nextHopCfg := "10.0.0.11,fc00:f853:ccd:e793::1" + nextHops := strings.Split(nextHopCfg, ",") + gwIPs := []net.IP{net.ParseIP(nextHops[0]), net.ParseIP(nextHops[1])} + config.Gateway.Interface = ifName + config.Gateway.NextHop = nextHopCfg + config.IPv4Mode = true + config.IPv6Mode = true + + gatewayNextHops, gatewayIntf, err := getGatewayNextHops() + Expect(err).NotTo(HaveOccurred()) + Expect(gatewayIntf).To(Equal(ifName)) + Expect(gatewayNextHops).To(Equal(gwIPs)) + }) + }) }) diff --git a/go-controller/pkg/node/helper_linux.go b/go-controller/pkg/node/helper_linux.go index dc9497765e..0b3282db1b 100644 --- a/go-controller/pkg/node/helper_linux.go +++ b/go-controller/pkg/node/helper_linux.go @@ -58,7 +58,7 @@ func getDefaultGatewayInterfaceByFamily(family int, gwIface string) (string, net gwIfIdx := 0 // gw interface provided if len(gwIface) > 0 { - link, err := netlink.LinkByName(gwIface) + link, err := util.GetNetLinkOps().LinkByName(gwIface) if err != nil { return "", nil, fmt.Errorf("error looking up gw interface: %q, error: %w", gwIface, err) } @@ -69,7 +69,7 @@ func getDefaultGatewayInterfaceByFamily(family int, gwIface string) (string, net klog.Infof("Provided gateway interface %q, found as index: %d", gwIface, gwIfIdx) } - routeList, err := netlink.RouteListFiltered(family, filter, mask) + routeList, err := util.GetNetLinkOps().RouteListFiltered(family, filter, mask) if err != nil { return "", nil, errors.Wrapf(err, "failed to get routing table in node") } @@ -78,7 +78,7 @@ func getDefaultGatewayInterfaceByFamily(family int, gwIface string) (string, net for _, r := range routes { // no multipath if len(r.MultiPath) == 0 { - intfLink, err := netlink.LinkByIndex(r.LinkIndex) + intfLink, err := util.GetNetLinkOps().LinkByIndex(r.LinkIndex) if err != nil { klog.Warningf("Failed to get interface link for route %v : %v", r, err) continue @@ -104,7 +104,7 @@ func getDefaultGatewayInterfaceByFamily(family int, gwIface string) (string, net // TODO: revisit for full multipath support // xref: https://github.com/vishvananda/netlink/blob/6ffafa9fc19b848776f4fd608c4ad09509aaacb4/route.go#L137-L145 for _, nh := range r.MultiPath { - intfLink, err := netlink.LinkByIndex(nh.LinkIndex) + intfLink, err := util.GetNetLinkOps().LinkByIndex(nh.LinkIndex) if err != nil { klog.Warningf("Failed to get interface link for route %v : %v", nh, err) continue From 09f75485c0e39405ababc46f52873285933882ff Mon Sep 17 00:00:00 2001 From: Andrea Panattoni Date: Thu, 13 Apr 2023 18:14:29 +0200 Subject: [PATCH 31/40] Improve e2e test assertion message Signed-off-by: Andrea Panattoni --- test/e2e/e2e.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index 6d5afe77d9..6a9bfa68ec 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -1836,7 +1836,8 @@ var _ = ginkgo.Describe("e2e ingress traffic validation", func() { break } } - framework.ExpectEqual(valid, true, "Validation failed for node", node, responses, nodePort) + framework.ExpectEqual(valid, true, + fmt.Sprintf("Validation failed for node %s. Expected Responses=%v, Actual Responses=%v", node.Name, nodesHostnames, responses)) } } } @@ -1988,7 +1989,8 @@ var _ = ginkgo.Describe("e2e ingress traffic validation", func() { break } } - framework.ExpectEqual(valid, true, "Validation failed for node", nodeName, responses, port) + framework.ExpectEqual(valid, true, + fmt.Sprintf("Validation failed for node %s. Expected Responses=%v, Actual Responses=%v", nodeName, nodesHostnames, responses)) } } } @@ -2058,7 +2060,8 @@ var _ = ginkgo.Describe("e2e ingress traffic validation", func() { } } - framework.ExpectEqual(valid, true, "Validation failed for node", node.Name, responses, nodePort) + framework.ExpectEqual(valid, true, + fmt.Sprintf("Validation failed for node %s. Expected Responses=%v, Actual Responses=%v", node.Name, expectedResponses, responses)) } } } @@ -2118,7 +2121,7 @@ var _ = ginkgo.Describe("e2e ingress traffic validation", func() { protocol, externalPort)) valid = pokeExternalIpService(clientContainerName, protocol, externalAddress, externalPort, maxTries, nodesHostnames) - framework.ExpectEqual(valid, true, "Validation failed for external address", externalAddress) + framework.ExpectEqual(valid, true, "Validation failed for external address: %s", externalAddress) } } @@ -2143,7 +2146,7 @@ var _ = ginkgo.Describe("e2e ingress traffic validation", func() { protocol, externalPort)) valid = pokeExternalIpService(clientContainerName, protocol, externalAddress, externalPort, maxTries, nodesHostnames) - framework.ExpectEqual(valid, true, "Validation failed for external address", externalAddress) + framework.ExpectEqual(valid, true, "Validation failed for external address: %s", externalAddress) } } }) @@ -2259,7 +2262,7 @@ var _ = ginkgo.Describe("e2e ingress traffic validation", func() { break } } - framework.ExpectEqual(valid, true, "Validation failed for external address", externalAddress) + framework.ExpectEqual(valid, true, "Validation failed for external address: %s", externalAddress) } } }) @@ -2397,7 +2400,8 @@ var _ = ginkgo.Describe("e2e ingress to host-networked pods traffic validation", } } - framework.ExpectEqual(valid, true, "Validation failed for node", node.Name, responses, nodePort) + framework.ExpectEqual(valid, true, + fmt.Sprintf("Validation failed for node %s. Expected Responses=%v, Actual Responses=%v", node.Name, expectedResponses, responses)) } } } From dde7ea84ccc90447c20309dc9d1d63a77c2f497a Mon Sep 17 00:00:00 2001 From: Andrea Panattoni Date: Tue, 20 Dec 2022 15:59:19 +0100 Subject: [PATCH 32/40] Serve NodePort services on every `host-addresses` When a node gets a new IP addresses, node loadbalancers should be updated to serve NodePort services on the new address. Update node_tracker.go to track both `host-addresses` and `l3-gateway-config` IP addresses, as they are used in different ways for building load balancers. Add End2End test to control-plane suite. Update load balancer unit tests to include a multi address node. Signed-off-by: Andrea Panattoni --- .../pkg/ovn/controller/services/lb_config.go | 8 +- .../ovn/controller/services/lb_config_test.go | 100 ++++++++++++++---- .../ovn/controller/services/node_tracker.go | 74 +++++++++---- .../services/services_controller.go | 4 +- .../services/services_controller_test.go | 13 +-- go-controller/pkg/util/node_annotations.go | 5 + test/e2e/e2e.go | 66 +++++++++++- 7 files changed, 216 insertions(+), 54 deletions(-) diff --git a/go-controller/pkg/ovn/controller/services/lb_config.go b/go-controller/pkg/ovn/controller/services/lb_config.go index 6a7633919f..5a0c16451d 100644 --- a/go-controller/pkg/ovn/controller/services/lb_config.go +++ b/go-controller/pkg/ovn/controller/services/lb_config.go @@ -74,7 +74,7 @@ func (c *lbConfig) makeNodeRouterTargetIPs(node *nodeInfo, epIPs []string, hostM // any targets local to the node need to have a special // harpin IP added, but only for the router LB - targetIPs, updated := util.UpdateIPsSlice(targetIPs, node.nodeIPsStr(), []string{hostMasqueradeIP}) + targetIPs, updated := util.UpdateIPsSlice(targetIPs, node.l3gatewayAddressesStr(), []string{hostMasqueradeIP}) // We either only removed stuff from the original slice, or updated some IPs. if len(targetIPs) != len(epIPs) || updated { @@ -108,8 +108,8 @@ var protos = []v1.Protocol{ // - services with InternalTrafficPolicy=Local // // Template LBs will be created for -// - services with NodePort set but *without* ExternalTrafficPolicy=Local or -// affinity timeout set. +// - services with NodePort set but *without* ExternalTrafficPolicy=Local or +// affinity timeout set. func buildServiceLBConfigs(service *v1.Service, endpointSlices []*discovery.EndpointSlice, useLBGroup, useTemplates bool) (perNodeConfigs, templateConfigs, clusterConfigs []lbConfig) { needsAffinityTimeout := hasSessionAffinityTimeOut(service) @@ -573,7 +573,7 @@ func buildPerNodeLBs(service *v1.Service, configs []lbConfig, nodes []nodeInfo) vips := make([]string, 0, len(config.vips)) for _, vip := range config.vips { if vip == placeholderNodeIPs { - vips = append(vips, node.nodeIPsStr()...) + vips = append(vips, node.hostAddressesStr()...) } else { vips = append(vips, vip) } diff --git a/go-controller/pkg/ovn/controller/services/lb_config_test.go b/go-controller/pkg/ovn/controller/services/lb_config_test.go index 3df9846b4c..37166956ea 100644 --- a/go-controller/pkg/ovn/controller/services/lb_config_test.go +++ b/go-controller/pkg/ovn/controller/services/lb_config_test.go @@ -750,16 +750,18 @@ func Test_buildClusterLBs(t *testing.T) { defaultNodes := []nodeInfo{ { - name: "node-a", - nodeIPs: []net.IP{net.ParseIP("10.0.0.1")}, - gatewayRouterName: "gr-node-a", - switchName: "switch-node-a", + name: "node-a", + l3gatewayAddresses: []net.IP{net.ParseIP("10.0.0.1")}, + hostAddresses: []net.IP{net.ParseIP("10.0.0.1")}, + gatewayRouterName: "gr-node-a", + switchName: "switch-node-a", }, { - name: "node-b", - nodeIPs: []net.IP{net.ParseIP("10.0.0.2")}, - gatewayRouterName: "gr-node-b", - switchName: "switch-node-b", + name: "node-b", + l3gatewayAddresses: []net.IP{net.ParseIP("10.0.0.2")}, + hostAddresses: []net.IP{net.ParseIP("10.0.0.2")}, + gatewayRouterName: "gr-node-b", + switchName: "switch-node-b", }, } @@ -978,18 +980,20 @@ func Test_buildPerNodeLBs(t *testing.T) { defaultNodes := []nodeInfo{ { - name: "node-a", - nodeIPs: []net.IP{net.ParseIP("10.0.0.1")}, - gatewayRouterName: "gr-node-a", - switchName: "switch-node-a", - podSubnets: []net.IPNet{{IP: net.ParseIP("10.128.0.0"), Mask: net.CIDRMask(24, 32)}}, + name: "node-a", + l3gatewayAddresses: []net.IP{net.ParseIP("10.0.0.1")}, + hostAddresses: []net.IP{net.ParseIP("10.0.0.1"), net.ParseIP("10.0.0.111")}, + gatewayRouterName: "gr-node-a", + switchName: "switch-node-a", + podSubnets: []net.IPNet{{IP: net.ParseIP("10.128.0.0"), Mask: net.CIDRMask(24, 32)}}, }, { - name: "node-b", - nodeIPs: []net.IP{net.ParseIP("10.0.0.2")}, - gatewayRouterName: "gr-node-b", - switchName: "switch-node-b", - podSubnets: []net.IPNet{{IP: net.ParseIP("10.128.1.0"), Mask: net.CIDRMask(24, 32)}}, + name: "node-b", + l3gatewayAddresses: []net.IP{net.ParseIP("10.0.0.2")}, + hostAddresses: []net.IP{net.ParseIP("10.0.0.2")}, + gatewayRouterName: "gr-node-b", + switchName: "switch-node-b", + podSubnets: []net.IPNet{{IP: net.ParseIP("10.128.1.0"), Mask: net.CIDRMask(24, 32)}}, }, } @@ -1079,6 +1083,10 @@ func Test_buildPerNodeLBs(t *testing.T) { Source: Addr{IP: "10.0.0.1", Port: 80}, Targets: []Addr{{IP: "10.128.0.2", Port: 8080}}, }, + { + Source: Addr{IP: "10.0.0.111", Port: 80}, + Targets: []Addr{{IP: "10.128.0.2", Port: 8080}}, + }, }, Opts: defaultOpts, }, @@ -1109,6 +1117,10 @@ func Test_buildPerNodeLBs(t *testing.T) { Source: Addr{IP: "10.0.0.1", Port: 80}, Targets: []Addr{{IP: "10.128.0.2", Port: 8080}}, }, + { + Source: Addr{IP: "10.0.0.111", Port: 80}, + Targets: []Addr{{IP: "10.128.0.2", Port: 8080}}, + }, }, Opts: defaultOpts, }, @@ -1166,6 +1178,10 @@ func Test_buildPerNodeLBs(t *testing.T) { Source: Addr{IP: "10.0.0.1", Port: 80}, Targets: []Addr{{IP: "169.254.169.2", Port: 8080}}, }, + { + Source: Addr{IP: "10.0.0.111", Port: 80}, + Targets: []Addr{{IP: "169.254.169.2", Port: 8080}}, + }, }, Opts: defaultOpts, }, @@ -1183,6 +1199,10 @@ func Test_buildPerNodeLBs(t *testing.T) { Source: Addr{IP: "10.0.0.1", Port: 80}, Targets: []Addr{{IP: "10.0.0.1", Port: 8080}}, }, + { + Source: Addr{IP: "10.0.0.111", Port: 80}, + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}}, + }, }, Opts: defaultOpts, }, @@ -1220,6 +1240,10 @@ func Test_buildPerNodeLBs(t *testing.T) { Source: Addr{IP: "10.0.0.1", Port: 80}, Targets: []Addr{{IP: "169.254.169.2", Port: 8080}}, }, + { + Source: Addr{IP: "10.0.0.111", Port: 80}, + Targets: []Addr{{IP: "169.254.169.2", Port: 8080}}, + }, }, Opts: defaultOpts, }, @@ -1237,6 +1261,10 @@ func Test_buildPerNodeLBs(t *testing.T) { Source: Addr{IP: "10.0.0.1", Port: 80}, Targets: []Addr{{IP: "10.0.0.1", Port: 8080}}, }, + { + Source: Addr{IP: "10.0.0.111", Port: 80}, + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}}, + }, }, Opts: defaultOpts, }, @@ -1315,6 +1343,10 @@ func Test_buildPerNodeLBs(t *testing.T) { Source: Addr{IP: "10.0.0.1", Port: 80}, Targets: []Addr{{IP: "169.254.169.2", Port: 8080}}, }, + { + Source: Addr{IP: "10.0.0.111", Port: 80}, + Targets: []Addr{{IP: "169.254.169.2", Port: 8080}}, + }, }, }, { @@ -1335,6 +1367,14 @@ func Test_buildPerNodeLBs(t *testing.T) { Source: Addr{IP: "10.0.0.1", Port: 80}, Targets: []Addr{{IP: "10.0.0.1", Port: 8080}}, }, + { + Source: Addr{IP: "169.254.169.3", Port: 80}, + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}}, + }, + { + Source: Addr{IP: "10.0.0.111", Port: 80}, + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}}, + }, }, Opts: defaultOpts, }, @@ -1733,6 +1773,10 @@ func Test_buildPerNodeLBs(t *testing.T) { Source: Addr{IP: "10.0.0.1", Port: 34345}, Targets: []Addr{{IP: "169.254.169.2", Port: 8080}}, // special skip_snat=true LB for ETP=local; used in SGW mode }, + { + Source: Addr{IP: "10.0.0.111", Port: 34345}, + Targets: []Addr{{IP: "169.254.169.2", Port: 8080}}, + }, }, }, { @@ -1751,6 +1795,14 @@ func Test_buildPerNodeLBs(t *testing.T) { }, { Source: Addr{IP: "10.0.0.1", Port: 34345}, + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}}, + }, + { + Source: Addr{IP: "169.254.169.3", Port: 34345}, + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}}, + }, + { + Source: Addr{IP: "10.0.0.111", Port: 34345}, Targets: []Addr{{IP: "10.0.0.1", Port: 8080}}, // don't filter out eps for nodePorts on switches when ETP=local }, }, @@ -1820,6 +1872,10 @@ func Test_buildPerNodeLBs(t *testing.T) { Source: Addr{IP: "10.0.0.1", Port: 34345}, Targets: []Addr{{IP: "169.254.169.2", Port: 8080}}, // special skip_snat=true LB for ETP=local; used in SGW mode }, + { + Source: Addr{IP: "10.0.0.111", Port: 34345}, + Targets: []Addr{{IP: "169.254.169.2", Port: 8080}}, + }, }, }, { @@ -1838,6 +1894,14 @@ func Test_buildPerNodeLBs(t *testing.T) { }, { Source: Addr{IP: "10.0.0.1", Port: 34345}, + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}}, + }, + { + Source: Addr{IP: "169.254.169.3", Port: 34345}, + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}}, + }, + { + Source: Addr{IP: "10.0.0.111", Port: 34345}, Targets: []Addr{{IP: "10.0.0.1", Port: 8080}}, // don't filter out eps for nodePorts on switches when ETP=local }, }, diff --git a/go-controller/pkg/ovn/controller/services/node_tracker.go b/go-controller/pkg/ovn/controller/services/node_tracker.go index bae71b6f54..4fbb62966a 100644 --- a/go-controller/pkg/ovn/controller/services/node_tracker.go +++ b/go-controller/pkg/ovn/controller/services/node_tracker.go @@ -12,6 +12,7 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/sets" coreinformers "k8s.io/client-go/informers/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/klog/v2" @@ -34,8 +35,10 @@ type nodeTracker struct { type nodeInfo struct { // the node's Name name string - // The list of physical IPs the node has, as reported by the gatewayconf annotation - nodeIPs []net.IP + // The list of physical IPs reported by the gatewayconf annotation + l3gatewayAddresses []net.IP + // The list of physical IPs the node has, as reported by the host-address annotation + hostAddresses []net.IP // The pod network subnet(s) podSubnets []net.IPNet // the name of the node's GatewayRouter, or "" of non-existent @@ -46,10 +49,18 @@ type nodeInfo struct { chassisID string } -func (ni *nodeInfo) nodeIPsStr() []string { - out := make([]string, 0, len(ni.nodeIPs)) - for _, nodeIP := range ni.nodeIPs { - out = append(out, nodeIP.String()) +func (ni *nodeInfo) hostAddressesStr() []string { + out := make([]string, 0, len(ni.hostAddresses)) + for _, ip := range ni.hostAddresses { + out = append(out, ip.String()) + } + return out +} + +func (ni *nodeInfo) l3gatewayAddressesStr() []string { + out := make([]string, 0, len(ni.l3gatewayAddresses)) + for _, ip := range ni.l3gatewayAddresses { + out = append(out, ip.String()) } return out } @@ -58,7 +69,7 @@ func (ni *nodeInfo) nodeIPsStr() []string { // includes node IPs, still as a mask-1 net func (ni *nodeInfo) nodeSubnets() []net.IPNet { out := append([]net.IPNet{}, ni.podSubnets...) - for _, ip := range ni.nodeIPs { + for _, ip := range ni.hostAddresses { if ipv4 := ip.To4(); ipv4 != nil { out = append(out, net.IPNet{ IP: ip, @@ -102,11 +113,16 @@ func newNodeTracker(nodeInformer coreinformers.NodeInformer) (*nodeTracker, erro return } - // updateNode needs to be called only when hostSubnet annotation has changed or - // if L3Gateway annotation's ip addresses have changed or the name of the node (very rare) - // has changed. No need to trigger update for any other field change. - if util.NodeSubnetAnnotationChanged(oldObj, newObj) || util.NodeL3GatewayAnnotationChanged(oldObj, newObj) || - util.NodeChassisIDAnnotationChanged(oldObj, newObj) || oldObj.Name != newObj.Name { + // updateNode needs to be called in the following cases: + // - hostSubnet annotation has changed + // - L3Gateway annotation's ip addresses have changed + // - the name of the node (very rare) has changed + // - the `host-addresses` annotation changed + // . No need to trigger update for any other field change. + if util.NodeSubnetAnnotationChanged(oldObj, newObj) || + util.NodeL3GatewayAnnotationChanged(oldObj, newObj) || + oldObj.Name != newObj.Name || + util.NodeHostAddressesAnnotationChanged(oldObj, newObj) { nt.updateNode(newObj) } }, @@ -136,14 +152,15 @@ func newNodeTracker(nodeInformer coreinformers.NodeInformer) (*nodeTracker, erro // updateNodeInfo updates the node info cache, and syncs all services // if it changed. -func (nt *nodeTracker) updateNodeInfo(nodeName, switchName, routerName, chassisID string, nodeIPs []net.IP, podSubnets []*net.IPNet) { +func (nt *nodeTracker) updateNodeInfo(nodeName, switchName, routerName, chassisID string, l3gatewayAddresses, hostAddresses []net.IP, podSubnets []*net.IPNet) { ni := nodeInfo{ - name: nodeName, - nodeIPs: nodeIPs, - podSubnets: make([]net.IPNet, 0, len(podSubnets)), - gatewayRouterName: routerName, - switchName: switchName, - chassisID: chassisID, + name: nodeName, + l3gatewayAddresses: l3gatewayAddresses, + hostAddresses: hostAddresses, + podSubnets: make([]net.IPNet, 0, len(podSubnets)), + gatewayRouterName: routerName, + switchName: switchName, + chassisID: chassisID, } for i := range podSubnets { ni.podSubnets = append(ni.podSubnets, *podSubnets[i]) // de-pointer @@ -199,7 +216,7 @@ func (nt *nodeTracker) updateNode(node *v1.Node) { switchName := node.Name grName := "" - ips := []net.IP{} + l3gatewayAddresses := []net.IP{} chassisID := "" // if the node has a gateway config, it will soon have a gateway router @@ -211,18 +228,31 @@ func (nt *nodeTracker) updateNode(node *v1.Node) { grName = util.GetGatewayRouterFromNode(node.Name) if gwConf.NodePortEnable { for _, ip := range gwConf.IPAddresses { - ips = append(ips, ip.IP) + l3gatewayAddresses = append(l3gatewayAddresses, ip.IP) } } chassisID = gwConf.ChassisID } + hostAddresses, err := util.ParseNodeHostAddresses(node) + if err != nil { + klog.Warningf("Failed to get node host addresses for [%s]: %s", node.Name, err.Error()) + hostAddresses = sets.New[string]() + } + + hostAddressesIPs := make([]net.IP, 0, len(hostAddresses)) + for _, ipStr := range hostAddresses.UnsortedList() { + ip := net.ParseIP(ipStr) + hostAddressesIPs = append(hostAddressesIPs, ip) + } + nt.updateNodeInfo( node.Name, switchName, grName, chassisID, - ips, + l3gatewayAddresses, + hostAddressesIPs, hsn, ) } diff --git a/go-controller/pkg/ovn/controller/services/services_controller.go b/go-controller/pkg/ovn/controller/services/services_controller.go index f6d94dafd0..81c2e5e73c 100644 --- a/go-controller/pkg/ovn/controller/services/services_controller.go +++ b/go-controller/pkg/ovn/controller/services/services_controller.go @@ -459,12 +459,12 @@ func (c *Controller) syncNodeInfos(nodeInfos []nodeInfo) { // Services are currently supported only on the node's first IP. // Extract that one and populate the node's IP template value. if globalconfig.IPv4Mode { - if ipv4, err := util.MatchFirstIPFamily(false, node.nodeIPs); err == nil { + if ipv4, err := util.MatchFirstIPFamily(false, node.l3gatewayAddresses); err == nil { c.nodeIPv4Template.Value[node.chassisID] = ipv4.String() } } if globalconfig.IPv6Mode { - if ipv6, err := util.MatchFirstIPFamily(true, node.nodeIPs); err == nil { + if ipv6, err := util.MatchFirstIPFamily(true, node.l3gatewayAddresses); err == nil { c.nodeIPv6Template.Value[node.chassisID] = ipv6.String() } } diff --git a/go-controller/pkg/ovn/controller/services/services_controller_test.go b/go-controller/pkg/ovn/controller/services/services_controller_test.go index c3b50d8a2d..cc0ed13092 100644 --- a/go-controller/pkg/ovn/controller/services/services_controller_test.go +++ b/go-controller/pkg/ovn/controller/services/services_controller_test.go @@ -635,7 +635,7 @@ func nodeIPTemplate(node *nodeInfo) *nbdb.ChassisTemplateVar { UUID: node.chassisID, Chassis: node.chassisID, Variables: map[string]string{ - makeLBNodeIPTemplateName(v1.IPv4Protocol): node.nodeIPs[0].String(), + makeLBNodeIPTemplateName(v1.IPv4Protocol): node.hostAddresses[0].String(), }, } } @@ -679,11 +679,12 @@ func endpoint(ip string, port int32) string { func nodeConfig(nodeName string, nodeIP string) *nodeInfo { return &nodeInfo{ - name: nodeName, - nodeIPs: []net.IP{net.ParseIP(nodeIP)}, - gatewayRouterName: nodeGWRouterName(nodeName), - switchName: nodeSwitchName(nodeName), - chassisID: nodeName, + name: nodeName, + l3gatewayAddresses: []net.IP{net.ParseIP(nodeIP)}, + hostAddresses: []net.IP{net.ParseIP(nodeIP)}, + gatewayRouterName: nodeGWRouterName(nodeName), + switchName: nodeSwitchName(nodeName), + chassisID: nodeName, } } diff --git a/go-controller/pkg/util/node_annotations.go b/go-controller/pkg/util/node_annotations.go index ce6b2d6426..201c6cb656 100644 --- a/go-controller/pkg/util/node_annotations.go +++ b/go-controller/pkg/util/node_annotations.go @@ -8,6 +8,7 @@ import ( "strconv" kapi "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" @@ -540,6 +541,10 @@ func SetNodeHostAddresses(nodeAnnotator kube.Annotator, addresses sets.Set[strin return nodeAnnotator.Set(ovnNodeHostAddresses, addresses.UnsortedList()) } +func NodeHostAddressesAnnotationChanged(oldNode, newNode *v1.Node) bool { + return oldNode.Annotations[ovnNodeHostAddresses] != newNode.Annotations[ovnNodeHostAddresses] +} + // ParseNodeHostAddresses returns the parsed host addresses living on a node func ParseNodeHostAddresses(node *kapi.Node) (sets.Set[string], error) { addrAnnotation, ok := node.Annotations[ovnNodeHostAddresses] diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index 6a9bfa68ec..37d112df24 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -2151,7 +2151,8 @@ var _ = ginkgo.Describe("e2e ingress traffic validation", func() { } }) }) - ginkgo.Context("Validating ExternalIP ingress traffic to manually added node IPs", func() { + + ginkgo.Context("Validating ingress traffic to manually added node IPs", func() { ginkgo.BeforeEach(func() { endPoints = make([]*v1.Pod, 0) nodesHostnames = sets.NewString() @@ -2191,14 +2192,18 @@ var _ = ginkgo.Describe("e2e ingress traffic validation", func() { // addresses. createClusterExternalContainer(clientContainerName, agnhostImage, []string{"--network", "kind", "-P"}, []string{"netexec", "--http-port=80"}) + // If `kindexgw` exists, connect client container to it + runCommand(containerRuntime, "network", "connect", "kindexgw", clientContainerName) + ginkgo.By("Adding ip addresses to each node") // add new secondary IP from node subnet to all nodes, if the cluster is v6 add an ipv6 address var newIP string + newNodeAddresses = make([]string, 0) for i, node := range nodes.Items { if utilnet.IsIPv6String(e2enode.GetAddresses(&node, v1.NodeInternalIP)[0]) { newIP = "fc00:f853:ccd:e794::" + strconv.Itoa(i) } else { - newIP = "172.18.1." + strconv.Itoa(i) + newIP = "172.18.1." + strconv.Itoa(i+1) } // manually add the a secondary IP to each node _, err := runCommand(containerRuntime, "exec", node.Name, "ip", "addr", "add", newIP, "dev", "breth0") @@ -2221,6 +2226,7 @@ var _ = ginkgo.Describe("e2e ingress traffic validation", func() { } } }) + // This test validates ingress traffic to externalservices after a new node Ip is added. // It creates a service on both udp and tcp and assigns the new node IPs as // external Addresses. Then, creates a backend pod on each node. @@ -2266,6 +2272,62 @@ var _ = ginkgo.Describe("e2e ingress traffic validation", func() { } } }) + + // This test verifies a NodePort service is reachable on manually added IP addresses. + ginkgo.It("for NodePort services", func() { + isIPv6Cluster := IsIPv6Cluster(f.ClientSet) + serviceName := "nodeportservice" + + ginkgo.By("Creating NodePort service") + svcSpec := nodePortServiceSpecFrom(serviceName, v1.IPFamilyPolicyPreferDualStack, endpointHTTPPort, endpointUDPPort, clusterHTTPPort, clusterUDPPort, endpointsSelector, v1.ServiceExternalTrafficPolicyTypeLocal) + svcSpec, err := f.ClientSet.CoreV1().Services(f.Namespace.Name).Create(context.Background(), svcSpec, metav1.CreateOptions{}) + framework.ExpectNoError(err) + + ginkgo.By("Waiting for the endpoints to pop up") + err = framework.WaitForServiceEndpointsNum(f.ClientSet, f.Namespace.Name, serviceName, len(endPoints), time.Second, wait.ForeverTestTimeout) + framework.ExpectNoError(err, "failed to validate endpoints for service %s in namespace: %s", serviceName, f.Namespace.Name) + + tcpNodePort, udpNodePort := nodePortsFromService(svcSpec) + + toCheckNodesAddresses := sets.NewString() + for _, node := range nodes.Items { + + addrAnnotation, ok := node.Annotations["k8s.ovn.org/host-addresses"] + gomega.Expect(ok).To(gomega.BeTrue()) + + var addrs []string + err := json.Unmarshal([]byte(addrAnnotation), &addrs) + framework.ExpectNoError(err, "failed to parse node[%s] host-address annotation[%s]", node.Name, addrAnnotation) + + toCheckNodesAddresses.Insert(addrs...) + } + + // Ensure newly added IP address are in the host-addresses annotation + for _, newAddress := range newNodeAddresses { + if !toCheckNodesAddresses.Has(newAddress) { + toCheckNodesAddresses.Insert(newAddress) + } + } + + for _, protocol := range []string{"http", "udp"} { + toCurlPort := int32(tcpNodePort) + if protocol == "udp" { + toCurlPort = int32(udpNodePort) + } + + for _, address := range toCheckNodesAddresses.List() { + if !isIPv6Cluster && utilnet.IsIPv6String(address) { + continue + } + ginkgo.By("Hitting the service on " + address + " via " + protocol) + gomega.Eventually(func() bool { + epHostname := pokeEndpoint("", clientContainerName, protocol, address, toCurlPort, "hostname") + // Expect to receive a valid hostname + return nodesHostnames.Has(epHostname) + }, "20s", "1s").Should(gomega.BeTrue()) + } + } + }) }) }) From 2adc58f6a0252467cdacb234043448f4ca3853e7 Mon Sep 17 00:00:00 2001 From: Ben Pickard Date: Wed, 12 Apr 2023 15:31:17 -0400 Subject: [PATCH 33/40] Convert unstortedList() to sets.List() During the kube 1.26 bump the sets[type].List() call was deprecated and replaced with unsortedList(). This could cause some flakes as we depend on the order of the set in some cases. This fix updates the unsortedList calls to use the sets List function Signed-off-by: Ben Pickard s s --- go-controller/pkg/node/node_ip_handler_linux.go | 2 +- go-controller/pkg/ovn/address_set/address_set.go | 2 +- go-controller/pkg/ovn/egressgw.go | 2 +- go-controller/pkg/ovn/gateway_init.go | 2 +- go-controller/pkg/ovn/master.go | 2 +- go-controller/pkg/ovn/namespace.go | 4 ++-- go-controller/pkg/util/node_annotations.go | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/go-controller/pkg/node/node_ip_handler_linux.go b/go-controller/pkg/node/node_ip_handler_linux.go index d26c1450cf..fa5d645f49 100644 --- a/go-controller/pkg/node/node_ip_handler_linux.go +++ b/go-controller/pkg/node/node_ip_handler_linux.go @@ -93,7 +93,7 @@ func (c *addressManager) delAddr(ip net.IP) bool { func (c *addressManager) ListAddresses() []net.IP { c.Lock() defer c.Unlock() - addrs := c.addresses.UnsortedList() + addrs := sets.List(c.addresses) out := make([]net.IP, 0, len(addrs)) for _, addr := range addrs { ip := net.ParseIP(addr) diff --git a/go-controller/pkg/ovn/address_set/address_set.go b/go-controller/pkg/ovn/address_set/address_set.go index 61a2c08ef9..52069ba088 100644 --- a/go-controller/pkg/ovn/address_set/address_set.go +++ b/go-controller/pkg/ovn/address_set/address_set.go @@ -613,7 +613,7 @@ func splitIPsByFamily(ips []net.IP) (v4 []net.IP, v6 []net.IP) { // Takes a slice of IPs and returns a slice with unique IPs func ipsToStringUnique(ips []net.IP) []string { - s := sets.NewString() + s := sets.New[string]() for _, ip := range ips { s.Insert(ip.String()) } diff --git a/go-controller/pkg/ovn/egressgw.go b/go-controller/pkg/ovn/egressgw.go index d3014fc6c7..c63735d56b 100644 --- a/go-controller/pkg/ovn/egressgw.go +++ b/go-controller/pkg/ovn/egressgw.go @@ -409,7 +409,7 @@ func (oc *DefaultNetworkController) deletePodGWRoutesForNamespace(pod *kapi.Pod, return fmt.Errorf("failed to delete GW routes for pod %s: %w", pod.Name, err) } // remove the exgw podIP from the namespace's k8s.ovn.org/external-gw-pod-ips list - if err := util.UpdateExternalGatewayPodIPsAnnotation(oc.kube, namespace, existingGWs.UnsortedList()); err != nil { + if err := util.UpdateExternalGatewayPodIPsAnnotation(oc.kube, namespace, sets.List(existingGWs)); err != nil { klog.Errorf("Unable to update %s/%v annotation for namespace %s: %v", util.ExternalGatewayPodIPsAnnotation, existingGWs, namespace, err) } return nil diff --git a/go-controller/pkg/ovn/gateway_init.go b/go-controller/pkg/ovn/gateway_init.go index 35f90da6b8..7ad8607ac2 100644 --- a/go-controller/pkg/ovn/gateway_init.go +++ b/go-controller/pkg/ovn/gateway_init.go @@ -582,7 +582,7 @@ func (oc *DefaultNetworkController) addPolicyBasedRoutes(nodeName, mgmtPortIP st // N+2 is fully made. func (oc *DefaultNetworkController) syncPolicyBasedRoutes(nodeName string, matches sets.Set[string], priority, nexthop string) error { // create a map to track matches found - matchTracker := sets.NewString(matches.UnsortedList()...) + matchTracker := sets.New(sets.List(matches)...) if priority == types.NodeSubnetPolicyPriority { policies, err := oc.findPolicyBasedRoutes(priority) diff --git a/go-controller/pkg/ovn/master.go b/go-controller/pkg/ovn/master.go index 065cfac974..56c9746727 100644 --- a/go-controller/pkg/ovn/master.go +++ b/go-controller/pkg/ovn/master.go @@ -341,7 +341,7 @@ func (oc *DefaultNetworkController) syncGatewayLogicalNetwork(node *kapi.Node, l if err != nil { return err } - relevantHostIPs, err := util.MatchAllIPStringFamily(utilnet.IsIPv6(hostIfAddr.IP), hostAddrs.UnsortedList()) + relevantHostIPs, err := util.MatchAllIPStringFamily(utilnet.IsIPv6(hostIfAddr.IP), sets.List(hostAddrs)) if err != nil && err != util.NoIPError { return err } diff --git a/go-controller/pkg/ovn/namespace.go b/go-controller/pkg/ovn/namespace.go index df7b1d4d18..b4e3fa344c 100644 --- a/go-controller/pkg/ovn/namespace.go +++ b/go-controller/pkg/ovn/namespace.go @@ -104,7 +104,7 @@ func (oc *DefaultNetworkController) getRoutingExternalGWs(nsInfo *namespaceInfo) // return a copy of the object so it can be handled without the // namespace locked res.bfdEnabled = nsInfo.routingExternalGWs.bfdEnabled - res.gws = sets.New[string](nsInfo.routingExternalGWs.gws.UnsortedList()...) + res.gws = sets.New(nsInfo.routingExternalGWs.gws.UnsortedList()...) return &res } @@ -131,7 +131,7 @@ func (oc *DefaultNetworkController) getRoutingPodGWs(nsInfo *namespaceInfo) map[ for k, v := range nsInfo.routingExternalPodGWs { item := gatewayInfo{ bfdEnabled: v.bfdEnabled, - gws: sets.New[string](v.gws.UnsortedList()...), + gws: sets.New(v.gws.UnsortedList()...), } res[k] = item } diff --git a/go-controller/pkg/util/node_annotations.go b/go-controller/pkg/util/node_annotations.go index ce6b2d6426..bb03d1d5de 100644 --- a/go-controller/pkg/util/node_annotations.go +++ b/go-controller/pkg/util/node_annotations.go @@ -537,7 +537,7 @@ func GetNodeEgressLabel() string { } func SetNodeHostAddresses(nodeAnnotator kube.Annotator, addresses sets.Set[string]) error { - return nodeAnnotator.Set(ovnNodeHostAddresses, addresses.UnsortedList()) + return nodeAnnotator.Set(ovnNodeHostAddresses, sets.List(addresses)) } // ParseNodeHostAddresses returns the parsed host addresses living on a node From ad052c1c0061f9734d78c93cb2454557238dac25 Mon Sep 17 00:00:00 2001 From: Nadia Pinaeva Date: Mon, 17 Apr 2023 10:55:45 +0200 Subject: [PATCH 34/40] Use loadbalancer.Name as client index Revert predicate search optimization 3369d387417ede05033c002b782b0279b201ea76 as predicate search should not bee needed anymore Signed-off-by: Nadia Pinaeva --- go-controller/pkg/libovsdb/libovsdb.go | 3 +- go-controller/pkg/libovsdbops/loadbalancer.go | 29 ++----------------- go-controller/pkg/libovsdbops/model.go | 5 ++-- .../ovn/controller/services/loadbalancer.go | 13 +-------- 4 files changed, 8 insertions(+), 42 deletions(-) diff --git a/go-controller/pkg/libovsdb/libovsdb.go b/go-controller/pkg/libovsdb/libovsdb.go index 28a49ec62c..2238aacecd 100644 --- a/go-controller/pkg/libovsdb/libovsdb.go +++ b/go-controller/pkg/libovsdb/libovsdb.go @@ -150,7 +150,8 @@ func NewNBClientWithConfig(cfg config.OvnAuthConfig, promRegistry prometheus.Reg // define client indexes for objects that are using dbIDs dbModel.SetIndexes(map[string][]model.ClientIndex{ - "ACL": {{Columns: []model.ColumnKey{{Column: "external_ids", Key: types.PrimaryIDKey}}}}, + nbdb.ACLTable: {{Columns: []model.ColumnKey{{Column: "external_ids", Key: types.PrimaryIDKey}}}}, + nbdb.LoadBalancerTable: {{Columns: []model.ColumnKey{{Column: "name"}}}}, }) c, err := newClient(cfg, dbModel, stopCh, enableMetricsOption) diff --git a/go-controller/pkg/libovsdbops/loadbalancer.go b/go-controller/pkg/libovsdbops/loadbalancer.go index a6b909b4cc..5cc074c9fd 100644 --- a/go-controller/pkg/libovsdbops/loadbalancer.go +++ b/go-controller/pkg/libovsdbops/loadbalancer.go @@ -57,25 +57,6 @@ func BuildLoadBalancer(name string, protocol nbdb.LoadBalancerProtocol, vips, op } } -// CreateLoadBalancersOps creates the provided load balancers returning the -// corresponding ops -func CreateLoadBalancersOps(nbClient libovsdbclient.Client, ops []libovsdb.Operation, lbs ...*nbdb.LoadBalancer) ([]libovsdb.Operation, error) { - opModels := make([]operationModel, 0, len(lbs)) - for i := range lbs { - lb := lbs[i] - opModel := operationModel{ - Model: lb, - OnModelUpdates: onModelUpdatesNone(), - ErrNotFound: false, - BulkOp: false, - } - opModels = append(opModels, opModel) - } - - modelClient := newModelClient(nbClient) - return modelClient.CreateOrUpdateOps(ops, opModels...) -} - // CreateOrUpdateLoadBalancersOps creates or updates the provided load balancers // returning the corresponding ops func CreateOrUpdateLoadBalancersOps(nbClient libovsdbclient.Client, ops []libovsdb.Operation, lbs ...*nbdb.LoadBalancer) ([]libovsdb.Operation, error) { @@ -85,7 +66,6 @@ func CreateOrUpdateLoadBalancersOps(nbClient libovsdbclient.Client, ops []libovs lb := lbs[i] opModel := operationModel{ Model: lb, - ModelPredicate: func(item *nbdb.LoadBalancer) bool { return item.Name == lb.Name }, OnModelUpdates: getNonZeroLoadBalancerMutableFields(lb), ErrNotFound: false, BulkOp: false, @@ -107,7 +87,6 @@ func RemoveLoadBalancerVipsOps(nbClient libovsdbclient.Client, ops []libovsdb.Op } opModel := operationModel{ Model: lb, - ModelPredicate: func(item *nbdb.LoadBalancer) bool { return item.Name == lb.Name }, OnModelMutations: []interface{}{&lb.Vips}, ErrNotFound: true, BulkOp: false, @@ -127,11 +106,9 @@ func DeleteLoadBalancersOps(nbClient libovsdbclient.Client, ops []libovsdb.Opera // can't use i in the predicate, for loop replaces it in-memory lb := lbs[i] opModel := operationModel{ - Model: lb, - // TODO: remove UUID match from predicate once model_client prioritizes indexed search over predicate - ModelPredicate: func(item *nbdb.LoadBalancer) bool { return item.UUID == lb.UUID || item.Name == lb.Name }, - ErrNotFound: false, - BulkOp: false, + Model: lb, + ErrNotFound: false, + BulkOp: false, } opModels = append(opModels, opModel) } diff --git a/go-controller/pkg/libovsdbops/model.go b/go-controller/pkg/libovsdbops/model.go index bda31f3e58..876c26dc11 100644 --- a/go-controller/pkg/libovsdbops/model.go +++ b/go-controller/pkg/libovsdbops/model.go @@ -159,6 +159,8 @@ func copyIndexes(model model.Model) model.Model { case *nbdb.LoadBalancer: return &nbdb.LoadBalancer{ UUID: t.UUID, + // client index + Name: t.Name, } case *nbdb.LoadBalancerGroup: return &nbdb.LoadBalancerGroup{ @@ -376,9 +378,6 @@ func buildFailOnDuplicateOps(c client.Client, m model.Model) ([]ovsdb.Operation, var field interface{} var value string switch t := m.(type) { - case *nbdb.LoadBalancer: - field = &t.Name - value = t.Name case *nbdb.LogicalRouter: field = &t.Name value = t.Name diff --git a/go-controller/pkg/ovn/controller/services/loadbalancer.go b/go-controller/pkg/ovn/controller/services/loadbalancer.go index e8a718d027..2571a51b73 100644 --- a/go-controller/pkg/ovn/controller/services/loadbalancer.go +++ b/go-controller/pkg/ovn/controller/services/loadbalancer.go @@ -142,8 +142,6 @@ func EnsureLBs(nbClient libovsdbclient.Client, service *corev1.Service, existing } tlbs := make([]*templateLoadBalancer, 0, len(LBs)) - existinglbs := make([]*templateLoadBalancer, 0, len(LBs)) - newlbs := make([]*templateLoadBalancer, 0, len(LBs)) addLBsToSwitch := map[string][]*templateLoadBalancer{} removeLBsFromSwitch := map[string][]*templateLoadBalancer{} addLBsToRouter := map[string][]*templateLoadBalancer{} @@ -161,13 +159,10 @@ func EnsureLBs(nbClient libovsdbclient.Client, service *corev1.Service, existing existingGroups := sets.Set[string]{} if existingLB != nil { blb.nbLB.UUID = existingLB.UUID - existinglbs = append(existinglbs, blb) delete(toDelete, existingLB.UUID) existingRouters = sets.New[string](existingLB.Routers...) existingSwitches = sets.New[string](existingLB.Switches...) existingGroups = sets.New[string](existingLB.Groups...) - } else { - newlbs = append(newlbs, blb) } wantRouters := sets.New(lb.Routers...) wantSwitches := sets.New(lb.Switches...) @@ -180,17 +175,11 @@ func EnsureLBs(nbClient libovsdbclient.Client, service *corev1.Service, existing mapLBDifferenceByKey(removeLBsFromGroups, existingGroups, wantGroups, blb) } - ops, err := libovsdbops.CreateOrUpdateLoadBalancersOps(nbClient, nil, toNBLoadBalancerList(existinglbs)...) + ops, err := libovsdbops.CreateOrUpdateLoadBalancersOps(nbClient, nil, toNBLoadBalancerList(tlbs)...) if err != nil { return err } - ops, err = libovsdbops.CreateLoadBalancersOps(nbClient, ops, toNBLoadBalancerList(newlbs)...) - if err != nil { - return fmt.Errorf("failed to create ops for ensuring update of service %s/%s load balancers: %w", - service.Namespace, service.Name, err) - } - ops, err = svcCreateOrUpdateTemplateVarOps(nbClient, ops, toNBTemplateList(tlbs)) if err != nil { return fmt.Errorf("failed to create ops for ensuring creation of service %s/%s load balancers: %w", From c6dc8e29e0457c2984428e84570c0fc40e5314bb Mon Sep 17 00:00:00 2001 From: Peng Liu Date: Tue, 18 Apr 2023 11:06:56 +0800 Subject: [PATCH 35/40] Detect bridge name when 'gateway-interface' is specified If the interface specified with '--gateway-interface' is already a bridge port, use the bridge name as the gateway interface. Signed-off-by: Peng Liu --- go-controller/pkg/node/gateway.go | 1 + go-controller/pkg/node/gateway_init.go | 7 +++ .../pkg/node/gateway_init_linux_test.go | 54 +++++++++++++++++++ 3 files changed, 62 insertions(+) diff --git a/go-controller/pkg/node/gateway.go b/go-controller/pkg/node/gateway.go index b4b235d430..5a9f1e2913 100644 --- a/go-controller/pkg/node/gateway.go +++ b/go-controller/pkg/node/gateway.go @@ -396,6 +396,7 @@ func bridgeForInterface(intfName, nodeName, physicalNetworkName string, gwIPs [] } res.bridgeName = bridgeName res.uplinkName = uplinkName + gwIntf = bridgeName } else if _, _, err := util.RunOVSVsctl("br-exists", intfName); err != nil { // This is not a OVS bridge. We need to create a OVS bridge // and add cluster.GatewayIntf as a port of that bridge. diff --git a/go-controller/pkg/node/gateway_init.go b/go-controller/pkg/node/gateway_init.go index f97d9bdaad..039f512dc5 100644 --- a/go-controller/pkg/node/gateway_init.go +++ b/go-controller/pkg/node/gateway_init.go @@ -179,6 +179,13 @@ func getGatewayNextHops() ([]net.IP, string, error) { } } gatewayIntf := config.Gateway.Interface + if gatewayIntf != "" { + if bridgeName, _, err := util.RunOVSVsctl("port-to-br", gatewayIntf); err == nil { + // This is an OVS bridge's internal port + gatewayIntf = bridgeName + } + } + if needIPv4NextHop || needIPv6NextHop || gatewayIntf == "" { defaultGatewayIntf, defaultGatewayNextHops, err := getDefaultGatewayInterfaceDetails(gatewayIntf) if err != nil { diff --git a/go-controller/pkg/node/gateway_init_linux_test.go b/go-controller/pkg/node/gateway_init_linux_test.go index 789ce8e4f9..07275dba46 100644 --- a/go-controller/pkg/node/gateway_init_linux_test.go +++ b/go-controller/pkg/node/gateway_init_linux_test.go @@ -70,6 +70,10 @@ func shareGatewayInterfaceTest(app *cli.App, testNS ns.NetNS, Cmd: "ovs-vsctl --timeout=15 port-to-br eth0", Err: fmt.Errorf(""), }) + fexec.AddFakeCmd(&ovntest.ExpectedCmd{ + Cmd: "ovs-vsctl --timeout=15 port-to-br eth0", + Err: fmt.Errorf(""), + }) fexec.AddFakeCmd(&ovntest.ExpectedCmd{ Cmd: "ovs-vsctl --timeout=15 br-exists eth0", Err: fmt.Errorf(""), @@ -406,6 +410,11 @@ func shareGatewayInterfaceDPUTest(app *cli.App, testNS ns.NetNS, Cmd: "ovs-vsctl --timeout=15 port-to-br " + brphys, Err: fmt.Errorf(""), }) + fexec.AddFakeCmd(&ovntest.ExpectedCmd{ + Cmd: "ovs-vsctl --timeout=15 port-to-br " + brphys, + Err: fmt.Errorf(""), + Output: brphys, + }) fexec.AddFakeCmd(&ovntest.ExpectedCmd{ Cmd: "ovs-vsctl --timeout=15 br-exists " + brphys, Err: nil, @@ -781,6 +790,10 @@ func localGatewayInterfaceTest(app *cli.App, testNS ns.NetNS, Cmd: "ovs-vsctl --timeout=15 port-to-br eth0", Err: fmt.Errorf(""), }) + fexec.AddFakeCmd(&ovntest.ExpectedCmd{ + Cmd: "ovs-vsctl --timeout=15 port-to-br eth0", + Err: fmt.Errorf(""), + }) fexec.AddFakeCmd(&ovntest.ExpectedCmd{ Cmd: "ovs-vsctl --timeout=15 br-exists eth0", Err: fmt.Errorf(""), @@ -1638,6 +1651,15 @@ var _ = Describe("Gateway unit tests", func() { It("Finds correct gateway interface and nexthops with single stack configuration", func() { ifName := "enf1f0" nextHopCfg := "10.0.0.11" + + fexec := ovntest.NewLooseCompareFakeExec() + fexec.AddFakeCmd(&ovntest.ExpectedCmd{ + Cmd: fmt.Sprintf("ovs-vsctl --timeout=15 port-to-br %s", ifName), + Err: fmt.Errorf(""), + }) + err := util.SetExec(fexec) + Expect(err).NotTo(HaveOccurred()) + gwIPs := []net.IP{net.ParseIP(nextHopCfg)} config.Gateway.Interface = ifName config.Gateway.NextHop = nextHopCfg @@ -1651,6 +1673,15 @@ var _ = Describe("Gateway unit tests", func() { It("Finds correct gateway interface and nexthops with dual stack configuration", func() { ifName := "enf1f0" nextHopCfg := "10.0.0.11,fc00:f853:ccd:e793::1" + + fexec := ovntest.NewLooseCompareFakeExec() + fexec.AddFakeCmd(&ovntest.ExpectedCmd{ + Cmd: fmt.Sprintf("ovs-vsctl --timeout=15 port-to-br %s", ifName), + Err: fmt.Errorf(""), + }) + err := util.SetExec(fexec) + Expect(err).NotTo(HaveOccurred()) + nextHops := strings.Split(nextHopCfg, ",") gwIPs := []net.IP{net.ParseIP(nextHops[0]), net.ParseIP(nextHops[1])} config.Gateway.Interface = ifName @@ -1663,5 +1694,28 @@ var _ = Describe("Gateway unit tests", func() { Expect(gatewayIntf).To(Equal(ifName)) Expect(gatewayNextHops).To(Equal(gwIPs)) }) + + ovntest.OnSupportedPlatformsIt("Finds correct gateway interface and nexthops when gateway bridge is created", func() { + ifName := "enf1f0" + nextHopCfg := "10.0.0.11" + + fexec := ovntest.NewLooseCompareFakeExec() + fexec.AddFakeCmd(&ovntest.ExpectedCmd{ + Cmd: fmt.Sprintf("ovs-vsctl --timeout=15 port-to-br %s", ifName), + Err: fmt.Errorf(""), + Output: "br" + ifName, + }) + err := util.SetExec(fexec) + Expect(err).NotTo(HaveOccurred()) + + gwIPs := []net.IP{net.ParseIP(nextHopCfg)} + config.Gateway.Interface = ifName + config.Gateway.NextHop = nextHopCfg + + gatewayNextHops, gatewayIntf, err := getGatewayNextHops() + Expect(err).NotTo(HaveOccurred()) + Expect(gatewayIntf).To(Equal(ifName)) + Expect(gatewayNextHops[0]).To(Equal(gwIPs[0])) + }) }) }) From b37a99e1f094a4ec35b207b5dabd455a1c48093f Mon Sep 17 00:00:00 2001 From: Dmytro Linkin Date: Thu, 8 Sep 2022 16:18:39 +0300 Subject: [PATCH 36/40] Add Scalable Functions support Scalable Functions (SFs) are "lightweight" Virtual Functions (VFs), meaning they are netdevices that implement representor model and can be used as container network interfaces. Unlike of VFs, SFs are Auxiliary devices and located on the respective system bus. Change the current code base to also accomodate the use of SF Auxiliary devices instead of only VF PCI devices for containers. Signed-off-by: Dmytro Linkin Signed-off-by: Alin-Gabriel Serdean Signed-off-by: Hareesh Puthalath --- go-controller/pkg/cni/cni.go | 25 +++---- go-controller/pkg/cni/cniserver.go | 16 +++++ go-controller/pkg/cni/helper_linux.go | 68 ++++++++----------- go-controller/pkg/cni/helper_linux_test.go | 42 ++++++------ go-controller/pkg/cni/types.go | 2 +- go-controller/pkg/cni/types/types.go | 7 +- go-controller/pkg/cni/utils.go | 4 +- .../node_network_controller_manager.go | 2 +- .../base_node_network_controller_dpu_test.go | 6 +- go-controller/pkg/util/mocks/SriovnetOps.go | 44 ++++++++++++ go-controller/pkg/util/sriovnet_linux.go | 12 ++++ 11 files changed, 145 insertions(+), 83 deletions(-) diff --git a/go-controller/pkg/cni/cni.go b/go-controller/pkg/cni/cni.go index eb7699685d..d199b435c1 100644 --- a/go-controller/pkg/cni/cni.go +++ b/go-controller/pkg/cni/cni.go @@ -107,23 +107,23 @@ func (pr *PodRequest) cmdAdd(kubeAuth *KubeAPIAuth, clientset *ClientSet, useOVS kubecli := &kube.Kube{KClient: clientset.kclient} annotCondFn := isOvnReady - vfNetdevName := "" + netdevName := "" if pr.CNIConf.DeviceID != "" { var err error - vfNetdevName, err = util.GetNetdevNameFromDeviceId(pr.CNIConf.DeviceID) + netdevName, err = util.GetNetdevNameFromDeviceId(pr.CNIConf.DeviceID) if err != nil { - return nil, fmt.Errorf("failed in cmdAdd while getting VF Netdevice name: %v", err) + return nil, fmt.Errorf("failed in cmdAdd while getting Netdevice name: %v", err) } if config.OvnKubeNode.Mode == types.NodeModeDPUHost { // Add DPU connection-details annotation so ovnkube-node running on DPU // performs the needed network plumbing. - if err = pr.addDPUConnectionDetailsAnnot(kubecli, clientset.podLister, vfNetdevName); err != nil { + if err = pr.addDPUConnectionDetailsAnnot(kubecli, clientset.podLister, netdevName); err != nil { return nil, err } annotCondFn = isDPUReady } - // In the case of SmartNIC (CX5), we store the VFNetdevname in the VF representor's + // In the case of SmartNIC (CX5), we store the netdevname in the representor's // OVS interface's external_id column. This is done in ConfigureInterface(). } // Get the IP address and MAC address of the pod @@ -136,7 +136,8 @@ func (pr *PodRequest) cmdAdd(kubeAuth *KubeAPIAuth, clientset *ClientSet, useOVS if err = pr.checkOrUpdatePodUID(pod); err != nil { return nil, err } - podInterfaceInfo, err := PodAnnotation2PodInfo(annotations, podNADAnnotation, useOVSExternalIDs, pr.PodUID, vfNetdevName, + + podInterfaceInfo, err := PodAnnotation2PodInfo(annotations, podNADAnnotation, useOVSExternalIDs, pr.PodUID, netdevName, pr.nadName, pr.netName, pr.CNIConf.MTU) if err != nil { return nil, err @@ -166,7 +167,7 @@ func (pr *PodRequest) cmdDel(clientset *ClientSet) (*Response, error) { return nil, fmt.Errorf("required CNI variable missing") } - vfNetdevName := "" + netdevName := "" if pr.CNIConf.DeviceID != "" { if config.OvnKubeNode.Mode == types.NodeModeDPUHost { pod, err := clientset.getPod(pr.PodNamespace, pr.PodName) @@ -180,9 +181,9 @@ func (pr *PodRequest) cmdDel(clientset *ClientSet) (*Response, error) { pr.PodName, pr.nadName, err) return response, nil } - vfNetdevName = dpuCD.VfNetdevName + netdevName = dpuCD.VfNetdevName } else { - // Find the the hostInterface name + // Find the hostInterface name condString := "external-ids:sandbox=" + pr.SandboxID if pr.netName != types.DefaultNetworkName { condString += fmt.Sprintf(" external_ids:%s=%s", types.NADExternalID, pr.nadName) @@ -197,10 +198,10 @@ func (pr *PodRequest) cmdDel(clientset *ClientSet) (*Response, error) { ovsIfName := ovsIfNames[0] out, err := ovsGet("interface", ovsIfName, "external_ids", "vf-netdev-name") if err != nil { - klog.Warningf("Couldn't find the original VF Netdev name from OVS interface %s for pod %s/%s: %v", + klog.Warningf("Couldn't find the original Netdev name from OVS interface %s for pod %s/%s: %v", ovsIfName, pr.PodNamespace, pr.PodName, err) } else { - vfNetdevName = out + netdevName = out } } } @@ -208,7 +209,7 @@ func (pr *PodRequest) cmdDel(clientset *ClientSet) (*Response, error) { podInterfaceInfo := &PodInterfaceInfo{ IsDPUHostMode: config.OvnKubeNode.Mode == types.NodeModeDPUHost, - VfNetdevName: vfNetdevName, + NetdevName: netdevName, } if !config.UnprivilegedMode { err := pr.UnconfigureInterface(podInterfaceInfo) diff --git a/go-controller/pkg/cni/cniserver.go b/go-controller/pkg/cni/cniserver.go index b79f593994..0413211c0f 100644 --- a/go-controller/pkg/cni/cniserver.go +++ b/go-controller/pkg/cni/cniserver.go @@ -20,6 +20,7 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" ) // *** The Server is PRIVATE API between OVN components and may be @@ -183,6 +184,21 @@ func cniRequestToPodRequest(cr *Request) (*PodRequest, error) { } else { req.nadName = conf.NADName } + + if conf.DeviceID != "" { + if util.IsPCIDeviceName(conf.DeviceID) { + // DeviceID is a PCI address + } else if util.IsAuxDeviceName(conf.DeviceID) { + // DeviceID is an Auxiliary device name - .. + chunks := strings.Split(conf.DeviceID, ".") + if chunks[1] != "sf" { + return nil, fmt.Errorf("only SF auxiliary devices are supported") + } + } else { + return nil, fmt.Errorf("expected PCI or Auxiliary device name, got - %s", conf.DeviceID) + } + } + req.CNIConf = conf req.timestamp = time.Now() // Match the Kubelet default CRI operation timeout of 2m diff --git a/go-controller/pkg/cni/helper_linux.go b/go-controller/pkg/cni/helper_linux.go index 478cc19c0d..b007369d04 100644 --- a/go-controller/pkg/cni/helper_linux.go +++ b/go-controller/pkg/cni/helper_linux.go @@ -134,13 +134,13 @@ func setSysctl(sysctl string, newVal int) error { } func moveIfToNetns(ifname string, netns ns.NetNS) error { - vfDev, err := util.GetNetLinkOps().LinkByName(ifname) + dev, err := util.GetNetLinkOps().LinkByName(ifname) if err != nil { - return fmt.Errorf("failed to lookup vf device %v: %q", ifname, err) + return fmt.Errorf("failed to lookup device %v: %q", ifname, err) } - // move VF device to ns - if err = util.GetNetLinkOps().LinkSetNsFd(vfDev, int(netns.Fd())); err != nil { + // move netdevice to ns + if err = util.GetNetLinkOps().LinkSetNsFd(dev, int(netns.Fd())); err != nil { return fmt.Errorf("failed to move device %+v to netns: %q", ifname, err) } @@ -249,22 +249,22 @@ func setupInterface(netns ns.NetNS, containerID, ifName string, ifInfo *PodInter } // Setup sriov interface in the pod -func setupSriovInterface(netns ns.NetNS, containerID, ifName string, ifInfo *PodInterfaceInfo, pciAddrs string) (*current.Interface, *current.Interface, error) { +func setupSriovInterface(netns ns.NetNS, containerID, ifName string, ifInfo *PodInterfaceInfo, deviceID string) (*current.Interface, *current.Interface, error) { hostIface := ¤t.Interface{} contIface := ¤t.Interface{} ifnameSuffix := "" - vfNetdevice := ifInfo.VfNetdevName + netdevice := ifInfo.NetdevName - // 1. Move VF to Container namespace - err := moveIfToNetns(vfNetdevice, netns) + // 1. Move netdevice to Container namespace + err := moveIfToNetns(netdevice, netns) if err != nil { return nil, nil, err } err = netns.Do(func(hostNS ns.NetNS) error { contIface.Name = ifName - err = renameLink(vfNetdevice, contIface.Name) + err = renameLink(netdevice, contIface.Name) if err != nil { return err } @@ -305,32 +305,19 @@ func setupSriovInterface(netns ns.NetNS, containerID, ifName string, ifInfo *Pod } if !ifInfo.IsDPUHostMode { - // 2. get Uplink netdevice - uplink, err := util.GetSriovnetOps().GetUplinkRepresentor(pciAddrs) + // 2. get device representor name + oldHostRepName, err := util.GetFunctionRepresentorName(deviceID) if err != nil { return nil, nil, err } - // 3. get VF index from PCI - vfIndex, err := util.GetSriovnetOps().GetVfIndexByPciAddress(pciAddrs) - if err != nil { - return nil, nil, err - } - - // 4. lookup representor - rep, err := util.GetSriovnetOps().GetVfRepresentor(uplink, vfIndex) - if err != nil { - return nil, nil, err - } - oldHostRepName := rep - - // 5. make sure it's not a port managed by OVS to avoid conflicts when renaming the VF representor + // 3. make sure it's not a port managed by OVS to avoid conflicts when renaming the representor _, err = ovsExec("--if-exists", "del-port", oldHostRepName) if err != nil { return nil, nil, err } - // 6. rename the host VF representor + // 4. rename the host representor hostIface.Name = containerID[:(15-len(ifnameSuffix))] + ifnameSuffix if err = renameLink(oldHostRepName, hostIface.Name); err != nil { return nil, nil, fmt.Errorf("failed to rename %s to %s: %v", oldHostRepName, hostIface.Name, err) @@ -342,7 +329,7 @@ func setupSriovInterface(netns ns.NetNS, containerID, ifName string, ifInfo *Pod } hostIface.Mac = link.Attrs().HardwareAddr.String() - // 7. set MTU on VF representor + // 5. set MTU on the representor if err = util.GetNetLinkOps().LinkSetMTU(link, ifInfo.MTU); err != nil { return nil, nil, fmt.Errorf("failed to set MTU on %s: %v", hostIface.Name, err) } @@ -395,8 +382,10 @@ func ConfigureOVS(ctx context.Context, namespace, podName, hostIfaceName string, ovsArgs = append(ovsArgs, fmt.Sprintf("external_ids:ip_addresses=%s", strings.Join(ipStrs, ","))) } - if len(ifInfo.VfNetdevName) != 0 { - ovsArgs = append(ovsArgs, fmt.Sprintf("external_ids:vf-netdev-name=%s", ifInfo.VfNetdevName)) + if len(ifInfo.NetdevName) != 0 { + // NOTE: For SF representor same external_id is used due to https://github.com/ovn-org/ovn-kubernetes/pull/3054 + // Review this line when upgrade mechanism will be implemented + ovsArgs = append(ovsArgs, fmt.Sprintf("external_ids:vf-netdev-name=%s", ifInfo.NetdevName)) } if ifInfo.NetName != types.DefaultNetworkName { @@ -520,13 +509,12 @@ func (pr *PodRequest) UnconfigureInterface(ifInfo *PodInterfaceInfo) error { klog.Warningf("Unexpected configuration %s, Device ID must be present for pod request on smart-nic host", podDesc) return nil } - // 1. For SRIOV case, we'd need to move the VF from container namespace back to the host namespace - // 2. If it is secondary network and non-dpu mode, needs to get the container interface index + // 1. For SRIOV case, we'd need to move the netdevice from container namespace back to the host namespace + // 2. If it is secondary network and non-dpu mode, then get the container interface index // so that we know the host-side interface name. ifnameSuffix := "" isSecondary := pr.netName != types.DefaultNetworkName if pr.CNIConf.DeviceID != "" || (isSecondary && !ifInfo.IsDPUHostMode) { - // For SRIOV case, we'd need to move the VF from container namespace back to the host namespace netns, err := ns.GetNS(pr.Netns) if err != nil { return fmt.Errorf("failed to get container namespace %s: %v", podDesc, err) @@ -551,19 +539,19 @@ func (pr *PodRequest) UnconfigureInterface(ifInfo *PodInterfaceInfo) error { if err != nil { return fmt.Errorf("failed to bring down container interface %s %s: %v", pr.IfName, podDesc, err) } - // rename VF device to make sure it is unique in the host namespace: - // if the VF's original name is empty, sandbox id and a '0' letter prefix is used to make up the unique name. - oldVfName := ifInfo.VfNetdevName - if oldVfName == "" { + // rename netdevice to make sure it is unique in the host namespace: + // if original name of netdevice is empty, sandbox id and a '0' letter prefix is used to make up the unique name. + oldName := ifInfo.NetdevName + if oldName == "" { id := fmt.Sprintf("_0%d", link.Attrs().Index) - oldVfName = pr.SandboxID[:(15-len(id))] + id + oldName = pr.SandboxID[:(15-len(id))] + id } - err = util.GetNetLinkOps().LinkSetName(link, oldVfName) + err = util.GetNetLinkOps().LinkSetName(link, oldName) if err != nil { return fmt.Errorf("failed to rename container interface %s to %s %s: %v", - pr.IfName, oldVfName, podDesc, err) + pr.IfName, oldName, podDesc, err) } - // move VF device to host netns + // move netdevice to host netns err = util.GetNetLinkOps().LinkSetNsFd(link, int(hostNS.Fd())) if err != nil { return fmt.Errorf("failed to move container interface %s back to host namespace %s: %v", diff --git a/go-controller/pkg/cni/helper_linux_test.go b/go-controller/pkg/cni/helper_linux_test.go index a2887906bd..c8ef24c7dc 100644 --- a/go-controller/pkg/cni/helper_linux_test.go +++ b/go-controller/pkg/cni/helper_linux_test.go @@ -126,7 +126,7 @@ func TestMoveIfToNetns(t *testing.T) { desc: "test code path when LinkByName() returns error", inpIfaceName: "testIfaceName", inpNetNs: nil, - errMatch: fmt.Errorf("failed to lookup vf device"), + errMatch: fmt.Errorf("failed to lookup device"), netLinkOpsMockHelper: []ovntest.TestifyMockHelper{ {OnCallMethodName: "LinkByName", OnCallMethodArgType: []string{"string"}, RetArgList: []interface{}{nil, fmt.Errorf("mock error")}}, }, @@ -478,7 +478,7 @@ func TestSetupSriovInterface(t *testing.T) { inpPodIfaceInfo: &PodInterfaceInfo{ PodAnnotation: util.PodAnnotation{}, MTU: 1500, - VfNetdevName: "en01", + NetdevName: "en01", }, inpPCIAddrs: "0000:03:00.1", errExp: true, @@ -498,7 +498,7 @@ func TestSetupSriovInterface(t *testing.T) { inpPodIfaceInfo: &PodInterfaceInfo{ PodAnnotation: util.PodAnnotation{}, MTU: 1500, - VfNetdevName: "en01", + NetdevName: "en01", }, inpPCIAddrs: "0000:03:00.1", errExp: true, @@ -525,7 +525,7 @@ func TestSetupSriovInterface(t *testing.T) { inpPodIfaceInfo: &PodInterfaceInfo{ PodAnnotation: util.PodAnnotation{}, MTU: 1500, - VfNetdevName: "en01", + NetdevName: "en01", }, inpPCIAddrs: "0000:03:00.1", errExp: true, @@ -554,7 +554,7 @@ func TestSetupSriovInterface(t *testing.T) { inpPodIfaceInfo: &PodInterfaceInfo{ PodAnnotation: util.PodAnnotation{}, MTU: 1500, - VfNetdevName: "en01", + NetdevName: "en01", }, inpPCIAddrs: "0000:03:00.1", errExp: true, @@ -586,7 +586,7 @@ func TestSetupSriovInterface(t *testing.T) { inpPodIfaceInfo: &PodInterfaceInfo{ PodAnnotation: util.PodAnnotation{}, MTU: 1500, - VfNetdevName: "en01", + NetdevName: "en01", }, inpPCIAddrs: "0000:03:00.1", errExp: true, @@ -619,7 +619,7 @@ func TestSetupSriovInterface(t *testing.T) { inpPodIfaceInfo: &PodInterfaceInfo{ PodAnnotation: util.PodAnnotation{}, MTU: 1500, - VfNetdevName: "en01", + NetdevName: "en01", }, inpPCIAddrs: "0000:03:00.1", errMatch: fmt.Errorf("failed to rename"), @@ -659,7 +659,7 @@ func TestSetupSriovInterface(t *testing.T) { inpPodIfaceInfo: &PodInterfaceInfo{ PodAnnotation: util.PodAnnotation{}, MTU: 1500, - VfNetdevName: "en01", + NetdevName: "en01", }, inpPCIAddrs: "0000:03:00.1", errExp: true, @@ -704,7 +704,7 @@ func TestSetupSriovInterface(t *testing.T) { inpPodIfaceInfo: &PodInterfaceInfo{ PodAnnotation: util.PodAnnotation{}, MTU: 1500, - VfNetdevName: "en01", + NetdevName: "en01", }, inpPCIAddrs: "0000:03:00.1", errMatch: fmt.Errorf("failed to set MTU on"), @@ -756,7 +756,7 @@ func TestSetupSriovInterface(t *testing.T) { PodAnnotation: util.PodAnnotation{}, MTU: 1500, IsDPUHostMode: true, - VfNetdevName: "en01", + NetdevName: "en01", }, inpPCIAddrs: "0000:03:00.1", errExp: false, @@ -790,7 +790,7 @@ func TestSetupSriovInterface(t *testing.T) { PodAnnotation: util.PodAnnotation{}, MTU: 1500, IsDPUHostMode: true, - VfNetdevName: "en01", + NetdevName: "en01", }, inpPCIAddrs: "0000:03:00.1", errExp: true, @@ -813,7 +813,7 @@ func TestSetupSriovInterface(t *testing.T) { PodAnnotation: util.PodAnnotation{}, MTU: 1500, IsDPUHostMode: true, - VfNetdevName: "en01", + NetdevName: "en01", }, inpPCIAddrs: "0000:03:00.1", errExp: true, @@ -837,7 +837,7 @@ func TestSetupSriovInterface(t *testing.T) { PodAnnotation: util.PodAnnotation{}, MTU: 1500, IsDPUHostMode: true, - VfNetdevName: "en01", + NetdevName: "en01", }, inpPCIAddrs: "0000:03:00.1", errExp: true, @@ -862,7 +862,7 @@ func TestSetupSriovInterface(t *testing.T) { PodAnnotation: util.PodAnnotation{}, MTU: 1500, IsDPUHostMode: true, - VfNetdevName: "en01", + NetdevName: "en01", }, inpPCIAddrs: "0000:03:00.1", errExp: true, @@ -888,7 +888,7 @@ func TestSetupSriovInterface(t *testing.T) { PodAnnotation: util.PodAnnotation{}, MTU: 1500, IsDPUHostMode: true, - VfNetdevName: "en01", + NetdevName: "en01", }, inpPCIAddrs: "0000:03:00.1", errExp: true, @@ -915,7 +915,7 @@ func TestSetupSriovInterface(t *testing.T) { PodAnnotation: util.PodAnnotation{}, MTU: 1500, IsDPUHostMode: true, - VfNetdevName: "en01", + NetdevName: "en01", }, inpPCIAddrs: "0000:03:00.1", errExp: true, @@ -943,7 +943,7 @@ func TestSetupSriovInterface(t *testing.T) { PodAnnotation: util.PodAnnotation{}, MTU: 1500, IsDPUHostMode: true, - VfNetdevName: "en01", + NetdevName: "en01", }, inpPCIAddrs: "0000:03:00.1", errExp: true, @@ -972,7 +972,7 @@ func TestSetupSriovInterface(t *testing.T) { PodAnnotation: util.PodAnnotation{}, MTU: 1500, IsDPUHostMode: true, - VfNetdevName: "en01", + NetdevName: "en01", }, inpPCIAddrs: "0000:03:00.1", errExp: true, @@ -1004,7 +1004,7 @@ func TestSetupSriovInterface(t *testing.T) { }, MTU: 1500, IsDPUHostMode: true, - VfNetdevName: "en01", + NetdevName: "en01", }, inpPCIAddrs: "0000:03:00.1", errExp: true, @@ -1041,7 +1041,7 @@ func TestSetupSriovInterface(t *testing.T) { }, MTU: 1500, IsDPUHostMode: true, - VfNetdevName: "en01", + NetdevName: "en01", }, inpPCIAddrs: "0000:03:00.1", errExp: true, @@ -1086,7 +1086,7 @@ func TestSetupSriovInterface(t *testing.T) { }, MTU: 1500, IsDPUHostMode: true, - VfNetdevName: "en01", + NetdevName: "en01", }, inpPCIAddrs: "0000:03:00.1", errExp: true, diff --git a/go-controller/pkg/cni/types.go b/go-controller/pkg/cni/types.go index 6458d0aae7..ac65f0c4a3 100644 --- a/go-controller/pkg/cni/types.go +++ b/go-controller/pkg/cni/types.go @@ -49,7 +49,7 @@ type PodInterfaceInfo struct { CheckExtIDs bool `json:"check-external-ids"` IsDPUHostMode bool `json:"is-dpu-host-mode"` PodUID string `json:"pod-uid"` - VfNetdevName string `json:"vf-netdev-name"` + NetdevName string `json:"vf-netdev-name"` EnableUDPAggregation bool `json:"enable-udp-aggregation"` // network name, for default network, it is "default", otherwise it is net-attach-def's netconf spec name diff --git a/go-controller/pkg/cni/types/types.go b/go-controller/pkg/cni/types/types.go index 5b0da70203..4b776ba58c 100644 --- a/go-controller/pkg/cni/types/types.go +++ b/go-controller/pkg/cni/types/types.go @@ -1,8 +1,9 @@ package types import ( - "github.com/containernetworking/cni/pkg/types" "net" + + "github.com/containernetworking/cni/pkg/types" ) // NetConf is CNI NetConf with DeviceID @@ -27,7 +28,7 @@ type NetConf struct { // VLANID, valid in localnet topology network only VLANID int `json:"vlanID,omitempty"` - // PciAddrs in case of using sriov + // PciAddrs in case of using sriov or Auxiliry device name in case of SF DeviceID string `json:"deviceID,omitempty"` // LogFile to log all the messages from cni shim binary to LogFile string `json:"logFile,omitempty"` @@ -36,7 +37,7 @@ type NetConf struct { // LogFileMaxSize is the maximum size in bytes of the logfile // before it gets rolled. LogFileMaxSize int `json:"logfile-maxsize"` - // LogFileMaxBackups represents the the maximum number of + // LogFileMaxBackups represents the maximum number of // old log files to retain LogFileMaxBackups int `json:"logfile-maxbackups"` // LogFileMaxAge represents the maximum number diff --git a/go-controller/pkg/cni/utils.go b/go-controller/pkg/cni/utils.go index 06c3c8b1e8..a594ec7e91 100644 --- a/go-controller/pkg/cni/utils.go +++ b/go-controller/pkg/cni/utils.go @@ -99,7 +99,7 @@ func GetPodWithAnnotations(ctx context.Context, getter PodInfoGetter, // PodAnnotation2PodInfo creates PodInterfaceInfo from Pod annotations and additional attributes func PodAnnotation2PodInfo(podAnnotation map[string]string, podNADAnnotation *util.PodAnnotation, checkExtIDs bool, podUID, - vfNetdevname, nadName string, netName string, mtu int) (*PodInterfaceInfo, error) { + netdevname, nadName, netName string, mtu int) (*PodInterfaceInfo, error) { var err error // get pod's annotation of the given NAD if it is not available if podNADAnnotation == nil { @@ -126,7 +126,7 @@ func PodAnnotation2PodInfo(podAnnotation map[string]string, podNADAnnotation *ut CheckExtIDs: checkExtIDs, IsDPUHostMode: config.OvnKubeNode.Mode == types.NodeModeDPUHost, PodUID: podUID, - VfNetdevName: vfNetdevname, + NetdevName: netdevname, NetName: netName, NADName: nadName, EnableUDPAggregation: config.Default.EnableUDPAggregation, diff --git a/go-controller/pkg/network-controller-manager/node_network_controller_manager.go b/go-controller/pkg/network-controller-manager/node_network_controller_manager.go index f33f59c02f..34997ed2ff 100644 --- a/go-controller/pkg/network-controller-manager/node_network_controller_manager.go +++ b/go-controller/pkg/network-controller-manager/node_network_controller_manager.go @@ -187,7 +187,7 @@ func (ncm *nodeNetworkControllerManager) Stop() { // derive iface-id from pod name and namespace then remove any interfaces assoicated with a sandbox that are // not scheduled to the node. func (ncm *nodeNetworkControllerManager) checkForStaleOVSRepresentorInterfaces() { - // Get all ovn-kuberntes Pod interfaces. these are OVS interfaces that have their external_ids:sandbox set. + // Get all representor interfaces. these are OVS interfaces that have their external_ids:sandbox and vf-netdev-name set. out, stderr, err := util.RunOVSVsctl("--columns=name,external_ids", "--data=bare", "--no-headings", "--format=csv", "find", "Interface", "external_ids:sandbox!=\"\"", "external_ids:vf-netdev-name!=\"\"") if err != nil { diff --git a/go-controller/pkg/node/base_node_network_controller_dpu_test.go b/go-controller/pkg/node/base_node_network_controller_dpu_test.go index d6d34be39a..505fbe3e74 100644 --- a/go-controller/pkg/node/base_node_network_controller_dpu_test.go +++ b/go-controller/pkg/node/base_node_network_controller_dpu_test.go @@ -208,8 +208,8 @@ var _ = Describe("Node DPU tests", func() { }) It("Fails if configure OVS fails (with netdev external-ids)", func() { - // set vfRep netdev name - ifInfo.VfNetdevName = "netdev-vf9" + // set representor netdev name + ifInfo.NetdevName = "netdev-vf9" // set ovs CMD output execMock.AddFakeCmd(&ovntest.ExpectedCmd{ @@ -217,7 +217,7 @@ var _ = Describe("Node DPU tests", func() { "external-ids:iface-id="+genIfaceID(pod.Namespace, pod.Name)), }) execMock.AddFakeCmd(&ovntest.ExpectedCmd{ - Cmd: genOVSAddPortCmdWithNetdev(vfRep, ifInfo.VfNetdevName, genIfaceID(pod.Namespace, pod.Name), "", "", "a8d09931", string(pod.UID)), + Cmd: genOVSAddPortCmdWithNetdev(vfRep, ifInfo.NetdevName, genIfaceID(pod.Namespace, pod.Name), "", "", "a8d09931", string(pod.UID)), Err: fmt.Errorf("failed to run ovs command"), }) // Mock netlink/ovs calls for cleanup diff --git a/go-controller/pkg/util/mocks/SriovnetOps.go b/go-controller/pkg/util/mocks/SriovnetOps.go index 8b06594964..609cac1379 100644 --- a/go-controller/pkg/util/mocks/SriovnetOps.go +++ b/go-controller/pkg/util/mocks/SriovnetOps.go @@ -15,6 +15,29 @@ type SriovnetOps struct { mock.Mock } +// GetNetDevicesFromAux provides a mock function with given fields: auxDev +func (_m *SriovnetOps) GetNetDevicesFromAux(auxDev string) ([]string, error) { + ret := _m.Called(auxDev) + + var r0 []string + if rf, ok := ret.Get(0).(func(string) []string); ok { + r0 = rf(auxDev) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]string) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(auxDev) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetNetDevicesFromPci provides a mock function with given fields: pciAddress func (_m *SriovnetOps) GetNetDevicesFromPci(pciAddress string) ([]string, error) { ret := _m.Called(pciAddress) @@ -38,6 +61,27 @@ func (_m *SriovnetOps) GetNetDevicesFromPci(pciAddress string) ([]string, error) return r0, r1 } +// GetPfPciFromAux provides a mock function with given fields: auxDev +func (_m *SriovnetOps) GetPfPciFromAux(auxDev string) (string, error) { + ret := _m.Called(auxDev) + + var r0 string + if rf, ok := ret.Get(0).(func(string) string); ok { + r0 = rf(auxDev) + } else { + r0 = ret.Get(0).(string) + } + + var r1 error + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(auxDev) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetPfPciFromVfPci provides a mock function with given fields: vfPciAddress func (_m *SriovnetOps) GetPfPciFromVfPci(vfPciAddress string) (string, error) { ret := _m.Called(vfPciAddress) diff --git a/go-controller/pkg/util/sriovnet_linux.go b/go-controller/pkg/util/sriovnet_linux.go index 472b26d532..42bda3a6e1 100644 --- a/go-controller/pkg/util/sriovnet_linux.go +++ b/go-controller/pkg/util/sriovnet_linux.go @@ -13,6 +13,7 @@ import ( type SriovnetOps interface { GetNetDevicesFromPci(pciAddress string) ([]string, error) + GetNetDevicesFromAux(auxDev string) ([]string, error) GetUplinkRepresentor(vfPciAddress string) (string, error) GetUplinkRepresentorFromAux(auxDev string) (string, error) GetVfIndexByPciAddress(vfPciAddress string) (int, error) @@ -20,6 +21,7 @@ type SriovnetOps interface { GetVfRepresentor(uplink string, vfIndex int) (string, error) GetSfRepresentor(uplink string, sfIndex int) (string, error) GetPfPciFromVfPci(vfPciAddress string) (string, error) + GetPfPciFromAux(auxDev string) (string, error) GetVfRepresentorDPU(pfID, vfIndex string) (string, error) GetRepresentorPeerMacAddress(netdev string) (net.HardwareAddr, error) GetRepresentorPortFlavour(netdev string) (sriovnet.PortFlavour, error) @@ -44,6 +46,10 @@ func (defaultSriovnetOps) GetNetDevicesFromPci(pciAddress string) ([]string, err return sriovnet.GetNetDevicesFromPci(pciAddress) } +func (defaultSriovnetOps) GetNetDevicesFromAux(auxDev string) ([]string, error) { + return sriovnet.GetNetDevicesFromAux(auxDev) +} + func (defaultSriovnetOps) GetUplinkRepresentor(vfPciAddress string) (string, error) { return sriovnet.GetUplinkRepresentor(vfPciAddress) } @@ -72,6 +78,10 @@ func (defaultSriovnetOps) GetPfPciFromVfPci(vfPciAddress string) (string, error) return sriovnet.GetPfPciFromVfPci(vfPciAddress) } +func (defaultSriovnetOps) GetPfPciFromAux(auxDev string) (string, error) { + return sriovnet.GetPfPciFromAux(auxDev) +} + func (defaultSriovnetOps) GetVfRepresentorDPU(pfID, vfIndex string) (string, error) { return sriovnet.GetVfRepresentorDPU(pfID, vfIndex) } @@ -134,6 +144,8 @@ func GetNetdevNameFromDeviceId(deviceId string) (string, error) { } netdevices, err = GetSriovnetOps().GetNetDevicesFromPci(deviceId) + } else { // Auxiliary network device + netdevices, err = GetSriovnetOps().GetNetDevicesFromAux(deviceId) } if err != nil { return "", err From 8bd9b3e4f9477f1b86efbcf91c8777cc91307968 Mon Sep 17 00:00:00 2001 From: Andrea Panattoni Date: Fri, 7 Apr 2023 18:36:08 +0200 Subject: [PATCH 37/40] Fix ETP iptables for shared gateway mode For NodePort Services with ExternalTrafficPolicy = Local, incoming connections shouldn't be DNATted to masqueradeIP. Signed-off-by: Andrea Panattoni --- go-controller/pkg/node/gateway_iptables.go | 4 +++- go-controller/pkg/node/gateway_localnet_linux_test.go | 11 +++-------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/go-controller/pkg/node/gateway_iptables.go b/go-controller/pkg/node/gateway_iptables.go index 14c32a785a..17eba5ef5c 100644 --- a/go-controller/pkg/node/gateway_iptables.go +++ b/go-controller/pkg/node/gateway_iptables.go @@ -611,7 +611,9 @@ func getGatewayIPTRules(service *kapi.Service, localEndpoints []string, svcHasLo if svcTypeIsETPLocal && !svcHasLocalHostNetEndPnt { // case1 (see function description for details) // A DNAT rule to masqueradeIP is added that takes priority over DNAT to clusterIP. - rules = append(rules, getNodePortIPTRules(svcPort, clusterIP, svcPort.NodePort, svcHasLocalHostNetEndPnt, svcTypeIsETPLocal)...) + if config.Gateway.Mode == config.GatewayModeLocal { + rules = append(rules, getNodePortIPTRules(svcPort, clusterIP, svcPort.NodePort, svcHasLocalHostNetEndPnt, svcTypeIsETPLocal)...) + } // add a skip SNAT rule to OVN-KUBE-SNAT-MGMTPORT to preserve sourceIP for etp=local traffic. rules = append(rules, getNodePortETPLocalIPTRules(svcPort, clusterIP)...) } diff --git a/go-controller/pkg/node/gateway_localnet_linux_test.go b/go-controller/pkg/node/gateway_localnet_linux_test.go index 27ae4ddd1c..8202673026 100644 --- a/go-controller/pkg/node/gateway_localnet_linux_test.go +++ b/go-controller/pkg/node/gateway_localnet_linux_test.go @@ -1155,7 +1155,6 @@ var _ = Describe("Node Operations", func() { "OVN-KUBE-ETP": []string{ fmt.Sprintf("-p %s -d %s --dport %v -j DNAT --to-destination %s:%v", service.Spec.Ports[0].Protocol, service.Status.LoadBalancer.Ingress[0].IP, service.Spec.Ports[0].Port, types.V4HostETPLocalMasqueradeIP, service.Spec.Ports[0].NodePort), fmt.Sprintf("-p %s -d %s --dport %v -j DNAT --to-destination %s:%v", service.Spec.Ports[0].Protocol, externalIP, service.Spec.Ports[0].Port, types.V4HostETPLocalMasqueradeIP, service.Spec.Ports[0].NodePort), - fmt.Sprintf("-p %s -m addrtype --dst-type LOCAL --dport %v -j DNAT --to-destination %s:%v", service.Spec.Ports[0].Protocol, service.Spec.Ports[0].NodePort, types.V4HostETPLocalMasqueradeIP, service.Spec.Ports[0].NodePort), }, "OVN-KUBE-ITP": []string{}, "OVN-KUBE-EGRESS-SVC": []string{"-m mark --mark 0x3f0 -m comment --comment Do not SNAT to SVC VIP -j RETURN"}, @@ -2116,9 +2115,7 @@ var _ = Describe("Node Operations", func() { "OVN-KUBE-SNAT-MGMTPORT": []string{ fmt.Sprintf("-p TCP --dport %v -j RETURN", service.Spec.Ports[0].NodePort), }, - "OVN-KUBE-ETP": []string{ - fmt.Sprintf("-p %s -m addrtype --dst-type LOCAL --dport %v -j DNAT --to-destination %s:%v", service.Spec.Ports[0].Protocol, service.Spec.Ports[0].NodePort, types.V4HostETPLocalMasqueradeIP, service.Spec.Ports[0].NodePort), - }, + "OVN-KUBE-ETP": []string{}, "OVN-KUBE-ITP": []string{}, "OVN-KUBE-EGRESS-SVC": []string{"-m mark --mark 0x3f0 -m comment --comment Do not SNAT to SVC VIP -j RETURN"}, }, @@ -2405,10 +2402,8 @@ var _ = Describe("Node Operations", func() { "OVN-KUBE-SNAT-MGMTPORT": []string{ fmt.Sprintf("-p TCP --dport %v -j RETURN", service.Spec.Ports[0].NodePort), }, - "OVN-KUBE-ITP": []string{}, - "OVN-KUBE-ETP": []string{ - fmt.Sprintf("-p %s -m addrtype --dst-type LOCAL --dport %v -j DNAT --to-destination %s:%v", service.Spec.Ports[0].Protocol, service.Spec.Ports[0].NodePort, types.V4HostETPLocalMasqueradeIP, service.Spec.Ports[0].NodePort), - }, + "OVN-KUBE-ITP": []string{}, + "OVN-KUBE-ETP": []string{}, "OVN-KUBE-EGRESS-SVC": []string{"-m mark --mark 0x3f0 -m comment --comment Do not SNAT to SVC VIP -j RETURN"}, }, "filter": {}, From c55c557a9457705fea61ca1cf80325d31531c19b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Caama=C3=B1o=20Ruiz?= Date: Fri, 21 Apr 2023 16:01:06 +0000 Subject: [PATCH 38/40] Temporary fix for metallb issue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jaime Caamaño Ruiz --- contrib/kind.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/contrib/kind.sh b/contrib/kind.sh index 04849a6be6..c2ae4e3721 100755 --- a/contrib/kind.sh +++ b/contrib/kind.sh @@ -886,6 +886,9 @@ install_metallb() { fi git clone https://github.com/metallb/metallb.git pushd metallb + # temporary fix for metallb issue + # https://github.com/metallb/metallb/commit/fdf92741c7fac20eedf3caa0aa922f9ff0f0e7dd#r110009241 + git reset --hard f5ba918 pip install -r dev-env/requirements.txt inv dev-env -n ovn -b frr -p bgp docker network create --driver bridge clientnet From 3b57312d98d0e2ad99bba4f83a13dec4829a0354 Mon Sep 17 00:00:00 2001 From: Surya Seetharaman Date: Wed, 31 May 2023 23:23:40 +0200 Subject: [PATCH 39/40] Fix cleanupStalePodSNATs if no podIPs are found In CI we are observing: E0531 19:52:52.309098 1 obj_retry.go:627] Failed to update *v1.Node, old=ip-10-0-133-36.ec2.internal, new=ip-10-0-133-36.ec2.internal, error: error creating gateway for node ip-10-0-133-36.ec2.internal: failed to init shared interface gateway: failed to sync stale SNATs on node ip-10-0-133-36.ec2.internal: unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-cr75v: pod openshift-multus/network-metrics-daemon-cr75v: no pod IPs found I0531 19:52:52.309185 1 event.go:285] Event(v1.ObjectReference{Kind:"Node", Namespace:"", Name:"ip-10-0-133-36.ec2.internal", UID:"65797716-b2a8-43fc-a216-8154fecee781", APIVersion:"v1", ResourceVersion:"203498", FieldPath:""}): type: 'Warning' reason: 'ErrorUpdatingResource' error creating gateway for node ip-10-0-133-36.ec2.internal: failed to init shared interface gateway: failed to sync stale SNATs on node ip-10-0-133-36.ec2.internal: unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-cr75v: pod openshift-multus/network-metrics-daemon-cr75v: no pod IPs found which is happening because pod is scheduled but IP allocation has not happened and since this sync is called from watch nodes, node add does not succeed, so we have a chicken-egg problem where watch pods can't start. In reality we don't care about pods that don't have an IP because there is nothing to cleanup for that pod in that case. Signed-off-by: Surya Seetharaman (cherry picked from commit 91e8a3bf787168a26020a0fac66b6fb97cd97805) (cherry picked from commit fe63418d89e376a74b11013c37552decc9a3ffc5) --- go-controller/pkg/ovn/gateway_init.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/go-controller/pkg/ovn/gateway_init.go b/go-controller/pkg/ovn/gateway_init.go index dded7d0c15..9b163d6c90 100644 --- a/go-controller/pkg/ovn/gateway_init.go +++ b/go-controller/pkg/ovn/gateway_init.go @@ -10,6 +10,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog/v2" utilnet "k8s.io/utils/net" libovsdbclient "github.com/ovn-org/libovsdb/client" @@ -72,7 +73,15 @@ func (oc *DefaultNetworkController) cleanupStalePodSNATs(nodeName string, nodeIP } } podIPs, err := util.GetPodIPsOfNetwork(&pod, oc.NetInfo) - if err != nil { + if err != nil && errors.Is(err, util.ErrNoPodIPFound) { + // It is possible that the pod is scheduled during this time, but the LSP add or + // IP Allocation has not happened and it is waiting for the WatchPods to start + // after WatchNodes completes (This function is called during syncNodes). So since + // the pod doesn't have any IPs, there is no SNAT here to keep for this pod so we skip + // this pod from processing and move onto the next one. + klog.Warningf("Unable to fetch podIPs for pod %s/%s: %v", pod.Namespace, pod.Name, err) + continue // no-op + } else if err != nil { return fmt.Errorf("unable to fetch podIPs for pod %s/%s: %w", pod.Namespace, pod.Name, err) } for _, podIP := range podIPs { From 2390d0f7a72cda890a09168df46f19ffd8360503 Mon Sep 17 00:00:00 2001 From: Riccardo Ravaioli Date: Fri, 26 May 2023 14:55:45 +0200 Subject: [PATCH 40/40] Fix unit test for localWithFallback Signed-off-by: Riccardo Ravaioli (cherry picked from commit d556bde435423732f8547fc8b509986fc21af7b3) --- .../services/load_balancer_ocphack_test.go | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/go-controller/pkg/ovn/controller/services/load_balancer_ocphack_test.go b/go-controller/pkg/ovn/controller/services/load_balancer_ocphack_test.go index 61ce641298..c99ce70e3f 100644 --- a/go-controller/pkg/ovn/controller/services/load_balancer_ocphack_test.go +++ b/go-controller/pkg/ovn/controller/services/load_balancer_ocphack_test.go @@ -38,14 +38,14 @@ func Test_buildPerNodeLBs_OCPHackForDNS(t *testing.T) { defaultNodes := []nodeInfo{ { name: "node-a", - nodeIPs: []string{"10.0.0.1"}, + nodeIPs: []net.IP{net.ParseIP("10.0.0.1")}, gatewayRouterName: "gr-node-a", switchName: "switch-node-a", podSubnets: []net.IPNet{{IP: net.ParseIP("10.128.0.0"), Mask: net.CIDRMask(24, 32)}}, }, { name: "node-b", - nodeIPs: []string{"10.0.0.2"}, + nodeIPs: []net.IP{net.ParseIP("10.0.0.2")}, gatewayRouterName: "gr-node-b", switchName: "switch-node-b", podSubnets: []net.IPNet{{IP: net.ParseIP("10.128.1.0"), Mask: net.CIDRMask(24, 32)}}, @@ -90,8 +90,8 @@ func Test_buildPerNodeLBs_OCPHackForDNS(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"192.168.1.1", 80}, - Targets: []Addr{{"10.128.0.2", 8080}, {"10.128.1.2", 8080}}, + Source: Addr{"192.168.1.1", 80, nil}, + Targets: []Addr{{"10.128.0.2", 8080, nil}, {"10.128.1.2", 8080, nil}}, }, }, Opts: defaultOpts, @@ -103,8 +103,8 @@ func Test_buildPerNodeLBs_OCPHackForDNS(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"192.168.1.1", 80}, - Targets: []Addr{{"10.128.0.2", 8080}}, + Source: Addr{"192.168.1.1", 80, nil}, + Targets: []Addr{{"10.128.0.2", 8080, nil}}, }, }, Opts: defaultOpts, @@ -116,8 +116,8 @@ func Test_buildPerNodeLBs_OCPHackForDNS(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"192.168.1.1", 80}, - Targets: []Addr{{"10.128.1.2", 8080}}, + Source: Addr{"192.168.1.1", 80, nil}, + Targets: []Addr{{"10.128.1.2", 8080, nil}}, }, }, Opts: defaultOpts, @@ -189,14 +189,14 @@ func Test_buildPerNodeLBs_OCPHackForLocalWithFallback(t *testing.T) { defaultNodes := []nodeInfo{ { name: "node-a", - nodeIPs: []string{"10.0.0.1"}, + nodeIPs: []net.IP{net.ParseIP("10.0.0.1")}, gatewayRouterName: "gr-node-a", switchName: "switch-node-a", podSubnets: []net.IPNet{{IP: net.ParseIP("10.128.0.0"), Mask: net.CIDRMask(24, 32)}}, }, { name: "node-b", - nodeIPs: []string{"10.0.0.2"}, + nodeIPs: []net.IP{net.ParseIP("10.0.0.2")}, gatewayRouterName: "gr-node-b", switchName: "switch-node-b", podSubnets: []net.IPNet{{IP: net.ParseIP("10.128.1.0"), Mask: net.CIDRMask(24, 32)}},