diff --git a/Dockerfile.base b/Dockerfile.base index 2e17911795..f55e24c603 100644 --- a/Dockerfile.base +++ b/Dockerfile.base @@ -13,7 +13,7 @@ RUN dnf install -y --nodocs \ dnf clean all ARG ovsver=3.1.0-10.el9fdp -ARG ovnver=23.03.0-7.el9fdp +ARG ovnver=23.03.0-49.el9fdp RUN INSTALL_PKGS="iptables" && \ ovsver_short=$(echo "$ovsver" | cut -d'.' -f1,2) && \ diff --git a/dist/images/Dockerfile.fedora b/dist/images/Dockerfile.fedora index c14b487791..9cb7682022 100644 --- a/dist/images/Dockerfile.fedora +++ b/dist/images/Dockerfile.fedora @@ -15,7 +15,7 @@ USER root ENV PYTHONDONTWRITEBYTECODE yes -ARG ovnver=ovn-23.03.0-4.fc37 +ARG ovnver=ovn-23.03.0-16.fc37 # Automatically populated when using docker buildx ARG TARGETPLATFORM ARG BUILDPLATFORM diff --git a/go-controller/Makefile b/go-controller/Makefile index 3c7234f4a3..d192dc0a25 100644 --- a/go-controller/Makefile +++ b/go-controller/Makefile @@ -22,7 +22,7 @@ else CONTAINER_RUNTIME=docker endif CONTAINER_RUNNABLE ?= $(shell $(CONTAINER_RUNTIME) -v > /dev/null 2>&1; echo $$?) -OVN_VERSION ?= v22.03.0 +OVN_SCHEMA_VERSION ?= v23.03.0 ifeq ($(NOROOT),TRUE) C_ARGS = -e NOROOT=TRUE else @@ -88,7 +88,7 @@ else endif pkg/nbdb/ovn-nb.ovsschema: - curl -sSL https://raw.githubusercontent.com/ovn-org/ovn/$(OVN_VERSION)/ovn-nb.ovsschema -o $@ + curl -sSL https://raw.githubusercontent.com/ovn-org/ovn/$(OVN_SCHEMA_VERSION)/ovn-nb.ovsschema -o $@ pkg/sbdb/ovn-sb.ovsschema: - curl -sSL https://raw.githubusercontent.com/ovn-org/ovn/$(OVN_VERSION)/ovn-sb.ovsschema -o $@ + curl -sSL https://raw.githubusercontent.com/ovn-org/ovn/$(OVN_SCHEMA_VERSION)/ovn-sb.ovsschema -o $@ diff --git a/go-controller/pkg/libovsdbops/model.go b/go-controller/pkg/libovsdbops/model.go index d2c116d85f..2f800b59ea 100644 --- a/go-controller/pkg/libovsdbops/model.go +++ b/go-controller/pkg/libovsdbops/model.go @@ -62,6 +62,8 @@ func getUUID(model model.Model) string { return t.UUID case *nbdb.QoS: return t.UUID + case *nbdb.ChassisTemplateVar: + return t.UUID default: panic(fmt.Sprintf("getUUID: unknown model %T", t)) } @@ -117,6 +119,8 @@ func setUUID(model model.Model, uuid string) { t.UUID = uuid case *nbdb.QoS: t.UUID = uuid + case *nbdb.ChassisTemplateVar: + t.UUID = uuid default: panic(fmt.Sprintf("setUUID: unknown model %T", t)) } @@ -236,6 +240,11 @@ func copyIndexes(model model.Model) model.Model { return &nbdb.QoS{ UUID: t.UUID, } + case *nbdb.ChassisTemplateVar: + return &nbdb.ChassisTemplateVar{ + UUID: t.UUID, + Chassis: t.Chassis, + } default: panic(fmt.Sprintf("copyIndexes: unknown model %T", t)) } @@ -289,6 +298,8 @@ func getListFromModel(model model.Model) interface{} { return &[]*sbdb.MACBinding{} case *nbdb.QoS: return &[]nbdb.QoS{} + case *nbdb.ChassisTemplateVar: + return &[]*nbdb.ChassisTemplateVar{} default: panic(fmt.Sprintf("getModelList: unknown model %T", t)) } diff --git a/go-controller/pkg/libovsdbops/model_client.go b/go-controller/pkg/libovsdbops/model_client.go index e0102aa4db..9badfe11b9 100644 --- a/go-controller/pkg/libovsdbops/model_client.go +++ b/go-controller/pkg/libovsdbops/model_client.go @@ -86,6 +86,20 @@ func buildMutationsFromFields(fields []interface{}, mutator ovsdb.Mutator) ([]mo } continue } + // RFC 7047, section 5.1: a MutateOperationDelete is generated + // automatically for every updated key. + removeKeys := make([]string, 0, len(*v)) + for key := range *v { + removeKeys = append(removeKeys, key) + } + if len(removeKeys) > 0 { + mutation := model.Mutation{ + Field: field, + Mutator: ovsdb.MutateOperationDelete, + Value: removeKeys, + } + mutations = append(mutations, mutation) + } mutation := model.Mutation{ Field: field, Mutator: mutator, diff --git a/go-controller/pkg/libovsdbops/model_client_test.go b/go-controller/pkg/libovsdbops/model_client_test.go index c375c219fa..bb4d417450 100644 --- a/go-controller/pkg/libovsdbops/model_client_test.go +++ b/go-controller/pkg/libovsdbops/model_client_test.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/onsi/ginkgo" "reflect" + "sort" "testing" "github.com/onsi/gomega/types" @@ -1458,6 +1459,11 @@ func TestBuildMutationsFromFields(t *testing.T) { fields: []interface{}{&mapField}, mutator: ovsdb.MutateOperationInsert, mutations: []model.Mutation{ + { + Field: &mapField, + Mutator: ovsdb.MutateOperationDelete, + Value: []string{"", "key1", "key2"}, + }, { Field: &mapField, Mutator: ovsdb.MutateOperationInsert, @@ -1534,6 +1540,11 @@ func TestBuildMutationsFromFields(t *testing.T) { if err != nil && !tCase.err { t.Fatalf("%s: got unexpected error: %v", tCase.name, err) } + for _, m := range mutations { + if v, ok := m.Value.([]string); ok { + sort.Strings(v) + } + } if !reflect.DeepEqual(mutations, tCase.mutations) { t.Fatalf("%s: unexpected mutations, got: %+v expected: %+v", tCase.name, mutations, tCase.mutations) } diff --git a/go-controller/pkg/libovsdbops/template_var.go b/go-controller/pkg/libovsdbops/template_var.go new file mode 100644 index 0000000000..95d3f49d69 --- /dev/null +++ b/go-controller/pkg/libovsdbops/template_var.go @@ -0,0 +1,112 @@ +package libovsdbops + +import ( + "context" + + libovsdbclient "github.com/ovn-org/libovsdb/client" + libovsdb "github.com/ovn-org/libovsdb/ovsdb" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" +) + +type chassisTemplateVarPredicate func(*nbdb.ChassisTemplateVar) bool + +// ListTemplateVar looks up all chassis template variables. +func ListTemplateVar(nbClient libovsdbclient.Client) ([]*nbdb.ChassisTemplateVar, error) { + ctx, cancel := context.WithTimeout(context.Background(), types.OVSDBTimeout) + defer cancel() + + templatesList := []*nbdb.ChassisTemplateVar{} + err := nbClient.List(ctx, &templatesList) + return templatesList, err +} + +// CreateOrUpdateChassisTemplateVarOps creates or updates the provided +// 'template' variable and returns the corresponding ops. +func CreateOrUpdateChassisTemplateVarOps(nbClient libovsdbclient.Client, + ops []libovsdb.Operation, template *nbdb.ChassisTemplateVar) ([]libovsdb.Operation, error) { + + modelClient := newModelClient(nbClient) + return modelClient.CreateOrUpdateOps(ops, operationModel{ + Model: template, + OnModelMutations: []interface{}{&template.Variables}, + ErrNotFound: false, + BulkOp: false, + }) +} + +// deleteChassisTemplateVarVariablesOps removes the variables listed as +// keys of 'template.Variables' and returns the corresponding ops. +// It applies the mutation to all records that are selected by 'predicate'. +func deleteChassisTemplateVarVariablesOps(nbClient libovsdbclient.Client, + ops []libovsdb.Operation, template *nbdb.ChassisTemplateVar, + predicate chassisTemplateVarPredicate) ([]libovsdb.Operation, error) { + + deleteTemplate := &nbdb.ChassisTemplateVar{ + Chassis: template.Chassis, + Variables: map[string]string{}, + } + for name := range template.Variables { + deleteTemplate.Variables[name] = "" + } + modelClient := newModelClient(nbClient) + return modelClient.DeleteOps(ops, operationModel{ + Model: deleteTemplate, + ModelPredicate: predicate, + OnModelMutations: []interface{}{&deleteTemplate.Variables}, + ErrNotFound: false, + BulkOp: true, + }) +} + +// DeleteChassisTemplateVarVariablesOps removes all variables listed as +// keys of 'template.Variables' from the record matching the same chassis +// as 'template'. It returns the corresponding ops. +func DeleteChassisTemplateVarVariablesOps(nbClient libovsdbclient.Client, + ops []libovsdb.Operation, template *nbdb.ChassisTemplateVar) ([]libovsdb.Operation, error) { + + return deleteChassisTemplateVarVariablesOps(nbClient, ops, template, nil) +} + +// DeleteAllChassisTemplateVarVariables removes the variables listed as +// in 'varNames' and commits the transaction to the database. It applies +// the mutation to all records that contain these variable names. +func DeleteAllChassisTemplateVarVariables(nbClient libovsdbclient.Client, varNames []string) error { + deleteTemplateVar := &nbdb.ChassisTemplateVar{ + Variables: make(map[string]string, len(varNames)), + } + for _, name := range varNames { + deleteTemplateVar.Variables[name] = "" + } + ops, err := deleteChassisTemplateVarVariablesOps(nbClient, nil, deleteTemplateVar, + func(item *nbdb.ChassisTemplateVar) bool { + for _, name := range varNames { + if _, found := item.Variables[name]; found { + return true + } + } + return false + }) + if err != nil { + return err + } + + _, err = TransactAndCheck(nbClient, ops) + return err +} + +// DeleteChassisTemplateVar deletes all complete Chassis_Template_Var +// records matching 'templates'. +func DeleteChassisTemplateVar(nbClient libovsdbclient.Client, templates ...*nbdb.ChassisTemplateVar) error { + opModels := make([]operationModel, 0, len(templates)) + for i := range templates { + template := templates[i] + opModels = append(opModels, operationModel{ + Model: template, + ErrNotFound: false, + BulkOp: false, + }) + } + m := newModelClient(nbClient) + return m.Delete(opModels...) +} diff --git a/go-controller/pkg/nbdb/chassis_template_var.go b/go-controller/pkg/nbdb/chassis_template_var.go new file mode 100644 index 0000000000..602c3f5223 --- /dev/null +++ b/go-controller/pkg/nbdb/chassis_template_var.go @@ -0,0 +1,120 @@ +// Code generated by "libovsdb.modelgen" +// DO NOT EDIT. + +package nbdb + +import "github.com/ovn-org/libovsdb/model" + +const ChassisTemplateVarTable = "Chassis_Template_Var" + +// ChassisTemplateVar defines an object in Chassis_Template_Var table +type ChassisTemplateVar struct { + UUID string `ovsdb:"_uuid"` + Chassis string `ovsdb:"chassis"` + ExternalIDs map[string]string `ovsdb:"external_ids"` + Variables map[string]string `ovsdb:"variables"` +} + +func (a *ChassisTemplateVar) GetUUID() string { + return a.UUID +} + +func (a *ChassisTemplateVar) GetChassis() string { + return a.Chassis +} + +func (a *ChassisTemplateVar) GetExternalIDs() map[string]string { + return a.ExternalIDs +} + +func copyChassisTemplateVarExternalIDs(a map[string]string) map[string]string { + if a == nil { + return nil + } + b := make(map[string]string, len(a)) + for k, v := range a { + b[k] = v + } + return b +} + +func equalChassisTemplateVarExternalIDs(a, b map[string]string) bool { + if (a == nil) != (b == nil) { + return false + } + if len(a) != len(b) { + return false + } + for k, v := range a { + if w, ok := b[k]; !ok || v != w { + return false + } + } + return true +} + +func (a *ChassisTemplateVar) GetVariables() map[string]string { + return a.Variables +} + +func copyChassisTemplateVarVariables(a map[string]string) map[string]string { + if a == nil { + return nil + } + b := make(map[string]string, len(a)) + for k, v := range a { + b[k] = v + } + return b +} + +func equalChassisTemplateVarVariables(a, b map[string]string) bool { + if (a == nil) != (b == nil) { + return false + } + if len(a) != len(b) { + return false + } + for k, v := range a { + if w, ok := b[k]; !ok || v != w { + return false + } + } + return true +} + +func (a *ChassisTemplateVar) DeepCopyInto(b *ChassisTemplateVar) { + *b = *a + b.ExternalIDs = copyChassisTemplateVarExternalIDs(a.ExternalIDs) + b.Variables = copyChassisTemplateVarVariables(a.Variables) +} + +func (a *ChassisTemplateVar) DeepCopy() *ChassisTemplateVar { + b := new(ChassisTemplateVar) + a.DeepCopyInto(b) + return b +} + +func (a *ChassisTemplateVar) CloneModelInto(b model.Model) { + c := b.(*ChassisTemplateVar) + a.DeepCopyInto(c) +} + +func (a *ChassisTemplateVar) CloneModel() model.Model { + return a.DeepCopy() +} + +func (a *ChassisTemplateVar) Equals(b *ChassisTemplateVar) bool { + return a.UUID == b.UUID && + a.Chassis == b.Chassis && + equalChassisTemplateVarExternalIDs(a.ExternalIDs, b.ExternalIDs) && + equalChassisTemplateVarVariables(a.Variables, b.Variables) +} + +func (a *ChassisTemplateVar) EqualsModel(b model.Model) bool { + c := b.(*ChassisTemplateVar) + return a.Equals(c) +} + +var _ model.CloneableModel = &ChassisTemplateVar{} +var _ model.ComparableModel = &ChassisTemplateVar{} diff --git a/go-controller/pkg/nbdb/logical_switch_port.go b/go-controller/pkg/nbdb/logical_switch_port.go index 547bdaee7e..c048f76541 100644 --- a/go-controller/pkg/nbdb/logical_switch_port.go +++ b/go-controller/pkg/nbdb/logical_switch_port.go @@ -17,6 +17,7 @@ type LogicalSwitchPort struct { Enabled *bool `ovsdb:"enabled"` ExternalIDs map[string]string `ovsdb:"external_ids"` HaChassisGroup *string `ovsdb:"ha_chassis_group"` + MirrorRules []string `ovsdb:"mirror_rules"` Name string `ovsdb:"name"` Options map[string]string `ovsdb:"options"` ParentName *string `ovsdb:"parent_name"` @@ -199,6 +200,34 @@ func equalLogicalSwitchPortHaChassisGroup(a, b *string) bool { return *a == *b } +func (a *LogicalSwitchPort) GetMirrorRules() []string { + return a.MirrorRules +} + +func copyLogicalSwitchPortMirrorRules(a []string) []string { + if a == nil { + return nil + } + b := make([]string, len(a)) + copy(b, a) + return b +} + +func equalLogicalSwitchPortMirrorRules(a, b []string) bool { + if (a == nil) != (b == nil) { + return false + } + if len(a) != len(b) { + return false + } + for i, v := range a { + if b[i] != v { + return false + } + } + return true +} + func (a *LogicalSwitchPort) GetName() string { return a.Name } @@ -362,6 +391,7 @@ func (a *LogicalSwitchPort) DeepCopyInto(b *LogicalSwitchPort) { b.Enabled = copyLogicalSwitchPortEnabled(a.Enabled) b.ExternalIDs = copyLogicalSwitchPortExternalIDs(a.ExternalIDs) b.HaChassisGroup = copyLogicalSwitchPortHaChassisGroup(a.HaChassisGroup) + b.MirrorRules = copyLogicalSwitchPortMirrorRules(a.MirrorRules) b.Options = copyLogicalSwitchPortOptions(a.Options) b.ParentName = copyLogicalSwitchPortParentName(a.ParentName) b.PortSecurity = copyLogicalSwitchPortPortSecurity(a.PortSecurity) @@ -394,6 +424,7 @@ func (a *LogicalSwitchPort) Equals(b *LogicalSwitchPort) bool { equalLogicalSwitchPortEnabled(a.Enabled, b.Enabled) && equalLogicalSwitchPortExternalIDs(a.ExternalIDs, b.ExternalIDs) && equalLogicalSwitchPortHaChassisGroup(a.HaChassisGroup, b.HaChassisGroup) && + equalLogicalSwitchPortMirrorRules(a.MirrorRules, b.MirrorRules) && a.Name == b.Name && equalLogicalSwitchPortOptions(a.Options, b.Options) && equalLogicalSwitchPortParentName(a.ParentName, b.ParentName) && diff --git a/go-controller/pkg/nbdb/mirror.go b/go-controller/pkg/nbdb/mirror.go new file mode 100644 index 0000000000..7b9e237996 --- /dev/null +++ b/go-controller/pkg/nbdb/mirror.go @@ -0,0 +1,123 @@ +// Code generated by "libovsdb.modelgen" +// DO NOT EDIT. + +package nbdb + +import "github.com/ovn-org/libovsdb/model" + +const MirrorTable = "Mirror" + +type ( + MirrorFilter = string + MirrorType = string +) + +var ( + MirrorFilterFromLport MirrorFilter = "from-lport" + MirrorFilterToLport MirrorFilter = "to-lport" + MirrorTypeGre MirrorType = "gre" + MirrorTypeErspan MirrorType = "erspan" +) + +// Mirror defines an object in Mirror table +type Mirror struct { + UUID string `ovsdb:"_uuid"` + ExternalIDs map[string]string `ovsdb:"external_ids"` + Filter MirrorFilter `ovsdb:"filter"` + Index int `ovsdb:"index"` + Name string `ovsdb:"name"` + Sink string `ovsdb:"sink"` + Type MirrorType `ovsdb:"type"` +} + +func (a *Mirror) GetUUID() string { + return a.UUID +} + +func (a *Mirror) GetExternalIDs() map[string]string { + return a.ExternalIDs +} + +func copyMirrorExternalIDs(a map[string]string) map[string]string { + if a == nil { + return nil + } + b := make(map[string]string, len(a)) + for k, v := range a { + b[k] = v + } + return b +} + +func equalMirrorExternalIDs(a, b map[string]string) bool { + if (a == nil) != (b == nil) { + return false + } + if len(a) != len(b) { + return false + } + for k, v := range a { + if w, ok := b[k]; !ok || v != w { + return false + } + } + return true +} + +func (a *Mirror) GetFilter() MirrorFilter { + return a.Filter +} + +func (a *Mirror) GetIndex() int { + return a.Index +} + +func (a *Mirror) GetName() string { + return a.Name +} + +func (a *Mirror) GetSink() string { + return a.Sink +} + +func (a *Mirror) GetType() MirrorType { + return a.Type +} + +func (a *Mirror) DeepCopyInto(b *Mirror) { + *b = *a + b.ExternalIDs = copyMirrorExternalIDs(a.ExternalIDs) +} + +func (a *Mirror) DeepCopy() *Mirror { + b := new(Mirror) + a.DeepCopyInto(b) + return b +} + +func (a *Mirror) CloneModelInto(b model.Model) { + c := b.(*Mirror) + a.DeepCopyInto(c) +} + +func (a *Mirror) CloneModel() model.Model { + return a.DeepCopy() +} + +func (a *Mirror) Equals(b *Mirror) bool { + return a.UUID == b.UUID && + equalMirrorExternalIDs(a.ExternalIDs, b.ExternalIDs) && + a.Filter == b.Filter && + a.Index == b.Index && + a.Name == b.Name && + a.Sink == b.Sink && + a.Type == b.Type +} + +func (a *Mirror) EqualsModel(b model.Model) bool { + c := b.(*Mirror) + return a.Equals(c) +} + +var _ model.CloneableModel = &Mirror{} +var _ model.ComparableModel = &Mirror{} diff --git a/go-controller/pkg/nbdb/model.go b/go-controller/pkg/nbdb/model.go index 3381a37cde..e8f3009726 100644 --- a/go-controller/pkg/nbdb/model.go +++ b/go-controller/pkg/nbdb/model.go @@ -16,6 +16,7 @@ func FullDatabaseModel() (model.ClientDBModel, error) { "ACL": &ACL{}, "Address_Set": &AddressSet{}, "BFD": &BFD{}, + "Chassis_Template_Var": &ChassisTemplateVar{}, "Connection": &Connection{}, "Copp": &Copp{}, "DHCP_Options": &DHCPOptions{}, @@ -35,17 +36,19 @@ func FullDatabaseModel() (model.ClientDBModel, error) { "Logical_Switch_Port": &LogicalSwitchPort{}, "Meter": &Meter{}, "Meter_Band": &MeterBand{}, + "Mirror": &Mirror{}, "NAT": &NAT{}, "NB_Global": &NBGlobal{}, "Port_Group": &PortGroup{}, "QoS": &QoS{}, "SSL": &SSL{}, + "Static_MAC_Binding": &StaticMACBinding{}, }) } var schema = `{ "name": "OVN_Northbound", - "version": "6.1.0", + "version": "7.0.0", "tables": { "ACL": { "columns": { @@ -289,6 +292,42 @@ var schema = `{ ] ] }, + "Chassis_Template_Var": { + "columns": { + "chassis": { + "type": "string" + }, + "external_ids": { + "type": { + "key": { + "type": "string" + }, + "value": { + "type": "string" + }, + "min": 0, + "max": "unlimited" + } + }, + "variables": { + "type": { + "key": { + "type": "string" + }, + "value": { + "type": "string" + }, + "min": 0, + "max": "unlimited" + } + } + }, + "indexes": [ + [ + "chassis" + ] + ] + }, "Connection": { "columns": { "external_ids": { @@ -1324,6 +1363,17 @@ var schema = `{ "max": 1 } }, + "mirror_rules": { + "type": { + "key": { + "type": "uuid", + "refTable": "Mirror", + "refType": "weak" + }, + "min": 0, + "max": "unlimited" + } + }, "name": { "type": "string" }, @@ -1498,6 +1548,64 @@ var schema = `{ } } }, + "Mirror": { + "columns": { + "external_ids": { + "type": { + "key": { + "type": "string" + }, + "value": { + "type": "string" + }, + "min": 0, + "max": "unlimited" + } + }, + "filter": { + "type": { + "key": { + "type": "string", + "enum": [ + "set", + [ + "from-lport", + "to-lport" + ] + ] + } + } + }, + "index": { + "type": "integer" + }, + "name": { + "type": "string" + }, + "sink": { + "type": "string" + }, + "type": { + "type": { + "key": { + "type": "string", + "enum": [ + "set", + [ + "gre", + "erspan" + ] + ] + } + } + } + }, + "indexes": [ + [ + "name" + ] + ] + }, "NAT": { "columns": { "allowed_ext_ips": { @@ -1549,6 +1657,17 @@ var schema = `{ "external_port_range": { "type": "string" }, + "gateway_port": { + "type": { + "key": { + "type": "uuid", + "refTable": "Logical_Router_Port", + "refType": "weak" + }, + "min": 0, + "max": 1 + } + }, "logical_ip": { "type": "string" }, @@ -1819,6 +1938,28 @@ var schema = `{ "type": "string" } } + }, + "Static_MAC_Binding": { + "columns": { + "ip": { + "type": "string" + }, + "logical_port": { + "type": "string" + }, + "mac": { + "type": "string" + }, + "override_dynamic_mac": { + "type": "boolean" + } + }, + "indexes": [ + [ + "logical_port", + "ip" + ] + ] } } }` diff --git a/go-controller/pkg/nbdb/nat.go b/go-controller/pkg/nbdb/nat.go index 2f08b862fd..3286bbde2c 100644 --- a/go-controller/pkg/nbdb/nat.go +++ b/go-controller/pkg/nbdb/nat.go @@ -26,6 +26,7 @@ type NAT struct { ExternalIP string `ovsdb:"external_ip"` ExternalMAC *string `ovsdb:"external_mac"` ExternalPortRange string `ovsdb:"external_port_range"` + GatewayPort *string `ovsdb:"gateway_port"` LogicalIP string `ovsdb:"logical_ip"` LogicalPort *string `ovsdb:"logical_port"` Options map[string]string `ovsdb:"options"` @@ -140,6 +141,28 @@ func (a *NAT) GetExternalPortRange() string { return a.ExternalPortRange } +func (a *NAT) GetGatewayPort() *string { + return a.GatewayPort +} + +func copyNATGatewayPort(a *string) *string { + if a == nil { + return nil + } + b := *a + return &b +} + +func equalNATGatewayPort(a, b *string) bool { + if (a == nil) != (b == nil) { + return false + } + if a == b { + return true + } + return *a == *b +} + func (a *NAT) GetLogicalIP() string { return a.LogicalIP } @@ -206,6 +229,7 @@ func (a *NAT) DeepCopyInto(b *NAT) { b.ExemptedExtIPs = copyNATExemptedExtIPs(a.ExemptedExtIPs) b.ExternalIDs = copyNATExternalIDs(a.ExternalIDs) b.ExternalMAC = copyNATExternalMAC(a.ExternalMAC) + b.GatewayPort = copyNATGatewayPort(a.GatewayPort) b.LogicalPort = copyNATLogicalPort(a.LogicalPort) b.Options = copyNATOptions(a.Options) } @@ -233,6 +257,7 @@ func (a *NAT) Equals(b *NAT) bool { a.ExternalIP == b.ExternalIP && equalNATExternalMAC(a.ExternalMAC, b.ExternalMAC) && a.ExternalPortRange == b.ExternalPortRange && + equalNATGatewayPort(a.GatewayPort, b.GatewayPort) && a.LogicalIP == b.LogicalIP && equalNATLogicalPort(a.LogicalPort, b.LogicalPort) && equalNATOptions(a.Options, b.Options) && diff --git a/go-controller/pkg/nbdb/static_mac_binding.go b/go-controller/pkg/nbdb/static_mac_binding.go new file mode 100644 index 0000000000..15207e6484 --- /dev/null +++ b/go-controller/pkg/nbdb/static_mac_binding.go @@ -0,0 +1,72 @@ +// Code generated by "libovsdb.modelgen" +// DO NOT EDIT. + +package nbdb + +import "github.com/ovn-org/libovsdb/model" + +const StaticMACBindingTable = "Static_MAC_Binding" + +// StaticMACBinding defines an object in Static_MAC_Binding table +type StaticMACBinding struct { + UUID string `ovsdb:"_uuid"` + IP string `ovsdb:"ip"` + LogicalPort string `ovsdb:"logical_port"` + MAC string `ovsdb:"mac"` + OverrideDynamicMAC bool `ovsdb:"override_dynamic_mac"` +} + +func (a *StaticMACBinding) GetUUID() string { + return a.UUID +} + +func (a *StaticMACBinding) GetIP() string { + return a.IP +} + +func (a *StaticMACBinding) GetLogicalPort() string { + return a.LogicalPort +} + +func (a *StaticMACBinding) GetMAC() string { + return a.MAC +} + +func (a *StaticMACBinding) GetOverrideDynamicMAC() bool { + return a.OverrideDynamicMAC +} + +func (a *StaticMACBinding) DeepCopyInto(b *StaticMACBinding) { + *b = *a +} + +func (a *StaticMACBinding) DeepCopy() *StaticMACBinding { + b := new(StaticMACBinding) + a.DeepCopyInto(b) + return b +} + +func (a *StaticMACBinding) CloneModelInto(b model.Model) { + c := b.(*StaticMACBinding) + a.DeepCopyInto(c) +} + +func (a *StaticMACBinding) CloneModel() model.Model { + return a.DeepCopy() +} + +func (a *StaticMACBinding) Equals(b *StaticMACBinding) bool { + return a.UUID == b.UUID && + a.IP == b.IP && + a.LogicalPort == b.LogicalPort && + a.MAC == b.MAC && + a.OverrideDynamicMAC == b.OverrideDynamicMAC +} + +func (a *StaticMACBinding) EqualsModel(b model.Model) bool { + c := b.(*StaticMACBinding) + return a.Equals(c) +} + +var _ model.CloneableModel = &StaticMACBinding{} +var _ model.ComparableModel = &StaticMACBinding{} diff --git a/go-controller/pkg/network-controller-manager/network_controller_manager.go b/go-controller/pkg/network-controller-manager/network_controller_manager.go index 8ea05c91d1..285d09dd80 100644 --- a/go-controller/pkg/network-controller-manager/network_controller_manager.go +++ b/go-controller/pkg/network-controller-manager/network_controller_manager.go @@ -41,6 +41,8 @@ type networkControllerManager struct { SCTPSupport bool // Supports multicast? multicastSupport bool + // Supports OVN Template Load Balancers? + svcTemplateSupport bool stopChan chan struct{} wg *sync.WaitGroup @@ -230,6 +232,16 @@ func (cm *networkControllerManager) configureMulticastSupport() { } } +func (cm *networkControllerManager) configureSvcTemplateSupport() { + if _, _, err := util.RunOVNNbctl("--columns=_uuid", "list", "Chassis_Template_Var"); err != nil { + klog.Warningf("Version of OVN in use does not support Chassis_Template_Var. " + + "Disabling Templates Support") + cm.svcTemplateSupport = false + } else { + cm.svcTemplateSupport = true + } +} + // enableOVNLogicalDataPathGroups sets an OVN flag to enable logical datapath // groups on OVN 20.12 and later. The option is ignored if OVN doesn't // understand it. Logical datapath groups reduce the size of the southbound @@ -255,7 +267,7 @@ func (cm *networkControllerManager) configureMetrics(stopChan <-chan struct{}) { // newCommonNetworkControllerInfo creates and returns the common networkController info func (cm *networkControllerManager) newCommonNetworkControllerInfo() (*ovn.CommonNetworkControllerInfo, error) { return ovn.NewCommonNetworkControllerInfo(cm.client, cm.kube, cm.watchFactory, cm.recorder, cm.nbClient, - cm.sbClient, cm.podRecorder, cm.SCTPSupport, cm.multicastSupport) + cm.sbClient, cm.podRecorder, cm.SCTPSupport, cm.multicastSupport, cm.svcTemplateSupport) } // initDefaultNetworkController creates the controller for default network @@ -282,6 +294,7 @@ func (cm *networkControllerManager) Start(ctx context.Context) error { } cm.configureMulticastSupport() + cm.configureSvcTemplateSupport() err = cm.enableOVNLogicalDataPathGroups() if err != nil { diff --git a/go-controller/pkg/ovn/base_network_controller.go b/go-controller/pkg/ovn/base_network_controller.go index 1e28c80465..766d24d0b5 100644 --- a/go-controller/pkg/ovn/base_network_controller.go +++ b/go-controller/pkg/ovn/base_network_controller.go @@ -56,6 +56,9 @@ type CommonNetworkControllerInfo struct { // Supports multicast? multicastSupport bool + + // Supports OVN Template Load Balancers? + svcTemplateSupport bool } // BaseNetworkController structure holds per-network fields and network specific configuration @@ -116,17 +119,18 @@ type BaseSecondaryNetworkController struct { // NewCommonNetworkControllerInfo creates CommonNetworkControllerInfo shared by controllers func NewCommonNetworkControllerInfo(client clientset.Interface, kube *kube.KubeOVN, wf *factory.WatchFactory, recorder record.EventRecorder, nbClient libovsdbclient.Client, sbClient libovsdbclient.Client, - podRecorder *metrics.PodRecorder, SCTPSupport, multicastSupport bool) (*CommonNetworkControllerInfo, error) { + podRecorder *metrics.PodRecorder, SCTPSupport, multicastSupport, svcTemplateSupport bool) (*CommonNetworkControllerInfo, error) { return &CommonNetworkControllerInfo{ - client: client, - kube: kube, - watchFactory: wf, - recorder: recorder, - nbClient: nbClient, - sbClient: sbClient, - podRecorder: podRecorder, - SCTPSupport: SCTPSupport, - multicastSupport: multicastSupport, + client: client, + kube: kube, + watchFactory: wf, + recorder: recorder, + nbClient: nbClient, + sbClient: sbClient, + podRecorder: podRecorder, + SCTPSupport: SCTPSupport, + multicastSupport: multicastSupport, + svcTemplateSupport: svcTemplateSupport, }, nil } @@ -252,7 +256,7 @@ func (bnc *BaseNetworkController) syncNodeClusterRouterPort(node *kapi.Node, hos } func (bnc *BaseNetworkController) createNodeLogicalSwitch(nodeName string, hostSubnets []*net.IPNet, - loadBalancerGroupUUID string) error { + clusterLoadBalancerGroupUUID, switchLoadBalancerGroupUUID string) error { // logical router port MAC is based on IPv4 subnet if there is one, else IPv6 var nodeLRPMAC net.HardwareAddr switchName := bnc.GetNetworkScopedName(nodeName) @@ -297,8 +301,8 @@ func (bnc *BaseNetworkController) createNodeLogicalSwitch(nodeName string, hostS } } - if loadBalancerGroupUUID != "" { - logicalSwitch.LoadBalancerGroup = []string{loadBalancerGroupUUID} + if clusterLoadBalancerGroupUUID != "" && switchLoadBalancerGroupUUID != "" { + logicalSwitch.LoadBalancerGroup = []string{clusterLoadBalancerGroupUUID, switchLoadBalancerGroupUUID} } // If supported, enable IGMP/MLD snooping and querier on the node. diff --git a/go-controller/pkg/ovn/controller/services/lb_config.go b/go-controller/pkg/ovn/controller/services/lb_config.go index 88d2b385ac..fe355b76f4 100644 --- a/go-controller/pkg/ovn/controller/services/lb_config.go +++ b/go-controller/pkg/ovn/controller/services/lb_config.go @@ -40,6 +40,72 @@ type lbConfig struct { hasNodePort bool } +func (c *lbConfig) makeNodeSwitchTargetIPs(service *v1.Service, node *nodeInfo, epIPs []string) (targetIPs []string, changed bool) { + targetIPs = epIPs + changed = false + + if c.externalTrafficLocal { + // for ExternalTrafficPolicy=Local, remove non-local endpoints from the router/switch targets + // NOTE: on the switches, filtered eps are used only by masqueradeVIP + targetIPs = util.FilterIPsSlice(targetIPs, node.nodeSubnets(), true) + } + + if c.internalTrafficLocal { + // for InternalTrafficPolicy=Local, remove non-local endpoints from the switch targets only + targetIPs = util.FilterIPsSlice(targetIPs, node.nodeSubnets(), true) + } + + // OCP HACK BEGIN + if _, set := service.Annotations[localWithFallbackAnnotation]; set && c.externalTrafficLocal { + // if service is annotated and is ETP=local, fallback to ETP=cluster on nodes with no local endpoints: + // include endpoints from other nodes + if len(targetIPs) == 0 { + targetIPs = epIPs + } + } + // OCP HACK END + + // We potentially only removed stuff from the original slice, so just + // comparing lenghts is enough. + if len(targetIPs) != len(epIPs) { + changed = true + } + return +} + +func (c *lbConfig) makeNodeRouterTargetIPs(service *v1.Service, node *nodeInfo, epIPs []string, hostMasqueradeIP string) (targetIPs []string, changed, zeroRouterLocalEndpoints bool) { + targetIPs = epIPs + changed = false + + if c.externalTrafficLocal { + // for ExternalTrafficPolicy=Local, remove non-local endpoints from the router/switch targets + // NOTE: on the switches, filtered eps are used only by masqueradeVIP + targetIPs = util.FilterIPsSlice(targetIPs, node.nodeSubnets(), true) + } + + // OCP HACK BEGIN + zeroRouterLocalEndpoints = false + if _, set := service.Annotations[localWithFallbackAnnotation]; set && c.externalTrafficLocal { + // if service is annotated and is ETP=local, fallback to ETP=cluster on nodes with no local endpoints: + // include endpoints from other nodes + if len(targetIPs) == 0 { + zeroRouterLocalEndpoints = true + targetIPs = epIPs + } + } + // OCP HACK END + + // any targets local to the node need to have a special + // harpin IP added, but only for the router LB + targetIPs, updated := util.UpdateIPsSlice(targetIPs, node.nodeIPsStr(), []string{hostMasqueradeIP}) + + // We either only removed stuff from the original slice, or updated some IPs. + if len(targetIPs) != len(epIPs) || updated { + changed = true + } + return +} + // just used for consistent ordering var protos = []v1.Protocol{ v1.ProtocolTCP, @@ -50,8 +116,9 @@ var protos = []v1.Protocol{ // buildServiceLBConfigs generates the abstract load balancer(s) configurations for each service. The abstract configurations // are then expanded in buildClusterLBs and buildPerNodeLBs to the full list of OVN LBs desired. // -// It creates two lists of configurations: +// It creates three lists of configurations: // - the per-node configs, which are load balancers that, for some reason must be expanded per-node. (see below for why) +// - the template configs, which are template load balancers that are similar across the whole cluster. // - the cluster-wide configs, which are load balancers that can be the same across the whole cluster. // // For a "standard" ClusterIP service (possibly with ExternalIPS or external LoadBalancer Status IPs), @@ -62,7 +129,13 @@ var protos = []v1.Protocol{ // - services with host-network endpoints // - services with ExternalTrafficPolicy=Local // - services with InternalTrafficPolicy=Local -func buildServiceLBConfigs(service *v1.Service, endpointSlices []*discovery.EndpointSlice) (perNodeConfigs []lbConfig, clusterConfigs []lbConfig) { +// +// Template LBs will be created for +// - services with NodePort set but *without* ExternalTrafficPolicy=Local or +// affinity timeout set. +func buildServiceLBConfigs(service *v1.Service, endpointSlices []*discovery.EndpointSlice, useLBGroup, useTemplates bool) (perNodeConfigs, templateConfigs, clusterConfigs []lbConfig) { + needsAffinityTimeout := hasSessionAffinityTimeOut(service) + // For each svcPort, determine if it will be applied per-node or cluster-wide for _, svcPort := range service.Spec.Ports { eps := util.GetLbEndpoints(endpointSlices, svcPort, service.Spec.PublishNotReadyAddresses) @@ -84,7 +157,14 @@ func buildServiceLBConfigs(service *v1.Service, endpointSlices []*discovery.Endp internalTrafficLocal: false, // always false for non-ClusterIPs hasNodePort: true, } - perNodeConfigs = append(perNodeConfigs, nodePortLBConfig) + // Only "plain" NodePort services (no ETP, no affinity timeout) + // can use load balancer templates. + if !useLBGroup || !useTemplates || externalTrafficLocal || + needsAffinityTimeout { + perNodeConfigs = append(perNodeConfigs, nodePortLBConfig) + } else { + templateConfigs = append(templateConfigs, nodePortLBConfig) + } } // Build up list of vips and externalVips @@ -248,6 +328,221 @@ func buildClusterLBs(service *v1.Service, configs []lbConfig, nodeInfos []nodeIn return out } +// buildTemplateLBs takes a list of lbConfigs and expands them to one template +// LB per protocol (per address family). +// +// Template LBs are created for nodeport services and are attached to each +// node's gateway router + switch via load balancer groups. Their vips and +// backends are OVN chassis template variables that expand to the chassis' +// node IP and set of backends. +// +// Note: +// NodePort services with ETP=local or affinity timeout set still need +// non-template per-node LBs. +func buildTemplateLBs(service *v1.Service, configs []lbConfig, nodes []nodeInfo, + nodeIPv4Template, nodeIPv6Template *Template) []LB { + + cbp := configsByProto(configs) + eids := util.ExternalIDsForObject(service) + out := make([]LB, 0, len(configs)) + + for _, proto := range protos { + configs, ok := cbp[proto] + if !ok { + continue + } + + switchV4Rules := make([]LBRule, 0, len(configs)) + switchV6Rules := make([]LBRule, 0, len(configs)) + routerV4Rules := make([]LBRule, 0, len(configs)) + routerV6Rules := make([]LBRule, 0, len(configs)) + + optsV4 := lbTemplateOpts(service, v1.IPv4Protocol) + optsV6 := lbTemplateOpts(service, v1.IPv6Protocol) + + for _, config := range configs { + switchV4TemplateTarget := + makeTemplate( + makeLBTargetTemplateName( + service, proto, config.inport, + optsV4.AddressFamily, "node_switch_template")) + switchV6TemplateTarget := + makeTemplate( + makeLBTargetTemplateName( + service, proto, config.inport, + optsV6.AddressFamily, "node_switch_template")) + + routerV4TemplateTarget := + makeTemplate( + makeLBTargetTemplateName( + service, proto, config.inport, + optsV4.AddressFamily, "node_router_template")) + routerV6TemplateTarget := + makeTemplate( + makeLBTargetTemplateName( + service, proto, config.inport, + optsV6.AddressFamily, "node_router_template")) + + for range config.vips { + klog.V(5).Infof(" buildTemplateLBs() service %s/%s adding rules", service.Namespace, service.Name) + + // If all targets have exactly the same IPs on all nodes there's + // no need to use a template, just use the same list of explicit + // targets on all nodes. + switchV4TargetNeedsTemplate := false + switchV6TargetNeedsTemplate := false + routerV4TargetNeedsTemplate := false + routerV6TargetNeedsTemplate := false + + for _, node := range nodes { + switchV4targetips, changed := config.makeNodeSwitchTargetIPs(service, &node, config.eps.V4IPs) + if !switchV4TargetNeedsTemplate && changed { + switchV4TargetNeedsTemplate = true + } + switchV6targetips, changed := config.makeNodeSwitchTargetIPs(service, &node, config.eps.V6IPs) + if !switchV6TargetNeedsTemplate && changed { + switchV6TargetNeedsTemplate = true + } + + routerV4targetips, changed, _ := config.makeNodeRouterTargetIPs(service, &node, config.eps.V4IPs, types.V4HostMasqueradeIP) + if !routerV4TargetNeedsTemplate && changed { + routerV4TargetNeedsTemplate = true + } + routerV6targetips, changed, _ := config.makeNodeRouterTargetIPs(service, &node, config.eps.V6IPs, types.V6HostMasqueradeIP) + if !routerV6TargetNeedsTemplate && changed { + routerV6TargetNeedsTemplate = true + } + + switchV4TemplateTarget.Value[node.chassisID] = addrsToString( + joinHostsPort(switchV4targetips, config.eps.Port)) + switchV6TemplateTarget.Value[node.chassisID] = addrsToString( + joinHostsPort(switchV6targetips, config.eps.Port)) + + routerV4TemplateTarget.Value[node.chassisID] = addrsToString( + joinHostsPort(routerV4targetips, config.eps.Port)) + routerV6TemplateTarget.Value[node.chassisID] = addrsToString( + joinHostsPort(routerV6targetips, config.eps.Port)) + } + + sharedV4Targets := []Addr{} + sharedV6Targets := []Addr{} + if !switchV4TargetNeedsTemplate || !routerV4TargetNeedsTemplate { + sharedV4Targets = joinHostsPort(config.eps.V4IPs, config.eps.Port) + } + if !switchV6TargetNeedsTemplate || !routerV6TargetNeedsTemplate { + sharedV6Targets = joinHostsPort(config.eps.V6IPs, config.eps.Port) + } + + if switchV4TargetNeedsTemplate { + switchV4Rules = append(switchV4Rules, LBRule{ + Source: Addr{Template: nodeIPv4Template, Port: config.inport}, + Targets: []Addr{{Template: switchV4TemplateTarget}}, + }) + } else { + switchV4Rules = append(switchV4Rules, LBRule{ + Source: Addr{Template: nodeIPv4Template, Port: config.inport}, + Targets: sharedV4Targets, + }) + } + + if switchV6TargetNeedsTemplate { + switchV6Rules = append(switchV6Rules, LBRule{ + Source: Addr{Template: nodeIPv6Template, Port: config.inport}, + Targets: []Addr{{Template: switchV6TemplateTarget}}, + }) + } else { + switchV6Rules = append(switchV6Rules, LBRule{ + Source: Addr{Template: nodeIPv6Template, Port: config.inport}, + Targets: sharedV6Targets, + }) + } + + if routerV4TargetNeedsTemplate { + routerV4Rules = append(routerV4Rules, LBRule{ + Source: Addr{Template: nodeIPv4Template, Port: config.inport}, + Targets: []Addr{{Template: routerV4TemplateTarget}}, + }) + } else { + routerV4Rules = append(routerV4Rules, LBRule{ + Source: Addr{Template: nodeIPv4Template, Port: config.inport}, + Targets: sharedV4Targets, + }) + } + + if routerV6TargetNeedsTemplate { + routerV6Rules = append(routerV6Rules, LBRule{ + Source: Addr{Template: nodeIPv6Template, Port: config.inport}, + Targets: []Addr{{Template: routerV6TemplateTarget}}, + }) + } else { + routerV6Rules = append(routerV6Rules, LBRule{ + Source: Addr{Template: nodeIPv6Template, Port: config.inport}, + Targets: sharedV6Targets, + }) + } + } + } + + if nodeIPv4Template.len() > 0 { + if len(switchV4Rules) > 0 { + out = append(out, LB{ + Name: makeLBName(service, proto, "node_switch_template_IPv4"), + Protocol: string(proto), + ExternalIDs: eids, + Opts: optsV4, + Groups: []string{types.ClusterSwitchLBGroupName}, + Rules: switchV4Rules, + Templates: getTemplatesFromRulesTargets(switchV4Rules), + }) + } + if len(routerV4Rules) > 0 { + out = append(out, LB{ + Name: makeLBName(service, proto, "node_router_template_IPv4"), + Protocol: string(proto), + ExternalIDs: eids, + Opts: optsV4, + Groups: []string{types.ClusterRouterLBGroupName}, + Rules: routerV4Rules, + Templates: getTemplatesFromRulesTargets(routerV4Rules), + }) + } + } + if nodeIPv6Template.len() > 0 { + if len(switchV6Rules) > 0 { + out = append(out, LB{ + Name: makeLBName(service, proto, "node_switch_template_IPv6"), + Protocol: string(proto), + ExternalIDs: eids, + Opts: optsV6, + Groups: []string{types.ClusterSwitchLBGroupName}, + Rules: switchV6Rules, + Templates: getTemplatesFromRulesTargets(switchV6Rules), + }) + } + if len(routerV6Rules) > 0 { + out = append(out, LB{ + Name: makeLBName(service, proto, "node_router_template_IPv6"), + Protocol: string(proto), + ExternalIDs: eids, + Opts: optsV6, + Groups: []string{types.ClusterRouterLBGroupName}, + Rules: routerV6Rules, + Templates: getTemplatesFromRulesTargets(routerV6Rules), + }) + } + } + } + + merged := mergeLBs(out) + if len(merged) != len(out) { + klog.V(5).Infof("Service %s/%s merged %d LBs to %d", + service.Namespace, service.Name, + len(out), len(merged)) + } + + return merged +} + // buildPerNodeLBs takes a list of lbConfigs and expands them to one LB per protocol per node // // Per-node lbs are created for @@ -288,60 +583,17 @@ func buildPerNodeLBs(service *v1.Service, configs []lbConfig, nodes []nodeInfo) switchRules := make([]LBRule, 0, len(configs)) for _, config := range configs { - vips := config.vips - - routerV4targetips := config.eps.V4IPs - routerV6targetips := config.eps.V6IPs - switchV4targetips := config.eps.V4IPs - switchV6targetips := config.eps.V6IPs - - if config.externalTrafficLocal { - // for ExternalTrafficPolicy=Local, remove non-local endpoints from the router/switch targets - // NOTE: on the switches, filtered eps are used only by masqueradeVIP - routerV4targetips = util.FilterIPsSlice(routerV4targetips, node.nodeSubnets(), true) - routerV6targetips = util.FilterIPsSlice(routerV6targetips, node.nodeSubnets(), true) - switchV4targetips = util.FilterIPsSlice(switchV4targetips, node.nodeSubnets(), true) - switchV6targetips = util.FilterIPsSlice(switchV6targetips, node.nodeSubnets(), true) - } - if config.internalTrafficLocal { - // for InternalTrafficPolicy=Local, remove non-local endpoints from the switch targets only - switchV4targetips = util.FilterIPsSlice(switchV4targetips, node.nodeSubnets(), true) - switchV6targetips = util.FilterIPsSlice(switchV6targetips, node.nodeSubnets(), true) - } - // OCP HACK BEGIN - zeroRouterV4LocalEndpoints := false - zeroRouterV6LocalEndpoints := false - if _, set := service.Annotations[localWithFallbackAnnotation]; set && config.externalTrafficLocal { - // if service is annotated and is ETP=local, fallback to ETP=cluster on nodes with no local endpoints: - // include endpoints from other nodes - if len(routerV4targetips) == 0 { - zeroRouterV4LocalEndpoints = true - routerV4targetips = config.eps.V4IPs - } - if len(routerV6targetips) == 0 { - zeroRouterV6LocalEndpoints = true - routerV6targetips = config.eps.V6IPs - } - if len(switchV4targetips) == 0 { - switchV4targetips = config.eps.V4IPs - } - if len(switchV6targetips) == 0 { - switchV6targetips = config.eps.V6IPs - } - } - // OCP HACK END - // at this point, the targets may be empty + switchV4targetips, _ := config.makeNodeSwitchTargetIPs(service, &node, config.eps.V4IPs) + switchV6targetips, _ := config.makeNodeSwitchTargetIPs(service, &node, config.eps.V6IPs) - // any targets local to the node need to have a special - // harpin IP added, but only for the router LB - routerV4targetips = util.UpdateIPsSlice(routerV4targetips, node.nodeIPs, []string{types.V4HostMasqueradeIP}) - routerV6targetips = util.UpdateIPsSlice(routerV6targetips, node.nodeIPs, []string{types.V6HostMasqueradeIP}) + routerV4targetips, _, zeroRouterV4LocalEndpoints := config.makeNodeRouterTargetIPs(service, &node, config.eps.V4IPs, types.V4HostMasqueradeIP) + routerV6targetips, _, zeroRouterV6LocalEndpoints := config.makeNodeRouterTargetIPs(service, &node, config.eps.V6IPs, types.V6HostMasqueradeIP) routerV4targets := joinHostsPort(routerV4targetips, config.eps.Port) routerV6targets := joinHostsPort(routerV6targetips, config.eps.Port) - switchV4Targets := joinHostsPort(config.eps.V4IPs, config.eps.Port) - switchV6Targets := joinHostsPort(config.eps.V6IPs, config.eps.Port) + switchV4targets := joinHostsPort(config.eps.V4IPs, config.eps.Port) + switchV6targets := joinHostsPort(config.eps.V6IPs, config.eps.Port) // OCP HACK begin // TODO: Remove this hack once we add support for ITP:preferLocal and DNS operator starts using it. @@ -356,17 +608,17 @@ func buildPerNodeLBs(service *v1.Service, configs []lbConfig, nodes []nodeInfo) if len(switchV6targetDNSips) == 0 { switchV6targetDNSips = config.eps.V6IPs } - switchV4Targets = joinHostsPort(switchV4targetDNSips, config.eps.Port) - switchV6Targets = joinHostsPort(switchV6targetDNSips, config.eps.Port) + switchV4targets = joinHostsPort(switchV4targetDNSips, config.eps.Port) + switchV6targets = joinHostsPort(switchV6targetDNSips, config.eps.Port) } // OCP HACK end // Substitute the special vip "node" for the node's physical ips // This is used for nodeport - vips = make([]string, 0, len(vips)) + vips := make([]string, 0, len(config.vips)) for _, vip := range config.vips { if vip == placeholderNodeIPs { - vips = append(vips, node.nodeIPs...) + vips = append(vips, node.nodeIPsStr()...) } else { vips = append(vips, vip) } @@ -375,9 +627,9 @@ func buildPerNodeLBs(service *v1.Service, configs []lbConfig, nodes []nodeInfo) for _, vip := range vips { isv6 := utilnet.IsIPv6String((vip)) // build switch rules - targets := switchV4Targets + targets := switchV4targets if isv6 { - targets = switchV6Targets + targets = switchV6targets } if config.externalTrafficLocal && config.hasNodePort { @@ -513,6 +765,11 @@ func getSessionAffinityTimeOut(service *v1.Service) int32 { return *service.Spec.SessionAffinityConfig.ClientIP.TimeoutSeconds } +func hasSessionAffinityTimeOut(service *v1.Service) bool { + return service.Spec.SessionAffinity == v1.ServiceAffinityClientIP && + getSessionAffinityTimeOut(service) > 0 +} + // lbOpts generates the OVN load balancer options from the kubernetes Service. func lbOpts(service *v1.Service) LBOpts { affinity := service.Spec.SessionAffinity == v1.ServiceAffinityClientIP @@ -547,6 +804,15 @@ func lbOpts(service *v1.Service) LBOpts { return lbOptions } +func lbTemplateOpts(service *v1.Service, addressFamily v1.IPFamily) LBOpts { + lbOptions := lbOpts(service) + + // Only template LBs need an explicit address family. + lbOptions.AddressFamily = addressFamily + lbOptions.Template = true + return lbOptions +} + // mergeLBs joins two LBs together if it is safe to do so. // // an LB can be merged if the protocol, rules, and options are the same, @@ -560,11 +826,12 @@ func mergeLBs(lbs []LB) []LB { outer: for _, lb := range lbs { for i := range out { - // If mergeable, rather than inserting lb to out, just add switches and routers + // If mergeable, rather than inserting lb to out, just add switches, routers, groups // and drop if canMergeLB(lb, out[i]) { out[i].Switches = append(out[i].Switches, lb.Switches...) out[i].Routers = append(out[i].Routers, lb.Routers...) + out[i].Groups = append(out[i].Groups, lb.Groups...) if !strings.HasSuffix(out[i].Name, "_merged") { out[i].Name += "_merged" diff --git a/go-controller/pkg/ovn/controller/services/lb_config_test.go b/go-controller/pkg/ovn/controller/services/lb_config_test.go index fabb6f33d2..3df9846b4c 100644 --- a/go-controller/pkg/ovn/controller/services/lb_config_test.go +++ b/go-controller/pkg/ovn/controller/services/lb_config_test.go @@ -129,11 +129,13 @@ func Test_buildServiceLBConfigs(t *testing.T) { name string args args - resultSharedGatewayCluster []lbConfig - resultSharedGatewayNode []lbConfig + resultSharedGatewayCluster []lbConfig + resultSharedGatewayTemplate []lbConfig + resultSharedGatewayNode []lbConfig - resultLocalGatewayNode []lbConfig - resultLocalGatewayCluster []lbConfig + resultLocalGatewayNode []lbConfig + resultLocalGatewayTemplate []lbConfig + resultLocalGatewayCluster []lbConfig resultsSame bool //if true, then just use the SharedGateway results for the LGW test }{ @@ -503,7 +505,7 @@ func Test_buildServiceLBConfigs(t *testing.T) { Port: outport, }, }}, - resultSharedGatewayNode: []lbConfig{{ + resultSharedGatewayTemplate: []lbConfig{{ vips: []string{"node"}, protocol: v1.ProtocolTCP, inport: 5, @@ -538,49 +540,53 @@ func Test_buildServiceLBConfigs(t *testing.T) { // In shared and local gateway modes, nodeport and host-network-pods must be per-node resultSharedGatewayNode: []lbConfig{ { - vips: []string{"node"}, + vips: []string{"192.168.1.1", "2002::1"}, protocol: v1.ProtocolTCP, - inport: 5, + inport: inport, eps: util.LbEndpoints{ V4IPs: []string{"192.168.0.1"}, V6IPs: []string{"2001::1"}, Port: outport, }, - hasNodePort: true, }, + }, + resultSharedGatewayTemplate: []lbConfig{ { - vips: []string{"192.168.1.1", "2002::1"}, + vips: []string{"node"}, protocol: v1.ProtocolTCP, - inport: inport, + inport: 5, eps: util.LbEndpoints{ V4IPs: []string{"192.168.0.1"}, V6IPs: []string{"2001::1"}, Port: outport, }, + hasNodePort: true, }, }, // in local gateway mode, only nodePort is per-node resultLocalGatewayNode: []lbConfig{ { - vips: []string{"node"}, + vips: []string{"192.168.1.1", "2002::1"}, protocol: v1.ProtocolTCP, - inport: 5, + inport: inport, eps: util.LbEndpoints{ V4IPs: []string{"192.168.0.1"}, V6IPs: []string{"2001::1"}, Port: outport, }, - hasNodePort: true, }, + }, + resultLocalGatewayTemplate: []lbConfig{ { - vips: []string{"192.168.1.1", "2002::1"}, + vips: []string{"node"}, protocol: v1.ProtocolTCP, - inport: inport, + inport: 5, eps: util.LbEndpoints{ V4IPs: []string{"192.168.0.1"}, V6IPs: []string{"2001::1"}, Port: outport, }, + hasNodePort: true, }, }, }, @@ -705,17 +711,20 @@ func Test_buildServiceLBConfigs(t *testing.T) { for i, tt := range tests { t.Run(fmt.Sprintf("%d_%s", i, tt.name), func(t *testing.T) { globalconfig.Gateway.Mode = globalconfig.GatewayModeShared - perNode, clusterWide := buildServiceLBConfigs(tt.args.service, tt.args.slices) + perNode, template, clusterWide := buildServiceLBConfigs(tt.args.service, tt.args.slices, true, true) assert.EqualValues(t, tt.resultSharedGatewayNode, perNode, "SGW per-node configs should be equal") + assert.EqualValues(t, tt.resultSharedGatewayTemplate, template, "SGW template configs should be equal") assert.EqualValues(t, tt.resultSharedGatewayCluster, clusterWide, "SGW cluster-wide configs should be equal") globalconfig.Gateway.Mode = globalconfig.GatewayModeLocal - perNode, clusterWide = buildServiceLBConfigs(tt.args.service, tt.args.slices) + perNode, template, clusterWide = buildServiceLBConfigs(tt.args.service, tt.args.slices, true, true) if tt.resultsSame { assert.EqualValues(t, tt.resultSharedGatewayNode, perNode, "LGW per-node configs should be equal") + assert.EqualValues(t, tt.resultSharedGatewayTemplate, template, "LGW template configs should be equal") assert.EqualValues(t, tt.resultSharedGatewayCluster, clusterWide, "LGW cluster-wide configs should be equal") } else { assert.EqualValues(t, tt.resultLocalGatewayNode, perNode, "LGW per-node configs should be equal") + assert.EqualValues(t, tt.resultLocalGatewayTemplate, template, "LGW template configs should be equal") assert.EqualValues(t, tt.resultLocalGatewayCluster, clusterWide, "LGW cluster-wide configs should be equal") } }) @@ -742,13 +751,13 @@ func Test_buildClusterLBs(t *testing.T) { defaultNodes := []nodeInfo{ { name: "node-a", - nodeIPs: []string{"10.0.0.1"}, + nodeIPs: []net.IP{net.ParseIP("10.0.0.1")}, gatewayRouterName: "gr-node-a", switchName: "switch-node-a", }, { name: "node-b", - nodeIPs: []string{"10.0.0.2"}, + nodeIPs: []net.IP{net.ParseIP("10.0.0.2")}, gatewayRouterName: "gr-node-b", switchName: "switch-node-b", }, @@ -802,12 +811,12 @@ func Test_buildClusterLBs(t *testing.T) { ExternalIDs: defaultExternalIDs, Rules: []LBRule{ { - Source: Addr{"1.2.3.4", 80}, - Targets: []Addr{{"192.168.0.1", 8080}, {"192.168.0.2", 8080}}, + Source: Addr{IP: "1.2.3.4", Port: 80}, + Targets: []Addr{{IP: "192.168.0.1", Port: 8080}, {IP: "192.168.0.2", Port: 8080}}, }, { - Source: Addr{"1.2.3.4", 443}, - Targets: []Addr{{"192.168.0.1", 8043}}, + Source: Addr{IP: "1.2.3.4", Port: 443}, + Targets: []Addr{{IP: "192.168.0.1", Port: 8043}}, }, }, @@ -849,8 +858,8 @@ func Test_buildClusterLBs(t *testing.T) { ExternalIDs: defaultExternalIDs, Rules: []LBRule{ { - Source: Addr{"1.2.3.4", 80}, - Targets: []Addr{{"192.168.0.1", 8080}, {"192.168.0.2", 8080}}, + Source: Addr{IP: "1.2.3.4", Port: 80}, + Targets: []Addr{{IP: "192.168.0.1", Port: 8080}, {IP: "192.168.0.2", Port: 8080}}, }, }, @@ -865,8 +874,8 @@ func Test_buildClusterLBs(t *testing.T) { ExternalIDs: defaultExternalIDs, Rules: []LBRule{ { - Source: Addr{"1.2.3.4", 443}, - Targets: []Addr{{"192.168.0.1", 8043}}, + Source: Addr{IP: "1.2.3.4", Port: 443}, + Targets: []Addr{{IP: "192.168.0.1", Port: 8043}}, }, }, @@ -910,20 +919,20 @@ func Test_buildClusterLBs(t *testing.T) { ExternalIDs: defaultExternalIDs, Rules: []LBRule{ { - Source: Addr{"1.2.3.4", 80}, - Targets: []Addr{{"192.168.0.1", 8080}, {"192.168.0.2", 8080}}, + Source: Addr{IP: "1.2.3.4", Port: 80}, + Targets: []Addr{{IP: "192.168.0.1", Port: 8080}, {IP: "192.168.0.2", Port: 8080}}, }, { - Source: Addr{"fe80::1", 80}, - Targets: []Addr{{"fe90::1", 8080}, {"fe91::1", 8080}}, + Source: Addr{IP: "fe80::1", Port: 80}, + Targets: []Addr{{IP: "fe90::1", Port: 8080}, {IP: "fe91::1", Port: 8080}}, }, { - Source: Addr{"1.2.3.4", 443}, - Targets: []Addr{{"192.168.0.1", 8043}}, + Source: Addr{IP: "1.2.3.4", Port: 443}, + Targets: []Addr{{IP: "192.168.0.1", Port: 8043}}, }, { - Source: Addr{"fe80::1", 443}, - Targets: []Addr{{"fe90::1", 8043}}, + Source: Addr{IP: "fe80::1", Port: 443}, + Targets: []Addr{{IP: "fe90::1", Port: 8043}}, }, }, @@ -970,14 +979,14 @@ func Test_buildPerNodeLBs(t *testing.T) { defaultNodes := []nodeInfo{ { name: "node-a", - nodeIPs: []string{"10.0.0.1"}, + nodeIPs: []net.IP{net.ParseIP("10.0.0.1")}, gatewayRouterName: "gr-node-a", switchName: "switch-node-a", podSubnets: []net.IPNet{{IP: net.ParseIP("10.128.0.0"), Mask: net.CIDRMask(24, 32)}}, }, { name: "node-b", - nodeIPs: []string{"10.0.0.2"}, + nodeIPs: []net.IP{net.ParseIP("10.0.0.2")}, gatewayRouterName: "gr-node-b", switchName: "switch-node-b", podSubnets: []net.IPNet{{IP: net.ParseIP("10.128.1.0"), Mask: net.CIDRMask(24, 32)}}, @@ -1022,8 +1031,8 @@ func Test_buildPerNodeLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"1.2.3.4", 80}, - Targets: []Addr{{"169.254.169.2", 8080}}, + Source: Addr{IP: "1.2.3.4", Port: 80}, + Targets: []Addr{{IP: "169.254.169.2", Port: 8080}}, }, }, Opts: defaultOpts, @@ -1036,8 +1045,8 @@ func Test_buildPerNodeLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"1.2.3.4", 80}, - Targets: []Addr{{"10.0.0.1", 8080}}, + Source: Addr{IP: "1.2.3.4", Port: 80}, + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}}, }, }, Opts: defaultOpts, @@ -1067,8 +1076,8 @@ func Test_buildPerNodeLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"10.0.0.1", 80}, - Targets: []Addr{{"10.128.0.2", 8080}}, + Source: Addr{IP: "10.0.0.1", Port: 80}, + Targets: []Addr{{IP: "10.128.0.2", Port: 8080}}, }, }, Opts: defaultOpts, @@ -1081,8 +1090,8 @@ func Test_buildPerNodeLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"10.0.0.2", 80}, - Targets: []Addr{{"10.128.0.2", 8080}}, + Source: Addr{IP: "10.0.0.2", Port: 80}, + Targets: []Addr{{IP: "10.128.0.2", Port: 8080}}, }, }, Opts: defaultOpts, @@ -1097,8 +1106,8 @@ func Test_buildPerNodeLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"10.0.0.1", 80}, - Targets: []Addr{{"10.128.0.2", 8080}}, + Source: Addr{IP: "10.0.0.1", Port: 80}, + Targets: []Addr{{IP: "10.128.0.2", Port: 8080}}, }, }, Opts: defaultOpts, @@ -1111,8 +1120,8 @@ func Test_buildPerNodeLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"10.0.0.2", 80}, - Targets: []Addr{{"10.128.0.2", 8080}}, + Source: Addr{IP: "10.0.0.2", Port: 80}, + Targets: []Addr{{IP: "10.128.0.2", Port: 8080}}, }, }, Opts: defaultOpts, @@ -1150,12 +1159,12 @@ func Test_buildPerNodeLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"192.168.0.1", 80}, - Targets: []Addr{{"169.254.169.2", 8080}}, + Source: Addr{IP: "192.168.0.1", Port: 80}, + Targets: []Addr{{IP: "169.254.169.2", Port: 8080}}, }, { - Source: Addr{"10.0.0.1", 80}, - Targets: []Addr{{"169.254.169.2", 8080}}, + Source: Addr{IP: "10.0.0.1", Port: 80}, + Targets: []Addr{{IP: "169.254.169.2", Port: 8080}}, }, }, Opts: defaultOpts, @@ -1167,12 +1176,12 @@ func Test_buildPerNodeLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"192.168.0.1", 80}, - Targets: []Addr{{"10.0.0.1", 8080}}, + Source: Addr{IP: "192.168.0.1", Port: 80}, + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}}, }, { - Source: Addr{"10.0.0.1", 80}, - Targets: []Addr{{"10.0.0.1", 8080}}, + Source: Addr{IP: "10.0.0.1", Port: 80}, + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}}, }, }, Opts: defaultOpts, @@ -1185,12 +1194,12 @@ func Test_buildPerNodeLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"192.168.0.1", 80}, - Targets: []Addr{{"10.0.0.1", 8080}}, + Source: Addr{IP: "192.168.0.1", Port: 80}, + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}}, }, { - Source: Addr{"10.0.0.2", 80}, - Targets: []Addr{{"10.0.0.1", 8080}}, + Source: Addr{IP: "10.0.0.2", Port: 80}, + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}}, }, }, Opts: defaultOpts, @@ -1204,12 +1213,12 @@ func Test_buildPerNodeLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"192.168.0.1", 80}, - Targets: []Addr{{"169.254.169.2", 8080}}, + Source: Addr{IP: "192.168.0.1", Port: 80}, + Targets: []Addr{{IP: "169.254.169.2", Port: 8080}}, }, { - Source: Addr{"10.0.0.1", 80}, - Targets: []Addr{{"169.254.169.2", 8080}}, + Source: Addr{IP: "10.0.0.1", Port: 80}, + Targets: []Addr{{IP: "169.254.169.2", Port: 8080}}, }, }, Opts: defaultOpts, @@ -1221,12 +1230,12 @@ func Test_buildPerNodeLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"192.168.0.1", 80}, - Targets: []Addr{{"10.0.0.1", 8080}}, + Source: Addr{IP: "192.168.0.1", Port: 80}, + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}}, }, { - Source: Addr{"10.0.0.1", 80}, - Targets: []Addr{{"10.0.0.1", 8080}}, + Source: Addr{IP: "10.0.0.1", Port: 80}, + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}}, }, }, Opts: defaultOpts, @@ -1239,12 +1248,12 @@ func Test_buildPerNodeLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"192.168.0.1", 80}, - Targets: []Addr{{"10.0.0.1", 8080}}, + Source: Addr{IP: "192.168.0.1", Port: 80}, + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}}, }, { - Source: Addr{"10.0.0.2", 80}, - Targets: []Addr{{"10.0.0.1", 8080}}, + Source: Addr{IP: "10.0.0.2", Port: 80}, + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}}, }, }, Opts: defaultOpts, @@ -1289,8 +1298,8 @@ func Test_buildPerNodeLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"192.168.0.1", 80}, - Targets: []Addr{{"169.254.169.2", 8080}}, + Source: Addr{IP: "192.168.0.1", Port: 80}, + Targets: []Addr{{IP: "169.254.169.2", Port: 8080}}, }, }, Opts: defaultOpts, @@ -1303,8 +1312,8 @@ func Test_buildPerNodeLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"10.0.0.1", 80}, - Targets: []Addr{{"169.254.169.2", 8080}}, + Source: Addr{IP: "10.0.0.1", Port: 80}, + Targets: []Addr{{IP: "169.254.169.2", Port: 8080}}, }, }, }, @@ -1315,16 +1324,16 @@ func Test_buildPerNodeLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"192.168.0.1", 80}, - Targets: []Addr{{"10.0.0.1", 8080}}, + Source: Addr{IP: "192.168.0.1", Port: 80}, + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}}, }, { - Source: Addr{"169.254.169.3", 80}, - Targets: []Addr{{"10.0.0.1", 8080}}, + Source: Addr{IP: "169.254.169.3", Port: 80}, + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}}, }, { - Source: Addr{"10.0.0.1", 80}, - Targets: []Addr{{"10.0.0.1", 8080}}, + Source: Addr{IP: "10.0.0.1", Port: 80}, + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}}, }, }, Opts: defaultOpts, @@ -1341,11 +1350,11 @@ func Test_buildPerNodeLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"192.168.0.1", 80}, - Targets: []Addr{{"10.0.0.1", 8080}}, + Source: Addr{IP: "192.168.0.1", Port: 80}, + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}}, }, { - Source: Addr{"10.0.0.2", 80}, + Source: Addr{IP: "10.0.0.2", Port: 80}, Targets: []Addr{}, }, }, @@ -1358,16 +1367,16 @@ func Test_buildPerNodeLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"192.168.0.1", 80}, - Targets: []Addr{{"10.0.0.1", 8080}}, + Source: Addr{IP: "192.168.0.1", Port: 80}, + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}}, }, { - Source: Addr{"169.254.169.3", 80}, + Source: Addr{IP: "169.254.169.3", Port: 80}, Targets: []Addr{}, }, { - Source: Addr{"10.0.0.2", 80}, - Targets: []Addr{{"10.0.0.1", 8080}}, + Source: Addr{IP: "10.0.0.2", Port: 80}, + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}}, }, }, Opts: defaultOpts, @@ -1406,12 +1415,12 @@ func Test_buildPerNodeLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"192.168.0.1", 80}, - Targets: []Addr{{"10.128.0.1", 8080}, {"10.128.1.1", 8080}}, // no filtering on GR LBs for ITP=local + Source: Addr{IP: "192.168.0.1", Port: 80}, + Targets: []Addr{{IP: "10.128.0.1", Port: 8080}, {IP: "10.128.1.1", Port: 8080}}, // no filtering on GR LBs for ITP=local }, { - Source: Addr{"1.2.3.4", 80}, - Targets: []Addr{{"10.128.0.1", 8080}, {"10.128.1.1", 8080}}, + Source: Addr{IP: "1.2.3.4", Port: 80}, + Targets: []Addr{{IP: "10.128.0.1", Port: 8080}, {IP: "10.128.1.1", Port: 8080}}, }, }, Opts: defaultOpts, @@ -1423,12 +1432,12 @@ func Test_buildPerNodeLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"192.168.0.1", 80}, - Targets: []Addr{{"10.128.0.1", 8080}}, // filters out the ep present only on node-a + Source: Addr{IP: "192.168.0.1", Port: 80}, + Targets: []Addr{{IP: "10.128.0.1", Port: 8080}}, // filters out the ep present only on node-a }, { - Source: Addr{"1.2.3.4", 80}, - Targets: []Addr{{"10.128.0.1", 8080}, {"10.128.1.1", 8080}}, // ITP is only applicable for clusterIPs + Source: Addr{IP: "1.2.3.4", Port: 80}, + Targets: []Addr{{IP: "10.128.0.1", Port: 8080}, {IP: "10.128.1.1", Port: 8080}}, // ITP is only applicable for clusterIPs }, }, Opts: defaultOpts, @@ -1440,12 +1449,12 @@ func Test_buildPerNodeLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"192.168.0.1", 80}, - Targets: []Addr{{"10.128.1.1", 8080}}, // filters out the ep present only on node-b + Source: Addr{IP: "192.168.0.1", Port: 80}, + Targets: []Addr{{IP: "10.128.1.1", Port: 8080}}, // filters out the ep present only on node-b }, { - Source: Addr{"1.2.3.4", 80}, - Targets: []Addr{{"10.128.0.1", 8080}, {"10.128.1.1", 8080}}, // ITP is only applicable for clusterIPs + Source: Addr{IP: "1.2.3.4", Port: 80}, + Targets: []Addr{{IP: "10.128.0.1", Port: 8080}, {IP: "10.128.1.1", Port: 8080}}, // ITP is only applicable for clusterIPs }, }, Opts: defaultOpts, @@ -1459,12 +1468,12 @@ func Test_buildPerNodeLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"192.168.0.1", 80}, - Targets: []Addr{{"10.128.0.1", 8080}, {"10.128.1.1", 8080}}, // no filtering on GR LBs for ITP=local + Source: Addr{IP: "192.168.0.1", Port: 80}, + Targets: []Addr{{IP: "10.128.0.1", Port: 8080}, {IP: "10.128.1.1", Port: 8080}}, // no filtering on GR LBs for ITP=local }, { - Source: Addr{"1.2.3.4", 80}, - Targets: []Addr{{"10.128.0.1", 8080}, {"10.128.1.1", 8080}}, + Source: Addr{IP: "1.2.3.4", Port: 80}, + Targets: []Addr{{IP: "10.128.0.1", Port: 8080}, {IP: "10.128.1.1", Port: 8080}}, }, }, Opts: defaultOpts, @@ -1476,12 +1485,12 @@ func Test_buildPerNodeLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"192.168.0.1", 80}, - Targets: []Addr{{"10.128.0.1", 8080}}, // filters out the ep present only on node-a + Source: Addr{IP: "192.168.0.1", Port: 80}, + Targets: []Addr{{IP: "10.128.0.1", Port: 8080}}, // filters out the ep present only on node-a }, { - Source: Addr{"1.2.3.4", 80}, - Targets: []Addr{{"10.128.0.1", 8080}, {"10.128.1.1", 8080}}, // ITP is only applicable for clusterIPs + Source: Addr{IP: "1.2.3.4", Port: 80}, + Targets: []Addr{{IP: "10.128.0.1", Port: 8080}, {IP: "10.128.1.1", Port: 8080}}, // ITP is only applicable for clusterIPs }, }, Opts: defaultOpts, @@ -1493,12 +1502,12 @@ func Test_buildPerNodeLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"192.168.0.1", 80}, - Targets: []Addr{{"10.128.1.1", 8080}}, // filters out the ep present only on node-b + Source: Addr{IP: "192.168.0.1", Port: 80}, + Targets: []Addr{{IP: "10.128.1.1", Port: 8080}}, // filters out the ep present only on node-b }, { - Source: Addr{"1.2.3.4", 80}, - Targets: []Addr{{"10.128.0.1", 8080}, {"10.128.1.1", 8080}}, // ITP is only applicable for clusterIPs + Source: Addr{IP: "1.2.3.4", Port: 80}, + Targets: []Addr{{IP: "10.128.0.1", Port: 8080}, {IP: "10.128.1.1", Port: 8080}}, // ITP is only applicable for clusterIPs }, }, Opts: defaultOpts, @@ -1537,12 +1546,12 @@ func Test_buildPerNodeLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"192.168.0.1", 80}, - Targets: []Addr{{"169.254.169.2", 8080}, {"10.0.0.2", 8080}}, // no filtering on GR LBs for ITP=local + Source: Addr{IP: "192.168.0.1", Port: 80}, + Targets: []Addr{{IP: "169.254.169.2", Port: 8080}, {IP: "10.0.0.2", Port: 8080}}, // no filtering on GR LBs for ITP=local }, { - Source: Addr{"1.2.3.4", 80}, - Targets: []Addr{{"169.254.169.2", 8080}, {"10.0.0.2", 8080}}, + Source: Addr{IP: "1.2.3.4", Port: 80}, + Targets: []Addr{{IP: "169.254.169.2", Port: 8080}, {IP: "10.0.0.2", Port: 8080}}, }, }, Opts: defaultOpts, @@ -1554,12 +1563,12 @@ func Test_buildPerNodeLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"192.168.0.1", 80}, - Targets: []Addr{{"10.0.0.1", 8080}}, // filters out the ep present only on node-a + Source: Addr{IP: "192.168.0.1", Port: 80}, + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}}, // filters out the ep present only on node-a }, { - Source: Addr{"1.2.3.4", 80}, - Targets: []Addr{{"10.0.0.1", 8080}, {"10.0.0.2", 8080}}, // ITP is only applicable for clusterIPs + Source: Addr{IP: "1.2.3.4", Port: 80}, + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}, {IP: "10.0.0.2", Port: 8080}}, // ITP is only applicable for clusterIPs }, }, Opts: defaultOpts, @@ -1571,12 +1580,12 @@ func Test_buildPerNodeLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"192.168.0.1", 80}, - Targets: []Addr{{"10.0.0.1", 8080}, {"169.254.169.2", 8080}}, // no filtering on GR LBs for ITP=local + Source: Addr{IP: "192.168.0.1", Port: 80}, + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}, {IP: "169.254.169.2", Port: 8080}}, // no filtering on GR LBs for ITP=local }, { - Source: Addr{"1.2.3.4", 80}, - Targets: []Addr{{"10.0.0.1", 8080}, {"169.254.169.2", 8080}}, + Source: Addr{IP: "1.2.3.4", Port: 80}, + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}, {IP: "169.254.169.2", Port: 8080}}, }, }, Opts: defaultOpts, @@ -1588,12 +1597,12 @@ func Test_buildPerNodeLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"192.168.0.1", 80}, - Targets: []Addr{{"10.0.0.2", 8080}}, // filters out the ep present only on node-b + Source: Addr{IP: "192.168.0.1", Port: 80}, + Targets: []Addr{{IP: "10.0.0.2", Port: 8080}}, // filters out the ep present only on node-b }, { - Source: Addr{"1.2.3.4", 80}, - Targets: []Addr{{"10.0.0.1", 8080}, {"10.0.0.2", 8080}}, // ITP is only applicable for clusterIPs + Source: Addr{IP: "1.2.3.4", Port: 80}, + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}, {IP: "10.0.0.2", Port: 8080}}, // ITP is only applicable for clusterIPs }, }, Opts: defaultOpts, @@ -1607,12 +1616,12 @@ func Test_buildPerNodeLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"192.168.0.1", 80}, - Targets: []Addr{{"169.254.169.2", 8080}, {"10.0.0.2", 8080}}, // no filtering on GR LBs for ITP=local + Source: Addr{IP: "192.168.0.1", Port: 80}, + Targets: []Addr{{IP: "169.254.169.2", Port: 8080}, {IP: "10.0.0.2", Port: 8080}}, // no filtering on GR LBs for ITP=local }, { - Source: Addr{"1.2.3.4", 80}, - Targets: []Addr{{"169.254.169.2", 8080}, {"10.0.0.2", 8080}}, + Source: Addr{IP: "1.2.3.4", Port: 80}, + Targets: []Addr{{IP: "169.254.169.2", Port: 8080}, {IP: "10.0.0.2", Port: 8080}}, }, }, Opts: defaultOpts, @@ -1624,12 +1633,12 @@ func Test_buildPerNodeLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"192.168.0.1", 80}, - Targets: []Addr{{"10.0.0.1", 8080}}, // filters out the ep present only on node-a + Source: Addr{IP: "192.168.0.1", Port: 80}, + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}}, // filters out the ep present only on node-a }, { - Source: Addr{"1.2.3.4", 80}, - Targets: []Addr{{"10.0.0.1", 8080}, {"10.0.0.2", 8080}}, // ITP is only applicable for clusterIPs + Source: Addr{IP: "1.2.3.4", Port: 80}, + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}, {IP: "10.0.0.2", Port: 8080}}, // ITP is only applicable for clusterIPs }, }, Opts: defaultOpts, @@ -1641,12 +1650,12 @@ func Test_buildPerNodeLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"192.168.0.1", 80}, - Targets: []Addr{{"10.0.0.1", 8080}, {"169.254.169.2", 8080}}, // no filtering on GR LBs for ITP=local + Source: Addr{IP: "192.168.0.1", Port: 80}, + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}, {IP: "169.254.169.2", Port: 8080}}, // no filtering on GR LBs for ITP=local }, { - Source: Addr{"1.2.3.4", 80}, - Targets: []Addr{{"10.0.0.1", 8080}, {"169.254.169.2", 8080}}, + Source: Addr{IP: "1.2.3.4", Port: 80}, + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}, {IP: "169.254.169.2", Port: 8080}}, }, }, Opts: defaultOpts, @@ -1658,12 +1667,12 @@ func Test_buildPerNodeLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"192.168.0.1", 80}, - Targets: []Addr{{"10.0.0.2", 8080}}, // filters out the ep present only on node-b + Source: Addr{IP: "192.168.0.1", Port: 80}, + Targets: []Addr{{IP: "10.0.0.2", Port: 8080}}, // filters out the ep present only on node-b }, { - Source: Addr{"1.2.3.4", 80}, - Targets: []Addr{{"10.0.0.1", 8080}, {"10.0.0.2", 8080}}, // ITP is only applicable for clusterIPs + Source: Addr{IP: "1.2.3.4", Port: 80}, + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}, {IP: "10.0.0.2", Port: 8080}}, // ITP is only applicable for clusterIPs }, }, Opts: defaultOpts, @@ -1707,8 +1716,8 @@ func Test_buildPerNodeLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"192.168.0.1", 80}, - Targets: []Addr{{"169.254.169.2", 8080}}, // we don't filter clusterIPs at GR for ETP/ITP=local + Source: Addr{IP: "192.168.0.1", Port: 80}, + Targets: []Addr{{IP: "169.254.169.2", Port: 8080}}, // we don't filter clusterIPs at GR for ETP/ITP=local }, }, Opts: defaultOpts, @@ -1721,8 +1730,8 @@ func Test_buildPerNodeLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"10.0.0.1", 34345}, - Targets: []Addr{{"169.254.169.2", 8080}}, // special skip_snat=true LB for ETP=local; used in SGW mode + Source: Addr{IP: "10.0.0.1", Port: 34345}, + Targets: []Addr{{IP: "169.254.169.2", Port: 8080}}, // special skip_snat=true LB for ETP=local; used in SGW mode }, }, }, @@ -1733,16 +1742,16 @@ func Test_buildPerNodeLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"192.168.0.1", 80}, - Targets: []Addr{{"10.0.0.1", 8080}}, // filter out eps only on node-a for clusterIP + Source: Addr{IP: "192.168.0.1", Port: 80}, + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}}, // filter out eps only on node-a for clusterIP }, { - Source: Addr{"169.254.169.3", 34345}, // add special masqueradeIP VIP for nodePort/LB traffic coming from node via mp0 when ETP=local - Targets: []Addr{{"10.0.0.1", 8080}}, // filter out eps only on node-a for nodePorts + Source: Addr{IP: "169.254.169.3", Port: 34345}, // add special masqueradeIP VIP for nodePort/LB traffic coming from node via mp0 when ETP=local + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}}, // filter out eps only on node-a for nodePorts }, { - Source: Addr{"10.0.0.1", 34345}, - Targets: []Addr{{"10.0.0.1", 8080}}, // don't filter out eps for nodePorts on switches when ETP=local + Source: Addr{IP: "10.0.0.1", Port: 34345}, + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}}, // don't filter out eps for nodePorts on switches when ETP=local }, }, Opts: defaultOpts, @@ -1754,11 +1763,11 @@ func Test_buildPerNodeLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"192.168.0.1", 80}, - Targets: []Addr{{"10.0.0.1", 8080}}, // we don't filter clusterIPs at GR for ETP/ITP=local + Source: Addr{IP: "192.168.0.1", Port: 80}, + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}}, // we don't filter clusterIPs at GR for ETP/ITP=local }, { - Source: Addr{"10.0.0.2", 34345}, + Source: Addr{IP: "10.0.0.2", Port: 34345}, Targets: []Addr{}, // filter out eps only on node-b for nodePort on GR when ETP=local }, }, @@ -1771,16 +1780,16 @@ func Test_buildPerNodeLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"192.168.0.1", 80}, + Source: Addr{IP: "192.168.0.1", Port: 80}, Targets: []Addr{}, // filter out eps only on node-b for clusterIP }, { - Source: Addr{"169.254.169.3", 34345}, // add special masqueradeIP VIP for nodePort/LB traffic coming from node via mp0 when ETP=local - Targets: []Addr{}, // filter out eps only on node-b for nodePorts + Source: Addr{IP: "169.254.169.3", Port: 34345}, // add special masqueradeIP VIP for nodePort/LB traffic coming from node via mp0 when ETP=local + Targets: []Addr{}, // filter out eps only on node-b for nodePorts }, { - Source: Addr{"10.0.0.2", 34345}, - Targets: []Addr{{"10.0.0.1", 8080}}, // don't filter out eps for nodePorts on switches when ETP=local + Source: Addr{IP: "10.0.0.2", Port: 34345}, + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}}, // don't filter out eps for nodePorts on switches when ETP=local }, }, Opts: defaultOpts, @@ -1794,8 +1803,8 @@ func Test_buildPerNodeLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"192.168.0.1", 80}, - Targets: []Addr{{"169.254.169.2", 8080}}, // we don't filter clusterIPs at GR for ETP/ITP=local + Source: Addr{IP: "192.168.0.1", Port: 80}, + Targets: []Addr{{IP: "169.254.169.2", Port: 8080}}, // we don't filter clusterIPs at GR for ETP/ITP=local }, }, Opts: defaultOpts, @@ -1808,8 +1817,8 @@ func Test_buildPerNodeLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"10.0.0.1", 34345}, - Targets: []Addr{{"169.254.169.2", 8080}}, // special skip_snat=true LB for ETP=local; used in SGW mode + Source: Addr{IP: "10.0.0.1", Port: 34345}, + Targets: []Addr{{IP: "169.254.169.2", Port: 8080}}, // special skip_snat=true LB for ETP=local; used in SGW mode }, }, }, @@ -1820,16 +1829,16 @@ func Test_buildPerNodeLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"192.168.0.1", 80}, - Targets: []Addr{{"10.0.0.1", 8080}}, // filter out eps only on node-a for clusterIP + Source: Addr{IP: "192.168.0.1", Port: 80}, + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}}, // filter out eps only on node-a for clusterIP }, { - Source: Addr{"169.254.169.3", 34345}, // add special masqueradeIP VIP for nodePort/LB traffic coming from node via mp0 when ETP=local - Targets: []Addr{{"10.0.0.1", 8080}}, // filter out eps only on node-a for nodePorts + Source: Addr{IP: "169.254.169.3", Port: 34345}, // add special masqueradeIP VIP for nodePort/LB traffic coming from node via mp0 when ETP=local + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}}, // filter out eps only on node-a for nodePorts }, { - Source: Addr{"10.0.0.1", 34345}, - Targets: []Addr{{"10.0.0.1", 8080}}, // don't filter out eps for nodePorts on switches when ETP=local + Source: Addr{IP: "10.0.0.1", Port: 34345}, + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}}, // don't filter out eps for nodePorts on switches when ETP=local }, }, Opts: defaultOpts, @@ -1841,11 +1850,11 @@ func Test_buildPerNodeLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"192.168.0.1", 80}, - Targets: []Addr{{"10.0.0.1", 8080}}, // we don't filter clusterIPs at GR for ETP/ITP=local + Source: Addr{IP: "192.168.0.1", Port: 80}, + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}}, // we don't filter clusterIPs at GR for ETP/ITP=local }, { - Source: Addr{"10.0.0.2", 34345}, + Source: Addr{IP: "10.0.0.2", Port: 34345}, Targets: []Addr{}, // filter out eps only on node-b for nodePort on GR when ETP=local }, }, @@ -1858,16 +1867,16 @@ func Test_buildPerNodeLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"192.168.0.1", 80}, + Source: Addr{IP: "192.168.0.1", Port: 80}, Targets: []Addr{}, // filter out eps only on node-b for clusterIP }, { - Source: Addr{"169.254.169.3", 34345}, // add special masqueradeIP VIP for nodePort/LB traffic coming from node via mp0 when ETP=local - Targets: []Addr{}, // filter out eps only on node-b for nodePorts + Source: Addr{IP: "169.254.169.3", Port: 34345}, // add special masqueradeIP VIP for nodePort/LB traffic coming from node via mp0 when ETP=local + Targets: []Addr{}, // filter out eps only on node-b for nodePorts }, { - Source: Addr{"10.0.0.2", 34345}, - Targets: []Addr{{"10.0.0.1", 8080}}, // don't filter out eps for nodePorts on switches when ETP=local + Source: Addr{IP: "10.0.0.2", Port: 34345}, + Targets: []Addr{{IP: "10.0.0.1", Port: 8080}}, // don't filter out eps for nodePorts on switches when ETP=local }, }, Opts: defaultOpts, diff --git a/go-controller/pkg/ovn/controller/services/loadbalancer.go b/go-controller/pkg/ovn/controller/services/loadbalancer.go index 7657bdba78..2571a51b73 100644 --- a/go-controller/pkg/ovn/controller/services/loadbalancer.go +++ b/go-controller/pkg/ovn/controller/services/loadbalancer.go @@ -28,6 +28,8 @@ type LB struct { Rules []LBRule + Templates TemplateMap // Templates that this LB uses as backends. + // the names of logical switches, routers and LB groups that this LB should be attached to Switches []string Routers []string @@ -46,11 +48,18 @@ type LBOpts struct { // If true, then disable SNAT entirely SkipSNAT bool + + // If true, this is a LB template. + Template bool + + // Only useful for template LBs. + AddressFamily corev1.IPFamily } type Addr struct { - IP string - Port int32 + IP string + Port int32 + Template *Template } type LBRule struct { @@ -59,11 +68,46 @@ type LBRule struct { } func (a *Addr) String() string { - return util.JoinHostPortInt32(a.IP, a.Port) + if a.Template == nil { + return util.JoinHostPortInt32(a.IP, a.Port) + } else if a.Port != 0 { + return fmt.Sprintf("%s:%d", a.Template.toReferenceString(), a.Port) + } else { + return a.Template.toReferenceString() + } +} + +// addrsToString joins together the textual representation of a list of Addr. +func addrsToString(addrs []Addr) string { + s := "" + for _, a := range addrs { + s += a.String() + "," + } + s = strings.TrimRight(s, ",") + return s } -func (a *Addr) Equals(b *Addr) bool { - return a.Port == b.Port && a.IP == b.IP +// templateLoadBalancer enriches a NB load balancer record with the +// associated template maps it requires provisioned in the NB database. +type templateLoadBalancer struct { + nbLB *nbdb.LoadBalancer + templates TemplateMap +} + +func toNBLoadBalancerList(tlbs []*templateLoadBalancer) []*nbdb.LoadBalancer { + result := make([]*nbdb.LoadBalancer, 0, len(tlbs)) + for _, tlb := range tlbs { + result = append(result, tlb.nbLB) + } + return result +} + +func toNBTemplateList(tlbs []*templateLoadBalancer) []TemplateMap { + templateVars := make([]TemplateMap, 0, len(tlbs)) + for _, tlb := range tlbs { + templateVars = append(templateVars, tlb.templates) + } + return templateVars } // EnsureLBs provides a generic load-balancer reconciliation engine. @@ -89,33 +133,33 @@ func (a *Addr) Equals(b *Addr) bool { func EnsureLBs(nbClient libovsdbclient.Client, service *corev1.Service, existingCacheLBs []LB, LBs []LB) error { externalIDs := util.ExternalIDsForObject(service) existingByName := make(map[string]*LB, len(existingCacheLBs)) - toDelete := sets.New[string]() + toDelete := make(map[string]*LB, len(existingCacheLBs)) for i := range existingCacheLBs { lb := &existingCacheLBs[i] existingByName[lb.Name] = lb - toDelete.Insert(lb.UUID) + toDelete[lb.UUID] = lb } - lbs := make([]*nbdb.LoadBalancer, 0, len(LBs)) - addLBsToSwitch := map[string][]*nbdb.LoadBalancer{} - removeLBsFromSwitch := map[string][]*nbdb.LoadBalancer{} - addLBsToRouter := map[string][]*nbdb.LoadBalancer{} - removesLBsFromRouter := map[string][]*nbdb.LoadBalancer{} - addLBsToGroups := map[string][]*nbdb.LoadBalancer{} - removeLBsFromGroups := map[string][]*nbdb.LoadBalancer{} + tlbs := make([]*templateLoadBalancer, 0, len(LBs)) + addLBsToSwitch := map[string][]*templateLoadBalancer{} + removeLBsFromSwitch := map[string][]*templateLoadBalancer{} + addLBsToRouter := map[string][]*templateLoadBalancer{} + removesLBsFromRouter := map[string][]*templateLoadBalancer{} + addLBsToGroups := map[string][]*templateLoadBalancer{} + removeLBsFromGroups := map[string][]*templateLoadBalancer{} wantedByName := make(map[string]*LB, len(LBs)) for i, lb := range LBs { wantedByName[lb.Name] = &LBs[i] blb := buildLB(&lb) - lbs = append(lbs, blb) + tlbs = append(tlbs, blb) existingLB := existingByName[lb.Name] existingRouters := sets.Set[string]{} existingSwitches := sets.Set[string]{} existingGroups := sets.Set[string]{} if existingLB != nil { - blb.UUID = existingLB.UUID - toDelete.Delete(existingLB.UUID) + blb.nbLB.UUID = existingLB.UUID + delete(toDelete, existingLB.UUID) existingRouters = sets.New[string](existingLB.Routers...) existingSwitches = sets.New[string](existingLB.Switches...) existingGroups = sets.New[string](existingLB.Groups...) @@ -131,9 +175,14 @@ func EnsureLBs(nbClient libovsdbclient.Client, service *corev1.Service, existing mapLBDifferenceByKey(removeLBsFromGroups, existingGroups, wantGroups, blb) } - ops, err := libovsdbops.CreateOrUpdateLoadBalancersOps(nbClient, nil, lbs...) + ops, err := libovsdbops.CreateOrUpdateLoadBalancersOps(nbClient, nil, toNBLoadBalancerList(tlbs)...) if err != nil { - return fmt.Errorf("failed to create ops for ensuring update of service %s/%s load balancers: %w", + return err + } + + ops, err = svcCreateOrUpdateTemplateVarOps(nbClient, ops, toNBTemplateList(tlbs)) + if err != nil { + return fmt.Errorf("failed to create ops for ensuring creation of service %s/%s load balancers: %w", service.Namespace, service.Name, err) } @@ -149,14 +198,14 @@ func EnsureLBs(nbClient libovsdbclient.Client, service *corev1.Service, existing return lswitch } for k, v := range addLBsToSwitch { - ops, err = libovsdbops.AddLoadBalancersToLogicalSwitchOps(nbClient, ops, getSwitch(k), v...) + ops, err = libovsdbops.AddLoadBalancersToLogicalSwitchOps(nbClient, ops, getSwitch(k), toNBLoadBalancerList(v)...) if err != nil { return fmt.Errorf("failed to create ops for adding load balancers to switch %s for service %s/%s: %w", k, service.Namespace, service.Name, err) } } for k, v := range removeLBsFromSwitch { - ops, err = libovsdbops.RemoveLoadBalancersFromLogicalSwitchOps(nbClient, ops, getSwitch(k), v...) + ops, err = libovsdbops.RemoveLoadBalancersFromLogicalSwitchOps(nbClient, ops, getSwitch(k), toNBLoadBalancerList(v)...) if err != nil { return fmt.Errorf("failed to create ops for removing load balancers from switch %s for service %s/%s: %w", k, service.Namespace, service.Name, err) @@ -175,14 +224,14 @@ func EnsureLBs(nbClient libovsdbclient.Client, service *corev1.Service, existing return router } for k, v := range addLBsToRouter { - ops, err = libovsdbops.AddLoadBalancersToLogicalRouterOps(nbClient, ops, getRouter(k), v...) + ops, err = libovsdbops.AddLoadBalancersToLogicalRouterOps(nbClient, ops, getRouter(k), toNBLoadBalancerList(v)...) if err != nil { return fmt.Errorf("failed to create ops for adding load balancers to router %s for service %s/%s: %w", k, service.Namespace, service.Name, err) } } for k, v := range removesLBsFromRouter { - ops, err = libovsdbops.RemoveLoadBalancersFromLogicalRouterOps(nbClient, ops, getRouter(k), v...) + ops, err = libovsdbops.RemoveLoadBalancersFromLogicalRouterOps(nbClient, ops, getRouter(k), toNBLoadBalancerList(v)...) if err != nil { return fmt.Errorf("failed to create ops for removing load balancers from router %s for service %s/%s: %w", k, service.Namespace, service.Name, err) @@ -201,14 +250,14 @@ func EnsureLBs(nbClient libovsdbclient.Client, service *corev1.Service, existing return group } for k, v := range addLBsToGroups { - ops, err = libovsdbops.AddLoadBalancersToGroupOps(nbClient, ops, getGroup(k), v...) + ops, err = libovsdbops.AddLoadBalancersToGroupOps(nbClient, ops, getGroup(k), toNBLoadBalancerList(v)...) if err != nil { return fmt.Errorf("failed to create ops for adding load balancers to group %s for service %s/%s: %w", k, service.Namespace, service.Name, err) } } for k, v := range removeLBsFromGroups { - ops, err = libovsdbops.RemoveLoadBalancersFromGroupOps(nbClient, ops, getGroup(k), v...) + ops, err = libovsdbops.RemoveLoadBalancersFromGroupOps(nbClient, ops, getGroup(k), toNBLoadBalancerList(v)...) if err != nil { return fmt.Errorf("failed to create ops for removing load balancers from group %s for service %s/%s: %w", k, service.Namespace, service.Name, err) @@ -216,8 +265,10 @@ func EnsureLBs(nbClient libovsdbclient.Client, service *corev1.Service, existing } deleteLBs := make([]*nbdb.LoadBalancer, 0, len(toDelete)) - for uuid := range toDelete { - deleteLBs = append(deleteLBs, &nbdb.LoadBalancer{UUID: uuid}) + deleteTemplates := make([]TemplateMap, 0, len(toDelete)) + for _, clb := range toDelete { + deleteLBs = append(deleteLBs, &nbdb.LoadBalancer{UUID: clb.UUID}) + deleteTemplates = append(deleteTemplates, clb.Templates) } ops, err = libovsdbops.DeleteLoadBalancersOps(nbClient, ops, deleteLBs...) if err != nil { @@ -225,6 +276,11 @@ func EnsureLBs(nbClient libovsdbclient.Client, service *corev1.Service, existing len(deleteLBs), service.Namespace, service.Name, err) } + ops, err = svcDeleteTemplateVarOps(nbClient, ops, deleteTemplates) + if err != nil { + return err + } + recordOps, txOkCallBack, _, err := metrics.GetConfigDurationRecorder().AddOVN(nbClient, "service", service.Namespace, service.Name) if err != nil { @@ -232,7 +288,7 @@ func EnsureLBs(nbClient libovsdbclient.Client, service *corev1.Service, existing } ops = append(ops, recordOps...) - _, err = libovsdbops.TransactAndCheckAndSetUUIDs(nbClient, lbs, ops) + _, err = libovsdbops.TransactAndCheckAndSetUUIDs(nbClient, toNBLoadBalancerList(tlbs), ops) if err != nil { return fmt.Errorf("failed to ensure load balancers for service %s/%s: %w", service.Namespace, service.Name, err) } @@ -240,8 +296,8 @@ func EnsureLBs(nbClient libovsdbclient.Client, service *corev1.Service, existing // Store UUID of newly created load balancers for future calls. // This is accomplished by the caching of LBs by the caller of this function. - for _, lb := range lbs { - wantedByName[lb.Name].UUID = lb.UUID + for _, tlb := range tlbs { + wantedByName[tlb.nbLB.Name].UUID = tlb.nbLB.UUID } klog.V(5).Infof("Deleted %d stale LBs for %#v", len(toDelete), externalIDs) @@ -268,18 +324,18 @@ func LoadBalancersEqualNoUUID(lbs1, lbs2 []LB) bool { return reflect.DeepEqual(new1, new2) } -func mapLBDifferenceByKey(keyMap map[string][]*nbdb.LoadBalancer, keyIn sets.Set[string], keyNotIn sets.Set[string], lb *nbdb.LoadBalancer) { +func mapLBDifferenceByKey(keyMap map[string][]*templateLoadBalancer, keyIn sets.Set[string], keyNotIn sets.Set[string], lb *templateLoadBalancer) { for _, k := range keyIn.Difference(keyNotIn).UnsortedList() { l := keyMap[k] if l == nil { - l = []*nbdb.LoadBalancer{} + l = []*templateLoadBalancer{} } l = append(l, lb) keyMap[k] = l } } -func buildLB(lb *LB) *nbdb.LoadBalancer { +func buildLB(lb *LB) *templateLoadBalancer { skipSNAT := "false" if lb.Opts.SkipSNAT { skipSNAT = "true" @@ -310,10 +366,20 @@ func buildLB(lb *LB) *nbdb.LoadBalancer { options["affinity_timeout"] = fmt.Sprintf("%d", lb.Opts.AffinityTimeOut) } - // vipMap - vips := buildVipMap(lb.Rules) + if lb.Opts.Template { + options["template"] = "true" + + // Address family is used only for template LBs. + if lb.Opts.AddressFamily != "" { + // OVN expects the family as lowercase... + options["address-family"] = strings.ToLower(fmt.Sprintf("%v", lb.Opts.AddressFamily)) + } + } - return libovsdbops.BuildLoadBalancer(lb.Name, strings.ToLower(lb.Protocol), vips, options, lb.ExternalIDs) + return &templateLoadBalancer{ + nbLB: libovsdbops.BuildLoadBalancer(lb.Name, strings.ToLower(lb.Protocol), buildVipMap(lb.Rules), options, lb.ExternalIDs), + templates: lb.Templates, + } } // buildVipMap returns a viups map from a set of rules @@ -351,17 +417,17 @@ func DeleteLBs(nbClient libovsdbclient.Client, uuids []string) error { } // getLBs returns a slice of load balancers found in OVN. -func getLBs(nbClient libovsdbclient.Client) ([]*LB, error) { - _, out, err := _getLBsCommon(nbClient, false) +func getLBs(nbClient libovsdbclient.Client, allTemplates TemplateMap) ([]*LB, error) { + _, out, err := _getLBsCommon(nbClient, allTemplates, false) return out, err } // getServiceLBs returns a set of services as well as a slice of load balancers found in OVN. -func getServiceLBs(nbClient libovsdbclient.Client) (sets.Set[string], []*LB, error) { - return _getLBsCommon(nbClient, true) +func getServiceLBs(nbClient libovsdbclient.Client, allTemplates TemplateMap) (sets.Set[string], []*LB, error) { + return _getLBsCommon(nbClient, allTemplates, true) } -func _getLBsCommon(nbClient libovsdbclient.Client, withServiceOwner bool) (sets.Set[string], []*LB, error) { +func _getLBsCommon(nbClient libovsdbclient.Client, allTemplates TemplateMap, withServiceOwner bool) (sets.Set[string], []*LB, error) { lbs, err := libovsdbops.ListLoadBalancers(nbClient) if err != nil { return nil, nil, fmt.Errorf("could not list load_balancer: %w", err) @@ -392,6 +458,7 @@ func _getLBsCommon(nbClient libovsdbclient.Client, withServiceOwner bool) (sets. ExternalIDs: lb.ExternalIDs, Opts: LBOpts{}, Rules: []LBRule{}, + Templates: getLoadBalancerTemplates(lb, allTemplates), Switches: []string{}, Routers: []string{}, Groups: []string{}, diff --git a/go-controller/pkg/ovn/controller/services/loadbalancer_test.go b/go-controller/pkg/ovn/controller/services/loadbalancer_test.go index 5a7599024c..27d47a874f 100644 --- a/go-controller/pkg/ovn/controller/services/loadbalancer_test.go +++ b/go-controller/pkg/ovn/controller/services/loadbalancer_test.go @@ -40,8 +40,8 @@ func TestEnsureLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"1.2.3.4", 80}, - Targets: []Addr{{"169.254.169.2", 8080}}, + Source: Addr{IP: "1.2.3.4", Port: 80}, + Targets: []Addr{{IP: "169.254.169.2", Port: 8080}}, }, }, UUID: "test-UUID", @@ -55,8 +55,8 @@ func TestEnsureLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"4.5.6.7", 80}, - Targets: []Addr{{"169.254.169.2", 8080}}, + Source: Addr{IP: "4.5.6.7", Port: 80}, + Targets: []Addr{{IP: "169.254.169.2", Port: 8080}}, }, }, UUID: "test-UUID2", @@ -73,8 +73,8 @@ func TestEnsureLBs(t *testing.T) { Protocol: "TCP", Rules: []LBRule{ { - Source: Addr{"1.2.3.4", 80}, - Targets: []Addr{{"169.254.169.2", 8080}}, + Source: Addr{IP: "1.2.3.4", Port: 80}, + Targets: []Addr{{IP: "169.254.169.2", Port: 8080}}, }, }, UUID: "", // intentionally left empty to make sure EnsureLBs sets it properly diff --git a/go-controller/pkg/ovn/controller/services/node_tracker.go b/go-controller/pkg/ovn/controller/services/node_tracker.go index 31846230ba..bae71b6f54 100644 --- a/go-controller/pkg/ovn/controller/services/node_tracker.go +++ b/go-controller/pkg/ovn/controller/services/node_tracker.go @@ -28,28 +28,37 @@ type nodeTracker struct { nodes map[string]nodeInfo // resyncFn is the function to call so that all service are resynced - resyncFn func() + resyncFn func(nodes []nodeInfo) } type nodeInfo struct { // the node's Name name string // The list of physical IPs the node has, as reported by the gatewayconf annotation - nodeIPs []string + nodeIPs []net.IP // The pod network subnet(s) podSubnets []net.IPNet // the name of the node's GatewayRouter, or "" of non-existent gatewayRouterName string // The name of the node's switch - never empty switchName string + // The chassisID of the node (ovs.external-ids:system-id) + chassisID string +} + +func (ni *nodeInfo) nodeIPsStr() []string { + out := make([]string, 0, len(ni.nodeIPs)) + for _, nodeIP := range ni.nodeIPs { + out = append(out, nodeIP.String()) + } + return out } // returns a list of all ip blocks "assigned" to this node // includes node IPs, still as a mask-1 net func (ni *nodeInfo) nodeSubnets() []net.IPNet { out := append([]net.IPNet{}, ni.podSubnets...) - for _, ipStr := range ni.nodeIPs { - ip := net.ParseIP(ipStr) + for _, ip := range ni.nodeIPs { if ipv4 := ip.To4(); ipv4 != nil { out = append(out, net.IPNet{ IP: ip, @@ -96,7 +105,8 @@ func newNodeTracker(nodeInformer coreinformers.NodeInformer) (*nodeTracker, erro // updateNode needs to be called only when hostSubnet annotation has changed or // if L3Gateway annotation's ip addresses have changed or the name of the node (very rare) // has changed. No need to trigger update for any other field change. - if util.NodeSubnetAnnotationChanged(oldObj, newObj) || util.NodeL3GatewayAnnotationChanged(oldObj, newObj) || oldObj.Name != newObj.Name { + if util.NodeSubnetAnnotationChanged(oldObj, newObj) || util.NodeL3GatewayAnnotationChanged(oldObj, newObj) || + util.NodeChassisIDAnnotationChanged(oldObj, newObj) || oldObj.Name != newObj.Name { nt.updateNode(newObj) } }, @@ -126,39 +136,42 @@ func newNodeTracker(nodeInformer coreinformers.NodeInformer) (*nodeTracker, erro // updateNodeInfo updates the node info cache, and syncs all services // if it changed. -func (nt *nodeTracker) updateNodeInfo(nodeName, switchName, routerName string, nodeIPs []string, podSubnets []*net.IPNet) { +func (nt *nodeTracker) updateNodeInfo(nodeName, switchName, routerName, chassisID string, nodeIPs []net.IP, podSubnets []*net.IPNet) { ni := nodeInfo{ name: nodeName, nodeIPs: nodeIPs, podSubnets: make([]net.IPNet, 0, len(podSubnets)), gatewayRouterName: routerName, switchName: switchName, + chassisID: chassisID, } for i := range podSubnets { ni.podSubnets = append(ni.podSubnets, *podSubnets[i]) // de-pointer } + klog.Infof("Node %s switch + router changed, syncing services", nodeName) + nt.Lock() + defer nt.Unlock() if existing, ok := nt.nodes[nodeName]; ok { if reflect.DeepEqual(existing, ni) { - nt.Unlock() return } } nt.nodes[nodeName] = ni - nt.Unlock() - klog.Infof("Node %s switch + router changed, syncing services", nodeName) // Resync all services - nt.resyncFn() + nt.resyncFn(nt.allNodes()) } // removeNodeWithServiceReSync removes a node from the LB -> node mapper // *and* forces full reconciliation of services. func (nt *nodeTracker) removeNodeWithServiceReSync(nodeName string) { nt.removeNode(nodeName) - nt.resyncFn() + nt.Lock() + nt.resyncFn(nt.allNodes()) + nt.Unlock() } // RemoveNode removes a node from the LB -> node mapper @@ -186,7 +199,8 @@ func (nt *nodeTracker) updateNode(node *v1.Node) { switchName := node.Name grName := "" - ips := []string{} + ips := []net.IP{} + chassisID := "" // if the node has a gateway config, it will soon have a gateway router // so, set the router name @@ -197,15 +211,17 @@ func (nt *nodeTracker) updateNode(node *v1.Node) { grName = util.GetGatewayRouterFromNode(node.Name) if gwConf.NodePortEnable { for _, ip := range gwConf.IPAddresses { - ips = append(ips, ip.IP.String()) + ips = append(ips, ip.IP) } } + chassisID = gwConf.ChassisID } nt.updateNodeInfo( node.Name, switchName, grName, + chassisID, ips, hsn, ) @@ -213,9 +229,6 @@ func (nt *nodeTracker) updateNode(node *v1.Node) { // allNodes returns a list of all nodes (and their relevant information) func (nt *nodeTracker) allNodes() []nodeInfo { - nt.Lock() - defer nt.Unlock() - out := make([]nodeInfo, 0, len(nt.nodes)) for _, node := range nt.nodes { out = append(out, node) diff --git a/go-controller/pkg/ovn/controller/services/repair.go b/go-controller/pkg/ovn/controller/services/repair.go index 8e3f4e05d6..9687880144 100644 --- a/go-controller/pkg/ovn/controller/services/repair.go +++ b/go-controller/pkg/ovn/controller/services/repair.go @@ -67,13 +67,30 @@ func (r *repair) runBeforeSync() { r.unsyncedServices.Insert(key) } + // Find all templates. + allTemplates, err := listSvcTemplates(r.nbClient) + if err != nil { + klog.Errorf("Unable to get templates for repair: %v", err) + } + // Find all load-balancers associated with Services - existingLBs, err := getLBs(r.nbClient) + existingLBs, err := getLBs(r.nbClient, allTemplates) if err != nil { klog.Errorf("Unable to get service lbs for repair: %v", err) } // Look for any load balancers whose Service no longer exists in the apiserver + // and for any chassis template vars whose Service no longer exists. + staleTemplateNames := sets.Set[string]{} + for templateName := range allTemplates { + // NodeIP templates are always valid, skip those. + if isLBNodeIPTemplateName(templateName) { + continue + } + + // All others are potentially stale. + staleTemplateNames.Insert(templateName) + } staleLBs := []string{} for _, lb := range existingLBs { // Extract namespace + name, look to see if it exists @@ -82,12 +99,19 @@ func (r *repair) runBeforeSync() { if err != nil || namespace == "" { klog.Warningf("Service LB %#v has unreadable owner, deleting", lb) staleLBs = append(staleLBs, lb.UUID) + continue } _, err = r.serviceLister.Services(namespace).Get(name) if apierrors.IsNotFound(err) { klog.V(5).Infof("Found stale service LB %#v", lb) staleLBs = append(staleLBs, lb.UUID) + continue + } + + // All of the LB's template vars are still useful. + for _, t := range lb.Templates { + staleTemplateNames.Delete(t.Name) } } @@ -97,6 +121,12 @@ func (r *repair) runBeforeSync() { } klog.V(2).Infof("Deleted %d stale service LBs", len(staleLBs)) + // Delete those stale template vars + if err := libovsdbops.DeleteAllChassisTemplateVarVariables(r.nbClient, staleTemplateNames.UnsortedList()); err != nil { + klog.Errorf("Failed to delete stale Chassis Template Vars: %v", err) + } + klog.V(2).Infof("Deleted %d stale Chassis Template Vars", len(staleTemplateNames)) + // Remove existing reject rules. They are not used anymore // given the introduction of idling loadbalancers p := func(item *nbdb.ACL) bool { diff --git a/go-controller/pkg/ovn/controller/services/services_controller.go b/go-controller/pkg/ovn/controller/services/services_controller.go index e3e4745efb..f6d94dafd0 100644 --- a/go-controller/pkg/ovn/controller/services/services_controller.go +++ b/go-controller/pkg/ovn/controller/services/services_controller.go @@ -7,6 +7,7 @@ import ( "time" libovsdbclient "github.com/ovn-org/libovsdb/client" + globalconfig "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" @@ -62,6 +63,8 @@ func NewController(client clientset.Interface, queue: workqueue.NewNamedRateLimitingQueue(newRatelimiter(100), controllerName), workerLoopPeriod: time.Second, alreadyApplied: map[string][]LB{}, + nodeIPv4Template: makeTemplate(makeLBNodeIPTemplateName(v1.IPv4Protocol)), + nodeIPv6Template: makeTemplate(makeLBNodeIPTemplateName(v1.IPv6Protocol)), } // services @@ -147,18 +150,34 @@ type Controller struct { // nodeTracker nodeTracker *nodeTracker + // Per node information and template variables. The latter expand to each + // chassis' node IP (v4 and v6). + // Must be accessed only with the nodeInfo mutex taken. + // These are written in RequestFullSync(). + nodeInfos []nodeInfo + nodeIPv4Template *Template + nodeIPv6Template *Template + nodeInfoRWLock sync.RWMutex + // alreadyApplied is a map of service key -> already applied configuration, so we can short-circuit // if a service's config hasn't changed - alreadyApplied map[string][]LB - alreadyAppliedLock sync.Mutex + alreadyApplied map[string][]LB + alreadyAppliedRWLock sync.RWMutex + + // Lock order considerations: if both nodeInfoRWLock and alreadyAppliedRWLock + // need to be taken for some reason then the order in which they're taken is + // always: first nodeInfoRWLock and then alreadyAppliedRWLock. // 'true' if Load_Balancer_Group is supported. useLBGroups bool + + // 'true' if Chassis_Template_Var is supported. + useTemplates bool } // Run will not return until stopCh is closed. workers determines how many // endpoints will be handled in parallel. -func (c *Controller) Run(workers int, stopCh <-chan struct{}, runRepair, useLBGroups bool) error { +func (c *Controller) Run(workers int, stopCh <-chan struct{}, runRepair, useLBGroups, useTemplates bool) error { defer utilruntime.HandleCrash() defer c.queue.ShutDown() @@ -166,6 +185,7 @@ func (c *Controller) Run(workers int, stopCh <-chan struct{}, runRepair, useLBGr defer klog.Infof("Shutting down controller %s", controllerName) c.useLBGroups = useLBGroups + c.useTemplates = useTemplates // Wait for the caches to be synced klog.Info("Waiting for informer caches to sync") @@ -246,11 +266,19 @@ func (c *Controller) handleErr(err error, key interface{}) { // That is because such work will be performed in syncService, so all that is needed here is the ability // to distinguish what is present in ovn database and this 'dirty' initial value. func (c *Controller) initTopLevelCache() error { - c.alreadyAppliedLock.Lock() - defer c.alreadyAppliedLock.Unlock() + var err error + + c.alreadyAppliedRWLock.Lock() + defer c.alreadyAppliedRWLock.Unlock() - // first, list all load balancers and their respective services - services, lbs, err := getServiceLBs(c.nbClient) + // First list all the templates. + allTemplates, err := listSvcTemplates(c.nbClient) + if err != nil { + return fmt.Errorf("failed to load templates: %w", err) + } + + // Then list all load balancers and their respective services. + services, lbs, err := getServiceLBs(c.nbClient, allTemplates) if err != nil { return fmt.Errorf("failed to load balancers: %w", err) } @@ -288,6 +316,11 @@ func (c *Controller) syncService(key string) error { metrics.MetricSyncServiceLatency.Observe(time.Since(startTime).Seconds()) }() + // Shared node information (c.nodeInfos, c.nodeIPv4Template, c.nodeIPv6Template) + // needs to be accessed with the nodeInfoRWLock taken for read. + c.nodeInfoRWLock.RLock() + defer c.nodeInfoRWLock.RUnlock() + // Get current Service from the cache service, err := c.serviceLister.Services(namespace).Get(name) // It´s unlikely that we have an error different that "Not Found Object" @@ -307,14 +340,14 @@ func (c *Controller) syncService(key string) error { }, } - c.alreadyAppliedLock.Lock() + c.alreadyAppliedRWLock.RLock() alreadyAppliedLbs, alreadyAppliedKeyExists := c.alreadyApplied[key] var existingLBs []LB if alreadyAppliedKeyExists { existingLBs = make([]LB, len(alreadyAppliedLbs)) copy(existingLBs, alreadyAppliedLbs) } - c.alreadyAppliedLock.Unlock() + c.alreadyAppliedRWLock.RUnlock() if alreadyAppliedKeyExists { // @@ -328,9 +361,9 @@ func (c *Controller) syncService(key string) error { namespace, name, err) } - c.alreadyAppliedLock.Lock() + c.alreadyAppliedRWLock.Lock() delete(c.alreadyApplied, key) - c.alreadyAppliedLock.Unlock() + c.alreadyAppliedRWLock.Unlock() } c.repair.serviceSynced(key) @@ -355,29 +388,35 @@ func (c *Controller) syncService(key string) error { } // Build the abstract LB configs for this service - perNodeConfigs, clusterConfigs := buildServiceLBConfigs(service, endpointSlices) + perNodeConfigs, templateConfigs, clusterConfigs := buildServiceLBConfigs(service, endpointSlices, + c.useLBGroups, c.useTemplates) klog.V(5).Infof("Built service %s LB cluster-wide configs %#v", key, clusterConfigs) klog.V(5).Infof("Built service %s LB per-node configs %#v", key, perNodeConfigs) + klog.V(5).Infof("Built service %s LB template configs %#v", key, templateConfigs) // Convert the LB configs in to load-balancer objects - nodeInfos := c.nodeTracker.allNodes() - clusterLBs := buildClusterLBs(service, clusterConfigs, nodeInfos, c.useLBGroups) - perNodeLBs := buildPerNodeLBs(service, perNodeConfigs, nodeInfos) + clusterLBs := buildClusterLBs(service, clusterConfigs, c.nodeInfos, c.useLBGroups) + templateLBs := buildTemplateLBs(service, templateConfigs, c.nodeInfos, + c.nodeIPv4Template, c.nodeIPv6Template) + perNodeLBs := buildPerNodeLBs(service, perNodeConfigs, c.nodeInfos) klog.V(5).Infof("Built service %s cluster-wide LB %#v", key, clusterLBs) klog.V(5).Infof("Built service %s per-node LB %#v", key, perNodeLBs) - klog.V(3).Infof("Service %s has %d cluster-wide and %d per-node configs, making %d and %d load balancers", - key, len(clusterConfigs), len(perNodeConfigs), len(clusterLBs), len(perNodeLBs)) - lbs := append(clusterLBs, perNodeLBs...) + klog.V(5).Infof("Built service %s template LB %#v", key, templateLBs) + klog.V(3).Infof("Service %s has %d cluster-wide, %d per-node configs, %d template configs, making %d (cluster) %d (per node) and %d (template) load balancers", + key, len(clusterConfigs), len(perNodeConfigs), len(templateConfigs), + len(clusterLBs), len(perNodeLBs), len(templateLBs)) + lbs := append(clusterLBs, templateLBs...) + lbs = append(lbs, perNodeLBs...) // Short-circuit if nothing has changed - c.alreadyAppliedLock.Lock() + c.alreadyAppliedRWLock.RLock() alreadyAppliedLbs, alreadyAppliedKeyExists := c.alreadyApplied[key] var existingLBs []LB if alreadyAppliedKeyExists { existingLBs = make([]LB, len(alreadyAppliedLbs)) copy(existingLBs, alreadyAppliedLbs) } - c.alreadyAppliedLock.Unlock() + c.alreadyAppliedRWLock.RUnlock() if alreadyAppliedKeyExists && LoadBalancersEqualNoUUID(existingLBs, lbs) { klog.V(3).Infof("Skipping no-op change for service %s", key) @@ -391,23 +430,78 @@ func (c *Controller) syncService(key string) error { return fmt.Errorf("failed to ensure service %s load balancers: %w", key, err) } - c.alreadyAppliedLock.Lock() + c.alreadyAppliedRWLock.Lock() c.alreadyApplied[key] = lbs - c.alreadyAppliedLock.Unlock() + c.alreadyAppliedRWLock.Unlock() } c.repair.serviceSynced(key) return nil } +func (c *Controller) syncNodeInfos(nodeInfos []nodeInfo) { + c.nodeInfoRWLock.Lock() + defer c.nodeInfoRWLock.Unlock() + + c.nodeInfos = nodeInfos + if !c.useTemplates { + return + } + + // Compute the nodeIP template values. + c.nodeIPv4Template = makeTemplate(makeLBNodeIPTemplateName(v1.IPv4Protocol)) + c.nodeIPv6Template = makeTemplate(makeLBNodeIPTemplateName(v1.IPv6Protocol)) + + for _, node := range c.nodeInfos { + if node.chassisID == "" { + continue + } + // Services are currently supported only on the node's first IP. + // Extract that one and populate the node's IP template value. + if globalconfig.IPv4Mode { + if ipv4, err := util.MatchFirstIPFamily(false, node.nodeIPs); err == nil { + c.nodeIPv4Template.Value[node.chassisID] = ipv4.String() + } + } + if globalconfig.IPv6Mode { + if ipv6, err := util.MatchFirstIPFamily(true, node.nodeIPs); err == nil { + c.nodeIPv6Template.Value[node.chassisID] = ipv6.String() + } + } + } + + // Sync the nodeIP template values to the DB. + nodeIPTemplateMap := TemplateMap{} + if c.nodeIPv4Template.len() > 0 { + nodeIPTemplateMap[c.nodeIPv4Template.Name] = c.nodeIPv4Template + } + if c.nodeIPv6Template.len() > 0 { + nodeIPTemplateMap[c.nodeIPv6Template.Name] = c.nodeIPv6Template + } + + nodeIPTemplates := []TemplateMap{ + nodeIPTemplateMap, + } + if err := svcCreateOrUpdateTemplateVar(c.nbClient, nodeIPTemplates); err != nil { + klog.Errorf("Could not sync node IP templates") + return + } +} + // RequestFullSync re-syncs every service that currently exists -func (c *Controller) RequestFullSync() { +func (c *Controller) RequestFullSync(nodeInfos []nodeInfo) { klog.Info("Full service sync requested") + + // Resync node infos and node IP templates. + c.syncNodeInfos(nodeInfos) + + // Resync services. services, err := c.serviceLister.List(labels.Everything()) if err != nil { klog.Errorf("Cached lister failed!? %v", err) return } + for _, service := range services { c.onServiceAdd(service) } diff --git a/go-controller/pkg/ovn/controller/services/services_controller_test.go b/go-controller/pkg/ovn/controller/services/services_controller_test.go index 457d5d006d..c3b50d8a2d 100644 --- a/go-controller/pkg/ovn/controller/services/services_controller_test.go +++ b/go-controller/pkg/ovn/controller/services/services_controller_test.go @@ -1,7 +1,6 @@ package services import ( - "context" "fmt" "net" "strings" @@ -10,18 +9,15 @@ import ( "github.com/onsi/ginkgo" "github.com/onsi/gomega" "github.com/onsi/gomega/format" - libovsdbclient "github.com/ovn-org/libovsdb/client" globalconfig "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" libovsdbtest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/libovsdb" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" - "github.com/pkg/errors" v1 "k8s.io/api/core/v1" discovery "k8s.io/api/discovery/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/tools/cache" @@ -66,6 +62,8 @@ func newControllerWithDBSetup(dbSetup libovsdbtest.TestSetup) (*serviceControlle controller.servicesSynced = alwaysReady controller.endpointSlicesSynced = alwaysReady controller.initTopLevelCache() + controller.useLBGroups = true + controller.useTemplates = true return &serviceController{ controller, informerFactory.Core().V1().Services().Informer().GetStore(), @@ -88,6 +86,8 @@ func TestSyncServices(t *testing.T) { ns := "testns" serviceName := "foo" + initialLsGroups := []string{types.ClusterLBGroupName, types.ClusterSwitchLBGroupName} + initialLrGroups := []string{types.ClusterLBGroupName, types.ClusterRouterLBGroupName} oldGateway := globalconfig.Gateway.Mode oldClusterSubnet := globalconfig.Default.ClusterSubnets @@ -161,10 +161,13 @@ func TestSyncServices(t *testing.T) { }, }, initialDb: []libovsdbtest.TestData{ - nodeLogicalSwitch(nodeA), - nodeLogicalSwitch(nodeB), - nodeLogicalRouter(nodeA), - nodeLogicalRouter(nodeB), + nodeLogicalSwitch(nodeA, initialLsGroups), + nodeLogicalSwitch(nodeB, initialLsGroups), + nodeLogicalRouter(nodeA, initialLrGroups), + nodeLogicalRouter(nodeB, initialLrGroups), + lbGroup(types.ClusterLBGroupName), + lbGroup(types.ClusterSwitchLBGroupName), + lbGroup(types.ClusterRouterLBGroupName), }, expectedDb: []libovsdbtest.TestData{ &nbdb.LoadBalancer{ @@ -177,10 +180,15 @@ func TestSyncServices(t *testing.T) { }, ExternalIDs: serviceExternalIDs(namespacedServiceName(ns, serviceName)), }, - nodeLogicalSwitch(nodeA, loadBalancerClusterWideTCPServiceName(ns, serviceName)), - nodeLogicalSwitch(nodeB, loadBalancerClusterWideTCPServiceName(ns, serviceName)), - nodeLogicalRouter(nodeA, loadBalancerClusterWideTCPServiceName(ns, serviceName)), - nodeLogicalRouter(nodeB, loadBalancerClusterWideTCPServiceName(ns, serviceName)), + nodeLogicalSwitch(nodeA, initialLsGroups), + nodeLogicalSwitch(nodeB, initialLsGroups), + nodeLogicalRouter(nodeA, initialLrGroups), + nodeLogicalRouter(nodeB, initialLrGroups), + lbGroup(types.ClusterLBGroupName, loadBalancerClusterWideTCPServiceName(ns, serviceName)), + lbGroup(types.ClusterSwitchLBGroupName), + lbGroup(types.ClusterRouterLBGroupName), + nodeIPTemplate(firstNode), + nodeIPTemplate(secondNode), }, }, { @@ -220,12 +228,15 @@ func TestSyncServices(t *testing.T) { }, ExternalIDs: serviceExternalIDs(namespacedServiceName(ns, serviceName)), }, - nodeLogicalSwitch(nodeA), - nodeLogicalSwitch(nodeB), - nodeLogicalSwitch("wrong-switch", loadBalancerClusterWideTCPServiceName(ns, serviceName)), - nodeLogicalRouter(nodeA, loadBalancerClusterWideTCPServiceName(ns, serviceName)), - nodeLogicalRouter(nodeB, loadBalancerClusterWideTCPServiceName(ns, serviceName)), - nodeLogicalRouter("node-c", loadBalancerClusterWideTCPServiceName(ns, serviceName)), + nodeLogicalSwitch(nodeA, initialLsGroups), + nodeLogicalSwitch(nodeB, initialLsGroups), + nodeLogicalSwitch("wrong-switch", []string{}, loadBalancerClusterWideTCPServiceName(ns, serviceName)), + nodeLogicalRouter(nodeA, initialLrGroups, loadBalancerClusterWideTCPServiceName(ns, serviceName)), + nodeLogicalRouter(nodeB, initialLrGroups, loadBalancerClusterWideTCPServiceName(ns, serviceName)), + nodeLogicalRouter("node-c", []string{}, loadBalancerClusterWideTCPServiceName(ns, serviceName)), + lbGroup(types.ClusterLBGroupName), + lbGroup(types.ClusterSwitchLBGroupName), + lbGroup(types.ClusterRouterLBGroupName), }, expectedDb: []libovsdbtest.TestData{ &nbdb.LoadBalancer{ @@ -238,12 +249,17 @@ func TestSyncServices(t *testing.T) { }, ExternalIDs: serviceExternalIDs(namespacedServiceName(ns, serviceName)), }, - nodeLogicalSwitch(nodeA, loadBalancerClusterWideTCPServiceName(ns, serviceName)), - nodeLogicalSwitch(nodeB, loadBalancerClusterWideTCPServiceName(ns, serviceName)), - nodeLogicalSwitch("wrong-switch"), - nodeLogicalRouter(nodeA, loadBalancerClusterWideTCPServiceName(ns, serviceName)), - nodeLogicalRouter(nodeB, loadBalancerClusterWideTCPServiceName(ns, serviceName)), - nodeLogicalRouter("node-c"), + nodeLogicalSwitch(nodeA, initialLsGroups), + nodeLogicalSwitch(nodeB, initialLsGroups), + nodeLogicalSwitch("wrong-switch", []string{}), + nodeLogicalRouter(nodeA, initialLrGroups), + nodeLogicalRouter(nodeB, initialLrGroups), + nodeLogicalRouter("node-c", []string{}), + lbGroup(types.ClusterLBGroupName, loadBalancerClusterWideTCPServiceName(ns, serviceName)), + lbGroup(types.ClusterSwitchLBGroupName), + lbGroup(types.ClusterRouterLBGroupName), + nodeIPTemplate(firstNode), + nodeIPTemplate(secondNode), }, }, { @@ -296,10 +312,13 @@ func TestSyncServices(t *testing.T) { }, ExternalIDs: serviceExternalIDs(namespacedServiceName(ns, serviceName)), }, - nodeLogicalSwitch(nodeA, loadBalancerClusterWideTCPServiceName(ns, serviceName)), - nodeLogicalSwitch(nodeB, loadBalancerClusterWideTCPServiceName(ns, serviceName)), - nodeLogicalRouter(nodeA, loadBalancerClusterWideTCPServiceName(ns, serviceName)), - nodeLogicalRouter(nodeB, loadBalancerClusterWideTCPServiceName(ns, serviceName)), + nodeLogicalSwitch(nodeA, initialLsGroups, loadBalancerClusterWideTCPServiceName(ns, serviceName)), + nodeLogicalSwitch(nodeB, initialLsGroups, loadBalancerClusterWideTCPServiceName(ns, serviceName)), + nodeLogicalRouter(nodeA, initialLrGroups, loadBalancerClusterWideTCPServiceName(ns, serviceName)), + nodeLogicalRouter(nodeB, initialLrGroups, loadBalancerClusterWideTCPServiceName(ns, serviceName)), + lbGroup(types.ClusterLBGroupName), + lbGroup(types.ClusterSwitchLBGroupName), + lbGroup(types.ClusterRouterLBGroupName), }, expectedDb: []libovsdbtest.TestData{ &nbdb.LoadBalancer{ @@ -312,20 +331,16 @@ func TestSyncServices(t *testing.T) { }, ExternalIDs: serviceExternalIDs(namespacedServiceName(ns, serviceName)), }, - nodeRouterLoadBalancer(firstNode, nodePort, serviceName, ns, outport, nodeAEndpointIP, nodeBEndpointIP), - nodeRouterLoadBalancer(secondNode, nodePort, serviceName, ns, outport, nodeAEndpointIP, nodeBEndpointIP), - nodeLogicalSwitch(nodeA, - loadBalancerClusterWideTCPServiceName(ns, serviceName), - nodeSwitchRouterLoadBalancerName(nodeA, ns, serviceName)), - nodeLogicalSwitch(nodeB, - loadBalancerClusterWideTCPServiceName(ns, serviceName), - nodeSwitchRouterLoadBalancerName(nodeB, ns, serviceName)), - nodeLogicalRouter(nodeA, - loadBalancerClusterWideTCPServiceName(ns, serviceName), - nodeSwitchRouterLoadBalancerName(nodeA, ns, serviceName)), - nodeLogicalRouter(nodeB, - loadBalancerClusterWideTCPServiceName(ns, serviceName), - nodeSwitchRouterLoadBalancerName(nodeB, ns, serviceName)), + nodeMergedTemplateLoadBalancer(nodePort, serviceName, ns, outport, nodeAEndpointIP, nodeBEndpointIP), + nodeLogicalSwitch(nodeA, initialLsGroups), + nodeLogicalSwitch(nodeB, initialLsGroups), + nodeLogicalRouter(nodeA, initialLrGroups), + nodeLogicalRouter(nodeB, initialLrGroups), + lbGroup(types.ClusterLBGroupName, loadBalancerClusterWideTCPServiceName(ns, serviceName)), + lbGroup(types.ClusterSwitchLBGroupName, nodeMergedTemplateLoadBalancerName(ns, serviceName, v1.IPv4Protocol)), + lbGroup(types.ClusterRouterLBGroupName, nodeMergedTemplateLoadBalancerName(ns, serviceName, v1.IPv4Protocol)), + nodeIPTemplate(firstNode), + nodeIPTemplate(secondNode), }, }, { @@ -378,14 +393,14 @@ func TestSyncServices(t *testing.T) { }, ExternalIDs: serviceExternalIDs(namespacedServiceName(ns, serviceName)), }, - nodeLogicalSwitch(nodeA, - loadBalancerClusterWideTCPServiceName(ns, serviceName)), - nodeLogicalSwitch(nodeB, - loadBalancerClusterWideTCPServiceName(ns, serviceName)), - nodeLogicalRouter(nodeA, - loadBalancerClusterWideTCPServiceName(ns, serviceName)), - nodeLogicalRouter(nodeB, + nodeLogicalSwitch(nodeA, initialLsGroups), + nodeLogicalSwitch(nodeB, initialLsGroups), + nodeLogicalRouter(nodeA, initialLrGroups), + nodeLogicalRouter(nodeB, initialLrGroups), + lbGroup(types.ClusterLBGroupName, loadBalancerClusterWideTCPServiceName(ns, serviceName)), + lbGroup(types.ClusterSwitchLBGroupName), + lbGroup(types.ClusterRouterLBGroupName), }, expectedDb: []libovsdbtest.TestData{ &nbdb.LoadBalancer{ @@ -398,20 +413,16 @@ func TestSyncServices(t *testing.T) { }, ExternalIDs: serviceExternalIDs(namespacedServiceName(ns, serviceName)), }, - nodeRouterLoadBalancer(firstNode, nodePort, serviceName, ns, outport, nodeAEndpointIP, nodeBEndpointIP), - nodeRouterLoadBalancer(secondNode, nodePort, serviceName, ns, outport, nodeAEndpointIP, nodeBEndpointIP), - nodeLogicalSwitch(nodeA, - loadBalancerClusterWideTCPServiceName(ns, serviceName), - nodeSwitchRouterLoadBalancerName(nodeA, ns, serviceName)), - nodeLogicalSwitch(nodeB, - loadBalancerClusterWideTCPServiceName(ns, serviceName), - nodeSwitchRouterLoadBalancerName(nodeB, ns, serviceName)), - nodeLogicalRouter(nodeA, - loadBalancerClusterWideTCPServiceName(ns, serviceName), - nodeSwitchRouterLoadBalancerName(nodeA, ns, serviceName)), - nodeLogicalRouter(nodeB, - loadBalancerClusterWideTCPServiceName(ns, serviceName), - nodeSwitchRouterLoadBalancerName(nodeB, ns, serviceName)), + nodeMergedTemplateLoadBalancer(nodePort, serviceName, ns, outport, nodeAEndpointIP, nodeBEndpointIP), + nodeLogicalSwitch(nodeA, initialLsGroups), + nodeLogicalSwitch(nodeB, initialLsGroups), + nodeLogicalRouter(nodeA, initialLrGroups), + nodeLogicalRouter(nodeB, initialLrGroups), + lbGroup(types.ClusterLBGroupName, loadBalancerClusterWideTCPServiceName(ns, serviceName)), + lbGroup(types.ClusterSwitchLBGroupName, nodeMergedTemplateLoadBalancerName(ns, serviceName, v1.IPv4Protocol)), + lbGroup(types.ClusterRouterLBGroupName, nodeMergedTemplateLoadBalancerName(ns, serviceName, v1.IPv4Protocol)), + nodeIPTemplate(firstNode), + nodeIPTemplate(secondNode), }, nodeToDelete: nodeConfig(nodeA, nodeAHostIP), dbStateAfterDeleting: []libovsdbtest.TestData{ @@ -425,15 +436,16 @@ func TestSyncServices(t *testing.T) { }, ExternalIDs: serviceExternalIDs(namespacedServiceName(ns, serviceName)), }, - nodeRouterLoadBalancer(secondNode, nodePort, serviceName, ns, outport, nodeAEndpointIP, nodeBEndpointIP), - nodeLogicalSwitch(nodeA), - nodeLogicalSwitch(nodeB, - loadBalancerClusterWideTCPServiceName(ns, serviceName), - nodeSwitchRouterLoadBalancerName(nodeB, ns, serviceName)), - nodeLogicalRouter(nodeA), - nodeLogicalRouter(nodeB, - loadBalancerClusterWideTCPServiceName(ns, serviceName), - nodeSwitchRouterLoadBalancerName(nodeB, ns, serviceName)), + nodeMergedTemplateLoadBalancer(nodePort, serviceName, ns, outport, nodeAEndpointIP, nodeBEndpointIP), + nodeLogicalSwitch(nodeA, initialLsGroups), + nodeLogicalSwitch(nodeB, initialLsGroups), + nodeLogicalRouter(nodeA, initialLrGroups), + nodeLogicalRouter(nodeB, initialLrGroups), + lbGroup(types.ClusterLBGroupName, loadBalancerClusterWideTCPServiceName(ns, serviceName)), + lbGroup(types.ClusterSwitchLBGroupName, nodeMergedTemplateLoadBalancerName(ns, serviceName, v1.IPv4Protocol)), + lbGroup(types.ClusterRouterLBGroupName, nodeMergedTemplateLoadBalancerName(ns, serviceName, v1.IPv4Protocol)), + nodeIPTemplate(firstNode), + nodeIPTemplate(secondNode), }, }, } @@ -458,6 +470,7 @@ func TestSyncServices(t *testing.T) { controller.serviceStore.Add(tt.service) controller.nodeTracker.nodes = defaultNodes + controller.RequestFullSync(controller.nodeTracker.allNodes()) err = controller.syncService(ns + "/" + serviceName) if err != nil { @@ -468,40 +481,18 @@ func TestSyncServices(t *testing.T) { if tt.nodeToDelete != nil { controller.nodeTracker.removeNode(tt.nodeToDelete.name) - - // we need to extract the deleted load balancer UUID, because - // of a test library limitation: it does not delete the weak - // references to the load balancers on the logical switches / - // logical routers. - nodeLoadBalancerUUID, err := extractLoadBalancerRealUUID( - controller.nbClient, - nodeSwitchRouterLoadBalancerName(nodeA, ns, serviceName)) - g.Expect(err).NotTo(gomega.HaveOccurred()) - g.Expect(nodeLoadBalancerUUID).NotTo(gomega.BeEmpty()) - g.Expect(controller.syncService(namespacedServiceName(ns, serviceName))).To(gomega.Succeed()) - - // here, we need to patch the original expected data with the - // real load balancer UUID, on the logical switch / GW router - // of the deleted node, since the test library will be unable - // to correlate the name of the load balancer with its UUID. - // This happens because the load balancer was deleted. - // Patching up the load balancer UUIDs allows us to use the - // `HaveData` matcher below, thus increasing the test - // accurateness. - expectedData := patchLogicalRouterAndSwitchLoadBalancerUUIDs( - nodeLoadBalancerUUID, tt.dbStateAfterDeleting, nodeSwitchName(nodeA), nodeGWRouterName(nodeA)) - - g.Expect(controller.nbClient).To(libovsdbtest.HaveData(expectedData)) + g.Expect(controller.nbClient).To(libovsdbtest.HaveData(tt.dbStateAfterDeleting)) } }) } } -func nodeLogicalSwitch(nodeName string, namespacedServiceNames ...string) *nbdb.LogicalSwitch { +func nodeLogicalSwitch(nodeName string, lbGroups []string, namespacedServiceNames ...string) *nbdb.LogicalSwitch { ls := &nbdb.LogicalSwitch{ - UUID: nodeSwitchName(nodeName), - Name: nodeSwitchName(nodeName), + UUID: nodeSwitchName(nodeName), + Name: nodeSwitchName(nodeName), + LoadBalancerGroup: lbGroups, } if len(namespacedServiceNames) > 0 { ls.LoadBalancer = namespacedServiceNames @@ -509,15 +500,16 @@ func nodeLogicalSwitch(nodeName string, namespacedServiceNames ...string) *nbdb. return ls } -func nodeLogicalRouter(nodeName string, namespacedServiceNames ...string) *nbdb.LogicalRouter { - ls := &nbdb.LogicalRouter{ - UUID: nodeGWRouterName(nodeName), - Name: nodeGWRouterName(nodeName), +func nodeLogicalRouter(nodeName string, lbGroups []string, namespacedServiceNames ...string) *nbdb.LogicalRouter { + lr := &nbdb.LogicalRouter{ + UUID: nodeGWRouterName(nodeName), + Name: nodeGWRouterName(nodeName), + LoadBalancerGroup: lbGroups, } if len(namespacedServiceNames) > 0 { - ls.LoadBalancer = namespacedServiceNames + lr.LoadBalancer = namespacedServiceNames } - return ls + return lr } func nodeSwitchName(nodeName string) string { @@ -528,6 +520,17 @@ func nodeGWRouterName(nodeName string) string { return fmt.Sprintf("gr-%s", nodeName) } +func lbGroup(name string, namespacedServiceNames ...string) *nbdb.LoadBalancerGroup { + lbg := &nbdb.LoadBalancerGroup{ + UUID: name, + Name: name, + } + if len(namespacedServiceNames) > 0 { + lbg.LoadBalancer = namespacedServiceNames + } + return lbg +} + func loadBalancerClusterWideTCPServiceName(ns string, serviceName string) string { return fmt.Sprintf("Service_%s_TCP_cluster", namespacedServiceName(ns, serviceName)) } @@ -544,6 +547,30 @@ func nodeSwitchRouterLoadBalancerName(nodeName string, serviceNamespace string, nodeName) } +func nodeSwitchTemplateLoadBalancerName(serviceNamespace string, serviceName string, addressFamily v1.IPFamily) string { + return fmt.Sprintf( + "Service_%s/%s_TCP_node_switch_template_%s", + serviceNamespace, + serviceName, + addressFamily) +} + +func nodeRouterTemplateLoadBalancerName(serviceNamespace string, serviceName string, addressFamily v1.IPFamily) string { + return fmt.Sprintf( + "Service_%s/%s_TCP_node_router_template_%s", + serviceNamespace, + serviceName, + addressFamily) +} + +func nodeMergedTemplateLoadBalancerName(serviceNamespace string, serviceName string, addressFamily v1.IPFamily) string { + return fmt.Sprintf( + "Service_%s/%s_TCP_node_switch_template_%s_merged", + serviceNamespace, + serviceName, + addressFamily) +} + func servicesOptions() map[string]string { return map[string]string{ "event": "false", @@ -554,6 +581,14 @@ func servicesOptions() map[string]string { } } +func templateServicesOptions() map[string]string { + // Template LBs need "options:template=true" and "options:address-family" set. + opts := servicesOptions() + opts["template"] = "true" + opts["address-family"] = "ipv4" + return opts +} + func tcpGatewayRouterExternalIDs() map[string]string { return map[string]string{ "TCP_lb_gateway_router": "", @@ -567,19 +602,69 @@ func serviceExternalIDs(namespacedServiceName string) map[string]string { } } -func nodeRouterLoadBalancer(node *nodeInfo, nodePort int32, serviceName string, serviceNamespace string, outputPort int32, endpointIPs ...string) *nbdb.LoadBalancer { +func nodeSwitchTemplateLoadBalancer(nodePort int32, serviceName string, serviceNamespace string) *nbdb.LoadBalancer { + nodeTemplateIP := makeTemplate(makeLBNodeIPTemplateName(v1.IPv4Protocol)) + return &nbdb.LoadBalancer{ + UUID: nodeSwitchTemplateLoadBalancerName(serviceNamespace, serviceName, v1.IPv4Protocol), + Name: nodeSwitchTemplateLoadBalancerName(serviceNamespace, serviceName, v1.IPv4Protocol), + Options: templateServicesOptions(), + Protocol: &nbdb.LoadBalancerProtocolTCP, + Vips: map[string]string{ + endpoint(refTemplate(nodeTemplateIP.Name), nodePort): refTemplate(makeTarget(serviceName, serviceNamespace, "TCP", nodePort, "node_switch_template", v1.IPv4Protocol)), + }, + ExternalIDs: serviceExternalIDs(namespacedServiceName(serviceNamespace, serviceName)), + } +} + +func nodeRouterTemplateLoadBalancer(nodePort int32, serviceName string, serviceNamespace string) *nbdb.LoadBalancer { + nodeTemplateIP := makeTemplate(makeLBNodeIPTemplateName(v1.IPv4Protocol)) + return &nbdb.LoadBalancer{ + UUID: nodeRouterTemplateLoadBalancerName(serviceNamespace, serviceName, v1.IPv4Protocol), + Name: nodeRouterTemplateLoadBalancerName(serviceNamespace, serviceName, v1.IPv4Protocol), + Options: templateServicesOptions(), + Protocol: &nbdb.LoadBalancerProtocolTCP, + Vips: map[string]string{ + endpoint(refTemplate(nodeTemplateIP.Name), nodePort): refTemplate(makeTarget(serviceName, serviceNamespace, "TCP", nodePort, "node_router_template", v1.IPv4Protocol)), + }, + ExternalIDs: serviceExternalIDs(namespacedServiceName(serviceNamespace, serviceName)), + } +} + +func nodeIPTemplate(node *nodeInfo) *nbdb.ChassisTemplateVar { + return &nbdb.ChassisTemplateVar{ + UUID: node.chassisID, + Chassis: node.chassisID, + Variables: map[string]string{ + makeLBNodeIPTemplateName(v1.IPv4Protocol): node.nodeIPs[0].String(), + }, + } +} + +func nodeMergedTemplateLoadBalancer(nodePort int32, serviceName string, serviceNamespace string, outputPort int32, endpointIPs ...string) *nbdb.LoadBalancer { + nodeTemplateIP := makeTemplate(makeLBNodeIPTemplateName(v1.IPv4Protocol)) return &nbdb.LoadBalancer{ - UUID: nodeSwitchRouterLoadBalancerName(node.name, serviceNamespace, serviceName), - Name: nodeSwitchRouterLoadBalancerName(node.name, serviceNamespace, serviceName), - Options: servicesOptions(), + UUID: nodeMergedTemplateLoadBalancerName(serviceNamespace, serviceName, v1.IPv4Protocol), + Name: nodeMergedTemplateLoadBalancerName(serviceNamespace, serviceName, v1.IPv4Protocol), + Options: templateServicesOptions(), Protocol: &nbdb.LoadBalancerProtocolTCP, Vips: map[string]string{ - endpoint(node.nodeIPs[0], nodePort): computeEndpoints(outputPort, endpointIPs...), + endpoint(refTemplate(nodeTemplateIP.Name), nodePort): computeEndpoints(outputPort, endpointIPs...), }, ExternalIDs: serviceExternalIDs(namespacedServiceName(serviceNamespace, serviceName)), } } +func refTemplate(template string) string { + return "^" + template +} + +func makeTarget(serviceName, serviceNamespace string, proto v1.Protocol, outputPort int32, scope string, addressFamily v1.IPFamily) string { + return makeTemplateName( + fmt.Sprintf("Service_%s/%s_%s_%d_%s_%s", + serviceNamespace, serviceName, + proto, outputPort, scope, addressFamily)) +} + func computeEndpoints(outputPort int32, ips ...string) string { var endpoints []string for _, ip := range ips { @@ -595,9 +680,10 @@ func endpoint(ip string, port int32) string { func nodeConfig(nodeName string, nodeIP string) *nodeInfo { return &nodeInfo{ name: nodeName, - nodeIPs: []string{nodeIP}, + nodeIPs: []net.IP{net.ParseIP(nodeIP)}, gatewayRouterName: nodeGWRouterName(nodeName), switchName: nodeSwitchName(nodeName), + chassisID: nodeName, } } @@ -608,33 +694,3 @@ func temporarilyEnableGomegaMaxLengthFormat() { func restoreGomegaMaxLengthFormat(originalLength int) { format.MaxLength = originalLength } - -func extractLoadBalancerRealUUID(nbClient libovsdbclient.Client, lbName string) (string, error) { - var lbs []nbdb.LoadBalancer - predicate := func(lb *nbdb.LoadBalancer) bool { - return lb.Name == lbName - } - if err := nbClient.WhereCache(predicate).List(context.TODO(), &lbs); err != nil { - return "", errors.Wrapf(err, "failed to find load balancer %s", lbName) - } - - return lbs[0].UUID, nil -} - -func patchLogicalRouterAndSwitchLoadBalancerUUIDs(lbUUID string, testData []libovsdbtest.TestData, logicalEntityNames ...string) []libovsdbtest.TestData { - entitiesToPatchUUIDs := sets.NewString(logicalEntityNames...) - for _, ovnNBEntity := range testData { - if logicalRouter, isLogicalRouter := ovnNBEntity.(*nbdb.LogicalRouter); isLogicalRouter { - if entitiesToPatchUUIDs.Has(logicalRouter.Name) { - logicalRouter.LoadBalancer = []string{lbUUID} - } - } else if logicalSwitch, isLogicalSwitch := ovnNBEntity.(*nbdb.LogicalSwitch); isLogicalSwitch { - if entitiesToPatchUUIDs.Has(logicalSwitch.Name) { - logicalSwitch.LoadBalancer = []string{lbUUID} - } - } else { - continue - } - } - return testData -} diff --git a/go-controller/pkg/ovn/controller/services/svc_template_var.go b/go-controller/pkg/ovn/controller/services/svc_template_var.go new file mode 100644 index 0000000000..7c52bd365d --- /dev/null +++ b/go-controller/pkg/ovn/controller/services/svc_template_var.go @@ -0,0 +1,217 @@ +package services + +import ( + "fmt" + "regexp" + "strings" + + libovsdbclient "github.com/ovn-org/libovsdb/client" + libovsdb "github.com/ovn-org/libovsdb/ovsdb" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdbops" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" + corev1 "k8s.io/api/core/v1" +) + +const LBVipNodeTemplate string = "NODEIP" +const TemplatePrefix = "^" + +// Chassis_Template_Var records store (for efficiency reasons) a 'variables' +// map for each chassis. For simpler CMS code, store templates in memory +// as a (Name, Value) tuple where Value is a map with potentially different +// values on each chassis. +type Template struct { + // The name of the template. + Name string + + // Per chassis template value, indexed by chassisID. + Value map[string]string +} + +type TemplateMap map[string]*Template +type ChassisTemplateVarMap map[string]*nbdb.ChassisTemplateVar + +// len returns the number of chasis on which this Template variable is +// instantiated (has a value). +func (t *Template) len() int { + return len(t.Value) +} + +// toReferenceString returns the textual representation of a template +// reference, that is, '^'. +func (t *Template) toReferenceString() string { + return TemplatePrefix + t.Name +} + +// isTemplateReference returns true if 'name' is a valid template reference. +func isTemplateReference(name string) bool { + return strings.HasPrefix(name, TemplatePrefix) +} + +// templateNameFromReference extracts the template name from a textual +// reference to a template. +func templateNameFromReference(template string) string { + return strings.TrimPrefix(template, TemplatePrefix) +} + +// makeTemplateName creates a valid template name by replacing invalid +// characters in the original 'name' with '_'. Existing '_' are doubled. +// +// For example: +// "Service_default/node-port-svc_UDP_30957_node_router_template_IPv4" +// +// gets mangled to: +// "Service__default_node_port_svc__UDP__30957__node__router__template__IPv4" +func makeTemplateName(name string) string { + invalidChars := regexp.MustCompile(`[/\-$@]`) + name = strings.Replace(name, "_", "__", -1) + return invalidChars.ReplaceAllString(name, "_") +} + +// makeTemplate intializes a named Template struct with 0 values. +func makeTemplate(name string) *Template { + return &Template{Name: name, Value: map[string]string{}} +} + +func forEachNBTemplateInMaps(templateMaps []TemplateMap, callback func(nbTemplate *nbdb.ChassisTemplateVar) bool) { + // First flatten the maps into *nbdb.ChassisTemplateVar records: + flattened := ChassisTemplateVarMap{} + for _, templateMap := range templateMaps { + for _, template := range templateMap { + for chassisName, templateValue := range template.Value { + nbTemplate, found := flattened[chassisName] + if !found { + nbTemplate = &nbdb.ChassisTemplateVar{ + Chassis: chassisName, + Variables: map[string]string{}, + } + flattened[chassisName] = nbTemplate + } + nbTemplate.Variables[template.Name] = templateValue + } + } + } + + // Now walk the flattened records: + for _, nbTemplate := range flattened { + if !callback(nbTemplate) { + break + } + } +} + +// getLoadBalancerTemplates returns the TemplateMap of templates referenced +// by the load balancer. These are pointers to Templates already read from +// the database which are stored in 'allTemplates'. +func getLoadBalancerTemplates(lb *nbdb.LoadBalancer, allTemplates TemplateMap) TemplateMap { + result := TemplateMap{} + for vip, backend := range lb.Vips { + for _, s := range []string{vip, backend} { + if isTemplateReference(s) { + templateName := templateNameFromReference(s) + if template, found := allTemplates[templateName]; found { + result[template.Name] = template + } + } + } + } + return result +} + +// listSvcTemplates looks up all chassis template variables. It returns +// libovsdb.Templates values indexed by template name. +func listSvcTemplates(nbClient libovsdbclient.Client) (templatesByName TemplateMap, err error) { + templatesByName = TemplateMap{} + + templatesList, err := libovsdbops.ListTemplateVar(nbClient) + if err != nil { + return + } + + for _, nbTemplate := range templatesList { + for name, perChassisValue := range nbTemplate.Variables { + tv, found := templatesByName[name] + if !found { + tv = makeTemplate(name) + templatesByName[name] = tv + } + tv.Value[nbTemplate.Chassis] = perChassisValue + } + } + return +} + +func svcCreateOrUpdateTemplateVarOps(nbClient libovsdbclient.Client, + ops []libovsdb.Operation, templateVars []TemplateMap) ([]libovsdb.Operation, error) { + + var err error + + forEachNBTemplateInMaps(templateVars, func(nbTemplate *nbdb.ChassisTemplateVar) bool { + ops, err = libovsdbops.CreateOrUpdateChassisTemplateVarOps(nbClient, ops, nbTemplate) + return err == nil + }) + if err != nil { + return nil, err + } + return ops, nil +} + +func svcDeleteTemplateVarOps(nbClient libovsdbclient.Client, + ops []libovsdb.Operation, templateVars []TemplateMap) ([]libovsdb.Operation, error) { + var err error + + forEachNBTemplateInMaps(templateVars, func(nbTemplate *nbdb.ChassisTemplateVar) bool { + ops, err = libovsdbops.DeleteChassisTemplateVarVariablesOps(nbClient, ops, nbTemplate) + return err == nil + }) + if err != nil { + return nil, err + } + return ops, nil +} + +func svcCreateOrUpdateTemplateVar(nbClient libovsdbclient.Client, templateVars []TemplateMap) error { + ops, err := svcCreateOrUpdateTemplateVarOps(nbClient, nil, templateVars) + if err != nil { + return err + } + + _, err = libovsdbops.TransactAndCheck(nbClient, ops) + return err +} + +// makeLBNodeIPTemplateName creates a template name for the node IP (per family) +func makeLBNodeIPTemplateName(family corev1.IPFamily) string { + return fmt.Sprintf("%s_%v", LBVipNodeTemplate, family) +} + +// isLBNodeIPTemplateName returns true if 'name' is the node IP template name +// for any IP family. +func isLBNodeIPTemplateName(name string) bool { + return name == makeLBNodeIPTemplateName(corev1.IPv4Protocol) || + name == makeLBNodeIPTemplateName(corev1.IPv6Protocol) +} + +// makeLBTargetTemplateName builds a load balancer target template name. +func makeLBTargetTemplateName(service *corev1.Service, proto corev1.Protocol, port int32, + family corev1.IPFamily, scope string) string { + return makeTemplateName( + makeLBName(service, proto, + fmt.Sprintf("%d_%s_%v", port, scope, family))) +} + +// getTemplatesFromRulesTargets returns the map of template variables referred +// to by the targets in 'rules'. +func getTemplatesFromRulesTargets(rules []LBRule) TemplateMap { + templates := TemplateMap{} + for _, r := range rules { + for _, tgt := range r.Targets { + if tgt.Template == nil || tgt.Template.Name == "" { + continue + } + templates[tgt.Template.Name] = tgt.Template + } + // No need to return source templates, those are managed in the + // node-tracker. + } + return templates +} diff --git a/go-controller/pkg/ovn/default_network_controller.go b/go-controller/pkg/ovn/default_network_controller.go index f253b03ad3..91032af97c 100644 --- a/go-controller/pkg/ovn/default_network_controller.go +++ b/go-controller/pkg/ovn/default_network_controller.go @@ -85,7 +85,16 @@ type DefaultNetworkController struct { podSelectorAddressSets *syncmap.SyncMap[*PodSelectorAddressSet] // Cluster wide Load_Balancer_Group UUID. - loadBalancerGroupUUID string + // Includes all node switches and node gateway routers. + clusterLoadBalancerGroupUUID string + + // Cluster wide switch Load_Balancer_Group UUID. + // Includes all node switches. + switchLoadBalancerGroupUUID string + + // Cluster wide router Load_Balancer_Group UUID. + // Includes all node gateway routers. + routerLoadBalancerGroupUUID string // Cluster-wide router default Control Plane Protection (COPP) UUID defaultCOPPUUID string @@ -200,13 +209,15 @@ func newDefaultNetworkControllerCommon(cnci *CommonNetworkControllerInfo, reachabilityCheckInterval: egressIPReachabilityCheckInterval, egressIPNodeHealthCheckPort: config.OVNKubernetesFeature.EgressIPNodeHealthCheckPort, }, - loadbalancerClusterCache: make(map[kapi.Protocol]string), - loadBalancerGroupUUID: "", - aclLoggingEnabled: true, - joinSwIPManager: nil, - svcController: svcController, - svcFactory: svcFactory, - egressSvcController: egressSvcController, + loadbalancerClusterCache: make(map[kapi.Protocol]string), + clusterLoadBalancerGroupUUID: "", + switchLoadBalancerGroupUUID: "", + routerLoadBalancerGroupUUID: "", + aclLoggingEnabled: true, + joinSwIPManager: nil, + svcController: svcController, + svcFactory: svcFactory, + egressSvcController: egressSvcController, } oc.initRetryFramework() @@ -343,7 +354,27 @@ func (oc *DefaultNetworkController) Init() error { klog.Errorf("Error creating cluster-wide load balancer group %s: %v", ovntypes.ClusterLBGroupName, err) return err } - oc.loadBalancerGroupUUID = loadBalancerGroup.UUID + oc.clusterLoadBalancerGroupUUID = loadBalancerGroup.UUID + + loadBalancerGroup = nbdb.LoadBalancerGroup{ + Name: ovntypes.ClusterSwitchLBGroupName, + } + err = libovsdbops.CreateOrUpdateLoadBalancerGroup(oc.nbClient, &loadBalancerGroup) + if err != nil { + klog.Errorf("Error creating cluster-wide switch load balancer group %s: %v", ovntypes.ClusterSwitchLBGroupName, err) + return err + } + oc.switchLoadBalancerGroupUUID = loadBalancerGroup.UUID + + loadBalancerGroup = nbdb.LoadBalancerGroup{ + Name: ovntypes.ClusterRouterLBGroupName, + } + err = libovsdbops.CreateOrUpdateLoadBalancerGroup(oc.nbClient, &loadBalancerGroup) + if err != nil { + klog.Errorf("Error creating cluster-wide router load balancer group %s: %v", ovntypes.ClusterRouterLBGroupName, err) + return err + } + oc.routerLoadBalancerGroupUUID = loadBalancerGroup.UUID } nodeNames := []string{} diff --git a/go-controller/pkg/ovn/gateway_init.go b/go-controller/pkg/ovn/gateway_init.go index 76dfec325e..7086d0b82c 100644 --- a/go-controller/pkg/ovn/gateway_init.go +++ b/go-controller/pkg/ovn/gateway_init.go @@ -137,8 +137,8 @@ func (oc *DefaultNetworkController) gatewayInit(nodeName string, clusterIPSubnet Copp: &oc.defaultCOPPUUID, } - if oc.loadBalancerGroupUUID != "" { - logicalRouter.LoadBalancerGroup = []string{oc.loadBalancerGroupUUID} + if oc.clusterLoadBalancerGroupUUID != "" && oc.routerLoadBalancerGroupUUID != "" { + logicalRouter.LoadBalancerGroup = []string{oc.clusterLoadBalancerGroupUUID, oc.routerLoadBalancerGroupUUID} } // If l3gatewayAnnotation.IPAddresses changed, we need to update the perPodSNATs, diff --git a/go-controller/pkg/ovn/gateway_test.go b/go-controller/pkg/ovn/gateway_test.go index 43de59d01c..6947467092 100644 --- a/go-controller/pkg/ovn/gateway_test.go +++ b/go-controller/pkg/ovn/gateway_test.go @@ -260,7 +260,7 @@ func generateGatewayInitExpectedNB(testData []libovsdb.TestData, expectedOVNClus Ports: []string{gwRouterPort + "-UUID", externalRouterPort + "-UUID"}, StaticRoutes: grStaticRoutes, Nat: natUUIDs, - LoadBalancerGroup: []string{types.ClusterLBGroupName + "-UUID"}, + LoadBalancerGroup: []string{types.ClusterLBGroupName + "-UUID", types.ClusterRouterLBGroupName + "-UUID"}, Copp: &copp.UUID, }) @@ -321,6 +321,14 @@ func generateGatewayInitExpectedNB(testData []libovsdb.TestData, expectedOVNClus &nbdb.LoadBalancerGroup{ Name: types.ClusterLBGroupName, UUID: types.ClusterLBGroupName + "-UUID", + }, + &nbdb.LoadBalancerGroup{ + Name: types.ClusterSwitchLBGroupName, + UUID: types.ClusterSwitchLBGroupName + "-UUID", + }, + &nbdb.LoadBalancerGroup{ + Name: types.ClusterRouterLBGroupName, + UUID: types.ClusterRouterLBGroupName + "-UUID", }) return testData } @@ -365,6 +373,14 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { UUID: types.ClusterLBGroupName + "-UUID", Name: types.ClusterLBGroupName, } + expectedSwitchLBGroup := &nbdb.LoadBalancerGroup{ + UUID: types.ClusterSwitchLBGroupName + "-UUID", + Name: types.ClusterSwitchLBGroupName, + } + expectedRouterLBGroup := &nbdb.LoadBalancerGroup{ + UUID: types.ClusterRouterLBGroupName + "-UUID", + Name: types.ClusterRouterLBGroupName, + } gr := types.GWRouterPrefix + nodeName datapath := &sbdb.DatapathBinding{ UUID: gr + "-UUID", @@ -381,6 +397,8 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { expectedOVNClusterRouter, expectedNodeSwitch, expectedClusterLBGroup, + expectedSwitchLBGroup, + expectedRouterLBGroup, }, SBData: []libovsdbtest.TestData{ datapath, @@ -444,6 +462,14 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { UUID: types.ClusterLBGroupName + "-UUID", Name: types.ClusterLBGroupName, } + expectedSwitchLBGroup := &nbdb.LoadBalancerGroup{ + UUID: types.ClusterSwitchLBGroupName + "-UUID", + Name: types.ClusterSwitchLBGroupName, + } + expectedRouterLBGroup := &nbdb.LoadBalancerGroup{ + UUID: types.ClusterRouterLBGroupName + "-UUID", + Name: types.ClusterRouterLBGroupName, + } gr := types.GWRouterPrefix + nodeName datapath := &sbdb.DatapathBinding{ UUID: gr + "-UUID", @@ -460,6 +486,8 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { expectedOVNClusterRouter, expectedNodeSwitch, expectedClusterLBGroup, + expectedSwitchLBGroup, + expectedRouterLBGroup, }, SBData: []libovsdbtest.TestData{ datapath, @@ -517,6 +545,14 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { UUID: types.ClusterLBGroupName + "-UUID", Name: types.ClusterLBGroupName, } + expectedSwitchLBGroup := &nbdb.LoadBalancerGroup{ + UUID: types.ClusterSwitchLBGroupName + "-UUID", + Name: types.ClusterSwitchLBGroupName, + } + expectedRouterLBGroup := &nbdb.LoadBalancerGroup{ + UUID: types.ClusterRouterLBGroupName + "-UUID", + Name: types.ClusterRouterLBGroupName, + } gr := types.GWRouterPrefix + nodeName datapath := &sbdb.DatapathBinding{ UUID: gr + "-UUID", @@ -531,6 +567,8 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { expectedOVNClusterRouter, expectedNodeSwitch, expectedClusterLBGroup, + expectedSwitchLBGroup, + expectedRouterLBGroup, }, SBData: []libovsdbtest.TestData{ datapath, @@ -600,6 +638,14 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { UUID: types.ClusterLBGroupName + "-UUID", Name: types.ClusterLBGroupName, } + expectedSwitchLBGroup := &nbdb.LoadBalancerGroup{ + UUID: types.ClusterSwitchLBGroupName + "-UUID", + Name: types.ClusterSwitchLBGroupName, + } + expectedRouterLBGroup := &nbdb.LoadBalancerGroup{ + UUID: types.ClusterRouterLBGroupName + "-UUID", + Name: types.ClusterRouterLBGroupName, + } gr := types.GWRouterPrefix + nodeName datapath := &sbdb.DatapathBinding{ UUID: gr + "-UUID", @@ -614,6 +660,8 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { expectedOVNClusterRouter, expectedNodeSwitch, expectedClusterLBGroup, + expectedSwitchLBGroup, + expectedRouterLBGroup, }, SBData: []libovsdbtest.TestData{ datapath, @@ -672,6 +720,14 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { UUID: types.ClusterLBGroupName + "-UUID", Name: types.ClusterLBGroupName, } + expectedSwitchLBGroup := &nbdb.LoadBalancerGroup{ + UUID: types.ClusterSwitchLBGroupName + "-UUID", + Name: types.ClusterSwitchLBGroupName, + } + expectedRouterLBGroup := &nbdb.LoadBalancerGroup{ + UUID: types.ClusterRouterLBGroupName + "-UUID", + Name: types.ClusterRouterLBGroupName, + } gr := types.GWRouterPrefix + nodeName datapath := &sbdb.DatapathBinding{ UUID: gr + "-UUID", @@ -686,6 +742,8 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { expectedOVNClusterRouter, expectedNodeSwitch, expectedClusterLBGroup, + expectedSwitchLBGroup, + expectedRouterLBGroup, }, SBData: []libovsdbtest.TestData{ datapath, @@ -744,6 +802,14 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { UUID: types.ClusterLBGroupName + "-UUID", Name: types.ClusterLBGroupName, } + expectedSwitchLBGroup := &nbdb.LoadBalancerGroup{ + UUID: types.ClusterSwitchLBGroupName + "-UUID", + Name: types.ClusterSwitchLBGroupName, + } + expectedRouterLBGroup := &nbdb.LoadBalancerGroup{ + UUID: types.ClusterRouterLBGroupName + "-UUID", + Name: types.ClusterRouterLBGroupName, + } gr := types.GWRouterPrefix + nodeName datapath := &sbdb.DatapathBinding{ UUID: gr + "-UUID", @@ -758,6 +824,8 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { expectedOVNClusterRouter, expectedNodeSwitch, expectedClusterLBGroup, + expectedSwitchLBGroup, + expectedRouterLBGroup, }, SBData: []libovsdbtest.TestData{ datapath, @@ -817,6 +885,14 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { UUID: types.ClusterLBGroupName + "-UUID", Name: types.ClusterLBGroupName, } + expectedSwitchLBGroup := &nbdb.LoadBalancerGroup{ + UUID: types.ClusterSwitchLBGroupName + "-UUID", + Name: types.ClusterSwitchLBGroupName, + } + expectedRouterLBGroup := &nbdb.LoadBalancerGroup{ + UUID: types.ClusterRouterLBGroupName + "-UUID", + Name: types.ClusterRouterLBGroupName, + } gr := types.GWRouterPrefix + nodeName datapath := &sbdb.DatapathBinding{ UUID: gr + "-UUID", @@ -831,6 +907,8 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { expectedOVNClusterRouter, expectedNodeSwitch, expectedClusterLBGroup, + expectedSwitchLBGroup, + expectedRouterLBGroup, }, SBData: []libovsdbtest.TestData{ datapath, @@ -920,6 +998,14 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { UUID: types.ClusterLBGroupName + "-UUID", Name: types.ClusterLBGroupName, } + expectedSwitchLBGroup := &nbdb.LoadBalancerGroup{ + UUID: types.ClusterSwitchLBGroupName + "-UUID", + Name: types.ClusterSwitchLBGroupName, + } + expectedRouterLBGroup := &nbdb.LoadBalancerGroup{ + UUID: types.ClusterRouterLBGroupName + "-UUID", + Name: types.ClusterRouterLBGroupName, + } gr := types.GWRouterPrefix + nodeName datapath := &sbdb.DatapathBinding{ UUID: gr + "-UUID", @@ -938,6 +1024,8 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { expectedOVNClusterRouter, expectedNodeSwitch, expectedClusterLBGroup, + expectedSwitchLBGroup, + expectedRouterLBGroup, }, SBData: []libovsdbtest.TestData{ datapath, @@ -1018,6 +1106,14 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { Name: types.ClusterLBGroupName, UUID: types.ClusterLBGroupName + "-UUID", } + expectedSwitchLBGroup := &nbdb.LoadBalancerGroup{ + UUID: types.ClusterSwitchLBGroupName + "-UUID", + Name: types.ClusterSwitchLBGroupName, + } + expectedRouterLBGroup := &nbdb.LoadBalancerGroup{ + UUID: types.ClusterRouterLBGroupName + "-UUID", + Name: types.ClusterRouterLBGroupName, + } gr := types.GWRouterPrefix + nodeName datapath := &sbdb.DatapathBinding{ UUID: gr + "-UUID", @@ -1035,6 +1131,8 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { expectedOVNClusterRouter, expectedNodeSwitch, expectedClusterLBGroup, + expectedSwitchLBGroup, + expectedRouterLBGroup, }, SBData: []libovsdbtest.TestData{ datapath, @@ -1105,6 +1203,14 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { UUID: types.ClusterLBGroupName + "-UUID", Name: types.ClusterLBGroupName, } + expectedSwitchLBGroup := &nbdb.LoadBalancerGroup{ + UUID: types.ClusterSwitchLBGroupName + "-UUID", + Name: types.ClusterSwitchLBGroupName, + } + expectedRouterLBGroup := &nbdb.LoadBalancerGroup{ + UUID: types.ClusterRouterLBGroupName + "-UUID", + Name: types.ClusterRouterLBGroupName, + } gr := types.GWRouterPrefix + nodeName datapath := &sbdb.DatapathBinding{ UUID: gr + "-UUID", @@ -1120,6 +1226,8 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { expectedOVNClusterRouter, expectedNodeSwitch, expectedClusterLBGroup, + expectedSwitchLBGroup, + expectedRouterLBGroup, }, SBData: []libovsdbtest.TestData{ datapath, diff --git a/go-controller/pkg/ovn/hybrid_test.go b/go-controller/pkg/ovn/hybrid_test.go index b6d016d48c..77df209307 100644 --- a/go-controller/pkg/ovn/hybrid_test.go +++ b/go-controller/pkg/ovn/hybrid_test.go @@ -128,10 +128,12 @@ func setupHybridOverlayOVNObjects(node tNode, hoSubnet, nodeHOIP, nodeHOMAC stri } -func setupClusterController(clusterController *DefaultNetworkController, clusterLBUUID, expectedNodeSwitchUUID, node1Name string) { +func setupClusterController(clusterController *DefaultNetworkController, clusterLBUUID, switchLBUUID, routerLBUUID, expectedNodeSwitchUUID, node1Name string) { var err error clusterController.SCTPSupport = true - clusterController.loadBalancerGroupUUID = clusterLBUUID + clusterController.clusterLoadBalancerGroupUUID = clusterLBUUID + clusterController.switchLoadBalancerGroupUUID = switchLBUUID + clusterController.routerLoadBalancerGroupUUID = routerLBUUID clusterController.defaultCOPPUUID, err = EnsureDefaultCOPP(clusterController.nbClient) gomega.Expect(err).NotTo(gomega.HaveOccurred()) clusterController.joinSwIPManager, _ = lsm.NewJoinLogicalSwitchIPManager(clusterController.nbClient, expectedNodeSwitchUUID, []string{node1Name}) @@ -338,7 +340,9 @@ var _ = ginkgo.Describe("Hybrid SDN Master Operations", func() { err = f.Start() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - expectedClusterLBGroup := newLoadBalancerGroup() + expectedClusterLBGroup := newLoadBalancerGroup(types.ClusterLBGroupName) + expectedSwitchLBGroup := newLoadBalancerGroup(types.ClusterSwitchLBGroupName) + expectedRouterLBGroup := newLoadBalancerGroup(types.ClusterRouterLBGroupName) expectedOVNClusterRouter := newOVNClusterRouter() ovnClusterRouterLRP := &nbdb.LogicalRouterPort{ Name: types.GWRouterToJoinSwitchPrefix + types.OVNClusterRouter, @@ -346,7 +350,7 @@ var _ = ginkgo.Describe("Hybrid SDN Master Operations", func() { UUID: types.GWRouterToJoinSwitchPrefix + types.OVNClusterRouter + "-UUID", } expectedOVNClusterRouter.Ports = []string{ovnClusterRouterLRP.UUID} - expectedNodeSwitch := node1.logicalSwitch(expectedClusterLBGroup.UUID) + expectedNodeSwitch := node1.logicalSwitch([]string{expectedClusterLBGroup.UUID, expectedSwitchLBGroup.UUID}) expectedClusterRouterPortGroup := newRouterPortGroup() expectedClusterPortGroup := newClusterPortGroup() @@ -369,6 +373,8 @@ var _ = ginkgo.Describe("Hybrid SDN Master Operations", func() { expectedClusterRouterPortGroup, expectedClusterPortGroup, expectedClusterLBGroup, + expectedSwitchLBGroup, + expectedRouterLBGroup, }, SBData: []libovsdbtest.TestData{ clusterRouterDatapath, @@ -385,7 +391,7 @@ var _ = ginkgo.Describe("Hybrid SDN Master Operations", func() { clusterController, err := NewOvnController(fakeClient.GetMasterClientset(), f, stopChan, nil, libovsdbOvnNBClient, libovsdbOvnSBClient, record.NewFakeRecorder(10), wg) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - setupClusterController(clusterController, expectedClusterLBGroup.UUID, expectedNodeSwitch.UUID, node1.Name) + setupClusterController(clusterController, expectedClusterLBGroup.UUID, expectedSwitchLBGroup.UUID, expectedRouterLBGroup.UUID, expectedNodeSwitch.UUID, node1.Name) _, _ = clusterController.joinSwIPManager.EnsureJoinLRPIPs(types.OVNClusterRouter) @@ -623,7 +629,9 @@ var _ = ginkgo.Describe("Hybrid SDN Master Operations", func() { err = f.Start() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - expectedClusterLBGroup := newLoadBalancerGroup() + expectedClusterLBGroup := newLoadBalancerGroup(types.ClusterLBGroupName) + expectedSwitchLBGroup := newLoadBalancerGroup(types.ClusterSwitchLBGroupName) + expectedRouterLBGroup := newLoadBalancerGroup(types.ClusterRouterLBGroupName) expectedOVNClusterRouter := newOVNClusterRouter() ovnClusterRouterLRP := &nbdb.LogicalRouterPort{ Name: types.GWRouterToJoinSwitchPrefix + types.OVNClusterRouter, @@ -631,7 +639,7 @@ var _ = ginkgo.Describe("Hybrid SDN Master Operations", func() { UUID: types.GWRouterToJoinSwitchPrefix + types.OVNClusterRouter + "-UUID", } expectedOVNClusterRouter.Ports = []string{ovnClusterRouterLRP.UUID} - expectedNodeSwitch := node1.logicalSwitch(expectedClusterLBGroup.UUID) + expectedNodeSwitch := node1.logicalSwitch([]string{expectedClusterLBGroup.UUID, expectedSwitchLBGroup.UUID}) expectedClusterRouterPortGroup := newRouterPortGroup() expectedClusterPortGroup := newClusterPortGroup() @@ -702,7 +710,7 @@ var _ = ginkgo.Describe("Hybrid SDN Master Operations", func() { clusterController, err := NewOvnController(fakeClient.GetMasterClientset(), f, stopChan, nil, libovsdbOvnNBClient, libovsdbOvnSBClient, record.NewFakeRecorder(10), wg) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - setupClusterController(clusterController, expectedClusterLBGroup.UUID, expectedNodeSwitch.UUID, node1.Name) + setupClusterController(clusterController, expectedClusterLBGroup.UUID, expectedSwitchLBGroup.UUID, expectedRouterLBGroup.UUID, expectedNodeSwitch.UUID, node1.Name) _, _ = clusterController.joinSwIPManager.EnsureJoinLRPIPs(types.OVNClusterRouter) err = clusterController.syncGatewayLogicalNetwork(updatedNode, l3GatewayConfig, []*net.IPNet{subnet}, hostAddrs) @@ -833,7 +841,9 @@ var _ = ginkgo.Describe("Hybrid SDN Master Operations", func() { err = f.Start() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - expectedClusterLBGroup := newLoadBalancerGroup() + expectedClusterLBGroup := newLoadBalancerGroup(types.ClusterLBGroupName) + expectedSwitchLBGroup := newLoadBalancerGroup(types.ClusterSwitchLBGroupName) + expectedRouterLBGroup := newLoadBalancerGroup(types.ClusterRouterLBGroupName) expectedOVNClusterRouter := newOVNClusterRouter() ovnClusterRouterLRP := &nbdb.LogicalRouterPort{ Name: types.GWRouterToJoinSwitchPrefix + types.OVNClusterRouter, @@ -841,7 +851,7 @@ var _ = ginkgo.Describe("Hybrid SDN Master Operations", func() { UUID: types.GWRouterToJoinSwitchPrefix + types.OVNClusterRouter + "-UUID", } expectedOVNClusterRouter.Ports = []string{ovnClusterRouterLRP.UUID} - expectedNodeSwitch := node1.logicalSwitch(expectedClusterLBGroup.UUID) + expectedNodeSwitch := node1.logicalSwitch([]string{expectedClusterLBGroup.UUID, expectedSwitchLBGroup.UUID}) expectedClusterRouterPortGroup := newRouterPortGroup() expectedClusterPortGroup := newClusterPortGroup() @@ -864,6 +874,8 @@ var _ = ginkgo.Describe("Hybrid SDN Master Operations", func() { expectedClusterRouterPortGroup, expectedClusterPortGroup, expectedClusterLBGroup, + expectedSwitchLBGroup, + expectedRouterLBGroup, }, SBData: []libovsdbtest.TestData{ clusterRouterDatapath, @@ -880,7 +892,7 @@ var _ = ginkgo.Describe("Hybrid SDN Master Operations", func() { clusterController, err := NewOvnController(fakeClient.GetMasterClientset(), f, stopChan, nil, libovsdbOvnNBClient, libovsdbOvnSBClient, record.NewFakeRecorder(10), wg) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - setupClusterController(clusterController, expectedClusterLBGroup.UUID, expectedNodeSwitch.UUID, node1.Name) + setupClusterController(clusterController, expectedClusterLBGroup.UUID, expectedSwitchLBGroup.UUID, expectedRouterLBGroup.UUID, expectedNodeSwitch.UUID, node1.Name) _, _ = clusterController.joinSwIPManager.EnsureJoinLRPIPs(types.OVNClusterRouter) @@ -1110,7 +1122,9 @@ var _ = ginkgo.Describe("Hybrid SDN Master Operations", func() { err = f.Start() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - expectedClusterLBGroup := newLoadBalancerGroup() + expectedClusterLBGroup := newLoadBalancerGroup(types.ClusterLBGroupName) + expectedSwitchLBGroup := newLoadBalancerGroup(types.ClusterSwitchLBGroupName) + expectedRouterLBGroup := newLoadBalancerGroup(types.ClusterRouterLBGroupName) expectedOVNClusterRouter := newOVNClusterRouter() ovnClusterRouterLRP := &nbdb.LogicalRouterPort{ Name: types.GWRouterToJoinSwitchPrefix + types.OVNClusterRouter, @@ -1118,7 +1132,7 @@ var _ = ginkgo.Describe("Hybrid SDN Master Operations", func() { UUID: types.GWRouterToJoinSwitchPrefix + types.OVNClusterRouter + "-UUID", } expectedOVNClusterRouter.Ports = []string{ovnClusterRouterLRP.UUID} - expectedNodeSwitch := node1.logicalSwitch(expectedClusterLBGroup.UUID) + expectedNodeSwitch := node1.logicalSwitch([]string{expectedClusterLBGroup.UUID, expectedSwitchLBGroup.UUID}) expectedClusterRouterPortGroup := newRouterPortGroup() expectedClusterPortGroup := newClusterPortGroup() @@ -1141,6 +1155,8 @@ var _ = ginkgo.Describe("Hybrid SDN Master Operations", func() { expectedClusterRouterPortGroup, expectedClusterPortGroup, expectedClusterLBGroup, + expectedSwitchLBGroup, + expectedRouterLBGroup, }, SBData: []libovsdbtest.TestData{ clusterRouterDatapath, @@ -1157,7 +1173,7 @@ var _ = ginkgo.Describe("Hybrid SDN Master Operations", func() { clusterController, err := NewOvnController(fakeClient, f, stopChan, nil, libovsdbOvnNBClient, libovsdbOvnSBClient, record.NewFakeRecorder(10), wg) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - setupClusterController(clusterController, expectedClusterLBGroup.UUID, expectedNodeSwitch.UUID, node1.Name) + setupClusterController(clusterController, expectedClusterLBGroup.UUID, expectedSwitchLBGroup.UUID, expectedRouterLBGroup.UUID, expectedNodeSwitch.UUID, node1.Name) _, _ = clusterController.joinSwIPManager.EnsureJoinLRPIPs(types.OVNClusterRouter) @@ -1322,7 +1338,9 @@ var _ = ginkgo.Describe("Hybrid SDN Master Operations", func() { err = f.Start() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - expectedClusterLBGroup := newLoadBalancerGroup() + expectedClusterLBGroup := newLoadBalancerGroup(types.ClusterLBGroupName) + expectedSwitchLBGroup := newLoadBalancerGroup(types.ClusterSwitchLBGroupName) + expectedRouterLBGroup := newLoadBalancerGroup(types.ClusterRouterLBGroupName) expectedOVNClusterRouter := newOVNClusterRouter() ovnClusterRouterLRP := &nbdb.LogicalRouterPort{ Name: types.GWRouterToJoinSwitchPrefix + types.OVNClusterRouter, @@ -1330,7 +1348,7 @@ var _ = ginkgo.Describe("Hybrid SDN Master Operations", func() { UUID: types.GWRouterToJoinSwitchPrefix + types.OVNClusterRouter + "-UUID", } expectedOVNClusterRouter.Ports = []string{ovnClusterRouterLRP.UUID} - expectedNodeSwitch := node1.logicalSwitch(expectedClusterLBGroup.UUID) + expectedNodeSwitch := node1.logicalSwitch([]string{expectedClusterLBGroup.UUID, expectedSwitchLBGroup.UUID}) expectedClusterRouterPortGroup := newRouterPortGroup() expectedClusterPortGroup := newClusterPortGroup() @@ -1353,6 +1371,8 @@ var _ = ginkgo.Describe("Hybrid SDN Master Operations", func() { expectedClusterRouterPortGroup, expectedClusterPortGroup, expectedClusterLBGroup, + expectedSwitchLBGroup, + expectedRouterLBGroup, }, SBData: []libovsdbtest.TestData{ clusterRouterDatapath, @@ -1369,7 +1389,7 @@ var _ = ginkgo.Describe("Hybrid SDN Master Operations", func() { clusterController, err := NewOvnController(fakeClient, f, stopChan, nil, libovsdbOvnNBClient, libovsdbOvnSBClient, record.NewFakeRecorder(10), wg) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - setupClusterController(clusterController, expectedClusterLBGroup.UUID, expectedNodeSwitch.UUID, node1.Name) + setupClusterController(clusterController, expectedClusterLBGroup.UUID, expectedSwitchLBGroup.UUID, expectedRouterLBGroup.UUID, expectedNodeSwitch.UUID, node1.Name) _, _ = clusterController.joinSwIPManager.EnsureJoinLRPIPs(types.OVNClusterRouter) diff --git a/go-controller/pkg/ovn/master.go b/go-controller/pkg/ovn/master.go index 115a3a7ae7..bad0cced62 100644 --- a/go-controller/pkg/ovn/master.go +++ b/go-controller/pkg/ovn/master.go @@ -386,7 +386,7 @@ func (oc *DefaultNetworkController) ensureNodeLogicalNetwork(node *kapi.Node, ho return err } - return oc.createNodeLogicalSwitch(node.Name, hostSubnets, oc.loadBalancerGroupUUID) + return oc.createNodeLogicalSwitch(node.Name, hostSubnets, oc.clusterLoadBalancerGroupUUID, oc.switchLoadBalancerGroupUUID) } func (oc *DefaultNetworkController) addNode(node *kapi.Node) ([]*net.IPNet, error) { @@ -483,6 +483,13 @@ func (oc *DefaultNetworkController) deleteStaleNodeChassis(node *kapi.Node) erro p := func(item *sbdb.Chassis) bool { return item.Name == staleChassis } + if err = libovsdbops.DeleteChassisTemplateVar(oc.nbClient, &nbdb.ChassisTemplateVar{Chassis: staleChassis}); err != nil { + // Send an event and Log on failure + oc.recorder.Eventf(node, kapi.EventTypeWarning, "ErrorMismatchChassis", + "Node %s is now with a new chassis ID. Its stale chassis template vars are still in the NBDB", + node.Name) + return fmt.Errorf("node %s is now with a new chassis ID. Its stale chassis template vars are still in the NBDB", node.Name) + } if err = libovsdbops.DeleteChassisWithPredicate(oc.sbClient, p); err != nil { // Send an event and Log on failure oc.recorder.Eventf(node, kapi.EventTypeWarning, "ErrorMismatchChassis", @@ -507,12 +514,20 @@ func (oc *DefaultNetworkController) deleteNode(nodeName string) error { return fmt.Errorf("failed to clean up GR LRP IPs for node %s: %v", nodeName, err) } + chassisTemplateVars := make([]*nbdb.ChassisTemplateVar, 0) p := func(item *sbdb.Chassis) bool { - return item.Hostname == nodeName + if item.Hostname == nodeName { + chassisTemplateVars = append(chassisTemplateVars, &nbdb.ChassisTemplateVar{Chassis: item.Name}) + return true + } + return false } if err := libovsdbops.DeleteChassisWithPredicate(oc.sbClient, p); err != nil { return fmt.Errorf("failed to remove the chassis associated with node %s in the OVN SB Chassis table: %v", nodeName, err) } + if err := libovsdbops.DeleteChassisTemplateVar(oc.nbClient, chassisTemplateVars...); err != nil { + return fmt.Errorf("failed deleting chassis template variables for %s: %v", nodeName, err) + } return nil } @@ -568,42 +583,19 @@ func (oc *DefaultNetworkController) clearInitialNodeNetworkUnavailableCondition( // and sbdb and deletes chassis that are stale func (oc *DefaultNetworkController) syncNodesPeriodic() { //node names is a slice of all node names - nodes, err := oc.kube.GetNodes() + kNodes, err := oc.kube.GetNodes() if err != nil { klog.Errorf("Error getting existing nodes from kube API: %v", err) return } - nodeNames := make([]string, 0, len(nodes.Items)) - - for _, node := range nodes.Items { - nodeNames = append(nodeNames, node.Name) - } - - chassisList, err := libovsdbops.ListChassis(oc.sbClient) - if err != nil { - klog.Errorf("Failed to get chassis list: error: %v", err) - return - } - - chassisHostNameMap := map[string]*sbdb.Chassis{} - for _, chassis := range chassisList { - chassisHostNameMap[chassis.Hostname] = chassis + nodes := make([]*kapi.Node, 0, len(kNodes.Items)) + for i := range kNodes.Items { + nodes = append(nodes, &kNodes.Items[i]) } - //delete existing nodes from the chassis map. - for _, nodeName := range nodeNames { - delete(chassisHostNameMap, nodeName) - } - - staleChassis := []*sbdb.Chassis{} - for _, v := range chassisHostNameMap { - staleChassis = append(staleChassis, v) - } - - if err = libovsdbops.DeleteChassis(oc.sbClient, staleChassis...); err != nil { - klog.Errorf("Failed Deleting chassis %v error: %v", chassisHostNameMap, err) - return + if err := oc.syncChassis(nodes); err != nil { + klog.Errorf("Failed to sync chassis: error: %v", err) } } @@ -611,9 +603,10 @@ func (oc *DefaultNetworkController) syncNodesPeriodic() { // watchNodes() will be called for all existing nodes at startup anyway. // Note that this list will include the 'join' cluster switch, which we // do not want to delete. -func (oc *DefaultNetworkController) syncNodes(nodes []interface{}) error { +func (oc *DefaultNetworkController) syncNodes(kNodes []interface{}) error { foundNodes := sets.New[string]() - for _, tmp := range nodes { + nodes := make([]*kapi.Node, 0, len(kNodes)) + for _, tmp := range kNodes { node, ok := tmp.(*kapi.Node) if !ok { return fmt.Errorf("spurious object in syncNodes: %v", tmp) @@ -623,6 +616,7 @@ func (oc *DefaultNetworkController) syncNodes(nodes []interface{}) error { continue } foundNodes.Insert(node.Name) + nodes = append(nodes, node) // For each existing node, reserve its joinSwitch LRP IPs if they already exist. if _, err := oc.joinSwIPManager.EnsureJoinLRPIPs(node.Name); err != nil { @@ -647,44 +641,82 @@ func (oc *DefaultNetworkController) syncNodes(nodes []interface{}) error { } } - // cleanup stale chassis with no corresponding nodes - chassisList, err := libovsdbops.ListChassis(oc.sbClient) - if err != nil { - return fmt.Errorf("failed to get chassis list: %v", err) + if err := oc.syncChassis(nodes); err != nil { + return fmt.Errorf("failed to sync chassis: error: %v", err) } + return nil +} - knownChassisNames := sets.NewString() - chassisDeleteList := []*sbdb.Chassis{} - for _, chassis := range chassisList { - knownChassisNames.Insert(chassis.Name) - // skip chassis that have a corresponding node - if foundNodes.Has(chassis.Hostname) { - continue - } - chassisDeleteList = append(chassisDeleteList, chassis) +// Cleanup stale chassis and chassis template variables with no +// corresponding nodes. +func (oc *DefaultNetworkController) syncChassis(nodes []*kapi.Node) error { + chassisList, err := libovsdbops.ListChassis(oc.sbClient) + if err != nil { + return fmt.Errorf("failed to get chassis list: error: %v", err) } - // cleanup stale chassis private with no corresponding chassis + // Cleanup stale chassis private with no corresponding chassis chassisPrivateList, err := libovsdbops.ListChassisPrivate(oc.sbClient) if err != nil { return fmt.Errorf("failed to get chassis private list: %v", err) } - for _, chassis := range chassisPrivateList { - // skip chassis private that have a corresponding chassis - if knownChassisNames.Has(chassis.Name) { + templateVarList, err := libovsdbops.ListTemplateVar(oc.nbClient) + if err != nil { + return fmt.Errorf("failed to get template var list: error: %v", err) + } + + chassisHostNameMap := map[string]*sbdb.Chassis{} + chassisNameMap := map[string]*sbdb.Chassis{} + for _, chassis := range chassisList { + chassisHostNameMap[chassis.Hostname] = chassis + chassisNameMap[chassis.Name] = chassis + } + + for _, chassisPrivate := range chassisPrivateList { + // Skip chassis private that have a corresponding chassis + if _, ok := chassisNameMap[chassisPrivate.Name]; ok { continue } - // we add to the list what would be the corresponding Chassis. Even if + // We add to the map what would be the corresponding Chassis. Even if // the Chassis does not exist in SBDB, DeleteChassis will remove the // ChassisPrivate. - chassisDeleteList = append(chassisDeleteList, &sbdb.Chassis{Name: chassis.Name}) + chassisNameMap[chassisPrivate.Name] = &sbdb.Chassis{Name: chassisPrivate.Name} } - // Delete stale chassis and associated chassis private - if err := libovsdbops.DeleteChassis(oc.sbClient, chassisDeleteList...); err != nil { - return fmt.Errorf("failed deleting chassis %v: %v", chassisDeleteList, err) + templateChassisMap := map[string]*nbdb.ChassisTemplateVar{} + for _, templateVar := range templateVarList { + templateChassisMap[templateVar.Chassis] = templateVar } + + // Delete existing nodes from the chassis map. + // Also delete existing templateVars from the template map. + for _, node := range nodes { + if chassis, ok := chassisHostNameMap[node.Name]; ok { + delete(chassisNameMap, chassis.Name) + delete(chassisHostNameMap, chassis.Hostname) + delete(templateChassisMap, chassis.Name) + } + } + + staleChassis := make([]*sbdb.Chassis, 0, len(chassisHostNameMap)) + for _, chassis := range chassisNameMap { + staleChassis = append(staleChassis, chassis) + } + + staleChassisTemplateVars := make([]*nbdb.ChassisTemplateVar, 0, len(templateChassisMap)) + for _, template := range templateChassisMap { + staleChassisTemplateVars = append(staleChassisTemplateVars, template) + } + + if err := libovsdbops.DeleteChassis(oc.sbClient, staleChassis...); err != nil { + return fmt.Errorf("failed Deleting chassis %v error: %v", chassisHostNameMap, err) + } + + if err := libovsdbops.DeleteChassisTemplateVar(oc.nbClient, staleChassisTemplateVars...); err != nil { + return fmt.Errorf("failed Deleting chassis template vars %v error: %v", chassisHostNameMap, err) + } + return nil } @@ -834,6 +866,7 @@ func (oc *DefaultNetworkController) deleteNodeEvent(node *kapi.Node) error { if err := oc.deleteNode(node.Name); err != nil { return err } + oc.lsManager.DeleteSwitch(node.Name) oc.addNodeFailed.Delete(node.Name) oc.mgmtPortFailed.Delete(node.Name) diff --git a/go-controller/pkg/ovn/master_test.go b/go-controller/pkg/ovn/master_test.go index ffa8520887..9acf5fe393 100644 --- a/go-controller/pkg/ovn/master_test.go +++ b/go-controller/pkg/ovn/master_test.go @@ -98,12 +98,12 @@ func (n tNode) gatewayConfig(gatewayMode config.GatewayMode, vlanID uint) *util. } } -func (n tNode) logicalSwitch(loadBalancerGroupUUID string) *nbdb.LogicalSwitch { +func (n tNode) logicalSwitch(loadBalancerGroupUUIDs []string) *nbdb.LogicalSwitch { return &nbdb.LogicalSwitch{ UUID: n.Name + "-UUID", Name: n.Name, OtherConfig: map[string]string{"subnet": n.NodeSubnet}, - LoadBalancerGroup: []string{loadBalancerGroupUUID}, + LoadBalancerGroup: loadBalancerGroupUUIDs, } } @@ -885,6 +885,8 @@ var _ = ginkgo.Describe("Default network controller operations", func() { recorder *record.FakeRecorder expectedClusterLBGroup *nbdb.LoadBalancerGroup + expectedSwitchLBGroup *nbdb.LoadBalancerGroup + expectedRouterLBGroup *nbdb.LoadBalancerGroup expectedNodeSwitch *nbdb.LogicalSwitch expectedOVNClusterRouter *nbdb.LogicalRouter expectedClusterRouterPortGroup *nbdb.PortGroup @@ -935,8 +937,10 @@ var _ = ginkgo.Describe("Default network controller operations", func() { DnatSnatIP: "169.254.0.1", } - expectedClusterLBGroup = newLoadBalancerGroup() - expectedNodeSwitch = node1.logicalSwitch(expectedClusterLBGroup.UUID) + expectedClusterLBGroup = newLoadBalancerGroup(types.ClusterLBGroupName) + expectedSwitchLBGroup = newLoadBalancerGroup(types.ClusterSwitchLBGroupName) + expectedRouterLBGroup = newLoadBalancerGroup(types.ClusterRouterLBGroupName) + expectedNodeSwitch = node1.logicalSwitch([]string{expectedClusterLBGroup.UUID, expectedSwitchLBGroup.UUID}) expectedOVNClusterRouter = newOVNClusterRouter() expectedClusterRouterPortGroup = newRouterPortGroup() expectedClusterPortGroup = newClusterPortGroup() @@ -955,6 +959,8 @@ var _ = ginkgo.Describe("Default network controller operations", func() { newRouterPortGroup(), newClusterPortGroup(), expectedClusterLBGroup, + expectedSwitchLBGroup, + expectedRouterLBGroup, }, SBData: []libovsdbtest.TestData{ datapath, @@ -1002,7 +1008,9 @@ var _ = ginkgo.Describe("Default network controller operations", func() { recorder = record.NewFakeRecorder(10) oc, _ = NewOvnController(fakeClient, f, stopChan, nil, nbClient, sbClient, recorder, wg) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - oc.loadBalancerGroupUUID = expectedClusterLBGroup.UUID + oc.clusterLoadBalancerGroupUUID = expectedClusterLBGroup.UUID + oc.switchLoadBalancerGroupUUID = expectedSwitchLBGroup.UUID + oc.routerLoadBalancerGroupUUID = expectedRouterLBGroup.UUID gomega.Expect(oc).NotTo(gomega.BeNil()) oc.defaultCOPPUUID, err = EnsureDefaultCOPP(nbClient) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -1559,10 +1567,9 @@ func newOVNClusterRouter() *nbdb.LogicalRouter { } } -func newLoadBalancerGroup() *nbdb.LoadBalancerGroup { +func newLoadBalancerGroup(name string) *nbdb.LoadBalancerGroup { return &nbdb.LoadBalancerGroup{ - Name: types.ClusterLBGroupName, - UUID: types.ClusterLBGroupName + "-UUID", + Name: name, UUID: name + "-UUID", } } diff --git a/go-controller/pkg/ovn/namespace_test.go b/go-controller/pkg/ovn/namespace_test.go index fcc18a053d..f380611f45 100644 --- a/go-controller/pkg/ovn/namespace_test.go +++ b/go-controller/pkg/ovn/namespace_test.go @@ -215,9 +215,11 @@ var _ = ginkgo.Describe("OVN Namespace Operations", func() { hostNetworkNamespace := "test-host-network-ns" config.Kubernetes.HostNetworkNamespace = hostNetworkNamespace - expectedClusterLBGroup := newLoadBalancerGroup() + expectedClusterLBGroup := newLoadBalancerGroup(ovntypes.ClusterLBGroupName) + expectedSwitchLBGroup := newLoadBalancerGroup(ovntypes.ClusterSwitchLBGroupName) + expectedRouterLBGroup := newLoadBalancerGroup(ovntypes.ClusterRouterLBGroupName) expectedOVNClusterRouter := newOVNClusterRouter() - expectedNodeSwitch := node1.logicalSwitch(expectedClusterLBGroup.UUID) + expectedNodeSwitch := node1.logicalSwitch([]string{expectedClusterLBGroup.UUID, expectedSwitchLBGroup.UUID}) expectedClusterRouterPortGroup := newRouterPortGroup() expectedClusterPortGroup := newClusterPortGroup() gr := ovntypes.GWRouterPrefix + node1.Name @@ -234,6 +236,8 @@ var _ = ginkgo.Describe("OVN Namespace Operations", func() { expectedClusterRouterPortGroup, expectedClusterPortGroup, expectedClusterLBGroup, + expectedSwitchLBGroup, + expectedRouterLBGroup, }, SBData: []libovsdbtest.TestData{ datapath, diff --git a/go-controller/pkg/ovn/ovn.go b/go-controller/pkg/ovn/ovn.go index 529ab3803d..0aed62b642 100644 --- a/go-controller/pkg/ovn/ovn.go +++ b/go-controller/pkg/ovn/ovn.go @@ -440,10 +440,10 @@ func (oc *DefaultNetworkController) StartServiceController(wg *sync.WaitGroup, r wg.Add(1) go func() { defer wg.Done() - useLBGroups := oc.loadBalancerGroupUUID != "" + useLBGroups := oc.clusterLoadBalancerGroupUUID != "" // use 5 workers like most of the kubernetes controllers in the // kubernetes controller-manager - err := oc.svcController.Run(5, oc.stopChan, runRepair, useLBGroups) + err := oc.svcController.Run(5, oc.stopChan, runRepair, useLBGroups, oc.svcTemplateSupport) if err != nil { klog.Errorf("Error running OVN Kubernetes Services controller: %v", err) } diff --git a/go-controller/pkg/ovn/ovn_test.go b/go-controller/pkg/ovn/ovn_test.go index e58a64d377..50f99ba3bc 100644 --- a/go-controller/pkg/ovn/ovn_test.go +++ b/go-controller/pkg/ovn/ovn_test.go @@ -123,7 +123,9 @@ func (o *FakeOVN) init() { o.fakeRecorder, o.wg) gomega.Expect(err).NotTo(gomega.HaveOccurred()) o.controller.multicastSupport = true - o.controller.loadBalancerGroupUUID = types.ClusterLBGroupName + "-UUID" + o.controller.clusterLoadBalancerGroupUUID = types.ClusterLBGroupName + "-UUID" + o.controller.switchLoadBalancerGroupUUID = types.ClusterSwitchLBGroupName + "-UUID" + o.controller.routerLoadBalancerGroupUUID = types.ClusterRouterLBGroupName + "-UUID" } func resetNBClient(ctx context.Context, nbClient libovsdbclient.Client) { @@ -165,8 +167,9 @@ func NewOvnController(ovnClient *util.OVNMasterClientset, wf *factory.WatchFacto libovsdbOvnNBClient, libovsdbOvnSBClient, &podRecorder, - false, - false, + false, // sctp support + false, // multicast support + true, // templates support ) if err != nil { return nil, err diff --git a/go-controller/pkg/ovn/secondary_layer3_network_controller.go b/go-controller/pkg/ovn/secondary_layer3_network_controller.go index 7206a28f3d..623dba7946 100644 --- a/go-controller/pkg/ovn/secondary_layer3_network_controller.go +++ b/go-controller/pkg/ovn/secondary_layer3_network_controller.go @@ -409,7 +409,7 @@ func (oc *SecondaryLayer3NetworkController) addNode(node *kapi.Node) ([]*net.IPN return nil, fmt.Errorf("subnet annotation in the node %q for the layer3 secondary network %s is missing : %w", node.Name, oc.GetNetworkName(), err) } - err = oc.createNodeLogicalSwitch(node.Name, hostSubnets, "") + err = oc.createNodeLogicalSwitch(node.Name, hostSubnets, "", "") if err != nil { return nil, err } diff --git a/go-controller/pkg/sbdb/chassis_template_var.go b/go-controller/pkg/sbdb/chassis_template_var.go new file mode 100644 index 0000000000..212e772be6 --- /dev/null +++ b/go-controller/pkg/sbdb/chassis_template_var.go @@ -0,0 +1,87 @@ +// Code generated by "libovsdb.modelgen" +// DO NOT EDIT. + +package sbdb + +import "github.com/ovn-org/libovsdb/model" + +const ChassisTemplateVarTable = "Chassis_Template_Var" + +// ChassisTemplateVar defines an object in Chassis_Template_Var table +type ChassisTemplateVar struct { + UUID string `ovsdb:"_uuid"` + Chassis string `ovsdb:"chassis"` + Variables map[string]string `ovsdb:"variables"` +} + +func (a *ChassisTemplateVar) GetUUID() string { + return a.UUID +} + +func (a *ChassisTemplateVar) GetChassis() string { + return a.Chassis +} + +func (a *ChassisTemplateVar) GetVariables() map[string]string { + return a.Variables +} + +func copyChassisTemplateVarVariables(a map[string]string) map[string]string { + if a == nil { + return nil + } + b := make(map[string]string, len(a)) + for k, v := range a { + b[k] = v + } + return b +} + +func equalChassisTemplateVarVariables(a, b map[string]string) bool { + if (a == nil) != (b == nil) { + return false + } + if len(a) != len(b) { + return false + } + for k, v := range a { + if w, ok := b[k]; !ok || v != w { + return false + } + } + return true +} + +func (a *ChassisTemplateVar) DeepCopyInto(b *ChassisTemplateVar) { + *b = *a + b.Variables = copyChassisTemplateVarVariables(a.Variables) +} + +func (a *ChassisTemplateVar) DeepCopy() *ChassisTemplateVar { + b := new(ChassisTemplateVar) + a.DeepCopyInto(b) + return b +} + +func (a *ChassisTemplateVar) CloneModelInto(b model.Model) { + c := b.(*ChassisTemplateVar) + a.DeepCopyInto(c) +} + +func (a *ChassisTemplateVar) CloneModel() model.Model { + return a.DeepCopy() +} + +func (a *ChassisTemplateVar) Equals(b *ChassisTemplateVar) bool { + return a.UUID == b.UUID && + a.Chassis == b.Chassis && + equalChassisTemplateVarVariables(a.Variables, b.Variables) +} + +func (a *ChassisTemplateVar) EqualsModel(b model.Model) bool { + c := b.(*ChassisTemplateVar) + return a.Equals(c) +} + +var _ model.CloneableModel = &ChassisTemplateVar{} +var _ model.ComparableModel = &ChassisTemplateVar{} diff --git a/go-controller/pkg/sbdb/load_balancer.go b/go-controller/pkg/sbdb/load_balancer.go index 26c59ea089..76b6de3f5c 100644 --- a/go-controller/pkg/sbdb/load_balancer.go +++ b/go-controller/pkg/sbdb/load_balancer.go @@ -19,19 +19,42 @@ var ( // LoadBalancer defines an object in Load_Balancer table type LoadBalancer struct { - UUID string `ovsdb:"_uuid"` - Datapaths []string `ovsdb:"datapaths"` - ExternalIDs map[string]string `ovsdb:"external_ids"` - Name string `ovsdb:"name"` - Options map[string]string `ovsdb:"options"` - Protocol *LoadBalancerProtocol `ovsdb:"protocol"` - Vips map[string]string `ovsdb:"vips"` + UUID string `ovsdb:"_uuid"` + DatapathGroup *string `ovsdb:"datapath_group"` + Datapaths []string `ovsdb:"datapaths"` + ExternalIDs map[string]string `ovsdb:"external_ids"` + Name string `ovsdb:"name"` + Options map[string]string `ovsdb:"options"` + Protocol *LoadBalancerProtocol `ovsdb:"protocol"` + Vips map[string]string `ovsdb:"vips"` } func (a *LoadBalancer) GetUUID() string { return a.UUID } +func (a *LoadBalancer) GetDatapathGroup() *string { + return a.DatapathGroup +} + +func copyLoadBalancerDatapathGroup(a *string) *string { + if a == nil { + return nil + } + b := *a + return &b +} + +func equalLoadBalancerDatapathGroup(a, b *string) bool { + if (a == nil) != (b == nil) { + return false + } + if a == b { + return true + } + return *a == *b +} + func (a *LoadBalancer) GetDatapaths() []string { return a.Datapaths } @@ -178,6 +201,7 @@ func equalLoadBalancerVips(a, b map[string]string) bool { func (a *LoadBalancer) DeepCopyInto(b *LoadBalancer) { *b = *a + b.DatapathGroup = copyLoadBalancerDatapathGroup(a.DatapathGroup) b.Datapaths = copyLoadBalancerDatapaths(a.Datapaths) b.ExternalIDs = copyLoadBalancerExternalIDs(a.ExternalIDs) b.Options = copyLoadBalancerOptions(a.Options) @@ -202,6 +226,7 @@ func (a *LoadBalancer) CloneModel() model.Model { func (a *LoadBalancer) Equals(b *LoadBalancer) bool { return a.UUID == b.UUID && + equalLoadBalancerDatapathGroup(a.DatapathGroup, b.DatapathGroup) && equalLoadBalancerDatapaths(a.Datapaths, b.Datapaths) && equalLoadBalancerExternalIDs(a.ExternalIDs, b.ExternalIDs) && a.Name == b.Name && diff --git a/go-controller/pkg/sbdb/mac_binding.go b/go-controller/pkg/sbdb/mac_binding.go index ff266fdf4d..705431f1d0 100644 --- a/go-controller/pkg/sbdb/mac_binding.go +++ b/go-controller/pkg/sbdb/mac_binding.go @@ -14,6 +14,7 @@ type MACBinding struct { IP string `ovsdb:"ip"` LogicalPort string `ovsdb:"logical_port"` MAC string `ovsdb:"mac"` + Timestamp int `ovsdb:"timestamp"` } func (a *MACBinding) GetUUID() string { @@ -36,6 +37,10 @@ func (a *MACBinding) GetMAC() string { return a.MAC } +func (a *MACBinding) GetTimestamp() int { + return a.Timestamp +} + func (a *MACBinding) DeepCopyInto(b *MACBinding) { *b = *a } @@ -60,7 +65,8 @@ func (a *MACBinding) Equals(b *MACBinding) bool { a.Datapath == b.Datapath && a.IP == b.IP && a.LogicalPort == b.LogicalPort && - a.MAC == b.MAC + a.MAC == b.MAC && + a.Timestamp == b.Timestamp } func (a *MACBinding) EqualsModel(b model.Model) bool { diff --git a/go-controller/pkg/sbdb/mirror.go b/go-controller/pkg/sbdb/mirror.go new file mode 100644 index 0000000000..90216a67c7 --- /dev/null +++ b/go-controller/pkg/sbdb/mirror.go @@ -0,0 +1,123 @@ +// Code generated by "libovsdb.modelgen" +// DO NOT EDIT. + +package sbdb + +import "github.com/ovn-org/libovsdb/model" + +const MirrorTable = "Mirror" + +type ( + MirrorFilter = string + MirrorType = string +) + +var ( + MirrorFilterFromLport MirrorFilter = "from-lport" + MirrorFilterToLport MirrorFilter = "to-lport" + MirrorTypeGre MirrorType = "gre" + MirrorTypeErspan MirrorType = "erspan" +) + +// Mirror defines an object in Mirror table +type Mirror struct { + UUID string `ovsdb:"_uuid"` + ExternalIDs map[string]string `ovsdb:"external_ids"` + Filter MirrorFilter `ovsdb:"filter"` + Index int `ovsdb:"index"` + Name string `ovsdb:"name"` + Sink string `ovsdb:"sink"` + Type MirrorType `ovsdb:"type"` +} + +func (a *Mirror) GetUUID() string { + return a.UUID +} + +func (a *Mirror) GetExternalIDs() map[string]string { + return a.ExternalIDs +} + +func copyMirrorExternalIDs(a map[string]string) map[string]string { + if a == nil { + return nil + } + b := make(map[string]string, len(a)) + for k, v := range a { + b[k] = v + } + return b +} + +func equalMirrorExternalIDs(a, b map[string]string) bool { + if (a == nil) != (b == nil) { + return false + } + if len(a) != len(b) { + return false + } + for k, v := range a { + if w, ok := b[k]; !ok || v != w { + return false + } + } + return true +} + +func (a *Mirror) GetFilter() MirrorFilter { + return a.Filter +} + +func (a *Mirror) GetIndex() int { + return a.Index +} + +func (a *Mirror) GetName() string { + return a.Name +} + +func (a *Mirror) GetSink() string { + return a.Sink +} + +func (a *Mirror) GetType() MirrorType { + return a.Type +} + +func (a *Mirror) DeepCopyInto(b *Mirror) { + *b = *a + b.ExternalIDs = copyMirrorExternalIDs(a.ExternalIDs) +} + +func (a *Mirror) DeepCopy() *Mirror { + b := new(Mirror) + a.DeepCopyInto(b) + return b +} + +func (a *Mirror) CloneModelInto(b model.Model) { + c := b.(*Mirror) + a.DeepCopyInto(c) +} + +func (a *Mirror) CloneModel() model.Model { + return a.DeepCopy() +} + +func (a *Mirror) Equals(b *Mirror) bool { + return a.UUID == b.UUID && + equalMirrorExternalIDs(a.ExternalIDs, b.ExternalIDs) && + a.Filter == b.Filter && + a.Index == b.Index && + a.Name == b.Name && + a.Sink == b.Sink && + a.Type == b.Type +} + +func (a *Mirror) EqualsModel(b model.Model) bool { + c := b.(*Mirror) + return a.Equals(c) +} + +var _ model.CloneableModel = &Mirror{} +var _ model.ComparableModel = &Mirror{} diff --git a/go-controller/pkg/sbdb/model.go b/go-controller/pkg/sbdb/model.go index 2837f80118..5e0b164444 100644 --- a/go-controller/pkg/sbdb/model.go +++ b/go-controller/pkg/sbdb/model.go @@ -13,43 +13,46 @@ import ( // FullDatabaseModel returns the DatabaseModel object to be used in libovsdb func FullDatabaseModel() (model.ClientDBModel, error) { return model.NewClientDBModel("OVN_Southbound", map[string]model.Model{ - "Address_Set": &AddressSet{}, - "BFD": &BFD{}, - "Chassis": &Chassis{}, - "Chassis_Private": &ChassisPrivate{}, - "Connection": &Connection{}, - "Controller_Event": &ControllerEvent{}, - "DHCP_Options": &DHCPOptions{}, - "DHCPv6_Options": &DHCPv6Options{}, - "DNS": &DNS{}, - "Datapath_Binding": &DatapathBinding{}, - "Encap": &Encap{}, - "FDB": &FDB{}, - "Gateway_Chassis": &GatewayChassis{}, - "HA_Chassis": &HAChassis{}, - "HA_Chassis_Group": &HAChassisGroup{}, - "IGMP_Group": &IGMPGroup{}, - "IP_Multicast": &IPMulticast{}, - "Load_Balancer": &LoadBalancer{}, - "Logical_DP_Group": &LogicalDPGroup{}, - "Logical_Flow": &LogicalFlow{}, - "MAC_Binding": &MACBinding{}, - "Meter": &Meter{}, - "Meter_Band": &MeterBand{}, - "Multicast_Group": &MulticastGroup{}, - "Port_Binding": &PortBinding{}, - "Port_Group": &PortGroup{}, - "RBAC_Permission": &RBACPermission{}, - "RBAC_Role": &RBACRole{}, - "SB_Global": &SBGlobal{}, - "SSL": &SSL{}, - "Service_Monitor": &ServiceMonitor{}, + "Address_Set": &AddressSet{}, + "BFD": &BFD{}, + "Chassis": &Chassis{}, + "Chassis_Private": &ChassisPrivate{}, + "Chassis_Template_Var": &ChassisTemplateVar{}, + "Connection": &Connection{}, + "Controller_Event": &ControllerEvent{}, + "DHCP_Options": &DHCPOptions{}, + "DHCPv6_Options": &DHCPv6Options{}, + "DNS": &DNS{}, + "Datapath_Binding": &DatapathBinding{}, + "Encap": &Encap{}, + "FDB": &FDB{}, + "Gateway_Chassis": &GatewayChassis{}, + "HA_Chassis": &HAChassis{}, + "HA_Chassis_Group": &HAChassisGroup{}, + "IGMP_Group": &IGMPGroup{}, + "IP_Multicast": &IPMulticast{}, + "Load_Balancer": &LoadBalancer{}, + "Logical_DP_Group": &LogicalDPGroup{}, + "Logical_Flow": &LogicalFlow{}, + "MAC_Binding": &MACBinding{}, + "Meter": &Meter{}, + "Meter_Band": &MeterBand{}, + "Mirror": &Mirror{}, + "Multicast_Group": &MulticastGroup{}, + "Port_Binding": &PortBinding{}, + "Port_Group": &PortGroup{}, + "RBAC_Permission": &RBACPermission{}, + "RBAC_Role": &RBACRole{}, + "SB_Global": &SBGlobal{}, + "SSL": &SSL{}, + "Service_Monitor": &ServiceMonitor{}, + "Static_MAC_Binding": &StaticMACBinding{}, }) } var schema = `{ "name": "OVN_Southbound", - "version": "20.21.0", + "version": "20.27.0", "tables": { "Address_Set": { "columns": { @@ -262,6 +265,30 @@ var schema = `{ ] ] }, + "Chassis_Template_Var": { + "columns": { + "chassis": { + "type": "string" + }, + "variables": { + "type": { + "key": { + "type": "string" + }, + "value": { + "type": "string" + }, + "min": 0, + "max": "unlimited" + } + } + }, + "indexes": [ + [ + "chassis" + ] + ] + }, "Connection": { "columns": { "external_ids": { @@ -868,6 +895,16 @@ var schema = `{ }, "Load_Balancer": { "columns": { + "datapath_group": { + "type": { + "key": { + "type": "uuid", + "refTable": "Logical_DP_Group" + }, + "min": 0, + "max": 1 + } + }, "datapaths": { "type": { "key": { @@ -1064,6 +1101,9 @@ var schema = `{ }, "mac": { "type": "string" + }, + "timestamp": { + "type": "integer" } }, "indexes": [ @@ -1140,6 +1180,64 @@ var schema = `{ } } }, + "Mirror": { + "columns": { + "external_ids": { + "type": { + "key": { + "type": "string" + }, + "value": { + "type": "string" + }, + "min": 0, + "max": "unlimited" + } + }, + "filter": { + "type": { + "key": { + "type": "string", + "enum": [ + "set", + [ + "from-lport", + "to-lport" + ] + ] + } + } + }, + "index": { + "type": "integer" + }, + "name": { + "type": "string" + }, + "sink": { + "type": "string" + }, + "type": { + "type": { + "key": { + "type": "string", + "enum": [ + "set", + [ + "gre", + "erspan" + ] + ] + } + } + } + }, + "indexes": [ + [ + "name" + ] + ] + }, "Multicast_Group": { "columns": { "datapath": { @@ -1187,6 +1285,28 @@ var schema = `{ }, "Port_Binding": { "columns": { + "additional_chassis": { + "type": { + "key": { + "type": "uuid", + "refTable": "Chassis", + "refType": "weak" + }, + "min": 0, + "max": "unlimited" + } + }, + "additional_encap": { + "type": { + "key": { + "type": "uuid", + "refTable": "Encap", + "refType": "weak" + }, + "min": 0, + "max": "unlimited" + } + }, "chassis": { "type": { "key": { @@ -1263,6 +1383,17 @@ var schema = `{ "max": "unlimited" } }, + "mirror_rules": { + "type": { + "key": { + "type": "uuid", + "refTable": "Mirror", + "refType": "weak" + }, + "min": 0, + "max": "unlimited" + } + }, "nat_addresses": { "type": { "key": { @@ -1293,6 +1424,26 @@ var schema = `{ "max": 1 } }, + "port_security": { + "type": { + "key": { + "type": "string" + }, + "min": 0, + "max": "unlimited" + } + }, + "requested_additional_chassis": { + "type": { + "key": { + "type": "uuid", + "refTable": "Chassis", + "refType": "weak" + }, + "min": 0, + "max": "unlimited" + } + }, "requested_chassis": { "type": { "key": { @@ -1603,6 +1754,36 @@ var schema = `{ "protocol" ] ] + }, + "Static_MAC_Binding": { + "columns": { + "datapath": { + "type": { + "key": { + "type": "uuid", + "refTable": "Datapath_Binding" + } + } + }, + "ip": { + "type": "string" + }, + "logical_port": { + "type": "string" + }, + "mac": { + "type": "string" + }, + "override_dynamic_mac": { + "type": "boolean" + } + }, + "indexes": [ + [ + "logical_port", + "ip" + ] + ] } } }` diff --git a/go-controller/pkg/sbdb/port_binding.go b/go-controller/pkg/sbdb/port_binding.go index 4f5179292b..b3d30f843a 100644 --- a/go-controller/pkg/sbdb/port_binding.go +++ b/go-controller/pkg/sbdb/port_binding.go @@ -9,30 +9,91 @@ const PortBindingTable = "Port_Binding" // PortBinding defines an object in Port_Binding table type PortBinding struct { - UUID string `ovsdb:"_uuid"` - Chassis *string `ovsdb:"chassis"` - Datapath string `ovsdb:"datapath"` - Encap *string `ovsdb:"encap"` - ExternalIDs map[string]string `ovsdb:"external_ids"` - GatewayChassis []string `ovsdb:"gateway_chassis"` - HaChassisGroup *string `ovsdb:"ha_chassis_group"` - LogicalPort string `ovsdb:"logical_port"` - MAC []string `ovsdb:"mac"` - NatAddresses []string `ovsdb:"nat_addresses"` - Options map[string]string `ovsdb:"options"` - ParentPort *string `ovsdb:"parent_port"` - RequestedChassis *string `ovsdb:"requested_chassis"` - Tag *int `ovsdb:"tag"` - TunnelKey int `ovsdb:"tunnel_key"` - Type string `ovsdb:"type"` - Up *bool `ovsdb:"up"` - VirtualParent *string `ovsdb:"virtual_parent"` + UUID string `ovsdb:"_uuid"` + AdditionalChassis []string `ovsdb:"additional_chassis"` + AdditionalEncap []string `ovsdb:"additional_encap"` + Chassis *string `ovsdb:"chassis"` + Datapath string `ovsdb:"datapath"` + Encap *string `ovsdb:"encap"` + ExternalIDs map[string]string `ovsdb:"external_ids"` + GatewayChassis []string `ovsdb:"gateway_chassis"` + HaChassisGroup *string `ovsdb:"ha_chassis_group"` + LogicalPort string `ovsdb:"logical_port"` + MAC []string `ovsdb:"mac"` + MirrorRules []string `ovsdb:"mirror_rules"` + NatAddresses []string `ovsdb:"nat_addresses"` + Options map[string]string `ovsdb:"options"` + ParentPort *string `ovsdb:"parent_port"` + PortSecurity []string `ovsdb:"port_security"` + RequestedAdditionalChassis []string `ovsdb:"requested_additional_chassis"` + RequestedChassis *string `ovsdb:"requested_chassis"` + Tag *int `ovsdb:"tag"` + TunnelKey int `ovsdb:"tunnel_key"` + Type string `ovsdb:"type"` + Up *bool `ovsdb:"up"` + VirtualParent *string `ovsdb:"virtual_parent"` } func (a *PortBinding) GetUUID() string { return a.UUID } +func (a *PortBinding) GetAdditionalChassis() []string { + return a.AdditionalChassis +} + +func copyPortBindingAdditionalChassis(a []string) []string { + if a == nil { + return nil + } + b := make([]string, len(a)) + copy(b, a) + return b +} + +func equalPortBindingAdditionalChassis(a, b []string) bool { + if (a == nil) != (b == nil) { + return false + } + if len(a) != len(b) { + return false + } + for i, v := range a { + if b[i] != v { + return false + } + } + return true +} + +func (a *PortBinding) GetAdditionalEncap() []string { + return a.AdditionalEncap +} + +func copyPortBindingAdditionalEncap(a []string) []string { + if a == nil { + return nil + } + b := make([]string, len(a)) + copy(b, a) + return b +} + +func equalPortBindingAdditionalEncap(a, b []string) bool { + if (a == nil) != (b == nil) { + return false + } + if len(a) != len(b) { + return false + } + for i, v := range a { + if b[i] != v { + return false + } + } + return true +} + func (a *PortBinding) GetChassis() *string { return a.Chassis } @@ -193,6 +254,34 @@ func equalPortBindingMAC(a, b []string) bool { return true } +func (a *PortBinding) GetMirrorRules() []string { + return a.MirrorRules +} + +func copyPortBindingMirrorRules(a []string) []string { + if a == nil { + return nil + } + b := make([]string, len(a)) + copy(b, a) + return b +} + +func equalPortBindingMirrorRules(a, b []string) bool { + if (a == nil) != (b == nil) { + return false + } + if len(a) != len(b) { + return false + } + for i, v := range a { + if b[i] != v { + return false + } + } + return true +} + func (a *PortBinding) GetNatAddresses() []string { return a.NatAddresses } @@ -273,6 +362,62 @@ func equalPortBindingParentPort(a, b *string) bool { return *a == *b } +func (a *PortBinding) GetPortSecurity() []string { + return a.PortSecurity +} + +func copyPortBindingPortSecurity(a []string) []string { + if a == nil { + return nil + } + b := make([]string, len(a)) + copy(b, a) + return b +} + +func equalPortBindingPortSecurity(a, b []string) bool { + if (a == nil) != (b == nil) { + return false + } + if len(a) != len(b) { + return false + } + for i, v := range a { + if b[i] != v { + return false + } + } + return true +} + +func (a *PortBinding) GetRequestedAdditionalChassis() []string { + return a.RequestedAdditionalChassis +} + +func copyPortBindingRequestedAdditionalChassis(a []string) []string { + if a == nil { + return nil + } + b := make([]string, len(a)) + copy(b, a) + return b +} + +func equalPortBindingRequestedAdditionalChassis(a, b []string) bool { + if (a == nil) != (b == nil) { + return false + } + if len(a) != len(b) { + return false + } + for i, v := range a { + if b[i] != v { + return false + } + } + return true +} + func (a *PortBinding) GetRequestedChassis() *string { return a.RequestedChassis } @@ -371,15 +516,20 @@ func equalPortBindingVirtualParent(a, b *string) bool { func (a *PortBinding) DeepCopyInto(b *PortBinding) { *b = *a + b.AdditionalChassis = copyPortBindingAdditionalChassis(a.AdditionalChassis) + b.AdditionalEncap = copyPortBindingAdditionalEncap(a.AdditionalEncap) b.Chassis = copyPortBindingChassis(a.Chassis) b.Encap = copyPortBindingEncap(a.Encap) b.ExternalIDs = copyPortBindingExternalIDs(a.ExternalIDs) b.GatewayChassis = copyPortBindingGatewayChassis(a.GatewayChassis) b.HaChassisGroup = copyPortBindingHaChassisGroup(a.HaChassisGroup) b.MAC = copyPortBindingMAC(a.MAC) + b.MirrorRules = copyPortBindingMirrorRules(a.MirrorRules) b.NatAddresses = copyPortBindingNatAddresses(a.NatAddresses) b.Options = copyPortBindingOptions(a.Options) b.ParentPort = copyPortBindingParentPort(a.ParentPort) + b.PortSecurity = copyPortBindingPortSecurity(a.PortSecurity) + b.RequestedAdditionalChassis = copyPortBindingRequestedAdditionalChassis(a.RequestedAdditionalChassis) b.RequestedChassis = copyPortBindingRequestedChassis(a.RequestedChassis) b.Tag = copyPortBindingTag(a.Tag) b.Up = copyPortBindingUp(a.Up) @@ -403,6 +553,8 @@ func (a *PortBinding) CloneModel() model.Model { func (a *PortBinding) Equals(b *PortBinding) bool { return a.UUID == b.UUID && + equalPortBindingAdditionalChassis(a.AdditionalChassis, b.AdditionalChassis) && + equalPortBindingAdditionalEncap(a.AdditionalEncap, b.AdditionalEncap) && equalPortBindingChassis(a.Chassis, b.Chassis) && a.Datapath == b.Datapath && equalPortBindingEncap(a.Encap, b.Encap) && @@ -411,9 +563,12 @@ func (a *PortBinding) Equals(b *PortBinding) bool { equalPortBindingHaChassisGroup(a.HaChassisGroup, b.HaChassisGroup) && a.LogicalPort == b.LogicalPort && equalPortBindingMAC(a.MAC, b.MAC) && + equalPortBindingMirrorRules(a.MirrorRules, b.MirrorRules) && equalPortBindingNatAddresses(a.NatAddresses, b.NatAddresses) && equalPortBindingOptions(a.Options, b.Options) && equalPortBindingParentPort(a.ParentPort, b.ParentPort) && + equalPortBindingPortSecurity(a.PortSecurity, b.PortSecurity) && + equalPortBindingRequestedAdditionalChassis(a.RequestedAdditionalChassis, b.RequestedAdditionalChassis) && equalPortBindingRequestedChassis(a.RequestedChassis, b.RequestedChassis) && equalPortBindingTag(a.Tag, b.Tag) && a.TunnelKey == b.TunnelKey && diff --git a/go-controller/pkg/sbdb/static_mac_binding.go b/go-controller/pkg/sbdb/static_mac_binding.go new file mode 100644 index 0000000000..370968f604 --- /dev/null +++ b/go-controller/pkg/sbdb/static_mac_binding.go @@ -0,0 +1,78 @@ +// Code generated by "libovsdb.modelgen" +// DO NOT EDIT. + +package sbdb + +import "github.com/ovn-org/libovsdb/model" + +const StaticMACBindingTable = "Static_MAC_Binding" + +// StaticMACBinding defines an object in Static_MAC_Binding table +type StaticMACBinding struct { + UUID string `ovsdb:"_uuid"` + Datapath string `ovsdb:"datapath"` + IP string `ovsdb:"ip"` + LogicalPort string `ovsdb:"logical_port"` + MAC string `ovsdb:"mac"` + OverrideDynamicMAC bool `ovsdb:"override_dynamic_mac"` +} + +func (a *StaticMACBinding) GetUUID() string { + return a.UUID +} + +func (a *StaticMACBinding) GetDatapath() string { + return a.Datapath +} + +func (a *StaticMACBinding) GetIP() string { + return a.IP +} + +func (a *StaticMACBinding) GetLogicalPort() string { + return a.LogicalPort +} + +func (a *StaticMACBinding) GetMAC() string { + return a.MAC +} + +func (a *StaticMACBinding) GetOverrideDynamicMAC() bool { + return a.OverrideDynamicMAC +} + +func (a *StaticMACBinding) DeepCopyInto(b *StaticMACBinding) { + *b = *a +} + +func (a *StaticMACBinding) DeepCopy() *StaticMACBinding { + b := new(StaticMACBinding) + a.DeepCopyInto(b) + return b +} + +func (a *StaticMACBinding) CloneModelInto(b model.Model) { + c := b.(*StaticMACBinding) + a.DeepCopyInto(c) +} + +func (a *StaticMACBinding) CloneModel() model.Model { + return a.DeepCopy() +} + +func (a *StaticMACBinding) Equals(b *StaticMACBinding) bool { + return a.UUID == b.UUID && + a.Datapath == b.Datapath && + a.IP == b.IP && + a.LogicalPort == b.LogicalPort && + a.MAC == b.MAC && + a.OverrideDynamicMAC == b.OverrideDynamicMAC +} + +func (a *StaticMACBinding) EqualsModel(b model.Model) bool { + c := b.(*StaticMACBinding) + return a.Equals(c) +} + +var _ model.CloneableModel = &StaticMACBinding{} +var _ model.ComparableModel = &StaticMACBinding{} diff --git a/go-controller/pkg/types/const.go b/go-controller/pkg/types/const.go index 2c877a61b0..3ac8fedb66 100644 --- a/go-controller/pkg/types/const.go +++ b/go-controller/pkg/types/const.go @@ -160,7 +160,9 @@ const ( OVSDBTimeout = 10 * time.Second OVSDBWaitTimeout = 0 - ClusterLBGroupName = "clusterLBGroup" + ClusterLBGroupName = "clusterLBGroup" + ClusterSwitchLBGroupName = "clusterSwitchLBGroup" + ClusterRouterLBGroupName = "clusterRouterLBGroup" // key for network name external-id NetworkExternalID = OvnK8sPrefix + "/" + "network" diff --git a/go-controller/pkg/util/node_annotations.go b/go-controller/pkg/util/node_annotations.go index a42fa5865b..ce6b2d6426 100644 --- a/go-controller/pkg/util/node_annotations.go +++ b/go-controller/pkg/util/node_annotations.go @@ -308,6 +308,10 @@ func ParseNodeChassisIDAnnotation(node *kapi.Node) (string, error) { return chassisID, nil } +func NodeChassisIDAnnotationChanged(oldNode, newNode *kapi.Node) bool { + return oldNode.Annotations[ovnNodeChassisID] != newNode.Annotations[ovnNodeChassisID] +} + func SetNodeManagementPortMACAddress(nodeAnnotator kube.Annotator, macAddress net.HardwareAddr) error { return nodeAnnotator.Set(ovnNodeManagementPortMacAddress, macAddress.String()) } diff --git a/go-controller/pkg/util/util.go b/go-controller/pkg/util/util.go index b002f965d8..b5bd023da4 100644 --- a/go-controller/pkg/util/util.go +++ b/go-controller/pkg/util/util.go @@ -168,9 +168,10 @@ func HashForOVN(s string) string { } // UpdateIPsSlice will search for values of oldIPs in the slice "s" and update it with newIPs values of same IP family -func UpdateIPsSlice(s, oldIPs, newIPs []string) []string { +func UpdateIPsSlice(s, oldIPs, newIPs []string) ([]string, bool) { n := make([]string, len(s)) copy(n, s) + updated := false for i, entry := range s { for _, oldIP := range oldIPs { if entry == oldIP { @@ -178,11 +179,13 @@ func UpdateIPsSlice(s, oldIPs, newIPs []string) []string { if utilnet.IsIPv6(net.ParseIP(oldIP)) { if utilnet.IsIPv6(net.ParseIP(newIP)) { n[i] = newIP + updated = true break } } else { if !utilnet.IsIPv6(net.ParseIP(newIP)) { n[i] = newIP + updated = true break } } @@ -191,7 +194,7 @@ func UpdateIPsSlice(s, oldIPs, newIPs []string) []string { } } } - return n + return n, updated } // FilterIPsSlice will filter a list of IPs by a list of CIDRs. By default, diff --git a/go-controller/pkg/util/util_unit_test.go b/go-controller/pkg/util/util_unit_test.go index 4de5f936a5..4e8437aa7a 100644 --- a/go-controller/pkg/util/util_unit_test.go +++ b/go-controller/pkg/util/util_unit_test.go @@ -110,6 +110,7 @@ func TestUpdateIPsSlice(t *testing.T) { name string s, oldIPs, newIPs []string want []string + changed bool }{ { "Tests no matching IPs to remove", @@ -117,6 +118,7 @@ func TestUpdateIPsSlice(t *testing.T) { []string{"1.1.1.1"}, []string{"9.9.9.9", "fe99::1"}, []string{"192.168.1.1", "10.0.0.1", "127.0.0.2"}, + false, }, { "Tests some matching IPs to replace", @@ -124,6 +126,7 @@ func TestUpdateIPsSlice(t *testing.T) { []string{"10.0.0.1"}, []string{"9.9.9.9", "fe99::1"}, []string{"192.168.1.1", "9.9.9.9", "127.0.0.2"}, + true, }, { "Tests matching IPv6 to replace", @@ -131,6 +134,7 @@ func TestUpdateIPsSlice(t *testing.T) { []string{"3dfd::99ac"}, []string{"9.9.9.9", "fe99::1"}, []string{"fed9::5", "ab13::1e15", "fe99::1"}, + true, }, { "Tests match but nothing to replace with", @@ -138,6 +142,7 @@ func TestUpdateIPsSlice(t *testing.T) { []string{"3dfd::99ac"}, []string{"9.9.9.9"}, []string{"fed9::5", "ab13::1e15", "3dfd::99ac"}, + false, }, { "Tests with no newIPs", @@ -145,6 +150,7 @@ func TestUpdateIPsSlice(t *testing.T) { []string{"3dfd::99ac"}, []string{}, []string{"fed9::5", "ab13::1e15", "3dfd::99ac"}, + false, }, { "Tests with no newIPs or oldIPs", @@ -152,15 +158,20 @@ func TestUpdateIPsSlice(t *testing.T) { []string{}, []string{}, []string{"fed9::5", "ab13::1e15", "3dfd::99ac"}, + false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - ans := UpdateIPsSlice(tt.s, tt.oldIPs, tt.newIPs) + ans, changed := UpdateIPsSlice(tt.s, tt.oldIPs, tt.newIPs) if !reflect.DeepEqual(ans, tt.want) { t.Errorf("got %v, want %v", ans, tt.want) } + + if tt.changed != changed { + t.Errorf("got changed %t, want %t", changed, tt.changed) + } }) } }