diff --git a/api/models/loadbalance_entry.go b/api/models/loadbalance_entry.go index cb2a5ac00..956281cdf 100644 --- a/api/models/loadbalance_entry.go +++ b/api/models/loadbalance_entry.go @@ -22,6 +22,9 @@ type LoadbalanceEntry struct { // values of End point servers Endpoints []*LoadbalanceEntryEndpointsItems0 `json:"endpoints"` + // Secondary IPs in for multi-homed SCTP service + SecondaryIPs []*LoadbalanceEntrySecondaryIPsItems0 `json:"secondaryIPs"` + // service arguments ServiceArguments *LoadbalanceEntryServiceArguments `json:"serviceArguments,omitempty"` } @@ -34,6 +37,10 @@ func (m *LoadbalanceEntry) Validate(formats strfmt.Registry) error { res = append(res, err) } + if err := m.validateSecondaryIPs(formats); err != nil { + res = append(res, err) + } + if err := m.validateServiceArguments(formats); err != nil { res = append(res, err) } @@ -70,6 +77,32 @@ func (m *LoadbalanceEntry) validateEndpoints(formats strfmt.Registry) error { return nil } +func (m *LoadbalanceEntry) validateSecondaryIPs(formats strfmt.Registry) error { + if swag.IsZero(m.SecondaryIPs) { // not required + return nil + } + + for i := 0; i < len(m.SecondaryIPs); i++ { + if swag.IsZero(m.SecondaryIPs[i]) { // not required + continue + } + + if m.SecondaryIPs[i] != nil { + if err := m.SecondaryIPs[i].Validate(formats); err != nil { + if ve, ok := err.(*errors.Validation); ok { + return ve.ValidateName("secondaryIPs" + "." + strconv.Itoa(i)) + } else if ce, ok := err.(*errors.CompositeError); ok { + return ce.ValidateName("secondaryIPs" + "." + strconv.Itoa(i)) + } + return err + } + } + + } + + return nil +} + func (m *LoadbalanceEntry) validateServiceArguments(formats strfmt.Registry) error { if swag.IsZero(m.ServiceArguments) { // not required return nil @@ -97,6 +130,10 @@ func (m *LoadbalanceEntry) ContextValidate(ctx context.Context, formats strfmt.R res = append(res, err) } + if err := m.contextValidateSecondaryIPs(ctx, formats); err != nil { + res = append(res, err) + } + if err := m.contextValidateServiceArguments(ctx, formats); err != nil { res = append(res, err) } @@ -127,6 +164,26 @@ func (m *LoadbalanceEntry) contextValidateEndpoints(ctx context.Context, formats return nil } +func (m *LoadbalanceEntry) contextValidateSecondaryIPs(ctx context.Context, formats strfmt.Registry) error { + + for i := 0; i < len(m.SecondaryIPs); i++ { + + if m.SecondaryIPs[i] != nil { + if err := m.SecondaryIPs[i].ContextValidate(ctx, formats); err != nil { + if ve, ok := err.(*errors.Validation); ok { + return ve.ValidateName("secondaryIPs" + "." + strconv.Itoa(i)) + } else if ce, ok := err.(*errors.CompositeError); ok { + return ce.ValidateName("secondaryIPs" + "." + strconv.Itoa(i)) + } + return err + } + } + + } + + return nil +} + func (m *LoadbalanceEntry) contextValidateServiceArguments(ctx context.Context, formats strfmt.Registry) error { if m.ServiceArguments != nil { @@ -207,6 +264,43 @@ func (m *LoadbalanceEntryEndpointsItems0) UnmarshalBinary(b []byte) error { return nil } +// LoadbalanceEntrySecondaryIPsItems0 loadbalance entry secondary i ps items0 +// +// swagger:model LoadbalanceEntrySecondaryIPsItems0 +type LoadbalanceEntrySecondaryIPsItems0 struct { + + // IP address for externel access + SecondaryIP string `json:"secondaryIP,omitempty"` +} + +// Validate validates this loadbalance entry secondary i ps items0 +func (m *LoadbalanceEntrySecondaryIPsItems0) Validate(formats strfmt.Registry) error { + return nil +} + +// ContextValidate validates this loadbalance entry secondary i ps items0 based on context it is used +func (m *LoadbalanceEntrySecondaryIPsItems0) ContextValidate(ctx context.Context, formats strfmt.Registry) error { + return nil +} + +// MarshalBinary interface implementation +func (m *LoadbalanceEntrySecondaryIPsItems0) MarshalBinary() ([]byte, error) { + if m == nil { + return nil, nil + } + return swag.WriteJSON(m) +} + +// UnmarshalBinary interface implementation +func (m *LoadbalanceEntrySecondaryIPsItems0) UnmarshalBinary(b []byte) error { + var res LoadbalanceEntrySecondaryIPsItems0 + if err := swag.ReadJSON(b, &res); err != nil { + return err + } + *m = res + return nil +} + // LoadbalanceEntryServiceArguments loadbalance entry service arguments // // swagger:model LoadbalanceEntryServiceArguments diff --git a/api/restapi/embedded_spec.go b/api/restapi/embedded_spec.go index 3e7f952be..7b43e9758 100644 --- a/api/restapi/embedded_spec.go +++ b/api/restapi/embedded_spec.go @@ -7607,6 +7607,14 @@ func init() { } } }, + "LoadbalanceEntrySecondaryIPsItems0": { + "properties": { + "secondaryIP": { + "description": "IP address for externel access", + "type": "string" + } + } + }, "LoadbalanceEntryServiceArguments": { "type": "object", "properties": { diff --git a/api/restapi/handler/loadbalancer.go b/api/restapi/handler/loadbalancer.go index 120d735a8..040aa7d06 100644 --- a/api/restapi/handler/loadbalancer.go +++ b/api/restapi/handler/loadbalancer.go @@ -38,6 +38,14 @@ func ConfigPostLoadbalancer(params operations.PostConfigLoadbalancerParams) midd lbRules.Serv.Mode = cmn.LBMode(params.Attr.ServiceArguments.Mode) lbRules.Serv.InactiveTimeout = uint32(params.Attr.ServiceArguments.InactiveTimeOut) + if lbRules.Serv.Proto == "sctp" { + for _, data := range params.Attr.SecondaryIPs { + lbRules.SecIPs = append(lbRules.SecIPs, cmn.LbSecIpArg{ + SecIP: data.SecondaryIP, + }) + } + } + for _, data := range params.Attr.Endpoints { lbRules.Eps = append(lbRules.Eps, cmn.LbEndPointArg{ EpIP: data.EndpointIP, @@ -112,6 +120,12 @@ func ConfigGetLoadbalancer(params operations.GetConfigLoadbalancerAllParams) mid tmpLB.ServiceArguments = &tmpSvc + for _, sip := range lb.SecIPs { + tmpSIP := new(models.LoadbalanceEntrySecondaryIPsItems0) + tmpSIP.SecondaryIP = sip.SecIP + tmpLB.SecondaryIPs = append(tmpLB.SecondaryIPs, tmpSIP) + } + // Endpoints match for _, ep := range lb.Eps { tmpEp := new(models.LoadbalanceEntryEndpointsItems0) diff --git a/common/common.go b/common/common.go index b2fc91549..968bc5f32 100644 --- a/common/common.go +++ b/common/common.go @@ -513,10 +513,18 @@ type LbEndPointArg struct { State string `json:"state"` } +// LbSecIpArg - Secondary IP +type LbSecIpArg struct { + // SecIP - Secondary IP address + SecIP string `json:"secondaryIP"` +} + // LbRuleMod - Info related to a load-balancer entry type LbRuleMod struct { // Serv - service argument of type LbServiceArg Serv LbServiceArg `json:"serviceArguments"` + // SecIPs - Secondary IPs for SCTP multi-homed service + SecIPs []LbSecIpArg `json:"secondaryIPs"` // Eps - slice containing LbEndPointArg Eps []LbEndPointArg `json:"endpoints"` } diff --git a/loxilb-ebpf b/loxilb-ebpf index 2982953a7..c64a3fb91 160000 --- a/loxilb-ebpf +++ b/loxilb-ebpf @@ -1 +1 @@ -Subproject commit 2982953a745de866c019b3d1203e465bd3c47ebe +Subproject commit c64a3fb91ed0fe4b3ad77624cfbda9cac0ac0a75 diff --git a/loxinet/apiclient.go b/loxinet/apiclient.go index aaefff7ca..a51130af0 100644 --- a/loxinet/apiclient.go +++ b/loxinet/apiclient.go @@ -253,11 +253,15 @@ func (*NetAPIStruct) NetRouteDel(rm *cmn.RouteMod) (int, error) { func (*NetAPIStruct) NetLbRuleAdd(lm *cmn.LbRuleMod) (int, error) { mh.mtx.Lock() defer mh.mtx.Unlock() - - ret, err := mh.zr.Rules.AddNatLbRule(lm.Serv, lm.Eps[:]) + var ips []string + ret, err := mh.zr.Rules.AddNatLbRule(lm.Serv, lm.SecIPs[:], lm.Eps[:]) if err == nil && lm.Serv.Bgp { if mh.bgp != nil { - mh.bgp.AddBGPRule("default", lm.Serv.ServIP) + ips = append(ips, lm.Serv.ServIP) + for _, ip := range lm.SecIPs { + ips = append(ips, ip.SecIP) + } + mh.bgp.AddBGPRule("default", ips) } else { tk.LogIt(tk.LogDebug, "loxilb BGP mode is disabled \n") } @@ -270,10 +274,12 @@ func (*NetAPIStruct) NetLbRuleDel(lm *cmn.LbRuleMod) (int, error) { mh.mtx.Lock() defer mh.mtx.Unlock() + ips := mh.zr.Rules.GetNatLbRuleSecIPs(lm.Serv) ret, err := mh.zr.Rules.DeleteNatLbRule(lm.Serv) if lm.Serv.Bgp { if mh.bgp != nil { - mh.bgp.DelBGPRule("default", lm.Serv.ServIP) + ips = append(ips, lm.Serv.ServIP) + mh.bgp.DelBGPRule("default", ips) } else { tk.LogIt(tk.LogDebug, "loxilb BGP mode is disabled \n") } diff --git a/loxinet/dpbroker.go b/loxinet/dpbroker.go index 44696c843..f3703b761 100644 --- a/loxinet/dpbroker.go +++ b/loxinet/dpbroker.go @@ -288,6 +288,7 @@ type NatDpWorkQ struct { EpSel NatSel InActTo uint64 endPoints []NatEP + secIP []net.IP } // DpCtInfo - representation of a datapath conntrack information diff --git a/loxinet/dpebpf_linux.go b/loxinet/dpebpf_linux.go index 21e5dffd7..ce03bb71d 100644 --- a/loxinet/dpebpf_linux.go +++ b/loxinet/dpebpf_linux.go @@ -768,6 +768,14 @@ func DpNatLbRuleMod(w *NatDpWorkQ) int { // seconds to nanoseconds dat.ito = C.uint64_t(w.InActTo * 1000000000) + /*dat.npmhh = 2 + dat.pmhh[0] = 0x64646464 + dat.pmhh[1] = 0x65656565*/ + for i, k := range w.secIP { + dat.pmhh[i] = C.uint(tk.IPtonl(k)) + } + dat.npmhh = C.uchar(len(w.secIP)) + switch { case w.EpSel == EpRR: dat.sel_type = C.NAT_LB_SEL_RR @@ -787,7 +795,7 @@ func DpNatLbRuleMod(w *NatDpWorkQ) int { nxfa := (*nxfrmAct)(unsafe.Pointer(&dat.nxfrms[0])) for _, k := range w.endPoints { - nxfa.wprio = C.ushort(k.Weight) + nxfa.wprio = C.uchar(k.Weight) nxfa.nat_xport = C.ushort(tk.Htons(k.XPort)) if tk.IsNetIPv6(k.XIP.String()) { convNetIP2DPv6Addr(unsafe.Pointer(&nxfa.nat_xip[0]), k.XIP) @@ -1656,9 +1664,11 @@ func dpCTMapChkUpdates() { if time.Duration(tc.Sub(cti.LTs).Seconds()) >= time.Duration(5*60) { tk.LogIt(tk.LogInfo, "[CT] out-of-sync %s:%s:%v\n", cti.Key(), cti.CState, cti.XSync) if C.bpf_map_lookup_elem(C.int(fd), unsafe.Pointer(&cti.PKey[0]), unsafe.Pointer(&tact)) != 0 { + tk.LogIt(tk.LogInfo, "[CT] out-of-sync not found %s:%s:%v\n", cti.Key(), cti.CState, cti.XSync) delete(mh.dpEbpf.ctMap, cti.Key()) continue } + cti.PVal = C.GoBytes(unsafe.Pointer(&tact), C.sizeof_struct_dp_ct_tact) cti.LTs = tc } @@ -1666,14 +1676,18 @@ func dpCTMapChkUpdates() { if time.Duration(tc.Sub(cti.NTs).Seconds()) < time.Duration(60) { continue } + if C.bpf_map_lookup_elem(C.int(fd), unsafe.Pointer(&cti.PKey[0]), unsafe.Pointer(&tact)) != 0 { + tk.LogIt(tk.LogInfo, "[CT] ent not found %s\n", cti.Key()) + delete(mh.dpEbpf.ctMap, cti.Key()) + continue + } ptact := (*C.struct_dp_ct_tact)(unsafe.Pointer(&cti.PVal[0])) ret := C.llb_fetch_map_stats_cached(C.int(C.LL_DP_CT_STATS_MAP), C.uint(ptact.ca.cidx), C.int(0), (unsafe.Pointer(&b)), unsafe.Pointer(&p)) - if ret == 0 { - if cti.Packets != p { - cti.Bytes = b - cti.Packets = p + if cti.Packets != p+uint64(tact.ctd.pb.packets) { + cti.Bytes = b + uint64(tact.ctd.pb.bytes) + cti.Packets = p + uint64(tact.ctd.pb.packets) cti.XSync = true cti.NTs = tc cti.LTs = tc diff --git a/loxinet/gobgpclient.go b/loxinet/gobgpclient.go index 44907cbe0..8e76b75a9 100644 --- a/loxinet/gobgpclient.go +++ b/loxinet/gobgpclient.go @@ -48,10 +48,17 @@ const ( bgpTO ) +type goBgpRouteInfo struct { + nlri bgp.AddrPrefixInterface + attrs []bgp.PathAttributeInterface + withdraw bool + pathId uint32 +} + type goBgpEvent struct { EventType goBgpEventType Src string - Data api.Path + Data goBgpRouteInfo conn *grpc.ClientConn } @@ -66,7 +73,7 @@ type goCI struct { name string hastate int vip net.IP - rules map[string]bool + rules map[string]int } // GoBgpH - context container @@ -119,35 +126,33 @@ func (gbh *GoBgpH) getNextHopFromPathAttributes(attrs []bgp.PathAttributeInterfa return nil } -func (gbh *GoBgpH) makeMonitorRouteArgs(p *api.Path, showIdentifier bgp.BGPAddPathMode) []interface{} { +func (gbh *GoBgpH) makeMonitorRouteArgs(p *goBgpRouteInfo, showIdentifier bgp.BGPAddPathMode) []interface{} { pathStr := make([]interface{}, 0) // Title title := "ADDROUTE" - if p.IsWithdraw { + if p.withdraw { title = "DELROUTE" } pathStr = append(pathStr, title) // NLRI // If Add-Path required, append Path Identifier. - nlri, _ := apiutil.GetNativeNlri(p) if showIdentifier != bgp.BGP_ADD_PATH_NONE { - pathStr = append(pathStr, p.GetIdentifier()) + pathStr = append(pathStr, p.pathId) } - pathStr = append(pathStr, nlri) + pathStr = append(pathStr, p.nlri) - attrs, _ := apiutil.GetNativePathAttributes(p) // Next Hop nexthop := "fictitious" - if n := gbh.getNextHopFromPathAttributes(attrs); n != nil { + if n := gbh.getNextHopFromPathAttributes(p.attrs); n != nil { nexthop = n.String() } pathStr = append(pathStr, nexthop) // AS_PATH aspathstr := func() string { - for _, attr := range attrs { + for _, attr := range p.attrs { switch a := attr.(type) { case *bgp.PathAttributeAsPath: return bgp.AsPathString(a) @@ -158,12 +163,12 @@ func (gbh *GoBgpH) makeMonitorRouteArgs(p *api.Path, showIdentifier bgp.BGPAddPa pathStr = append(pathStr, aspathstr) // Path Attributes - pathStr = append(pathStr, gbh.getPathAttributeString(nlri, attrs)) + pathStr = append(pathStr, gbh.getPathAttributeString(p.nlri, p.attrs)) return pathStr } -func (gbh *GoBgpH) processRouteSingle(p *api.Path, showIdentifier bgp.BGPAddPathMode) { +func (gbh *GoBgpH) processRouteSingle(p *goBgpRouteInfo, showIdentifier bgp.BGPAddPathMode) { //pathStr := make([]interface{}, 1) pathStr := gbh.makeMonitorRouteArgs(p, showIdentifier) @@ -178,31 +183,22 @@ func (gbh *GoBgpH) processRouteSingle(p *api.Path, showIdentifier bgp.BGPAddPath tk.LogIt(tk.LogInfo, format, pathStr...) if err := gbh.syncRoute(p, showIdentifier); err != nil { - tk.LogIt(tk.LogError, " failed to"+format, pathStr...) + tk.LogIt(tk.LogError, " failed to "+format, pathStr...) } } -func (gbh *GoBgpH) syncRoute(p *api.Path, showIdentifier bgp.BGPAddPathMode) error { +func (gbh *GoBgpH) syncRoute(p *goBgpRouteInfo, showIdentifier bgp.BGPAddPathMode) error { if gbh.noNlp { return nil } - // NLRI have destination CIDR info - nlri, err := apiutil.GetNativeNlri(p) - if err != nil { - return err - } - _, dstIPN, err := net.ParseCIDR(nlri.String()) + _, dstIPN, err := net.ParseCIDR(p.nlri.String()) if err != nil { return err } // NextHop - attrs, err := apiutil.GetNativePathAttributes(p) - if err != nil { - return err - } - nexthop := gbh.getNextHopFromPathAttributes(attrs) + nexthop := gbh.getNextHopFromPathAttributes(p.attrs) // Make netlink route and add route := &nlp.Route{ @@ -214,7 +210,7 @@ func (gbh *GoBgpH) syncRoute(p *api.Path, showIdentifier bgp.BGPAddPathMode) err return nil } - if p.GetIsWithdraw() { + if p.withdraw { tk.LogIt(tk.LogDebug, "[GoBGP] ip route delete %s via %s\n", route.Dst.String(), route.Gw.String()) if err := nlp.RouteDel(route); err != nil { tk.LogIt(tk.LogError, "[GoBGP] failed to ip route delete. err: %s\n", err.Error()) @@ -234,10 +230,23 @@ func (gbh *GoBgpH) syncRoute(p *api.Path, showIdentifier bgp.BGPAddPathMode) err func (gbh *GoBgpH) processRoute(pathList []*api.Path) { for _, p := range pathList { + // NLRI have destination CIDR info + nlri, err := apiutil.GetNativeNlri(p) + if err != nil { + return + } + // NextHop + attrs, err := apiutil.GetNativePathAttributes(p) + if err != nil { + return + } + + data := goBgpRouteInfo{nlri: nlri, attrs: attrs, withdraw: p.GetIsWithdraw(), pathId: p.GetIdentifier()} + gbh.eventCh <- goBgpEvent{ EventType: bgpRtRecvd, Src: "", - Data: *p, + Data: data, conn: &grpc.ClientConn{}, } } @@ -470,32 +479,33 @@ func (gbh *GoBgpH) goBgpConnect(host string) { } // AddBGPRule - add a bgp rule in goBGP -func (gbh *GoBgpH) AddBGPRule(instance string, IP string) { +func (gbh *GoBgpH) AddBGPRule(instance string, IP []string) { var pref uint32 gbh.mtx.Lock() ci := gbh.ciMap[instance] if ci == nil { ci = new(goCI) - ci.rules = make(map[string]bool) + ci.rules = make(map[string]int) ci.name = instance ci.hastate = cmn.CIStateBackup ci.vip = net.IPv4zero gbh.ciMap[instance] = ci } - if !ci.rules[IP] { - ci.rules[IP] = true - } - if gbh.state == BGPConnected { - if ci.hastate == cmn.CIStateBackup { - pref = cmn.LowLocalPref - } else { - pref = cmn.HighLocalPref - } - if net.ParseIP(IP).To4() != nil { - gbh.AdvertiseRoute(IP, 32, "0.0.0.0", pref, true) - } else { - gbh.AdvertiseRoute(IP, 128, "::", pref, false) + for _, ip := range IP { + ci.rules[ip]++ + + if gbh.state == BGPConnected { + if ci.hastate == cmn.CIStateBackup { + pref = cmn.LowLocalPref + } else { + pref = cmn.HighLocalPref + } + if net.ParseIP(ip).To4() != nil { + gbh.AdvertiseRoute(ip, 32, "0.0.0.0", pref, true) + } else { + gbh.AdvertiseRoute(ip, 128, "::", pref, false) + } } } @@ -503,29 +513,32 @@ func (gbh *GoBgpH) AddBGPRule(instance string, IP string) { } // DelBGPRule - delete a bgp rule in goBGP -func (gbh *GoBgpH) DelBGPRule(instance string, IP string) { +func (gbh *GoBgpH) DelBGPRule(instance string, IP []string) { var pref uint32 gbh.mtx.Lock() ci := gbh.ciMap[instance] if ci == nil { tk.LogIt(tk.LogError, "[GoBGP] Del BGP Rule - Invalid instance %s\n", instance) + gbh.mtx.Unlock() return } - if ci.rules[IP] { - ci.rules[IP] = false - } - - if gbh.state == BGPConnected { - if ci.hastate == cmn.CIStateBackup { - pref = cmn.LowLocalPref - } else { - pref = cmn.HighLocalPref + for _, ip := range IP { + if ci.rules[ip] > 0 { + ci.rules[ip]-- } - if net.ParseIP(IP).To4() != nil { - gbh.DelAdvertiseRoute(IP, 32, "0.0.0.0", pref) - } else { - gbh.DelAdvertiseRoute(IP, 128, "::", pref) + if gbh.state == BGPConnected && ci.rules[ip] == 0 { + if ci.hastate == cmn.CIStateBackup { + pref = cmn.LowLocalPref + } else { + pref = cmn.HighLocalPref + } + if net.ParseIP(ip).To4() != nil { + gbh.DelAdvertiseRoute(ip, 32, "0.0.0.0", pref) + } else { + gbh.DelAdvertiseRoute(ip, 128, "::", pref) + } + tk.LogIt(tk.LogDebug, "[GoBGP] Del BGP Rule %s\n", ip) } } gbh.mtx.Unlock() @@ -617,9 +630,9 @@ func (gbh *GoBgpH) advertiseAllRoutes(instance string) { gbh.AdvertiseRoute(ci.vip.String(), 32, "0.0.0.0", pref, true) } - for ip, valid := range ci.rules { - tk.LogIt(tk.LogDebug, "[GoBGP] connected BGP rules ip %s is valid(%v)\n", ip, valid) - if valid { + for ip, count := range ci.rules { + tk.LogIt(tk.LogDebug, "[GoBGP] connected BGP rules ip %s ref count(%d)\n", ip, count) + if count > 0 { if net.ParseIP(ip).To4() != nil { gbh.AdvertiseRoute(ip, 32, "0.0.0.0", pref, true) } else { @@ -690,7 +703,7 @@ func (gbh *GoBgpH) UpdateCIState(instance string, state int, vip net.IP) { ci := gbh.ciMap[instance] if ci == nil { ci = new(goCI) - ci.rules = make(map[string]bool) + ci.rules = make(map[string]int) } ci.name = instance ci.hastate = state diff --git a/loxinet/loxinettest.go b/loxinet/loxinettest.go index 59125e940..ae09c3bc8 100644 --- a/loxinet/loxinettest.go +++ b/loxinet/loxinettest.go @@ -286,7 +286,7 @@ func TestLoxinet(t *testing.T) { Weight: 2, }, } - _, err = mh.zr.Rules.AddNatLbRule(lbServ, lbEps[:]) + _, err = mh.zr.Rules.AddNatLbRule(lbServ, nil, lbEps[:]) if err != nil { t.Errorf("failed to add nat lb rule for 10.10.10.1\n") } diff --git a/loxinet/rules.go b/loxinet/rules.go index bed69f5bf..7a4b08887 100644 --- a/loxinet/rules.go +++ b/loxinet/rules.go @@ -27,6 +27,7 @@ import ( probing "github.com/prometheus-community/pro-bing" "io/ioutil" "net" + "reflect" "sort" "strconv" "sync" @@ -207,6 +208,10 @@ type ruleNatEp struct { Mark bool } +type ruleNatSIP struct { + sIP net.IP +} + type ruleNatActs struct { mode cmn.LBMode sel cmn.EpSelect @@ -248,6 +253,7 @@ type ruleEnt struct { sT time.Time iTo uint32 act ruleAct + secIP []ruleNatSIP stat ruleStat } @@ -714,6 +720,10 @@ func (R *RuleH) GetNatLbRule() ([]cmn.LbRuleMod, error) { ret.Serv.Bgp = data.BGP ret.Serv.BlockNum = data.tuples.pref + for _, sip := range data.secIP { + ret.SecIPs = append(ret.SecIPs, cmn.LbSecIpArg{SecIP: sip.sIP.String()}) + } + // Make Endpoints tmpEp := data.act.action.(*ruleNatActs).endPoints for _, ep := range tmpEp { @@ -834,6 +844,39 @@ func (R *RuleH) GetNatLbRuleByServArgs(serv cmn.LbServiceArg) *ruleEnt { return R.Tables[RtLB].eMap[rt.ruleKey()] } +// GetNatLbRuleSecIPs - Get secondary IPs for SCTP NAT rule by its service args +func (R *RuleH) GetNatLbRuleSecIPs(serv cmn.LbServiceArg) []string { + var ipProto uint8 + var ips []string + service := "" + if tk.IsNetIPv4(serv.ServIP) { + service = serv.ServIP + "/32" + } else { + service = serv.ServIP + "/128" + } + _, sNetAddr, err := net.ParseCIDR(service) + if err != nil { + return nil + } + + if serv.Proto == "sctp" { + ipProto = 132 + } else { + return nil + } + + l4prot := rule8Tuple{ipProto, 0xff} + l3dst := ruleIPTuple{*sNetAddr} + l4dst := rule16Tuple{serv.ServPort, 0xffff} + rt := ruleTuples{l3Dst: l3dst, l4Prot: l4prot, l4Dst: l4dst, pref: serv.BlockNum} + if R.Tables[RtLB].eMap[rt.ruleKey()] != nil { + for _, ip := range R.Tables[RtLB].eMap[rt.ruleKey()].secIP { + ips = append(ips, ip.sIP.String()) + } + } + return ips +} + func (R *RuleH) syncEPHostState2Rule(rule *ruleEnt, checkNow bool) bool { var sType string rChg := false @@ -880,8 +923,9 @@ func (R *RuleH) syncEPHostState2Rule(rule *ruleEnt, checkNow bool) bool { // AddNatLbRule - Add a service LB nat rule. The service details are passed in serv argument, // and end-point information is passed in the slice servEndPoints. On success, // it will return 0 and nil error, else appropriate return code and error string will be set -func (R *RuleH) AddNatLbRule(serv cmn.LbServiceArg, servEndPoints []cmn.LbEndPointArg) (int, error) { +func (R *RuleH) AddNatLbRule(serv cmn.LbServiceArg, servSecIPs []cmn.LbSecIpArg, servEndPoints []cmn.LbEndPointArg) (int, error) { var natActs ruleNatActs + var nSecIP []ruleNatSIP var ipProto uint8 // Validate service args @@ -925,6 +969,32 @@ func (R *RuleH) AddNatLbRule(serv cmn.LbServiceArg, servEndPoints []cmn.LbEndPoi return RuleUnknownServiceErr, errors.New("malformed-proto error") } + if serv.Proto != "sctp" && len(servSecIPs) > 0 { + return RuleArgsErr, errors.New("secondaryIP-args error") + } + + if len(servSecIPs) > 3 { + return RuleArgsErr, errors.New("secondaryIP-args len error") + } + + for _, k := range servSecIPs { + pNetAddr := net.ParseIP(k.SecIP) + if pNetAddr == nil { + return RuleUnknownServiceErr, errors.New("malformed-secIP error") + } + if tk.IsNetIPv4(serv.ServIP) && tk.IsNetIPv6(k.SecIP) { + return RuleUnknownServiceErr, errors.New("malformed-secIP nat46 error") + } + sip := ruleNatSIP{pNetAddr} + nSecIP = append(nSecIP, sip) + } + + sort.SliceStable(nSecIP, func(i, j int) bool { + a := tk.IPtonl(nSecIP[i].sIP) + b := tk.IPtonl(nSecIP[j].sIP) + return a < b + }) + natActs.sel = serv.Sel natActs.mode = cmn.LBMode(serv.Mode) @@ -961,6 +1031,9 @@ func (R *RuleH) AddNatLbRule(serv cmn.LbServiceArg, servEndPoints []cmn.LbEndPoi eRule := R.Tables[RtLB].eMap[rt.ruleKey()] if eRule != nil { + if !reflect.DeepEqual(eRule.secIP, nSecIP) { + return RuleUnknownServiceErr, errors.New("secIP modify error") + } // If a NAT rule already exists, we try not reschuffle the order of the end-points. // We will try to append the new end-points at the end, while marking any other end-points // not in the new list as inactive @@ -1029,11 +1102,11 @@ func (R *RuleH) AddNatLbRule(serv cmn.LbServiceArg, servEndPoints []cmn.LbEndPoi } else { r.act.actType = RtActDnat } + r.secIP = nSecIP // Per LB end-point health-check is supposed to be handled at CCM, // but it certain cases like stand-alone mode, loxilb can do its own // lb end-point health monitoring r.ActChk = serv.Monitor - r.act.action = &natActs r.ruleNum, err = R.Tables[RtLB].Mark.GetCounter() if err != nil { @@ -1463,7 +1536,7 @@ func (R *RuleH) AddEPHost(apiCall bool, hostName string, name string, args epHos ep.ruleCount = 1 } ep.hID = R.lepHID % MaxEndPointCheckers - ep.sT = time.Now() + //ep.sT = time.Now() R.lepHID++ R.epMap[epKey] = ep @@ -1824,6 +1897,10 @@ func (r *ruleEnt) Nat2DP(work DpWorkT) int { mode := cmn.LBModeDefault + for _, sip := range r.secIP { + nWork.secIP = append(nWork.secIP, sip.sIP) + } + switch at := r.act.action.(type) { case *ruleNatActs: switch {