From c3c5c6d52cf56ebde8d7013408cf736ceafa8152 Mon Sep 17 00:00:00 2001 From: Trekkie Coder Date: Wed, 24 Jan 2024 23:03:17 +0900 Subject: [PATCH 1/6] Updated sub-module --- loxilb-ebpf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loxilb-ebpf b/loxilb-ebpf index e40eadc6..08a7b2ab 160000 --- a/loxilb-ebpf +++ b/loxilb-ebpf @@ -1 +1 @@ -Subproject commit e40eadc60e531f2c025f4700ff3e918e16154890 +Subproject commit 08a7b2abbbd53b503a8dd2d6a5732f69d9ff6661 From 159fcb4da95964b220958714639560f3c5b850ae Mon Sep 17 00:00:00 2001 From: Trekkie Coder Date: Wed, 24 Jan 2024 23:15:50 +0900 Subject: [PATCH 2/6] Removed unwanted img from dockerfile --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9bbc4a91..1a4db09d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,7 +45,6 @@ RUN mkdir -p /opt/loxilb && \ git clone --recurse-submodules https://github.com/loxilb-io/loxilb /root/loxilb-io/loxilb/ && \ cd /root/loxilb-io/loxilb/ && go get . && make && \ cp loxilb-ebpf/utils/mkllb_bpffs.sh /usr/local/sbin/mkllb_bpffs && \ - cp loxilb-ebpf/utils/loxilb_dp_tool /usr/local/sbin/loxilb_dp_tool && \ cp api/certification/* /opt/loxilb/cert/ && cd - && \ cp /root/loxilb-io/loxilb/loxilb-ebpf/kernel/loxilb_dp_debug /usr/local/sbin/loxilb_dp_debug && \ cp /root/loxilb-io/loxilb/loxilb /usr/local/sbin/loxilb && \ From a582a9b2795550af231d03c01825ab6415abe98d Mon Sep 17 00:00:00 2001 From: Trekkie Coder Date: Thu, 25 Jan 2024 01:56:50 +0900 Subject: [PATCH 3/6] gh-494 Continued Fixes for ipvs-compat mode --- api/api.go | 2 ++ api/loxinlp/nlp.go | 36 ++++++++++++++++++++++++++++-------- common/common.go | 7 ++++--- loxinet/apiclient.go | 5 +++++ loxinet/dpbroker.go | 6 ++++++ loxinet/dpebpf_linux.go | 9 +++++++++ loxinet/layer2.go | 9 ++++++++- loxinet/loxinet.go | 23 +++++++++++++++++++++++ loxinet/port.go | 15 ++++++++++----- loxinet/xsync_server.go | 5 +++++ 10 files changed, 100 insertions(+), 17 deletions(-) diff --git a/api/api.go b/api/api.go index 3b519fdc..fa08237d 100644 --- a/api/api.go +++ b/api/api.go @@ -77,6 +77,8 @@ func RunAPIServer() { if e := recover(); e != nil { tk.LogIt(tk.LogCritical, "%s: %s", e, debug.Stack()) } + handler.ApiHooks.NetHandlePanic() + os.Exit(1) }() if ApiShutOk == nil { diff --git a/api/loxinlp/nlp.go b/api/loxinlp/nlp.go index ffd7bd36..e7560f92 100644 --- a/api/loxinlp/nlp.go +++ b/api/loxinlp/nlp.go @@ -27,6 +27,7 @@ import ( "os" "os/exec" "regexp" + "runtime/debug" "strconv" "strings" "syscall" @@ -1378,8 +1379,18 @@ func RUWorker(ch chan nlp.RouteUpdate, f chan struct{}) { } } -func NLWorker(nNl *NlH, bgpPeerMode bool, ch chan bool) { +func NLWorker(nNl *NlH, bgpPeerMode bool, ch chan bool, wch chan bool) { ch <- true + <-wch + + defer func() { + if e := recover(); e != nil { + tk.LogIt(tk.LogCritical, "%s: %s", e, debug.Stack()) + } + hooks.NetHandlePanic() + os.Exit(1) + }() + if bgpPeerMode { for { /* Single thread for reading route NL msgs in below order */ RUWorker(nNl.FromRUCh, nNl.FromRUDone) @@ -1426,6 +1437,18 @@ func NlpGet(ch chan bool) int { ret = -1 } + for _, link := range links { + + if iSBlackListedIntf(link.Attrs().Name, link.Attrs().MasterIndex) { + continue + } + + ret = ModLink(link, true) + if ret == -1 { + continue + } + } + for _, link := range links { if iSBlackListedIntf(link.Attrs().Name, link.Attrs().MasterIndex) { @@ -1446,11 +1469,6 @@ func NlpGet(ch chan bool) int { continue } - ret = ModLink(link, true) - if ret == -1 { - continue - } - /* Get FDBs */ _, ok := link.(*nlp.Vxlan) if link.Attrs().MasterIndex > 0 || ok { @@ -1580,6 +1598,7 @@ func NlpInit(bgpPeerMode bool, blackList string, ipvsCompat bool) *NlH { nNl.BlackList = blackList nNl.BLRgx = regexp.MustCompile(blackList) checkInit := make(chan bool) + waitInit := make(chan bool) if bgpPeerMode { nNl.FromRUCh = make(chan nlp.RouteUpdate, cmn.RuWorkQLen) @@ -1590,7 +1609,7 @@ func NlpInit(bgpPeerMode bool, blackList string, ipvsCompat bool) *NlH { tk.LogIt(tk.LogInfo, "[NLP] Route msgs subscribed\n") } - go NLWorker(nNl, bgpPeerMode, checkInit) + go NLWorker(nNl, bgpPeerMode, checkInit, waitInit) <-checkInit return nNl } @@ -1605,7 +1624,7 @@ func NlpInit(bgpPeerMode bool, blackList string, ipvsCompat bool) *NlH { nNl.FromRUCh = make(chan nlp.RouteUpdate, cmn.RuWorkQLen) nNl.IMap = make(map[string]Intf) - go NLWorker(nNl, bgpPeerMode, checkInit) + go NLWorker(nNl, bgpPeerMode, checkInit, waitInit) <-checkInit err := nlp.LinkSubscribe(nNl.FromLUCh, nNl.FromLUDone) @@ -1637,6 +1656,7 @@ func NlpInit(bgpPeerMode bool, blackList string, ipvsCompat bool) *NlH { go NlpGet(checkInit) done := <-checkInit + waitInit <- true go LbSessionGet(done) diff --git a/common/common.go b/common/common.go index 286864f3..086a835a 100644 --- a/common/common.go +++ b/common/common.go @@ -65,11 +65,11 @@ const ( const ( // AuWorkqLen - Address worker channel depth - AuWorkqLen = 1024 + AuWorkqLen = 2048 // LuWorkQLen - Link worker channel depth - LuWorkQLen = 1024 + LuWorkQLen = 2048 // NuWorkQLen - Neigh worker channel depth - NuWorkQLen = 1024 + NuWorkQLen = 2048 // RuWorkQLen - Route worker channel depth RuWorkQLen = 40827 ) @@ -833,4 +833,5 @@ type NetHookInterface interface { NetGoBGPNeighAdd(nm *GoBGPNeighMod) (int, error) NetGoBGPNeighDel(nm *GoBGPNeighMod) (int, error) NetGoBGPGCAdd(gc *GoBGPGlobalConfig) (int, error) + NetHandlePanic() } diff --git a/loxinet/apiclient.go b/loxinet/apiclient.go index 80ba65b0..f4aee5ea 100644 --- a/loxinet/apiclient.go +++ b/loxinet/apiclient.go @@ -629,3 +629,8 @@ func (na *NetAPIStruct) NetGoBGPGCAdd(param *cmn.GoBGPGlobalConfig) (int, error) return 0, errors.New("loxilb BGP mode is disabled") } + +// NetHandlePanic - Handle panics +func (na *NetAPIStruct) NetHandlePanic() { + mh.dp.DpHooks.DpEbpfUnInit() +} diff --git a/loxinet/dpbroker.go b/loxinet/dpbroker.go index f7e228ad..09e2f617 100644 --- a/loxinet/dpbroker.go +++ b/loxinet/dpbroker.go @@ -19,6 +19,7 @@ package loxinet import ( "fmt" "net" + "os" "runtime/debug" "sync" "time" @@ -411,6 +412,7 @@ type DpHookInterface interface { DpCtGetAsync() DpGetLock() DpRelLock() + DpEbpfUnInit() } // DpPeer - Remote DP Peer information @@ -786,6 +788,10 @@ func DpWorker(dp *DpH, f chan int, ch chan interface{}) { if e := recover(); e != nil { tk.LogIt(tk.LogCritical, "%s: %s", e, debug.Stack()) } + if mh.dp != nil { + mh.dp.DpHooks.DpEbpfUnInit() + } + os.Exit(1) }() for { for n := 0; n < DpWorkQLen; n++ { diff --git a/loxinet/dpebpf_linux.go b/loxinet/dpebpf_linux.go index c2017d94..82c89f9d 100644 --- a/loxinet/dpebpf_linux.go +++ b/loxinet/dpebpf_linux.go @@ -45,6 +45,7 @@ import ( "errors" "fmt" "net" + "os" "runtime/debug" "strings" "sync" @@ -157,6 +158,10 @@ func dpEbpfTicker() { if e := recover(); e != nil { tk.LogIt(tk.LogCritical, "%s: %s", e, debug.Stack()) } + if mh.dp != nil { + mh.dp.DpHooks.DpEbpfUnInit() + } + os.Exit(1) }() tbls := []int{int(C.LL_DP_RTV4_STATS_MAP), @@ -1918,6 +1923,10 @@ func dpMapNotifierWorker(f chan int, ch chan interface{}) { if e := recover(); e != nil { tk.LogIt(tk.LogCritical, "%s: %s", e, debug.Stack()) } + if mh.dp != nil { + mh.dp.DpHooks.DpEbpfUnInit() + } + os.Exit(1) }() for { diff --git a/loxinet/layer2.go b/loxinet/layer2.go index cc4a2be9..f2611480 100644 --- a/loxinet/layer2.go +++ b/loxinet/layer2.go @@ -291,7 +291,7 @@ func (l2 *L2H) L2FdbDel(key FdbKey) (int, error) { // FdbTicker - Ticker routine for a fwd entry func (l2 *L2H) FdbTicker(f *FdbEnt) { - if time.Now().Sub(f.stime) > FdbGts { + if time.Since(f.stime) > FdbGts { // This scans for inconsistencies in a fdb // 1. Do garbage cleaning if underlying oif or vlan is not valid anymore // 2. If FDB is a TunFDB, we need to make sure NH is reachable @@ -304,6 +304,8 @@ func (l2 *L2H) FdbTicker(f *FdbEnt) { f.unReach = unRch f.DP(DpCreate) } + } else if f.Sync != 0 { + f.DP(DpCreate) } f.stime = time.Now() } @@ -385,6 +387,10 @@ func (f *FdbEnt) DP(work DpWorkT) int { l2Wq.Tagged = 0 } else { l2Wq.Tagged = 1 + if f.Port.SInfo.PortReal == nil { + f.Sync = DpUknownErr + return -1 + } l2Wq.PortNum = f.Port.SInfo.PortReal.PortNo } mh.dp.ToDpCh <- l2Wq @@ -396,6 +402,7 @@ func (f *FdbEnt) DP(work DpWorkT) int { if f.Port.SInfo.PortReal == nil || f.FdbTun.ep == nil { + f.Sync = DpUknownErr return -1 } diff --git a/loxinet/loxinet.go b/loxinet/loxinet.go index 80e96719..3e9b6fb3 100644 --- a/loxinet/loxinet.go +++ b/loxinet/loxinet.go @@ -129,6 +129,17 @@ func (mh *loxiNetH) ParamGet(param *cmn.ParamMod) (int, error) { // loxiNetTicker - this ticker routine runs every LOXINET_TIVAL seconds func loxiNetTicker(bgpPeerMode bool) { + + defer func() { + if e := recover(); e != nil { + tk.LogIt(tk.LogCritical, "%s: %s", e, debug.Stack()) + } + if mh.dp != nil { + mh.dp.DpHooks.DpEbpfUnInit() + } + os.Exit(1) + }() + for { select { case <-mh.tDone: @@ -187,6 +198,10 @@ func loxiNetInit() { if e := recover(); e != nil { tk.LogIt(tk.LogCritical, "%s: %s", e, debug.Stack()) } + if mh.dp != nil { + mh.dp.DpHooks.DpEbpfUnInit() + } + os.Exit(1) }() // It is important to make sure loxilb's eBPF filesystem @@ -310,6 +325,14 @@ func loxiNetInit() { mh.ready = true } +type Shark struct { + Name string +} + +func (s *Shark) SayHello() { + fmt.Println("Hi! My name is", s.Name) +} + // loxiNetRun - This routine will not return func loxiNetRun() { mh.wg.Wait() diff --git a/loxinet/port.go b/loxinet/port.go index c8683faf..f211dd68 100644 --- a/loxinet/port.go +++ b/loxinet/port.go @@ -914,7 +914,8 @@ func (p *Port) DP(work DpWorkT) int { ipts.Qfi = 0 ipts.TTeID = 0 - mh.dp.ToDpCh <- ipts + //mh.dp.ToDpCh <- ipts + DpWorkSingle(mh.dp, ipts) return 0 } @@ -944,7 +945,8 @@ func (p *Port) DP(work DpWorkT) int { rmWq.TunType = DpTunVxlan rmWq.BD = p.L2.Vid - mh.dp.ToDpCh <- rmWq + //mh.dp.ToDpCh <- rmWq + DpWorkSingle(mh.dp, rmWq) return 0 } @@ -961,7 +963,8 @@ func (p *Port) DP(work DpWorkT) int { pWq.IngVlan = p.L2.Vid pWq.SetBD = p.L2.Vid pWq.SetZoneNum = zoneNum - mh.dp.ToDpCh <- pWq + //mh.dp.ToDpCh <- pWq + DpWorkSingle(mh.dp, pWq) return 0 } @@ -981,7 +984,8 @@ func (p *Port) DP(work DpWorkT) int { pWq.SetPol = p.SInfo.PortPolNum pWq.SetMirr = p.SInfo.PortMirNum - mh.dp.ToDpCh <- pWq + //mh.dp.ToDpCh <- pWq + DpWorkSingle(mh.dp, pWq) } return 0 } @@ -1039,7 +1043,8 @@ func (p *Port) DP(work DpWorkT) int { } // TODO - Need to unload eBPF when port properties change - mh.dp.ToDpCh <- pWq + //mh.dp.ToDpCh <- pWq + DpWorkSingle(mh.dp, pWq) return 0 } diff --git a/loxinet/xsync_server.go b/loxinet/xsync_server.go index 3661994c..fd118caa 100644 --- a/loxinet/xsync_server.go +++ b/loxinet/xsync_server.go @@ -24,6 +24,7 @@ import ( "net" "net/http" "net/rpc" + "os" "runtime/debug" opts "github.com/loxilb-io/loxilb/options" @@ -218,6 +219,10 @@ func LoxiXsyncMain(mode string) { if mh.logger != nil { tk.LogIt(tk.LogCritical, "%s: %s", e, debug.Stack()) } + if mh.dp != nil { + mh.dp.DpHooks.DpEbpfUnInit() + } + os.Exit(1) } }() if mode == "netrpc" { From ebafceb210b59321f83ceae58dba64e1cd38134c Mon Sep 17 00:00:00 2001 From: Trekkie Coder Date: Thu, 25 Jan 2024 01:57:46 +0900 Subject: [PATCH 4/6] gh-494 Continued Fixes for ipvs-compat mode --- loxinet/loxinet.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/loxinet/loxinet.go b/loxinet/loxinet.go index 3e9b6fb3..29e45aed 100644 --- a/loxinet/loxinet.go +++ b/loxinet/loxinet.go @@ -325,14 +325,6 @@ func loxiNetInit() { mh.ready = true } -type Shark struct { - Name string -} - -func (s *Shark) SayHello() { - fmt.Println("Hi! My name is", s.Name) -} - // loxiNetRun - This routine will not return func loxiNetRun() { mh.wg.Wait() From b2bce3858d923488493e6f76eafc63563ae9f680 Mon Sep 17 00:00:00 2001 From: Trekkie Coder Date: Thu, 25 Jan 2024 02:32:20 +0900 Subject: [PATCH 5/6] gh-494 In any panic, we need to gracefully exit and unload eBPF at the min --- loxinet/dpebpf_linux.go | 2 +- loxinet/loxinet.go | 21 ++++++++++----------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/loxinet/dpebpf_linux.go b/loxinet/dpebpf_linux.go index 82c89f9d..b0313359 100644 --- a/loxinet/dpebpf_linux.go +++ b/loxinet/dpebpf_linux.go @@ -317,7 +317,7 @@ func (e *DpEbpfH) DpEbpfUnInit() { e.ToFinCh[i] <- 1 } - tk.LogIt(tk.LogInfo, "ebpf uninit \n") + tk.LogIt(tk.LogInfo, "ebpf uninit : %s\n", debug.Stack()) // Make sure to unload eBPF programs ifList, err := net.Interfaces() diff --git a/loxinet/loxinet.go b/loxinet/loxinet.go index 29e45aed..5c06ceaf 100644 --- a/loxinet/loxinet.go +++ b/loxinet/loxinet.go @@ -193,17 +193,6 @@ func loxiNetInit() { logLevel := LogString2Level(opts.Opts.LogLevel) mh.logger = tk.LogItInit(logfile, logLevel, true) - // Stack trace logger - defer func() { - if e := recover(); e != nil { - tk.LogIt(tk.LogCritical, "%s: %s", e, debug.Stack()) - } - if mh.dp != nil { - mh.dp.DpHooks.DpEbpfUnInit() - } - os.Exit(1) - }() - // It is important to make sure loxilb's eBPF filesystem // is in place and mounted to make sure maps are pinned properly if !FileExists(BpfFsCheckFile) { @@ -327,6 +316,16 @@ func loxiNetInit() { // loxiNetRun - This routine will not return func loxiNetRun() { + // Stack trace logger + defer func() { + if e := recover(); e != nil { + tk.LogIt(tk.LogCritical, "%s: %s", e, debug.Stack()) + } + if mh.dp != nil { + mh.dp.DpHooks.DpEbpfUnInit() + } + os.Exit(1) + }() mh.wg.Wait() } From 822c323e6bb06ce88fd9c4abd4203c55bea180b3 Mon Sep 17 00:00:00 2001 From: Trekkie Coder Date: Thu, 25 Jan 2024 09:23:07 +0900 Subject: [PATCH 6/6] Changed to latest lib --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 4186cd73..249520a6 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/go-openapi/validate v0.22.0 github.com/jessevdk/go-flags v1.5.0 github.com/loxilb-io/ipvs v0.1.0 - github.com/loxilb-io/loxilib v0.8.9-0.20231211082246-dc641488569f + github.com/loxilb-io/loxilib v0.8.9-0.20240124072521-f37fbddfb4e8 github.com/osrg/gobgp/v3 v3.5.0 github.com/prometheus-community/pro-bing v0.1.0 github.com/prometheus/client_model v0.3.0 diff --git a/go.sum b/go.sum index 6563b726..bd5663f0 100644 --- a/go.sum +++ b/go.sum @@ -262,6 +262,8 @@ github.com/loxilb-io/ipvs v0.1.0 h1:TpTkwh5CLgJ7YW86rvWyqJPEpQFqs2TNbRG/IECeq+w= github.com/loxilb-io/ipvs v0.1.0/go.mod h1:EKjimnzyVL9AXMMNfPWeokxF1uNeuDrEGF5gPFMdmIo= github.com/loxilb-io/loxilib v0.8.9-0.20231211082246-dc641488569f h1:q745LqB2G97bOpf6MCa34VyNJEuvVx4rTFVut3kz9oo= github.com/loxilb-io/loxilib v0.8.9-0.20231211082246-dc641488569f/go.mod h1:LoQCxBz+N0fO9rGwRmPHrQPHol/jUf4MNpph63Cydkg= +github.com/loxilb-io/loxilib v0.8.9-0.20240124072521-f37fbddfb4e8 h1:PYkgIX2fv1UluRR82SAot3O5YnwwCbcWc7VULvPv3QM= +github.com/loxilb-io/loxilib v0.8.9-0.20240124072521-f37fbddfb4e8/go.mod h1:LoQCxBz+N0fO9rGwRmPHrQPHol/jUf4MNpph63Cydkg= github.com/loxilb-io/sctp v0.0.0-20230519081703-6d1baec82fd4 h1:oDc2lsbfuQEcVP3k+Pw4v6Xdm3t4M9vBc1Y9egszv6g= github.com/loxilb-io/sctp v0.0.0-20230519081703-6d1baec82fd4/go.mod h1:1a6hv8ISVQhnW5IVpW9o+OL6BAFlWiVpC0O4d19g+wQ= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=