From 0f31b5a0c46e05eb3b3dd57b923dace150045f47 Mon Sep 17 00:00:00 2001 From: Alvaro Date: Wed, 18 Sep 2019 10:57:44 +0200 Subject: [PATCH 01/31] Proposed solution for peers load balancing in Kademlia --- network/kademlia.go | 148 ++++++++++++++++++++++++++++++++++----- network/kademlia_test.go | 45 ++++++++++++ pot/pot.go | 2 +- pss/pss.go | 17 +++-- 4 files changed, 187 insertions(+), 25 deletions(-) diff --git a/network/kademlia.go b/network/kademlia.go index 6f82fbce74..21458e8fd7 100644 --- a/network/kademlia.go +++ b/network/kademlia.go @@ -21,6 +21,7 @@ import ( "encoding/hex" "errors" "fmt" + "github.com/ethereum/go-ethereum/common/hexutil" "math/rand" "sort" "strings" @@ -98,6 +99,7 @@ type Kademlia struct { nDepth int // stores the last neighbourhood depth nDepthMu sync.RWMutex // protects neighbourhood depth nDepth nDepthSig []chan struct{} // signals when neighbourhood depth nDepth is changed + searching color // Color of entries ion global index (conns) to search for } type KademliaInfo struct { @@ -125,6 +127,7 @@ func NewKademlia(addr []byte, params *KadParams) *Kademlia { capabilityIndex: make(map[string]*capabilityIndex), addrs: pot.NewPot(nil, 0), conns: pot.NewPot(nil, 0), + searching: black, } k.RegisterCapabilityIndex("full", *fullCapability) k.RegisterCapabilityIndex("light", *lightCapability) @@ -164,9 +167,9 @@ func (k *Kademlia) addToCapabilityIndex(p interface{}) { if vCap.Match(idxItem.Capability) { log.Trace("Added peer to capability index", "conn", ok, "s", s, "v", vCap, "p", p) if ok { - k.capabilityIndex[s].conns, _, _ = pot.Add(idxItem.conns, ePeer, Pof) + k.capabilityIndex[s].conns, _, _ = pot.Add(idxItem.conns, newEntryFromPeer(ePeer, idxItem.searching), Pof) } else { - k.capabilityIndex[s].addrs, _, _ = pot.Add(idxItem.addrs, newEntry(eAddr), Pof) + k.capabilityIndex[s].addrs, _, _ = pot.Add(idxItem.addrs, newEntryFromBzzAddress(eAddr, idxItem.searching), Pof) } } } @@ -188,7 +191,8 @@ func (k *Kademlia) removeFromCapabilityIndex(p interface{}, disconnectOnly bool) } for s, idxItem := range k.capabilityIndex { if ok { - conns, _, found, _ := pot.Swap(idxItem.conns, ePeer, Pof, func(_ pot.Val) pot.Val { + peerEntry := newEntryFromPeer(ePeer, idxItem.searching) + conns, _, found, _ := pot.Swap(idxItem.conns, peerEntry, Pof, func(_ pot.Val) pot.Val { return nil }) if found { @@ -208,28 +212,47 @@ func (k *Kademlia) removeFromCapabilityIndex(p interface{}, disconnectOnly bool) } } +const ( + black = iota + red +) +type color int + // entry represents a Kademlia table entry (an extension of BzzAddr) type entry struct { *BzzAddr conn *Peer seenAt time.Time retries int + use color } -// newEntry creates a kademlia peer from a *Peer -func newEntry(p *BzzAddr) *entry { +// newEntryFromBzzAddress creates a kademlia entry from a *BzzAddr +func newEntryFromBzzAddress(p *BzzAddr, initialColor color) *entry { return &entry{ BzzAddr: p, seenAt: time.Now(), + use: initialColor, + } +} + +// newEntryFromPeer creates a kademlia entry from a *Peer +func newEntryFromPeer(p *Peer, initialColor color) *entry { + return &entry{ + BzzAddr: p.BzzAddr, + conn: p, + seenAt: time.Now(), + use: initialColor, } } // index providing quick access to all peers having a certain capability set type capabilityIndex struct { *capability.Capability - conns *pot.Pot - addrs *pot.Pot - depth int + conns *pot.Pot + addrs *pot.Pot + depth int + searching color } // NewCapabilityIndex creates a new capability index with a copy the provided capabilities array @@ -251,6 +274,17 @@ func (e *entry) Hex() string { return hex.EncodeToString(e.Address()) } +func (e *entry) flipUse() { + e.use = flipColor(e.use) +} + +func flipColor(color color) color { + if color == black { + return red + } else { + return black + } +} // Register enters each address as kademlia peer record into the // database of known peer addresses func (k *Kademlia) Register(peers ...*BzzAddr) error { @@ -272,7 +306,7 @@ func (k *Kademlia) Register(peers ...*BzzAddr) error { if v == nil { log.Trace("registering new peer", "addr", p) // insert new offline peer into conns - return newEntry(p) + return newEntryFromBzzAddress(p, k.searching) } e := v.(*entry) @@ -281,12 +315,12 @@ func (k *Kademlia) Register(peers ...*BzzAddr) error { if !bytes.Equal(e.BzzAddr.UAddr, p.UAddr) { log.Trace("underlay addr is different, so add again", "new", p, "old", e.BzzAddr) // insert new offline peer into conns - return newEntry(p) + return newEntryFromBzzAddress(p, k.searching) } return v }) - k.addToCapabilityIndex(newEntry(p)) + k.addToCapabilityIndex(newEntryFromBzzAddress(p, k.searching)) size++ } @@ -419,22 +453,22 @@ func (k *Kademlia) On(p *Peer) (uint8, bool) { metrics.GetOrRegisterCounter("kad.on", nil).Inc(1) var ins bool - k.conns, _, _, _ = pot.Swap(k.conns, p, Pof, func(v pot.Val) pot.Val { + peerEntry := newEntryFromPeer(p, k.searching) + k.conns, _, _, _ = pot.Swap(k.conns, peerEntry, Pof, func(v pot.Val) pot.Val { // if not found live if v == nil { ins = true // insert new online peer into conns - return p + return peerEntry } // found among live peers, do nothing return v }) k.addToCapabilityIndex(p) if ins { - a := newEntry(p.BzzAddr) - a.conn = p + a := newEntryFromBzzAddress(p.BzzAddr, k.searching) // insert new online peer into addrs - k.addrs, _, _, _ = pot.Swap(k.addrs, p, Pof, func(v pot.Val) pot.Val { + k.addrs, _, _, _ = pot.Swap(k.addrs, a, Pof, func(v pot.Val) pot.Val { return a }) } @@ -537,7 +571,7 @@ func (k *Kademlia) Off(p *Peer) { if v == nil { panic(fmt.Sprintf("connected peer not found %v", p)) } - return newEntry(p.BzzAddr) + return newEntryFromBzzAddress(p.BzzAddr, k.searching) }) // note the following only ran if the peer was a lightnode k.conns, _, _, _ = pot.Swap(k.conns, p, Pof, func(_ pot.Val) pot.Val { @@ -561,6 +595,21 @@ func (k *Kademlia) EachConnFiltered(base []byte, capKey string, o int, f func(*P return nil } +// EachConnFiltered performs the same action as EachConn +// with the difference that it will only return peers that matches the specified capability index filter +func (k *Kademlia) EachConnFilteredLB(base []byte, capKey string, o int, f func(*Peer, int) (bool, bool)) error { + k.lock.RLock() + defer k.lock.RUnlock() + c, ok := k.capabilityIndex[capKey] + if !ok { + return fmt.Errorf("Unregistered capability index '%s'", capKey) + } + if k.eachConnLB(base, c.conns, c.searching, o, f) { + c.searching = flipColor(c.searching) + } + return nil +} + // EachConn is an iterator with args (base, po, f) applies f to each live peer // that has proximity order po or less as measured from the base // if base is nil, kademlia base address is used @@ -570,6 +619,21 @@ func (k *Kademlia) EachConn(base []byte, o int, f func(*Peer, int) bool) { k.eachConn(base, k.conns, o, f) } +// EachConnLB is an iterator with args (base, po, f) applies f to each live peer +// that has proximity order po or less as measured from the base +// if base is nil, kademlia base address is used. +// f function should return two booleans: First one indicates if it wants to continue iterating, +// second one signals if the peer has been used and a request should be accounted to that peer. +// It return first the peers not used in the last iteration (entry.color == k.searching) +func (k *Kademlia) EachConnLB(base []byte, o int, f func(*Peer, int) (bool, bool)) { + k.lock.RLock() + defer k.lock.RUnlock() + if k.eachConnLB(base, k.conns, k.searching, o, f) { + k.searching = flipColor(k.searching) + log.Debug("Flipping color of kademlia", "newColor", k.searching) + } +} + func (k *Kademlia) eachConn(base []byte, db *pot.Pot, o int, f func(*Peer, int) bool) { if len(base) == 0 { base = k.base @@ -581,10 +645,56 @@ func (k *Kademlia) eachConn(base []byte, db *pot.Pot, o int, f func(*Peer, int) if po > o { return true } - return f(val.(*Peer), po) + return f(val.(*entry).conn, po) }) } +// returns true if it has completed a iteration (so the current searching color must change) +func (k *Kademlia) eachConnLB(base []byte, db *pot.Pot, searching color, o int, f func(*Peer, int) (bool, bool)) bool{ + if len(base) == 0 { + base = k.base + } + if db == nil { + db = k.conns + } + oneUsed := false + anySkip := false + f2 := func(val pot.Val, po int) bool { + if po > o { + return true + } + + entry, _ := val.(*entry) + if entry.use != searching { + anySkip = true + log.Debug("Skipping peer because of color", "peer", hexutil.Encode(entry.BzzAddr.OAddr[:8]), "color", entry.use, "searching", searching) + // Skip it because LB + return true + } else { + continueIterating, markUsed := f(entry.conn, po) + if markUsed { + // count as used + entry.flipUse() + log.Warn("Using peer", "peer", hexutil.Encode(entry.BzzAddr.OAddr[:8]), "color", flipColor(entry.use), "newColor", entry.use) + oneUsed = true + } + return continueIterating + } + } + db.EachNeighbour(base, Pof, f2) + if anySkip && !oneUsed { + // We skip some entries because LB color but we didn't found one to use, + // so we repeat again with flipped color + searching = flipColor(searching) + db.EachNeighbour(base, Pof, f2) + return true + } else { + return false + } +} + + + // EachAddrFiltered performs the same action as EachAddr // with the difference that it will only return peers that matches the specified capability index filter func (k *Kademlia) EachAddrFiltered(base []byte, capKey string, o int, f func(*BzzAddr, int) bool) error { @@ -815,7 +925,7 @@ func (k *Kademlia) string() string { row := []string{fmt.Sprintf("%2d", size)} rest -= size bin.ValIterator(func(val pot.Val) bool { - e := val.(*Peer) + e := val.(*entry) row = append(row, hex.EncodeToString(e.Address()[:2])) rowlen++ return rowlen < 4 diff --git a/network/kademlia_test.go b/network/kademlia_test.go index c721ba450d..1260c912e3 100644 --- a/network/kademlia_test.go +++ b/network/kademlia_test.go @@ -18,6 +18,7 @@ package network import ( "fmt" + "math/rand" "os" "testing" "time" @@ -370,6 +371,50 @@ func binStr(a *BzzAddr) string { return pot.ToBin(a.Address())[:8] } +func TestEachConnLB(t *testing.T) { + tk := newTestKademlia(t, "00000000") + base := tk.base + tk.On("00100000") + tk.On("00100001") + tk.On("00010101") + stats := make(map[string]int) + f := func(peer *Peer, po int) (bool, bool) { + stats[peer.String()] = stats[peer.String()] + 1 + // return false to only use one peer and mark it as used + return false, true + } + // Let n be the number of peers (3) + // Executing 2 * n times. We expect 2 access to each peer + tk.EachConnLB(base, 255, f) + tk.EachConnLB(base, 255, f) + tk.EachConnLB(base, 255, f) + tk.EachConnLB(base, 255, f) + tk.EachConnLB(base, 255, f) + tk.EachConnLB(base, 255, f) + for addr, count := range stats { + log.Warn("Stats", "addr", addr, "count", count) + if count != 2 { + t.Errorf("Expected 2 access to peer %v but got %v", addr, count) + } + } + + stats = make(map[string]int) + // For each number k where m * n <= k < (m+1) *n, if we request k access, + // each peer should have m or m+1 access + k := int(rand.Int31n(200)) + m := k / 3 + log.Warn("k/3", "k", k, "m", m) + for i := 0 ; i < k ; i++ { + tk.EachConnLB(base, 255, f) + } + for addr, count := range stats { + log.Warn("Stats", "addr", addr, "count", count) + if count != m && count != (m+1) { + t.Errorf("Expected %v or %v access to peer %v but got %v", m, m+1, addr, count) + } + } +} + func TestSuggestPeerFindPeers(t *testing.T) { tk := newTestKademlia(t, "00000000") tk.On("00100000") diff --git a/pot/pot.go b/pot/pot.go index b3a0e03178..152c48273c 100644 --- a/pot/pot.go +++ b/pot/pot.go @@ -209,7 +209,7 @@ func remove(t *Pot, val Val, pof Pof) (r *Pot, po int, found bool) { // if f(v) returns v' <> v then v' is inserted into the Pot // if (v) == v the Pot is not changed // it panics if Pof(f(v), k) show that v' and v are not key-equal -// BUG if "default" empty pot is supplied (created with NewPot(nil, 0), quieried address NOT found, then returned pot will be a nil value +// BUG if "default" empty pot is supplied (created with NewPot(nil, 0), queried address NOT found, then returned pot will be a nil value func Swap(t *Pot, k Val, pof Pof, f func(v Val) Val) (r *Pot, po int, found bool, change bool) { var val Val if t.pin == nil { diff --git a/pss/pss.go b/pss/pss.go index 4c53389ebc..974a0728b3 100644 --- a/pss/pss.go +++ b/pss/pss.go @@ -803,21 +803,28 @@ func (p *Pss) forward(msg *message.Message) error { onlySendOnce = true } - p.Kademlia.EachConn(to, addressLength*8, func(sp *network.Peer, po int) bool { + p.Kademlia.EachConnLB(to, addressLength*8, func(sp *network.Peer, po int) (continueIt bool, peerUsed bool) { if po < broadcastThreshold && sent > 0 { - return false // stop iterating + return // stop iterating, peer not used } + peerUsed = true if sendFunc(p, sp, msg) { sent++ if onlySendOnce { - return false + continueIt = false + // stop iterating peer used + return } if po == addressLength*8 { + continueIt = false // stop iterating if successfully sent to the exact recipient (perfect match of full address) - return false + // peer used + return } } - return true + continueIt = true + // continue iterating, peer used + return }) // cache the message From c1c9ac5b50f8352f1426d50d54f45b2c0895e8b7 Mon Sep 17 00:00:00 2001 From: Alvaro Date: Thu, 19 Sep 2019 08:36:56 +0200 Subject: [PATCH 02/31] network: created global capabilityIndex --- network/kademlia.go | 143 ++++++++++++++++++++------------------ network/kademlia_test.go | 12 ++-- network/networkid_test.go | 4 +- 3 files changed, 83 insertions(+), 76 deletions(-) diff --git a/network/kademlia.go b/network/kademlia.go index 21458e8fd7..65a72339d5 100644 --- a/network/kademlia.go +++ b/network/kademlia.go @@ -91,15 +91,13 @@ func NewKadParams() *KadParams { type Kademlia struct { lock sync.RWMutex capabilityIndex map[string]*capabilityIndex + globalIndex *capabilityIndex // Index with pots and searching color *KadParams // Kademlia configuration parameters base []byte // immutable baseaddress of the table - addrs *pot.Pot // pots container for known peer addresses - conns *pot.Pot // pots container for live peer connections depth uint8 // stores the last current depth of saturation nDepth int // stores the last neighbourhood depth nDepthMu sync.RWMutex // protects neighbourhood depth nDepth nDepthSig []chan struct{} // signals when neighbourhood depth nDepth is changed - searching color // Color of entries ion global index (conns) to search for } type KademliaInfo struct { @@ -125,9 +123,7 @@ func NewKademlia(addr []byte, params *KadParams) *Kademlia { base: addr, KadParams: params, capabilityIndex: make(map[string]*capabilityIndex), - addrs: pot.NewPot(nil, 0), - conns: pot.NewPot(nil, 0), - searching: black, + globalIndex: NewGlobalIndex(), } k.RegisterCapabilityIndex("full", *fullCapability) k.RegisterCapabilityIndex("light", *lightCapability) @@ -255,12 +251,23 @@ type capabilityIndex struct { searching color } +// NewCGlobalIndex creates a new index for no capability with black starting color and nil capabilities +func NewGlobalIndex() *capabilityIndex { + return &capabilityIndex{ + Capability: nil, + conns: pot.NewPot(nil, 0), + addrs: pot.NewPot(nil, 0), + searching: black, + } +} + // NewCapabilityIndex creates a new capability index with a copy the provided capabilities array func NewCapabilityIndex(c capability.Capability) *capabilityIndex { return &capabilityIndex{ Capability: &c, conns: pot.NewPot(nil, 0), addrs: pot.NewPot(nil, 0), + searching: black, } } @@ -301,12 +308,13 @@ func (k *Kademlia) Register(peers ...*BzzAddr) error { if bytes.Equal(p.Address(), k.base) { return fmt.Errorf("add peers: %x is self", k.base) } - k.addrs, _, _, _ = pot.Swap(k.addrs, p, Pof, func(v pot.Val) pot.Val { + index := k.globalIndex + index.addrs, _, _, _ = pot.Swap(index.addrs, p, Pof, func(v pot.Val) pot.Val { // if not found if v == nil { log.Trace("registering new peer", "addr", p) - // insert new offline peer into conns - return newEntryFromBzzAddress(p, k.searching) + // insert new offline peer into addrs + return newEntryFromBzzAddress(p, index.searching) } e := v.(*entry) @@ -314,13 +322,13 @@ func (k *Kademlia) Register(peers ...*BzzAddr) error { // if underlay address is different, still add if !bytes.Equal(e.BzzAddr.UAddr, p.UAddr) { log.Trace("underlay addr is different, so add again", "new", p, "old", e.BzzAddr) - // insert new offline peer into conns - return newEntryFromBzzAddress(p, k.searching) + // insert new offline peer into addrs + return newEntryFromBzzAddress(p, index.searching) } return v }) - k.addToCapabilityIndex(newEntryFromBzzAddress(p, k.searching)) + k.addToCapabilityIndex(newEntryFromBzzAddress(p, index.searching)) size++ } @@ -335,7 +343,7 @@ func (k *Kademlia) SuggestPeer() (suggestedPeer *BzzAddr, saturationDepth int, c metrics.GetOrRegisterCounter("kad.suggestpeer", nil).Inc(1) - radius := neighbourhoodRadiusForPot(k.conns, k.NeighbourhoodSize, k.base) + radius := neighbourhoodRadiusForPot(k.globalIndex.conns, k.NeighbourhoodSize, k.base) // collect undersaturated bins in ascending order of number of connected peers // and from shallow to deep (ascending order of PO) // insert them in a map of bin arrays, keyed with the number of connected peers @@ -343,7 +351,7 @@ func (k *Kademlia) SuggestPeer() (suggestedPeer *BzzAddr, saturationDepth int, c var lastPO int // the last non-empty PO bin in the iteration saturationDepth = -1 // the deepest PO such that all shallower bins have >= k.MinBinSize peers var pastDepth bool // whether po of iteration >= depth - k.conns.EachBin(k.base, Pof, 0, func(bin *pot.Bin) bool { + k.globalIndex.conns.EachBin(k.base, Pof, 0, func(bin *pot.Bin) bool { // process skipped empty bins po := bin.ProximityOrder size := bin.Size @@ -379,7 +387,7 @@ func (k *Kademlia) SuggestPeer() (suggestedPeer *BzzAddr, saturationDepth int, c // to trigger peer requests for peers closer than closest connection, include // all bins from nearest connection upto nearest address as unsaturated var nearestAddrAt int - k.addrs.EachNeighbour(k.base, Pof, func(_ pot.Val, po int) bool { + k.globalIndex.addrs.EachNeighbour(k.base, Pof, func(_ pot.Val, po int) bool { nearestAddrAt = po return false }) @@ -403,7 +411,7 @@ func (k *Kademlia) SuggestPeer() (suggestedPeer *BzzAddr, saturationDepth int, c } cur := 0 curPO := bins[0] - k.addrs.EachBin(k.base, Pof, curPO, func(bin *pot.Bin) bool { + k.globalIndex.addrs.EachBin(k.base, Pof, curPO, func(bin *pot.Bin) bool { curPO = bins[cur] // find the next bin that has size size po := bin.ProximityOrder @@ -453,8 +461,9 @@ func (k *Kademlia) On(p *Peer) (uint8, bool) { metrics.GetOrRegisterCounter("kad.on", nil).Inc(1) var ins bool - peerEntry := newEntryFromPeer(p, k.searching) - k.conns, _, _, _ = pot.Swap(k.conns, peerEntry, Pof, func(v pot.Val) pot.Val { + index := k.globalIndex + peerEntry := newEntryFromPeer(p, index.searching) + index.conns, _, _, _ = pot.Swap(index.conns, peerEntry, Pof, func(v pot.Val) pot.Val { // if not found live if v == nil { ins = true @@ -466,9 +475,9 @@ func (k *Kademlia) On(p *Peer) (uint8, bool) { }) k.addToCapabilityIndex(p) if ins { - a := newEntryFromBzzAddress(p.BzzAddr, k.searching) + a := newEntryFromBzzAddress(p.BzzAddr, index.searching) // insert new online peer into addrs - k.addrs, _, _, _ = pot.Swap(k.addrs, a, Pof, func(v pot.Val) pot.Val { + index.addrs, _, _, _ = pot.Swap(index.addrs, a, Pof, func(v pot.Val) pot.Val { return a }) } @@ -486,7 +495,7 @@ func (k *Kademlia) On(p *Peer) (uint8, bool) { // setNeighbourhoodDepth calculates neighbourhood depth with depthForPot, // sets it to the nDepth and sends a signal to every nDepthSig channel. func (k *Kademlia) setNeighbourhoodDepth() { - nDepth := depthForPot(k.conns, k.NeighbourhoodSize, k.base) + nDepth := depthForPot(k.globalIndex.conns, k.NeighbourhoodSize, k.base) var changed bool k.nDepthMu.Lock() if nDepth != k.nDepth { @@ -566,15 +575,16 @@ func (k *Kademlia) SubscribeToNeighbourhoodDepthChange() (c <-chan struct{}, uns func (k *Kademlia) Off(p *Peer) { k.lock.Lock() defer k.lock.Unlock() - k.addrs, _, _, _ = pot.Swap(k.addrs, p, Pof, func(v pot.Val) pot.Val { + index := k.globalIndex + index.addrs, _, _, _ = pot.Swap(index.addrs, p, Pof, func(v pot.Val) pot.Val { // v cannot be nil, must check otherwise we overwrite entry if v == nil { panic(fmt.Sprintf("connected peer not found %v", p)) } - return newEntryFromBzzAddress(p.BzzAddr, k.searching) + return newEntryFromBzzAddress(p.BzzAddr, index.searching) }) // note the following only ran if the peer was a lightnode - k.conns, _, _, _ = pot.Swap(k.conns, p, Pof, func(_ pot.Val) pot.Val { + index.conns, _, _, _ = pot.Swap(index.conns, p, Pof, func(_ pot.Val) pot.Val { // v cannot be nil, but no need to check return nil }) @@ -604,9 +614,7 @@ func (k *Kademlia) EachConnFilteredLB(base []byte, capKey string, o int, f func( if !ok { return fmt.Errorf("Unregistered capability index '%s'", capKey) } - if k.eachConnLB(base, c.conns, c.searching, o, f) { - c.searching = flipColor(c.searching) - } + k.eachConnLB(base, c, o, f) return nil } @@ -616,7 +624,7 @@ func (k *Kademlia) EachConnFilteredLB(base []byte, capKey string, o int, f func( func (k *Kademlia) EachConn(base []byte, o int, f func(*Peer, int) bool) { k.lock.RLock() defer k.lock.RUnlock() - k.eachConn(base, k.conns, o, f) + k.eachConn(base, k.globalIndex.conns, o, f) } // EachConnLB is an iterator with args (base, po, f) applies f to each live peer @@ -628,10 +636,7 @@ func (k *Kademlia) EachConn(base []byte, o int, f func(*Peer, int) bool) { func (k *Kademlia) EachConnLB(base []byte, o int, f func(*Peer, int) (bool, bool)) { k.lock.RLock() defer k.lock.RUnlock() - if k.eachConnLB(base, k.conns, k.searching, o, f) { - k.searching = flipColor(k.searching) - log.Debug("Flipping color of kademlia", "newColor", k.searching) - } + k.eachConnLB(base, k.globalIndex, o, f) } func (k *Kademlia) eachConn(base []byte, db *pot.Pot, o int, f func(*Peer, int) bool) { @@ -639,7 +644,7 @@ func (k *Kademlia) eachConn(base []byte, db *pot.Pot, o int, f func(*Peer, int) base = k.base } if db == nil { - db = k.conns + db = k.globalIndex.conns } db.EachNeighbour(base, Pof, func(val pot.Val, po int) bool { if po > o { @@ -649,47 +654,47 @@ func (k *Kademlia) eachConn(base []byte, db *pot.Pot, o int, f func(*Peer, int) }) } -// returns true if it has completed a iteration (so the current searching color must change) -func (k *Kademlia) eachConnLB(base []byte, db *pot.Pot, searching color, o int, f func(*Peer, int) (bool, bool)) bool{ +// eachConn iterates over index connection Neighbourghs, skipping most recent used peers to balance requests among peers +// The function f provided by the caller should return whether to obtain more peers and if the peer has been used or not +// in order to change the color +func (k *Kademlia) eachConnLB(base []byte, index *capabilityIndex, o int, f func(*Peer, int) (bool, bool)) { + db := index.conns if len(base) == 0 { base = k.base } - if db == nil { - db = k.conns - } - oneUsed := false + oneFound := false anySkip := false + lastContinue := true f2 := func(val pot.Val, po int) bool { if po > o { return true } entry, _ := val.(*entry) - if entry.use != searching { + if entry.use != index.searching { anySkip = true - log.Debug("Skipping peer because of color", "peer", hexutil.Encode(entry.BzzAddr.OAddr[:8]), "color", entry.use, "searching", searching) + log.Debug("Skipping peer because of color", "peer", hexutil.Encode(entry.BzzAddr.OAddr[:8]), "color", entry.use, "searching", index.searching) // Skip it because LB return true } else { + // Regardless of the user using or not this peer we set oneFound to true + oneFound = true continueIterating, markUsed := f(entry.conn, po) if markUsed { // count as used entry.flipUse() log.Warn("Using peer", "peer", hexutil.Encode(entry.BzzAddr.OAddr[:8]), "color", flipColor(entry.use), "newColor", entry.use) - oneUsed = true } + lastContinue = continueIterating return continueIterating } } db.EachNeighbour(base, Pof, f2) - if anySkip && !oneUsed { - // We skip some entries because LB color but we didn't found one to use, - // so we repeat again with flipped color - searching = flipColor(searching) + if anySkip && !oneFound && lastContinue { + // In this situation we had a complete iteration of the table skipping all peers. We should flip searching color + // and give another lap + index.searching = flipColor(index.searching) db.EachNeighbour(base, Pof, f2) - return true - } else { - return false } } @@ -715,7 +720,7 @@ func (k *Kademlia) EachAddrFiltered(base []byte, capKey string, o int, f func(*B func (k *Kademlia) EachAddr(base []byte, o int, f func(*BzzAddr, int) bool) { k.lock.RLock() defer k.lock.RUnlock() - k.eachAddr(base, k.addrs, o, f) + k.eachAddr(base, k.globalIndex.addrs, o, f) } func (k *Kademlia) eachAddr(base []byte, db *pot.Pot, o int, f func(*BzzAddr, int) bool) { @@ -723,7 +728,7 @@ func (k *Kademlia) eachAddr(base []byte, db *pot.Pot, o int, f func(*BzzAddr, in base = k.base } if db == nil { - db = k.addrs + db = k.globalIndex.addrs } db.EachNeighbour(base, Pof, func(val pot.Val, po int) bool { if po > o { @@ -844,14 +849,14 @@ func (k *Kademlia) KademliaInfo() KademliaInfo { } func (k *Kademlia) kademliaInfo() (ki KademliaInfo) { - ki.Self = hex.EncodeToString(k.BaseAddr()) - ki.Depth = depthForPot(k.conns, k.NeighbourhoodSize, k.base) - ki.TotalConnections = k.conns.Size() - ki.TotalKnown = k.addrs.Size() + ki.Self = hex.EncodeToString(k.BaseAddr()) + ki.Depth = depthForPot(k.globalIndex.conns, k.NeighbourhoodSize, k.base) + ki.TotalConnections = k.globalIndex.conns.Size() + ki.TotalKnown = k.globalIndex.addrs.Size() ki.Connections = make([][]string, k.MaxProxDisplay) ki.Known = make([][]string, k.MaxProxDisplay) - k.conns.EachBin(k.base, Pof, 0, func(bin *pot.Bin) bool { + k.globalIndex.conns.EachBin(k.base, Pof, 0, func(bin *pot.Bin) bool { po := bin.ProximityOrder if po >= k.MaxProxDisplay { po = k.MaxProxDisplay - 1 @@ -869,7 +874,7 @@ func (k *Kademlia) kademliaInfo() (ki KademliaInfo) { return true }) - k.addrs.EachBin(k.base, Pof, 0, func(bin *pot.Bin) bool { + k.globalIndex.addrs.EachBin(k.base, Pof, 0, func(bin *pot.Bin) bool { po := bin.ProximityOrder if po >= k.MaxProxDisplay { po = k.MaxProxDisplay - 1 @@ -908,14 +913,14 @@ func (k *Kademlia) string() string { rows = append(rows, fmt.Sprintf("commit hash: %s", sv.GitCommit)) } rows = append(rows, fmt.Sprintf("%v KΛÐΞMLIΛ hive: queen's address: %x", time.Now().UTC().Format(time.UnixDate), k.BaseAddr())) - rows = append(rows, fmt.Sprintf("population: %d (%d), NeighbourhoodSize: %d, MinBinSize: %d, MaxBinSize: %d", k.conns.Size(), k.addrs.Size(), k.NeighbourhoodSize, k.MinBinSize, k.MaxBinSize)) + rows = append(rows, fmt.Sprintf("population: %d (%d), NeighbourhoodSize: %d, MinBinSize: %d, MaxBinSize: %d", k.globalIndex.conns.Size(), k.globalIndex.addrs.Size(), k.NeighbourhoodSize, k.MinBinSize, k.MaxBinSize)) liverows := make([]string, k.MaxProxDisplay) peersrows := make([]string, k.MaxProxDisplay) - depth := depthForPot(k.conns, k.NeighbourhoodSize, k.base) - rest := k.conns.Size() - k.conns.EachBin(k.base, Pof, 0, func(bin *pot.Bin) bool { + depth := depthForPot(k.globalIndex.conns, k.NeighbourhoodSize, k.base) + rest := k.globalIndex.conns.Size() + k.globalIndex.conns.EachBin(k.base, Pof, 0, func(bin *pot.Bin) bool { var rowlen int po := bin.ProximityOrder if po >= k.MaxProxDisplay { @@ -936,7 +941,7 @@ func (k *Kademlia) string() string { return true }) - k.addrs.EachBin(k.base, Pof, 0, func(bin *pot.Bin) bool { + k.globalIndex.addrs.EachBin(k.base, Pof, 0, func(bin *pot.Bin) bool { var rowlen int po := bin.ProximityOrder if po >= k.MaxProxDisplay { @@ -1049,8 +1054,8 @@ func (k *Kademlia) Saturation() int { func (k *Kademlia) saturation() int { prev := -1 - radius := neighbourhoodRadiusForPot(k.conns, k.NeighbourhoodSize, k.base) - k.conns.EachBin(k.base, Pof, 0, func(bin *pot.Bin) bool { + radius := neighbourhoodRadiusForPot(k.globalIndex.conns, k.NeighbourhoodSize, k.base) + k.globalIndex.conns.EachBin(k.base, Pof, 0, func(bin *pot.Bin) bool { prev++ po := bin.ProximityOrder if po >= radius { @@ -1079,7 +1084,7 @@ func (k *Kademlia) isSaturated(peersPerBin []int, depth int) bool { return false } unsaturatedBins := make([]int, 0) - k.conns.EachBin(k.base, Pof, 0, func(bin *pot.Bin) bool { + k.globalIndex.conns.EachBin(k.base, Pof, 0, func(bin *pot.Bin) bool { po := bin.ProximityOrder if po >= depth { return false @@ -1104,9 +1109,9 @@ func (k *Kademlia) isSaturated(peersPerBin []int, depth int) bool { // TODO move to separate testing tools file func (k *Kademlia) knowNeighbours(addrs [][]byte) (got bool, n int, missing [][]byte) { pm := make(map[string]bool) - depth := depthForPot(k.conns, k.NeighbourhoodSize, k.base) + depth := depthForPot(k.globalIndex.conns, k.NeighbourhoodSize, k.base) // create a map with all peers at depth and deeper known in the kademlia - k.eachAddr(nil, k.addrs, 255, func(p *BzzAddr, po int) bool { + k.eachAddr(nil, k.globalIndex.addrs, 255, func(p *BzzAddr, po int) bool { // in order deepest to shallowest compared to the kademlia base address // all bins (except self) are included (0 <= bin <= 255) if po < depth { @@ -1144,7 +1149,7 @@ func (k *Kademlia) connectedNeighbours(peers [][]byte) (got bool, n int, missing // create a map with all peers at depth and deeper that are connected in the kademlia // in order deepest to shallowest compared to the kademlia base address // all bins (except self) are included (0 <= bin <= 255) - depth := depthForPot(k.conns, k.NeighbourhoodSize, k.base) + depth := depthForPot(k.globalIndex.conns, k.NeighbourhoodSize, k.base) k.eachConn(nil, nil, 255, func(p *Peer, po int) bool { if po < depth { return false @@ -1202,7 +1207,7 @@ func (k *Kademlia) GetHealthInfo(pp *PeerPot) *Health { } gotnn, countgotnn, culpritsgotnn := k.connectedNeighbours(pp.NNSet) knownn, countknownn, culpritsknownn := k.knowNeighbours(pp.NNSet) - depth := depthForPot(k.conns, k.NeighbourhoodSize, k.base) + depth := depthForPot(k.globalIndex.conns, k.NeighbourhoodSize, k.base) // check saturation saturated := k.isSaturated(pp.PeersPerBin, depth) diff --git a/network/kademlia_test.go b/network/kademlia_test.go index 1260c912e3..55d4a4aa62 100644 --- a/network/kademlia_test.go +++ b/network/kademlia_test.go @@ -18,6 +18,7 @@ package network import ( "fmt" + "github.com/ethereum/go-ethereum/common/hexutil" "math/rand" "os" "testing" @@ -379,7 +380,8 @@ func TestEachConnLB(t *testing.T) { tk.On("00010101") stats := make(map[string]int) f := func(peer *Peer, po int) (bool, bool) { - stats[peer.String()] = stats[peer.String()] + 1 + key := hexutil.Encode(peer.Address()[:8]) + stats[key] = stats[key] + 1 // return false to only use one peer and mark it as used return false, true } @@ -499,21 +501,21 @@ func TestOffEffectingAddressBookNormalNode(t *testing.T) { // peer added to kademlia tk.On("01000000") // peer should be in the address book - if tk.addrs.Size() != 1 { + if tk.globalIndex.addrs.Size() != 1 { t.Fatal("known peer addresses should contain 1 entry") } // peer should be among live connections - if tk.conns.Size() != 1 { + if tk.globalIndex.conns.Size() != 1 { t.Fatal("live peers should contain 1 entry") } // remove peer from kademlia tk.Off("01000000") // peer should be in the address book - if tk.addrs.Size() != 1 { + if tk.globalIndex.addrs.Size() != 1 { t.Fatal("known peer addresses should contain 1 entry") } // peer should not be among live connections - if tk.conns.Size() != 0 { + if tk.globalIndex.conns.Size() != 0 { t.Fatal("live peers should contain 0 entry") } } diff --git a/network/networkid_test.go b/network/networkid_test.go index 614092f634..13075ad6aa 100644 --- a/network/networkid_test.go +++ b/network/networkid_test.go @@ -88,8 +88,8 @@ func TestNetworkID(t *testing.T) { //...check that their size of the kademlia is of the expected size //the assumption is that it should be the size of the group minus 1 (the node itself) for _, node := range netIDGroup { - if kademlias[node].addrs.Size() != len(netIDGroup)-1 { - t.Fatalf("Kademlia size has not expected peer size. Kademlia size: %d, expected size: %d", kademlias[node].addrs.Size(), len(netIDGroup)-1) + if kademlias[node].globalIndex.addrs.Size() != len(netIDGroup)-1 { + t.Fatalf("Kademlia size has not expected peer size. Kademlia size: %d, expected size: %d", kademlias[node].globalIndex.addrs.Size(), len(netIDGroup)-1) } kademlias[node].EachAddr(nil, 0, func(addr *BzzAddr, _ int) bool { found := false From fc65d5a569b0c15b14b5258517399652325e5c08 Mon Sep 17 00:00:00 2001 From: Alvaro Date: Thu, 19 Sep 2019 09:01:47 +0200 Subject: [PATCH 03/31] typo --- network/kademlia.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/kademlia.go b/network/kademlia.go index 65a72339d5..6a49fc2a90 100644 --- a/network/kademlia.go +++ b/network/kademlia.go @@ -251,7 +251,7 @@ type capabilityIndex struct { searching color } -// NewCGlobalIndex creates a new index for no capability with black starting color and nil capabilities +// NewGlobalIndex creates a new index for no capability with black starting color and nil capabilities func NewGlobalIndex() *capabilityIndex { return &capabilityIndex{ Capability: nil, From 7dc95689f496595b5e382419b60fe62c376bc17c Mon Sep 17 00:00:00 2001 From: Alvaro Date: Thu, 19 Sep 2019 09:34:15 +0200 Subject: [PATCH 04/31] renamed globalIndex to defaultIndex --- network/kademlia.go | 99 ++++++++++++++++++++------------------- network/kademlia_test.go | 18 +++---- network/networkid_test.go | 4 +- 3 files changed, 61 insertions(+), 60 deletions(-) diff --git a/network/kademlia.go b/network/kademlia.go index 6a49fc2a90..66183a71ef 100644 --- a/network/kademlia.go +++ b/network/kademlia.go @@ -21,7 +21,7 @@ import ( "encoding/hex" "errors" "fmt" - "github.com/ethereum/go-ethereum/common/hexutil" + "math/rand" "sort" "strings" @@ -29,6 +29,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/metrics" "github.com/ethersphere/swarm/log" "github.com/ethersphere/swarm/network/capability" @@ -91,13 +92,13 @@ func NewKadParams() *KadParams { type Kademlia struct { lock sync.RWMutex capabilityIndex map[string]*capabilityIndex - globalIndex *capabilityIndex // Index with pots and searching color - *KadParams // Kademlia configuration parameters - base []byte // immutable baseaddress of the table - depth uint8 // stores the last current depth of saturation - nDepth int // stores the last neighbourhood depth - nDepthMu sync.RWMutex // protects neighbourhood depth nDepth - nDepthSig []chan struct{} // signals when neighbourhood depth nDepth is changed + defaultIndex *capabilityIndex // Index with pots and searching color + *KadParams // Kademlia configuration parameters + base []byte // immutable baseaddress of the table + depth uint8 // stores the last current depth of saturation + nDepth int // stores the last neighbourhood depth + nDepthMu sync.RWMutex // protects neighbourhood depth nDepth + nDepthSig []chan struct{} // signals when neighbourhood depth nDepth is changed } type KademliaInfo struct { @@ -123,7 +124,7 @@ func NewKademlia(addr []byte, params *KadParams) *Kademlia { base: addr, KadParams: params, capabilityIndex: make(map[string]*capabilityIndex), - globalIndex: NewGlobalIndex(), + defaultIndex: NewDefaultIndex(), } k.RegisterCapabilityIndex("full", *fullCapability) k.RegisterCapabilityIndex("light", *lightCapability) @@ -212,6 +213,7 @@ const ( black = iota red ) + type color int // entry represents a Kademlia table entry (an extension of BzzAddr) @@ -220,7 +222,7 @@ type entry struct { conn *Peer seenAt time.Time retries int - use color + use color } // newEntryFromBzzAddress creates a kademlia entry from a *BzzAddr @@ -228,7 +230,7 @@ func newEntryFromBzzAddress(p *BzzAddr, initialColor color) *entry { return &entry{ BzzAddr: p, seenAt: time.Now(), - use: initialColor, + use: initialColor, } } @@ -236,9 +238,9 @@ func newEntryFromBzzAddress(p *BzzAddr, initialColor color) *entry { func newEntryFromPeer(p *Peer, initialColor color) *entry { return &entry{ BzzAddr: p.BzzAddr, - conn: p, + conn: p, seenAt: time.Now(), - use: initialColor, + use: initialColor, } } @@ -251,8 +253,8 @@ type capabilityIndex struct { searching color } -// NewGlobalIndex creates a new index for no capability with black starting color and nil capabilities -func NewGlobalIndex() *capabilityIndex { +// NewDefaultIndex creates a new index for no capability with black starting color and nil capabilities +func NewDefaultIndex() *capabilityIndex { return &capabilityIndex{ Capability: nil, conns: pot.NewPot(nil, 0), @@ -292,6 +294,7 @@ func flipColor(color color) color { return black } } + // Register enters each address as kademlia peer record into the // database of known peer addresses func (k *Kademlia) Register(peers ...*BzzAddr) error { @@ -308,7 +311,7 @@ func (k *Kademlia) Register(peers ...*BzzAddr) error { if bytes.Equal(p.Address(), k.base) { return fmt.Errorf("add peers: %x is self", k.base) } - index := k.globalIndex + index := k.defaultIndex index.addrs, _, _, _ = pot.Swap(index.addrs, p, Pof, func(v pot.Val) pot.Val { // if not found if v == nil { @@ -343,7 +346,7 @@ func (k *Kademlia) SuggestPeer() (suggestedPeer *BzzAddr, saturationDepth int, c metrics.GetOrRegisterCounter("kad.suggestpeer", nil).Inc(1) - radius := neighbourhoodRadiusForPot(k.globalIndex.conns, k.NeighbourhoodSize, k.base) + radius := neighbourhoodRadiusForPot(k.defaultIndex.conns, k.NeighbourhoodSize, k.base) // collect undersaturated bins in ascending order of number of connected peers // and from shallow to deep (ascending order of PO) // insert them in a map of bin arrays, keyed with the number of connected peers @@ -351,7 +354,7 @@ func (k *Kademlia) SuggestPeer() (suggestedPeer *BzzAddr, saturationDepth int, c var lastPO int // the last non-empty PO bin in the iteration saturationDepth = -1 // the deepest PO such that all shallower bins have >= k.MinBinSize peers var pastDepth bool // whether po of iteration >= depth - k.globalIndex.conns.EachBin(k.base, Pof, 0, func(bin *pot.Bin) bool { + k.defaultIndex.conns.EachBin(k.base, Pof, 0, func(bin *pot.Bin) bool { // process skipped empty bins po := bin.ProximityOrder size := bin.Size @@ -387,7 +390,7 @@ func (k *Kademlia) SuggestPeer() (suggestedPeer *BzzAddr, saturationDepth int, c // to trigger peer requests for peers closer than closest connection, include // all bins from nearest connection upto nearest address as unsaturated var nearestAddrAt int - k.globalIndex.addrs.EachNeighbour(k.base, Pof, func(_ pot.Val, po int) bool { + k.defaultIndex.addrs.EachNeighbour(k.base, Pof, func(_ pot.Val, po int) bool { nearestAddrAt = po return false }) @@ -411,7 +414,7 @@ func (k *Kademlia) SuggestPeer() (suggestedPeer *BzzAddr, saturationDepth int, c } cur := 0 curPO := bins[0] - k.globalIndex.addrs.EachBin(k.base, Pof, curPO, func(bin *pot.Bin) bool { + k.defaultIndex.addrs.EachBin(k.base, Pof, curPO, func(bin *pot.Bin) bool { curPO = bins[cur] // find the next bin that has size size po := bin.ProximityOrder @@ -461,7 +464,7 @@ func (k *Kademlia) On(p *Peer) (uint8, bool) { metrics.GetOrRegisterCounter("kad.on", nil).Inc(1) var ins bool - index := k.globalIndex + index := k.defaultIndex peerEntry := newEntryFromPeer(p, index.searching) index.conns, _, _, _ = pot.Swap(index.conns, peerEntry, Pof, func(v pot.Val) pot.Val { // if not found live @@ -495,7 +498,7 @@ func (k *Kademlia) On(p *Peer) (uint8, bool) { // setNeighbourhoodDepth calculates neighbourhood depth with depthForPot, // sets it to the nDepth and sends a signal to every nDepthSig channel. func (k *Kademlia) setNeighbourhoodDepth() { - nDepth := depthForPot(k.globalIndex.conns, k.NeighbourhoodSize, k.base) + nDepth := depthForPot(k.defaultIndex.conns, k.NeighbourhoodSize, k.base) var changed bool k.nDepthMu.Lock() if nDepth != k.nDepth { @@ -575,7 +578,7 @@ func (k *Kademlia) SubscribeToNeighbourhoodDepthChange() (c <-chan struct{}, uns func (k *Kademlia) Off(p *Peer) { k.lock.Lock() defer k.lock.Unlock() - index := k.globalIndex + index := k.defaultIndex index.addrs, _, _, _ = pot.Swap(index.addrs, p, Pof, func(v pot.Val) pot.Val { // v cannot be nil, must check otherwise we overwrite entry if v == nil { @@ -624,7 +627,7 @@ func (k *Kademlia) EachConnFilteredLB(base []byte, capKey string, o int, f func( func (k *Kademlia) EachConn(base []byte, o int, f func(*Peer, int) bool) { k.lock.RLock() defer k.lock.RUnlock() - k.eachConn(base, k.globalIndex.conns, o, f) + k.eachConn(base, k.defaultIndex.conns, o, f) } // EachConnLB is an iterator with args (base, po, f) applies f to each live peer @@ -636,7 +639,7 @@ func (k *Kademlia) EachConn(base []byte, o int, f func(*Peer, int) bool) { func (k *Kademlia) EachConnLB(base []byte, o int, f func(*Peer, int) (bool, bool)) { k.lock.RLock() defer k.lock.RUnlock() - k.eachConnLB(base, k.globalIndex, o, f) + k.eachConnLB(base, k.defaultIndex, o, f) } func (k *Kademlia) eachConn(base []byte, db *pot.Pot, o int, f func(*Peer, int) bool) { @@ -644,7 +647,7 @@ func (k *Kademlia) eachConn(base []byte, db *pot.Pot, o int, f func(*Peer, int) base = k.base } if db == nil { - db = k.globalIndex.conns + db = k.defaultIndex.conns } db.EachNeighbour(base, Pof, func(val pot.Val, po int) bool { if po > o { @@ -654,7 +657,7 @@ func (k *Kademlia) eachConn(base []byte, db *pot.Pot, o int, f func(*Peer, int) }) } -// eachConn iterates over index connection Neighbourghs, skipping most recent used peers to balance requests among peers +// eachConn iterates over index connection Neighbours, skipping most recent used peers to balance requests among peers // The function f provided by the caller should return whether to obtain more peers and if the peer has been used or not // in order to change the color func (k *Kademlia) eachConnLB(base []byte, index *capabilityIndex, o int, f func(*Peer, int) (bool, bool)) { @@ -698,8 +701,6 @@ func (k *Kademlia) eachConnLB(base []byte, index *capabilityIndex, o int, f func } } - - // EachAddrFiltered performs the same action as EachAddr // with the difference that it will only return peers that matches the specified capability index filter func (k *Kademlia) EachAddrFiltered(base []byte, capKey string, o int, f func(*BzzAddr, int) bool) error { @@ -720,7 +721,7 @@ func (k *Kademlia) EachAddrFiltered(base []byte, capKey string, o int, f func(*B func (k *Kademlia) EachAddr(base []byte, o int, f func(*BzzAddr, int) bool) { k.lock.RLock() defer k.lock.RUnlock() - k.eachAddr(base, k.globalIndex.addrs, o, f) + k.eachAddr(base, k.defaultIndex.addrs, o, f) } func (k *Kademlia) eachAddr(base []byte, db *pot.Pot, o int, f func(*BzzAddr, int) bool) { @@ -728,7 +729,7 @@ func (k *Kademlia) eachAddr(base []byte, db *pot.Pot, o int, f func(*BzzAddr, in base = k.base } if db == nil { - db = k.globalIndex.addrs + db = k.defaultIndex.addrs } db.EachNeighbour(base, Pof, func(val pot.Val, po int) bool { if po > o { @@ -849,14 +850,14 @@ func (k *Kademlia) KademliaInfo() KademliaInfo { } func (k *Kademlia) kademliaInfo() (ki KademliaInfo) { - ki.Self = hex.EncodeToString(k.BaseAddr()) - ki.Depth = depthForPot(k.globalIndex.conns, k.NeighbourhoodSize, k.base) - ki.TotalConnections = k.globalIndex.conns.Size() - ki.TotalKnown = k.globalIndex.addrs.Size() + ki.Self = hex.EncodeToString(k.BaseAddr()) + ki.Depth = depthForPot(k.defaultIndex.conns, k.NeighbourhoodSize, k.base) + ki.TotalConnections = k.defaultIndex.conns.Size() + ki.TotalKnown = k.defaultIndex.addrs.Size() ki.Connections = make([][]string, k.MaxProxDisplay) ki.Known = make([][]string, k.MaxProxDisplay) - k.globalIndex.conns.EachBin(k.base, Pof, 0, func(bin *pot.Bin) bool { + k.defaultIndex.conns.EachBin(k.base, Pof, 0, func(bin *pot.Bin) bool { po := bin.ProximityOrder if po >= k.MaxProxDisplay { po = k.MaxProxDisplay - 1 @@ -874,7 +875,7 @@ func (k *Kademlia) kademliaInfo() (ki KademliaInfo) { return true }) - k.globalIndex.addrs.EachBin(k.base, Pof, 0, func(bin *pot.Bin) bool { + k.defaultIndex.addrs.EachBin(k.base, Pof, 0, func(bin *pot.Bin) bool { po := bin.ProximityOrder if po >= k.MaxProxDisplay { po = k.MaxProxDisplay - 1 @@ -913,14 +914,14 @@ func (k *Kademlia) string() string { rows = append(rows, fmt.Sprintf("commit hash: %s", sv.GitCommit)) } rows = append(rows, fmt.Sprintf("%v KΛÐΞMLIΛ hive: queen's address: %x", time.Now().UTC().Format(time.UnixDate), k.BaseAddr())) - rows = append(rows, fmt.Sprintf("population: %d (%d), NeighbourhoodSize: %d, MinBinSize: %d, MaxBinSize: %d", k.globalIndex.conns.Size(), k.globalIndex.addrs.Size(), k.NeighbourhoodSize, k.MinBinSize, k.MaxBinSize)) + rows = append(rows, fmt.Sprintf("population: %d (%d), NeighbourhoodSize: %d, MinBinSize: %d, MaxBinSize: %d", k.defaultIndex.conns.Size(), k.defaultIndex.addrs.Size(), k.NeighbourhoodSize, k.MinBinSize, k.MaxBinSize)) liverows := make([]string, k.MaxProxDisplay) peersrows := make([]string, k.MaxProxDisplay) - depth := depthForPot(k.globalIndex.conns, k.NeighbourhoodSize, k.base) - rest := k.globalIndex.conns.Size() - k.globalIndex.conns.EachBin(k.base, Pof, 0, func(bin *pot.Bin) bool { + depth := depthForPot(k.defaultIndex.conns, k.NeighbourhoodSize, k.base) + rest := k.defaultIndex.conns.Size() + k.defaultIndex.conns.EachBin(k.base, Pof, 0, func(bin *pot.Bin) bool { var rowlen int po := bin.ProximityOrder if po >= k.MaxProxDisplay { @@ -941,7 +942,7 @@ func (k *Kademlia) string() string { return true }) - k.globalIndex.addrs.EachBin(k.base, Pof, 0, func(bin *pot.Bin) bool { + k.defaultIndex.addrs.EachBin(k.base, Pof, 0, func(bin *pot.Bin) bool { var rowlen int po := bin.ProximityOrder if po >= k.MaxProxDisplay { @@ -1054,8 +1055,8 @@ func (k *Kademlia) Saturation() int { func (k *Kademlia) saturation() int { prev := -1 - radius := neighbourhoodRadiusForPot(k.globalIndex.conns, k.NeighbourhoodSize, k.base) - k.globalIndex.conns.EachBin(k.base, Pof, 0, func(bin *pot.Bin) bool { + radius := neighbourhoodRadiusForPot(k.defaultIndex.conns, k.NeighbourhoodSize, k.base) + k.defaultIndex.conns.EachBin(k.base, Pof, 0, func(bin *pot.Bin) bool { prev++ po := bin.ProximityOrder if po >= radius { @@ -1084,7 +1085,7 @@ func (k *Kademlia) isSaturated(peersPerBin []int, depth int) bool { return false } unsaturatedBins := make([]int, 0) - k.globalIndex.conns.EachBin(k.base, Pof, 0, func(bin *pot.Bin) bool { + k.defaultIndex.conns.EachBin(k.base, Pof, 0, func(bin *pot.Bin) bool { po := bin.ProximityOrder if po >= depth { return false @@ -1109,9 +1110,9 @@ func (k *Kademlia) isSaturated(peersPerBin []int, depth int) bool { // TODO move to separate testing tools file func (k *Kademlia) knowNeighbours(addrs [][]byte) (got bool, n int, missing [][]byte) { pm := make(map[string]bool) - depth := depthForPot(k.globalIndex.conns, k.NeighbourhoodSize, k.base) + depth := depthForPot(k.defaultIndex.conns, k.NeighbourhoodSize, k.base) // create a map with all peers at depth and deeper known in the kademlia - k.eachAddr(nil, k.globalIndex.addrs, 255, func(p *BzzAddr, po int) bool { + k.eachAddr(nil, k.defaultIndex.addrs, 255, func(p *BzzAddr, po int) bool { // in order deepest to shallowest compared to the kademlia base address // all bins (except self) are included (0 <= bin <= 255) if po < depth { @@ -1149,7 +1150,7 @@ func (k *Kademlia) connectedNeighbours(peers [][]byte) (got bool, n int, missing // create a map with all peers at depth and deeper that are connected in the kademlia // in order deepest to shallowest compared to the kademlia base address // all bins (except self) are included (0 <= bin <= 255) - depth := depthForPot(k.globalIndex.conns, k.NeighbourhoodSize, k.base) + depth := depthForPot(k.defaultIndex.conns, k.NeighbourhoodSize, k.base) k.eachConn(nil, nil, 255, func(p *Peer, po int) bool { if po < depth { return false @@ -1207,7 +1208,7 @@ func (k *Kademlia) GetHealthInfo(pp *PeerPot) *Health { } gotnn, countgotnn, culpritsgotnn := k.connectedNeighbours(pp.NNSet) knownn, countknownn, culpritsknownn := k.knowNeighbours(pp.NNSet) - depth := depthForPot(k.globalIndex.conns, k.NeighbourhoodSize, k.base) + depth := depthForPot(k.defaultIndex.conns, k.NeighbourhoodSize, k.base) // check saturation saturated := k.isSaturated(pp.PeersPerBin, depth) diff --git a/network/kademlia_test.go b/network/kademlia_test.go index 55d4a4aa62..54e5a5ba65 100644 --- a/network/kademlia_test.go +++ b/network/kademlia_test.go @@ -18,13 +18,13 @@ package network import ( "fmt" - "github.com/ethereum/go-ethereum/common/hexutil" "math/rand" "os" "testing" "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" @@ -381,7 +381,7 @@ func TestEachConnLB(t *testing.T) { stats := make(map[string]int) f := func(peer *Peer, po int) (bool, bool) { key := hexutil.Encode(peer.Address()[:8]) - stats[key] = stats[key] + 1 + stats[key] = stats[key] + 1 // return false to only use one peer and mark it as used return false, true } @@ -395,7 +395,7 @@ func TestEachConnLB(t *testing.T) { tk.EachConnLB(base, 255, f) for addr, count := range stats { log.Warn("Stats", "addr", addr, "count", count) - if count != 2 { + if count != 2 { t.Errorf("Expected 2 access to peer %v but got %v", addr, count) } } @@ -406,12 +406,12 @@ func TestEachConnLB(t *testing.T) { k := int(rand.Int31n(200)) m := k / 3 log.Warn("k/3", "k", k, "m", m) - for i := 0 ; i < k ; i++ { + for i := 0; i < k; i++ { tk.EachConnLB(base, 255, f) } for addr, count := range stats { log.Warn("Stats", "addr", addr, "count", count) - if count != m && count != (m+1) { + if count != m && count != (m+1) { t.Errorf("Expected %v or %v access to peer %v but got %v", m, m+1, addr, count) } } @@ -501,21 +501,21 @@ func TestOffEffectingAddressBookNormalNode(t *testing.T) { // peer added to kademlia tk.On("01000000") // peer should be in the address book - if tk.globalIndex.addrs.Size() != 1 { + if tk.defaultIndex.addrs.Size() != 1 { t.Fatal("known peer addresses should contain 1 entry") } // peer should be among live connections - if tk.globalIndex.conns.Size() != 1 { + if tk.defaultIndex.conns.Size() != 1 { t.Fatal("live peers should contain 1 entry") } // remove peer from kademlia tk.Off("01000000") // peer should be in the address book - if tk.globalIndex.addrs.Size() != 1 { + if tk.defaultIndex.addrs.Size() != 1 { t.Fatal("known peer addresses should contain 1 entry") } // peer should not be among live connections - if tk.globalIndex.conns.Size() != 0 { + if tk.defaultIndex.conns.Size() != 0 { t.Fatal("live peers should contain 0 entry") } } diff --git a/network/networkid_test.go b/network/networkid_test.go index 13075ad6aa..09359788ce 100644 --- a/network/networkid_test.go +++ b/network/networkid_test.go @@ -88,8 +88,8 @@ func TestNetworkID(t *testing.T) { //...check that their size of the kademlia is of the expected size //the assumption is that it should be the size of the group minus 1 (the node itself) for _, node := range netIDGroup { - if kademlias[node].globalIndex.addrs.Size() != len(netIDGroup)-1 { - t.Fatalf("Kademlia size has not expected peer size. Kademlia size: %d, expected size: %d", kademlias[node].globalIndex.addrs.Size(), len(netIDGroup)-1) + if kademlias[node].defaultIndex.addrs.Size() != len(netIDGroup)-1 { + t.Fatalf("Kademlia size has not expected peer size. Kademlia size: %d, expected size: %d", kademlias[node].defaultIndex.addrs.Size(), len(netIDGroup)-1) } kademlias[node].EachAddr(nil, 0, func(addr *BzzAddr, _ int) bool { found := false From ca11c5288aadfc908638ccc5902df2b35adcca90 Mon Sep 17 00:00:00 2001 From: Alvaro Date: Fri, 20 Sep 2019 08:47:48 +0200 Subject: [PATCH 05/31] Load balancing capability test --- network/kademlia.go | 2 +- network/kademlia_test.go | 68 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 64 insertions(+), 6 deletions(-) diff --git a/network/kademlia.go b/network/kademlia.go index 66183a71ef..dc9666951e 100644 --- a/network/kademlia.go +++ b/network/kademlia.go @@ -686,7 +686,7 @@ func (k *Kademlia) eachConnLB(base []byte, index *capabilityIndex, o int, f func if markUsed { // count as used entry.flipUse() - log.Warn("Using peer", "peer", hexutil.Encode(entry.BzzAddr.OAddr[:8]), "color", flipColor(entry.use), "newColor", entry.use) + log.Debug("Using peer", "peer", hexutil.Encode(entry.BzzAddr.OAddr[:8]), "color", flipColor(entry.use), "newColor", entry.use) } lastContinue = continueIterating return continueIterating diff --git a/network/kademlia_test.go b/network/kademlia_test.go index 54e5a5ba65..97bf4791cd 100644 --- a/network/kademlia_test.go +++ b/network/kademlia_test.go @@ -18,7 +18,6 @@ package network import ( "fmt" - "math/rand" "os" "testing" "time" @@ -67,6 +66,12 @@ func (tk *testKademlia) newTestKadPeer(s string) *Peer { return NewPeer(&BzzPeer{BzzAddr: testKadPeerAddr(s)}, tk.Kademlia) } +func (tk *testKademlia) newTestKadPeerWithCapabilities(s string, cap *capability.Capability) *Peer { + addr := testKadPeerAddr(s) + addr.Capabilities.Add(cap) + return NewPeer(&BzzPeer{BzzAddr: addr}, tk.Kademlia) +} + func (tk *testKademlia) On(ons ...string) { for _, s := range ons { tk.Kademlia.On(tk.newTestKadPeer(s)) @@ -394,7 +399,7 @@ func TestEachConnLB(t *testing.T) { tk.EachConnLB(base, 255, f) tk.EachConnLB(base, 255, f) for addr, count := range stats { - log.Warn("Stats", "addr", addr, "count", count) + log.Debug("Stats", "addr", addr, "count", count) if count != 2 { t.Errorf("Expected 2 access to peer %v but got %v", addr, count) } @@ -403,20 +408,73 @@ func TestEachConnLB(t *testing.T) { stats = make(map[string]int) // For each number k where m * n <= k < (m+1) *n, if we request k access, // each peer should have m or m+1 access - k := int(rand.Int31n(200)) + k := 43 m := k / 3 - log.Warn("k/3", "k", k, "m", m) + log.Debug("k/3", "k", k, "m", m) for i := 0; i < k; i++ { tk.EachConnLB(base, 255, f) } for addr, count := range stats { - log.Warn("Stats", "addr", addr, "count", count) + log.Debug("Stats", "addr", addr, "count", count) if count != m && count != (m+1) { t.Errorf("Expected %v or %v access to peer %v but got %v", m, m+1, addr, count) } } } +func TestEachConnFilteredLB(t *testing.T) { + tk := newTestKademlia(t, "00000000") + base := tk.base + caps := make(map[string]*capability.Capability) + + capKey := "42:101" + caps[capKey] = capability.NewCapability(42, 3) + caps[capKey].Set(0) + caps[capKey].Set(2) + tk.RegisterCapabilityIndex(capKey, *caps[capKey]) + + peer := tk.newTestKadPeerWithCapabilities("00100000", caps[capKey]) + capPeerAddress := hexutil.Encode(peer.Address()[:8]) + tk.Kademlia.On(peer) + tk.On("00100000") + tk.On("00100001") + tk.On("00010101") + stats := make(map[string]int) + f := func(peer *Peer, po int) (bool, bool) { + key := hexutil.Encode(peer.Address()[:8]) + stats[key] = stats[key] + 1 + // return false to only use one peer and mark it as used + return false, true + } + stats = make(map[string]int) + // Let n is the total number of peers (3), let na is the number of peers with + // capability a (1) + // For each number k where m * n <= k < (m+1) *n, if we request k access without capability, + // and ka where ma * na <= ka < (ma+1) access for capability a each peer of na should have m or m+1 access + // plus ma or ma+1 access + // + n := 3 + na := 1 + k := 130 + ka := 47 + m := k / n + ma := ka / na + + for i := 0; i < k; i++ { + tk.EachConnLB(base, 255, f) + } + for i := 0; i < ka; i++ { + tk.EachConnFilteredLB(base, capKey,255, f) + } + min := m + ma + max := m +1 + ma + 1 + log.Debug("Expected capability Accesses", "min", min, "max", max) + capAccesses := stats[capPeerAddress] + if min > capAccesses || capAccesses > max { + t.Errorf("Expected [%v , %v ] access to peer %v but got %v", min, max, capPeerAddress, capAccesses) + } +} + func TestSuggestPeerFindPeers(t *testing.T) { tk := newTestKademlia(t, "00000000") tk.On("00100000") From 97ab47e98f7700bbebe909682eeda3f3d552cf52 Mon Sep 17 00:00:00 2001 From: Alvaro Date: Wed, 25 Sep 2019 15:51:51 +0200 Subject: [PATCH 06/31] Removed color balancing and using a KademliaLoadBalancer --- network/kademlia.go | 257 +++++++++++---------- network/kademlia_test.go | 99 -------- network/kademlialoadbalancer.go | 330 +++++++++++++++++++++++++++ network/kademlialoadbalancer_test.go | 265 +++++++++++++++++++++ pot/pot.go | 118 ++++++++-- pot/pot_test.go | 61 +++++ 6 files changed, 901 insertions(+), 229 deletions(-) create mode 100644 network/kademlialoadbalancer.go create mode 100644 network/kademlialoadbalancer_test.go diff --git a/network/kademlia.go b/network/kademlia.go index dc9666951e..837fe63059 100644 --- a/network/kademlia.go +++ b/network/kademlia.go @@ -21,7 +21,6 @@ import ( "encoding/hex" "errors" "fmt" - "math/rand" "sort" "strings" @@ -92,13 +91,15 @@ func NewKadParams() *KadParams { type Kademlia struct { lock sync.RWMutex capabilityIndex map[string]*capabilityIndex - defaultIndex *capabilityIndex // Index with pots and searching color - *KadParams // Kademlia configuration parameters - base []byte // immutable baseaddress of the table - depth uint8 // stores the last current depth of saturation - nDepth int // stores the last neighbourhood depth - nDepthMu sync.RWMutex // protects neighbourhood depth nDepth - nDepthSig []chan struct{} // signals when neighbourhood depth nDepth is changed + defaultIndex *capabilityIndex // Index with pots and searching color + *KadParams // Kademlia configuration parameters + base []byte // immutable baseaddress of the table + depth uint8 // stores the last current depth of saturation + nDepth int // stores the last neighbourhood depth + nDepthMu sync.RWMutex // protects neighbourhood depth nDepth + nDepthSig []chan struct{} // signals when neighbourhood depth nDepth is changed + newPeerSig []chan newPeerSignal // signals when new peer is added + removePeerSig []chan *Peer // signals when peer is removed } type KademliaInfo struct { @@ -131,6 +132,11 @@ func NewKademlia(addr []byte, params *KadParams) *Kademlia { return k } +type newPeerSignal struct { + peer *Peer + po int +} + // RegisterCapabilityIndex adds an entry to the capability index of the kademlia // The capability index is associated with the supplied string s // Any peers matching any bits set in the capability in the index, will be added to the index (or removed on removal) @@ -164,9 +170,9 @@ func (k *Kademlia) addToCapabilityIndex(p interface{}) { if vCap.Match(idxItem.Capability) { log.Trace("Added peer to capability index", "conn", ok, "s", s, "v", vCap, "p", p) if ok { - k.capabilityIndex[s].conns, _, _ = pot.Add(idxItem.conns, newEntryFromPeer(ePeer, idxItem.searching), Pof) + k.capabilityIndex[s].conns, _, _ = pot.Add(idxItem.conns, newEntryFromPeer(ePeer), Pof) } else { - k.capabilityIndex[s].addrs, _, _ = pot.Add(idxItem.addrs, newEntryFromBzzAddress(eAddr, idxItem.searching), Pof) + k.capabilityIndex[s].addrs, _, _ = pot.Add(idxItem.addrs, newEntryFromBzzAddress(eAddr), Pof) } } } @@ -188,7 +194,7 @@ func (k *Kademlia) removeFromCapabilityIndex(p interface{}, disconnectOnly bool) } for s, idxItem := range k.capabilityIndex { if ok { - peerEntry := newEntryFromPeer(ePeer, idxItem.searching) + peerEntry := newEntryFromPeer(ePeer) conns, _, found, _ := pot.Swap(idxItem.conns, peerEntry, Pof, func(_ pot.Val) pot.Val { return nil }) @@ -222,35 +228,31 @@ type entry struct { conn *Peer seenAt time.Time retries int - use color } // newEntryFromBzzAddress creates a kademlia entry from a *BzzAddr -func newEntryFromBzzAddress(p *BzzAddr, initialColor color) *entry { +func newEntryFromBzzAddress(p *BzzAddr) *entry { return &entry{ BzzAddr: p, seenAt: time.Now(), - use: initialColor, } } // newEntryFromPeer creates a kademlia entry from a *Peer -func newEntryFromPeer(p *Peer, initialColor color) *entry { +func newEntryFromPeer(p *Peer) *entry { return &entry{ BzzAddr: p.BzzAddr, conn: p, seenAt: time.Now(), - use: initialColor, } } // index providing quick access to all peers having a certain capability set type capabilityIndex struct { *capability.Capability - conns *pot.Pot - addrs *pot.Pot - depth int - searching color + conns *pot.Pot + addrs *pot.Pot + depth int } // NewDefaultIndex creates a new index for no capability with black starting color and nil capabilities @@ -259,7 +261,6 @@ func NewDefaultIndex() *capabilityIndex { Capability: nil, conns: pot.NewPot(nil, 0), addrs: pot.NewPot(nil, 0), - searching: black, } } @@ -269,7 +270,6 @@ func NewCapabilityIndex(c capability.Capability) *capabilityIndex { Capability: &c, conns: pot.NewPot(nil, 0), addrs: pot.NewPot(nil, 0), - searching: black, } } @@ -283,18 +283,6 @@ func (e *entry) Hex() string { return hex.EncodeToString(e.Address()) } -func (e *entry) flipUse() { - e.use = flipColor(e.use) -} - -func flipColor(color color) color { - if color == black { - return red - } else { - return black - } -} - // Register enters each address as kademlia peer record into the // database of known peer addresses func (k *Kademlia) Register(peers ...*BzzAddr) error { @@ -317,7 +305,7 @@ func (k *Kademlia) Register(peers ...*BzzAddr) error { if v == nil { log.Trace("registering new peer", "addr", p) // insert new offline peer into addrs - return newEntryFromBzzAddress(p, index.searching) + return newEntryFromBzzAddress(p) } e := v.(*entry) @@ -326,12 +314,12 @@ func (k *Kademlia) Register(peers ...*BzzAddr) error { if !bytes.Equal(e.BzzAddr.UAddr, p.UAddr) { log.Trace("underlay addr is different, so add again", "new", p, "old", e.BzzAddr) // insert new offline peer into addrs - return newEntryFromBzzAddress(p, index.searching) + return newEntryFromBzzAddress(p) } return v }) - k.addToCapabilityIndex(newEntryFromBzzAddress(p, index.searching)) + k.addToCapabilityIndex(newEntryFromBzzAddress(p)) size++ } @@ -386,7 +374,7 @@ func (k *Kademlia) SuggestPeer() (suggestedPeer *BzzAddr, saturationDepth int, c saturation[size] = append(saturation[size], po) } return true - }) + }, true) // to trigger peer requests for peers closer than closest connection, include // all bins from nearest connection upto nearest address as unsaturated var nearestAddrAt int @@ -446,7 +434,7 @@ func (k *Kademlia) SuggestPeer() (suggestedPeer *BzzAddr, saturationDepth int, c return true }) return cur < len(bins) && suggestedPeer == nil - }) + }, true) } if uint8(saturationDepth) < k.depth { @@ -465,8 +453,9 @@ func (k *Kademlia) On(p *Peer) (uint8, bool) { var ins bool index := k.defaultIndex - peerEntry := newEntryFromPeer(p, index.searching) - index.conns, _, _, _ = pot.Swap(index.conns, peerEntry, Pof, func(v pot.Val) pot.Val { + peerEntry := newEntryFromPeer(p) + var po int + index.conns, po, _, _ = pot.Swap(index.conns, peerEntry, Pof, func(v pot.Val) pot.Val { // if not found live if v == nil { ins = true @@ -477,8 +466,16 @@ func (k *Kademlia) On(p *Peer) (uint8, bool) { return v }) k.addToCapabilityIndex(p) + // notify subscribers asynchronously + go func(p *Peer, po int) { + for _, c := range k.newPeerSig { + c <- newPeerSignal{peer: p, po: po} + } + }(p, po) + if ins { - a := newEntryFromBzzAddress(p.BzzAddr, index.searching) + a := newEntryFromBzzAddress(p.BzzAddr) + a.conn = p // insert new online peer into addrs index.addrs, _, _, _ = pot.Swap(index.addrs, a, Pof, func(v pot.Val) pot.Val { return a @@ -495,6 +492,11 @@ func (k *Kademlia) On(p *Peer) (uint8, bool) { return k.depth, changed } +func (k *Kademlia) findPeerPo(peer *Peer) (po int, found bool) { + po, found = Pof(k.defaultIndex.conns.Pin(), peer, 0) + return po, found +} + // setNeighbourhoodDepth calculates neighbourhood depth with depthForPot, // sets it to the nDepth and sends a signal to every nDepthSig channel. func (k *Kademlia) setNeighbourhoodDepth() { @@ -574,6 +576,48 @@ func (k *Kademlia) SubscribeToNeighbourhoodDepthChange() (c <-chan struct{}, uns return channel, unsubscribe } +// SubscribeToPeerChanges returns the channel that signals +// when a new Peer is added or removed from the table. Returned function unsubscribes +// the channel from signaling and releases the resources. Returned function is safe +// to be called multiple times. +func (k *Kademlia) SubscribeToPeerChanges() (addedC <-chan newPeerSignal, removedC <-chan *Peer, unsubscribe func()) { + newPeerC := make(chan newPeerSignal, 1) + removedPeerC := make(chan *Peer, 1) + var closeOnce sync.Once + + k.lock.Lock() + defer k.lock.Unlock() + + k.newPeerSig = append(k.newPeerSig, newPeerC) + k.removePeerSig = append(k.removePeerSig, removedPeerC) + + unsubscribe = func() { + k.lock.Lock() + defer k.lock.Unlock() + + for i, c := range k.newPeerSig { + if c == newPeerC { + k.newPeerSig = append(k.newPeerSig[:i], k.newPeerSig[i+1:]...) + break + } + } + + for i, c := range k.removePeerSig { + if c == removedPeerC { + k.removePeerSig = append(k.removePeerSig[:i], k.removePeerSig[i+1:]...) + break + } + } + + closeOnce.Do(func() { + close(newPeerC) + close(removedPeerC) + }) + } + + return newPeerC, removedPeerC, unsubscribe +} + // Off removes a peer from among live peers func (k *Kademlia) Off(p *Peer) { k.lock.Lock() @@ -584,7 +628,7 @@ func (k *Kademlia) Off(p *Peer) { if v == nil { panic(fmt.Sprintf("connected peer not found %v", p)) } - return newEntryFromBzzAddress(p.BzzAddr, index.searching) + return newEntryFromBzzAddress(p.BzzAddr) }) // note the following only ran if the peer was a lightnode index.conns, _, _, _ = pot.Swap(index.conns, p, Pof, func(_ pot.Val) pot.Val { @@ -593,6 +637,11 @@ func (k *Kademlia) Off(p *Peer) { }) k.removeFromCapabilityIndex(p, true) k.setNeighbourhoodDepth() + go func(p *Peer) { + for _, c := range k.removePeerSig { + c <- p + } + }(p) } // EachConnFiltered performs the same action as EachConn @@ -608,19 +657,6 @@ func (k *Kademlia) EachConnFiltered(base []byte, capKey string, o int, f func(*P return nil } -// EachConnFiltered performs the same action as EachConn -// with the difference that it will only return peers that matches the specified capability index filter -func (k *Kademlia) EachConnFilteredLB(base []byte, capKey string, o int, f func(*Peer, int) (bool, bool)) error { - k.lock.RLock() - defer k.lock.RUnlock() - c, ok := k.capabilityIndex[capKey] - if !ok { - return fmt.Errorf("Unregistered capability index '%s'", capKey) - } - k.eachConnLB(base, c, o, f) - return nil -} - // EachConn is an iterator with args (base, po, f) applies f to each live peer // that has proximity order po or less as measured from the base // if base is nil, kademlia base address is used @@ -630,18 +666,6 @@ func (k *Kademlia) EachConn(base []byte, o int, f func(*Peer, int) bool) { k.eachConn(base, k.defaultIndex.conns, o, f) } -// EachConnLB is an iterator with args (base, po, f) applies f to each live peer -// that has proximity order po or less as measured from the base -// if base is nil, kademlia base address is used. -// f function should return two booleans: First one indicates if it wants to continue iterating, -// second one signals if the peer has been used and a request should be accounted to that peer. -// It return first the peers not used in the last iteration (entry.color == k.searching) -func (k *Kademlia) EachConnLB(base []byte, o int, f func(*Peer, int) (bool, bool)) { - k.lock.RLock() - defer k.lock.RUnlock() - k.eachConnLB(base, k.defaultIndex, o, f) -} - func (k *Kademlia) eachConn(base []byte, db *pot.Pot, o int, f func(*Peer, int) bool) { if len(base) == 0 { base = k.base @@ -657,48 +681,51 @@ func (k *Kademlia) eachConn(base []byte, db *pot.Pot, o int, f func(*Peer, int) }) } -// eachConn iterates over index connection Neighbours, skipping most recent used peers to balance requests among peers -// The function f provided by the caller should return whether to obtain more peers and if the peer has been used or not -// in order to change the color -func (k *Kademlia) eachConnLB(base []byte, index *capabilityIndex, o int, f func(*Peer, int) (bool, bool)) { - db := index.conns - if len(base) == 0 { - base = k.base - } - oneFound := false - anySkip := false - lastContinue := true - f2 := func(val pot.Val, po int) bool { - if po > o { - return true - } +type PeerConsumer func(entry *entry) bool +type PeerIterator func(PeerConsumer) bool +type PeerBin struct { + ProximityOrder int + Size int + PeerIterator PeerIterator +} +type PeerBinConsumer func(peerBin *PeerBin) bool - entry, _ := val.(*entry) - if entry.use != index.searching { - anySkip = true - log.Debug("Skipping peer because of color", "peer", hexutil.Encode(entry.BzzAddr.OAddr[:8]), "color", entry.use, "searching", index.searching) - // Skip it because LB - return true - } else { - // Regardless of the user using or not this peer we set oneFound to true - oneFound = true - continueIterating, markUsed := f(entry.conn, po) - if markUsed { - // count as used - entry.flipUse() - log.Debug("Using peer", "peer", hexutil.Encode(entry.BzzAddr.OAddr[:8]), "color", flipColor(entry.use), "newColor", entry.use) - } - lastContinue = continueIterating - return continueIterating - } - } - db.EachNeighbour(base, Pof, f2) - if anySkip && !oneFound && lastContinue { - // In this situation we had a complete iteration of the table skipping all peers. We should flip searching color - // and give another lap - index.searching = flipColor(index.searching) - db.EachNeighbour(base, Pof, f2) +func (k *Kademlia) EachBinDesc(base []byte, minProximityOrder int, consumer PeerBinConsumer) { + k.lock.RLock() + defer k.lock.RUnlock() + k.eachBinDesc(k.defaultIndex, base, minProximityOrder, consumer) +} + +func (k *Kademlia) EachBinDescFiltered(base []byte, capKey string, minProximityOrder int, consumer PeerBinConsumer) error { + k.lock.RLock() + defer k.lock.RUnlock() + c, ok := k.capabilityIndex[capKey] + if !ok { + return fmt.Errorf("Unregistered capability index '%s'", capKey) } + k.eachBinDesc(c, base, minProximityOrder, consumer) + return nil +} + +func (k *Kademlia) eachBinDesc(index *capabilityIndex, base []byte, minProximityOrder int, consumer PeerBinConsumer) { + index.conns.EachBin(base, Pof, minProximityOrder, func(bin *pot.Bin) bool { + return consumer(&PeerBin{ + PeerIterator: func(consume PeerConsumer) bool { + return bin.ValIterator(func(val pot.Val) bool { + return consume(val.(*entry)) + }) + }, + ProximityOrder: bin.ProximityOrder, + Size: bin.Size, + }) + }, false) +} + +type LBPeerConsumer func(peer *Peer, po int) (bool, bool) +type LBPeerIterator func(consumer LBPeerConsumer) bool +type LBLapStatus struct { + oneFound bool + anySkip bool } // EachAddrFiltered performs the same action as EachAddr @@ -802,7 +829,7 @@ func depthForPot(p *pot.Pot, neighbourhoodSize int, pivotAddr []byte) (depth int return true } return false - }) + }, true) return depth } @@ -873,7 +900,7 @@ func (k *Kademlia) kademliaInfo() (ki KademliaInfo) { ki.Connections[po] = row return true - }) + }, true) k.defaultIndex.addrs.EachBin(k.base, Pof, 0, func(bin *pot.Bin) bool { po := bin.ProximityOrder @@ -891,7 +918,7 @@ func (k *Kademlia) kademliaInfo() (ki KademliaInfo) { ki.Known[po] = row return true - }) + }, true) return } @@ -940,7 +967,7 @@ func (k *Kademlia) string() string { r = r + wsrow liverows[po] = r[:31] return true - }) + }, true) k.defaultIndex.addrs.EachBin(k.base, Pof, 0, func(bin *pot.Bin) bool { var rowlen int @@ -962,7 +989,7 @@ func (k *Kademlia) string() string { }) peersrows[po] = strings.Join(row, " ") return true - }) + }, true) for i := 0; i < k.MaxProxDisplay; i++ { if i == depth { @@ -1063,7 +1090,7 @@ func (k *Kademlia) saturation() int { return false } return prev == po && bin.Size >= k.MinBinSize - }) + }, true) if prev < 0 { return 0 } @@ -1098,7 +1125,7 @@ func (k *Kademlia) isSaturated(peersPerBin []int, depth int) bool { unsaturatedBins = append(unsaturatedBins, po) } return true - }) + }, true) log.Trace("list of unsaturated bins", "unsaturatedBins", unsaturatedBins) return len(unsaturatedBins) == 0 diff --git a/network/kademlia_test.go b/network/kademlia_test.go index 97bf4791cd..99515ea30e 100644 --- a/network/kademlia_test.go +++ b/network/kademlia_test.go @@ -23,7 +23,6 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" @@ -377,104 +376,6 @@ func binStr(a *BzzAddr) string { return pot.ToBin(a.Address())[:8] } -func TestEachConnLB(t *testing.T) { - tk := newTestKademlia(t, "00000000") - base := tk.base - tk.On("00100000") - tk.On("00100001") - tk.On("00010101") - stats := make(map[string]int) - f := func(peer *Peer, po int) (bool, bool) { - key := hexutil.Encode(peer.Address()[:8]) - stats[key] = stats[key] + 1 - // return false to only use one peer and mark it as used - return false, true - } - // Let n be the number of peers (3) - // Executing 2 * n times. We expect 2 access to each peer - tk.EachConnLB(base, 255, f) - tk.EachConnLB(base, 255, f) - tk.EachConnLB(base, 255, f) - tk.EachConnLB(base, 255, f) - tk.EachConnLB(base, 255, f) - tk.EachConnLB(base, 255, f) - for addr, count := range stats { - log.Debug("Stats", "addr", addr, "count", count) - if count != 2 { - t.Errorf("Expected 2 access to peer %v but got %v", addr, count) - } - } - - stats = make(map[string]int) - // For each number k where m * n <= k < (m+1) *n, if we request k access, - // each peer should have m or m+1 access - k := 43 - m := k / 3 - log.Debug("k/3", "k", k, "m", m) - for i := 0; i < k; i++ { - tk.EachConnLB(base, 255, f) - } - for addr, count := range stats { - log.Debug("Stats", "addr", addr, "count", count) - if count != m && count != (m+1) { - t.Errorf("Expected %v or %v access to peer %v but got %v", m, m+1, addr, count) - } - } -} - -func TestEachConnFilteredLB(t *testing.T) { - tk := newTestKademlia(t, "00000000") - base := tk.base - caps := make(map[string]*capability.Capability) - - capKey := "42:101" - caps[capKey] = capability.NewCapability(42, 3) - caps[capKey].Set(0) - caps[capKey].Set(2) - tk.RegisterCapabilityIndex(capKey, *caps[capKey]) - - peer := tk.newTestKadPeerWithCapabilities("00100000", caps[capKey]) - capPeerAddress := hexutil.Encode(peer.Address()[:8]) - tk.Kademlia.On(peer) - tk.On("00100000") - tk.On("00100001") - tk.On("00010101") - stats := make(map[string]int) - f := func(peer *Peer, po int) (bool, bool) { - key := hexutil.Encode(peer.Address()[:8]) - stats[key] = stats[key] + 1 - // return false to only use one peer and mark it as used - return false, true - } - stats = make(map[string]int) - // Let n is the total number of peers (3), let na is the number of peers with - // capability a (1) - // For each number k where m * n <= k < (m+1) *n, if we request k access without capability, - // and ka where ma * na <= ka < (ma+1) access for capability a each peer of na should have m or m+1 access - // plus ma or ma+1 access - // - n := 3 - na := 1 - k := 130 - ka := 47 - m := k / n - ma := ka / na - - for i := 0; i < k; i++ { - tk.EachConnLB(base, 255, f) - } - for i := 0; i < ka; i++ { - tk.EachConnFilteredLB(base, capKey,255, f) - } - min := m + ma - max := m +1 + ma + 1 - log.Debug("Expected capability Accesses", "min", min, "max", max) - capAccesses := stats[capPeerAddress] - if min > capAccesses || capAccesses > max { - t.Errorf("Expected [%v , %v ] access to peer %v but got %v", min, max, capPeerAddress, capAccesses) - } -} - func TestSuggestPeerFindPeers(t *testing.T) { tk := newTestKademlia(t, "00000000") tk.On("00100000") diff --git a/network/kademlialoadbalancer.go b/network/kademlialoadbalancer.go new file mode 100644 index 0000000000..3b607d2b58 --- /dev/null +++ b/network/kademlialoadbalancer.go @@ -0,0 +1,330 @@ +package network + +import ( + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethersphere/swarm/log" + "sort" + "strconv" + "sync" +) + +// KademliaBackend is the required interface of KademliaLoadBalancer. +type KademliaBackend interface { + SubscribeToPeerChanges() (addedC <-chan newPeerSignal, removedC <-chan *Peer, unsubscribe func()) + BaseAddr() []byte + EachBinDesc(base []byte, minProximityOrder int, consumer PeerBinConsumer) + EachBinDescFiltered(base []byte, capKey string, minProximityOrder int, consumer PeerBinConsumer) error +} + +// Creates a new KademliaLoadBalancer from a KademliaBackend +func NewKademliaLoadBalancer(kademlia KademliaBackend) *KademliaLoadBalancer { + chanNewPeerSignals, chanOffPeerSignals, unsubscribe := kademlia.SubscribeToPeerChanges() + klb := &KademliaLoadBalancer{ + kademlia: kademlia, + resourceUseStats: newResourceLoadBalancer(), + newPeerChannel: chanNewPeerSignals, + offPeerChannel: chanOffPeerSignals, + unsubscribeNotifier: unsubscribe, + quitC: make(chan struct{}), + } + go klb.listenNewPeers() + go klb.listenOffPeers() + return klb +} + +// Consumer functions + +// An LBPeer represents a peer with a Use() function to signal that the peer has been used in order +// to account it for LB sorting criteria +type LBPeer struct { + Peer *Peer + Use func() +} + +// LBBin represents a Bin of LBPeer's +type LBBin struct { + LBPeers []LBPeer + ProximityOrder int +} + +// LBBinConsumer will be provided with a list of LBPeer's usually in LB criteria ordering +type LBBinConsumer func(bin LBBin) bool + +// KademliaLoadBalancer struct and methods + +// KademliaLoadBalancer tries to balance request to the peers in Kademlia returning the peers sorted +// by least recent used whenever several will be returned with the same po to a particular address. +// The user of KademliaLoadBalancer should signal if the returned element (LBPeer) has been used with the +// function lbPeer.Use() +type KademliaLoadBalancer struct { + kademlia KademliaBackend //kademlia to obtain bins of peers + resourceUseStats *resourceUseStats //a resourceUseStats to count uses + newPeerChannel <-chan newPeerSignal //a channel to be notified of new peers in kademlia + offPeerChannel <-chan *Peer //a channel to be notified of removed peers in kademlia + unsubscribeNotifier func() //an unsubscribe function provided when subscribe to kademlia notifiers + quitC chan struct{} +} + +// Stop unsubscribe from notifiers +func (klb KademliaLoadBalancer) Stop() { + klb.unsubscribeNotifier() + close(klb.quitC) +} + +// EachBinNodeAddress calls EachBin with the base address of kademlia (the node address) +func (klb KademliaLoadBalancer) EachBinNodeAddress(consumeBin LBBinConsumer) { + klb.EachBin(klb.kademlia.BaseAddr(), consumeBin) +} + +// EachBinFiltered returns all bins in descending order from the perspective of base address. +// Only peers with the provided capabilities capKey are considered. +// All peers in that bin will be provided to the LBBinConsumer sorted by least used first. +func (klb KademliaLoadBalancer) EachBinFiltered(base []byte, capKey string, consumeBin LBBinConsumer) { + klb.kademlia.EachBinDescFiltered(base, capKey, 0, func(peerBin *PeerBin) bool { + peers := klb.peerBinToPeerList(peerBin) + return consumeBin(LBBin{LBPeers: peers, ProximityOrder: peerBin.ProximityOrder}) + }) +} + +// EachBin returns all bins in descending order from the perspective of base address. +// All peers in that bin will be provided to the LBBinConsumer sorted by least used first. +func (klb KademliaLoadBalancer) EachBin(base []byte, consumeBin LBBinConsumer) { + klb.kademlia.EachBinDesc(base, 0, func(peerBin *PeerBin) bool { + peers := klb.peerBinToPeerList(peerBin) + return consumeBin(LBBin{LBPeers: peers, ProximityOrder: peerBin.ProximityOrder}) + }) +} + +func (klb *KademliaLoadBalancer) peerBinToPeerList(bin *PeerBin) []LBPeer { + resources := make([]Resource, bin.Size) + var i int + bin.PeerIterator(func(entry *entry) bool { + resources[i] = entry.conn + i++ + return true + }) + return klb.resourcesToLbPeers(resources) +} + +func (klb *KademliaLoadBalancer) resourcesToLbPeers(resources []Resource) []LBPeer { + sorted := klb.resourceUseStats.sortResources(resources) + peers := klb.toLBPeers(sorted) + return peers +} + +func (klb *KademliaLoadBalancer) listenNewPeers() { + for { + select { + case <-klb.quitC: + return + case signal, ok := <-klb.newPeerChannel: + if !ok { + return + } + klb.addedPeer(signal.peer, signal.po) + } + } +} + +func (klb *KademliaLoadBalancer) listenOffPeers() { + for { + select { + case <-klb.quitC: + return + case peer := <-klb.offPeerChannel: + if peer != nil { + klb.removedPeer(peer) + } + } + } +} + +// addedPeer is called back when a new peer is added to the kademlia. Its uses will be initialized +// to the use count of the least used peer in its bin. The po of the new peer is passed to avoid having +// to calculate it again. +func (klb *KademliaLoadBalancer) addedPeer(peer *Peer, po int) { + leastUsedCount := klb.leastUsedCountInBin(po, peer) + log.Debug("Adding peer", "key", peer.Key()[:4], "leastUsedCount", leastUsedCount) + klb.resourceUseStats.initKey(peer.Key(), leastUsedCount) +} + +// leastUsedCountInBin returns the use count for the least used peer in this bin excluding the excludePeer. +func (klb *KademliaLoadBalancer) leastUsedCountInBin(po int, excludePeer *Peer) int { + addr := klb.kademlia.BaseAddr() + peersInSamePo := klb.getPeersForPo(addr, po) + idx := 0 + leastUsedCount := 0 + for idx < len(peersInSamePo) { + leastUsed := peersInSamePo[idx] + if leastUsed.Peer.Key() != excludePeer.Key() { + leastUsedCount = klb.resourceUseStats.getUses(leastUsed.Peer) + log.Debug("Least used peer is", "peer", leastUsed.Peer.Key()[:4], "leastUsedCount", leastUsedCount) + break + } + idx++ + } + return leastUsedCount +} + +func (klb *KademliaLoadBalancer) removedPeer(peer *Peer) { + klb.resourceUseStats.lock.Lock() + defer klb.resourceUseStats.lock.Lock() + delete(klb.resourceUseStats.resourceUses, peer.Key()) +} + +func (klb *KademliaLoadBalancer) getPeersForPo(base []byte, po int) []LBPeer { + resources := make([]Resource, 0) + klb.kademlia.EachBinDesc(base, po, func(bin *PeerBin) bool { + if bin.ProximityOrder == po { + return bin.PeerIterator(func(entry *entry) bool { + resources = append(resources, entry.conn) + return true + }) + } else { + return true + } + }) + return klb.resourcesToLbPeers(resources) +} + +func (klb *KademliaLoadBalancer) toLBPeers(resources []Resource) []LBPeer { + peers := make([]LBPeer, len(resources)) + for i, res := range resources { + peer := res.(*Peer) + peers[i].Peer = peer + peers[i].Use = func() { + klb.resourceUseStats.addUse(peer) + } + } + return peers +} + +// Resource Use Stats + +// resourceUseStats can be used to count uses of resources. A Resource is anything with a Key() +type resourceUseStats struct { + resourceUses map[string]int + waiting map[string]chan struct{} + lock sync.RWMutex +} + +type Resource interface { + Key() string +} + +// Adding Resource interface to Peer +func (d *Peer) Key() string { + return hexutil.Encode(d.Address()) +} + +type ResourceCount struct { + resource Resource + count int +} + +func newResourceLoadBalancer() *resourceUseStats { + return &resourceUseStats{ + resourceUses: make(map[string]int), + waiting: make(map[string]chan struct{}), + } +} + +func (lb *resourceUseStats) sortResources(resources []Resource) []Resource { + sorted := make([]Resource, len(resources)) + resourceCounts := lb.getAllUses(resources) + sort.Slice(resourceCounts, func(i, j int) bool { + return resourceCounts[i].count < resourceCounts[j].count + }) + for i, resourceCount := range resourceCounts { + sorted[i] = resourceCount.resource + } + return sorted +} + +func (lbp ResourceCount) String() string { + return lbp.resource.Key() + ":" + strconv.Itoa(lbp.count) +} + +func (lb *resourceUseStats) dumpAllUses() map[string]int { + lb.lock.RLock() + defer lb.lock.RUnlock() + dump := make(map[string]int) + for k, v := range lb.resourceUses { + dump[k] = v + } + return dump +} + +func (lb *resourceUseStats) getAllUses(resources []Resource) []ResourceCount { + peerUses := make([]ResourceCount, len(resources)) + for i, resource := range resources { + peerUses[i] = ResourceCount{ + resource: resource, + count: lb.getUses(resource), + } + } + return peerUses +} + +func (lb *resourceUseStats) getUses(keyed Resource) int { + return lb.getKeyUses(keyed.Key()) +} + +func (lb *resourceUseStats) getKeyUses(key string) int { + lb.lock.RLock() + defer lb.lock.RUnlock() + return lb.resourceUses[key] +} + +func (lb *resourceUseStats) addUse(resource Resource) int { + lb.lock.Lock() + defer lb.lock.Unlock() + log.Debug("Adding use", "key", resource.Key()[:4]) + key := resource.Key() + lb.resourceUses[key] = lb.resourceUses[key] + 1 + return lb.resourceUses[key] +} + +// Used for testing. As peer resource initialization is asynchronous we need a way +// to know that the initial uses has been initialized for a new peer +func (lb *resourceUseStats) waitKey(key string) { + lb.lock.Lock() + defer lb.lock.Unlock() + if _, ok := lb.resourceUses[key]; ok { + return + } + lb.waiting[key] = make(chan struct{}) + <-lb.waiting[key] + delete(lb.waiting, key) +} + +func (lb *resourceUseStats) initKey(key string, count int) { + lb.lock.Lock() + defer lb.lock.Unlock() + lb.resourceUses[key] = count + if kChan, ok := lb.waiting[key]; ok { + kChan <- struct{}{} + } +} + +// Debug functions + +func stringBinaryToHex(binary string) string { + var byteSlice = make([]byte, 32) + i, _ := strconv.ParseInt(binary, 2, 0) + byteSlice[0] = byte(i) + return hexutil.Encode(byteSlice) +} +func peerToBinaryId(peer *Peer) string { + return byteToBinary(peer.Address()[0]) +} + +func byteToBinary(b byte) string { + binary := strconv.FormatUint(uint64(b), 2) + if len(binary) < 8 { + for i := 8 - len(binary); i > 0; i-- { + binary = "0" + binary + } + } + return binary +} diff --git a/network/kademlialoadbalancer_test.go b/network/kademlialoadbalancer_test.go new file mode 100644 index 0000000000..fa7f2b30a3 --- /dev/null +++ b/network/kademlialoadbalancer_test.go @@ -0,0 +1,265 @@ +package network + +import ( + "sort" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethersphere/swarm/network/capability" + "github.com/ethersphere/swarm/pot" +) + +// TestAddedNodes tests that when adding a node it is assigned the correct number of uses. +// This number of uses will be the least number of uses of a peer in its bin +func TestAddedNodes(t *testing.T) { + kademlia := newTestKademliaBackend("11110000") + first := newTestKadPeer("010101010") + kademlia.addPeer(first, 0) + second := newTestKadPeer("010101011") + kademlia.addPeer(second, 0) + klb := NewKademliaLoadBalancer(kademlia) + + defer klb.Stop() + firstUses := klb.resourceUseStats.getUses(first) + if firstUses != 0 { + t.Errorf("Expected 0 uses for new peer at start") + } + peersFor0 := klb.getPeersForPo(kademlia.baseAddr, 0) + peersFor0[0].Use() + // Now new peers still should have 0 uses + third := newTestKadPeer("011101011") + kademlia.addPeer(third, 0) + klb.resourceUseStats.waitKey(third.Key()) + thirdUses := klb.resourceUseStats.getUses(third) + if thirdUses != 0 { + t.Errorf("Expected 0 uses for new peer because minimum in bin is 0. Instead %v", thirdUses) + } + peersFor0 = klb.getPeersForPo(kademlia.baseAddr, 0) + peersFor0[0].Use() + peersFor0[1].Use() //Now all peers should have 1 use + //New peers should start with 1 use + fourth := newTestKadPeer("011100011") + kademlia.addPeer(fourth, 0) + klb.resourceUseStats.waitKey(fourth.Key()) + fourthUses := klb.resourceUseStats.getUses(fourth) + if fourthUses != 1 { + t.Errorf("Expected 1 use for new peer because minimum in bin should be 1. Instead %v", fourthUses) + } +} + +// TestEachBinBaseUses tests that EachBin returns first the least used peer in its bin +// We will create 3 bins with two peers each. We will call EachBin 6 times twice with an address +// on each bin, so at the end all peers should have 1 use (because the address in each bin is equidistant to +// the peers in that bin). +// Then wi will use an address in a bin that is nearer one of the peers and we will check that that peer is always +// returned first +func TestEachBinBaseUses(t *testing.T) { + tk := newTestKademlia(t, "11111111") + klb := NewKademliaLoadBalancer(tk) + tk.On("01010101") //Peer 1 dec 85 + tk.On("01010100") // 2 dec 84 + tk.On("10010100") // 3 dec 148 + tk.On("10010001") // 4 dec 145 + tk.On("11010100") // 5 dec 212 + tk.On("11010101") // 6 dec 213 + + pivotAddressBin0 := pot.NewAddressFromString("00000000") // Two nearest peers (1,2) + pivotAddressBin1 := pot.NewAddressFromString("10000000") // Two nearest peers (3,4) + pivotAddressBin2 := pot.NewAddressFromString("11000000") // Two nearest peers (5,6) + countUse := func(bin LBBin) bool { + bin.LBPeers[0].Use() + return false + } + // Use peer 1 and 2 + klb.EachBin(pivotAddressBin0, countUse) + klb.EachBin(pivotAddressBin0, countUse) + + // Use peers 3 and 4 + klb.EachBin(pivotAddressBin1, countUse) + klb.EachBin(pivotAddressBin1, countUse) + + // Use peers 5 and 6 + klb.EachBin(pivotAddressBin2, countUse) + klb.EachBin(pivotAddressBin2, countUse) + + resourceUses := klb.resourceUseStats.dumpAllUses() + if len(resourceUses) != 6 { + t.Errorf("Expected all 6 peers to be used but got %v", len(resourceUses)) + } + for key, uses := range resourceUses { + if uses != 1 { + bytes, _ := hexutil.Decode(key) + binaryKey := byteToBinary(bytes[0]) + byteToBinary(bytes[1]) + t.Errorf("Expected only 1 use of %v but got %v", binaryKey, uses) + } + } + + //Now a message that is nearer 10010001 than 10010100 in its bin. It will be taken always regardless of uses + pivotAddressBin3 := pot.NewAddressFromString("10010011") // Nearer 4 + + //Both calls to 4 + klb.EachBin(pivotAddressBin3, countUse) + klb.EachBin(pivotAddressBin3, countUse) + + count := klb.resourceUseStats.getKeyUses(stringBinaryToHex("10010001")) + if count != 3 { + t.Errorf("Expected 3 uses of 10010001 but got %v", count) + } +} + +func TestEachBinFiltered(t *testing.T) { + tk := newTestKademlia(t, "11111111") + klb := NewKademliaLoadBalancer(tk) + caps := make(map[string]*capability.Capability) + + capKey := "42:101" + caps[capKey] = capability.NewCapability(42, 3) + caps[capKey].Set(0) + caps[capKey].Set(2) + tk.RegisterCapabilityIndex(capKey, *caps[capKey]) + + capPeer := tk.newTestKadPeerWithCapabilities("10100000", caps[capKey]) + tk.Kademlia.On(capPeer) + + tk.On("01010101") // bin 0 dec 85 + tk.On("01010100") // bin 0 dec 84 + tk.On("10010100") // bin 1 dec 148 + tk.On("10010001") // bin 1 dec 145 + tk.On("11010100") // bin 2 dec 212 + tk.On("11010101") // bin 2 dec 213 + stats := make(map[string]int) + countUse := func(bin LBBin) bool { + peer := bin.LBPeers[0].Peer + bin.LBPeers[0].Use() + key := peerToBinaryId(peer) + stats[key] = stats[key] + 1 + return false + } + + pivotAddressBin1 := pot.NewAddressFromString("10000000") // Two nearest peers (1,2) + // Instead of selecting peers 10010100 or 10010001, capPeer is always chosen (10100000) + klb.EachBinFiltered(pivotAddressBin1, capKey, countUse) + klb.EachBinFiltered(pivotAddressBin1, capKey, countUse) + klb.EachBinFiltered(pivotAddressBin1, capKey, countUse) + + useStats := klb.resourceUseStats + count := useStats.getUses(capPeer) + if count != 3 || stats["10100000"] != 3 { + t.Errorf("Expected 3 uses of capability peer but got %v/%v", count, stats["10100000"]) + } + + secondCapPeer := tk.newTestKadPeerWithCapabilities("10100000", caps[capKey]) + tk.Kademlia.On(secondCapPeer) + useStats.waitKey(secondCapPeer.Key()) + klb.EachBinFiltered(pivotAddressBin1, capKey, countUse) + klb.EachBinFiltered(pivotAddressBin1, capKey, countUse) + secondCount := useStats.getUses(secondCapPeer) + if secondCount == 0 { + t.Errorf("Expected some use of second capability peer but got %v", secondCount) + } + +} + +type testKademliaBackend struct { + baseAddr []byte + addedChannel chan newPeerSignal + removedChannel chan *Peer + bins map[int][]*Peer + subscribed bool +} + +func newTestKademliaBackend(address string) *testKademliaBackend { + return &testKademliaBackend{ + baseAddr: pot.NewAddressFromString(address), + addedChannel: make(chan newPeerSignal, 1), + removedChannel: make(chan *Peer, 1), + bins: make(map[int][]*Peer), + } +} + +func (tkb testKademliaBackend) BaseAddr() []byte { + return tkb.baseAddr +} + +func (tkb *testKademliaBackend) SubscribeToPeerChanges() (addedC <-chan newPeerSignal, removedC <-chan *Peer, unsubscribe func()) { + unsubscribe = func() { + tkb.subscribed = false + close(tkb.addedChannel) + close(tkb.removedChannel) + } + tkb.subscribed = true + return tkb.addedChannel, tkb.removedChannel, unsubscribe +} + +func (tkb testKademliaBackend) EachBinDescFiltered(base []byte, capKey string, minProximityOrder int, consumer PeerBinConsumer) error { + tkb.EachBinDesc(base, minProximityOrder, consumer) + return nil +} + +func (tkb testKademliaBackend) EachBinDesc(_ []byte, minProximityOrder int, consumer PeerBinConsumer) { + type poPeers struct { + po int + peers []*Peer + } + var poPeersList []poPeers + for po, peers := range tkb.bins { + poPeersList = append(poPeersList, poPeers{po: po, peers: peers}) + } + sort.Slice(poPeersList, func(i, j int) bool { + return poPeersList[i].po > poPeersList[j].po + }) + for _, aPoPeers := range poPeersList { + peers := aPoPeers.peers + po := aPoPeers.po + if peers != nil && po >= minProximityOrder { + bin := &PeerBin{ + ProximityOrder: po, + Size: len(peers), + PeerIterator: func(consumePeer PeerConsumer) bool { + for _, peer := range peers { + if !consumePeer(&entry{conn: peer}) { + return false + } + } + return true + }, + } + if !consumer(bin) { + return + } + } + } +} + +func (tkb *testKademliaBackend) addPeer(peer *Peer, po int) { + if tkb.bins[po] == nil { + tkb.bins[po] = make([]*Peer, 0) + } + tkb.bins[po] = append(tkb.bins[po], peer) + if tkb.subscribed { + tkb.addedChannel <- newPeerSignal{ + peer: peer, + po: po, + } + } + time.Sleep(100 * time.Millisecond) +} + +func (tkb *testKademliaBackend) removePeer(peer *Peer) { + for po, bin := range tkb.bins { + for i, aPeer := range bin { + if aPeer == peer { + tkb.bins[po] = append(bin[:i], bin[i+1:]...) + break + } + } + } + if tkb.subscribed { + tkb.removedChannel <- peer + } +} + +func newTestKadPeer(s string) *Peer { + return NewPeer(&BzzPeer{BzzAddr: testKadPeerAddr(s)}, nil) +} diff --git a/pot/pot.go b/pot/pot.go index 152c48273c..008efbdbee 100644 --- a/pot/pot.go +++ b/pot/pot.go @@ -470,26 +470,26 @@ type Bin struct { // Consumer in generics notation type BinConsumer func(bin *Bin) bool -// Each is a synchronous iterator over the elements of pot with function f. +// Each is a synchronous iterator over the elements of pot with a consumer. func (t *Pot) Each(consumer ValConsumer) bool { return t.each(consumer) } -// each is a synchronous iterator over the elements of pot with consumer f. +// each is a synchronous iterator over the elements of pot with a consumer. // the iteration ends if the consumer return false or there are no more elements. -func (t *Pot) each(f ValConsumer) bool { +func (t *Pot) each(consume ValConsumer) bool { if t == nil || t.size == 0 { return false } for _, n := range t.bins { - if !n.each(f) { + if !n.each(consume) { return false } } - return f(t.pin) + return consume(t.pin) } -// eachFrom is a synchronous iterator over the elements of pot with consumer, +// eachFrom is a synchronous iterator over the elements of pot with a consumer, // starting from certain proximity order po, which is passed as a second parameter. // the iteration ends if the function return false or there are no more elements. func (t *Pot) eachFrom(consumer ValConsumer, po int) bool { @@ -511,10 +511,16 @@ func (t *Pot) eachFrom(consumer ValConsumer, po int) bool { // The order the bins are consumed depends on the bins po with respect to the pivot Val. // minProximityOrder gives the caller the possibility of filtering the bins by proximityOrder >= minProximityOrder // If pivotVal is the root val it iterates the bin as stored in this pot. -func (t *Pot) EachBin(pivotVal Val, pof Pof, minProximityOrder int, binConsumer BinConsumer) { - t.eachBin(pivotVal, pof, minProximityOrder, binConsumer) +// ascending flag controls the sorting of bins in the iterator. True => will be for farthest to closest, false => closest to farthest +func (t *Pot) EachBin(pivotVal Val, pof Pof, minProximityOrder int, binConsumer BinConsumer, ascending bool) { + if ascending { + t.eachBin(pivotVal, pof, minProximityOrder, binConsumer) + } else { + t.eachBinDesc(pivotVal, pof, minProximityOrder, binConsumer) + } } +// eachBin traverse bin in ascending order (farthest to nearest) func (t *Pot) eachBin(pivotVal Val, pof Pof, minProximityOrder int, consumeBin BinConsumer) { if t == nil || t.size == 0 { return @@ -585,14 +591,96 @@ func (t *Pot) eachBin(pivotVal Val, pof Pof, minProximityOrder int, consumeBin B } +// eachBinDesc traverse bins in descending po order (nearest to farthest). Returns if the user wants to continue iterating +func (t *Pot) eachBinDesc(pivotVal Val, pof Pof, minProximityOrder int, consumeBin BinConsumer) bool { + if t == nil || t.size == 0 { + return false + } + valProximityOrder, _ := pof(t.pin, pivotVal, t.po) + _, pivotBinIndex := t.getPos(valProximityOrder) + + var subPot *Pot + // If pivotBinIndex == len(t.bins), the pivotVal is the t.pin. We consume a virtual bin with max valProximityOrder + // and only one element. + if pivotBinIndex == len(t.bins) { + if valProximityOrder >= minProximityOrder { + bin := &Bin{ + ProximityOrder: valProximityOrder, + Size: 1, + // Only iterate the pin + ValIterator: func(consume ValConsumer) bool { + return consume(t.pin) + }, + } + if !consumeBin(bin) { + return false + } + } + } else { // pivotVal is anywhere on the subtree + subPot = t.bins[pivotBinIndex] + // Consume bin where the pivotVal is, there we will have closest bins and t.pin that will have valProximityOrder + if subPot.po == valProximityOrder { + if !subPot.eachBinDesc(pivotVal, pof, minProximityOrder, consumeBin) { + return false + } + } + + higherPo := valProximityOrder + nextBinsStart := pivotBinIndex + if subPot.po == valProximityOrder { + nextBinsStart++ + higherPo++ + } + var size int = 1 //One for the pin + for i := nextBinsStart; i < len(t.bins); i++ { + size += t.bins[i].size + } + // Consuming all bins after the bin where the pivotVal is + // (All bins will be provided to the user as one virtual bin with po = valProximityOrder) + if valProximityOrder >= minProximityOrder { + bin := &Bin{ + ProximityOrder: valProximityOrder, + Size: size, + ValIterator: func(consume ValConsumer) bool { + return t.eachFrom(consume, higherPo) + }, + } + if !consumeBin(bin) { + return false + } + } + } + + // Finally we will consume all bins before the pivotVal bin (or all bins if the pivotVal is the t.pin) + // Always filtering bins with proximityOrder < minProximityOrder + for i := pivotBinIndex - 1; i >= 0; i-- { + subPot = t.bins[i] + if subPot.po < minProximityOrder { + return true + } + bin := &Bin{ + ProximityOrder: subPot.po, + Size: subPot.size, + ValIterator: subPot.each, + } + if !consumeBin(bin) { + return false + } + } + return true + +} + +type NeighbourConsumer = func(Val, int) bool + // EachNeighbour is a synchronous iterator over neighbours of any target val // the order of elements retrieved reflect proximity order to the target // TODO: add maximum proxbin to start range of iteration -func (t *Pot) EachNeighbour(val Val, pof Pof, f func(Val, int) bool) bool { - return t.eachNeighbour(val, pof, f) +func (t *Pot) EachNeighbour(val Val, pof Pof, consume NeighbourConsumer) bool { + return t.eachNeighbour(val, pof, consume) } -func (t *Pot) eachNeighbour(val Val, pof Pof, f func(Val, int) bool) bool { +func (t *Pot) eachNeighbour(val Val, pof Pof, consume NeighbourConsumer) bool { if t == nil || t.size == 0 { return false } @@ -605,7 +693,7 @@ func (t *Pot) eachNeighbour(val Val, pof Pof, f func(Val, int) bool) bool { if !eq { n, il = t.getPos(po) if n != nil { - next = n.eachNeighbour(val, pof, f) + next = n.eachNeighbour(val, pof, consume) if !next { return false } @@ -615,14 +703,14 @@ func (t *Pot) eachNeighbour(val Val, pof Pof, f func(Val, int) bool) bool { } } - next = f(t.pin, po) + next = consume(t.pin, po) if !next { return false } for i := l - 1; i > ir; i-- { next = t.bins[i].each(func(v Val) bool { - return f(v, po) + return consume(v, po) }) if !next { return false @@ -632,7 +720,7 @@ func (t *Pot) eachNeighbour(val Val, pof Pof, f func(Val, int) bool) bool { for i := il - 1; i >= 0; i-- { n := t.bins[i] next = n.each(func(v Val) bool { - return f(v, n.po) + return consume(v, n.po) }) if !next { return false diff --git a/pot/pot_test.go b/pot/pot_test.go index 6189c7607a..de3636f9fe 100644 --- a/pot/pot_test.go +++ b/pot/pot_test.go @@ -20,6 +20,7 @@ import ( "fmt" "math/rand" "runtime" + "strconv" "sync" "testing" "time" @@ -590,6 +591,66 @@ func TestPotEachNeighbourAsync(t *testing.T) { } } +func TestEachBinDesc(t *testing.T) { + pof := DefaultPof(8) + baseAddr := newTestAddr("11111111", 0) + pot := NewPot(baseAddr, 0) + pot, _, _ = testAdd(pot, pof, 1, "01111111", "01000000", "10111111", "11011111", "11101111") + binConsumer := func(bin *Bin) bool { + fmt.Println("Bin", "pot", bin.ProximityOrder, "size", bin.Size) + bin.ValIterator(func(val Val) bool { + addr := val.(*testAddr) + fmt.Println(" val", toBinaryByte(addr), "pot", bin.ProximityOrder) + return true + }) + return true + } + fmt.Println("****************Forward order****************") + pot.eachBin(pot.pin, pof, 0, binConsumer) + fmt.Println("****************Reverse order****************") + pot.eachBinDesc(pot.pin, pof, 0, binConsumer) + + //Test eachBinDesc for a given address. For example one address with po 2 with respect to t.pin + pivotAddr := newTestAddr("11010000", 6) + fmt.Println("****************Forward order with pivot 11010000****************") + pot.eachBin(pivotAddr, pof, 0, binConsumer) + fmt.Println("****************Reverse order with pivot 11010000****************") + pot.eachBinDesc(pivotAddr, pof, 0, binConsumer) +} + +func TestEachBinDescPivotInAMissingBin(t *testing.T) { + pof := DefaultPof(8) + baseAddr := newTestAddr("11111111", 0) + pot := NewPot(baseAddr, 0) + pot, _, _ = testAdd(pot, pof, 1, "01111111", "01000000", "10111111", "11101111", "11101011", "11111110") + binConsumer := func(bin *Bin) bool { + fmt.Println("Bin", "pot", bin.ProximityOrder, "size", bin.Size) + bin.ValIterator(func(val Val) bool { + addr := val.(*testAddr) + fmt.Println(" val", toBinaryByte(addr), "pot", bin.ProximityOrder) + return true + }) + return true + } + + //Test eachBinDesc for an address with a po that we don't have a bin for + pivotAddr := newTestAddr("11010000", 7) + fmt.Println("****************Forward order with pivot 11010000****************") + pot.eachBin(pivotAddr, pof, 0, binConsumer) + fmt.Println("****************Reverse order with pivot 11010000****************") + pot.eachBinDesc(pivotAddr, pof, 0, binConsumer) +} + +func toBinaryByte(addr *testAddr) string { + formatted := strconv.FormatInt(int64(addr.Address()[0]), 2) + if len(formatted) < 8 { + for i := 0; i < 8-len(formatted); i++ { + formatted = "0" + formatted + } + } + return formatted +} + func benchmarkEachNeighbourSync(t *testing.B, max, count int, d time.Duration) { t.ReportAllocs() alen := maxkeylen From f0fc99db23b3ff204dc4af1da83f9ff8f039a13d Mon Sep 17 00:00:00 2001 From: Alvaro Date: Wed, 25 Sep 2019 16:05:08 +0200 Subject: [PATCH 07/31] Missing file in commit --- network/kademlia.go | 12 ++------- network/kademlialoadbalancer.go | 7 ++--- network/kademlialoadbalancer_test.go | 6 ++--- pss/forwarding_test.go | 10 +++++--- pss/pss.go | 38 ++++++++++++++-------------- 5 files changed, 34 insertions(+), 39 deletions(-) diff --git a/network/kademlia.go b/network/kademlia.go index 837fe63059..cae330f020 100644 --- a/network/kademlia.go +++ b/network/kademlia.go @@ -28,7 +28,6 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/metrics" "github.com/ethersphere/swarm/log" "github.com/ethersphere/swarm/network/capability" @@ -91,7 +90,7 @@ func NewKadParams() *KadParams { type Kademlia struct { lock sync.RWMutex capabilityIndex map[string]*capabilityIndex - defaultIndex *capabilityIndex // Index with pots and searching color + defaultIndex *capabilityIndex // Index with pots *KadParams // Kademlia configuration parameters base []byte // immutable baseaddress of the table depth uint8 // stores the last current depth of saturation @@ -215,13 +214,6 @@ func (k *Kademlia) removeFromCapabilityIndex(p interface{}, disconnectOnly bool) } } -const ( - black = iota - red -) - -type color int - // entry represents a Kademlia table entry (an extension of BzzAddr) type entry struct { *BzzAddr @@ -255,7 +247,7 @@ type capabilityIndex struct { depth int } -// NewDefaultIndex creates a new index for no capability with black starting color and nil capabilities +// NewDefaultIndex creates a new index for no capability func NewDefaultIndex() *capabilityIndex { return &capabilityIndex{ Capability: nil, diff --git a/network/kademlialoadbalancer.go b/network/kademlialoadbalancer.go index 3b607d2b58..21d8b0feee 100644 --- a/network/kademlialoadbalancer.go +++ b/network/kademlialoadbalancer.go @@ -1,11 +1,12 @@ package network import ( - "github.com/ethereum/go-ethereum/common/hexutil" - "github.com/ethersphere/swarm/log" "sort" "strconv" "sync" + + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethersphere/swarm/log" ) // KademliaBackend is the required interface of KademliaLoadBalancer. @@ -80,7 +81,7 @@ func (klb KademliaLoadBalancer) EachBinNodeAddress(consumeBin LBBinConsumer) { // Only peers with the provided capabilities capKey are considered. // All peers in that bin will be provided to the LBBinConsumer sorted by least used first. func (klb KademliaLoadBalancer) EachBinFiltered(base []byte, capKey string, consumeBin LBBinConsumer) { - klb.kademlia.EachBinDescFiltered(base, capKey, 0, func(peerBin *PeerBin) bool { + _ = klb.kademlia.EachBinDescFiltered(base, capKey, 0, func(peerBin *PeerBin) bool { peers := klb.peerBinToPeerList(peerBin) return consumeBin(LBBin{LBPeers: peers, ProximityOrder: peerBin.ProximityOrder}) }) diff --git a/network/kademlialoadbalancer_test.go b/network/kademlialoadbalancer_test.go index fa7f2b30a3..64a6b6b0e6 100644 --- a/network/kademlialoadbalancer_test.go +++ b/network/kademlialoadbalancer_test.go @@ -115,9 +115,9 @@ func TestEachBinFiltered(t *testing.T) { capKey := "42:101" caps[capKey] = capability.NewCapability(42, 3) - caps[capKey].Set(0) - caps[capKey].Set(2) - tk.RegisterCapabilityIndex(capKey, *caps[capKey]) + _ = caps[capKey].Set(0) + _ = caps[capKey].Set(2) + _ = tk.RegisterCapabilityIndex(capKey, *caps[capKey]) capPeer := tk.newTestKadPeerWithCapabilities("10100000", caps[capKey]) tk.Kademlia.On(capPeer) diff --git a/pss/forwarding_test.go b/pss/forwarding_test.go index ea56c489c4..b0c9d3da22 100644 --- a/pss/forwarding_test.go +++ b/pss/forwarding_test.go @@ -9,6 +9,7 @@ import ( ethCrypto "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethersphere/swarm/log" "github.com/ethersphere/swarm/network" "github.com/ethersphere/swarm/p2p/protocols" "github.com/ethersphere/swarm/pot" @@ -128,7 +129,7 @@ func TestForwardBasic(t *testing.T) { } testCases = append(testCases, c) - // test with partial addresses + //test with partial addresses const part = 12 for i := 0; i < firstNearest; i++ { @@ -198,7 +199,7 @@ func TestForwardBasic(t *testing.T) { expected: all[indexAtPo8:], exclusive: false, } - testCases = append(testCases, c) + //testCases = append(testCases, c) // luminous radius of 256 bits, proximity order 8 a4 := pot.Address{} @@ -210,7 +211,7 @@ func TestForwardBasic(t *testing.T) { expected: []int{indexAtPo8, indexAtPo8 + 1}, exclusive: true, } - testCases = append(testCases, c) + //testCases = append(testCases, c) // check correct behaviour in case send fails for i := 2; i < firstNearest-3; i += 2 { @@ -220,7 +221,7 @@ func TestForwardBasic(t *testing.T) { // msg should be received by only one of the deeper peers. a := pot.RandomAddressAt(base, po) c = testCase{ - name: fmt.Sprintf("Send direct to known, id: [%d]", i), + name: fmt.Sprintf("Send direct to known with errors, id: [%d] po=%v", i, po), recipient: a[:], peers: peerAddresses, expected: all[i+1:], @@ -238,6 +239,7 @@ func TestForwardBasic(t *testing.T) { // this function tests the forwarding of a single message. the recipient address is passed as param, // along with addresses of all peers, and indices of those peers which are expected to receive the message. func testForwardMsg(t *testing.T, ps *Pss, c *testCase) { + log.Debug("Test forward msg", "name", c.name) recipientAddr := c.recipient peers := c.peers expected := c.expected diff --git a/pss/pss.go b/pss/pss.go index 974a0728b3..d5c475aefc 100644 --- a/pss/pss.go +++ b/pss/pss.go @@ -202,6 +202,7 @@ func (o outbox) reenqueue(slot int) { type Pss struct { *network.Kademlia // we can get the Kademlia address from this *KeyStore + kademliaLB *network.KademliaLoadBalancer forwardCache *ttlset.TTLSet gcTicker *ticker.Ticker @@ -249,6 +250,7 @@ func New(k *network.Kademlia, params *Params) (*Pss, error) { Kademlia: k, KeyStore: loadKeyStore(), + kademliaLB: network.NewKademliaLoadBalancer(k), privateKey: params.privateKey, quitC: make(chan struct{}), @@ -318,6 +320,7 @@ func (p *Pss) Stop() error { return err } close(p.quitC) + p.kademliaLB.Stop() return nil } @@ -803,28 +806,25 @@ func (p *Pss) forward(msg *message.Message) error { onlySendOnce = true } - p.Kademlia.EachConnLB(to, addressLength*8, func(sp *network.Peer, po int) (continueIt bool, peerUsed bool) { - if po < broadcastThreshold && sent > 0 { - return // stop iterating, peer not used + p.kademliaLB.EachBin(to, func(bin network.LBBin) bool { + if bin.ProximityOrder < broadcastThreshold && sent > 0 { + // This bin is at the same distance as the node to the message. If already sent, we stop sending + return false } - peerUsed = true - if sendFunc(p, sp, msg) { - sent++ - if onlySendOnce { - continueIt = false - // stop iterating peer used - return - } - if po == addressLength*8 { - continueIt = false - // stop iterating if successfully sent to the exact recipient (perfect match of full address) - // peer used - return + for _, lbPeer := range bin.LBPeers { + if sendFunc(p, lbPeer.Peer, msg) { + lbPeer.Use() + sent++ + if onlySendOnce { + return false + } + if bin.ProximityOrder == addressLength*8 { + // stop iterating if successfully sent to the exact recipient (perfect match of full address) + return false //stop iterating + } } } - continueIt = true - // continue iterating, peer used - return + return true }) // cache the message From e9263d722b7bae88885647b628d00ec338a12322 Mon Sep 17 00:00:00 2001 From: Alvaro Date: Thu, 26 Sep 2019 08:32:29 +0200 Subject: [PATCH 08/31] Fixed lint and test when mergin master --- bzzeth/bzzeth_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bzzeth/bzzeth_test.go b/bzzeth/bzzeth_test.go index 92280432ea..97c67a4baa 100644 --- a/bzzeth/bzzeth_test.go +++ b/bzzeth/bzzeth_test.go @@ -221,7 +221,7 @@ func TestBzzBzzHandshakeWithMessage(t *testing.T) { } // after successful handshake, expect peer added to peer pool - p := getPeerAfterConnection + p := getPeerAfterConnection(node.ID(), b) if p == nil { t.Fatal("bzzeth peer not added") } From ed1f9a954450bccbf276cd75f8db3e549e16b40a Mon Sep 17 00:00:00 2001 From: Alvaro Date: Thu, 26 Sep 2019 09:34:02 +0200 Subject: [PATCH 09/31] Subscription to peer changed closed only by writer, not by subscriptors --- network/kademlia.go | 69 +++++++++++++++++++++++++-------------------- 1 file changed, 39 insertions(+), 30 deletions(-) diff --git a/network/kademlia.go b/network/kademlia.go index 619d230f69..d7ad5c4bf1 100644 --- a/network/kademlia.go +++ b/network/kademlia.go @@ -91,15 +91,14 @@ func NewKadParams() *KadParams { type Kademlia struct { lock sync.RWMutex capabilityIndex map[string]*capabilityIndex - defaultIndex *capabilityIndex // Index with pots - *KadParams // Kademlia configuration parameters - base []byte // immutable baseaddress of the table - depth uint8 // stores the last current depth of saturation - nDepth int // stores the last neighbourhood depth - nDepthMu sync.RWMutex // protects neighbourhood depth nDepth - nDepthSig []chan struct{} // signals when neighbourhood depth nDepth is changed - newPeerSig []chan newPeerSignal // signals when new peer is added - removePeerSig []chan *Peer // signals when peer is removed + defaultIndex *capabilityIndex // Index with pots + *KadParams // Kademlia configuration parameters + base []byte // immutable baseaddress of the table + depth uint8 // stores the last current depth of saturation + nDepth int // stores the last neighbourhood depth + nDepthMu sync.RWMutex // protects neighbourhood depth nDepth + nDepthSig []chan struct{} // signals when neighbourhood depth nDepth is changed + peerChangedSig []peerSubscription // signals when new peer is added or removed } type KademliaInfo struct { @@ -132,6 +131,12 @@ func NewKademlia(addr []byte, params *KadParams) *Kademlia { return k } +type peerSubscription struct { + newPeerC chan newPeerSignal + removedPeerC chan *Peer + closed bool +} + type newPeerSignal struct { peer *Peer po int @@ -461,8 +466,14 @@ func (k *Kademlia) On(p *Peer) (uint8, bool) { k.addToCapabilityIndex(p) // notify subscribers asynchronously go func(p *Peer, po int) { - for _, c := range k.newPeerSig { - c <- newPeerSignal{peer: p, po: po} + for _, c := range k.peerChangedSig { + if c.closed { + close(c.newPeerC) + close(c.removedPeerC) + } else { + c.newPeerC <- newPeerSignal{peer: p, po: po} + } + } }(p, po) @@ -576,36 +587,28 @@ func (k *Kademlia) SubscribeToNeighbourhoodDepthChange() (c <-chan struct{}, uns func (k *Kademlia) SubscribeToPeerChanges() (addedC <-chan newPeerSignal, removedC <-chan *Peer, unsubscribe func()) { newPeerC := make(chan newPeerSignal, 1) removedPeerC := make(chan *Peer, 1) - var closeOnce sync.Once k.lock.Lock() defer k.lock.Unlock() - k.newPeerSig = append(k.newPeerSig, newPeerC) - k.removePeerSig = append(k.removePeerSig, removedPeerC) + k.peerChangedSig = append(k.peerChangedSig, peerSubscription{ + newPeerC: newPeerC, + removedPeerC: removedPeerC, + closed: false, + }) unsubscribe = func() { k.lock.Lock() defer k.lock.Unlock() - for i, c := range k.newPeerSig { - if c == newPeerC { - k.newPeerSig = append(k.newPeerSig[:i], k.newPeerSig[i+1:]...) + for i, c := range k.peerChangedSig { + if c.newPeerC == newPeerC { + c.closed = true + k.peerChangedSig = append(k.peerChangedSig[:i], k.peerChangedSig[i+1:]...) break } } - for i, c := range k.removePeerSig { - if c == removedPeerC { - k.removePeerSig = append(k.removePeerSig[:i], k.removePeerSig[i+1:]...) - break - } - } - - closeOnce.Do(func() { - close(newPeerC) - close(removedPeerC) - }) } return newPeerC, removedPeerC, unsubscribe @@ -631,8 +634,14 @@ func (k *Kademlia) Off(p *Peer) { k.removeFromCapabilityIndex(p, true) k.setNeighbourhoodDepth() go func(p *Peer) { - for _, c := range k.removePeerSig { - c <- p + for _, c := range k.peerChangedSig { + if c.closed { + close(c.newPeerC) + close(c.removedPeerC) + } else { + c.removedPeerC <- p + } + } }(p) } From 904e204893a44345c349cb8571615e87e4b82d96 Mon Sep 17 00:00:00 2001 From: Alvaro Date: Fri, 27 Sep 2019 09:58:56 +0200 Subject: [PATCH 10/31] Added an alternative method for initializing a new peer count --- network/kademlialoadbalancer.go | 58 +++++++++++----- network/kademlialoadbalancer_test.go | 99 ++++++++++++++++++++++++++-- pss/pss.go | 2 +- 3 files changed, 137 insertions(+), 22 deletions(-) diff --git a/network/kademlialoadbalancer.go b/network/kademlialoadbalancer.go index 21d8b0feee..f0627dbcee 100644 --- a/network/kademlialoadbalancer.go +++ b/network/kademlialoadbalancer.go @@ -15,10 +15,11 @@ type KademliaBackend interface { BaseAddr() []byte EachBinDesc(base []byte, minProximityOrder int, consumer PeerBinConsumer) EachBinDescFiltered(base []byte, capKey string, minProximityOrder int, consumer PeerBinConsumer) error + EachConn(base []byte, o int, f func(*Peer, int) bool) } // Creates a new KademliaLoadBalancer from a KademliaBackend -func NewKademliaLoadBalancer(kademlia KademliaBackend) *KademliaLoadBalancer { +func NewKademliaLoadBalancer(kademlia KademliaBackend, useMostSimilarInit bool) *KademliaLoadBalancer { chanNewPeerSignals, chanOffPeerSignals, unsubscribe := kademlia.SubscribeToPeerChanges() klb := &KademliaLoadBalancer{ kademlia: kademlia, @@ -28,6 +29,12 @@ func NewKademliaLoadBalancer(kademlia KademliaBackend) *KademliaLoadBalancer { unsubscribeNotifier: unsubscribe, quitC: make(chan struct{}), } + if useMostSimilarInit { + klb.initCountFunc = klb.mostSimilarPeerCount + } else { + klb.initCountFunc = klb.leastUsedCountInBin + } + go klb.listenNewPeers() go klb.listenOffPeers() return klb @@ -64,6 +71,8 @@ type KademliaLoadBalancer struct { offPeerChannel <-chan *Peer //a channel to be notified of removed peers in kademlia unsubscribeNotifier func() //an unsubscribe function provided when subscribe to kademlia notifiers quitC chan struct{} + + initCountFunc func(peer *Peer, po int) int //Function to use for initializing a new peer count } // Stop unsubscribe from notifiers @@ -144,13 +153,13 @@ func (klb *KademliaLoadBalancer) listenOffPeers() { // to the use count of the least used peer in its bin. The po of the new peer is passed to avoid having // to calculate it again. func (klb *KademliaLoadBalancer) addedPeer(peer *Peer, po int) { - leastUsedCount := klb.leastUsedCountInBin(po, peer) - log.Debug("Adding peer", "key", peer.Key()[:4], "leastUsedCount", leastUsedCount) - klb.resourceUseStats.initKey(peer.Key(), leastUsedCount) + initCount := klb.initCountFunc(peer, 0) + log.Debug("Adding peer", "key", peer.Key()[:4], "initCount", initCount) + klb.resourceUseStats.initKey(peer.Key(), initCount) } // leastUsedCountInBin returns the use count for the least used peer in this bin excluding the excludePeer. -func (klb *KademliaLoadBalancer) leastUsedCountInBin(po int, excludePeer *Peer) int { +func (klb *KademliaLoadBalancer) leastUsedCountInBin(excludePeer *Peer, po int) int { addr := klb.kademlia.BaseAddr() peersInSamePo := klb.getPeersForPo(addr, po) idx := 0 @@ -167,12 +176,39 @@ func (klb *KademliaLoadBalancer) leastUsedCountInBin(po int, excludePeer *Peer) return leastUsedCount } +// mostSimilarPeerCount returns the use count for the closest peer count. +func (klb *KademliaLoadBalancer) mostSimilarPeerCount(newPeer *Peer, _ int) int { + var count int + klb.kademlia.EachConn(newPeer.Address(), 255, func(peer *Peer, po int) bool { + if peer != newPeer { + count = klb.resourceUseStats.getUses(peer) + log.Debug("Most similar peer is", "peer", peer.Key()[:4], "count", count) + return false + } + return true + }) + return count +} + + func (klb *KademliaLoadBalancer) removedPeer(peer *Peer) { klb.resourceUseStats.lock.Lock() defer klb.resourceUseStats.lock.Lock() delete(klb.resourceUseStats.resourceUses, peer.Key()) } +func (klb *KademliaLoadBalancer) toLBPeers(resources []Resource) []LBPeer { + peers := make([]LBPeer, len(resources)) + for i, res := range resources { + peer := res.(*Peer) + peers[i].Peer = peer + peers[i].Use = func() { + klb.resourceUseStats.addUse(peer) + } + } + return peers +} + func (klb *KademliaLoadBalancer) getPeersForPo(base []byte, po int) []LBPeer { resources := make([]Resource, 0) klb.kademlia.EachBinDesc(base, po, func(bin *PeerBin) bool { @@ -188,18 +224,6 @@ func (klb *KademliaLoadBalancer) getPeersForPo(base []byte, po int) []LBPeer { return klb.resourcesToLbPeers(resources) } -func (klb *KademliaLoadBalancer) toLBPeers(resources []Resource) []LBPeer { - peers := make([]LBPeer, len(resources)) - for i, res := range resources { - peer := res.(*Peer) - peers[i].Peer = peer - peers[i].Use = func() { - klb.resourceUseStats.addUse(peer) - } - } - return peers -} - // Resource Use Stats // resourceUseStats can be used to count uses of resources. A Resource is anything with a Key() diff --git a/network/kademlialoadbalancer_test.go b/network/kademlialoadbalancer_test.go index 64a6b6b0e6..0081bb5195 100644 --- a/network/kademlialoadbalancer_test.go +++ b/network/kademlialoadbalancer_test.go @@ -10,7 +10,7 @@ import ( "github.com/ethersphere/swarm/pot" ) -// TestAddedNodes tests that when adding a node it is assigned the correct number of uses. +// TestAddedNodes checks that when adding a node it is assigned the correct number of uses. // This number of uses will be the least number of uses of a peer in its bin func TestAddedNodes(t *testing.T) { kademlia := newTestKademliaBackend("11110000") @@ -18,7 +18,7 @@ func TestAddedNodes(t *testing.T) { kademlia.addPeer(first, 0) second := newTestKadPeer("010101011") kademlia.addPeer(second, 0) - klb := NewKademliaLoadBalancer(kademlia) + klb := NewKademliaLoadBalancer(kademlia, false) defer klb.Stop() firstUses := klb.resourceUseStats.getUses(first) @@ -48,6 +48,47 @@ func TestAddedNodes(t *testing.T) { } } +// TestAddedNodesMostSimilar checks that when adding a node it is assigned the correct number of uses. +// This number of uses will be the most similar peer uses. +func TestAddedNodesMostSimilar(t *testing.T) { + kademlia := newTestKademliaBackend("11110000") + first := newTestKadPeer("01010101") + kademlia.addPeer(first, 0) + second := newTestKadPeer("01110101") + kademlia.addPeer(second, 0) + klb := NewKademliaLoadBalancer(kademlia, true) + + defer klb.Stop() + firstUses := klb.resourceUseStats.getUses(first) + if firstUses != 0 { + t.Errorf("Expected 0 uses for new peer at start") + } + peersFor0 := klb.getPeersForPo(kademlia.baseAddr, 0) + peersFor0[0].Use() + // Now third peer should have the same uses as second + third := newTestKadPeer("01110111") // most similar peer is second 01110101 + kademlia.addPeer(third, 0) + klb.resourceUseStats.waitKey(third.Key()) + secondUses := klb.resourceUseStats.getUses(second) + thirdUses := klb.resourceUseStats.getUses(third) + if thirdUses != secondUses { + t.Errorf("Expected %v uses for new peer because is most similar to second. Instead %v", secondUses, thirdUses) + } + peersFor0 = klb.getPeersForPo(kademlia.baseAddr, 0) + peersFor0[0].Use() + peersFor0[1].Use() + //New peers should start with 1 use + fourth := newTestKadPeer("01110110") // most similar peer is third 01110111 + kademlia.addPeer(fourth, 0) + klb.resourceUseStats.waitKey(fourth.Key()) + fourthUses := klb.resourceUseStats.getUses(fourth) + thirdUses = klb.resourceUseStats.getUses(third) + if fourthUses != thirdUses { + t.Errorf("Expected %v use for new peer because most similiar is peer 3. Instead %v", thirdUses, fourthUses) + } + +} + // TestEachBinBaseUses tests that EachBin returns first the least used peer in its bin // We will create 3 bins with two peers each. We will call EachBin 6 times twice with an address // on each bin, so at the end all peers should have 1 use (because the address in each bin is equidistant to @@ -56,7 +97,7 @@ func TestAddedNodes(t *testing.T) { // returned first func TestEachBinBaseUses(t *testing.T) { tk := newTestKademlia(t, "11111111") - klb := NewKademliaLoadBalancer(tk) + klb := NewKademliaLoadBalancer(tk, false) tk.On("01010101") //Peer 1 dec 85 tk.On("01010100") // 2 dec 84 tk.On("10010100") // 3 dec 148 @@ -110,7 +151,7 @@ func TestEachBinBaseUses(t *testing.T) { func TestEachBinFiltered(t *testing.T) { tk := newTestKademlia(t, "11111111") - klb := NewKademliaLoadBalancer(tk) + klb := NewKademliaLoadBalancer(tk, false) caps := make(map[string]*capability.Capability) capKey := "42:101" @@ -167,6 +208,41 @@ type testKademliaBackend struct { removedChannel chan *Peer bins map[int][]*Peer subscribed bool + maxPo int +} + +func (tkb *testKademliaBackend) EachConn(base []byte, maxPo int, consume func(*Peer, int) bool) { + po, _ := Pof(base, tkb.baseAddr, 0) + bin := tkb.bins[po] + peersInBin := make([]*Peer, len(bin)) + copy(peersInBin, bin) + sort.Slice(peersInBin, func(i, j int) bool { + peerIPo, _ := Pof(base, peersInBin[i], 0) + peerJPo, _ := Pof(base, peersInBin[j], 0) + return peerIPo > peerJPo + }) + for _, peer := range peersInBin { + if !consume(peer, po) { + return + } + } + for i := po + 1; po < maxPo ; po++ { + bin = tkb.bins[i] + for _, peer := range bin { + if !consume(peer, po) { + return + } + } + } + for i := po - 1; po >= 0 ; po-- { + bin = tkb.bins[i] + for _, peer := range bin { + if !consume(peer, i) { + return + } + } + } + } func newTestKademliaBackend(address string) *testKademliaBackend { @@ -234,6 +310,9 @@ func (tkb testKademliaBackend) EachBinDesc(_ []byte, minProximityOrder int, cons func (tkb *testKademliaBackend) addPeer(peer *Peer, po int) { if tkb.bins[po] == nil { + if po > tkb.maxPo { + tkb.maxPo = po + } tkb.bins[po] = make([]*Peer, 0) } tkb.bins[po] = append(tkb.bins[po], peer) @@ -251,6 +330,9 @@ func (tkb *testKademliaBackend) removePeer(peer *Peer) { for i, aPeer := range bin { if aPeer == peer { tkb.bins[po] = append(bin[:i], bin[i+1:]...) + if len(tkb.bins[po]) == 0 && tkb.maxPo >= po{ + tkb.updateMaxPo() + } break } } @@ -260,6 +342,15 @@ func (tkb *testKademliaBackend) removePeer(peer *Peer) { } } +func (tkb *testKademliaBackend) updateMaxPo() { + tkb.maxPo = 0 + for k := range tkb.bins { + if k > tkb.maxPo { + tkb.maxPo = k + } + } +} + func newTestKadPeer(s string) *Peer { return NewPeer(&BzzPeer{BzzAddr: testKadPeerAddr(s)}, nil) } diff --git a/pss/pss.go b/pss/pss.go index d3bd6164ee..cf09e48bf0 100644 --- a/pss/pss.go +++ b/pss/pss.go @@ -257,7 +257,7 @@ func New(k *network.Kademlia, params *Params) (*Pss, error) { Kademlia: k, KeyStore: loadKeyStore(), - kademliaLB: network.NewKademliaLoadBalancer(k), + kademliaLB: network.NewKademliaLoadBalancer(k, false), privateKey: params.privateKey, quitC: make(chan struct{}), From e53ae259ef1cba5c216e8ca602e9dbc93268f0b0 Mon Sep 17 00:00:00 2001 From: Alvaro Date: Fri, 27 Sep 2019 10:54:47 +0200 Subject: [PATCH 11/31] go fmt --- network/kademlialoadbalancer.go | 3 +-- network/kademlialoadbalancer_test.go | 8 ++++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/network/kademlialoadbalancer.go b/network/kademlialoadbalancer.go index f0627dbcee..aee3a0bbcd 100644 --- a/network/kademlialoadbalancer.go +++ b/network/kademlialoadbalancer.go @@ -72,7 +72,7 @@ type KademliaLoadBalancer struct { unsubscribeNotifier func() //an unsubscribe function provided when subscribe to kademlia notifiers quitC chan struct{} - initCountFunc func(peer *Peer, po int) int //Function to use for initializing a new peer count + initCountFunc func(peer *Peer, po int) int //Function to use for initializing a new peer count } // Stop unsubscribe from notifiers @@ -190,7 +190,6 @@ func (klb *KademliaLoadBalancer) mostSimilarPeerCount(newPeer *Peer, _ int) int return count } - func (klb *KademliaLoadBalancer) removedPeer(peer *Peer) { klb.resourceUseStats.lock.Lock() defer klb.resourceUseStats.lock.Lock() diff --git a/network/kademlialoadbalancer_test.go b/network/kademlialoadbalancer_test.go index 0081bb5195..e7fe85cfd3 100644 --- a/network/kademlialoadbalancer_test.go +++ b/network/kademlialoadbalancer_test.go @@ -84,7 +84,7 @@ func TestAddedNodesMostSimilar(t *testing.T) { fourthUses := klb.resourceUseStats.getUses(fourth) thirdUses = klb.resourceUseStats.getUses(third) if fourthUses != thirdUses { - t.Errorf("Expected %v use for new peer because most similiar is peer 3. Instead %v", thirdUses, fourthUses) + t.Errorf("Expected %v use for new peer because most similar is peer 3. Instead %v", thirdUses, fourthUses) } } @@ -226,7 +226,7 @@ func (tkb *testKademliaBackend) EachConn(base []byte, maxPo int, consume func(*P return } } - for i := po + 1; po < maxPo ; po++ { + for i := po + 1; po < maxPo; po++ { bin = tkb.bins[i] for _, peer := range bin { if !consume(peer, po) { @@ -234,7 +234,7 @@ func (tkb *testKademliaBackend) EachConn(base []byte, maxPo int, consume func(*P } } } - for i := po - 1; po >= 0 ; po-- { + for i := po - 1; po >= 0; po-- { bin = tkb.bins[i] for _, peer := range bin { if !consume(peer, i) { @@ -330,7 +330,7 @@ func (tkb *testKademliaBackend) removePeer(peer *Peer) { for i, aPeer := range bin { if aPeer == peer { tkb.bins[po] = append(bin[:i], bin[i+1:]...) - if len(tkb.bins[po]) == 0 && tkb.maxPo >= po{ + if len(tkb.bins[po]) == 0 && tkb.maxPo >= po { tkb.updateMaxPo() } break From f82db7f7d9471a8a67a4ecffd9799c0df14af636 Mon Sep 17 00:00:00 2001 From: Alvaro Date: Mon, 30 Sep 2019 09:28:27 +0200 Subject: [PATCH 12/31] extracted pubsub channel to a package --- network/gopubsub/pubsub.go | 124 +++++++++++++++++++++++++++ network/kademlia.go | 92 ++++++-------------- network/kademlialoadbalancer.go | 42 +++++---- network/kademlialoadbalancer_test.go | 27 +++--- 4 files changed, 185 insertions(+), 100 deletions(-) create mode 100644 network/gopubsub/pubsub.go diff --git a/network/gopubsub/pubsub.go b/network/gopubsub/pubsub.go new file mode 100644 index 0000000000..47c875d371 --- /dev/null +++ b/network/gopubsub/pubsub.go @@ -0,0 +1,124 @@ +package gopubsub + +import ( + "fmt" + "strconv" + "sync" +) + +//PubSubChannel represents a pubsub system where subscriber can .Subscribe() and publishers can .Publish() or .Close() +type PubSubChannel struct { + subscriptions []*Subscription + subsMutex sync.RWMutex + nextId int +} + +//Subscription is created in PubSubChannel using pubSub.Ubscribe(). Subscriptors can receive using .ReceiveChannel() +// or .Unsubscribe() +type Subscription struct { + closed bool + removeSub func() + signal chan interface{} + closeOnce sync.Once + id string +} + +//Creates a new PubSubChannel +func New() *PubSubChannel { + return &PubSubChannel{ + subscriptions: make([]*Subscription, 0), + } +} + +//Subscribe to a channel, each subscriptor should keep its own Subscription instance +func (psc *PubSubChannel) Subscribe() *Subscription { + psc.subsMutex.Lock() + defer psc.subsMutex.Unlock() + newSubscription := newSubscription(strconv.Itoa(psc.nextId)) + psc.nextId++ + psc.subscriptions = append(psc.subscriptions, &newSubscription) + newSubscription.removeSub = func() { + psc.subsMutex.Lock() + defer psc.subsMutex.Unlock() + + for i, subscription := range psc.subscriptions { + if subscription.signal == newSubscription.signal { + fmt.Println("Unsubscribing", "id", subscription.id) + subscription.closed = true + psc.subscriptions = append(psc.subscriptions[:i], psc.subscriptions[i+1:]...) + } + } + } + return &newSubscription +} + +//Publish a message and broadcast asynchronously to each subscriptor +func (psc *PubSubChannel) Publish(msg interface{}) { + psc.subsMutex.RLock() + defer psc.subsMutex.RUnlock() + for i, sub := range psc.subscriptions { + if sub.closed { + fmt.Println("Subscription was closed", "id", sub.id) + sub.closeChannel() + } else { + go func(sub *Subscription, index int) { + sub.signal <- msg + }(sub, i) + + } + } +} + +//NumSubscriptions tells how many subcriptions are currently active +func (psc *PubSubChannel) NumSubscriptions() int { + psc.subsMutex.RLock() + defer psc.subsMutex.RUnlock() + return len(psc.subscriptions) +} + +// Close all subscriptions. Usually the publisher is in charge of closing it +func (psc *PubSubChannel) Close() { + psc.subsMutex.Lock() + defer psc.subsMutex.Unlock() + for _, sub := range psc.subscriptions { + sub.closed = true + sub.closeChannel() + } +} + +//Unsubscribe from a subscription +func (sub *Subscription) Unsubscribe() { + sub.closed = true + sub.removeSub() +} + +//ReveiveChannel returns the channel where the subscriptor will receive messages +func (sub *Subscription) ReceiveChannel() <-chan interface{} { + return sub.signal +} + +//IsClosed returns if the subscription is clossed via Unsubscribe() or Close() in the pubSub that creates it +func (sub *Subscription) IsClosed() bool { + return sub.closed +} + +//Returns the ID of the subscription. Useful for debugging +func (sub *Subscription) ID() string { + return sub.id +} + +func (sub *Subscription) closeChannel() { + sub.closeOnce.Do(func() { + close(sub.signal) + }) +} + +func newSubscription(id string) Subscription { + return Subscription{ + closed: false, + removeSub: nil, + signal: make(chan interface{}), + closeOnce: sync.Once{}, + id: id, + } +} diff --git a/network/kademlia.go b/network/kademlia.go index d7ad5c4bf1..cff75730f6 100644 --- a/network/kademlia.go +++ b/network/kademlia.go @@ -21,6 +21,7 @@ import ( "encoding/hex" "errors" "fmt" + "math/rand" "sort" "strings" @@ -32,6 +33,7 @@ import ( "github.com/ethersphere/swarm/chunk" "github.com/ethersphere/swarm/log" "github.com/ethersphere/swarm/network/capability" + "github.com/ethersphere/swarm/network/gopubsub" "github.com/ethersphere/swarm/pot" sv "github.com/ethersphere/swarm/version" ) @@ -91,14 +93,16 @@ func NewKadParams() *KadParams { type Kademlia struct { lock sync.RWMutex capabilityIndex map[string]*capabilityIndex - defaultIndex *capabilityIndex // Index with pots - *KadParams // Kademlia configuration parameters - base []byte // immutable baseaddress of the table - depth uint8 // stores the last current depth of saturation - nDepth int // stores the last neighbourhood depth - nDepthMu sync.RWMutex // protects neighbourhood depth nDepth - nDepthSig []chan struct{} // signals when neighbourhood depth nDepth is changed - peerChangedSig []peerSubscription // signals when new peer is added or removed + defaultIndex *capabilityIndex // Index with pots + *KadParams // Kademlia configuration parameters + base []byte // immutable baseaddress of the table + depth uint8 // stores the last current depth of saturation + nDepth int // stores the last neighbourhood depth + nDepthMu sync.RWMutex // protects neighbourhood depth nDepth + nDepthSig []chan struct{} // signals when neighbourhood depth nDepth is changed + + newPeerPubSub *gopubsub.PubSubChannel + removedPeerPubSub *gopubsub.PubSubChannel } type KademliaInfo struct { @@ -121,22 +125,18 @@ func NewKademlia(addr []byte, params *KadParams) *Kademlia { params.Capabilities = capability.NewCapabilities() } k := &Kademlia{ - base: addr, - KadParams: params, - capabilityIndex: make(map[string]*capabilityIndex), - defaultIndex: NewDefaultIndex(), + base: addr, + KadParams: params, + capabilityIndex: make(map[string]*capabilityIndex), + defaultIndex: NewDefaultIndex(), + newPeerPubSub: gopubsub.New(), + removedPeerPubSub: gopubsub.New(), } k.RegisterCapabilityIndex("full", *fullCapability) k.RegisterCapabilityIndex("light", *lightCapability) return k } -type peerSubscription struct { - newPeerC chan newPeerSignal - removedPeerC chan *Peer - closed bool -} - type newPeerSignal struct { peer *Peer po int @@ -465,17 +465,7 @@ func (k *Kademlia) On(p *Peer) (uint8, bool) { }) k.addToCapabilityIndex(p) // notify subscribers asynchronously - go func(p *Peer, po int) { - for _, c := range k.peerChangedSig { - if c.closed { - close(c.newPeerC) - close(c.removedPeerC) - } else { - c.newPeerC <- newPeerSignal{peer: p, po: po} - } - - } - }(p, po) + k.newPeerPubSub.Publish(newPeerSignal{peer: p, po: po}) if ins { a := newEntryFromBzzAddress(p.BzzAddr) @@ -584,34 +574,10 @@ func (k *Kademlia) SubscribeToNeighbourhoodDepthChange() (c <-chan struct{}, uns // when a new Peer is added or removed from the table. Returned function unsubscribes // the channel from signaling and releases the resources. Returned function is safe // to be called multiple times. -func (k *Kademlia) SubscribeToPeerChanges() (addedC <-chan newPeerSignal, removedC <-chan *Peer, unsubscribe func()) { - newPeerC := make(chan newPeerSignal, 1) - removedPeerC := make(chan *Peer, 1) - - k.lock.Lock() - defer k.lock.Unlock() - - k.peerChangedSig = append(k.peerChangedSig, peerSubscription{ - newPeerC: newPeerC, - removedPeerC: removedPeerC, - closed: false, - }) - - unsubscribe = func() { - k.lock.Lock() - defer k.lock.Unlock() - - for i, c := range k.peerChangedSig { - if c.newPeerC == newPeerC { - c.closed = true - k.peerChangedSig = append(k.peerChangedSig[:i], k.peerChangedSig[i+1:]...) - break - } - } - - } - - return newPeerC, removedPeerC, unsubscribe +func (k *Kademlia) SubscribeToPeerChanges() (addedSub *gopubsub.Subscription, removedPeerSub *gopubsub.Subscription) { + addedSub = k.newPeerPubSub.Subscribe() + removedPeerSub = k.removedPeerPubSub.Subscribe() + return } // Off removes a peer from among live peers @@ -633,17 +599,7 @@ func (k *Kademlia) Off(p *Peer) { }) k.removeFromCapabilityIndex(p, true) k.setNeighbourhoodDepth() - go func(p *Peer) { - for _, c := range k.peerChangedSig { - if c.closed { - close(c.newPeerC) - close(c.removedPeerC) - } else { - c.removedPeerC <- p - } - - } - }(p) + k.removedPeerPubSub.Publish(p) } // EachConnFiltered performs the same action as EachConn diff --git a/network/kademlialoadbalancer.go b/network/kademlialoadbalancer.go index aee3a0bbcd..8de43157f0 100644 --- a/network/kademlialoadbalancer.go +++ b/network/kademlialoadbalancer.go @@ -7,11 +7,13 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethersphere/swarm/log" + + "github.com/ethersphere/swarm/network/gopubsub" ) // KademliaBackend is the required interface of KademliaLoadBalancer. type KademliaBackend interface { - SubscribeToPeerChanges() (addedC <-chan newPeerSignal, removedC <-chan *Peer, unsubscribe func()) + SubscribeToPeerChanges() (addedSub *gopubsub.Subscription, removedPeerSub *gopubsub.Subscription) BaseAddr() []byte EachBinDesc(base []byte, minProximityOrder int, consumer PeerBinConsumer) EachBinDescFiltered(base []byte, capKey string, minProximityOrder int, consumer PeerBinConsumer) error @@ -20,14 +22,13 @@ type KademliaBackend interface { // Creates a new KademliaLoadBalancer from a KademliaBackend func NewKademliaLoadBalancer(kademlia KademliaBackend, useMostSimilarInit bool) *KademliaLoadBalancer { - chanNewPeerSignals, chanOffPeerSignals, unsubscribe := kademlia.SubscribeToPeerChanges() + newPeerSub, offPeerSub := kademlia.SubscribeToPeerChanges() klb := &KademliaLoadBalancer{ - kademlia: kademlia, - resourceUseStats: newResourceLoadBalancer(), - newPeerChannel: chanNewPeerSignals, - offPeerChannel: chanOffPeerSignals, - unsubscribeNotifier: unsubscribe, - quitC: make(chan struct{}), + kademlia: kademlia, + resourceUseStats: newResourceLoadBalancer(), + newPeerSub: newPeerSub, + offPeerSub: offPeerSub, + quitC: make(chan struct{}), } if useMostSimilarInit { klb.initCountFunc = klb.mostSimilarPeerCount @@ -65,19 +66,19 @@ type LBBinConsumer func(bin LBBin) bool // The user of KademliaLoadBalancer should signal if the returned element (LBPeer) has been used with the // function lbPeer.Use() type KademliaLoadBalancer struct { - kademlia KademliaBackend //kademlia to obtain bins of peers - resourceUseStats *resourceUseStats //a resourceUseStats to count uses - newPeerChannel <-chan newPeerSignal //a channel to be notified of new peers in kademlia - offPeerChannel <-chan *Peer //a channel to be notified of removed peers in kademlia - unsubscribeNotifier func() //an unsubscribe function provided when subscribe to kademlia notifiers - quitC chan struct{} + kademlia KademliaBackend //kademlia to obtain bins of peers + resourceUseStats *resourceUseStats //a resourceUseStats to count uses + newPeerSub *gopubsub.Subscription //a pubsub channel to be notified of new peers in kademlia + offPeerSub *gopubsub.Subscription //a pubsub channel to be notified of removed peers in kademlia + quitC chan struct{} initCountFunc func(peer *Peer, po int) int //Function to use for initializing a new peer count } // Stop unsubscribe from notifiers func (klb KademliaLoadBalancer) Stop() { - klb.unsubscribeNotifier() + klb.newPeerSub.Unsubscribe() + klb.offPeerSub.Unsubscribe() close(klb.quitC) } @@ -127,7 +128,11 @@ func (klb *KademliaLoadBalancer) listenNewPeers() { select { case <-klb.quitC: return - case signal, ok := <-klb.newPeerChannel: + case msg, ok := <-klb.newPeerSub.ReceiveChannel(): + if !ok { + return + } + signal, ok := msg.(newPeerSignal) if !ok { return } @@ -141,8 +146,9 @@ func (klb *KademliaLoadBalancer) listenOffPeers() { select { case <-klb.quitC: return - case peer := <-klb.offPeerChannel: - if peer != nil { + case msg := <-klb.offPeerSub.ReceiveChannel(): + peer, ok := msg.(*Peer) + if peer != nil && ok { klb.removedPeer(peer) } } diff --git a/network/kademlialoadbalancer_test.go b/network/kademlialoadbalancer_test.go index e7fe85cfd3..807d50f84c 100644 --- a/network/kademlialoadbalancer_test.go +++ b/network/kademlialoadbalancer_test.go @@ -8,6 +8,8 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethersphere/swarm/network/capability" "github.com/ethersphere/swarm/pot" + + "github.com/ethersphere/swarm/network/gopubsub" ) // TestAddedNodes checks that when adding a node it is assigned the correct number of uses. @@ -204,8 +206,8 @@ func TestEachBinFiltered(t *testing.T) { type testKademliaBackend struct { baseAddr []byte - addedChannel chan newPeerSignal - removedChannel chan *Peer + addedChannel *gopubsub.PubSubChannel + removedChannel *gopubsub.PubSubChannel bins map[int][]*Peer subscribed bool maxPo int @@ -248,8 +250,8 @@ func (tkb *testKademliaBackend) EachConn(base []byte, maxPo int, consume func(*P func newTestKademliaBackend(address string) *testKademliaBackend { return &testKademliaBackend{ baseAddr: pot.NewAddressFromString(address), - addedChannel: make(chan newPeerSignal, 1), - removedChannel: make(chan *Peer, 1), + addedChannel: gopubsub.New(), + removedChannel: gopubsub.New(), bins: make(map[int][]*Peer), } } @@ -258,14 +260,11 @@ func (tkb testKademliaBackend) BaseAddr() []byte { return tkb.baseAddr } -func (tkb *testKademliaBackend) SubscribeToPeerChanges() (addedC <-chan newPeerSignal, removedC <-chan *Peer, unsubscribe func()) { - unsubscribe = func() { - tkb.subscribed = false - close(tkb.addedChannel) - close(tkb.removedChannel) - } +func (tkb *testKademliaBackend) SubscribeToPeerChanges() (addedSub *gopubsub.Subscription, removedPeerSub *gopubsub.Subscription) { + addedSub = tkb.addedChannel.Subscribe() + removedPeerSub = tkb.removedChannel.Subscribe() tkb.subscribed = true - return tkb.addedChannel, tkb.removedChannel, unsubscribe + return } func (tkb testKademliaBackend) EachBinDescFiltered(base []byte, capKey string, minProximityOrder int, consumer PeerBinConsumer) error { @@ -317,10 +316,10 @@ func (tkb *testKademliaBackend) addPeer(peer *Peer, po int) { } tkb.bins[po] = append(tkb.bins[po], peer) if tkb.subscribed { - tkb.addedChannel <- newPeerSignal{ + tkb.addedChannel.Publish(newPeerSignal{ peer: peer, po: po, - } + }) } time.Sleep(100 * time.Millisecond) } @@ -338,7 +337,7 @@ func (tkb *testKademliaBackend) removePeer(peer *Peer) { } } if tkb.subscribed { - tkb.removedChannel <- peer + tkb.removedChannel.Publish(peer) } } From 0e346e69a150b8c84d643d243db2cf671d3d0831 Mon Sep 17 00:00:00 2001 From: Alvaro Date: Tue, 1 Oct 2019 11:25:25 +0200 Subject: [PATCH 13/31] network: fixed pr comments --- network/gopubsub/pubsub.go | 24 +++++++++++++----------- network/kademlia.go | 23 ++++++++++++++++------- 2 files changed, 29 insertions(+), 18 deletions(-) diff --git a/network/gopubsub/pubsub.go b/network/gopubsub/pubsub.go index 47c875d371..eb39000d18 100644 --- a/network/gopubsub/pubsub.go +++ b/network/gopubsub/pubsub.go @@ -6,14 +6,14 @@ import ( "sync" ) -//PubSubChannel represents a pubsub system where subscriber can .Subscribe() and publishers can .Publish() or .Close() +//PubSubChannel represents a pubsub system where subscriber can .Subscribe() and publishers can .Publish() or .Close(). type PubSubChannel struct { subscriptions []*Subscription subsMutex sync.RWMutex nextId int } -//Subscription is created in PubSubChannel using pubSub.Ubscribe(). Subscriptors can receive using .ReceiveChannel() +//Subscription is created in PubSubChannel using pubSub.Subscribe(). Subscribers can receive using .ReceiveChannel(). // or .Unsubscribe() type Subscription struct { closed bool @@ -23,14 +23,14 @@ type Subscription struct { id string } -//Creates a new PubSubChannel +//New creates a new PubSubChannel. func New() *PubSubChannel { return &PubSubChannel{ subscriptions: make([]*Subscription, 0), } } -//Subscribe to a channel, each subscriptor should keep its own Subscription instance +//Subscribe creates a subscription to a channel, each subscriber should keep its own Subscription instance. func (psc *PubSubChannel) Subscribe() *Subscription { psc.subsMutex.Lock() defer psc.subsMutex.Unlock() @@ -52,7 +52,8 @@ func (psc *PubSubChannel) Subscribe() *Subscription { return &newSubscription } -//Publish a message and broadcast asynchronously to each subscriptor +//Publish broadcasts a message asynchronously to each subscriber. +//If some of the subscriptions(channels) has been marked as closeable, it does it now. func (psc *PubSubChannel) Publish(msg interface{}) { psc.subsMutex.RLock() defer psc.subsMutex.RUnlock() @@ -69,14 +70,15 @@ func (psc *PubSubChannel) Publish(msg interface{}) { } } -//NumSubscriptions tells how many subcriptions are currently active +//NumSubscriptions returns how many subscriptions are currently active. func (psc *PubSubChannel) NumSubscriptions() int { psc.subsMutex.RLock() defer psc.subsMutex.RUnlock() return len(psc.subscriptions) } -// Close all subscriptions. Usually the publisher is in charge of closing it +//Close cancels all subscriptions closing the channels associated with them. +//Usually the publisher is in charge of calling Close(). func (psc *PubSubChannel) Close() { psc.subsMutex.Lock() defer psc.subsMutex.Unlock() @@ -86,23 +88,23 @@ func (psc *PubSubChannel) Close() { } } -//Unsubscribe from a subscription +//Unsubscribe cancels subscription from the subscriber side. Channel is marked as closed but only writer should close it. func (sub *Subscription) Unsubscribe() { sub.closed = true sub.removeSub() } -//ReveiveChannel returns the channel where the subscriptor will receive messages +//ReceiveChannel returns the channel where the subscriber will receive messages. func (sub *Subscription) ReceiveChannel() <-chan interface{} { return sub.signal } -//IsClosed returns if the subscription is clossed via Unsubscribe() or Close() in the pubSub that creates it +//IsClosed returns if the subscription is closed via Unsubscribe() or Close() in the pubSub that creates it. func (sub *Subscription) IsClosed() bool { return sub.closed } -//Returns the ID of the subscription. Useful for debugging +//ID returns a unique id in the PubSubChannel of this subscription. Useful for debugging. func (sub *Subscription) ID() string { return sub.id } diff --git a/network/kademlia.go b/network/kademlia.go index cff75730f6..72b1710335 100644 --- a/network/kademlia.go +++ b/network/kademlia.go @@ -639,21 +639,37 @@ func (k *Kademlia) eachConn(base []byte, db *pot.Pot, o int, f func(*Peer, int) }) } +//In order to clarify iterator functions, we have created several functions types to identify the purpose of each +//param to those functions. + +//PeerConsumer consumes a peer entry in a PeerIterator. The function should return true if it wishes to continue iterating. type PeerConsumer func(entry *entry) bool + +//PeerIterator receives a PeerConsumer and iterates over peer entry until some of the executions of PeerConsumer returns +//false or the entries run out. It returns the last value returned by the last PeerConsumer execution. type PeerIterator func(PeerConsumer) bool + +//PeerBin represents a bin in the Kademlia table. Contains a PeerIterator to traverse the peer entries inside it. type PeerBin struct { ProximityOrder int Size int PeerIterator PeerIterator } + +//PeerBinConsumer consumes a peerBin. It should return true if it wishes to continue iterating bins. type PeerBinConsumer func(peerBin *PeerBin) bool +//Traverse bins (PeerBin) in descending order of proximity (so closest first) with respect to a given address base. +//It will stop iterating whenever the supplied consumer returns false, the bins run out or a bin is found with proximity +//order less than minProximityOrder param. func (k *Kademlia) EachBinDesc(base []byte, minProximityOrder int, consumer PeerBinConsumer) { k.lock.RLock() defer k.lock.RUnlock() k.eachBinDesc(k.defaultIndex, base, minProximityOrder, consumer) } +//Traverse bins in descending order filtered by capabilities. Sane as EachBinDesc but taking into account only peers +//with those capabilities. func (k *Kademlia) EachBinDescFiltered(base []byte, capKey string, minProximityOrder int, consumer PeerBinConsumer) error { k.lock.RLock() defer k.lock.RUnlock() @@ -679,13 +695,6 @@ func (k *Kademlia) eachBinDesc(index *capabilityIndex, base []byte, minProximity }, false) } -type LBPeerConsumer func(peer *Peer, po int) (bool, bool) -type LBPeerIterator func(consumer LBPeerConsumer) bool -type LBLapStatus struct { - oneFound bool - anySkip bool -} - // EachAddrFiltered performs the same action as EachAddr // with the difference that it will only return peers that matches the specified capability index filter func (k *Kademlia) EachAddrFiltered(base []byte, capKey string, o int, f func(*BzzAddr, int) bool) error { From 4c81a2435b9730b3d78319c9efb71adb862aa133 Mon Sep 17 00:00:00 2001 From: Alvaro Date: Mon, 14 Oct 2019 14:45:42 +0200 Subject: [PATCH 14/31] network/kademlia: Pr comments, tests commented and fixed --- network/discovery.go | 8 ++++ network/kademlialoadbalancer.go | 23 +++++------- network/kademlialoadbalancer_test.go | 26 ++++++++----- pot/pot.go | 14 +++++-- pot/pot_test.go | 55 ++++++++++++++++++++-------- pss/pss.go | 2 +- 6 files changed, 83 insertions(+), 45 deletions(-) diff --git a/network/discovery.go b/network/discovery.go index 31b5f2a086..5252773fc7 100644 --- a/network/discovery.go +++ b/network/discovery.go @@ -21,6 +21,7 @@ import ( "fmt" "sync" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/rlp" "github.com/ethersphere/swarm/pot" ) @@ -37,6 +38,7 @@ type Peer struct { mtx sync.RWMutex // peers map[string]bool // tracks node records sent to the peer depth uint8 // the proximity order advertised by remote as depth of saturation + key string // peer key. Hex form of Address() } // NewPeer constructs a discovery peer @@ -45,6 +47,7 @@ func NewPeer(p *BzzPeer, kad *Kademlia) *Peer { kad: kad, BzzPeer: p, peers: make(map[string]bool), + key: hexutil.Encode(p.Address()), } // record remote as seen so we never send a peer its own record d.seen(p.BzzAddr) @@ -66,6 +69,11 @@ func (d *Peer) HandleMsg(ctx context.Context, msg interface{}) error { } } +// Key returns a string representation of this peer to be used in maps. +func (d *Peer) Key() string { + return d.key +} + // NotifyDepth sends a message to all connections if depth of saturation is changed func NotifyDepth(depth uint8, kad *Kademlia) { f := func(val *Peer, po int) bool { diff --git a/network/kademlialoadbalancer.go b/network/kademlialoadbalancer.go index 8de43157f0..4116a1f83d 100644 --- a/network/kademlialoadbalancer.go +++ b/network/kademlialoadbalancer.go @@ -31,7 +31,7 @@ func NewKademliaLoadBalancer(kademlia KademliaBackend, useMostSimilarInit bool) quitC: make(chan struct{}), } if useMostSimilarInit { - klb.initCountFunc = klb.mostSimilarPeerCount + klb.initCountFunc = klb.nearestNeighbourCount } else { klb.initCountFunc = klb.leastUsedCountInBin } @@ -56,7 +56,7 @@ type LBBin struct { ProximityOrder int } -// LBBinConsumer will be provided with a list of LBPeer's usually in LB criteria ordering +// LBBinConsumer will be provided with a list of LBPeer's in LB criteria ordering (currently in least used ordering). type LBBinConsumer func(bin LBBin) bool // KademliaLoadBalancer struct and methods @@ -82,9 +82,9 @@ func (klb KademliaLoadBalancer) Stop() { close(klb.quitC) } -// EachBinNodeAddress calls EachBin with the base address of kademlia (the node address) +// EachBinNodeAddress calls EachBinDesc with the base address of kademlia (the node address) func (klb KademliaLoadBalancer) EachBinNodeAddress(consumeBin LBBinConsumer) { - klb.EachBin(klb.kademlia.BaseAddr(), consumeBin) + klb.EachBinDesc(klb.kademlia.BaseAddr(), consumeBin) } // EachBinFiltered returns all bins in descending order from the perspective of base address. @@ -97,9 +97,9 @@ func (klb KademliaLoadBalancer) EachBinFiltered(base []byte, capKey string, cons }) } -// EachBin returns all bins in descending order from the perspective of base address. +// EachBinDesc returns all bins in descending order from the perspective of base address. // All peers in that bin will be provided to the LBBinConsumer sorted by least used first. -func (klb KademliaLoadBalancer) EachBin(base []byte, consumeBin LBBinConsumer) { +func (klb KademliaLoadBalancer) EachBinDesc(base []byte, consumeBin LBBinConsumer) { klb.kademlia.EachBinDesc(base, 0, func(peerBin *PeerBin) bool { peers := klb.peerBinToPeerList(peerBin) return consumeBin(LBBin{LBPeers: peers, ProximityOrder: peerBin.ProximityOrder}) @@ -182,13 +182,13 @@ func (klb *KademliaLoadBalancer) leastUsedCountInBin(excludePeer *Peer, po int) return leastUsedCount } -// mostSimilarPeerCount returns the use count for the closest peer count. -func (klb *KademliaLoadBalancer) mostSimilarPeerCount(newPeer *Peer, _ int) int { +// nearestNeighbourCount returns the use count for the closest peer count. +func (klb *KademliaLoadBalancer) nearestNeighbourCount(newPeer *Peer, _ int) int { var count int klb.kademlia.EachConn(newPeer.Address(), 255, func(peer *Peer, po int) bool { if peer != newPeer { count = klb.resourceUseStats.getUses(peer) - log.Debug("Most similar peer is", "peer", peer.Key()[:4], "count", count) + log.Debug("Nearest neighbour is", "peer", peer.Key()[:4], "count", count) return false } return true @@ -242,11 +242,6 @@ type Resource interface { Key() string } -// Adding Resource interface to Peer -func (d *Peer) Key() string { - return hexutil.Encode(d.Address()) -} - type ResourceCount struct { resource Resource count int diff --git a/network/kademlialoadbalancer_test.go b/network/kademlialoadbalancer_test.go index 807d50f84c..d327e919fb 100644 --- a/network/kademlialoadbalancer_test.go +++ b/network/kademlialoadbalancer_test.go @@ -91,8 +91,8 @@ func TestAddedNodesMostSimilar(t *testing.T) { } -// TestEachBinBaseUses tests that EachBin returns first the least used peer in its bin -// We will create 3 bins with two peers each. We will call EachBin 6 times twice with an address +// TestEachBinBaseUses tests that EachBinDesc returns first the least used peer in its bin +// We will create 3 bins with two peers each. We will call EachBinDesc 6 times twice with an address // on each bin, so at the end all peers should have 1 use (because the address in each bin is equidistant to // the peers in that bin). // Then wi will use an address in a bin that is nearer one of the peers and we will check that that peer is always @@ -115,16 +115,16 @@ func TestEachBinBaseUses(t *testing.T) { return false } // Use peer 1 and 2 - klb.EachBin(pivotAddressBin0, countUse) - klb.EachBin(pivotAddressBin0, countUse) + klb.EachBinDesc(pivotAddressBin0, countUse) + klb.EachBinDesc(pivotAddressBin0, countUse) // Use peers 3 and 4 - klb.EachBin(pivotAddressBin1, countUse) - klb.EachBin(pivotAddressBin1, countUse) + klb.EachBinDesc(pivotAddressBin1, countUse) + klb.EachBinDesc(pivotAddressBin1, countUse) // Use peers 5 and 6 - klb.EachBin(pivotAddressBin2, countUse) - klb.EachBin(pivotAddressBin2, countUse) + klb.EachBinDesc(pivotAddressBin2, countUse) + klb.EachBinDesc(pivotAddressBin2, countUse) resourceUses := klb.resourceUseStats.dumpAllUses() if len(resourceUses) != 6 { @@ -142,8 +142,8 @@ func TestEachBinBaseUses(t *testing.T) { pivotAddressBin3 := pot.NewAddressFromString("10010011") // Nearer 4 //Both calls to 4 - klb.EachBin(pivotAddressBin3, countUse) - klb.EachBin(pivotAddressBin3, countUse) + klb.EachBinDesc(pivotAddressBin3, countUse) + klb.EachBinDesc(pivotAddressBin3, countUse) count := klb.resourceUseStats.getKeyUses(stringBinaryToHex("10010001")) if count != 3 { @@ -213,6 +213,7 @@ type testKademliaBackend struct { maxPo int } +// EachConn iterates this test kademlia table bins in order from furthest to nearest. func (tkb *testKademliaBackend) EachConn(base []byte, maxPo int, consume func(*Peer, int) bool) { po, _ := Pof(base, tkb.baseAddr, 0) bin := tkb.bins[po] @@ -267,11 +268,14 @@ func (tkb *testKademliaBackend) SubscribeToPeerChanges() (addedSub *gopubsub.Sub return } +// EachBinDescFiltered ignores capKey as in this test context it won't be used. func (tkb testKademliaBackend) EachBinDescFiltered(base []byte, capKey string, minProximityOrder int, consumer PeerBinConsumer) error { tkb.EachBinDesc(base, minProximityOrder, consumer) return nil } +// EachBinDesc iterates bin in the table in descending po order (from nearest to furthest). Base is always supposed to be +// node address in this test context. func (tkb testKademliaBackend) EachBinDesc(_ []byte, minProximityOrder int, consumer PeerBinConsumer) { type poPeers struct { po int @@ -321,6 +325,8 @@ func (tkb *testKademliaBackend) addPeer(peer *Peer, po int) { po: po, }) } + // As the subscribers to add peer are asynchronous, we will sleep here to allow them to execute. + // Because this will be used in tests several times, we wait here so the code is not polluted with Sleep calls. time.Sleep(100 * time.Millisecond) } diff --git a/pot/pot.go b/pot/pot.go index 008efbdbee..4a7be01c1e 100644 --- a/pot/pot.go +++ b/pot/pot.go @@ -591,7 +591,12 @@ func (t *Pot) eachBin(pivotVal Val, pof Pof, minProximityOrder int, consumeBin B } -// eachBinDesc traverse bins in descending po order (nearest to farthest). Returns if the user wants to continue iterating +// eachBinDesc traverse bins in descending po order (nearest to farthest). Returns if the user wants to continue iterating. +// Bins are iterated in the inverse order of eachBin: +// 1 - Pin of the pot if pivotVal is closer than any other sub bin. +// 2 - Then the bin (recursively) where the pivotVal belongs if any. +// 3 - Then all the bins closer than the pivotVal bin will be joined into one big bin with the po of the base. +// 4 - Then, the further bins to pivotVal in descending order. func (t *Pot) eachBinDesc(pivotVal Val, pof Pof, minProximityOrder int, consumeBin BinConsumer) bool { if t == nil || t.size == 0 { return false @@ -601,7 +606,7 @@ func (t *Pot) eachBinDesc(pivotVal Val, pof Pof, minProximityOrder int, consumeB var subPot *Pot // If pivotBinIndex == len(t.bins), the pivotVal is the t.pin. We consume a virtual bin with max valProximityOrder - // and only one element. + // and only one element (Step 1 above). if pivotBinIndex == len(t.bins) { if valProximityOrder >= minProximityOrder { bin := &Bin{ @@ -619,6 +624,7 @@ func (t *Pot) eachBinDesc(pivotVal Val, pof Pof, minProximityOrder int, consumeB } else { // pivotVal is anywhere on the subtree subPot = t.bins[pivotBinIndex] // Consume bin where the pivotVal is, there we will have closest bins and t.pin that will have valProximityOrder + // (Step 2 above). if subPot.po == valProximityOrder { if !subPot.eachBinDesc(pivotVal, pof, minProximityOrder, consumeBin) { return false @@ -636,7 +642,7 @@ func (t *Pot) eachBinDesc(pivotVal Val, pof Pof, minProximityOrder int, consumeB size += t.bins[i].size } // Consuming all bins after the bin where the pivotVal is - // (All bins will be provided to the user as one virtual bin with po = valProximityOrder) + // (All bins will be provided to the user as one virtual bin with po = valProximityOrder). (Step 3 above). if valProximityOrder >= minProximityOrder { bin := &Bin{ ProximityOrder: valProximityOrder, @@ -652,7 +658,7 @@ func (t *Pot) eachBinDesc(pivotVal Val, pof Pof, minProximityOrder int, consumeB } // Finally we will consume all bins before the pivotVal bin (or all bins if the pivotVal is the t.pin) - // Always filtering bins with proximityOrder < minProximityOrder + // Always filtering bins with proximityOrder < minProximityOrder (Step 4 above). for i := pivotBinIndex - 1; i >= 0; i-- { subPot = t.bins[i] if subPot.po < minProximityOrder { diff --git a/pot/pot_test.go b/pot/pot_test.go index de3636f9fe..d7d58e51cd 100644 --- a/pot/pot_test.go +++ b/pot/pot_test.go @@ -591,53 +591,76 @@ func TestPotEachNeighbourAsync(t *testing.T) { } } +// TestEachBinDesc adds peer to a pot and checks that the iteration of bin is done in descending po order. func TestEachBinDesc(t *testing.T) { pof := DefaultPof(8) baseAddr := newTestAddr("11111111", 0) pot := NewPot(baseAddr, 0) pot, _, _ = testAdd(pot, pof, 1, "01111111", "01000000", "10111111", "11011111", "11101111") + pivotAddr := baseAddr + lastBin := 256 // Max po binConsumer := func(bin *Bin) bool { - fmt.Println("Bin", "pot", bin.ProximityOrder, "size", bin.Size) + log.Debug("Bin", "pot", bin.ProximityOrder, "size", bin.Size) + // Checking correct bin po + if bin.ProximityOrder > lastBin { + t.Errorf("Incorrect desc sorting of bins, last po: %v, current po: %v", lastBin, bin.ProximityOrder) + } + lastBin = bin.ProximityOrder + // Checking correct value po for all values in this bin bin.ValIterator(func(val Val) bool { addr := val.(*testAddr) - fmt.Println(" val", toBinaryByte(addr), "pot", bin.ProximityOrder) + log.Debug(" val", toBinaryByte(addr), "pot", bin.ProximityOrder) + valPo, _ := pof(pivotAddr, val, 0) + if valPo != bin.ProximityOrder { + t.Errorf("Incorrect value found in bin. Expected po %v, but was %v", bin.ProximityOrder, valPo) + } return true }) return true } - fmt.Println("****************Forward order****************") - pot.eachBin(pot.pin, pof, 0, binConsumer) - fmt.Println("****************Reverse order****************") + // First we iterate pivoting over the base address (pot.pin) + log.Debug("****************Reverse order****************") pot.eachBinDesc(pot.pin, pof, 0, binConsumer) //Test eachBinDesc for a given address. For example one address with po 2 with respect to t.pin - pivotAddr := newTestAddr("11010000", 6) - fmt.Println("****************Forward order with pivot 11010000****************") - pot.eachBin(pivotAddr, pof, 0, binConsumer) - fmt.Println("****************Reverse order with pivot 11010000****************") + pivotAddr = newTestAddr("11010000", 6) + //Reset back lastBin to 256 + lastBin = 256 + log.Debug("****************Reverse order with pivot 11010000****************") pot.eachBinDesc(pivotAddr, pof, 0, binConsumer) } +// TestEachBinDescPivotInAMissingBin checks that the sorting of bins is correct when the pivotVal po lies in a missing +// bin below the pin address. This methods completes the coverage on the eachBinDesc method. func TestEachBinDescPivotInAMissingBin(t *testing.T) { pof := DefaultPof(8) baseAddr := newTestAddr("11111111", 0) pot := NewPot(baseAddr, 0) pot, _, _ = testAdd(pot, pof, 1, "01111111", "01000000", "10111111", "11101111", "11101011", "11111110") + //Test eachBinDesc for an address with a po that we don't have a bin for + pivotAddr := newTestAddr("11010000", 7) + lastBin := 256 // Max po binConsumer := func(bin *Bin) bool { - fmt.Println("Bin", "pot", bin.ProximityOrder, "size", bin.Size) + log.Debug("Bin", "pot", bin.ProximityOrder, "size", bin.Size) + // Checking correct bin po + if bin.ProximityOrder > lastBin { + t.Errorf("Incorrect desc sorting of bins, last po: %v, current po: %v", lastBin, bin.ProximityOrder) + } + lastBin = bin.ProximityOrder + // Checking correct value po for all values in this bin bin.ValIterator(func(val Val) bool { addr := val.(*testAddr) - fmt.Println(" val", toBinaryByte(addr), "pot", bin.ProximityOrder) + log.Debug(" val", toBinaryByte(addr), "pot", bin.ProximityOrder) + valPo, _ := pof(pivotAddr, val, 0) + if valPo != bin.ProximityOrder { + t.Errorf("Incorrect value found in bin. Expected po %v, but was %v", bin.ProximityOrder, valPo) + } return true }) return true } - //Test eachBinDesc for an address with a po that we don't have a bin for - pivotAddr := newTestAddr("11010000", 7) - fmt.Println("****************Forward order with pivot 11010000****************") - pot.eachBin(pivotAddr, pof, 0, binConsumer) - fmt.Println("****************Reverse order with pivot 11010000****************") + log.Debug("****************Reverse order with pivot 11010000****************") pot.eachBinDesc(pivotAddr, pof, 0, binConsumer) } diff --git a/pss/pss.go b/pss/pss.go index a117dfbb8a..6ada73303e 100644 --- a/pss/pss.go +++ b/pss/pss.go @@ -829,7 +829,7 @@ func (p *Pss) forward(msg *message.Message) error { onlySendOnce = true } - p.kademliaLB.EachBin(to, func(bin network.LBBin) bool { + p.kademliaLB.EachBinDesc(to, func(bin network.LBBin) bool { if bin.ProximityOrder < broadcastThreshold && sent > 0 { // This bin is at the same distance as the node to the message. If already sent, we stop sending return false From 5b44ab328fc05dca5a9b4542f7707960400966de Mon Sep 17 00:00:00 2001 From: Alvaro Date: Mon, 14 Oct 2019 14:50:38 +0200 Subject: [PATCH 15/31] network/kademlia: better naming for pub/sub channels in kademlialoadbalancer --- network/kademlialoadbalancer.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/network/kademlialoadbalancer.go b/network/kademlialoadbalancer.go index 4116a1f83d..b6fe63315a 100644 --- a/network/kademlialoadbalancer.go +++ b/network/kademlialoadbalancer.go @@ -22,11 +22,11 @@ type KademliaBackend interface { // Creates a new KademliaLoadBalancer from a KademliaBackend func NewKademliaLoadBalancer(kademlia KademliaBackend, useMostSimilarInit bool) *KademliaLoadBalancer { - newPeerSub, offPeerSub := kademlia.SubscribeToPeerChanges() + onPeerSub, offPeerSub := kademlia.SubscribeToPeerChanges() klb := &KademliaLoadBalancer{ kademlia: kademlia, resourceUseStats: newResourceLoadBalancer(), - newPeerSub: newPeerSub, + onPeerSub: onPeerSub, offPeerSub: offPeerSub, quitC: make(chan struct{}), } @@ -68,7 +68,7 @@ type LBBinConsumer func(bin LBBin) bool type KademliaLoadBalancer struct { kademlia KademliaBackend //kademlia to obtain bins of peers resourceUseStats *resourceUseStats //a resourceUseStats to count uses - newPeerSub *gopubsub.Subscription //a pubsub channel to be notified of new peers in kademlia + onPeerSub *gopubsub.Subscription //a pubsub channel to be notified of new peers in kademlia offPeerSub *gopubsub.Subscription //a pubsub channel to be notified of removed peers in kademlia quitC chan struct{} @@ -77,7 +77,7 @@ type KademliaLoadBalancer struct { // Stop unsubscribe from notifiers func (klb KademliaLoadBalancer) Stop() { - klb.newPeerSub.Unsubscribe() + klb.onPeerSub.Unsubscribe() klb.offPeerSub.Unsubscribe() close(klb.quitC) } @@ -128,7 +128,7 @@ func (klb *KademliaLoadBalancer) listenNewPeers() { select { case <-klb.quitC: return - case msg, ok := <-klb.newPeerSub.ReceiveChannel(): + case msg, ok := <-klb.onPeerSub.ReceiveChannel(): if !ok { return } From 52dacb0cf4c6d2c83e9ac4d101c190ed80e584bf Mon Sep 17 00:00:00 2001 From: Alvaro Date: Tue, 15 Oct 2019 10:21:42 +0200 Subject: [PATCH 16/31] Fixed wrong test in kademlia load balancer. Also fixed waiting methods. --- network/kademlialoadbalancer.go | 16 +++++----- network/kademlialoadbalancer_test.go | 45 +++++++++++++++++----------- 2 files changed, 36 insertions(+), 25 deletions(-) diff --git a/network/kademlialoadbalancer.go b/network/kademlialoadbalancer.go index b6fe63315a..ee6eefd475 100644 --- a/network/kademlialoadbalancer.go +++ b/network/kademlialoadbalancer.go @@ -76,21 +76,21 @@ type KademliaLoadBalancer struct { } // Stop unsubscribe from notifiers -func (klb KademliaLoadBalancer) Stop() { +func (klb *KademliaLoadBalancer) Stop() { klb.onPeerSub.Unsubscribe() klb.offPeerSub.Unsubscribe() close(klb.quitC) } // EachBinNodeAddress calls EachBinDesc with the base address of kademlia (the node address) -func (klb KademliaLoadBalancer) EachBinNodeAddress(consumeBin LBBinConsumer) { +func (klb *KademliaLoadBalancer) EachBinNodeAddress(consumeBin LBBinConsumer) { klb.EachBinDesc(klb.kademlia.BaseAddr(), consumeBin) } // EachBinFiltered returns all bins in descending order from the perspective of base address. // Only peers with the provided capabilities capKey are considered. // All peers in that bin will be provided to the LBBinConsumer sorted by least used first. -func (klb KademliaLoadBalancer) EachBinFiltered(base []byte, capKey string, consumeBin LBBinConsumer) { +func (klb *KademliaLoadBalancer) EachBinFiltered(base []byte, capKey string, consumeBin LBBinConsumer) { _ = klb.kademlia.EachBinDescFiltered(base, capKey, 0, func(peerBin *PeerBin) bool { peers := klb.peerBinToPeerList(peerBin) return consumeBin(LBBin{LBPeers: peers, ProximityOrder: peerBin.ProximityOrder}) @@ -99,7 +99,7 @@ func (klb KademliaLoadBalancer) EachBinFiltered(base []byte, capKey string, cons // EachBinDesc returns all bins in descending order from the perspective of base address. // All peers in that bin will be provided to the LBBinConsumer sorted by least used first. -func (klb KademliaLoadBalancer) EachBinDesc(base []byte, consumeBin LBBinConsumer) { +func (klb *KademliaLoadBalancer) EachBinDesc(base []byte, consumeBin LBBinConsumer) { klb.kademlia.EachBinDesc(base, 0, func(peerBin *PeerBin) bool { peers := klb.peerBinToPeerList(peerBin) return consumeBin(LBBin{LBPeers: peers, ProximityOrder: peerBin.ProximityOrder}) @@ -314,12 +314,14 @@ func (lb *resourceUseStats) addUse(resource Resource) int { // to know that the initial uses has been initialized for a new peer func (lb *resourceUseStats) waitKey(key string) { lb.lock.Lock() - defer lb.lock.Unlock() if _, ok := lb.resourceUses[key]; ok { + lb.lock.Unlock() return } - lb.waiting[key] = make(chan struct{}) - <-lb.waiting[key] + waitChan := make(chan struct{}) + lb.waiting[key] = waitChan + lb.lock.Unlock() + <-waitChan delete(lb.waiting, key) } diff --git a/network/kademlialoadbalancer_test.go b/network/kademlialoadbalancer_test.go index d327e919fb..302ce3f3ab 100644 --- a/network/kademlialoadbalancer_test.go +++ b/network/kademlialoadbalancer_test.go @@ -17,9 +17,9 @@ import ( func TestAddedNodes(t *testing.T) { kademlia := newTestKademliaBackend("11110000") first := newTestKadPeer("010101010") - kademlia.addPeer(first, 0) + kademlia.addPeer(first) second := newTestKadPeer("010101011") - kademlia.addPeer(second, 0) + kademlia.addPeer(second) klb := NewKademliaLoadBalancer(kademlia, false) defer klb.Stop() @@ -31,7 +31,7 @@ func TestAddedNodes(t *testing.T) { peersFor0[0].Use() // Now new peers still should have 0 uses third := newTestKadPeer("011101011") - kademlia.addPeer(third, 0) + kademlia.addPeer(third) klb.resourceUseStats.waitKey(third.Key()) thirdUses := klb.resourceUseStats.getUses(third) if thirdUses != 0 { @@ -42,7 +42,7 @@ func TestAddedNodes(t *testing.T) { peersFor0[1].Use() //Now all peers should have 1 use //New peers should start with 1 use fourth := newTestKadPeer("011100011") - kademlia.addPeer(fourth, 0) + kademlia.addPeer(fourth) klb.resourceUseStats.waitKey(fourth.Key()) fourthUses := klb.resourceUseStats.getUses(fourth) if fourthUses != 1 { @@ -55,9 +55,9 @@ func TestAddedNodes(t *testing.T) { func TestAddedNodesMostSimilar(t *testing.T) { kademlia := newTestKademliaBackend("11110000") first := newTestKadPeer("01010101") - kademlia.addPeer(first, 0) + kademlia.addPeer(first) second := newTestKadPeer("01110101") - kademlia.addPeer(second, 0) + kademlia.addPeer(second) klb := NewKademliaLoadBalancer(kademlia, true) defer klb.Stop() @@ -69,7 +69,7 @@ func TestAddedNodesMostSimilar(t *testing.T) { peersFor0[0].Use() // Now third peer should have the same uses as second third := newTestKadPeer("01110111") // most similar peer is second 01110101 - kademlia.addPeer(third, 0) + kademlia.addPeer(third) klb.resourceUseStats.waitKey(third.Key()) secondUses := klb.resourceUseStats.getUses(second) thirdUses := klb.resourceUseStats.getUses(third) @@ -79,9 +79,9 @@ func TestAddedNodesMostSimilar(t *testing.T) { peersFor0 = klb.getPeersForPo(kademlia.baseAddr, 0) peersFor0[0].Use() peersFor0[1].Use() - //New peers should start with 1 use + fourth := newTestKadPeer("01110110") // most similar peer is third 01110111 - kademlia.addPeer(fourth, 0) + kademlia.addPeer(fourth) klb.resourceUseStats.waitKey(fourth.Key()) fourthUses := klb.resourceUseStats.getUses(fourth) thirdUses = klb.resourceUseStats.getUses(third) @@ -95,7 +95,7 @@ func TestAddedNodesMostSimilar(t *testing.T) { // We will create 3 bins with two peers each. We will call EachBinDesc 6 times twice with an address // on each bin, so at the end all peers should have 1 use (because the address in each bin is equidistant to // the peers in that bin). -// Then wi will use an address in a bin that is nearer one of the peers and we will check that that peer is always +// Then we will use an address in a bin that is nearer one of the peers and we will check that that peer is always // returned first func TestEachBinBaseUses(t *testing.T) { tk := newTestKademlia(t, "11111111") @@ -164,13 +164,20 @@ func TestEachBinFiltered(t *testing.T) { capPeer := tk.newTestKadPeerWithCapabilities("10100000", caps[capKey]) tk.Kademlia.On(capPeer) - - tk.On("01010101") // bin 0 dec 85 - tk.On("01010100") // bin 0 dec 84 + useStats := klb.resourceUseStats + useStats.waitKey(capPeer.Key()) + tk.On("01010101") // bin 0 dec 85 hex 55 + useStats.waitKey(stringBinaryToHex("01010101")) + tk.On("01010100") // bin 0 dec 84 hex 54 + useStats.waitKey(stringBinaryToHex("01010100")) tk.On("10010100") // bin 1 dec 148 + useStats.waitKey(stringBinaryToHex("10010100")) tk.On("10010001") // bin 1 dec 145 + useStats.waitKey(stringBinaryToHex("10010001")) tk.On("11010100") // bin 2 dec 212 + useStats.waitKey(stringBinaryToHex("11010100")) tk.On("11010101") // bin 2 dec 213 + useStats.waitKey(stringBinaryToHex("11010101")) stats := make(map[string]int) countUse := func(bin LBBin) bool { peer := bin.LBPeers[0].Peer @@ -186,20 +193,21 @@ func TestEachBinFiltered(t *testing.T) { klb.EachBinFiltered(pivotAddressBin1, capKey, countUse) klb.EachBinFiltered(pivotAddressBin1, capKey, countUse) - useStats := klb.resourceUseStats count := useStats.getUses(capPeer) if count != 3 || stats["10100000"] != 3 { t.Errorf("Expected 3 uses of capability peer but got %v/%v", count, stats["10100000"]) } - secondCapPeer := tk.newTestKadPeerWithCapabilities("10100000", caps[capKey]) + secondCapPeer := tk.newTestKadPeerWithCapabilities("10100001", caps[capKey]) tk.Kademlia.On(secondCapPeer) useStats.waitKey(secondCapPeer.Key()) + secondCountStart := useStats.getUses(secondCapPeer) + count = useStats.getUses(capPeer) klb.EachBinFiltered(pivotAddressBin1, capKey, countUse) klb.EachBinFiltered(pivotAddressBin1, capKey, countUse) secondCount := useStats.getUses(secondCapPeer) - if secondCount == 0 { - t.Errorf("Expected some use of second capability peer but got %v", secondCount) + if secondCount-secondCountStart != 2 { + t.Errorf("Expected 2 uses of second capability peer but got %v", secondCount-secondCountStart) } } @@ -311,7 +319,8 @@ func (tkb testKademliaBackend) EachBinDesc(_ []byte, minProximityOrder int, cons } } -func (tkb *testKademliaBackend) addPeer(peer *Peer, po int) { +func (tkb *testKademliaBackend) addPeer(peer *Peer) { + po, _ := Pof(peer.Address(), tkb.baseAddr, 0) if tkb.bins[po] == nil { if po > tkb.maxPo { tkb.maxPo = po From 49e7d0971946d861437500db53b8c2f7f083b435 Mon Sep 17 00:00:00 2001 From: Alvaro Date: Tue, 15 Oct 2019 15:14:33 +0200 Subject: [PATCH 17/31] Debug functions moved out of kademlialoadbalancer.go --- network/discovery.go | 5 +++ network/kademlialoadbalancer.go | 39 +++++----------------- network/kademlialoadbalancer_test.go | 48 +++++++++++++++++++++------- 3 files changed, 50 insertions(+), 42 deletions(-) diff --git a/network/discovery.go b/network/discovery.go index 5252773fc7..207dfcb267 100644 --- a/network/discovery.go +++ b/network/discovery.go @@ -74,6 +74,11 @@ func (d *Peer) Key() string { return d.key } +// Label returns a short string representation for debugging purposes +func (d *Peer) Label() string { + return d.key[:4] +} + // NotifyDepth sends a message to all connections if depth of saturation is changed func NotifyDepth(depth uint8, kad *Kademlia) { f := func(val *Peer, po int) bool { diff --git a/network/kademlialoadbalancer.go b/network/kademlialoadbalancer.go index ee6eefd475..ebf7bc5805 100644 --- a/network/kademlialoadbalancer.go +++ b/network/kademlialoadbalancer.go @@ -5,9 +5,7 @@ import ( "strconv" "sync" - "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethersphere/swarm/log" - "github.com/ethersphere/swarm/network/gopubsub" ) @@ -160,7 +158,7 @@ func (klb *KademliaLoadBalancer) listenOffPeers() { // to calculate it again. func (klb *KademliaLoadBalancer) addedPeer(peer *Peer, po int) { initCount := klb.initCountFunc(peer, 0) - log.Debug("Adding peer", "key", peer.Key()[:4], "initCount", initCount) + log.Debug("Adding peer", "key", peer.Label(), "initCount", initCount) klb.resourceUseStats.initKey(peer.Key(), initCount) } @@ -174,7 +172,7 @@ func (klb *KademliaLoadBalancer) leastUsedCountInBin(excludePeer *Peer, po int) leastUsed := peersInSamePo[idx] if leastUsed.Peer.Key() != excludePeer.Key() { leastUsedCount = klb.resourceUseStats.getUses(leastUsed.Peer) - log.Debug("Least used peer is", "peer", leastUsed.Peer.Key()[:4], "leastUsedCount", leastUsedCount) + log.Debug("Least used peer is", "peer", leastUsed.Peer.Label(), "leastUsedCount", leastUsedCount) break } idx++ @@ -188,7 +186,7 @@ func (klb *KademliaLoadBalancer) nearestNeighbourCount(newPeer *Peer, _ int) int klb.kademlia.EachConn(newPeer.Address(), 255, func(peer *Peer, po int) bool { if peer != newPeer { count = klb.resourceUseStats.getUses(peer) - log.Debug("Nearest neighbour is", "peer", peer.Key()[:4], "count", count) + log.Debug("Nearest neighbour is", "peer", peer.Label(), "count", count) return false } return true @@ -239,7 +237,8 @@ type resourceUseStats struct { } type Resource interface { - Key() string + Key() string // unique id in string format of the resource. + Label() string // short string format of the key for debugging purposes. } type ResourceCount struct { @@ -304,14 +303,14 @@ func (lb *resourceUseStats) getKeyUses(key string) int { func (lb *resourceUseStats) addUse(resource Resource) int { lb.lock.Lock() defer lb.lock.Unlock() - log.Debug("Adding use", "key", resource.Key()[:4]) + log.Debug("Adding use", "key", resource.Label()) key := resource.Key() lb.resourceUses[key] = lb.resourceUses[key] + 1 return lb.resourceUses[key] } -// Used for testing. As peer resource initialization is asynchronous we need a way -// to know that the initial uses has been initialized for a new peer +// WaitKey blocks until some key is added to the load balancer stats. +// As peer resource initialization is asynchronous we need a way to know that the initial uses has been initialized. func (lb *resourceUseStats) waitKey(key string) { lb.lock.Lock() if _, ok := lb.resourceUses[key]; ok { @@ -333,25 +332,3 @@ func (lb *resourceUseStats) initKey(key string, count int) { kChan <- struct{}{} } } - -// Debug functions - -func stringBinaryToHex(binary string) string { - var byteSlice = make([]byte, 32) - i, _ := strconv.ParseInt(binary, 2, 0) - byteSlice[0] = byte(i) - return hexutil.Encode(byteSlice) -} -func peerToBinaryId(peer *Peer) string { - return byteToBinary(peer.Address()[0]) -} - -func byteToBinary(b byte) string { - binary := strconv.FormatUint(uint64(b), 2) - if len(binary) < 8 { - for i := 8 - len(binary); i > 0; i-- { - binary = "0" + binary - } - } - return binary -} diff --git a/network/kademlialoadbalancer_test.go b/network/kademlialoadbalancer_test.go index 302ce3f3ab..6f52b742fe 100644 --- a/network/kademlialoadbalancer_test.go +++ b/network/kademlialoadbalancer_test.go @@ -2,14 +2,14 @@ package network import ( "sort" + "strconv" "testing" "time" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethersphere/swarm/network/capability" - "github.com/ethersphere/swarm/pot" - "github.com/ethersphere/swarm/network/gopubsub" + "github.com/ethersphere/swarm/pot" ) // TestAddedNodes checks that when adding a node it is assigned the correct number of uses. @@ -133,7 +133,7 @@ func TestEachBinBaseUses(t *testing.T) { for key, uses := range resourceUses { if uses != 1 { bytes, _ := hexutil.Decode(key) - binaryKey := byteToBinary(bytes[0]) + byteToBinary(bytes[1]) + binaryKey := byteToBitString(bytes[0]) + byteToBitString(bytes[1]) t.Errorf("Expected only 1 use of %v but got %v", binaryKey, uses) } } @@ -145,7 +145,7 @@ func TestEachBinBaseUses(t *testing.T) { klb.EachBinDesc(pivotAddressBin3, countUse) klb.EachBinDesc(pivotAddressBin3, countUse) - count := klb.resourceUseStats.getKeyUses(stringBinaryToHex("10010001")) + count := klb.resourceUseStats.getKeyUses(bitStringToHex("10010001")) if count != 3 { t.Errorf("Expected 3 uses of 10010001 but got %v", count) } @@ -167,22 +167,22 @@ func TestEachBinFiltered(t *testing.T) { useStats := klb.resourceUseStats useStats.waitKey(capPeer.Key()) tk.On("01010101") // bin 0 dec 85 hex 55 - useStats.waitKey(stringBinaryToHex("01010101")) + useStats.waitKey(bitStringToHex("01010101")) tk.On("01010100") // bin 0 dec 84 hex 54 - useStats.waitKey(stringBinaryToHex("01010100")) + useStats.waitKey(bitStringToHex("01010100")) tk.On("10010100") // bin 1 dec 148 - useStats.waitKey(stringBinaryToHex("10010100")) + useStats.waitKey(bitStringToHex("10010100")) tk.On("10010001") // bin 1 dec 145 - useStats.waitKey(stringBinaryToHex("10010001")) + useStats.waitKey(bitStringToHex("10010001")) tk.On("11010100") // bin 2 dec 212 - useStats.waitKey(stringBinaryToHex("11010100")) + useStats.waitKey(bitStringToHex("11010100")) tk.On("11010101") // bin 2 dec 213 - useStats.waitKey(stringBinaryToHex("11010101")) + useStats.waitKey(bitStringToHex("11010101")) stats := make(map[string]int) countUse := func(bin LBBin) bool { peer := bin.LBPeers[0].Peer bin.LBPeers[0].Use() - key := peerToBinaryId(peer) + key := peerToBitString(peer) stats[key] = stats[key] + 1 return false } @@ -368,3 +368,29 @@ func (tkb *testKademliaBackend) updateMaxPo() { func newTestKadPeer(s string) *Peer { return NewPeer(&BzzPeer{BzzAddr: testKadPeerAddr(s)}, nil) } + +// Debug functions + +// bitStringToHex converts an address in bit format (11001100) to hex format. BitString format is used to create test +// peers, hex format is used in the load balancer stats. +func bitStringToHex(binary string) string { + var byteSlice = make([]byte, 32) + i, _ := strconv.ParseInt(binary, 2, 0) + byteSlice[0] = byte(i) + return hexutil.Encode(byteSlice) +} + +// converts the peer address to bit string format +func peerToBitString(peer *Peer) string { + return byteToBitString(peer.Address()[0]) +} + +func byteToBitString(b byte) string { + binary := strconv.FormatUint(uint64(b), 2) + if len(binary) < 8 { + for i := 8 - len(binary); i > 0; i-- { + binary = "0" + binary + } + } + return binary +} From ad5eac5083668968a3ac6bba0841cb66f70d02bb Mon Sep 17 00:00:00 2001 From: Alvaro Date: Wed, 16 Oct 2019 10:15:59 +0200 Subject: [PATCH 18/31] More comments, fixed EachConn po for peers in the same bin as the pivot address, renamed findPeerPo to peerPo --- network/kademlia.go | 2 +- network/kademlialoadbalancer.go | 2 +- network/kademlialoadbalancer_test.go | 29 +++++++++++++++++++++------- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/network/kademlia.go b/network/kademlia.go index 72b1710335..cb479c77a2 100644 --- a/network/kademlia.go +++ b/network/kademlia.go @@ -486,7 +486,7 @@ func (k *Kademlia) On(p *Peer) (uint8, bool) { return k.depth, changed } -func (k *Kademlia) findPeerPo(peer *Peer) (po int, found bool) { +func (k *Kademlia) peerPo(peer *Peer) (po int, found bool) { po, found = Pof(k.defaultIndex.conns.Pin(), peer, 0) return po, found } diff --git a/network/kademlialoadbalancer.go b/network/kademlialoadbalancer.go index ebf7bc5805..deabc6e457 100644 --- a/network/kademlialoadbalancer.go +++ b/network/kademlialoadbalancer.go @@ -309,7 +309,7 @@ func (lb *resourceUseStats) addUse(resource Resource) int { return lb.resourceUses[key] } -// WaitKey blocks until some key is added to the load balancer stats. +// waitKey blocks until some key is added to the load balancer stats. // As peer resource initialization is asynchronous we need a way to know that the initial uses has been initialized. func (lb *resourceUseStats) waitKey(key string) { lb.lock.Lock() diff --git a/network/kademlialoadbalancer_test.go b/network/kademlialoadbalancer_test.go index 6f52b742fe..cf10a1ec6b 100644 --- a/network/kademlialoadbalancer_test.go +++ b/network/kademlialoadbalancer_test.go @@ -13,7 +13,7 @@ import ( ) // TestAddedNodes checks that when adding a node it is assigned the correct number of uses. -// This number of uses will be the least number of uses of a peer in its bin +// This number of uses will be the least number of uses of a peer in its bin. func TestAddedNodes(t *testing.T) { kademlia := newTestKademliaBackend("11110000") first := newTestKadPeer("010101010") @@ -95,8 +95,8 @@ func TestAddedNodesMostSimilar(t *testing.T) { // We will create 3 bins with two peers each. We will call EachBinDesc 6 times twice with an address // on each bin, so at the end all peers should have 1 use (because the address in each bin is equidistant to // the peers in that bin). -// Then we will use an address in a bin that is nearer one of the peers and we will check that that peer is always -// returned first +// Then we will use an address in a bin that is nearer to one of the peers and we will check that that peer is always +// returned first. func TestEachBinBaseUses(t *testing.T) { tk := newTestKademlia(t, "11111111") klb := NewKademliaLoadBalancer(tk, false) @@ -151,6 +151,7 @@ func TestEachBinBaseUses(t *testing.T) { } } +// TestEachBinFiltered checks that when load balancing peers, only those with the provided capabilities are chosen. func TestEachBinFiltered(t *testing.T) { tk := newTestKademlia(t, "11111111") klb := NewKademliaLoadBalancer(tk, false) @@ -221,7 +222,16 @@ type testKademliaBackend struct { maxPo int } -// EachConn iterates this test kademlia table bins in order from furthest to nearest. +// EachConn iterates this test kademlia table peers in order po from nearest to furthest with respect to the base address. +// - First it takes the po of the base address provided. With this po it takes the corresponding bin B (peers with same po +// as this base) copy the list and sort them using sort.Slice from furthest to lowest. Then it calls the provided consume +// function with those peers and its corresponding po (peerPo). +// - Then, it iterates all bins with higher po than B, and iterates all peers. All of these peers will have the +// same po as the base address with respect to the pin of the kademlia, that's why we don't need to sort them out and +// we can iterate them in any order. For each peer the consume function is called with the same po. +// - Finally, the bins furthest than B are iterated from highest po (nearest) to lowest (furthest). For each bin, all +// peers are iterated and called the consume function with the peer and the bin po. +// After any of the calls to the consume function, if that function returns false, the iteration stops. func (tkb *testKademliaBackend) EachConn(base []byte, maxPo int, consume func(*Peer, int) bool) { po, _ := Pof(base, tkb.baseAddr, 0) bin := tkb.bins[po] @@ -233,7 +243,8 @@ func (tkb *testKademliaBackend) EachConn(base []byte, maxPo int, consume func(*P return peerIPo > peerJPo }) for _, peer := range peersInBin { - if !consume(peer, po) { + peerPo, _ := Pof(base, peer, 0) + if !consume(peer, peerPo) { return } } @@ -265,10 +276,14 @@ func newTestKademliaBackend(address string) *testKademliaBackend { } } +// BaseAddr returns the node address of the kademlia table. This base address is the pivot address to which other addresses +// are sorted with respect to the proximity order function. func (tkb testKademliaBackend) BaseAddr() []byte { return tkb.baseAddr } +// SubscribeToPeerChanges returns a subscription to changes in the kademlia peers. It contains channels to notify about +// peers added/removed from this kademlia. func (tkb *testKademliaBackend) SubscribeToPeerChanges() (addedSub *gopubsub.Subscription, removedPeerSub *gopubsub.Subscription) { addedSub = tkb.addedChannel.Subscribe() removedPeerSub = tkb.removedChannel.Subscribe() @@ -276,14 +291,14 @@ func (tkb *testKademliaBackend) SubscribeToPeerChanges() (addedSub *gopubsub.Sub return } -// EachBinDescFiltered ignores capKey as in this test context it won't be used. +// EachBinDescFiltered ignores capKey as in this test context it won't be used. It ignores base as EachBinDesc ignores it. func (tkb testKademliaBackend) EachBinDescFiltered(base []byte, capKey string, minProximityOrder int, consumer PeerBinConsumer) error { tkb.EachBinDesc(base, minProximityOrder, consumer) return nil } // EachBinDesc iterates bin in the table in descending po order (from nearest to furthest). Base is always supposed to be -// node address in this test context. +// node address in this test context. For each bin found, the provided PeerBinConsumer function is called. func (tkb testKademliaBackend) EachBinDesc(_ []byte, minProximityOrder int, consumer PeerBinConsumer) { type poPeers struct { po int From e78e2b31dd6df3484f134b64198226de2b07a401 Mon Sep 17 00:00:00 2001 From: Alvaro Date: Wed, 16 Oct 2019 15:20:53 +0200 Subject: [PATCH 19/31] resourceUseStats moved to a diffrente file. Use() renamed to AddUseCount(). Several minor comments and fixes --- network/kademlia.go | 16 +- ...dbalancer.go => kademlia_load_balancer.go} | 174 +++++------------- ...test.go => kademlia_load_balancer_test.go} | 48 ++--- network/resource_use_stats.go | 135 ++++++++++++++ pss/forwarding_test.go | 4 +- pss/pss.go | 2 +- 6 files changed, 214 insertions(+), 165 deletions(-) rename network/{kademlialoadbalancer.go => kademlia_load_balancer.go} (61%) rename network/{kademlialoadbalancer_test.go => kademlia_load_balancer_test.go} (94%) create mode 100644 network/resource_use_stats.go diff --git a/network/kademlia.go b/network/kademlia.go index cb479c77a2..4530b62594 100644 --- a/network/kademlia.go +++ b/network/kademlia.go @@ -92,14 +92,14 @@ func NewKadParams() *KadParams { // Kademlia is a table of live peers and a db of known peers (node records) type Kademlia struct { lock sync.RWMutex - capabilityIndex map[string]*capabilityIndex - defaultIndex *capabilityIndex // Index with pots - *KadParams // Kademlia configuration parameters - base []byte // immutable baseaddress of the table - depth uint8 // stores the last current depth of saturation - nDepth int // stores the last neighbourhood depth - nDepthMu sync.RWMutex // protects neighbourhood depth nDepth - nDepthSig []chan struct{} // signals when neighbourhood depth nDepth is changed + capabilityIndex map[string]*capabilityIndex // index with pots for peers with a capability + defaultIndex *capabilityIndex // index with pots for all peers (no capability) + *KadParams // Kademlia configuration parameters + base []byte // immutable baseaddress of the table + depth uint8 // stores the last current depth of saturation + nDepth int // stores the last neighbourhood depth + nDepthMu sync.RWMutex // protects neighbourhood depth nDepth + nDepthSig []chan struct{} // signals when neighbourhood depth nDepth is changed newPeerPubSub *gopubsub.PubSubChannel removedPeerPubSub *gopubsub.PubSubChannel diff --git a/network/kademlialoadbalancer.go b/network/kademlia_load_balancer.go similarity index 61% rename from network/kademlialoadbalancer.go rename to network/kademlia_load_balancer.go index deabc6e457..3effea4f3b 100644 --- a/network/kademlialoadbalancer.go +++ b/network/kademlia_load_balancer.go @@ -1,10 +1,21 @@ +// Copyright 2019 The Swarm Authors +// This file is part of the Swarm library. +// +// The Swarm library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The Swarm library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the Swarm library. If not, see . package network import ( - "sort" - "strconv" - "sync" - "github.com/ethersphere/swarm/log" "github.com/ethersphere/swarm/network/gopubsub" ) @@ -18,18 +29,22 @@ type KademliaBackend interface { EachConn(base []byte, o int, f func(*Peer, int) bool) } -// Creates a new KademliaLoadBalancer from a KademliaBackend -func NewKademliaLoadBalancer(kademlia KademliaBackend, useMostSimilarInit bool) *KademliaLoadBalancer { +// Creates a new KademliaLoadBalancer from a KademliaBackend. +// If useNearestNeighbourInit is true the nearest neighbour peer use count will be used when a peer is initialized. +// If not, least used peer use count in same bin as new peer will be used. It is not clear which one is better, when +// this load balancer would be used in several use cases we could do take some decision. +func NewKademliaLoadBalancer(kademlia KademliaBackend, useNearestNeighbourInit bool) *KademliaLoadBalancer { onPeerSub, offPeerSub := kademlia.SubscribeToPeerChanges() + quitC := make(chan struct{}) klb := &KademliaLoadBalancer{ kademlia: kademlia, - resourceUseStats: newResourceLoadBalancer(), + resourceUseStats: newResourceUseStats(quitC), onPeerSub: onPeerSub, offPeerSub: offPeerSub, - quitC: make(chan struct{}), + quitC: quitC, } - if useMostSimilarInit { - klb.initCountFunc = klb.nearestNeighbourCount + if useNearestNeighbourInit { + klb.initCountFunc = klb.nearestNeighbourUseCount } else { klb.initCountFunc = klb.leastUsedCountInBin } @@ -39,13 +54,15 @@ func NewKademliaLoadBalancer(kademlia KademliaBackend, useMostSimilarInit bool) return klb } -// Consumer functions +// Consumer functions. A consumer is a function that uses an element returned by an iterator. It usually also returns +// a boolean signaling if it wants to iterate more or not. We created an alias for consumer function (LBBinConsumer) +// for code clarity. -// An LBPeer represents a peer with a Use() function to signal that the peer has been used in order -// to account it for LB sorting criteria +// An LBPeer represents a peer with a AddUseCount() function to signal that the peer has been used in order +// to account it for LB sorting criteria. type LBPeer struct { - Peer *Peer - Use func() + Peer *Peer + AddUseCount func() // called to account a use for these peer. Should be called if the peer is actually used. } // LBBin represents a Bin of LBPeer's @@ -55,19 +72,18 @@ type LBBin struct { } // LBBinConsumer will be provided with a list of LBPeer's in LB criteria ordering (currently in least used ordering). +// Should return true if it must continue iterating LBBin's or stops if false. type LBBinConsumer func(bin LBBin) bool -// KademliaLoadBalancer struct and methods - // KademliaLoadBalancer tries to balance request to the peers in Kademlia returning the peers sorted // by least recent used whenever several will be returned with the same po to a particular address. // The user of KademliaLoadBalancer should signal if the returned element (LBPeer) has been used with the -// function lbPeer.Use() +// function lbPeer.AddUseCount() type KademliaLoadBalancer struct { - kademlia KademliaBackend //kademlia to obtain bins of peers - resourceUseStats *resourceUseStats //a resourceUseStats to count uses - onPeerSub *gopubsub.Subscription //a pubsub channel to be notified of new peers in kademlia - offPeerSub *gopubsub.Subscription //a pubsub channel to be notified of removed peers in kademlia + kademlia KademliaBackend // kademlia to obtain bins of peers + resourceUseStats *resourceUseStats // a resourceUseStats to count uses + onPeerSub *gopubsub.Subscription // a pubsub channel to be notified of new peers in kademlia + offPeerSub *gopubsub.Subscription // a pubsub channel to be notified of removed peers in kademlia quitC chan struct{} initCountFunc func(peer *Peer, po int) int //Function to use for initializing a new peer count @@ -180,8 +196,8 @@ func (klb *KademliaLoadBalancer) leastUsedCountInBin(excludePeer *Peer, po int) return leastUsedCount } -// nearestNeighbourCount returns the use count for the closest peer count. -func (klb *KademliaLoadBalancer) nearestNeighbourCount(newPeer *Peer, _ int) int { +// nearestNeighbourUseCount returns the use count for the closest peer count. +func (klb *KademliaLoadBalancer) nearestNeighbourUseCount(newPeer *Peer, _ int) int { var count int klb.kademlia.EachConn(newPeer.Address(), 255, func(peer *Peer, po int) bool { if peer != newPeer { @@ -196,7 +212,7 @@ func (klb *KademliaLoadBalancer) nearestNeighbourCount(newPeer *Peer, _ int) int func (klb *KademliaLoadBalancer) removedPeer(peer *Peer) { klb.resourceUseStats.lock.Lock() - defer klb.resourceUseStats.lock.Lock() + defer klb.resourceUseStats.lock.Unlock() delete(klb.resourceUseStats.resourceUses, peer.Key()) } @@ -205,7 +221,7 @@ func (klb *KademliaLoadBalancer) toLBPeers(resources []Resource) []LBPeer { for i, res := range resources { peer := res.(*Peer) peers[i].Peer = peer - peers[i].Use = func() { + peers[i].AddUseCount = func() { klb.resourceUseStats.addUse(peer) } } @@ -226,109 +242,3 @@ func (klb *KademliaLoadBalancer) getPeersForPo(base []byte, po int) []LBPeer { }) return klb.resourcesToLbPeers(resources) } - -// Resource Use Stats - -// resourceUseStats can be used to count uses of resources. A Resource is anything with a Key() -type resourceUseStats struct { - resourceUses map[string]int - waiting map[string]chan struct{} - lock sync.RWMutex -} - -type Resource interface { - Key() string // unique id in string format of the resource. - Label() string // short string format of the key for debugging purposes. -} - -type ResourceCount struct { - resource Resource - count int -} - -func newResourceLoadBalancer() *resourceUseStats { - return &resourceUseStats{ - resourceUses: make(map[string]int), - waiting: make(map[string]chan struct{}), - } -} - -func (lb *resourceUseStats) sortResources(resources []Resource) []Resource { - sorted := make([]Resource, len(resources)) - resourceCounts := lb.getAllUses(resources) - sort.Slice(resourceCounts, func(i, j int) bool { - return resourceCounts[i].count < resourceCounts[j].count - }) - for i, resourceCount := range resourceCounts { - sorted[i] = resourceCount.resource - } - return sorted -} - -func (lbp ResourceCount) String() string { - return lbp.resource.Key() + ":" + strconv.Itoa(lbp.count) -} - -func (lb *resourceUseStats) dumpAllUses() map[string]int { - lb.lock.RLock() - defer lb.lock.RUnlock() - dump := make(map[string]int) - for k, v := range lb.resourceUses { - dump[k] = v - } - return dump -} - -func (lb *resourceUseStats) getAllUses(resources []Resource) []ResourceCount { - peerUses := make([]ResourceCount, len(resources)) - for i, resource := range resources { - peerUses[i] = ResourceCount{ - resource: resource, - count: lb.getUses(resource), - } - } - return peerUses -} - -func (lb *resourceUseStats) getUses(keyed Resource) int { - return lb.getKeyUses(keyed.Key()) -} - -func (lb *resourceUseStats) getKeyUses(key string) int { - lb.lock.RLock() - defer lb.lock.RUnlock() - return lb.resourceUses[key] -} - -func (lb *resourceUseStats) addUse(resource Resource) int { - lb.lock.Lock() - defer lb.lock.Unlock() - log.Debug("Adding use", "key", resource.Label()) - key := resource.Key() - lb.resourceUses[key] = lb.resourceUses[key] + 1 - return lb.resourceUses[key] -} - -// waitKey blocks until some key is added to the load balancer stats. -// As peer resource initialization is asynchronous we need a way to know that the initial uses has been initialized. -func (lb *resourceUseStats) waitKey(key string) { - lb.lock.Lock() - if _, ok := lb.resourceUses[key]; ok { - lb.lock.Unlock() - return - } - waitChan := make(chan struct{}) - lb.waiting[key] = waitChan - lb.lock.Unlock() - <-waitChan - delete(lb.waiting, key) -} - -func (lb *resourceUseStats) initKey(key string, count int) { - lb.lock.Lock() - defer lb.lock.Unlock() - lb.resourceUses[key] = count - if kChan, ok := lb.waiting[key]; ok { - kChan <- struct{}{} - } -} diff --git a/network/kademlialoadbalancer_test.go b/network/kademlia_load_balancer_test.go similarity index 94% rename from network/kademlialoadbalancer_test.go rename to network/kademlia_load_balancer_test.go index cf10a1ec6b..143b2a4832 100644 --- a/network/kademlialoadbalancer_test.go +++ b/network/kademlia_load_balancer_test.go @@ -28,7 +28,7 @@ func TestAddedNodes(t *testing.T) { t.Errorf("Expected 0 uses for new peer at start") } peersFor0 := klb.getPeersForPo(kademlia.baseAddr, 0) - peersFor0[0].Use() + peersFor0[0].AddUseCount() // Now new peers still should have 0 uses third := newTestKadPeer("011101011") kademlia.addPeer(third) @@ -38,8 +38,8 @@ func TestAddedNodes(t *testing.T) { t.Errorf("Expected 0 uses for new peer because minimum in bin is 0. Instead %v", thirdUses) } peersFor0 = klb.getPeersForPo(kademlia.baseAddr, 0) - peersFor0[0].Use() - peersFor0[1].Use() //Now all peers should have 1 use + peersFor0[0].AddUseCount() + peersFor0[1].AddUseCount() //Now all peers should have 1 use //New peers should start with 1 use fourth := newTestKadPeer("011100011") kademlia.addPeer(fourth) @@ -50,9 +50,9 @@ func TestAddedNodes(t *testing.T) { } } -// TestAddedNodesMostSimilar checks that when adding a node it is assigned the correct number of uses. +// TestAddedNodesNearestNeighbour checks that when adding a node it is assigned the correct number of uses. // This number of uses will be the most similar peer uses. -func TestAddedNodesMostSimilar(t *testing.T) { +func TestAddedNodesNearestNeighbour(t *testing.T) { kademlia := newTestKademliaBackend("11110000") first := newTestKadPeer("01010101") kademlia.addPeer(first) @@ -66,7 +66,7 @@ func TestAddedNodesMostSimilar(t *testing.T) { t.Errorf("Expected 0 uses for new peer at start") } peersFor0 := klb.getPeersForPo(kademlia.baseAddr, 0) - peersFor0[0].Use() + peersFor0[0].AddUseCount() // Now third peer should have the same uses as second third := newTestKadPeer("01110111") // most similar peer is second 01110101 kademlia.addPeer(third) @@ -76,13 +76,19 @@ func TestAddedNodesMostSimilar(t *testing.T) { if thirdUses != secondUses { t.Errorf("Expected %v uses for new peer because is most similar to second. Instead %v", secondUses, thirdUses) } + //Now we use third peer twice peersFor0 = klb.getPeersForPo(kademlia.baseAddr, 0) - peersFor0[0].Use() - peersFor0[1].Use() + for _, lbPeer := range peersFor0 { + if lbPeer.Peer.Key() == third.key { + lbPeer.AddUseCount() + lbPeer.AddUseCount() + } + } fourth := newTestKadPeer("01110110") // most similar peer is third 01110111 kademlia.addPeer(fourth) klb.resourceUseStats.waitKey(fourth.Key()) + //We expect fourth to be initialized with third peer use count fourthUses := klb.resourceUseStats.getUses(fourth) thirdUses = klb.resourceUseStats.getUses(third) if fourthUses != thirdUses { @@ -111,7 +117,7 @@ func TestEachBinBaseUses(t *testing.T) { pivotAddressBin1 := pot.NewAddressFromString("10000000") // Two nearest peers (3,4) pivotAddressBin2 := pot.NewAddressFromString("11000000") // Two nearest peers (5,6) countUse := func(bin LBBin) bool { - bin.LBPeers[0].Use() + bin.LBPeers[0].AddUseCount() return false } // Use peer 1 and 2 @@ -182,7 +188,7 @@ func TestEachBinFiltered(t *testing.T) { stats := make(map[string]int) countUse := func(bin LBBin) bool { peer := bin.LBPeers[0].Peer - bin.LBPeers[0].Use() + bin.LBPeers[0].AddUseCount() key := peerToBitString(peer) stats[key] = stats[key] + 1 return false @@ -218,7 +224,6 @@ type testKademliaBackend struct { addedChannel *gopubsub.PubSubChannel removedChannel *gopubsub.PubSubChannel bins map[int][]*Peer - subscribed bool maxPo int } @@ -287,7 +292,6 @@ func (tkb testKademliaBackend) BaseAddr() []byte { func (tkb *testKademliaBackend) SubscribeToPeerChanges() (addedSub *gopubsub.Subscription, removedPeerSub *gopubsub.Subscription) { addedSub = tkb.addedChannel.Subscribe() removedPeerSub = tkb.removedChannel.Subscribe() - tkb.subscribed = true return } @@ -343,18 +347,21 @@ func (tkb *testKademliaBackend) addPeer(peer *Peer) { tkb.bins[po] = make([]*Peer, 0) } tkb.bins[po] = append(tkb.bins[po], peer) - if tkb.subscribed { - tkb.addedChannel.Publish(newPeerSignal{ - peer: peer, - po: po, - }) - } + tkb.addedChannel.Publish(newPeerSignal{ + peer: peer, + po: po, + }) // As the subscribers to add peer are asynchronous, we will sleep here to allow them to execute. // Because this will be used in tests several times, we wait here so the code is not polluted with Sleep calls. time.Sleep(100 * time.Millisecond) } func (tkb *testKademliaBackend) removePeer(peer *Peer) { + tkb.removePeerFromBin(peer) + tkb.removedChannel.Publish(peer) +} + +func (tkb *testKademliaBackend) removePeerFromBin(peer *Peer) { for po, bin := range tkb.bins { for i, aPeer := range bin { if aPeer == peer { @@ -362,13 +369,10 @@ func (tkb *testKademliaBackend) removePeer(peer *Peer) { if len(tkb.bins[po]) == 0 && tkb.maxPo >= po { tkb.updateMaxPo() } - break + return } } } - if tkb.subscribed { - tkb.removedChannel.Publish(peer) - } } func (tkb *testKademliaBackend) updateMaxPo() { diff --git a/network/resource_use_stats.go b/network/resource_use_stats.go new file mode 100644 index 0000000000..f13df06857 --- /dev/null +++ b/network/resource_use_stats.go @@ -0,0 +1,135 @@ +// Copyright 2019 The Swarm Authors +// This file is part of the Swarm library. +// +// The Swarm library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The Swarm library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the Swarm library. If not, see . +package network + +import ( + "sort" + "strconv" + "sync" + + "github.com/ethersphere/swarm/log" +) + +// resourceUseStats can be used to count uses of resources. A Resource is anything with a Key() +type resourceUseStats struct { + resourceUses map[string]int + waiting map[string]chan struct{} + lock sync.RWMutex + quitC chan struct{} +} + +// Resource represents anything with a Key that can be accounted with some stat. +type Resource interface { + Key() string // unique id in string format of the resource. + Label() string // short string format of the key for debugging purposes. +} + +type ResourceCount struct { + resource Resource + count int +} + +func newResourceUseStats(quitC chan struct{}) *resourceUseStats { + return &resourceUseStats{ + resourceUses: make(map[string]int), + waiting: make(map[string]chan struct{}), + quitC: quitC, + } +} + +func (lb *resourceUseStats) sortResources(resources []Resource) []Resource { + sorted := make([]Resource, len(resources)) + resourceCounts := lb.getAllUseCounts(resources) + sort.Slice(resourceCounts, func(i, j int) bool { + return resourceCounts[i].count < resourceCounts[j].count + }) + for i, resourceCount := range resourceCounts { + sorted[i] = resourceCount.resource + } + return sorted +} + +func (lbp ResourceCount) String() string { + return lbp.resource.Key() + ":" + strconv.Itoa(lbp.count) +} + +func (lb *resourceUseStats) dumpAllUses() map[string]int { + lb.lock.RLock() + defer lb.lock.RUnlock() + dump := make(map[string]int) + for k, v := range lb.resourceUses { + dump[k] = v + } + return dump +} + +func (lb *resourceUseStats) getAllUseCounts(resources []Resource) []ResourceCount { + peerUses := make([]ResourceCount, len(resources)) + for i, resource := range resources { + peerUses[i] = ResourceCount{ + resource: resource, + count: lb.getUses(resource), + } + } + return peerUses +} + +func (lb *resourceUseStats) getUses(keyed Resource) int { + return lb.getKeyUses(keyed.Key()) +} + +func (lb *resourceUseStats) getKeyUses(key string) int { + lb.lock.RLock() + defer lb.lock.RUnlock() + return lb.resourceUses[key] +} + +func (lb *resourceUseStats) addUse(resource Resource) int { + lb.lock.Lock() + defer lb.lock.Unlock() + log.Debug("Adding use", "key", resource.Label()) + key := resource.Key() + lb.resourceUses[key] = lb.resourceUses[key] + 1 + return lb.resourceUses[key] +} + +// waitKey blocks until some key is added to the load balancer stats. +// As peer resource initialization is asynchronous we need a way to know that the initial uses has been initialized. +func (lb *resourceUseStats) waitKey(key string) { + lb.lock.Lock() + if _, ok := lb.resourceUses[key]; ok { + lb.lock.Unlock() + return + } + waitChan := make(chan struct{}) + lb.waiting[key] = waitChan + lb.lock.Unlock() + select { + case <-waitChan: + delete(lb.waiting, key) + case <-lb.quitC: + return + } +} + +func (lb *resourceUseStats) initKey(key string, count int) { + lb.lock.Lock() + defer lb.lock.Unlock() + lb.resourceUses[key] = count + if kChan, ok := lb.waiting[key]; ok { + kChan <- struct{}{} + } +} diff --git a/pss/forwarding_test.go b/pss/forwarding_test.go index b0c9d3da22..4436f1d472 100644 --- a/pss/forwarding_test.go +++ b/pss/forwarding_test.go @@ -199,7 +199,7 @@ func TestForwardBasic(t *testing.T) { expected: all[indexAtPo8:], exclusive: false, } - //testCases = append(testCases, c) + testCases = append(testCases, c) // luminous radius of 256 bits, proximity order 8 a4 := pot.Address{} @@ -211,7 +211,7 @@ func TestForwardBasic(t *testing.T) { expected: []int{indexAtPo8, indexAtPo8 + 1}, exclusive: true, } - //testCases = append(testCases, c) + testCases = append(testCases, c) // check correct behaviour in case send fails for i := 2; i < firstNearest-3; i += 2 { diff --git a/pss/pss.go b/pss/pss.go index 7c8784f507..11363bfea1 100644 --- a/pss/pss.go +++ b/pss/pss.go @@ -751,7 +751,7 @@ func (p *Pss) forward(msg *message.Message) error { } for _, lbPeer := range bin.LBPeers { if sendFunc(p, lbPeer.Peer, msg) { - lbPeer.Use() + lbPeer.AddUseCount() sent++ if onlySendOnce { return false From 21a0d2f6313e41aaae45646c9dec1c94266ddc7c Mon Sep 17 00:00:00 2001 From: Alvaro Date: Tue, 29 Oct 2019 08:23:15 +0100 Subject: [PATCH 20/31] added gopubsub unit tests --- network/gopubsub/pubsub_test.go | 70 +++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 network/gopubsub/pubsub_test.go diff --git a/network/gopubsub/pubsub_test.go b/network/gopubsub/pubsub_test.go new file mode 100644 index 0000000000..8789305417 --- /dev/null +++ b/network/gopubsub/pubsub_test.go @@ -0,0 +1,70 @@ +package gopubsub_test + +import ( + "fmt" + "github.com/ethersphere/swarm/network/gopubsub" + "sync" + "testing" +) + +func TestPubSeveralSub(t *testing.T) { + pubSub := gopubsub.New() + var group sync.WaitGroup + bucketSubs1, _ := testSubscriptor(pubSub, 2, &group) + bucketSubs2, _ := testSubscriptor(pubSub, 2, &group) + + fmt.Println("Adding message 0") + pubSub.Publish(struct {}{}) + fmt.Println("Adding message 1") + pubSub.Publish(struct {}{}) + group.Wait() + pubSub.Close() + if len(bucketSubs1) != 2 { + t.Errorf("Subscriptor 1 should have received 2 message, instead %v", len(bucketSubs1)) + } + + if len(bucketSubs2) != 2 { + t.Errorf("Subscriptor 1 should have received 2 message, instead %v", len(bucketSubs2)) + } + +} + +func TestPubUnsubscribe(t *testing.T) { + pubSub := gopubsub.New() + var group sync.WaitGroup + _, subscription := testSubscriptor(pubSub, 0, &group) + msgBucket2, _ := testSubscriptor(pubSub, 1, &group) + pubSub.Publish(struct {}{}) + group.Wait() + if len(msgBucket2) != 1 { + t.Errorf("Subscriptor 2 should have received 1 message regardless of sub 1 unsubscribing, instead %v", len(msgBucket2)) + } + + if pubSub.NumSubscriptions() == 2 || !subscription.IsClosed() { + t.Errorf("Subscription should have been closed") + } +} + +func testSubscriptor( pubsub *gopubsub.PubSubChannel, expectedMessages int, group *sync.WaitGroup) (map[int]interface{}, *gopubsub.Subscription) { + msgBucket := make(map[int]interface{}) + subscription := pubsub.Subscribe() + group.Add(1) + go func(subscription *gopubsub.Subscription) { + defer group.Done() + if expectedMessages == 0 { + subscription.Unsubscribe() + return + } + var i int + for msg := range subscription.ReceiveChannel() { + fmt.Println("Received message", "id", subscription.ID(), "msg", msg) + msgBucket[i] = msg + i++ + if i >= expectedMessages { + return + } + } + fmt.Println("Finishing subscriptor gofunc", "id", subscription.ID()) + }(subscription) + return msgBucket, subscription +} From c615cddd85cfff13a1b39dc2285219095eb8d811 Mon Sep 17 00:00:00 2001 From: Alvaro Date: Tue, 29 Oct 2019 12:44:14 +0100 Subject: [PATCH 21/31] Moved resource_use_stats to its own package. Renamed gopubsub to pubsubchannel. Some PR comments. Fixed TestEachBinBaseUses so it is more stable --- network/gopubsub/pubsub.go | 126 -------------- network/gopubsub/pubsub_test.go | 70 -------- network/kademlia.go | 17 +- network/kademlia_load_balancer.go | 64 ++++--- network/kademlia_load_balancer_test.go | 162 ++++++++++++------ network/pubsubchannel/pubsub.go | 159 +++++++++++++++++ network/pubsubchannel/pubsub_test.go | 86 ++++++++++ .../resource_use_stats.go | 64 ++++--- pss/forwarding_test.go | 2 +- 9 files changed, 445 insertions(+), 305 deletions(-) delete mode 100644 network/gopubsub/pubsub.go delete mode 100644 network/gopubsub/pubsub_test.go create mode 100644 network/pubsubchannel/pubsub.go create mode 100644 network/pubsubchannel/pubsub_test.go rename network/{ => resourceusestats}/resource_use_stats.go (65%) diff --git a/network/gopubsub/pubsub.go b/network/gopubsub/pubsub.go deleted file mode 100644 index eb39000d18..0000000000 --- a/network/gopubsub/pubsub.go +++ /dev/null @@ -1,126 +0,0 @@ -package gopubsub - -import ( - "fmt" - "strconv" - "sync" -) - -//PubSubChannel represents a pubsub system where subscriber can .Subscribe() and publishers can .Publish() or .Close(). -type PubSubChannel struct { - subscriptions []*Subscription - subsMutex sync.RWMutex - nextId int -} - -//Subscription is created in PubSubChannel using pubSub.Subscribe(). Subscribers can receive using .ReceiveChannel(). -// or .Unsubscribe() -type Subscription struct { - closed bool - removeSub func() - signal chan interface{} - closeOnce sync.Once - id string -} - -//New creates a new PubSubChannel. -func New() *PubSubChannel { - return &PubSubChannel{ - subscriptions: make([]*Subscription, 0), - } -} - -//Subscribe creates a subscription to a channel, each subscriber should keep its own Subscription instance. -func (psc *PubSubChannel) Subscribe() *Subscription { - psc.subsMutex.Lock() - defer psc.subsMutex.Unlock() - newSubscription := newSubscription(strconv.Itoa(psc.nextId)) - psc.nextId++ - psc.subscriptions = append(psc.subscriptions, &newSubscription) - newSubscription.removeSub = func() { - psc.subsMutex.Lock() - defer psc.subsMutex.Unlock() - - for i, subscription := range psc.subscriptions { - if subscription.signal == newSubscription.signal { - fmt.Println("Unsubscribing", "id", subscription.id) - subscription.closed = true - psc.subscriptions = append(psc.subscriptions[:i], psc.subscriptions[i+1:]...) - } - } - } - return &newSubscription -} - -//Publish broadcasts a message asynchronously to each subscriber. -//If some of the subscriptions(channels) has been marked as closeable, it does it now. -func (psc *PubSubChannel) Publish(msg interface{}) { - psc.subsMutex.RLock() - defer psc.subsMutex.RUnlock() - for i, sub := range psc.subscriptions { - if sub.closed { - fmt.Println("Subscription was closed", "id", sub.id) - sub.closeChannel() - } else { - go func(sub *Subscription, index int) { - sub.signal <- msg - }(sub, i) - - } - } -} - -//NumSubscriptions returns how many subscriptions are currently active. -func (psc *PubSubChannel) NumSubscriptions() int { - psc.subsMutex.RLock() - defer psc.subsMutex.RUnlock() - return len(psc.subscriptions) -} - -//Close cancels all subscriptions closing the channels associated with them. -//Usually the publisher is in charge of calling Close(). -func (psc *PubSubChannel) Close() { - psc.subsMutex.Lock() - defer psc.subsMutex.Unlock() - for _, sub := range psc.subscriptions { - sub.closed = true - sub.closeChannel() - } -} - -//Unsubscribe cancels subscription from the subscriber side. Channel is marked as closed but only writer should close it. -func (sub *Subscription) Unsubscribe() { - sub.closed = true - sub.removeSub() -} - -//ReceiveChannel returns the channel where the subscriber will receive messages. -func (sub *Subscription) ReceiveChannel() <-chan interface{} { - return sub.signal -} - -//IsClosed returns if the subscription is closed via Unsubscribe() or Close() in the pubSub that creates it. -func (sub *Subscription) IsClosed() bool { - return sub.closed -} - -//ID returns a unique id in the PubSubChannel of this subscription. Useful for debugging. -func (sub *Subscription) ID() string { - return sub.id -} - -func (sub *Subscription) closeChannel() { - sub.closeOnce.Do(func() { - close(sub.signal) - }) -} - -func newSubscription(id string) Subscription { - return Subscription{ - closed: false, - removeSub: nil, - signal: make(chan interface{}), - closeOnce: sync.Once{}, - id: id, - } -} diff --git a/network/gopubsub/pubsub_test.go b/network/gopubsub/pubsub_test.go deleted file mode 100644 index 8789305417..0000000000 --- a/network/gopubsub/pubsub_test.go +++ /dev/null @@ -1,70 +0,0 @@ -package gopubsub_test - -import ( - "fmt" - "github.com/ethersphere/swarm/network/gopubsub" - "sync" - "testing" -) - -func TestPubSeveralSub(t *testing.T) { - pubSub := gopubsub.New() - var group sync.WaitGroup - bucketSubs1, _ := testSubscriptor(pubSub, 2, &group) - bucketSubs2, _ := testSubscriptor(pubSub, 2, &group) - - fmt.Println("Adding message 0") - pubSub.Publish(struct {}{}) - fmt.Println("Adding message 1") - pubSub.Publish(struct {}{}) - group.Wait() - pubSub.Close() - if len(bucketSubs1) != 2 { - t.Errorf("Subscriptor 1 should have received 2 message, instead %v", len(bucketSubs1)) - } - - if len(bucketSubs2) != 2 { - t.Errorf("Subscriptor 1 should have received 2 message, instead %v", len(bucketSubs2)) - } - -} - -func TestPubUnsubscribe(t *testing.T) { - pubSub := gopubsub.New() - var group sync.WaitGroup - _, subscription := testSubscriptor(pubSub, 0, &group) - msgBucket2, _ := testSubscriptor(pubSub, 1, &group) - pubSub.Publish(struct {}{}) - group.Wait() - if len(msgBucket2) != 1 { - t.Errorf("Subscriptor 2 should have received 1 message regardless of sub 1 unsubscribing, instead %v", len(msgBucket2)) - } - - if pubSub.NumSubscriptions() == 2 || !subscription.IsClosed() { - t.Errorf("Subscription should have been closed") - } -} - -func testSubscriptor( pubsub *gopubsub.PubSubChannel, expectedMessages int, group *sync.WaitGroup) (map[int]interface{}, *gopubsub.Subscription) { - msgBucket := make(map[int]interface{}) - subscription := pubsub.Subscribe() - group.Add(1) - go func(subscription *gopubsub.Subscription) { - defer group.Done() - if expectedMessages == 0 { - subscription.Unsubscribe() - return - } - var i int - for msg := range subscription.ReceiveChannel() { - fmt.Println("Received message", "id", subscription.ID(), "msg", msg) - msgBucket[i] = msg - i++ - if i >= expectedMessages { - return - } - } - fmt.Println("Finishing subscriptor gofunc", "id", subscription.ID()) - }(subscription) - return msgBucket, subscription -} diff --git a/network/kademlia.go b/network/kademlia.go index 4530b62594..1e48c318aa 100644 --- a/network/kademlia.go +++ b/network/kademlia.go @@ -33,7 +33,7 @@ import ( "github.com/ethersphere/swarm/chunk" "github.com/ethersphere/swarm/log" "github.com/ethersphere/swarm/network/capability" - "github.com/ethersphere/swarm/network/gopubsub" + "github.com/ethersphere/swarm/network/pubsubchannel" "github.com/ethersphere/swarm/pot" sv "github.com/ethersphere/swarm/version" ) @@ -101,8 +101,8 @@ type Kademlia struct { nDepthMu sync.RWMutex // protects neighbourhood depth nDepth nDepthSig []chan struct{} // signals when neighbourhood depth nDepth is changed - newPeerPubSub *gopubsub.PubSubChannel - removedPeerPubSub *gopubsub.PubSubChannel + newPeerPubSub *pubsubchannel.PubSubChannel + removedPeerPubSub *pubsubchannel.PubSubChannel } type KademliaInfo struct { @@ -129,8 +129,8 @@ func NewKademlia(addr []byte, params *KadParams) *Kademlia { KadParams: params, capabilityIndex: make(map[string]*capabilityIndex), defaultIndex: NewDefaultIndex(), - newPeerPubSub: gopubsub.New(), - removedPeerPubSub: gopubsub.New(), + newPeerPubSub: pubsubchannel.New(), + removedPeerPubSub: pubsubchannel.New(), } k.RegisterCapabilityIndex("full", *fullCapability) k.RegisterCapabilityIndex("light", *lightCapability) @@ -487,8 +487,7 @@ func (k *Kademlia) On(p *Peer) (uint8, bool) { } func (k *Kademlia) peerPo(peer *Peer) (po int, found bool) { - po, found = Pof(k.defaultIndex.conns.Pin(), peer, 0) - return po, found + return Pof(k.defaultIndex.conns.Pin(), peer, 0) } // setNeighbourhoodDepth calculates neighbourhood depth with depthForPot, @@ -574,7 +573,7 @@ func (k *Kademlia) SubscribeToNeighbourhoodDepthChange() (c <-chan struct{}, uns // when a new Peer is added or removed from the table. Returned function unsubscribes // the channel from signaling and releases the resources. Returned function is safe // to be called multiple times. -func (k *Kademlia) SubscribeToPeerChanges() (addedSub *gopubsub.Subscription, removedPeerSub *gopubsub.Subscription) { +func (k *Kademlia) SubscribeToPeerChanges() (addedSub *pubsubchannel.Subscription, removedPeerSub *pubsubchannel.Subscription) { addedSub = k.newPeerPubSub.Subscribe() removedPeerSub = k.removedPeerPubSub.Subscribe() return @@ -675,7 +674,7 @@ func (k *Kademlia) EachBinDescFiltered(base []byte, capKey string, minProximityO defer k.lock.RUnlock() c, ok := k.capabilityIndex[capKey] if !ok { - return fmt.Errorf("Unregistered capability index '%s'", capKey) + return fmt.Errorf("unregistered capability index '%s'", capKey) } k.eachBinDesc(c, base, minProximityOrder, consumer) return nil diff --git a/network/kademlia_load_balancer.go b/network/kademlia_load_balancer.go index 3effea4f3b..00cacb2c9d 100644 --- a/network/kademlia_load_balancer.go +++ b/network/kademlia_load_balancer.go @@ -17,12 +17,13 @@ package network import ( "github.com/ethersphere/swarm/log" - "github.com/ethersphere/swarm/network/gopubsub" + "github.com/ethersphere/swarm/network/pubsubchannel" + "github.com/ethersphere/swarm/network/resourceusestats" ) // KademliaBackend is the required interface of KademliaLoadBalancer. type KademliaBackend interface { - SubscribeToPeerChanges() (addedSub *gopubsub.Subscription, removedPeerSub *gopubsub.Subscription) + SubscribeToPeerChanges() (addedSub *pubsubchannel.Subscription, removedPeerSub *pubsubchannel.Subscription) BaseAddr() []byte EachBinDesc(base []byte, minProximityOrder int, consumer PeerBinConsumer) EachBinDescFiltered(base []byte, capKey string, minProximityOrder int, consumer PeerBinConsumer) error @@ -38,7 +39,7 @@ func NewKademliaLoadBalancer(kademlia KademliaBackend, useNearestNeighbourInit b quitC := make(chan struct{}) klb := &KademliaLoadBalancer{ kademlia: kademlia, - resourceUseStats: newResourceUseStats(quitC), + resourceUseStats: resourceusestats.NewResourceUseStats(quitC), onPeerSub: onPeerSub, offPeerSub: offPeerSub, quitC: quitC, @@ -61,8 +62,13 @@ func NewKademliaLoadBalancer(kademlia KademliaBackend, useNearestNeighbourInit b // An LBPeer represents a peer with a AddUseCount() function to signal that the peer has been used in order // to account it for LB sorting criteria. type LBPeer struct { - Peer *Peer - AddUseCount func() // called to account a use for these peer. Should be called if the peer is actually used. + Peer *Peer + stats *resourceusestats.ResourceUseStats +} + +// AddUseCount is called to account a use for these peer. Should be called if the peer is actually used. +func (lbPeer LBPeer) AddUseCount() { + lbPeer.stats.AddUse(lbPeer.Peer) } // LBBin represents a Bin of LBPeer's @@ -80,10 +86,10 @@ type LBBinConsumer func(bin LBBin) bool // The user of KademliaLoadBalancer should signal if the returned element (LBPeer) has been used with the // function lbPeer.AddUseCount() type KademliaLoadBalancer struct { - kademlia KademliaBackend // kademlia to obtain bins of peers - resourceUseStats *resourceUseStats // a resourceUseStats to count uses - onPeerSub *gopubsub.Subscription // a pubsub channel to be notified of new peers in kademlia - offPeerSub *gopubsub.Subscription // a pubsub channel to be notified of removed peers in kademlia + kademlia KademliaBackend // kademlia to obtain bins of peers + resourceUseStats *resourceusestats.ResourceUseStats // a resourceUseStats to count uses + onPeerSub *pubsubchannel.Subscription // a pubsub channel to be notified of new peers in kademlia + offPeerSub *pubsubchannel.Subscription // a pubsub channel to be notified of removed peers in kademlia quitC chan struct{} initCountFunc func(peer *Peer, po int) int //Function to use for initializing a new peer count @@ -121,7 +127,7 @@ func (klb *KademliaLoadBalancer) EachBinDesc(base []byte, consumeBin LBBinConsum } func (klb *KademliaLoadBalancer) peerBinToPeerList(bin *PeerBin) []LBPeer { - resources := make([]Resource, bin.Size) + resources := make([]resourceusestats.Resource, bin.Size) var i int bin.PeerIterator(func(entry *entry) bool { resources[i] = entry.conn @@ -131,8 +137,8 @@ func (klb *KademliaLoadBalancer) peerBinToPeerList(bin *PeerBin) []LBPeer { return klb.resourcesToLbPeers(resources) } -func (klb *KademliaLoadBalancer) resourcesToLbPeers(resources []Resource) []LBPeer { - sorted := klb.resourceUseStats.sortResources(resources) +func (klb *KademliaLoadBalancer) resourcesToLbPeers(resources []resourceusestats.Resource) []LBPeer { + sorted := klb.resourceUseStats.SortResources(resources) peers := klb.toLBPeers(sorted) return peers } @@ -144,11 +150,13 @@ func (klb *KademliaLoadBalancer) listenNewPeers() { return case msg, ok := <-klb.onPeerSub.ReceiveChannel(): if !ok { + log.Warn("listenNewPeers closed channel, finishing subscriber to new peer") return } signal, ok := msg.(newPeerSignal) if !ok { - return + log.Warn("listenNewPeers received message is not a new peer signal") + continue } klb.addedPeer(signal.peer, signal.po) } @@ -162,9 +170,15 @@ func (klb *KademliaLoadBalancer) listenOffPeers() { return case msg := <-klb.offPeerSub.ReceiveChannel(): peer, ok := msg.(*Peer) - if peer != nil && ok { - klb.removedPeer(peer) + if peer == nil { + log.Warn("nil peer received listening for off peers. Ignoring.") + continue } + if !ok { + log.Warn("unexpected message received listening for off peers. Ignoring.") + continue + } + klb.removedPeer(peer) } } } @@ -173,9 +187,11 @@ func (klb *KademliaLoadBalancer) listenOffPeers() { // to the use count of the least used peer in its bin. The po of the new peer is passed to avoid having // to calculate it again. func (klb *KademliaLoadBalancer) addedPeer(peer *Peer, po int) { + //log.Warn("Adding peer", "key", peer.Label()) initCount := klb.initCountFunc(peer, 0) log.Debug("Adding peer", "key", peer.Label(), "initCount", initCount) - klb.resourceUseStats.initKey(peer.Key(), initCount) + klb.resourceUseStats.InitKey(peer.Key(), initCount) + //log.Warn("Peer added", "key", peer.Label()) } // leastUsedCountInBin returns the use count for the least used peer in this bin excluding the excludePeer. @@ -187,7 +203,7 @@ func (klb *KademliaLoadBalancer) leastUsedCountInBin(excludePeer *Peer, po int) for idx < len(peersInSamePo) { leastUsed := peersInSamePo[idx] if leastUsed.Peer.Key() != excludePeer.Key() { - leastUsedCount = klb.resourceUseStats.getUses(leastUsed.Peer) + leastUsedCount = klb.resourceUseStats.GetUses(leastUsed.Peer) log.Debug("Least used peer is", "peer", leastUsed.Peer.Label(), "leastUsedCount", leastUsedCount) break } @@ -201,7 +217,7 @@ func (klb *KademliaLoadBalancer) nearestNeighbourUseCount(newPeer *Peer, _ int) var count int klb.kademlia.EachConn(newPeer.Address(), 255, func(peer *Peer, po int) bool { if peer != newPeer { - count = klb.resourceUseStats.getUses(peer) + count = klb.resourceUseStats.GetUses(peer) log.Debug("Nearest neighbour is", "peer", peer.Label(), "count", count) return false } @@ -211,25 +227,21 @@ func (klb *KademliaLoadBalancer) nearestNeighbourUseCount(newPeer *Peer, _ int) } func (klb *KademliaLoadBalancer) removedPeer(peer *Peer) { - klb.resourceUseStats.lock.Lock() - defer klb.resourceUseStats.lock.Unlock() - delete(klb.resourceUseStats.resourceUses, peer.Key()) + klb.resourceUseStats.RemoveResource(peer) } -func (klb *KademliaLoadBalancer) toLBPeers(resources []Resource) []LBPeer { +func (klb *KademliaLoadBalancer) toLBPeers(resources []resourceusestats.Resource) []LBPeer { peers := make([]LBPeer, len(resources)) for i, res := range resources { peer := res.(*Peer) peers[i].Peer = peer - peers[i].AddUseCount = func() { - klb.resourceUseStats.addUse(peer) - } + peers[i].stats = klb.resourceUseStats } return peers } func (klb *KademliaLoadBalancer) getPeersForPo(base []byte, po int) []LBPeer { - resources := make([]Resource, 0) + resources := make([]resourceusestats.Resource, 0) klb.kademlia.EachBinDesc(base, po, func(bin *PeerBin) bool { if bin.ProximityOrder == po { return bin.PeerIterator(func(entry *entry) bool { diff --git a/network/kademlia_load_balancer_test.go b/network/kademlia_load_balancer_test.go index 143b2a4832..7c4f58e27a 100644 --- a/network/kademlia_load_balancer_test.go +++ b/network/kademlia_load_balancer_test.go @@ -1,3 +1,18 @@ +// Copyright 2019 The Swarm Authors +// This file is part of the Swarm library. +// +// The Swarm library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The Swarm library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the Swarm library. If not, see . package network import ( @@ -7,8 +22,9 @@ import ( "time" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethersphere/swarm/log" "github.com/ethersphere/swarm/network/capability" - "github.com/ethersphere/swarm/network/gopubsub" + "github.com/ethersphere/swarm/network/pubsubchannel" "github.com/ethersphere/swarm/pot" ) @@ -23,7 +39,7 @@ func TestAddedNodes(t *testing.T) { klb := NewKademliaLoadBalancer(kademlia, false) defer klb.Stop() - firstUses := klb.resourceUseStats.getUses(first) + firstUses := klb.resourceUseStats.GetUses(first) if firstUses != 0 { t.Errorf("Expected 0 uses for new peer at start") } @@ -32,8 +48,8 @@ func TestAddedNodes(t *testing.T) { // Now new peers still should have 0 uses third := newTestKadPeer("011101011") kademlia.addPeer(third) - klb.resourceUseStats.waitKey(third.Key()) - thirdUses := klb.resourceUseStats.getUses(third) + klb.resourceUseStats.WaitKey(third.Key()) + thirdUses := klb.resourceUseStats.GetUses(third) if thirdUses != 0 { t.Errorf("Expected 0 uses for new peer because minimum in bin is 0. Instead %v", thirdUses) } @@ -43,8 +59,8 @@ func TestAddedNodes(t *testing.T) { //New peers should start with 1 use fourth := newTestKadPeer("011100011") kademlia.addPeer(fourth) - klb.resourceUseStats.waitKey(fourth.Key()) - fourthUses := klb.resourceUseStats.getUses(fourth) + klb.resourceUseStats.WaitKey(fourth.Key()) + fourthUses := klb.resourceUseStats.GetUses(fourth) if fourthUses != 1 { t.Errorf("Expected 1 use for new peer because minimum in bin should be 1. Instead %v", fourthUses) } @@ -61,7 +77,7 @@ func TestAddedNodesNearestNeighbour(t *testing.T) { klb := NewKademliaLoadBalancer(kademlia, true) defer klb.Stop() - firstUses := klb.resourceUseStats.getUses(first) + firstUses := klb.resourceUseStats.GetUses(first) if firstUses != 0 { t.Errorf("Expected 0 uses for new peer at start") } @@ -70,9 +86,9 @@ func TestAddedNodesNearestNeighbour(t *testing.T) { // Now third peer should have the same uses as second third := newTestKadPeer("01110111") // most similar peer is second 01110101 kademlia.addPeer(third) - klb.resourceUseStats.waitKey(third.Key()) - secondUses := klb.resourceUseStats.getUses(second) - thirdUses := klb.resourceUseStats.getUses(third) + klb.resourceUseStats.WaitKey(third.Key()) + secondUses := klb.resourceUseStats.GetUses(second) + thirdUses := klb.resourceUseStats.GetUses(third) if thirdUses != secondUses { t.Errorf("Expected %v uses for new peer because is most similar to second. Instead %v", secondUses, thirdUses) } @@ -87,16 +103,18 @@ func TestAddedNodesNearestNeighbour(t *testing.T) { fourth := newTestKadPeer("01110110") // most similar peer is third 01110111 kademlia.addPeer(fourth) - klb.resourceUseStats.waitKey(fourth.Key()) + klb.resourceUseStats.WaitKey(fourth.Key()) //We expect fourth to be initialized with third peer use count - fourthUses := klb.resourceUseStats.getUses(fourth) - thirdUses = klb.resourceUseStats.getUses(third) + fourthUses := klb.resourceUseStats.GetUses(fourth) + thirdUses = klb.resourceUseStats.GetUses(third) if fourthUses != thirdUses { t.Errorf("Expected %v use for new peer because most similar is peer 3. Instead %v", thirdUses, fourthUses) } } +var testCount = 0 + // TestEachBinBaseUses tests that EachBinDesc returns first the least used peer in its bin // We will create 3 bins with two peers each. We will call EachBinDesc 6 times twice with an address // on each bin, so at the end all peers should have 1 use (because the address in each bin is equidistant to @@ -104,59 +122,99 @@ func TestAddedNodesNearestNeighbour(t *testing.T) { // Then we will use an address in a bin that is nearer to one of the peers and we will check that that peer is always // returned first. func TestEachBinBaseUses(t *testing.T) { + myCount := testCount + testCount++ tk := newTestKademlia(t, "11111111") klb := NewKademliaLoadBalancer(tk, false) - tk.On("01010101") //Peer 1 dec 85 - tk.On("01010100") // 2 dec 84 - tk.On("10010100") // 3 dec 148 - tk.On("10010001") // 4 dec 145 - tk.On("11010100") // 5 dec 212 - tk.On("11010101") // 6 dec 213 - - pivotAddressBin0 := pot.NewAddressFromString("00000000") // Two nearest peers (1,2) - pivotAddressBin1 := pot.NewAddressFromString("10000000") // Two nearest peers (3,4) - pivotAddressBin2 := pot.NewAddressFromString("11000000") // Two nearest peers (5,6) + tk.On("01010101") //Peer 1 dec 85 hex 55 + tk.On("01010100") // 2 dec 84 hex 54 + tk.On("10010100") // 3 dec 148 hex 94 + tk.On("10010001") // 4 dec 145 hex 91 + tk.On("11010100") // 5 dec 212 hex d4 + tk.On("11010101") // 6 dec 213 hex d5 + + //Waiting for all peers to be registered + resources := klb.resourceUseStats.Len() + for resources != 6 { + time.Sleep(10 * time.Millisecond) + resources = klb.resourceUseStats.Len() + } + + pivotAddressBin0 := pot.NewAddressFromString("00000000") // Two nearest peers (1,2) hex 00 + pivotAddressBin1 := pot.NewAddressFromString("10000000") // Two nearest peers (3,4) hex 80 + pivotAddressBin2 := pot.NewAddressFromString("11000000") // Two nearest peers (5,6) hex c0 countUse := func(bin LBBin) bool { - bin.LBPeers[0].AddUseCount() + peerLogLines := make([]string, 0) + for idx, lbPeer := range bin.LBPeers { + currentUses := klb.resourceUseStats.GetUses(lbPeer.Peer) + peerLogLine := "Peer " + peerToBitString(lbPeer.Peer) + " " + string(idx) + " currentUses " + strconv.FormatInt(int64(currentUses), 10) + peerLogLines = append(peerLogLines, peerLogLine) + } + + log.Debug("peers for address in bin", "peers", peerLogLines, "po", bin.ProximityOrder, "count", myCount) + chosen := bin.LBPeers[0] + log.Debug("Chosen peer is", "chosen", chosen.Peer.Label(), "uses", klb.resourceUseStats.GetUses(chosen.Peer), "count", myCount) + chosen.AddUseCount() return false } // Use peer 1 and 2 klb.EachBinDesc(pivotAddressBin0, countUse) klb.EachBinDesc(pivotAddressBin0, countUse) + peer1Uses := klb.resourceUseStats.GetKeyUses(bitStringToHex("01010101")) + if peer1Uses != 1 { + t.Errorf("expected %v uses of %v but got %v", 1, "01010101", peer1Uses) + } + peer2Uses := klb.resourceUseStats.GetKeyUses(bitStringToHex("01010100")) + if peer2Uses != 1 { + t.Errorf("expected %v uses of %v but got %v", 1, "01010100", peer2Uses) + } + // Use peers 3 and 4 klb.EachBinDesc(pivotAddressBin1, countUse) klb.EachBinDesc(pivotAddressBin1, countUse) + peer3Uses := klb.resourceUseStats.GetKeyUses(bitStringToHex("10010100")) + if peer3Uses != 1 { + t.Errorf("expected %v uses of %v but got %v", 1, "10010100", peer3Uses) + } + peer4Uses := klb.resourceUseStats.GetKeyUses(bitStringToHex("10010001")) + if peer4Uses != 1 { + t.Errorf("expected %v uses of %v but got %v", 1, "10010001", peer4Uses) + } + // Use peers 5 and 6 klb.EachBinDesc(pivotAddressBin2, countUse) klb.EachBinDesc(pivotAddressBin2, countUse) - resourceUses := klb.resourceUseStats.dumpAllUses() - if len(resourceUses) != 6 { - t.Errorf("Expected all 6 peers to be used but got %v", len(resourceUses)) + peer5Uses := klb.resourceUseStats.GetKeyUses(bitStringToHex("11010100")) + if peer5Uses != 1 { + t.Errorf("expected %v uses of %v but got %v", 1, "11010100", peer5Uses) } - for key, uses := range resourceUses { - if uses != 1 { - bytes, _ := hexutil.Decode(key) - binaryKey := byteToBitString(bytes[0]) + byteToBitString(bytes[1]) - t.Errorf("Expected only 1 use of %v but got %v", binaryKey, uses) - } + peer6Uses := klb.resourceUseStats.GetKeyUses(bitStringToHex("11010101")) + if peer6Uses != 1 { + t.Errorf("expected %v uses of %v but got %v", 1, "11010101", peer6Uses) } //Now a message that is nearer 10010001 than 10010100 in its bin. It will be taken always regardless of uses - pivotAddressBin3 := pot.NewAddressFromString("10010011") // Nearer 4 + pivotAddressBin3 := pot.NewAddressFromString("10010011") // Nearer 4 hex 93 //Both calls to 4 klb.EachBinDesc(pivotAddressBin3, countUse) klb.EachBinDesc(pivotAddressBin3, countUse) - count := klb.resourceUseStats.getKeyUses(bitStringToHex("10010001")) + count := klb.resourceUseStats.GetKeyUses(bitStringToHex("10010001")) if count != 3 { t.Errorf("Expected 3 uses of 10010001 but got %v", count) } } +func expectUses(actualUses int, expected int, peer string, t *testing.T) { + if actualUses != expected { + t.Errorf("expected %v uses of %v but got %v", expected, peer, actualUses) + } +} + // TestEachBinFiltered checks that when load balancing peers, only those with the provided capabilities are chosen. func TestEachBinFiltered(t *testing.T) { tk := newTestKademlia(t, "11111111") @@ -172,19 +230,19 @@ func TestEachBinFiltered(t *testing.T) { capPeer := tk.newTestKadPeerWithCapabilities("10100000", caps[capKey]) tk.Kademlia.On(capPeer) useStats := klb.resourceUseStats - useStats.waitKey(capPeer.Key()) + useStats.WaitKey(capPeer.Key()) tk.On("01010101") // bin 0 dec 85 hex 55 - useStats.waitKey(bitStringToHex("01010101")) + useStats.WaitKey(bitStringToHex("01010101")) tk.On("01010100") // bin 0 dec 84 hex 54 - useStats.waitKey(bitStringToHex("01010100")) + useStats.WaitKey(bitStringToHex("01010100")) tk.On("10010100") // bin 1 dec 148 - useStats.waitKey(bitStringToHex("10010100")) + useStats.WaitKey(bitStringToHex("10010100")) tk.On("10010001") // bin 1 dec 145 - useStats.waitKey(bitStringToHex("10010001")) + useStats.WaitKey(bitStringToHex("10010001")) tk.On("11010100") // bin 2 dec 212 - useStats.waitKey(bitStringToHex("11010100")) + useStats.WaitKey(bitStringToHex("11010100")) tk.On("11010101") // bin 2 dec 213 - useStats.waitKey(bitStringToHex("11010101")) + useStats.WaitKey(bitStringToHex("11010101")) stats := make(map[string]int) countUse := func(bin LBBin) bool { peer := bin.LBPeers[0].Peer @@ -200,19 +258,19 @@ func TestEachBinFiltered(t *testing.T) { klb.EachBinFiltered(pivotAddressBin1, capKey, countUse) klb.EachBinFiltered(pivotAddressBin1, capKey, countUse) - count := useStats.getUses(capPeer) + count := useStats.GetUses(capPeer) if count != 3 || stats["10100000"] != 3 { t.Errorf("Expected 3 uses of capability peer but got %v/%v", count, stats["10100000"]) } secondCapPeer := tk.newTestKadPeerWithCapabilities("10100001", caps[capKey]) tk.Kademlia.On(secondCapPeer) - useStats.waitKey(secondCapPeer.Key()) - secondCountStart := useStats.getUses(secondCapPeer) - count = useStats.getUses(capPeer) + useStats.WaitKey(secondCapPeer.Key()) + secondCountStart := useStats.GetUses(secondCapPeer) + count = useStats.GetUses(capPeer) klb.EachBinFiltered(pivotAddressBin1, capKey, countUse) klb.EachBinFiltered(pivotAddressBin1, capKey, countUse) - secondCount := useStats.getUses(secondCapPeer) + secondCount := useStats.GetUses(secondCapPeer) if secondCount-secondCountStart != 2 { t.Errorf("Expected 2 uses of second capability peer but got %v", secondCount-secondCountStart) } @@ -221,8 +279,8 @@ func TestEachBinFiltered(t *testing.T) { type testKademliaBackend struct { baseAddr []byte - addedChannel *gopubsub.PubSubChannel - removedChannel *gopubsub.PubSubChannel + addedChannel *pubsubchannel.PubSubChannel + removedChannel *pubsubchannel.PubSubChannel bins map[int][]*Peer maxPo int } @@ -275,8 +333,8 @@ func (tkb *testKademliaBackend) EachConn(base []byte, maxPo int, consume func(*P func newTestKademliaBackend(address string) *testKademliaBackend { return &testKademliaBackend{ baseAddr: pot.NewAddressFromString(address), - addedChannel: gopubsub.New(), - removedChannel: gopubsub.New(), + addedChannel: pubsubchannel.New(), + removedChannel: pubsubchannel.New(), bins: make(map[int][]*Peer), } } @@ -289,7 +347,7 @@ func (tkb testKademliaBackend) BaseAddr() []byte { // SubscribeToPeerChanges returns a subscription to changes in the kademlia peers. It contains channels to notify about // peers added/removed from this kademlia. -func (tkb *testKademliaBackend) SubscribeToPeerChanges() (addedSub *gopubsub.Subscription, removedPeerSub *gopubsub.Subscription) { +func (tkb *testKademliaBackend) SubscribeToPeerChanges() (addedSub *pubsubchannel.Subscription, removedPeerSub *pubsubchannel.Subscription) { addedSub = tkb.addedChannel.Subscribe() removedPeerSub = tkb.removedChannel.Subscribe() return diff --git a/network/pubsubchannel/pubsub.go b/network/pubsubchannel/pubsub.go new file mode 100644 index 0000000000..2a6bc75306 --- /dev/null +++ b/network/pubsubchannel/pubsub.go @@ -0,0 +1,159 @@ +// Copyright 2019 The Swarm Authors +// This file is part of the Swarm library. +// +// The Swarm library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The Swarm library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the Swarm library. If not, see . +package pubsubchannel + +import ( + "strconv" + "sync" + "time" + + "github.com/ethersphere/swarm/log" +) + +var subscriptionTimeout = 100 * time.Millisecond + +//PubSubChannel represents a pubsub system where subscriber can .Subscribe() and publishers can .Publish() or .Close(). +type PubSubChannel struct { + subscriptions []*Subscription + subsMutex sync.RWMutex + nextId int +} + +// Subscription is created in PubSubChannel using pubSub.Subscribe(). Subscribers can receive using .ReceiveChannel(). +// or .Unsubscribe() +type Subscription struct { + closed bool + pubSubC *PubSubChannel + //removeSub func() + signal chan interface{} + closeOnce sync.Once + id string + lock sync.RWMutex +} + +// New creates a new PubSubChannel. +func New() *PubSubChannel { + return &PubSubChannel{ + subscriptions: make([]*Subscription, 0), + } +} + +// Subscribe creates a subscription to a channel, each subscriber should keep its own Subscription instance. +func (psc *PubSubChannel) Subscribe() *Subscription { + psc.subsMutex.Lock() + defer psc.subsMutex.Unlock() + newSubscription := newSubscription(strconv.Itoa(psc.nextId), psc) + psc.nextId++ + psc.subscriptions = append(psc.subscriptions, &newSubscription) + + return &newSubscription +} + +func (psc *PubSubChannel) removeSub(s *Subscription) { + psc.subsMutex.Lock() + defer psc.subsMutex.Unlock() + + for i, subscription := range psc.subscriptions { + if subscription.signal == s.signal { + log.Debug("Unsubscribing", "id", subscription.id) + subscription.lock.Lock() + subscription.closed = true + subscription.lock.Unlock() + psc.subscriptions = append(psc.subscriptions[:i], psc.subscriptions[i+1:]...) + } + } +} + +// Publish broadcasts a message asynchronously to each subscriber. +// If some of the subscriptions(channels) has been marked as closeable, it does it now. +func (psc *PubSubChannel) Publish(msg interface{}) { + psc.subsMutex.RLock() + defer psc.subsMutex.RUnlock() + for _, sub := range psc.subscriptions { + sub.lock.RLock() + if sub.closed { + log.Debug("Subscription was closed", "id", sub.id) + sub.closeChannel() + } else { + go func(sub *Subscription) { + select { + case sub.signal <- msg: + case <-time.After(subscriptionTimeout): + log.Warn("Subscription unattended after timeout", "subId", sub.ID(), "timeout", subscriptionTimeout) + } + }(sub) + } + sub.lock.RUnlock() + } +} + +// NumSubscriptions returns how many subscriptions are currently active. +func (psc *PubSubChannel) NumSubscriptions() int { + psc.subsMutex.RLock() + defer psc.subsMutex.RUnlock() + return len(psc.subscriptions) +} + +// Close cancels all subscriptions closing the channels associated with them. +// Usually the publisher is in charge of calling Close(). +func (psc *PubSubChannel) Close() { + psc.subsMutex.Lock() + defer psc.subsMutex.Unlock() + for _, sub := range psc.subscriptions { + sub.lock.Lock() + sub.closed = true + sub.closeChannel() + sub.lock.Unlock() + } +} + +// Unsubscribe cancels subscription from the subscriber side. Channel is marked as closed but only writer should close it. +func (sub *Subscription) Unsubscribe() { + sub.pubSubC.removeSub(sub) +} + +// ReceiveChannel returns the channel where the subscriber will receive messages. +func (sub *Subscription) ReceiveChannel() <-chan interface{} { + return sub.signal +} + +// IsClosed returns if the subscription is closed via Unsubscribe() or Close() in the pubSub that creates it. +func (sub *Subscription) IsClosed() bool { + sub.lock.RLock() + defer sub.lock.RUnlock() + return sub.closed +} + +// ID returns a unique id in the PubSubChannel of this subscription. Useful for debugging. +func (sub *Subscription) ID() string { + return sub.id +} + +func (sub *Subscription) closeChannel() { + sub.closeOnce.Do(func() { + close(sub.signal) + }) +} + +func newSubscription(id string, psc *PubSubChannel) Subscription { + return Subscription{ + closed: false, + pubSubC: psc, + signal: make(chan interface{}, 20), + closeOnce: sync.Once{}, + id: id, + } +} diff --git a/network/pubsubchannel/pubsub_test.go b/network/pubsubchannel/pubsub_test.go new file mode 100644 index 0000000000..a917e1e558 --- /dev/null +++ b/network/pubsubchannel/pubsub_test.go @@ -0,0 +1,86 @@ +// Copyright 2019 The Swarm Authors +// This file is part of the Swarm library. +// +// The Swarm library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The Swarm library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the Swarm library. If not, see . +package pubsubchannel_test + +import ( + "sync" + "testing" + + "github.com/ethersphere/swarm/log" + "github.com/ethersphere/swarm/network/pubsubchannel" +) + +func TestPubSeveralSub(t *testing.T) { + pubSub := pubsubchannel.New() + var group sync.WaitGroup + bucketSubs1, _ := testSubscriptor(pubSub, 2, &group) + bucketSubs2, _ := testSubscriptor(pubSub, 2, &group) + + log.Debug("Adding message 0") + pubSub.Publish(struct{}{}) + log.Debug("Adding message 1") + pubSub.Publish(struct{}{}) + group.Wait() + pubSub.Close() + if len(bucketSubs1) != 2 { + t.Errorf("Subscriptor 1 should have received 2 message, instead %v", len(bucketSubs1)) + } + + if len(bucketSubs2) != 2 { + t.Errorf("Subscriptor 1 should have received 2 message, instead %v", len(bucketSubs2)) + } + +} + +func TestPubUnsubscribe(t *testing.T) { + pubSub := pubsubchannel.New() + var group sync.WaitGroup + _, subscription := testSubscriptor(pubSub, 0, &group) + msgBucket2, _ := testSubscriptor(pubSub, 1, &group) + pubSub.Publish(struct{}{}) + group.Wait() + if len(msgBucket2) != 1 { + t.Errorf("Subscriptor 2 should have received 1 message regardless of sub 1 unsubscribing, instead %v", len(msgBucket2)) + } + + if pubSub.NumSubscriptions() == 2 || !subscription.IsClosed() { + t.Errorf("Subscription should have been closed") + } +} + +func testSubscriptor(pubsub *pubsubchannel.PubSubChannel, expectedMessages int, group *sync.WaitGroup) (map[int]interface{}, *pubsubchannel.Subscription) { + msgBucket := make(map[int]interface{}) + subscription := pubsub.Subscribe() + group.Add(1) + go func(subscription *pubsubchannel.Subscription) { + defer group.Done() + if expectedMessages == 0 { + subscription.Unsubscribe() + return + } + var i int + for msg := range subscription.ReceiveChannel() { + log.Debug("Received message", "id", subscription.ID(), "msg", msg) + msgBucket[i] = msg + i++ + if i >= expectedMessages { + return + } + } + log.Debug("Finishing subscriptor gofunc", "id", subscription.ID()) + }(subscription) + return msgBucket, subscription +} diff --git a/network/resource_use_stats.go b/network/resourceusestats/resource_use_stats.go similarity index 65% rename from network/resource_use_stats.go rename to network/resourceusestats/resource_use_stats.go index f13df06857..3c9940dc24 100644 --- a/network/resource_use_stats.go +++ b/network/resourceusestats/resource_use_stats.go @@ -13,7 +13,7 @@ // // You should have received a copy of the GNU Lesser General Public License // along with the Swarm library. If not, see . -package network +package resourceusestats import ( "sort" @@ -23,12 +23,12 @@ import ( "github.com/ethersphere/swarm/log" ) -// resourceUseStats can be used to count uses of resources. A Resource is anything with a Key() -type resourceUseStats struct { +// ResourceUseStats can be used to count uses of resources. A Resource is anything with a Key() +type ResourceUseStats struct { resourceUses map[string]int waiting map[string]chan struct{} lock sync.RWMutex - quitC chan struct{} + quitC <-chan struct{} } // Resource represents anything with a Key that can be accounted with some stat. @@ -42,15 +42,15 @@ type ResourceCount struct { count int } -func newResourceUseStats(quitC chan struct{}) *resourceUseStats { - return &resourceUseStats{ +func NewResourceUseStats(quitC <-chan struct{}) *ResourceUseStats { + return &ResourceUseStats{ resourceUses: make(map[string]int), waiting: make(map[string]chan struct{}), quitC: quitC, } } -func (lb *resourceUseStats) sortResources(resources []Resource) []Resource { +func (lb *ResourceUseStats) SortResources(resources []Resource) []Resource { sorted := make([]Resource, len(resources)) resourceCounts := lb.getAllUseCounts(resources) sort.Slice(resourceCounts, func(i, j int) bool { @@ -66,7 +66,13 @@ func (lbp ResourceCount) String() string { return lbp.resource.Key() + ":" + strconv.Itoa(lbp.count) } -func (lb *resourceUseStats) dumpAllUses() map[string]int { +func (lb *ResourceUseStats) Len() int { + lb.lock.RLock() + defer lb.lock.RUnlock() + return len(lb.resourceUses) +} + +func (lb *ResourceUseStats) DumpAllUses() map[string]int { lb.lock.RLock() defer lb.lock.RUnlock() dump := make(map[string]int) @@ -76,39 +82,40 @@ func (lb *resourceUseStats) dumpAllUses() map[string]int { return dump } -func (lb *resourceUseStats) getAllUseCounts(resources []Resource) []ResourceCount { +func (lb *ResourceUseStats) getAllUseCounts(resources []Resource) []ResourceCount { peerUses := make([]ResourceCount, len(resources)) for i, resource := range resources { peerUses[i] = ResourceCount{ resource: resource, - count: lb.getUses(resource), + count: lb.resourceUses[resource.Key()], } } return peerUses } -func (lb *resourceUseStats) getUses(keyed Resource) int { - return lb.getKeyUses(keyed.Key()) +func (lb *ResourceUseStats) GetUses(keyed Resource) int { + return lb.GetKeyUses(keyed.Key()) } -func (lb *resourceUseStats) getKeyUses(key string) int { +func (lb *ResourceUseStats) GetKeyUses(key string) int { lb.lock.RLock() defer lb.lock.RUnlock() return lb.resourceUses[key] } -func (lb *resourceUseStats) addUse(resource Resource) int { +func (lb *ResourceUseStats) AddUse(resource Resource) int { lb.lock.Lock() defer lb.lock.Unlock() - log.Debug("Adding use", "key", resource.Label()) key := resource.Key() - lb.resourceUses[key] = lb.resourceUses[key] + 1 + prevCount := lb.resourceUses[key] + lb.resourceUses[key] = prevCount + 1 + log.Debug("Added use", "key", resource.Label(), "prevCount", prevCount, "newCount", lb.resourceUses[key]) return lb.resourceUses[key] } -// waitKey blocks until some key is added to the load balancer stats. +// WaitKey blocks until some key is added to the load balancer stats. // As peer resource initialization is asynchronous we need a way to know that the initial uses has been initialized. -func (lb *resourceUseStats) waitKey(key string) { +func (lb *ResourceUseStats) WaitKey(key string) { lb.lock.Lock() if _, ok := lb.resourceUses[key]; ok { lb.lock.Unlock() @@ -121,15 +128,30 @@ func (lb *resourceUseStats) waitKey(key string) { case <-waitChan: delete(lb.waiting, key) case <-lb.quitC: - return } } -func (lb *resourceUseStats) initKey(key string, count int) { +func (lb *ResourceUseStats) InitKey(key string, count int) { lb.lock.Lock() defer lb.lock.Unlock() lb.resourceUses[key] = count if kChan, ok := lb.waiting[key]; ok { - kChan <- struct{}{} + select { + case <-lb.quitC: + case kChan <- struct{}{}: + } + } } + +func (lb *ResourceUseStats) RemoveKey(key string) { + lb.lock.Lock() + defer lb.lock.Unlock() + delete(lb.resourceUses, key) +} + +func (lb *ResourceUseStats) RemoveResource(resource Resource) { + lb.lock.Lock() + defer lb.lock.Unlock() + delete(lb.resourceUses, resource.Key()) +} diff --git a/pss/forwarding_test.go b/pss/forwarding_test.go index 4436f1d472..2675fab259 100644 --- a/pss/forwarding_test.go +++ b/pss/forwarding_test.go @@ -129,7 +129,7 @@ func TestForwardBasic(t *testing.T) { } testCases = append(testCases, c) - //test with partial addresses + // test with partial addresses const part = 12 for i := 0; i < firstNearest; i++ { From 340ce15d25f87df1b03cdb5dfc03714abee816a4 Mon Sep 17 00:00:00 2001 From: Alvaro Date: Tue, 29 Oct 2019 14:52:48 +0100 Subject: [PATCH 22/31] Pubsub now closes all go routines when closing. Removed commented code --- network/kademlia_load_balancer.go | 6 ++---- network/pubsubchannel/pubsub.go | 14 ++++++-------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/network/kademlia_load_balancer.go b/network/kademlia_load_balancer.go index 00cacb2c9d..730c9d799a 100644 --- a/network/kademlia_load_balancer.go +++ b/network/kademlia_load_balancer.go @@ -110,8 +110,8 @@ func (klb *KademliaLoadBalancer) EachBinNodeAddress(consumeBin LBBinConsumer) { // EachBinFiltered returns all bins in descending order from the perspective of base address. // Only peers with the provided capabilities capKey are considered. // All peers in that bin will be provided to the LBBinConsumer sorted by least used first. -func (klb *KademliaLoadBalancer) EachBinFiltered(base []byte, capKey string, consumeBin LBBinConsumer) { - _ = klb.kademlia.EachBinDescFiltered(base, capKey, 0, func(peerBin *PeerBin) bool { +func (klb *KademliaLoadBalancer) EachBinFiltered(base []byte, capKey string, consumeBin LBBinConsumer) error { + return klb.kademlia.EachBinDescFiltered(base, capKey, 0, func(peerBin *PeerBin) bool { peers := klb.peerBinToPeerList(peerBin) return consumeBin(LBBin{LBPeers: peers, ProximityOrder: peerBin.ProximityOrder}) }) @@ -187,11 +187,9 @@ func (klb *KademliaLoadBalancer) listenOffPeers() { // to the use count of the least used peer in its bin. The po of the new peer is passed to avoid having // to calculate it again. func (klb *KademliaLoadBalancer) addedPeer(peer *Peer, po int) { - //log.Warn("Adding peer", "key", peer.Label()) initCount := klb.initCountFunc(peer, 0) log.Debug("Adding peer", "key", peer.Label(), "initCount", initCount) klb.resourceUseStats.InitKey(peer.Key(), initCount) - //log.Warn("Peer added", "key", peer.Label()) } // leastUsedCountInBin returns the use count for the least used peer in this bin excluding the excludePeer. diff --git a/network/pubsubchannel/pubsub.go b/network/pubsubchannel/pubsub.go index 2a6bc75306..233bf423f3 100644 --- a/network/pubsubchannel/pubsub.go +++ b/network/pubsubchannel/pubsub.go @@ -18,26 +18,23 @@ package pubsubchannel import ( "strconv" "sync" - "time" "github.com/ethersphere/swarm/log" ) -var subscriptionTimeout = 100 * time.Millisecond - //PubSubChannel represents a pubsub system where subscriber can .Subscribe() and publishers can .Publish() or .Close(). type PubSubChannel struct { subscriptions []*Subscription subsMutex sync.RWMutex nextId int + quitC chan struct{} } // Subscription is created in PubSubChannel using pubSub.Subscribe(). Subscribers can receive using .ReceiveChannel(). // or .Unsubscribe() type Subscription struct { - closed bool - pubSubC *PubSubChannel - //removeSub func() + closed bool + pubSubC *PubSubChannel signal chan interface{} closeOnce sync.Once id string @@ -48,6 +45,7 @@ type Subscription struct { func New() *PubSubChannel { return &PubSubChannel{ subscriptions: make([]*Subscription, 0), + quitC: make(chan struct{}, 1), } } @@ -91,8 +89,7 @@ func (psc *PubSubChannel) Publish(msg interface{}) { go func(sub *Subscription) { select { case sub.signal <- msg: - case <-time.After(subscriptionTimeout): - log.Warn("Subscription unattended after timeout", "subId", sub.ID(), "timeout", subscriptionTimeout) + case <-psc.quitC: } }(sub) } @@ -118,6 +115,7 @@ func (psc *PubSubChannel) Close() { sub.closeChannel() sub.lock.Unlock() } + psc.quitC <- struct{}{} } // Unsubscribe cancels subscription from the subscriber side. Channel is marked as closed but only writer should close it. From 697b049c6d8822d56b6dc98de35f8960e29405ed Mon Sep 17 00:00:00 2001 From: Alvaro Date: Tue, 29 Oct 2019 16:13:43 +0100 Subject: [PATCH 23/31] fix closing channel --- network/pubsubchannel/pubsub.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/network/pubsubchannel/pubsub.go b/network/pubsubchannel/pubsub.go index 233bf423f3..1805b5b8fd 100644 --- a/network/pubsubchannel/pubsub.go +++ b/network/pubsubchannel/pubsub.go @@ -45,7 +45,7 @@ type Subscription struct { func New() *PubSubChannel { return &PubSubChannel{ subscriptions: make([]*Subscription, 0), - quitC: make(chan struct{}, 1), + quitC: make(chan struct{}), } } @@ -115,7 +115,7 @@ func (psc *PubSubChannel) Close() { sub.closeChannel() sub.lock.Unlock() } - psc.quitC <- struct{}{} + close(psc.quitC) } // Unsubscribe cancels subscription from the subscriber side. Channel is marked as closed but only writer should close it. From 5cb6d46a711dcf5fcd35347ae422d4557bcb68a2 Mon Sep 17 00:00:00 2001 From: Alvaro Date: Wed, 30 Oct 2019 13:59:56 +0100 Subject: [PATCH 24/31] Added close channel for stopping blocked publishing goroutines --- network/pubsubchannel/pubsub.go | 7 ++++++- network/pubsubchannel/pubsub_test.go | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/network/pubsubchannel/pubsub.go b/network/pubsubchannel/pubsub.go index 1805b5b8fd..dfe3ff866b 100644 --- a/network/pubsubchannel/pubsub.go +++ b/network/pubsubchannel/pubsub.go @@ -39,6 +39,7 @@ type Subscription struct { closeOnce sync.Once id string lock sync.RWMutex + quitC chan struct{} // close channel for publisher goroutines } // New creates a new PubSubChannel. @@ -90,6 +91,8 @@ func (psc *PubSubChannel) Publish(msg interface{}) { select { case sub.signal <- msg: case <-psc.quitC: + case <-sub.quitC: + log.Warn("Subscription closed before message delivery") } }(sub) } @@ -120,6 +123,7 @@ func (psc *PubSubChannel) Close() { // Unsubscribe cancels subscription from the subscriber side. Channel is marked as closed but only writer should close it. func (sub *Subscription) Unsubscribe() { + close(sub.quitC) sub.pubSubC.removeSub(sub) } @@ -150,8 +154,9 @@ func newSubscription(id string, psc *PubSubChannel) Subscription { return Subscription{ closed: false, pubSubC: psc, - signal: make(chan interface{}, 20), + signal: make(chan interface{}), closeOnce: sync.Once{}, id: id, + quitC: make(chan struct{}), } } diff --git a/network/pubsubchannel/pubsub_test.go b/network/pubsubchannel/pubsub_test.go index a917e1e558..a70bec652a 100644 --- a/network/pubsubchannel/pubsub_test.go +++ b/network/pubsubchannel/pubsub_test.go @@ -16,8 +16,11 @@ package pubsubchannel_test import ( + "os" + "runtime/pprof" "sync" "testing" + "time" "github.com/ethersphere/swarm/log" "github.com/ethersphere/swarm/network/pubsubchannel" @@ -84,3 +87,18 @@ func testSubscriptor(pubsub *pubsubchannel.PubSubChannel, expectedMessages int, }(subscription) return msgBucket, subscription } + +func TestUnsubscribeBeforeReadingMessages(t *testing.T) { + ps := pubsubchannel.New() + s := ps.Subscribe() + defer ps.Close() + + for i := 0; i < 1000; i++ { + ps.Publish(struct{}{}) + } + + s.Unsubscribe() + // allow goroutines to finish + time.Sleep(100 * time.Millisecond) + pprof.Lookup("goroutine").WriteTo(os.Stdout, 1) +} From abc51abc99f6719420747a68e6581fe82d6e759b Mon Sep 17 00:00:00 2001 From: Alvaro Date: Wed, 30 Oct 2019 14:33:11 +0100 Subject: [PATCH 25/31] Added unit tests for pubsubchannel to check ongoing goroutines --- network/pubsubchannel/pubsub_test.go | 75 ++++++++++++++++++++++++++-- 1 file changed, 72 insertions(+), 3 deletions(-) diff --git a/network/pubsubchannel/pubsub_test.go b/network/pubsubchannel/pubsub_test.go index a70bec652a..a88b041348 100644 --- a/network/pubsubchannel/pubsub_test.go +++ b/network/pubsubchannel/pubsub_test.go @@ -16,7 +16,7 @@ package pubsubchannel_test import ( - "os" + "runtime" "runtime/pprof" "sync" "testing" @@ -88,17 +88,86 @@ func testSubscriptor(pubsub *pubsubchannel.PubSubChannel, expectedMessages int, return msgBucket, subscription } +// TestUnsubscribeBeforeReadingMessages tests that there is no goroutine leak when a subscription is finished +// before reading pending messages from the channel. func TestUnsubscribeBeforeReadingMessages(t *testing.T) { ps := pubsubchannel.New() s := ps.Subscribe() defer ps.Close() + numGoroutinesStart := runtime.NumGoroutine() + for i := 0; i < 1000; i++ { ps.Publish(struct{}{}) } s.Unsubscribe() // allow goroutines to finish - time.Sleep(100 * time.Millisecond) - pprof.Lookup("goroutine").WriteTo(os.Stdout, 1) + var newGoroutines int + for i := 0; i < 500; i++ { + time.Sleep(10 * time.Millisecond) + newGoroutines = runtime.NumGoroutine() - numGoroutinesStart + if newGoroutines <= 0 { + break + } + } + + if newGoroutines > 0 { + t.Errorf("%v new goroutines were active after unsubscribe, want none", newGoroutines) + pprof.Lookup("goroutine").WriteTo(newTestingErrorWriter(t), 1) + } +} + +type testingErrorWriter struct { + t *testing.T +} + +func newTestingErrorWriter(t *testing.T) testingErrorWriter { + return testingErrorWriter{t: t} +} + +func (w testingErrorWriter) Write(b []byte) (int, error) { + w.t.Error(string(b)) + return len(b), nil +} + +// TestMessageAfterUnsubscribe checks that if some pending message are still readable from the channel, after +// Unsubscribe(), the publishing goroutines will be exited and no message is received in the channel (even though the +// channel is still not closed). However, we need to wait a bit before extracting messages from the channel to allow +// the blocked publishers exit. In a real case, the moment a new message is published the channel will be closed. +func TestMessagesAfterUnsubscribe(t *testing.T) { + ps := pubsubchannel.New() + defer ps.Close() + + s := ps.Subscribe() + + for i := 0; i < 1000; i++ { + ps.Publish(i) + } + c := s.ReceiveChannel() + + s.Unsubscribe() + + time.Sleep(10 * time.Millisecond) + var n int + timeout := time.After(2 * time.Second) +loop: + for { + select { + case _, ok := <-c: + if !ok { + break loop + } + n++ + case <-timeout: + t.Log("timeout") + break loop + } + } + + t.Log("got", n, "messages") + if n > 0 { + t.Errorf("Expected no message received after unsubscribing but got %v", n) + } + } From 9f951d1c74a98e95b3730166d855f7d52e8df525 Mon Sep 17 00:00:00 2001 From: Alvaro Date: Wed, 30 Oct 2019 16:49:33 +0100 Subject: [PATCH 26/31] Better accounting of pending messages/goroutines --- network/pubsubchannel/pubsub.go | 36 ++++++++++++++++++++-------- network/pubsubchannel/pubsub_test.go | 29 ++++++++++++---------- 2 files changed, 42 insertions(+), 23 deletions(-) diff --git a/network/pubsubchannel/pubsub.go b/network/pubsubchannel/pubsub.go index dfe3ff866b..ee9a5cdb78 100644 --- a/network/pubsubchannel/pubsub.go +++ b/network/pubsubchannel/pubsub.go @@ -16,9 +16,11 @@ package pubsubchannel import ( + "fmt" "strconv" "sync" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethersphere/swarm/log" ) @@ -82,21 +84,27 @@ func (psc *PubSubChannel) Publish(msg interface{}) { psc.subsMutex.RLock() defer psc.subsMutex.RUnlock() for _, sub := range psc.subscriptions { - sub.lock.RLock() - if sub.closed { - log.Debug("Subscription was closed", "id", sub.id) - sub.closeChannel() - } else { - go func(sub *Subscription) { + go func(sub *Subscription) { + sub.lock.Lock() + defer sub.lock.Unlock() + metrics.GetOrRegisterCounter(fmt.Sprintf("pubsubchannel.%v.pending", sub.id), nil).Inc(1) + defer metrics.GetOrRegisterCounter(fmt.Sprintf("pubsubchannel.%v.pending", sub.id), nil).Inc(-1) + //atomic.AddInt64(sub.pending, 1) + //defer atomic.AddInt64(sub.pending, -1) + if sub.closed { + log.Debug("Subscription was closed", "id", sub.id) + sub.closeChannel() + } else { select { case sub.signal <- msg: + metrics.GetOrRegisterCounter(fmt.Sprintf("pubsubchannel.%v.delivered", sub.id), nil).Inc(1) + //atomic.AddInt64(sub.msgCount, 1) case <-psc.quitC: case <-sub.quitC: - log.Warn("Subscription closed before message delivery") } - }(sub) - } - sub.lock.RUnlock() + } + + }(sub) } } @@ -150,6 +158,14 @@ func (sub *Subscription) closeChannel() { }) } +func (sub *Subscription) MessageCount() int64 { + return metrics.GetOrRegisterCounter(fmt.Sprintf("pubsubchannel.%v.delivered", sub.id), nil).Count() +} + +func (sub *Subscription) Pending() int64 { + return metrics.GetOrRegisterCounter(fmt.Sprintf("pubsubchannel.%v.pending", sub.id), nil).Count() +} + func newSubscription(id string, psc *PubSubChannel) Subscription { return Subscription{ closed: false, diff --git a/network/pubsubchannel/pubsub_test.go b/network/pubsubchannel/pubsub_test.go index a88b041348..ed1f6e22fd 100644 --- a/network/pubsubchannel/pubsub_test.go +++ b/network/pubsubchannel/pubsub_test.go @@ -16,7 +16,7 @@ package pubsubchannel_test import ( - "runtime" + "fmt" "runtime/pprof" "sync" "testing" @@ -24,8 +24,13 @@ import ( "github.com/ethersphere/swarm/log" "github.com/ethersphere/swarm/network/pubsubchannel" + "github.com/ethersphere/swarm/testutil" ) +func init() { + testutil.Init() +} + func TestPubSeveralSub(t *testing.T) { pubSub := pubsubchannel.New() var group sync.WaitGroup @@ -83,7 +88,7 @@ func testSubscriptor(pubsub *pubsubchannel.PubSubChannel, expectedMessages int, return } } - log.Debug("Finishing subscriptor gofunc", "id", subscription.ID()) + log.Debug("Finishing subscriber gofunc", "id", subscription.ID()) }(subscription) return msgBucket, subscription } @@ -95,25 +100,23 @@ func TestUnsubscribeBeforeReadingMessages(t *testing.T) { s := ps.Subscribe() defer ps.Close() - numGoroutinesStart := runtime.NumGoroutine() - for i := 0; i < 1000; i++ { ps.Publish(struct{}{}) } s.Unsubscribe() - // allow goroutines to finish - var newGoroutines int - for i := 0; i < 500; i++ { + // allow goroutines to finish, no pending messages + var pendingMessages int64 + for i := 0; i < 500 && pendingMessages > 0; i++ { time.Sleep(10 * time.Millisecond) - newGoroutines = runtime.NumGoroutine() - numGoroutinesStart - if newGoroutines <= 0 { + pendingMessages = s.Pending() + if pendingMessages <= 0 { break } } - if newGoroutines > 0 { - t.Errorf("%v new goroutines were active after unsubscribe, want none", newGoroutines) + if pendingMessages > 0 { + t.Errorf("%v new goroutines were active after unsubscribe, want none", pendingMessages) pprof.Lookup("goroutine").WriteTo(newTestingErrorWriter(t), 1) } } @@ -142,13 +145,13 @@ func TestMessagesAfterUnsubscribe(t *testing.T) { s := ps.Subscribe() for i := 0; i < 1000; i++ { - ps.Publish(i) + ps.Publish(fmt.Sprintf("Message %v", i)) } c := s.ReceiveChannel() s.Unsubscribe() - time.Sleep(10 * time.Millisecond) + ps.Publish("End message") var n int timeout := time.After(2 * time.Second) loop: From d8f8059c9f166df5fc63841277063ff748aa8695 Mon Sep 17 00:00:00 2001 From: Alvaro Date: Wed, 30 Oct 2019 17:58:14 +0100 Subject: [PATCH 27/31] Reverted metrics to regular int counters --- network/hive_test.go | 6 +++--- network/pubsubchannel/pubsub.go | 23 +++++++++++++---------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/network/hive_test.go b/network/hive_test.go index 67410009a5..00fffe4690 100644 --- a/network/hive_test.go +++ b/network/hive_test.go @@ -254,7 +254,7 @@ func TestHiveStateConnections(t *testing.T) { } h1.Kademlia.lock.Lock() - numConns := h1.conns.Size() + numConns := h1.defaultIndex.conns.Size() h1.Kademlia.lock.Unlock() connAddresses := make(map[string]string) h1.EachConn(h1.base, 255, func(peer *Peer, i int) bool { @@ -271,12 +271,12 @@ func TestHiveStateConnections(t *testing.T) { connsAfterLoading := 0 iterations := 0 h2.Kademlia.lock.Lock() - connsAfterLoading = h2.conns.Size() + connsAfterLoading = h2.defaultIndex.conns.Size() h2.Kademlia.lock.Unlock() for connsAfterLoading != numConns && iterations < 5 { select { case <-addedChan: - connsAfterLoading = h2.conns.Size() + connsAfterLoading = h2.defaultIndex.conns.Size() case <-time.After(1 * time.Second): iterations++ } diff --git a/network/pubsubchannel/pubsub.go b/network/pubsubchannel/pubsub.go index ee9a5cdb78..11a7da1268 100644 --- a/network/pubsubchannel/pubsub.go +++ b/network/pubsubchannel/pubsub.go @@ -16,11 +16,10 @@ package pubsubchannel import ( - "fmt" "strconv" "sync" + "sync/atomic" - "github.com/ethereum/go-ethereum/metrics" "github.com/ethersphere/swarm/log" ) @@ -42,6 +41,8 @@ type Subscription struct { id string lock sync.RWMutex quitC chan struct{} // close channel for publisher goroutines + msgCount *int64 + pending *int64 } // New creates a new PubSubChannel. @@ -87,18 +88,15 @@ func (psc *PubSubChannel) Publish(msg interface{}) { go func(sub *Subscription) { sub.lock.Lock() defer sub.lock.Unlock() - metrics.GetOrRegisterCounter(fmt.Sprintf("pubsubchannel.%v.pending", sub.id), nil).Inc(1) - defer metrics.GetOrRegisterCounter(fmt.Sprintf("pubsubchannel.%v.pending", sub.id), nil).Inc(-1) - //atomic.AddInt64(sub.pending, 1) - //defer atomic.AddInt64(sub.pending, -1) + atomic.AddInt64(sub.pending, 1) + defer atomic.AddInt64(sub.pending, -1) if sub.closed { log.Debug("Subscription was closed", "id", sub.id) sub.closeChannel() } else { select { case sub.signal <- msg: - metrics.GetOrRegisterCounter(fmt.Sprintf("pubsubchannel.%v.delivered", sub.id), nil).Inc(1) - //atomic.AddInt64(sub.msgCount, 1) + atomic.AddInt64(sub.msgCount, 1) case <-psc.quitC: case <-sub.quitC: } @@ -159,14 +157,17 @@ func (sub *Subscription) closeChannel() { } func (sub *Subscription) MessageCount() int64 { - return metrics.GetOrRegisterCounter(fmt.Sprintf("pubsubchannel.%v.delivered", sub.id), nil).Count() + return *sub.msgCount } func (sub *Subscription) Pending() int64 { - return metrics.GetOrRegisterCounter(fmt.Sprintf("pubsubchannel.%v.pending", sub.id), nil).Count() + return *sub.pending } + func newSubscription(id string, psc *PubSubChannel) Subscription { + var count int64 + var pending int64 return Subscription{ closed: false, pubSubC: psc, @@ -174,5 +175,7 @@ func newSubscription(id string, psc *PubSubChannel) Subscription { closeOnce: sync.Once{}, id: id, quitC: make(chan struct{}), + msgCount: &count, + pending: &pending, } } From c34062cc2c46fb248b5e538e0778cb3c287c6a38 Mon Sep 17 00:00:00 2001 From: Alvaro Date: Thu, 31 Oct 2019 08:44:53 +0100 Subject: [PATCH 28/31] Removed testKademliaBackend, using testKademlia instead. Minor PR fixes --- network/kademlia_load_balancer.go | 24 +-- network/kademlia_load_balancer_test.go | 201 ++----------------------- network/pubsubchannel/pubsub.go | 1 - 3 files changed, 26 insertions(+), 200 deletions(-) diff --git a/network/kademlia_load_balancer.go b/network/kademlia_load_balancer.go index 730c9d799a..f3dca97fbc 100644 --- a/network/kademlia_load_balancer.go +++ b/network/kademlia_load_balancer.go @@ -16,6 +16,8 @@ package network import ( + "bytes" + "github.com/ethersphere/swarm/log" "github.com/ethersphere/swarm/network/pubsubchannel" "github.com/ethersphere/swarm/network/resourceusestats" @@ -67,7 +69,7 @@ type LBPeer struct { } // AddUseCount is called to account a use for these peer. Should be called if the peer is actually used. -func (lbPeer LBPeer) AddUseCount() { +func (lbPeer *LBPeer) AddUseCount() { lbPeer.stats.AddUse(lbPeer.Peer) } @@ -168,7 +170,11 @@ func (klb *KademliaLoadBalancer) listenOffPeers() { select { case <-klb.quitC: return - case msg := <-klb.offPeerSub.ReceiveChannel(): + case msg, ok := <-klb.offPeerSub.ReceiveChannel(): + if !ok { + log.Warn("listenOffPeers closed channel, finishing subscriber to off peers") + return + } peer, ok := msg.(*Peer) if peer == nil { log.Warn("nil peer received listening for off peers. Ignoring.") @@ -178,7 +184,7 @@ func (klb *KademliaLoadBalancer) listenOffPeers() { log.Warn("unexpected message received listening for off peers. Ignoring.") continue } - klb.removedPeer(peer) + klb.resourceUseStats.RemoveResource(peer) } } } @@ -196,16 +202,14 @@ func (klb *KademliaLoadBalancer) addedPeer(peer *Peer, po int) { func (klb *KademliaLoadBalancer) leastUsedCountInBin(excludePeer *Peer, po int) int { addr := klb.kademlia.BaseAddr() peersInSamePo := klb.getPeersForPo(addr, po) - idx := 0 leastUsedCount := 0 - for idx < len(peersInSamePo) { - leastUsed := peersInSamePo[idx] + for i := 0; i < len(peersInSamePo); i++ { + leastUsed := peersInSamePo[i] if leastUsed.Peer.Key() != excludePeer.Key() { leastUsedCount = klb.resourceUseStats.GetUses(leastUsed.Peer) log.Debug("Least used peer is", "peer", leastUsed.Peer.Label(), "leastUsedCount", leastUsedCount) break } - idx++ } return leastUsedCount } @@ -214,7 +218,7 @@ func (klb *KademliaLoadBalancer) leastUsedCountInBin(excludePeer *Peer, po int) func (klb *KademliaLoadBalancer) nearestNeighbourUseCount(newPeer *Peer, _ int) int { var count int klb.kademlia.EachConn(newPeer.Address(), 255, func(peer *Peer, po int) bool { - if peer != newPeer { + if !bytes.Equal(peer.OAddr, newPeer.OAddr) { count = klb.resourceUseStats.GetUses(peer) log.Debug("Nearest neighbour is", "peer", peer.Label(), "count", count) return false @@ -224,10 +228,6 @@ func (klb *KademliaLoadBalancer) nearestNeighbourUseCount(newPeer *Peer, _ int) return count } -func (klb *KademliaLoadBalancer) removedPeer(peer *Peer) { - klb.resourceUseStats.RemoveResource(peer) -} - func (klb *KademliaLoadBalancer) toLBPeers(resources []resourceusestats.Resource) []LBPeer { peers := make([]LBPeer, len(resources)) for i, res := range resources { diff --git a/network/kademlia_load_balancer_test.go b/network/kademlia_load_balancer_test.go index 7c4f58e27a..231871fe43 100644 --- a/network/kademlia_load_balancer_test.go +++ b/network/kademlia_load_balancer_test.go @@ -16,7 +16,6 @@ package network import ( - "sort" "strconv" "testing" "time" @@ -24,18 +23,17 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethersphere/swarm/log" "github.com/ethersphere/swarm/network/capability" - "github.com/ethersphere/swarm/network/pubsubchannel" "github.com/ethersphere/swarm/pot" ) // TestAddedNodes checks that when adding a node it is assigned the correct number of uses. // This number of uses will be the least number of uses of a peer in its bin. func TestAddedNodes(t *testing.T) { - kademlia := newTestKademliaBackend("11110000") + kademlia := newTestKademlia(t, "11110000") first := newTestKadPeer("010101010") - kademlia.addPeer(first) + kademlia.Kademlia.On(first) second := newTestKadPeer("010101011") - kademlia.addPeer(second) + kademlia.Kademlia.On(second) klb := NewKademliaLoadBalancer(kademlia, false) defer klb.Stop() @@ -43,22 +41,22 @@ func TestAddedNodes(t *testing.T) { if firstUses != 0 { t.Errorf("Expected 0 uses for new peer at start") } - peersFor0 := klb.getPeersForPo(kademlia.baseAddr, 0) + peersFor0 := klb.getPeersForPo(kademlia.base, 0) peersFor0[0].AddUseCount() // Now new peers still should have 0 uses third := newTestKadPeer("011101011") - kademlia.addPeer(third) + kademlia.Kademlia.On(third) klb.resourceUseStats.WaitKey(third.Key()) thirdUses := klb.resourceUseStats.GetUses(third) if thirdUses != 0 { t.Errorf("Expected 0 uses for new peer because minimum in bin is 0. Instead %v", thirdUses) } - peersFor0 = klb.getPeersForPo(kademlia.baseAddr, 0) + peersFor0 = klb.getPeersForPo(kademlia.base, 0) peersFor0[0].AddUseCount() peersFor0[1].AddUseCount() //Now all peers should have 1 use //New peers should start with 1 use fourth := newTestKadPeer("011100011") - kademlia.addPeer(fourth) + kademlia.Kademlia.On(fourth) klb.resourceUseStats.WaitKey(fourth.Key()) fourthUses := klb.resourceUseStats.GetUses(fourth) if fourthUses != 1 { @@ -69,11 +67,11 @@ func TestAddedNodes(t *testing.T) { // TestAddedNodesNearestNeighbour checks that when adding a node it is assigned the correct number of uses. // This number of uses will be the most similar peer uses. func TestAddedNodesNearestNeighbour(t *testing.T) { - kademlia := newTestKademliaBackend("11110000") + kademlia := newTestKademlia(t, "11110000") first := newTestKadPeer("01010101") - kademlia.addPeer(first) + kademlia.Kademlia.On(first) second := newTestKadPeer("01110101") - kademlia.addPeer(second) + kademlia.Kademlia.On(second) klb := NewKademliaLoadBalancer(kademlia, true) defer klb.Stop() @@ -81,11 +79,11 @@ func TestAddedNodesNearestNeighbour(t *testing.T) { if firstUses != 0 { t.Errorf("Expected 0 uses for new peer at start") } - peersFor0 := klb.getPeersForPo(kademlia.baseAddr, 0) + peersFor0 := klb.getPeersForPo(kademlia.base, 0) peersFor0[0].AddUseCount() // Now third peer should have the same uses as second third := newTestKadPeer("01110111") // most similar peer is second 01110101 - kademlia.addPeer(third) + kademlia.Kademlia.On(third) klb.resourceUseStats.WaitKey(third.Key()) secondUses := klb.resourceUseStats.GetUses(second) thirdUses := klb.resourceUseStats.GetUses(third) @@ -93,7 +91,7 @@ func TestAddedNodesNearestNeighbour(t *testing.T) { t.Errorf("Expected %v uses for new peer because is most similar to second. Instead %v", secondUses, thirdUses) } //Now we use third peer twice - peersFor0 = klb.getPeersForPo(kademlia.baseAddr, 0) + peersFor0 = klb.getPeersForPo(kademlia.base, 0) for _, lbPeer := range peersFor0 { if lbPeer.Peer.Key() == third.key { lbPeer.AddUseCount() @@ -102,7 +100,7 @@ func TestAddedNodesNearestNeighbour(t *testing.T) { } fourth := newTestKadPeer("01110110") // most similar peer is third 01110111 - kademlia.addPeer(fourth) + kademlia.Kademlia.On(fourth) klb.resourceUseStats.WaitKey(fourth.Key()) //We expect fourth to be initialized with third peer use count fourthUses := klb.resourceUseStats.GetUses(fourth) @@ -209,12 +207,6 @@ func TestEachBinBaseUses(t *testing.T) { } } -func expectUses(actualUses int, expected int, peer string, t *testing.T) { - if actualUses != expected { - t.Errorf("expected %v uses of %v but got %v", expected, peer, actualUses) - } -} - // TestEachBinFiltered checks that when load balancing peers, only those with the provided capabilities are chosen. func TestEachBinFiltered(t *testing.T) { tk := newTestKademlia(t, "11111111") @@ -277,171 +269,6 @@ func TestEachBinFiltered(t *testing.T) { } -type testKademliaBackend struct { - baseAddr []byte - addedChannel *pubsubchannel.PubSubChannel - removedChannel *pubsubchannel.PubSubChannel - bins map[int][]*Peer - maxPo int -} - -// EachConn iterates this test kademlia table peers in order po from nearest to furthest with respect to the base address. -// - First it takes the po of the base address provided. With this po it takes the corresponding bin B (peers with same po -// as this base) copy the list and sort them using sort.Slice from furthest to lowest. Then it calls the provided consume -// function with those peers and its corresponding po (peerPo). -// - Then, it iterates all bins with higher po than B, and iterates all peers. All of these peers will have the -// same po as the base address with respect to the pin of the kademlia, that's why we don't need to sort them out and -// we can iterate them in any order. For each peer the consume function is called with the same po. -// - Finally, the bins furthest than B are iterated from highest po (nearest) to lowest (furthest). For each bin, all -// peers are iterated and called the consume function with the peer and the bin po. -// After any of the calls to the consume function, if that function returns false, the iteration stops. -func (tkb *testKademliaBackend) EachConn(base []byte, maxPo int, consume func(*Peer, int) bool) { - po, _ := Pof(base, tkb.baseAddr, 0) - bin := tkb.bins[po] - peersInBin := make([]*Peer, len(bin)) - copy(peersInBin, bin) - sort.Slice(peersInBin, func(i, j int) bool { - peerIPo, _ := Pof(base, peersInBin[i], 0) - peerJPo, _ := Pof(base, peersInBin[j], 0) - return peerIPo > peerJPo - }) - for _, peer := range peersInBin { - peerPo, _ := Pof(base, peer, 0) - if !consume(peer, peerPo) { - return - } - } - for i := po + 1; po < maxPo; po++ { - bin = tkb.bins[i] - for _, peer := range bin { - if !consume(peer, po) { - return - } - } - } - for i := po - 1; po >= 0; po-- { - bin = tkb.bins[i] - for _, peer := range bin { - if !consume(peer, i) { - return - } - } - } - -} - -func newTestKademliaBackend(address string) *testKademliaBackend { - return &testKademliaBackend{ - baseAddr: pot.NewAddressFromString(address), - addedChannel: pubsubchannel.New(), - removedChannel: pubsubchannel.New(), - bins: make(map[int][]*Peer), - } -} - -// BaseAddr returns the node address of the kademlia table. This base address is the pivot address to which other addresses -// are sorted with respect to the proximity order function. -func (tkb testKademliaBackend) BaseAddr() []byte { - return tkb.baseAddr -} - -// SubscribeToPeerChanges returns a subscription to changes in the kademlia peers. It contains channels to notify about -// peers added/removed from this kademlia. -func (tkb *testKademliaBackend) SubscribeToPeerChanges() (addedSub *pubsubchannel.Subscription, removedPeerSub *pubsubchannel.Subscription) { - addedSub = tkb.addedChannel.Subscribe() - removedPeerSub = tkb.removedChannel.Subscribe() - return -} - -// EachBinDescFiltered ignores capKey as in this test context it won't be used. It ignores base as EachBinDesc ignores it. -func (tkb testKademliaBackend) EachBinDescFiltered(base []byte, capKey string, minProximityOrder int, consumer PeerBinConsumer) error { - tkb.EachBinDesc(base, minProximityOrder, consumer) - return nil -} - -// EachBinDesc iterates bin in the table in descending po order (from nearest to furthest). Base is always supposed to be -// node address in this test context. For each bin found, the provided PeerBinConsumer function is called. -func (tkb testKademliaBackend) EachBinDesc(_ []byte, minProximityOrder int, consumer PeerBinConsumer) { - type poPeers struct { - po int - peers []*Peer - } - var poPeersList []poPeers - for po, peers := range tkb.bins { - poPeersList = append(poPeersList, poPeers{po: po, peers: peers}) - } - sort.Slice(poPeersList, func(i, j int) bool { - return poPeersList[i].po > poPeersList[j].po - }) - for _, aPoPeers := range poPeersList { - peers := aPoPeers.peers - po := aPoPeers.po - if peers != nil && po >= minProximityOrder { - bin := &PeerBin{ - ProximityOrder: po, - Size: len(peers), - PeerIterator: func(consumePeer PeerConsumer) bool { - for _, peer := range peers { - if !consumePeer(&entry{conn: peer}) { - return false - } - } - return true - }, - } - if !consumer(bin) { - return - } - } - } -} - -func (tkb *testKademliaBackend) addPeer(peer *Peer) { - po, _ := Pof(peer.Address(), tkb.baseAddr, 0) - if tkb.bins[po] == nil { - if po > tkb.maxPo { - tkb.maxPo = po - } - tkb.bins[po] = make([]*Peer, 0) - } - tkb.bins[po] = append(tkb.bins[po], peer) - tkb.addedChannel.Publish(newPeerSignal{ - peer: peer, - po: po, - }) - // As the subscribers to add peer are asynchronous, we will sleep here to allow them to execute. - // Because this will be used in tests several times, we wait here so the code is not polluted with Sleep calls. - time.Sleep(100 * time.Millisecond) -} - -func (tkb *testKademliaBackend) removePeer(peer *Peer) { - tkb.removePeerFromBin(peer) - tkb.removedChannel.Publish(peer) -} - -func (tkb *testKademliaBackend) removePeerFromBin(peer *Peer) { - for po, bin := range tkb.bins { - for i, aPeer := range bin { - if aPeer == peer { - tkb.bins[po] = append(bin[:i], bin[i+1:]...) - if len(tkb.bins[po]) == 0 && tkb.maxPo >= po { - tkb.updateMaxPo() - } - return - } - } - } -} - -func (tkb *testKademliaBackend) updateMaxPo() { - tkb.maxPo = 0 - for k := range tkb.bins { - if k > tkb.maxPo { - tkb.maxPo = k - } - } -} - func newTestKadPeer(s string) *Peer { return NewPeer(&BzzPeer{BzzAddr: testKadPeerAddr(s)}, nil) } diff --git a/network/pubsubchannel/pubsub.go b/network/pubsubchannel/pubsub.go index 11a7da1268..17eb7ed134 100644 --- a/network/pubsubchannel/pubsub.go +++ b/network/pubsubchannel/pubsub.go @@ -164,7 +164,6 @@ func (sub *Subscription) Pending() int64 { return *sub.pending } - func newSubscription(id string, psc *PubSubChannel) Subscription { var count int64 var pending int64 From e7eefaf75bcbc9615cb74b525643958b2f2c31f0 Mon Sep 17 00:00:00 2001 From: Alvaro Date: Mon, 4 Nov 2019 11:24:14 +0100 Subject: [PATCH 29/31] PubSubChannel now publish messages semi-asynchronously filling first a subscription inbox before blocking. This will ensure order messages delivery if the inbox is never full --- network/kademlia.go | 27 ++++---- network/kademlia_load_balancer.go | 55 +++++----------- network/kademlia_load_balancer_test.go | 38 +++++++++++ network/pubsubchannel/pubsub.go | 89 +++++++++++++++----------- network/pubsubchannel/pubsub_test.go | 49 ++++++++++++-- 5 files changed, 160 insertions(+), 98 deletions(-) diff --git a/network/kademlia.go b/network/kademlia.go index e65ac3a140..a0a7793bda 100644 --- a/network/kademlia.go +++ b/network/kademlia.go @@ -100,8 +100,7 @@ type Kademlia struct { nDepthMu sync.RWMutex // protects neighbourhood depth nDepth nDepthSig []chan struct{} // signals when neighbourhood depth nDepth is changed - newPeerPubSub *pubsubchannel.PubSubChannel - removedPeerPubSub *pubsubchannel.PubSubChannel + onOffPeerPubSub *pubsubchannel.PubSubChannel // signals on and off peers in the table } type KademliaInfo struct { @@ -124,21 +123,21 @@ func NewKademlia(addr []byte, params *KadParams) *Kademlia { params.Capabilities = capability.NewCapabilities() } k := &Kademlia{ - base: addr, - KadParams: params, - capabilityIndex: make(map[string]*capabilityIndex), - defaultIndex: NewDefaultIndex(), - newPeerPubSub: pubsubchannel.New(), - removedPeerPubSub: pubsubchannel.New(), + base: addr, + KadParams: params, + capabilityIndex: make(map[string]*capabilityIndex), + defaultIndex: NewDefaultIndex(), + onOffPeerPubSub: pubsubchannel.New(100), } k.RegisterCapabilityIndex("full", *fullCapability) k.RegisterCapabilityIndex("light", *lightCapability) return k } -type newPeerSignal struct { +type onOffPeerSignal struct { peer *Peer po int + on bool } // RegisterCapabilityIndex adds an entry to the capability index of the kademlia @@ -472,7 +471,7 @@ func (k *Kademlia) On(p *Peer) (uint8, bool) { }) k.addToCapabilityIndex(p) // notify subscribers asynchronously - k.newPeerPubSub.Publish(newPeerSignal{peer: p, po: po}) + k.onOffPeerPubSub.Publish(onOffPeerSignal{peer: p, po: po, on: true}) if ins { a := newEntryFromBzzAddress(p.BzzAddr) @@ -580,10 +579,8 @@ func (k *Kademlia) SubscribeToNeighbourhoodDepthChange() (c <-chan struct{}, uns // when a new Peer is added or removed from the table. Returned function unsubscribes // the channel from signaling and releases the resources. Returned function is safe // to be called multiple times. -func (k *Kademlia) SubscribeToPeerChanges() (addedSub *pubsubchannel.Subscription, removedPeerSub *pubsubchannel.Subscription) { - addedSub = k.newPeerPubSub.Subscribe() - removedPeerSub = k.removedPeerPubSub.Subscribe() - return +func (k *Kademlia) SubscribeToPeerChanges() *pubsubchannel.Subscription { + return k.onOffPeerPubSub.Subscribe() } // Off removes a peer from among live peers @@ -605,7 +602,7 @@ func (k *Kademlia) Off(p *Peer) { }) k.removeFromCapabilityIndex(p, true) k.setNeighbourhoodDepth() - k.removedPeerPubSub.Publish(p) + k.onOffPeerPubSub.Publish(onOffPeerSignal{peer: p, po: -1, on: false}) } // EachConnFiltered performs the same action as EachConn diff --git a/network/kademlia_load_balancer.go b/network/kademlia_load_balancer.go index f3dca97fbc..a710554e60 100644 --- a/network/kademlia_load_balancer.go +++ b/network/kademlia_load_balancer.go @@ -25,7 +25,7 @@ import ( // KademliaBackend is the required interface of KademliaLoadBalancer. type KademliaBackend interface { - SubscribeToPeerChanges() (addedSub *pubsubchannel.Subscription, removedPeerSub *pubsubchannel.Subscription) + SubscribeToPeerChanges() *pubsubchannel.Subscription BaseAddr() []byte EachBinDesc(base []byte, minProximityOrder int, consumer PeerBinConsumer) EachBinDescFiltered(base []byte, capKey string, minProximityOrder int, consumer PeerBinConsumer) error @@ -37,13 +37,12 @@ type KademliaBackend interface { // If not, least used peer use count in same bin as new peer will be used. It is not clear which one is better, when // this load balancer would be used in several use cases we could do take some decision. func NewKademliaLoadBalancer(kademlia KademliaBackend, useNearestNeighbourInit bool) *KademliaLoadBalancer { - onPeerSub, offPeerSub := kademlia.SubscribeToPeerChanges() + onOffPeerSub := kademlia.SubscribeToPeerChanges() quitC := make(chan struct{}) klb := &KademliaLoadBalancer{ kademlia: kademlia, resourceUseStats: resourceusestats.NewResourceUseStats(quitC), - onPeerSub: onPeerSub, - offPeerSub: offPeerSub, + onOffPeerSub: onOffPeerSub, quitC: quitC, } if useNearestNeighbourInit { @@ -52,8 +51,7 @@ func NewKademliaLoadBalancer(kademlia KademliaBackend, useNearestNeighbourInit b klb.initCountFunc = klb.leastUsedCountInBin } - go klb.listenNewPeers() - go klb.listenOffPeers() + go klb.listenOnOffPeers() return klb } @@ -90,8 +88,7 @@ type LBBinConsumer func(bin LBBin) bool type KademliaLoadBalancer struct { kademlia KademliaBackend // kademlia to obtain bins of peers resourceUseStats *resourceusestats.ResourceUseStats // a resourceUseStats to count uses - onPeerSub *pubsubchannel.Subscription // a pubsub channel to be notified of new peers in kademlia - offPeerSub *pubsubchannel.Subscription // a pubsub channel to be notified of removed peers in kademlia + onOffPeerSub *pubsubchannel.Subscription // a pubsub channel to be notified of on/off peers in kademlia quitC chan struct{} initCountFunc func(peer *Peer, po int) int //Function to use for initializing a new peer count @@ -99,8 +96,7 @@ type KademliaLoadBalancer struct { // Stop unsubscribe from notifiers func (klb *KademliaLoadBalancer) Stop() { - klb.onPeerSub.Unsubscribe() - klb.offPeerSub.Unsubscribe() + klb.onOffPeerSub.Unsubscribe() close(klb.quitC) } @@ -145,46 +141,27 @@ func (klb *KademliaLoadBalancer) resourcesToLbPeers(resources []resourceusestats return peers } -func (klb *KademliaLoadBalancer) listenNewPeers() { +func (klb *KademliaLoadBalancer) listenOnOffPeers() { for { select { case <-klb.quitC: return - case msg, ok := <-klb.onPeerSub.ReceiveChannel(): + case msg, ok := <-klb.onOffPeerSub.ReceiveChannel(): if !ok { - log.Warn("listenNewPeers closed channel, finishing subscriber to new peer") + log.Debug("listenOnOffPeers closed channel, finishing subscriber to on/off peers") return } - signal, ok := msg.(newPeerSignal) + signal, ok := msg.(onOffPeerSignal) if !ok { - log.Warn("listenNewPeers received message is not a new peer signal") + log.Warn("listenOnOffPeers received message is not a on/off peer signal!") continue } - klb.addedPeer(signal.peer, signal.po) - } - } -} - -func (klb *KademliaLoadBalancer) listenOffPeers() { - for { - select { - case <-klb.quitC: - return - case msg, ok := <-klb.offPeerSub.ReceiveChannel(): - if !ok { - log.Warn("listenOffPeers closed channel, finishing subscriber to off peers") - return - } - peer, ok := msg.(*Peer) - if peer == nil { - log.Warn("nil peer received listening for off peers. Ignoring.") - continue - } - if !ok { - log.Warn("unexpected message received listening for off peers. Ignoring.") - continue + //log.Warn("OnOff peer", "key", signal.peer.Key(), "on", signal.on) + if signal.on { + klb.addedPeer(signal.peer, signal.po) + } else { + klb.resourceUseStats.RemoveResource(signal.peer) } - klb.resourceUseStats.RemoveResource(peer) } } } diff --git a/network/kademlia_load_balancer_test.go b/network/kademlia_load_balancer_test.go index 231871fe43..77a3757c5f 100644 --- a/network/kademlia_load_balancer_test.go +++ b/network/kademlia_load_balancer_test.go @@ -16,6 +16,7 @@ package network import ( + "encoding/binary" "strconv" "testing" "time" @@ -269,6 +270,43 @@ func TestEachBinFiltered(t *testing.T) { } +// TestResourceUseStats checks that on and off messages are delivered in order +func TestResourceUseStats(t *testing.T) { + + testResourceUseStats := func(t *testing.T, delay time.Duration) { + k := NewKademlia(make([]byte, 32), NewKadParams()) + lb := NewKademliaLoadBalancer(k, false) + for i := uint64(0); i < 10; i++ { + a := make([]byte, 8) + binary.BigEndian.PutUint64(a, i) + p := NewPeer(&BzzPeer{BzzAddr: NewBzzAddr(a, a)}, nil) + k.On(p) + if delay > 0 { + time.Sleep(delay) + } + k.Off(p) + if delay > 0 { + time.Sleep(delay) + } + } + + // we need to sleep to allow all messages to be received by lb + time.Sleep(100 * time.Millisecond) + count := lb.resourceUseStats.Len() + if count > 0 { + t.Errorf("got resourceUseStats %v, want 0, uses: %v", count, lb.resourceUseStats.DumpAllUses()) + } + lb.Stop() + } + + t.Run("no delay", func(t *testing.T) { + testResourceUseStats(t, 0) + }) + t.Run("1ms delay", func(t *testing.T) { + testResourceUseStats(t, time.Millisecond) + }) +} + func newTestKadPeer(s string) *Peer { return NewPeer(&BzzPeer{BzzAddr: testKadPeerAddr(s)}, nil) } diff --git a/network/pubsubchannel/pubsub.go b/network/pubsubchannel/pubsub.go index 17eb7ed134..a2036e99ea 100644 --- a/network/pubsubchannel/pubsub.go +++ b/network/pubsubchannel/pubsub.go @@ -23,12 +23,15 @@ import ( "github.com/ethersphere/swarm/log" ) -//PubSubChannel represents a pubsub system where subscriber can .Subscribe() and publishers can .Publish() or .Close(). +// PubSubChannel represents a pubsub system where subscriber can .Subscribe() and publishers can .Publish() or .Close(). +// When it publishes a message, it notifies all subscribers semi-asynchronously, meaning that each subscription will have +// an inbox of size inboxSize, but then a different goroutine will send those messages to the subscribers. type PubSubChannel struct { subscriptions []*Subscription subsMutex sync.RWMutex nextId int quitC chan struct{} + inboxSize int // size of the inbox channels in subscriptions. Depends on the number of pseudo-simultaneous messages expected to be published. } // Subscription is created in PubSubChannel using pubSub.Subscribe(). Subscribers can receive using .ReceiveChannel(). @@ -36,20 +39,22 @@ type PubSubChannel struct { type Subscription struct { closed bool pubSubC *PubSubChannel + inbox chan interface{} signal chan interface{} closeOnce sync.Once id string lock sync.RWMutex quitC chan struct{} // close channel for publisher goroutines - msgCount *int64 + msgCount int pending *int64 } // New creates a new PubSubChannel. -func New() *PubSubChannel { +func New(inboxSize int) *PubSubChannel { return &PubSubChannel{ subscriptions: make([]*Subscription, 0), quitC: make(chan struct{}), + inboxSize: inboxSize, } } @@ -57,11 +62,11 @@ func New() *PubSubChannel { func (psc *PubSubChannel) Subscribe() *Subscription { psc.subsMutex.Lock() defer psc.subsMutex.Unlock() - newSubscription := newSubscription(strconv.Itoa(psc.nextId), psc) + newSubscription := newSubscription(strconv.Itoa(psc.nextId), psc, psc.inboxSize) psc.nextId++ - psc.subscriptions = append(psc.subscriptions, &newSubscription) + psc.subscriptions = append(psc.subscriptions, newSubscription) - return &newSubscription + return newSubscription } func (psc *PubSubChannel) removeSub(s *Subscription) { @@ -79,30 +84,23 @@ func (psc *PubSubChannel) removeSub(s *Subscription) { } } -// Publish broadcasts a message asynchronously to each subscriber. -// If some of the subscriptions(channels) has been marked as closeable, it does it now. +// Publish broadcasts a message synchronously to each subscriber inbox. func (psc *PubSubChannel) Publish(msg interface{}) { psc.subsMutex.RLock() defer psc.subsMutex.RUnlock() for _, sub := range psc.subscriptions { - go func(sub *Subscription) { - sub.lock.Lock() - defer sub.lock.Unlock() - atomic.AddInt64(sub.pending, 1) - defer atomic.AddInt64(sub.pending, -1) - if sub.closed { - log.Debug("Subscription was closed", "id", sub.id) - sub.closeChannel() - } else { - select { - case sub.signal <- msg: - atomic.AddInt64(sub.msgCount, 1) - case <-psc.quitC: - case <-sub.quitC: - } - } + psc.publishToSub(sub, msg) + } +} - }(sub) +// publishToSub will block on the subscription inbox if there are more than inboxSize messages accumulated +func (psc *PubSubChannel) publishToSub(sub *Subscription, msg interface{}) { + atomic.AddInt64(sub.pending, 1) + defer atomic.AddInt64(sub.pending, -1) + select { + case <-psc.quitC: + case <-sub.quitC: + case sub.inbox <- msg: } } @@ -121,7 +119,7 @@ func (psc *PubSubChannel) Close() { for _, sub := range psc.subscriptions { sub.lock.Lock() sub.closed = true - sub.closeChannel() + close(sub.quitC) sub.lock.Unlock() } close(psc.quitC) @@ -150,31 +148,46 @@ func (sub *Subscription) ID() string { return sub.id } -func (sub *Subscription) closeChannel() { - sub.closeOnce.Do(func() { - close(sub.signal) - }) -} - -func (sub *Subscription) MessageCount() int64 { - return *sub.msgCount +func (sub *Subscription) MessageCount() int { + return sub.msgCount } func (sub *Subscription) Pending() int64 { return *sub.pending } -func newSubscription(id string, psc *PubSubChannel) Subscription { - var count int64 +func newSubscription(id string, psc *PubSubChannel, inboxSize int) *Subscription { var pending int64 - return Subscription{ + subscription := &Subscription{ closed: false, pubSubC: psc, + inbox: make(chan interface{}, inboxSize), signal: make(chan interface{}), closeOnce: sync.Once{}, id: id, quitC: make(chan struct{}), - msgCount: &count, + msgCount: 0, pending: &pending, } + // publishing goroutine. It closes the signal channel whenever it receives the quitC signal + go func(sub *Subscription) { + for { + select { + case <-sub.quitC: + close(sub.signal) + return + case msg := <-sub.inbox: + log.Debug("Retrieved inbox message", "msg", msg) + select { + case <-psc.quitC: + case <-sub.quitC: + close(sub.signal) + return + case sub.signal <- msg: + sub.msgCount++ + } + } + } + }(subscription) + return subscription } diff --git a/network/pubsubchannel/pubsub_test.go b/network/pubsubchannel/pubsub_test.go index ed1f6e22fd..937a17a208 100644 --- a/network/pubsubchannel/pubsub_test.go +++ b/network/pubsubchannel/pubsub_test.go @@ -32,7 +32,7 @@ func init() { } func TestPubSeveralSub(t *testing.T) { - pubSub := pubsubchannel.New() + pubSub := pubsubchannel.New(100) var group sync.WaitGroup bucketSubs1, _ := testSubscriptor(pubSub, 2, &group) bucketSubs2, _ := testSubscriptor(pubSub, 2, &group) @@ -54,7 +54,7 @@ func TestPubSeveralSub(t *testing.T) { } func TestPubUnsubscribe(t *testing.T) { - pubSub := pubsubchannel.New() + pubSub := pubsubchannel.New(100) var group sync.WaitGroup _, subscription := testSubscriptor(pubSub, 0, &group) msgBucket2, _ := testSubscriptor(pubSub, 1, &group) @@ -96,7 +96,7 @@ func testSubscriptor(pubsub *pubsubchannel.PubSubChannel, expectedMessages int, // TestUnsubscribeBeforeReadingMessages tests that there is no goroutine leak when a subscription is finished // before reading pending messages from the channel. func TestUnsubscribeBeforeReadingMessages(t *testing.T) { - ps := pubsubchannel.New() + ps := pubsubchannel.New(1001) s := ps.Subscribe() defer ps.Close() @@ -139,7 +139,7 @@ func (w testingErrorWriter) Write(b []byte) (int, error) { // channel is still not closed). However, we need to wait a bit before extracting messages from the channel to allow // the blocked publishers exit. In a real case, the moment a new message is published the channel will be closed. func TestMessagesAfterUnsubscribe(t *testing.T) { - ps := pubsubchannel.New() + ps := pubsubchannel.New(1001) defer ps.Close() s := ps.Subscribe() @@ -151,7 +151,6 @@ func TestMessagesAfterUnsubscribe(t *testing.T) { s.Unsubscribe() - ps.Publish("End message") var n int timeout := time.After(2 * time.Second) loop: @@ -169,8 +168,46 @@ loop: } t.Log("got", n, "messages") - if n > 0 { + if n > 1 { t.Errorf("Expected no message received after unsubscribing but got %v", n) } } + +// TestMessagesInOrder checks that messages are delivered in order to subscribers +func TestMessagesInOrder(t *testing.T) { + ps := pubsubchannel.New(1001) + defer ps.Close() + + s := ps.Subscribe() + + for i := 0; i < 1000; i++ { + ps.Publish(i) + } + c := s.ReceiveChannel() + + var n int + timeout := time.After(2 * time.Second) + var more = true + var last = -1 + for more && n < 1000 { + select { + case msg, ok := <-c: + if !ok { + more = false + } else { + newNum := msg.(int) + if newNum != last+1 { + t.Errorf("unsortered messages in pubsub channel. Expected %v, received %v", last+1, newNum) + more = false + } + last = last + 1 + n++ + } + case <-timeout: + t.Log("timeout") + more = false + } + } + +} From 30198220f47dd1b6138eb07b788ead00367d13c2 Mon Sep 17 00:00:00 2001 From: Alvaro Date: Wed, 6 Nov 2019 12:02:06 +0100 Subject: [PATCH 30/31] Exit publishing goroutine on psc.quitC close. Repeated sleep in test. --- network/kademlia_load_balancer_test.go | 9 +++++++-- network/pubsubchannel/pubsub.go | 3 +++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/network/kademlia_load_balancer_test.go b/network/kademlia_load_balancer_test.go index 77a3757c5f..37fb12f797 100644 --- a/network/kademlia_load_balancer_test.go +++ b/network/kademlia_load_balancer_test.go @@ -291,8 +291,13 @@ func TestResourceUseStats(t *testing.T) { } // we need to sleep to allow all messages to be received by lb - time.Sleep(100 * time.Millisecond) - count := lb.resourceUseStats.Len() + count := 0 + retries := 0 + for count == 0 && retries < 15 { + time.Sleep(10 * time.Millisecond) + count = lb.resourceUseStats.Len() + retries++ + } if count > 0 { t.Errorf("got resourceUseStats %v, want 0, uses: %v", count, lb.resourceUseStats.DumpAllUses()) } diff --git a/network/pubsubchannel/pubsub.go b/network/pubsubchannel/pubsub.go index a2036e99ea..5888467d6a 100644 --- a/network/pubsubchannel/pubsub.go +++ b/network/pubsubchannel/pubsub.go @@ -180,12 +180,15 @@ func newSubscription(id string, psc *PubSubChannel, inboxSize int) *Subscription log.Debug("Retrieved inbox message", "msg", msg) select { case <-psc.quitC: + return case <-sub.quitC: close(sub.signal) return case sub.signal <- msg: sub.msgCount++ } + case <-psc.quitC: + return } } }(subscription) From 2879509a00a5204cc5f9b63de40e54b6a63d0763 Mon Sep 17 00:00:00 2001 From: Alvaro Date: Tue, 12 Nov 2019 12:37:31 +0100 Subject: [PATCH 31/31] Avoid data race in getAllUseCounts --- network/resourceusestats/resource_use_stats.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/network/resourceusestats/resource_use_stats.go b/network/resourceusestats/resource_use_stats.go index 3c9940dc24..aac98e5abd 100644 --- a/network/resourceusestats/resource_use_stats.go +++ b/network/resourceusestats/resource_use_stats.go @@ -83,6 +83,8 @@ func (lb *ResourceUseStats) DumpAllUses() map[string]int { } func (lb *ResourceUseStats) getAllUseCounts(resources []Resource) []ResourceCount { + lb.lock.RLock() + defer lb.lock.RUnlock() peerUses := make([]ResourceCount, len(resources)) for i, resource := range resources { peerUses[i] = ResourceCount{