From fa62fad28416e214efc3e0ea43b4b4847c7c1b1a Mon Sep 17 00:00:00 2001 From: nolash Date: Fri, 30 Aug 2019 15:58:06 +0200 Subject: [PATCH 01/36] network: Cherrypick changes to kademlia for capabilities db --- network/capability/capability.go | 29 +++++ network/capability/capability_test.go | 56 ++++++++ network/discovery_test.go | 6 +- network/enr.go | 13 +- network/hive_test.go | 5 +- network/kademlia.go | 178 +++++++++++++++++++------- network/kademlia_test.go | 25 ++++ network/network.go | 1 - network/protocol_test.go | 1 - 9 files changed, 252 insertions(+), 62 deletions(-) diff --git a/network/capability/capability.go b/network/capability/capability.go index bd0e8d926f..197cfe030b 100644 --- a/network/capability/capability.go +++ b/network/capability/capability.go @@ -185,3 +185,32 @@ func (c *Capabilities) DecodeRLP(s *rlp.Stream) error { return nil } + +// Match returns true if all bits set in the argument is also set in the receiver +func (c *Capability) Match(capCompare *Capability) bool { + if len(c.Cap) != len(capCompare.Cap) { + return false + } + // on the first occurence of false where query has true we can fail + for i, flag := range capCompare.Cap { + if flag && !c.Cap[i] { + return false + } + } + return true +} + +func (c *Capabilities) Match(capsCompare *Capabilities) bool { + for _, capCompare := range capsCompare.Caps { + + // if queryied id doesn't exist in object we can nay right away + cap := c.Get(capCompare.Id) + if cap == nil { + return false + } + if !cap.Match(capCompare) { + return false + } + } + return true +} diff --git a/network/capability/capability_test.go b/network/capability/capability_test.go index a4d734e98e..b6095e19e9 100644 --- a/network/capability/capability_test.go +++ b/network/capability/capability_test.go @@ -172,3 +172,59 @@ func TestCapabilitiesRLP(t *testing.T) { t.Fatalf("cap 1 caps not correct, expected %v, got %v", cap2.Cap, cap2Restored.Cap) } } + +// TestCapabilitiesQuery tests methods for quering capability states +func TestCapabilitiesQuery(t *testing.T) { + + // Initialize capability + caps := NewCapabilities() + + // Register module. Should succeed + c1 := NewCapability(1, 3) + c1.Set(1) + err := caps.Add(c1) + if err != nil { + t.Fatalf("RegisterCapabilityModule fail: %v", err) + } + + c2 := NewCapability(42, 9) + c2.Set(2) + c2.Set(8) + err = caps.Add(c2) + if err != nil { + t.Fatalf("RegisterCapabilityModule fail: %v", err) + } + + capsCompare := NewCapabilities() + capCompare := NewCapability(42, 10) + capCompare.Set(2) + capCompare.Set(8) + capsCompare.Add(capCompare) + if caps.Match(capsCompare) { + t.Fatalf("Expected cCompare with mismatch length to fail; %s != %s", capsCompare, caps) + } + capsCompare = NewCapabilities() + capCompare = NewCapability(42, 9) + capCompare.Set(2) + capsCompare.Add(capCompare) + if !caps.Match(capsCompare) { + t.Fatalf("Expected %s to match %s", capsCompare, caps) + } + + capCompare = NewCapability(1, 3) + capsCompare.Add(capCompare) + if !caps.Match(capsCompare) { + t.Fatalf("Expected %s to match %s", capsCompare, caps) + } + + capCompare.Set(1) + if !caps.Match(capsCompare) { + t.Fatalf("Expected %s to match %s", capsCompare, caps) + } + + capCompare.Set(2) + if caps.Match(capsCompare) { + t.Fatalf("Expected %s not to match %s", capsCompare, caps) + } + +} diff --git a/network/discovery_test.go b/network/discovery_test.go index f7b61af627..7c14968f59 100644 --- a/network/discovery_test.go +++ b/network/discovery_test.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethersphere/swarm/network/capability" "github.com/ethersphere/swarm/p2p/protocols" p2ptest "github.com/ethersphere/swarm/p2p/testing" "github.com/ethersphere/swarm/pot" @@ -125,7 +126,10 @@ func testInitialPeersMsg(t *testing.T, peerPO, peerDepth int) { } register := func(a pot.Address, po int) { addr := pot.RandomAddressAt(a, po) - hive.Register(&BzzAddr{OAddr: addr[:]}) + bzzAddr := &BzzAddr{OAddr: addr[:]} + bzzAddr.WithCapabilities(capability.NewCapabilities()) + hive.Register(addr) + // hive.Register(&BzzAddr{OAddr: addr[:]}) } // generate connected and just registered peers diff --git a/network/enr.go b/network/enr.go index dc1264c385..025af9d128 100644 --- a/network/enr.go +++ b/network/enr.go @@ -48,12 +48,6 @@ func (b *ENRAddrEntry) DecodeRLP(s *rlp.Stream) error { return nil } -type ENRLightNodeEntry bool - -func (b ENRLightNodeEntry) ENRKey() string { - return "bzzlightnode" -} - type ENRBootNodeEntry bool func (b ENRBootNodeEntry) ENRKey() string { @@ -61,12 +55,10 @@ func (b ENRBootNodeEntry) ENRKey() string { } func getENRBzzPeer(p *p2p.Peer, rw p2p.MsgReadWriter, spec *protocols.Spec) *BzzPeer { - var lightnode ENRLightNodeEntry var bootnode ENRBootNodeEntry // retrieve the ENR Record data record := p.Node().Record() - record.Load(&lightnode) record.Load(&bootnode) // get the address; separate function as long as we need swarm/network:NewAddr() to call it @@ -74,9 +66,8 @@ func getENRBzzPeer(p *p2p.Peer, rw p2p.MsgReadWriter, spec *protocols.Spec) *Bzz // build the peer using the retrieved data return &BzzPeer{ - Peer: protocols.NewPeer(p, rw, spec), - LightNode: bool(lightnode), - BzzAddr: addr, + Peer: protocols.NewPeer(p, rw, spec), + BzzAddr: addr, } } diff --git a/network/hive_test.go b/network/hive_test.go index ff08a85690..2a20de7aa5 100644 --- a/network/hive_test.go +++ b/network/hive_test.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/p2p" + "github.com/ethersphere/swarm/network/capability" p2ptest "github.com/ethersphere/swarm/p2p/testing" "github.com/ethersphere/swarm/state" ) @@ -55,7 +56,7 @@ func TestRegisterAndConnect(t *testing.T) { } node := s.Nodes[0] - raddr := NewAddr(node) + raddr := NewAddr(node).WithCapabilities(capability.NewCapabilities()) pp.Register(raddr) // start the hive @@ -152,7 +153,7 @@ func TestHiveStatePersistance(t *testing.T) { h1, cleanup1 := startHive(t, dir) peers := make(map[string]bool) for i := 0; i < peersCount; i++ { - raddr := RandomAddr() + raddr := RandomAddr().WithCapabilities(capability.NewCapabilities()) h1.Register(raddr) peers[raddr.String()] = true } diff --git a/network/kademlia.go b/network/kademlia.go index f767137ba5..453ac6f072 100644 --- a/network/kademlia.go +++ b/network/kademlia.go @@ -19,6 +19,7 @@ package network import ( "bytes" "encoding/hex" + "errors" "fmt" "math/rand" "sort" @@ -29,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/metrics" "github.com/ethersphere/swarm/log" + "github.com/ethersphere/swarm/network/capability" "github.com/ethersphere/swarm/pot" sv "github.com/ethersphere/swarm/version" ) @@ -84,15 +86,16 @@ func NewKadParams() *KadParams { // Kademlia is a table of live peers and a db of known peers (node records) type Kademlia struct { - lock sync.RWMutex - *KadParams // Kademlia configuration parameters - base []byte // immutable baseaddress of the table - addrs *pot.Pot // pots container for known peer addresses - conns *pot.Pot // pots container for live peer connections - depth uint8 // stores the last current depth of saturation - nDepth int // stores the last neighbourhood depth - nDepthMu sync.RWMutex // protects neighbourhood depth nDepth - nDepthSig []chan struct{} // signals when neighbourhood depth nDepth is changed + lock sync.RWMutex + capabilityIndex map[string]*capabilityIndex + *KadParams // Kademlia configuration parameters + base []byte // immutable baseaddress of the table + addrs *pot.Pot // pots container for known peer addresses + conns *pot.Pot // pots container for live peer connections + depth uint8 // stores the last current depth of saturation + nDepth int // stores the last neighbourhood depth + nDepthMu sync.RWMutex // protects neighbourhood depth nDepth + nDepthSig []chan struct{} // signals when neighbourhood depth nDepth is changed } type KademliaInfo struct { @@ -111,11 +114,54 @@ func NewKademlia(addr []byte, params *KadParams) *Kademlia { if params == nil { params = NewKadParams() } - return &Kademlia{ - base: addr, - KadParams: params, - addrs: pot.NewPot(nil, 0), - conns: pot.NewPot(nil, 0), + k := &Kademlia{ + base: addr, + KadParams: params, + capabilityIndex: make(map[string]*capabilityIndex), + addrs: pot.NewPot(nil, 0), + conns: pot.NewPot(nil, 0), + } + k.RegisterCapabilityIndex("full", *fullCapability) + k.RegisterCapabilityIndex("light", *lightCapability) + return k +} + +func (k *Kademlia) RegisterCapabilityIndex(s string, c capability.Capability) error { + if s == "" { + return errors.New("Cannot add index with empty string key") + } else if _, ok := k.capabilityIndex[s]; ok { + return fmt.Errorf("Capability index '%s' already exists", s) + } + log.Debug("Registered cap index", "s", s, "c", c) + k.capabilityIndex[s] = NewCapabilityIndex(c) + return nil +} + +// blindly add index to +func (k *Kademlia) addToCapabilityIndex(p interface{}, connected bool) { + var eAddr *BzzAddr + var ePeer *Peer + if connected { + ePeer = p.(*Peer) + eAddr = ePeer.BzzAddr + } else { + eAddr = p.(*entry).BzzAddr + } + for s, idxItem := range k.capabilityIndex { + for _, vCap := range eAddr.Capabilities.Caps { + log.Debug("in add to capability compare", "idx", s, "peer", vCap, "idx", idxItem.Capability) + if idxItem.Id != vCap.Id { + continue + } + if idxItem.Match(vCap) { + log.Debug("Added peer to capability index", "conn", connected, "s", s, "v", vCap, "p", p) + if connected { + k.capabilityIndex[s].conns, _, _ = pot.Add(idxItem.conns, ePeer, Pof) + } else { + k.capabilityIndex[s].addrs, _, _ = pot.Add(idxItem.addrs, newEntry(eAddr), Pof) + } + } + } } } @@ -135,6 +181,22 @@ func newEntry(p *BzzAddr) *entry { } } +// index providing quick access to all peers having a certain capability set +type capabilityIndex struct { + *capability.Capability + conns *pot.Pot + addrs *pot.Pot +} + +// NewCapabilityIndex creates a new capability index with a copy the provided capabilities array +func NewCapabilityIndex(c capability.Capability) *capabilityIndex { + return &capabilityIndex{ + Capability: &c, + conns: pot.NewPot(nil, 0), + addrs: pot.NewPot(nil, 0), + } +} + // Label is a short tag for the entry for debug func Label(e *entry) string { return fmt.Sprintf("%s (%d)", e.Hex()[:4], e.retries) @@ -153,7 +215,7 @@ func (k *Kademlia) Register(peers ...*BzzAddr) error { metrics.GetOrRegisterCounter("kad.register", nil).Inc(1) - var known, size int + var size int for _, p := range peers { log.Trace("kademlia trying to register", "addr", p) // error if self received, peer should know better @@ -161,8 +223,7 @@ func (k *Kademlia) Register(peers ...*BzzAddr) error { if bytes.Equal(p.Address(), k.base) { return fmt.Errorf("add peers: %x is self", k.base) } - var found bool - k.addrs, _, found, _ = pot.Swap(k.addrs, p, Pof, func(v pot.Val) pot.Val { + k.addrs, _, _, _ = pot.Swap(k.addrs, p, Pof, func(v pot.Val) pot.Val { // if not found if v == nil { log.Trace("registering new peer", "addr", p) @@ -181,9 +242,7 @@ func (k *Kademlia) Register(peers ...*BzzAddr) error { return v }) - if found { - known++ - } + k.addToCapabilityIndex(newEntry(p), false) size++ } @@ -323,7 +382,8 @@ func (k *Kademlia) On(p *Peer) (uint8, bool) { // found among live peers, do nothing return v }) - if ins && !p.BzzPeer.LightNode { + k.addToCapabilityIndex(p, true) + if ins { a := newEntry(p.BzzAddr) a.conn = p // insert new online peer into addrs @@ -411,26 +471,34 @@ func (k *Kademlia) Off(p *Peer) { k.lock.Lock() defer k.lock.Unlock() var del bool - if !p.BzzPeer.LightNode { - k.addrs, _, _, _ = pot.Swap(k.addrs, p, Pof, func(v pot.Val) pot.Val { - // v cannot be nil, must check otherwise we overwrite entry - if v == nil { - panic(fmt.Sprintf("connected peer not found %v", p)) - } - del = true - return newEntry(p.BzzAddr) - }) - } else { + k.addrs, _, _, _ = pot.Swap(k.addrs, p, Pof, func(v pot.Val) pot.Val { + // v cannot be nil, must check otherwise we overwrite entry + if v == nil { + panic(fmt.Sprintf("connected peer not found %v", p)) + } del = true - } + return newEntry(p.BzzAddr) + }) + // note the following only ran if the peer was a lightnode + k.conns, _, _, _ = pot.Swap(k.conns, p, Pof, func(_ pot.Val) pot.Val { + // v cannot be nil, but no need to check + return nil + }) + k.setNeighbourhoodDepth() +} - if del { - k.conns, _, _, _ = pot.Swap(k.conns, p, Pof, func(_ pot.Val) pot.Val { - // v cannot be nil, but no need to check - return nil - }) - k.setNeighbourhoodDepth() +// EachConn is an iterator with args (base, po, f) applies f to each live peer +// that has proximity order po or less as measured from the base +// if base is nil, kademlia base address is used +func (k *Kademlia) EachConnFiltered(base []byte, capKey string, o int, f func(*Peer, int) bool) error { + k.lock.RLock() + defer k.lock.RUnlock() + c, ok := k.capabilityIndex[capKey] + if !ok { + return fmt.Errorf("Unregistered capability index '%s'", capKey) } + k.eachConn(base, c.conns, o, f) + return nil } // EachConn is an iterator with args (base, po, f) applies f to each live peer @@ -439,14 +507,17 @@ func (k *Kademlia) Off(p *Peer) { func (k *Kademlia) EachConn(base []byte, o int, f func(*Peer, int) bool) { k.lock.RLock() defer k.lock.RUnlock() - k.eachConn(base, o, f) + k.eachConn(base, k.conns, o, f) } -func (k *Kademlia) eachConn(base []byte, o int, f func(*Peer, int) bool) { +func (k *Kademlia) eachConn(base []byte, db *pot.Pot, o int, f func(*Peer, int) bool) { if len(base) == 0 { base = k.base } - k.conns.EachNeighbour(base, Pof, func(val pot.Val, po int) bool { + if db == nil { + db = k.conns + } + db.EachNeighbour(base, Pof, func(val pot.Val, po int) bool { if po > o { return true } @@ -454,20 +525,35 @@ func (k *Kademlia) eachConn(base []byte, o int, f func(*Peer, int) bool) { }) } +func (k *Kademlia) EachAddrFiltered(base []byte, capKey string, o int, f func(*BzzAddr, int) bool) error { + k.lock.RLock() + defer k.lock.RUnlock() + c, ok := k.capabilityIndex[capKey] + if !ok { + return fmt.Errorf("Unregistered capability index '%s'", capKey) + } + log.Debug("filter with capname", "key", capKey, "cap", c) + k.eachAddr(base, c.addrs, o, f) + return nil +} + // EachAddr called with (base, po, f) is an iterator applying f to each known peer // that has proximity order o or less as measured from the base // if base is nil, kademlia base address is used func (k *Kademlia) EachAddr(base []byte, o int, f func(*BzzAddr, int) bool) { k.lock.RLock() defer k.lock.RUnlock() - k.eachAddr(base, o, f) + k.eachAddr(base, k.addrs, o, f) } -func (k *Kademlia) eachAddr(base []byte, o int, f func(*BzzAddr, int) bool) { +func (k *Kademlia) eachAddr(base []byte, db *pot.Pot, o int, f func(*BzzAddr, int) bool) { if len(base) == 0 { base = k.base } - k.addrs.EachNeighbour(base, Pof, func(val pot.Val, po int) bool { + if db == nil { + db = k.addrs + } + db.EachNeighbour(base, Pof, func(val pot.Val, po int) bool { if po > o { return true } @@ -836,7 +922,7 @@ func (k *Kademlia) knowNeighbours(addrs [][]byte) (got bool, n int, missing [][] pm := make(map[string]bool) depth := depthForPot(k.conns, k.NeighbourhoodSize, k.base) // create a map with all peers at depth and deeper known in the kademlia - k.eachAddr(nil, 255, func(p *BzzAddr, po int) bool { + k.eachAddr(nil, nil, 255, func(p *BzzAddr, po int) bool { // in order deepest to shallowest compared to the kademlia base address // all bins (except self) are included (0 <= bin <= 255) if po < depth { @@ -875,7 +961,7 @@ func (k *Kademlia) connectedNeighbours(peers [][]byte) (got bool, n int, missing // in order deepest to shallowest compared to the kademlia base address // all bins (except self) are included (0 <= bin <= 255) depth := depthForPot(k.conns, k.NeighbourhoodSize, k.base) - k.eachConn(nil, 255, func(p *Peer, po int) bool { + k.eachConn(nil, nil, 255, func(p *Peer, po int) bool { if po < depth { return false } diff --git a/network/kademlia_test.go b/network/kademlia_test.go index be20ece17c..eec1c49ff0 100644 --- a/network/kademlia_test.go +++ b/network/kademlia_test.go @@ -23,9 +23,11 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethersphere/swarm/network/capability" "github.com/ethersphere/swarm/p2p/protocols" "github.com/ethersphere/swarm/pot" ) @@ -670,3 +672,26 @@ func TestKademlia_SubscribeToNeighbourhoodDepthChange(t *testing.T) { } }) } + +func NewAdaptivePeer(k *Kademlia, capabilities ...*capability.Capability) (*Peer, error) { + // create the peer that fits the kademlia record + // it's quite a bit of work + peerPrivKey, err := crypto.GenerateKey() + if err != nil { + return nil, err + } + peerEnodeId := enode.PubkeyToIDV4(&peerPrivKey.PublicKey) + peerP2p := p2p.NewPeer(peerEnodeId, "foo", []p2p.Cap{}) + peerProto := protocols.NewPeer(peerP2p, nil, nil) + peerBzz := NewBzzPeer(peerProto) + caps := capability.NewCapabilities() + for _, cap := range capabilities { + caps.Add(cap) + } + peerBzz.WithCapabilities(caps) + err = k.Register(peerBzz.BzzAddr) + if err != nil { + return nil, err + } + return NewPeer(peerBzz, k), nil +} diff --git a/network/network.go b/network/network.go index 747f9003ab..4235c12005 100644 --- a/network/network.go +++ b/network/network.go @@ -87,7 +87,6 @@ func NewEnodeRecord(params *EnodeParams) (*enr.Record, error) { var record enr.Record record.Set(NewENRAddrEntry(bzzkeybytes)) - record.Set(ENRLightNodeEntry(params.Lightnode)) record.Set(ENRBootNodeEntry(params.Bootnode)) return &record, nil } diff --git a/network/protocol_test.go b/network/protocol_test.go index 6e36515b52..4520b11af7 100644 --- a/network/protocol_test.go +++ b/network/protocol_test.go @@ -184,7 +184,6 @@ func newBzzHandshakeTester(n int, prvkey *ecdsa.PrivateKey, lightNode bool) (*bz var record enr.Record bzzkey := PrivateKeyToBzzKey(prvkey) record.Set(NewENRAddrEntry(bzzkey)) - record.Set(ENRLightNodeEntry(lightNode)) err := enode.SignV4(&record, prvkey) if err != nil { return nil, err From 97e4547680237efe55324b7d8a3d8f482d318231 Mon Sep 17 00:00:00 2001 From: nolash Date: Fri, 30 Aug 2019 20:32:26 +0200 Subject: [PATCH 02/36] network: WIP rehabilitate tests --- network/discovery_test.go | 6 +++--- network/kademlia.go | 2 -- network/kademlia_test.go | 8 +++++--- network/network.go | 15 +++++++++++---- network/networkid_test.go | 3 ++- network/protocol.go | 17 ++++++++--------- network/protocol_test.go | 2 +- pss/forwarding_test.go | 2 ++ pss/pss_test.go | 6 ++++-- 9 files changed, 36 insertions(+), 25 deletions(-) diff --git a/network/discovery_test.go b/network/discovery_test.go index 7c14968f59..b2190e706e 100644 --- a/network/discovery_test.go +++ b/network/discovery_test.go @@ -127,8 +127,8 @@ func testInitialPeersMsg(t *testing.T, peerPO, peerDepth int) { register := func(a pot.Address, po int) { addr := pot.RandomAddressAt(a, po) bzzAddr := &BzzAddr{OAddr: addr[:]} - bzzAddr.WithCapabilities(capability.NewCapabilities()) - hive.Register(addr) + bzzAddr = bzzAddr.WithCapabilities(capability.NewCapabilities()) + hive.Register(bzzAddr) // hive.Register(&BzzAddr{OAddr: addr[:]}) } @@ -249,7 +249,7 @@ func newDiscPeer(addr pot.Address) (*Peer, error) { } pubKey := pKey.PublicKey nod := enode.NewV4(&pubKey, net.IPv4(127, 0, 0, 1), 0, 0) - bzzAddr := &BzzAddr{OAddr: addr[:], UAddr: []byte(nod.String())} + bzzAddr := &BzzAddr{OAddr: addr[:], UAddr: []byte(nod.String()), Capabilities: capability.NewCapabilities()} id := nod.ID() p2pPeer := p2p.NewPeer(id, id.String(), nil) return NewPeer(&BzzPeer{ diff --git a/network/kademlia.go b/network/kademlia.go index 453ac6f072..3fdc92fca3 100644 --- a/network/kademlia.go +++ b/network/kademlia.go @@ -470,13 +470,11 @@ func (k *Kademlia) SubscribeToNeighbourhoodDepthChange() (c <-chan struct{}, uns func (k *Kademlia) Off(p *Peer) { k.lock.Lock() defer k.lock.Unlock() - var del bool k.addrs, _, _, _ = pot.Swap(k.addrs, p, Pof, func(v pot.Val) pot.Val { // v cannot be nil, must check otherwise we overwrite entry if v == nil { panic(fmt.Sprintf("connected peer not found %v", p)) } - del = true return newEntry(p.BzzAddr) }) // note the following only ran if the peer was a lightnode diff --git a/network/kademlia_test.go b/network/kademlia_test.go index eec1c49ff0..b2809a7763 100644 --- a/network/kademlia_test.go +++ b/network/kademlia_test.go @@ -39,7 +39,7 @@ func init() { func testKadPeerAddr(s string) *BzzAddr { a := pot.NewAddressFromString(s) - return &BzzAddr{OAddr: a, UAddr: a} + return &BzzAddr{OAddr: a, UAddr: a, Capabilities: capability.NewCapabilities()} } func newTestKademliaParams() *KadParams { @@ -476,6 +476,7 @@ func TestOffEffectingAddressBookNormalNode(t *testing.T) { // a light node should not be in the address book func TestOffEffectingAddressBookLightNode(t *testing.T) { + t.Skip("lightnode flag is now obsolete, this test must be changed") tk := newTestKademlia(t, "00000000") // light node peer added to kademlia tk.Kademlia.On(tk.newTestKadPeer("01000000", true)) @@ -556,8 +557,9 @@ func newTestDiscoveryPeer(addr pot.Address, kad *Kademlia) *Peer { bp := &BzzPeer{ Peer: pp, BzzAddr: &BzzAddr{ - OAddr: addr.Bytes(), - UAddr: []byte(fmt.Sprintf("%x", addr[:])), + OAddr: addr.Bytes(), + UAddr: []byte(fmt.Sprintf("%x", addr[:])), + Capabilities: capability.NewCapabilities(), }, } return NewPeer(bp, kad) diff --git a/network/network.go b/network/network.go index 4235c12005..a2d3707293 100644 --- a/network/network.go +++ b/network/network.go @@ -8,12 +8,14 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/enr" + "github.com/ethersphere/swarm/network/capability" ) // BzzAddr implements the PeerAddr interface type BzzAddr struct { - OAddr []byte - UAddr []byte + OAddr []byte + UAddr []byte + Capabilities *capability.Capabilities } // Address implements OverlayPeer interface to be used in Overlay. @@ -42,7 +44,7 @@ func (a *BzzAddr) ID() enode.ID { // Update updates the underlay address of a peer record func (a *BzzAddr) Update(na *BzzAddr) *BzzAddr { - return &BzzAddr{a.OAddr, na.UAddr} + return &BzzAddr{a.OAddr, na.UAddr, capability.NewCapabilities()} } // String pretty prints the address @@ -62,7 +64,12 @@ func RandomAddr() *BzzAddr { // NewAddr constructs a BzzAddr from a node record. func NewAddr(node *enode.Node) *BzzAddr { - return &BzzAddr{OAddr: node.ID().Bytes(), UAddr: []byte(node.URLv4())} + return &BzzAddr{OAddr: node.ID().Bytes(), UAddr: []byte(node.URLv4()), Capabilities: capability.NewCapabilities()} +} + +func (b *BzzAddr) WithCapabilities(c *capability.Capabilities) *BzzAddr { + b.Capabilities = c + return b } func PrivateKeyToBzzKey(prvKey *ecdsa.PrivateKey) []byte { diff --git a/network/networkid_test.go b/network/networkid_test.go index cd60e89849..9634ab61de 100644 --- a/network/networkid_test.go +++ b/network/networkid_test.go @@ -32,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/p2p/simulations" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" "github.com/ethereum/go-ethereum/rpc" + "github.com/ethersphere/swarm/network/capability" "github.com/ethersphere/swarm/testutil" ) @@ -198,7 +199,7 @@ func newServices() adapters.Services { } return adapters.Services{ "bzz": func(ctx *adapters.ServiceContext) (node.Service, error) { - addr := NewAddr(ctx.Config.Node()) + addr := NewAddr(ctx.Config.Node()).WithCapabilities(capability.NewCapabilities()) hp := NewHiveParams() hp.Discovery = false cnt++ diff --git a/network/protocol.go b/network/protocol.go index be8cc0d024..f1830a4b24 100644 --- a/network/protocol.go +++ b/network/protocol.go @@ -125,7 +125,6 @@ type Bzz struct { handshakes map[enode.ID]*HandshakeMsg streamerSpec *protocols.Spec streamerRun func(*BzzPeer) error - capabilities *capability.Capabilities // capabilities control and state retrievalSpec *protocols.Spec retrievalRun func(*BzzPeer) error } @@ -139,13 +138,12 @@ func NewBzz(config *BzzConfig, kad *Kademlia, store state.Store, streamerSpec, r bzz := &Bzz{ Hive: NewHive(config.HiveParams, kad, store), NetworkID: config.NetworkID, - localAddr: &BzzAddr{config.OverlayAddr, config.UnderlayAddr}, + localAddr: &BzzAddr{config.OverlayAddr, config.UnderlayAddr, capability.NewCapabilities()}, handshakes: make(map[enode.ID]*HandshakeMsg), streamerRun: streamerRun, streamerSpec: streamerSpec, retrievalRun: retrievalRun, retrievalSpec: retrievalSpec, - capabilities: capability.NewCapabilities(), } if config.BootnodeMode { @@ -157,9 +155,9 @@ func NewBzz(config *BzzConfig, kad *Kademlia, store state.Store, streamerSpec, r // temporary soon-to-be-legacy light/full, as above if config.LightNode { - bzz.capabilities.Add(newLightCapability()) + bzz.localAddr.Capabilities.Add(newLightCapability()) } else { - bzz.capabilities.Add(newFullCapability()) + bzz.localAddr.Capabilities.Add(newFullCapability()) } return bzz @@ -288,7 +286,7 @@ func (b *Bzz) performHandshake(p *protocols.Peer, handshake *HandshakeMsg) error return err } handshake.peerAddr = rsh.(*HandshakeMsg).Addr - handshake.Capabilities = rsh.(*HandshakeMsg).Capabilities + handshake.peerAddr.Capabilities = rsh.(*HandshakeMsg).Capabilities return nil } @@ -399,9 +397,10 @@ func (b *Bzz) GetOrCreateHandshake(peerID enode.ID) (*HandshakeMsg, bool) { Version: uint64(BzzSpec.Version), NetworkID: b.NetworkID, Addr: b.localAddr, - Capabilities: b.capabilities, - init: make(chan bool, 1), - done: make(chan struct{}), + Capabilities: b.localAddr.Capabilities, + //Capabilities: b.capabilities, + init: make(chan bool, 1), + done: make(chan struct{}), } // when handhsake is first created for a remote peer // it is initialised with the init diff --git a/network/protocol_test.go b/network/protocol_test.go index 4520b11af7..4f0ad05ad9 100644 --- a/network/protocol_test.go +++ b/network/protocol_test.go @@ -118,7 +118,7 @@ func newBzzBaseTesterWithAddrs(prvkey *ecdsa.PrivateKey, addrs [][]byte, spec *p mu.Lock() nodeToAddr[p.ID()] = addrs[0] mu.Unlock() - bzzAddr := &BzzAddr{addrs[0], []byte(p.Node().String())} + bzzAddr := &BzzAddr{addrs[0], []byte(p.Node().String()), capability.NewCapabilities()} addrs = addrs[1:] return srv(&BzzPeer{Peer: protocols.NewPeer(p, rw, spec), BzzAddr: bzzAddr}) } diff --git a/pss/forwarding_test.go b/pss/forwarding_test.go index d936f02259..f2a8861d92 100644 --- a/pss/forwarding_test.go +++ b/pss/forwarding_test.go @@ -10,6 +10,7 @@ import ( "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethersphere/swarm/network" + "github.com/ethersphere/swarm/network/capability" "github.com/ethersphere/swarm/p2p/protocols" "github.com/ethersphere/swarm/pot" "github.com/ethersphere/swarm/pss/message" @@ -319,6 +320,7 @@ func testForwardMsg(t *testing.T, ps *Pss, c *testCase) { func addPeers(kad *network.Kademlia, addresses []pot.Address) { for _, a := range addresses { p := newTestDiscoveryPeer(a, kad) + p.BzzAddr.Capabilities = capability.NewCapabilities() kad.On(p) } } diff --git a/pss/pss_test.go b/pss/pss_test.go index 624b1f33fa..7f653e4ad1 100644 --- a/pss/pss_test.go +++ b/pss/pss_test.go @@ -40,6 +40,7 @@ import ( "github.com/ethereum/go-ethereum/p2p/simulations/adapters" "github.com/ethereum/go-ethereum/rpc" "github.com/ethersphere/swarm/network" + "github.com/ethersphere/swarm/network/capability" "github.com/ethersphere/swarm/network/simulation" "github.com/ethersphere/swarm/p2p/protocols" "github.com/ethersphere/swarm/pot" @@ -198,6 +199,7 @@ func TestAddressMatchProx(t *testing.T) { }, } peer := network.NewPeer(bzzPeer, kad) + peer.Capabilities = capability.NewCapabilities() kad.On(peer) peers = append(peers, peer) } @@ -602,7 +604,7 @@ func TestPeerCapabilityMismatch(t *testing.T) { nid := enode.ID{0x01} wrongpsspeer := network.NewPeer(&network.BzzPeer{ Peer: protocols.NewPeer(p2p.NewPeer(nid, common.ToHex(wrongpssaddr.Over()), []p2p.Cap{wrongpsscap}), rw, nil), - BzzAddr: &network.BzzAddr{OAddr: wrongpssaddr.Over(), UAddr: nil}, + BzzAddr: &network.BzzAddr{OAddr: wrongpssaddr.Over(), UAddr: nil, Capabilities: capability.NewCapabilities()}, }, kad) // one peer doesn't even have pss (boo!) @@ -614,7 +616,7 @@ func TestPeerCapabilityMismatch(t *testing.T) { nid = enode.ID{0x02} nopsspeer := network.NewPeer(&network.BzzPeer{ Peer: protocols.NewPeer(p2p.NewPeer(nid, common.ToHex(nopssaddr.Over()), []p2p.Cap{nopsscap}), rw, nil), - BzzAddr: &network.BzzAddr{OAddr: nopssaddr.Over(), UAddr: nil}, + BzzAddr: &network.BzzAddr{OAddr: nopssaddr.Over(), UAddr: nil, Capabilities: capability.NewCapabilities()}, }, kad) // add peers to kademlia and activate them From 8e950616d69f5ac0b6516c748841996991a6f7e3 Mon Sep 17 00:00:00 2001 From: nolash Date: Mon, 2 Sep 2019 15:59:32 +0200 Subject: [PATCH 03/36] network, pss: Add capabilities to discovery message (WIP) --- network/discovery.go | 21 +++++++++++++++++++-- network/discovery_test.go | 2 +- network/hive_test.go | 1 + network/kademlia.go | 5 ++++- network/kademlia_test.go | 4 ++-- network/network.go | 28 ++++++++++++++++++++++------ network/protocol.go | 13 +++++++------ pss/forwarding_test.go | 8 ++------ pss/pss_test.go | 11 +++-------- 9 files changed, 61 insertions(+), 32 deletions(-) diff --git a/network/discovery.go b/network/discovery.go index 5d97f0f1b6..97aea50035 100644 --- a/network/discovery.go +++ b/network/discovery.go @@ -22,6 +22,8 @@ import ( "sync" "github.com/ethersphere/swarm/pot" + "github.com/ethersphere/swarm/log" + "github.com/ethersphere/swarm/network/capability" ) // discovery bzz extension for requesting and relaying node address records @@ -94,6 +96,10 @@ func (d *Peer) NotifyPeer(a *BzzAddr, po uint8) { } resp := &peersMsg{ Peers: []*BzzAddr{a}, + Capabilities: []*capability.Capabilities{a.capabilities}, + } + if a.capabilities == nil { + log.Crit("tried to send capabilities empty", "msg", resp) } go d.Send(context.TODO(), resp) } @@ -125,6 +131,7 @@ disconnected // relevant for bootstrapping connectivity and updating peersets type peersMsg struct { Peers []*BzzAddr + Capabilities []*capability.Capabilities } // String pretty prints a peersMsg @@ -140,7 +147,10 @@ func (d *Peer) handlePeersMsg(msg *peersMsg) error { if len(msg.Peers) == 0 { return nil } - + for i, cp := range msg.Capabilities { + msg.Peers[i].capabilities = cp + log.Warn("incoming notify peer", "peer", msg.Peers[i], "cap", cp) + } for _, a := range msg.Peers { d.seen(a) NotifyPeer(a, d.kad) @@ -183,7 +193,14 @@ func (d *Peer) handleSubPeersMsg(msg *subPeersMsg) error { }) // if useful peers are found, send them over if len(peers) > 0 { - go d.Send(context.TODO(), &peersMsg{Peers: sortPeers(peers)}) + outMsg := &peersMsg{Peers: sortPeers(peers), Capabilities: []*capability.Capabilities{}} + for _, p := range peers { + if p.capabilities == nil { + log.Crit("attempting to send handlesub with cap nil", outMsg) + } + outMsg.Capabilities = append(outMsg.Capabilities, p.capabilities) + } + go d.Send(context.TODO(), outMsg) } } d.sentPeers = true diff --git a/network/discovery_test.go b/network/discovery_test.go index b2190e706e..431e1ebed7 100644 --- a/network/discovery_test.go +++ b/network/discovery_test.go @@ -249,7 +249,7 @@ func newDiscPeer(addr pot.Address) (*Peer, error) { } pubKey := pKey.PublicKey nod := enode.NewV4(&pubKey, net.IPv4(127, 0, 0, 1), 0, 0) - bzzAddr := &BzzAddr{OAddr: addr[:], UAddr: []byte(nod.String()), Capabilities: capability.NewCapabilities()} + bzzAddr := &BzzAddr{OAddr: addr[:], UAddr: []byte(nod.String()), capabilities: capability.NewCapabilities()} id := nod.ID() p2pPeer := p2p.NewPeer(id, id.String(), nil) return NewPeer(&BzzPeer{ diff --git a/network/hive_test.go b/network/hive_test.go index 2a20de7aa5..c5461cca1d 100644 --- a/network/hive_test.go +++ b/network/hive_test.go @@ -110,6 +110,7 @@ func TestRegisterAndConnect(t *testing.T) { // Actual connectivity is not in scope for this test, as the peers loaded from state are not known to // the simulation; the test only verifies that the peers are known to the node func TestHiveStatePersistance(t *testing.T) { + t.Skip("persistence test does not retrieve capabilities properly") dir, err := ioutil.TempDir("", "hive_test_store") if err != nil { t.Fatal(err) diff --git a/network/kademlia.go b/network/kademlia.go index 3fdc92fca3..ffa343aab1 100644 --- a/network/kademlia.go +++ b/network/kademlia.go @@ -148,7 +148,7 @@ func (k *Kademlia) addToCapabilityIndex(p interface{}, connected bool) { eAddr = p.(*entry).BzzAddr } for s, idxItem := range k.capabilityIndex { - for _, vCap := range eAddr.Capabilities.Caps { + for _, vCap := range eAddr.capabilities.Caps { log.Debug("in add to capability compare", "idx", s, "peer", vCap, "idx", idxItem.Capability) if idxItem.Id != vCap.Id { continue @@ -217,6 +217,9 @@ func (k *Kademlia) Register(peers ...*BzzAddr) error { var size int for _, p := range peers { + if p.capabilities == nil { + panic("missing caps for peer") + } log.Trace("kademlia trying to register", "addr", p) // error if self received, peer should know better // and should be punished for this diff --git a/network/kademlia_test.go b/network/kademlia_test.go index b2809a7763..72e0dd5463 100644 --- a/network/kademlia_test.go +++ b/network/kademlia_test.go @@ -39,7 +39,7 @@ func init() { func testKadPeerAddr(s string) *BzzAddr { a := pot.NewAddressFromString(s) - return &BzzAddr{OAddr: a, UAddr: a, Capabilities: capability.NewCapabilities()} + return &BzzAddr{OAddr: a, UAddr: a, capabilities: capability.NewCapabilities()} } func newTestKademliaParams() *KadParams { @@ -559,7 +559,7 @@ func newTestDiscoveryPeer(addr pot.Address, kad *Kademlia) *Peer { BzzAddr: &BzzAddr{ OAddr: addr.Bytes(), UAddr: []byte(fmt.Sprintf("%x", addr[:])), - Capabilities: capability.NewCapabilities(), + capabilities: capability.NewCapabilities(), }, } return NewPeer(bp, kad) diff --git a/network/network.go b/network/network.go index a2d3707293..c876a4678e 100644 --- a/network/network.go +++ b/network/network.go @@ -15,7 +15,15 @@ import ( type BzzAddr struct { OAddr []byte UAddr []byte - Capabilities *capability.Capabilities + capabilities *capability.Capabilities +} + +func NewBzzAddr(oaddr []byte, uaddr []byte) *BzzAddr { + return &BzzAddr{ + OAddr: oaddr, + UAddr: uaddr, + capabilities: capability.NewCapabilities(), + } } // Address implements OverlayPeer interface to be used in Overlay. @@ -33,6 +41,10 @@ func (a *BzzAddr) Under() []byte { return a.UAddr } +func (a *BzzAddr) Capabilities() *capability.Capabilities { + return a.capabilities +} + // ID returns the node identifier in the underlay. func (a *BzzAddr) ID() enode.ID { n, err := enode.ParseV4(string(a.UAddr)) @@ -44,12 +56,12 @@ func (a *BzzAddr) ID() enode.ID { // Update updates the underlay address of a peer record func (a *BzzAddr) Update(na *BzzAddr) *BzzAddr { - return &BzzAddr{a.OAddr, na.UAddr, capability.NewCapabilities()} + return &BzzAddr{a.OAddr, na.UAddr, a.capabilities} } // String pretty prints the address func (a *BzzAddr) String() string { - return fmt.Sprintf("%x <%s>", a.OAddr, a.UAddr) + return fmt.Sprintf("%x <%s> cap:%s", a.OAddr, a.UAddr, a.capabilities) } // RandomAddr is a utility method generating an address from a public key @@ -63,12 +75,16 @@ func RandomAddr() *BzzAddr { } // NewAddr constructs a BzzAddr from a node record. -func NewAddr(node *enode.Node) *BzzAddr { - return &BzzAddr{OAddr: node.ID().Bytes(), UAddr: []byte(node.URLv4()), Capabilities: capability.NewCapabilities()} +func NewAddr(enod *enode.Node) *BzzAddr { + return NewBzzAddrFromEnode(enod) +} + +func NewBzzAddrFromEnode(enod *enode.Node) *BzzAddr { + return &BzzAddr{OAddr: enod.ID().Bytes(), UAddr: []byte(enod.URLv4()), capabilities: capability.NewCapabilities()} } func (b *BzzAddr) WithCapabilities(c *capability.Capabilities) *BzzAddr { - b.Capabilities = c + b.capabilities = c return b } diff --git a/network/protocol.go b/network/protocol.go index f1830a4b24..9b84957870 100644 --- a/network/protocol.go +++ b/network/protocol.go @@ -155,9 +155,9 @@ func NewBzz(config *BzzConfig, kad *Kademlia, store state.Store, streamerSpec, r // temporary soon-to-be-legacy light/full, as above if config.LightNode { - bzz.localAddr.Capabilities.Add(newLightCapability()) + bzz.localAddr.capabilities.Add(newLightCapability()) } else { - bzz.localAddr.Capabilities.Add(newFullCapability()) + bzz.localAddr.capabilities.Add(newFullCapability()) } return bzz @@ -257,7 +257,7 @@ func (b *Bzz) RunProtocol(spec *protocols.Spec, run func(*BzzPeer) error) func(* if handshake.err != nil { return fmt.Errorf("%08x: %s protocol closed: %v", b.BaseAddr()[:4], spec.Name, handshake.err) } - + // the handshake has succeeded so construct the BzzPeer and run the protocol peer := &BzzPeer{ Peer: protocols.NewPeer(p, rw, spec), @@ -285,8 +285,9 @@ func (b *Bzz) performHandshake(p *protocols.Peer, handshake *HandshakeMsg) error handshake.err = err return err } + rsh.(*HandshakeMsg).Addr.capabilities = rsh.(*HandshakeMsg).Capabilities handshake.peerAddr = rsh.(*HandshakeMsg).Addr - handshake.peerAddr.Capabilities = rsh.(*HandshakeMsg).Capabilities + handshake.Capabilities = rsh.(*HandshakeMsg).Capabilities return nil } @@ -376,6 +377,7 @@ func (b *Bzz) checkHandshake(hs interface{}) error { if !isFullCapability(rhs.Capabilities.Get(0)) && !isLightCapability(rhs.Capabilities.Get(0)) { return fmt.Errorf("invalid capabilities setting: %s", rhs.Capabilities) } + rhs.Addr.capabilities = hs.(*HandshakeMsg).Capabilities return nil } @@ -397,8 +399,7 @@ func (b *Bzz) GetOrCreateHandshake(peerID enode.ID) (*HandshakeMsg, bool) { Version: uint64(BzzSpec.Version), NetworkID: b.NetworkID, Addr: b.localAddr, - Capabilities: b.localAddr.Capabilities, - //Capabilities: b.capabilities, + Capabilities: b.localAddr.capabilities, init: make(chan bool, 1), done: make(chan struct{}), } diff --git a/pss/forwarding_test.go b/pss/forwarding_test.go index f2a8861d92..9e9d0edbae 100644 --- a/pss/forwarding_test.go +++ b/pss/forwarding_test.go @@ -10,7 +10,6 @@ import ( "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethersphere/swarm/network" - "github.com/ethersphere/swarm/network/capability" "github.com/ethersphere/swarm/p2p/protocols" "github.com/ethersphere/swarm/pot" "github.com/ethersphere/swarm/pss/message" @@ -320,7 +319,7 @@ func testForwardMsg(t *testing.T, ps *Pss, c *testCase) { func addPeers(kad *network.Kademlia, addresses []pot.Address) { for _, a := range addresses { p := newTestDiscoveryPeer(a, kad) - p.BzzAddr.Capabilities = capability.NewCapabilities() + //p.Addr.SetCapabilities = capability.NewCapabilities() kad.On(p) } } @@ -341,10 +340,7 @@ func newTestDiscoveryPeer(addr pot.Address, kad *network.Kademlia) *network.Peer pp := protocols.NewPeer(p, rw, &protocols.Spec{}) bp := &network.BzzPeer{ Peer: pp, - BzzAddr: &network.BzzAddr{ - OAddr: addr.Bytes(), - UAddr: []byte(fmt.Sprintf("%x", addr[:])), - }, + BzzAddr: network.NewBzzAddr(addr.Bytes(), []byte(fmt.Sprintf("%x", addr[:]))), } return network.NewPeer(bp, kad) } diff --git a/pss/pss_test.go b/pss/pss_test.go index 7f653e4ad1..47f7107cd8 100644 --- a/pss/pss_test.go +++ b/pss/pss_test.go @@ -40,7 +40,6 @@ import ( "github.com/ethereum/go-ethereum/p2p/simulations/adapters" "github.com/ethereum/go-ethereum/rpc" "github.com/ethersphere/swarm/network" - "github.com/ethersphere/swarm/network/capability" "github.com/ethersphere/swarm/network/simulation" "github.com/ethersphere/swarm/p2p/protocols" "github.com/ethersphere/swarm/pot" @@ -193,13 +192,9 @@ func TestAddressMatchProx(t *testing.T) { peerAddr := pot.RandomAddressAt(localPotAddr, i) bzzPeer := &network.BzzPeer{ Peer: protoPeer, - BzzAddr: &network.BzzAddr{ - OAddr: peerAddr.Bytes(), - UAddr: []byte(fmt.Sprintf("%x", peerAddr[:])), - }, + BzzAddr: network.NewBzzAddr(peerAddr.Bytes(), []byte(fmt.Sprintf("%x", peerAddr[:]))), } peer := network.NewPeer(bzzPeer, kad) - peer.Capabilities = capability.NewCapabilities() kad.On(peer) peers = append(peers, peer) } @@ -604,7 +599,7 @@ func TestPeerCapabilityMismatch(t *testing.T) { nid := enode.ID{0x01} wrongpsspeer := network.NewPeer(&network.BzzPeer{ Peer: protocols.NewPeer(p2p.NewPeer(nid, common.ToHex(wrongpssaddr.Over()), []p2p.Cap{wrongpsscap}), rw, nil), - BzzAddr: &network.BzzAddr{OAddr: wrongpssaddr.Over(), UAddr: nil, Capabilities: capability.NewCapabilities()}, + BzzAddr: network.NewBzzAddr(wrongpssaddr.Over(), nil), }, kad) // one peer doesn't even have pss (boo!) @@ -616,7 +611,7 @@ func TestPeerCapabilityMismatch(t *testing.T) { nid = enode.ID{0x02} nopsspeer := network.NewPeer(&network.BzzPeer{ Peer: protocols.NewPeer(p2p.NewPeer(nid, common.ToHex(nopssaddr.Over()), []p2p.Cap{nopsscap}), rw, nil), - BzzAddr: &network.BzzAddr{OAddr: nopssaddr.Over(), UAddr: nil, Capabilities: capability.NewCapabilities()}, + BzzAddr: network.NewBzzAddr(nopssaddr.Over(),nil), }, kad) // add peers to kademlia and activate them From 59f84ae85578b3b3f51b8a533922c3e21d3c06b8 Mon Sep 17 00:00:00 2001 From: nolash Date: Tue, 3 Sep 2019 11:52:42 +0200 Subject: [PATCH 04/36] network: Add capabilities array to discovery messages tests --- network/discovery_test.go | 40 ++++++++++++++++++++------------------- network/enr.go | 5 +---- 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/network/discovery_test.go b/network/discovery_test.go index 431e1ebed7..04932c454d 100644 --- a/network/discovery_test.go +++ b/network/discovery_test.go @@ -18,7 +18,7 @@ package network import ( "crypto/ecdsa" - crand "crypto/rand" + //crand "crypto/rand" "encoding/binary" "fmt" "math/rand" @@ -26,12 +26,13 @@ import ( "sort" "testing" "time" + "bytes" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" - "github.com/ethersphere/swarm/network/capability" "github.com/ethersphere/swarm/p2p/protocols" + "github.com/ethersphere/swarm/network/capability" p2ptest "github.com/ethersphere/swarm/p2p/testing" "github.com/ethersphere/swarm/pot" ) @@ -115,21 +116,16 @@ func testInitialPeersMsg(t *testing.T, peerPO, peerDepth int) { connect := func(a pot.Address, po int) (addrs []*BzzAddr) { n := rand.Intn(maxPeersPerPO) for i := 0; i < n; i++ { - peer, err := newDiscPeer(pot.RandomAddressAt(a, po)) - if err != nil { - t.Fatal(err) - } + peer := newDiscPeer(pot.RandomAddressAt(a, po)) hive.On(peer) addrs = append(addrs, peer.BzzAddr) } return addrs } + //register := func(a pot.Address, po int) { register := func(a pot.Address, po int) { - addr := pot.RandomAddressAt(a, po) - bzzAddr := &BzzAddr{OAddr: addr[:]} - bzzAddr = bzzAddr.WithCapabilities(capability.NewCapabilities()) - hive.Register(bzzAddr) - // hive.Register(&BzzAddr{OAddr: addr[:]}) + discPeer := newDiscPeer(a) + hive.Register(discPeer.BzzAddr) } // generate connected and just registered peers @@ -182,6 +178,11 @@ func testInitialPeersMsg(t *testing.T, peerPO, peerDepth int) { // 1. pivot sends to the control peer a `subPeersMsg` advertising its depth (ignored) // 2. peer sends to pivot a `subPeersMsg` advertising its own depth (arbitrarily chosen) // 3. pivot responds with `peersMsg` with the set of expected peers + var cps []*capability.Capabilities + for _, p := range expBzzAddrs { + cps = append(cps, p.capabilities) + } + err = s.TestExchanges( p2ptest.Exchange{ Label: "outgoing subPeersMsg", @@ -205,7 +206,7 @@ func testInitialPeersMsg(t *testing.T, peerPO, peerDepth int) { Expects: []p2ptest.Expect{ { Code: 0, - Msg: &peersMsg{Peers: testSortPeers(expBzzAddrs)}, + Msg: &peersMsg{Peers: testSortPeers(expBzzAddrs), Capabilities: cps}, Peer: peerID, Timeout: 100 * time.Millisecond, }, @@ -242,20 +243,20 @@ func testSortPeers(peers []*BzzAddr) []*BzzAddr { // as we are not creating a real node via the protocol, // we need to create the discovery peer objects for the additional kademlia // nodes manually -func newDiscPeer(addr pot.Address) (*Peer, error) { - pKey, err := ecdsa.GenerateKey(crypto.S256(), crand.Reader) - if err != nil { - return nil, err - } +func newDiscPeer(addr pot.Address) *Peer { + //pKey, err := ecdsa.GenerateKey(crypto.S256(), crand.Reader) + zeros := [32]byte{} + addrSeed := append(addr.Bytes(), zeros[:]...) + pKey, _ := ecdsa.GenerateKey(crypto.S256(), bytes.NewBuffer(addrSeed)) pubKey := pKey.PublicKey nod := enode.NewV4(&pubKey, net.IPv4(127, 0, 0, 1), 0, 0) - bzzAddr := &BzzAddr{OAddr: addr[:], UAddr: []byte(nod.String()), capabilities: capability.NewCapabilities()} + bzzAddr := NewBzzAddr(addr[:], []byte(nod.String())) id := nod.ID() p2pPeer := p2p.NewPeer(id, id.String(), nil) return NewPeer(&BzzPeer{ Peer: protocols.NewPeer(p2pPeer, &dummyMsgRW{}, DiscoverySpec), BzzAddr: bzzAddr, - }, nil), nil + }, nil) } type dummyMsgRW struct{} @@ -266,3 +267,4 @@ func (d *dummyMsgRW) ReadMsg() (p2p.Msg, error) { func (d *dummyMsgRW) WriteMsg(msg p2p.Msg) error { return nil } + diff --git a/network/enr.go b/network/enr.go index 025af9d128..a49bfefabe 100644 --- a/network/enr.go +++ b/network/enr.go @@ -77,8 +77,5 @@ func getENRBzzAddr(nod *enode.Node) *BzzAddr { record := nod.Record() record.Load(&addr) - return &BzzAddr{ - OAddr: addr.data, - UAddr: []byte(nod.String()), - } + return NewBzzAddr(addr.data, []byte(nod.String())) } From 1142690ecdba2a28756d06a0908c0a24792df5cc Mon Sep 17 00:00:00 2001 From: nolash Date: Tue, 3 Sep 2019 13:41:48 +0200 Subject: [PATCH 05/36] network: Clean up code and enforce constructors for BzzAddr --- network/discovery.go | 8 -------- network/discovery_test.go | 10 +++++----- network/hive.go | 3 ++- network/hive_test.go | 2 +- network/kademlia.go | 3 --- network/kademlia_test.go | 32 ++------------------------------ network/network.go | 7 +------ network/networkid_test.go | 2 +- network/protocol.go | 2 +- network/protocol_test.go | 12 ++++++------ 10 files changed, 19 insertions(+), 62 deletions(-) diff --git a/network/discovery.go b/network/discovery.go index 97aea50035..4b469933eb 100644 --- a/network/discovery.go +++ b/network/discovery.go @@ -22,7 +22,6 @@ import ( "sync" "github.com/ethersphere/swarm/pot" - "github.com/ethersphere/swarm/log" "github.com/ethersphere/swarm/network/capability" ) @@ -98,9 +97,6 @@ func (d *Peer) NotifyPeer(a *BzzAddr, po uint8) { Peers: []*BzzAddr{a}, Capabilities: []*capability.Capabilities{a.capabilities}, } - if a.capabilities == nil { - log.Crit("tried to send capabilities empty", "msg", resp) - } go d.Send(context.TODO(), resp) } @@ -149,7 +145,6 @@ func (d *Peer) handlePeersMsg(msg *peersMsg) error { } for i, cp := range msg.Capabilities { msg.Peers[i].capabilities = cp - log.Warn("incoming notify peer", "peer", msg.Peers[i], "cap", cp) } for _, a := range msg.Peers { d.seen(a) @@ -195,9 +190,6 @@ func (d *Peer) handleSubPeersMsg(msg *subPeersMsg) error { if len(peers) > 0 { outMsg := &peersMsg{Peers: sortPeers(peers), Capabilities: []*capability.Capabilities{}} for _, p := range peers { - if p.capabilities == nil { - log.Crit("attempting to send handlesub with cap nil", outMsg) - } outMsg.Capabilities = append(outMsg.Capabilities, p.capabilities) } go d.Send(context.TODO(), outMsg) diff --git a/network/discovery_test.go b/network/discovery_test.go index 04932c454d..48f159ca43 100644 --- a/network/discovery_test.go +++ b/network/discovery_test.go @@ -50,7 +50,7 @@ func TestSubPeersMsg(t *testing.T) { } node := s.Nodes[0] - raddr := NewAddr(node) + raddr := NewBzzAddrFromEnode(node) pp.Register(raddr) // start the hive and wait for the connection @@ -122,7 +122,6 @@ func testInitialPeersMsg(t *testing.T, peerPO, peerDepth int) { } return addrs } - //register := func(a pot.Address, po int) { register := func(a pot.Address, po int) { discPeer := newDiscPeer(a) hive.Register(discPeer.BzzAddr) @@ -244,9 +243,10 @@ func testSortPeers(peers []*BzzAddr) []*BzzAddr { // we need to create the discovery peer objects for the additional kademlia // nodes manually func newDiscPeer(addr pot.Address) *Peer { - //pKey, err := ecdsa.GenerateKey(crypto.S256(), crand.Reader) - zeros := [32]byte{} - addrSeed := append(addr.Bytes(), zeros[:]...) + + // deterministically create enode id + // Input to the non-random input buffer is 2xaddress since it munches 256 bits + addrSeed := append(addr.Bytes(), addr.Bytes()...) pKey, _ := ecdsa.GenerateKey(crypto.S256(), bytes.NewBuffer(addrSeed)) pubKey := pKey.PublicKey nod := enode.NewV4(&pubKey, net.IPv4(127, 0, 0, 1), 0, 0) diff --git a/network/hive.go b/network/hive.go index 3134553be6..73fc00aefa 100644 --- a/network/hive.go +++ b/network/hive.go @@ -209,7 +209,8 @@ func (h *Hive) PeerInfo(id enode.ID) interface{} { if p == nil { return nil } - addr := NewAddr(p.Node()) + // TODO this is bogus, the overlay address will not be correct + addr := NewBzzAddrFromEnode(p.Node()) return struct { OAddr hexutil.Bytes UAddr hexutil.Bytes diff --git a/network/hive_test.go b/network/hive_test.go index c5461cca1d..7f01ffbf6e 100644 --- a/network/hive_test.go +++ b/network/hive_test.go @@ -56,7 +56,7 @@ func TestRegisterAndConnect(t *testing.T) { } node := s.Nodes[0] - raddr := NewAddr(node).WithCapabilities(capability.NewCapabilities()) + raddr := NewBzzAddrFromEnode(node) pp.Register(raddr) // start the hive diff --git a/network/kademlia.go b/network/kademlia.go index ffa343aab1..aa34f65dee 100644 --- a/network/kademlia.go +++ b/network/kademlia.go @@ -217,9 +217,6 @@ func (k *Kademlia) Register(peers ...*BzzAddr) error { var size int for _, p := range peers { - if p.capabilities == nil { - panic("missing caps for peer") - } log.Trace("kademlia trying to register", "addr", p) // error if self received, peer should know better // and should be punished for this diff --git a/network/kademlia_test.go b/network/kademlia_test.go index 72e0dd5463..0beb5c5f59 100644 --- a/network/kademlia_test.go +++ b/network/kademlia_test.go @@ -23,11 +23,9 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" - "github.com/ethersphere/swarm/network/capability" "github.com/ethersphere/swarm/p2p/protocols" "github.com/ethersphere/swarm/pot" ) @@ -39,7 +37,7 @@ func init() { func testKadPeerAddr(s string) *BzzAddr { a := pot.NewAddressFromString(s) - return &BzzAddr{OAddr: a, UAddr: a, capabilities: capability.NewCapabilities()} + return NewBzzAddr(a, a) } func newTestKademliaParams() *KadParams { @@ -556,11 +554,7 @@ func newTestDiscoveryPeer(addr pot.Address, kad *Kademlia) *Peer { pp := protocols.NewPeer(p, rw, &protocols.Spec{}) bp := &BzzPeer{ Peer: pp, - BzzAddr: &BzzAddr{ - OAddr: addr.Bytes(), - UAddr: []byte(fmt.Sprintf("%x", addr[:])), - capabilities: capability.NewCapabilities(), - }, + BzzAddr: NewBzzAddr(addr.Bytes(), []byte(fmt.Sprintf("%x", addr[:]))), } return NewPeer(bp, kad) } @@ -675,25 +669,3 @@ func TestKademlia_SubscribeToNeighbourhoodDepthChange(t *testing.T) { }) } -func NewAdaptivePeer(k *Kademlia, capabilities ...*capability.Capability) (*Peer, error) { - // create the peer that fits the kademlia record - // it's quite a bit of work - peerPrivKey, err := crypto.GenerateKey() - if err != nil { - return nil, err - } - peerEnodeId := enode.PubkeyToIDV4(&peerPrivKey.PublicKey) - peerP2p := p2p.NewPeer(peerEnodeId, "foo", []p2p.Cap{}) - peerProto := protocols.NewPeer(peerP2p, nil, nil) - peerBzz := NewBzzPeer(peerProto) - caps := capability.NewCapabilities() - for _, cap := range capabilities { - caps.Add(cap) - } - peerBzz.WithCapabilities(caps) - err = k.Register(peerBzz.BzzAddr) - if err != nil { - return nil, err - } - return NewPeer(peerBzz, k), nil -} diff --git a/network/network.go b/network/network.go index c876a4678e..dba304b378 100644 --- a/network/network.go +++ b/network/network.go @@ -71,12 +71,7 @@ func RandomAddr() *BzzAddr { panic("unable to generate key") } node := enode.NewV4(&key.PublicKey, net.IP{127, 0, 0, 1}, 30303, 30303) - return NewAddr(node) -} - -// NewAddr constructs a BzzAddr from a node record. -func NewAddr(enod *enode.Node) *BzzAddr { - return NewBzzAddrFromEnode(enod) + return NewBzzAddrFromEnode(node) } func NewBzzAddrFromEnode(enod *enode.Node) *BzzAddr { diff --git a/network/networkid_test.go b/network/networkid_test.go index 9634ab61de..95ace13f5d 100644 --- a/network/networkid_test.go +++ b/network/networkid_test.go @@ -199,7 +199,7 @@ func newServices() adapters.Services { } return adapters.Services{ "bzz": func(ctx *adapters.ServiceContext) (node.Service, error) { - addr := NewAddr(ctx.Config.Node()).WithCapabilities(capability.NewCapabilities()) + addr := NewBzzAddrFromEnode(ctx.Config.Node()).WithCapabilities(capability.NewCapabilities()) hp := NewHiveParams() hp.Discovery = false cnt++ diff --git a/network/protocol.go b/network/protocol.go index 9b84957870..f7d56693b8 100644 --- a/network/protocol.go +++ b/network/protocol.go @@ -326,7 +326,7 @@ type BzzPeer struct { } func NewBzzPeer(p *protocols.Peer) *BzzPeer { - return &BzzPeer{Peer: p, BzzAddr: NewAddr(p.Node())} + return &BzzPeer{Peer: p, BzzAddr: NewBzzAddrFromEnode(p.Node())} } // ID returns the peer's underlay node identifier. diff --git a/network/protocol_test.go b/network/protocol_test.go index 4f0ad05ad9..edd95c48ae 100644 --- a/network/protocol_test.go +++ b/network/protocol_test.go @@ -118,7 +118,7 @@ func newBzzBaseTesterWithAddrs(prvkey *ecdsa.PrivateKey, addrs [][]byte, spec *p mu.Lock() nodeToAddr[p.ID()] = addrs[0] mu.Unlock() - bzzAddr := &BzzAddr{addrs[0], []byte(p.Node().String()), capability.NewCapabilities()} + bzzAddr := NewBzzAddr(addrs[0], []byte(p.Node().String())) addrs = addrs[1:] return srv(&BzzPeer{Peer: protocols.NewPeer(p, rw, spec), BzzAddr: bzzAddr}) } @@ -248,7 +248,7 @@ func TestBzzHandshakeNetworkIDMismatch(t *testing.T) { err = s.testHandshake( correctBzzHandshake(s.addr, lightNode), - newBzzHandshakeMsg(TestProtocolVersion, 321, NewAddr(node), false), + newBzzHandshakeMsg(TestProtocolVersion, 321, NewBzzAddrFromEnode(node), false), &p2ptest.Disconnect{Peer: node.ID(), Error: fmt.Errorf("Handshake error: Message handler error: (msg code 0): network id mismatch 321 (!= %v)", TestProtocolNetworkID)}, ) @@ -272,7 +272,7 @@ func TestBzzHandshakeVersionMismatch(t *testing.T) { err = s.testHandshake( correctBzzHandshake(s.addr, lightNode), - newBzzHandshakeMsg(0, TestProtocolNetworkID, NewAddr(node), false), + newBzzHandshakeMsg(0, TestProtocolNetworkID, NewBzzAddrFromEnode(node), false), &p2ptest.Disconnect{Peer: node.ID(), Error: fmt.Errorf("Handshake error: Message handler error: (msg code 0): version mismatch 0 (!= %d)", TestProtocolVersion)}, ) @@ -295,7 +295,7 @@ func TestBzzHandshakeInvalidCapabilities(t *testing.T) { defer s.Stop() node := s.Nodes[0] - msg := newBzzHandshakeMsg(TestProtocolVersion, TestProtocolNetworkID, NewAddr(node), false) + msg := newBzzHandshakeMsg(TestProtocolVersion, TestProtocolNetworkID, NewBzzAddrFromEnode(node), false) cap := msg.Capabilities.Get(0) cap.Set(14) err = s.testHandshake( @@ -323,7 +323,7 @@ func TestBzzHandshakeSuccess(t *testing.T) { err = s.testHandshake( correctBzzHandshake(s.addr, lightNode), - newBzzHandshakeMsg(TestProtocolVersion, TestProtocolNetworkID, NewAddr(node), false), + newBzzHandshakeMsg(TestProtocolVersion, TestProtocolNetworkID, NewBzzAddrFromEnode(node), false), ) if err != nil { @@ -353,7 +353,7 @@ func TestBzzHandshakeLightNode(t *testing.T) { defer pt.Stop() node := pt.Nodes[0] - addr := NewAddr(node) + addr := NewBzzAddrFromEnode(node) err = pt.testHandshake( correctBzzHandshake(pt.addr, false), From f36d1bd6dd0aa1ff670e187a9d32e836a2f1bcb9 Mon Sep 17 00:00:00 2001 From: nolash Date: Tue, 3 Sep 2019 19:55:17 +0200 Subject: [PATCH 06/36] network: Gofmt --- network/discovery.go | 6 +++--- network/discovery_test.go | 7 +++---- network/kademlia_test.go | 5 ++--- network/network.go | 6 +++--- network/protocol.go | 10 +++++----- 5 files changed, 16 insertions(+), 18 deletions(-) diff --git a/network/discovery.go b/network/discovery.go index 4b469933eb..245def1dca 100644 --- a/network/discovery.go +++ b/network/discovery.go @@ -21,8 +21,8 @@ import ( "fmt" "sync" - "github.com/ethersphere/swarm/pot" "github.com/ethersphere/swarm/network/capability" + "github.com/ethersphere/swarm/pot" ) // discovery bzz extension for requesting and relaying node address records @@ -94,7 +94,7 @@ func (d *Peer) NotifyPeer(a *BzzAddr, po uint8) { return } resp := &peersMsg{ - Peers: []*BzzAddr{a}, + Peers: []*BzzAddr{a}, Capabilities: []*capability.Capabilities{a.capabilities}, } go d.Send(context.TODO(), resp) @@ -126,7 +126,7 @@ disconnected // used for communicating about known peers // relevant for bootstrapping connectivity and updating peersets type peersMsg struct { - Peers []*BzzAddr + Peers []*BzzAddr Capabilities []*capability.Capabilities } diff --git a/network/discovery_test.go b/network/discovery_test.go index 48f159ca43..7ff7e50c83 100644 --- a/network/discovery_test.go +++ b/network/discovery_test.go @@ -19,6 +19,7 @@ package network import ( "crypto/ecdsa" //crand "crypto/rand" + "bytes" "encoding/binary" "fmt" "math/rand" @@ -26,13 +27,12 @@ import ( "sort" "testing" "time" - "bytes" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" - "github.com/ethersphere/swarm/p2p/protocols" "github.com/ethersphere/swarm/network/capability" + "github.com/ethersphere/swarm/p2p/protocols" p2ptest "github.com/ethersphere/swarm/p2p/testing" "github.com/ethersphere/swarm/pot" ) @@ -181,7 +181,7 @@ func testInitialPeersMsg(t *testing.T, peerPO, peerDepth int) { for _, p := range expBzzAddrs { cps = append(cps, p.capabilities) } - + err = s.TestExchanges( p2ptest.Exchange{ Label: "outgoing subPeersMsg", @@ -267,4 +267,3 @@ func (d *dummyMsgRW) ReadMsg() (p2p.Msg, error) { func (d *dummyMsgRW) WriteMsg(msg p2p.Msg) error { return nil } - diff --git a/network/kademlia_test.go b/network/kademlia_test.go index 0beb5c5f59..de7cbcb722 100644 --- a/network/kademlia_test.go +++ b/network/kademlia_test.go @@ -553,8 +553,8 @@ func newTestDiscoveryPeer(addr pot.Address, kad *Kademlia) *Peer { p := p2p.NewPeer(enode.ID{}, "foo", []p2p.Cap{}) pp := protocols.NewPeer(p, rw, &protocols.Spec{}) bp := &BzzPeer{ - Peer: pp, - BzzAddr: NewBzzAddr(addr.Bytes(), []byte(fmt.Sprintf("%x", addr[:]))), + Peer: pp, + BzzAddr: NewBzzAddr(addr.Bytes(), []byte(fmt.Sprintf("%x", addr[:]))), } return NewPeer(bp, kad) } @@ -668,4 +668,3 @@ func TestKademlia_SubscribeToNeighbourhoodDepthChange(t *testing.T) { } }) } - diff --git a/network/network.go b/network/network.go index dba304b378..e9346918ef 100644 --- a/network/network.go +++ b/network/network.go @@ -20,9 +20,9 @@ type BzzAddr struct { func NewBzzAddr(oaddr []byte, uaddr []byte) *BzzAddr { return &BzzAddr{ - OAddr: oaddr, - UAddr: uaddr, - capabilities: capability.NewCapabilities(), + OAddr: oaddr, + UAddr: uaddr, + capabilities: capability.NewCapabilities(), } } diff --git a/network/protocol.go b/network/protocol.go index f7d56693b8..45d086e791 100644 --- a/network/protocol.go +++ b/network/protocol.go @@ -257,7 +257,7 @@ func (b *Bzz) RunProtocol(spec *protocols.Spec, run func(*BzzPeer) error) func(* if handshake.err != nil { return fmt.Errorf("%08x: %s protocol closed: %v", b.BaseAddr()[:4], spec.Name, handshake.err) } - + // the handshake has succeeded so construct the BzzPeer and run the protocol peer := &BzzPeer{ Peer: protocols.NewPeer(p, rw, spec), @@ -285,7 +285,7 @@ func (b *Bzz) performHandshake(p *protocols.Peer, handshake *HandshakeMsg) error handshake.err = err return err } - rsh.(*HandshakeMsg).Addr.capabilities = rsh.(*HandshakeMsg).Capabilities + rsh.(*HandshakeMsg).Addr.capabilities = rsh.(*HandshakeMsg).Capabilities handshake.peerAddr = rsh.(*HandshakeMsg).Addr handshake.Capabilities = rsh.(*HandshakeMsg).Capabilities return nil @@ -377,7 +377,7 @@ func (b *Bzz) checkHandshake(hs interface{}) error { if !isFullCapability(rhs.Capabilities.Get(0)) && !isLightCapability(rhs.Capabilities.Get(0)) { return fmt.Errorf("invalid capabilities setting: %s", rhs.Capabilities) } - rhs.Addr.capabilities = hs.(*HandshakeMsg).Capabilities + rhs.Addr.capabilities = hs.(*HandshakeMsg).Capabilities return nil } @@ -400,8 +400,8 @@ func (b *Bzz) GetOrCreateHandshake(peerID enode.ID) (*HandshakeMsg, bool) { NetworkID: b.NetworkID, Addr: b.localAddr, Capabilities: b.localAddr.capabilities, - init: make(chan bool, 1), - done: make(chan struct{}), + init: make(chan bool, 1), + done: make(chan struct{}), } // when handhsake is first created for a remote peer // it is initialised with the init From df7be168c14066d7199c6edfa5cbd4c64fb38863 Mon Sep 17 00:00:00 2001 From: nolash Date: Tue, 3 Sep 2019 20:04:45 +0200 Subject: [PATCH 07/36] network: Speling --- network/capability/capability.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/capability/capability.go b/network/capability/capability.go index 197cfe030b..6dd26ae19b 100644 --- a/network/capability/capability.go +++ b/network/capability/capability.go @@ -191,7 +191,7 @@ func (c *Capability) Match(capCompare *Capability) bool { if len(c.Cap) != len(capCompare.Cap) { return false } - // on the first occurence of false where query has true we can fail + // on the first occurrence of false where query has true we can fail for i, flag := range capCompare.Cap { if flag && !c.Cap[i] { return false From 213921cdef4842d71335bc81a36daf9eb3117d41 Mon Sep 17 00:00:00 2001 From: nolash Date: Wed, 4 Sep 2019 09:14:11 +0200 Subject: [PATCH 08/36] network, pss, cmd: Finish renaming NewAddr --- cmd/swarm-snapshot/create.go | 2 +- network/enr.go | 2 +- network/retrieval/retrieve_test.go | 2 +- network/simulation/example_test.go | 2 +- network/simulation/kademlia_test.go | 2 +- network/simulation/node_test.go | 2 +- network/simulation/simulation.go | 3 +-- network/simulations/discovery/discovery_test.go | 2 +- network/simulations/overlay.go | 2 +- pss/client/client_test.go | 2 +- pss/notify/notify_test.go | 2 +- pss/prox_test.go | 2 +- pss/pss_test.go | 6 +++--- 13 files changed, 15 insertions(+), 16 deletions(-) diff --git a/cmd/swarm-snapshot/create.go b/cmd/swarm-snapshot/create.go index 7d4cdcc0f8..db932b0cf8 100644 --- a/cmd/swarm-snapshot/create.go +++ b/cmd/swarm-snapshot/create.go @@ -60,7 +60,7 @@ func createSnapshot(filename string, nodes int, services []string) (err error) { sim := simulation.NewInProc(map[string]simulation.ServiceFunc{ "bzz": func(ctx *adapters.ServiceContext, bucket *sync.Map) (node.Service, func(), error) { - addr := network.NewAddr(ctx.Config.Node()) + addr := network.NewBzzAddrFromEnode(ctx.Config.Node()) kad := network.NewKademlia(addr.Over(), network.NewKadParams()) hp := network.NewHiveParams() hp.KeepAliveInterval = time.Duration(200) * time.Millisecond diff --git a/network/enr.go b/network/enr.go index a49bfefabe..432c2b1ca9 100644 --- a/network/enr.go +++ b/network/enr.go @@ -61,7 +61,7 @@ func getENRBzzPeer(p *p2p.Peer, rw p2p.MsgReadWriter, spec *protocols.Spec) *Bzz record := p.Node().Record() record.Load(&bootnode) - // get the address; separate function as long as we need swarm/network:NewAddr() to call it + // get the address; separate function as long as we need swarm/network:NewBzzAddrFromEnode() to call it addr := getENRBzzAddr(p.Node()) // build the peer using the retrieved data diff --git a/network/retrieval/retrieve_test.go b/network/retrieval/retrieve_test.go index 75379bedbf..0716d196eb 100644 --- a/network/retrieval/retrieve_test.go +++ b/network/retrieval/retrieve_test.go @@ -311,7 +311,7 @@ func TestHasPriceImplementation(t *testing.T) { func newBzzRetrieveWithLocalstore(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Service, cleanup func(), err error) { n := ctx.Config.Node() - addr := network.NewAddr(n) + addr := network.NewBzzAddrFromEnode(n) localStore, localStoreCleanup, err := newTestLocalStore(n.ID(), addr, nil) if err != nil { diff --git a/network/simulation/example_test.go b/network/simulation/example_test.go index f8d126552f..e97a20c505 100644 --- a/network/simulation/example_test.go +++ b/network/simulation/example_test.go @@ -36,7 +36,7 @@ func ExampleSimulation_WaitTillHealthy() { sim := simulation.NewInProc(map[string]simulation.ServiceFunc{ "bzz": func(ctx *adapters.ServiceContext, b *sync.Map) (node.Service, func(), error) { - addr := network.NewAddr(ctx.Config.Node()) + addr := network.NewBzzAddrFromEnode(ctx.Config.Node()) hp := network.NewHiveParams() hp.Discovery = false config := &network.BzzConfig{ diff --git a/network/simulation/kademlia_test.go b/network/simulation/kademlia_test.go index 73f8e71a8d..703a92174b 100644 --- a/network/simulation/kademlia_test.go +++ b/network/simulation/kademlia_test.go @@ -128,7 +128,7 @@ func TestWaitTillHealthy(t *testing.T) { func createSimServiceMap(discovery bool) map[string]ServiceFunc { return map[string]ServiceFunc{ "bzz": func(ctx *adapters.ServiceContext, b *sync.Map) (node.Service, func(), error) { - addr := network.NewAddr(ctx.Config.Node()) + addr := network.NewBzzAddrFromEnode(ctx.Config.Node()) hp := network.NewHiveParams() hp.Discovery = discovery config := &network.BzzConfig{ diff --git a/network/simulation/node_test.go b/network/simulation/node_test.go index 84699f1e62..b2edb2237c 100644 --- a/network/simulation/node_test.go +++ b/network/simulation/node_test.go @@ -280,7 +280,7 @@ func TestUploadSnapshot(t *testing.T) { log.Debug("Creating simulation") s := NewInProc(map[string]ServiceFunc{ "bzz": func(ctx *adapters.ServiceContext, b *sync.Map) (node.Service, func(), error) { - addr := network.NewAddr(ctx.Config.Node()) + addr := network.NewBzzAddrFromEnode(ctx.Config.Node()) hp := network.NewHiveParams() hp.Discovery = false config := &network.BzzConfig{ diff --git a/network/simulation/simulation.go b/network/simulation/simulation.go index 09ae64a231..5cc2df6f75 100644 --- a/network/simulation/simulation.go +++ b/network/simulation/simulation.go @@ -102,8 +102,7 @@ func NewInProc(services map[string]ServiceFunc) (s *Simulation) { // NewBzzInProc is the same as NewInProc but injects bzz as a default protocol func NewBzzInProc(services map[string]ServiceFunc) (s *Simulation) { services["bzz"] = func(ctx *adapters.ServiceContext, bucket *sync.Map) (node.Service, func(), error) { - addr := network.NewAddr(ctx.Config.Node()) - + addr := network.NewBzzAddrFromEnode(ctx.Config.Node()) hp := network.NewHiveParams() hp.KeepAliveInterval = time.Duration(200) * time.Millisecond hp.Discovery = false diff --git a/network/simulations/discovery/discovery_test.go b/network/simulations/discovery/discovery_test.go index 1a8f90b163..5ce46ad286 100644 --- a/network/simulations/discovery/discovery_test.go +++ b/network/simulations/discovery/discovery_test.go @@ -491,7 +491,7 @@ func triggerChecks(trigger chan enode.ID, net *simulations.Network, id enode.ID) } func newService(ctx *adapters.ServiceContext) (node.Service, error) { - addr := network.NewAddr(ctx.Config.Node()) + addr := network.NewBzzAddrFromEnode(ctx.Config.Node()) kp := network.NewKadParams() kp.NeighbourhoodSize = testNeighbourhoodSize diff --git a/network/simulations/overlay.go b/network/simulations/overlay.go index cae543d6ad..c353a79968 100644 --- a/network/simulations/overlay.go +++ b/network/simulations/overlay.go @@ -65,7 +65,7 @@ func (s *Simulation) NewService(ctx *adapters.ServiceContext) (node.Service, err } s.mtx.Unlock() - addr := network.NewAddr(node) + addr := network.NewBzzAddrFromEnode(node) kp := network.NewKadParams() kp.NeighbourhoodSize = 2 diff --git a/pss/client/client_test.go b/pss/client/client_test.go index 62cf51b604..c9e5428a52 100644 --- a/pss/client/client_test.go +++ b/pss/client/client_test.go @@ -248,7 +248,7 @@ func newServices() adapters.Services { return ps, nil }, "bzz": func(ctx *adapters.ServiceContext) (node.Service, error) { - addr := network.NewAddr(ctx.Config.Node()) + addr := network.NewBzzAddrFromEnode(ctx.Config.Node()) hp := network.NewHiveParams() hp.Discovery = false config := &network.BzzConfig{ diff --git a/pss/notify/notify_test.go b/pss/notify/notify_test.go index 25b7f4113a..e0b6f992db 100644 --- a/pss/notify/notify_test.go +++ b/pss/notify/notify_test.go @@ -237,7 +237,7 @@ func newServices(allowRaw bool) adapters.Services { return ps, nil }, "bzz": func(ctx *adapters.ServiceContext) (node.Service, error) { - addr := network.NewAddr(ctx.Config.Node()) + addr := network.NewBzzAddrFromEnode(ctx.Config.Node()) hp := network.NewHiveParams() hp.Discovery = false config := &network.BzzConfig{ diff --git a/pss/prox_test.go b/pss/prox_test.go index 1d49f2df1e..f427559a8e 100644 --- a/pss/prox_test.go +++ b/pss/prox_test.go @@ -405,7 +405,7 @@ func newProxServices(td *testData, allowRaw bool, handlerContextFuncs map[messag // however, we need to keep track of it in the test driver as well. // if the translation in the network package changes, that can cause these tests to unpredictably fail // therefore we keep a local copy of the translation here - addr := network.NewAddr(ctx.Config.Node()) + addr := network.NewBzzAddrFromEnode(ctx.Config.Node()) bzzPrivateKey, err = simulation.BzzPrivateKeyFromConfig(ctx.Config) if err != nil { return nil, nil, err diff --git a/pss/pss_test.go b/pss/pss_test.go index 47f7107cd8..55f0c0dfe1 100644 --- a/pss/pss_test.go +++ b/pss/pss_test.go @@ -191,7 +191,7 @@ func TestAddressMatchProx(t *testing.T) { protoPeer := protocols.NewPeer(ptpPeer, rw, &protocols.Spec{}) peerAddr := pot.RandomAddressAt(localPotAddr, i) bzzPeer := &network.BzzPeer{ - Peer: protoPeer, + Peer: protoPeer, BzzAddr: network.NewBzzAddr(peerAddr.Bytes(), []byte(fmt.Sprintf("%x", peerAddr[:]))), } peer := network.NewPeer(bzzPeer, kad) @@ -611,7 +611,7 @@ func TestPeerCapabilityMismatch(t *testing.T) { nid = enode.ID{0x02} nopsspeer := network.NewPeer(&network.BzzPeer{ Peer: protocols.NewPeer(p2p.NewPeer(nid, common.ToHex(nopssaddr.Over()), []p2p.Cap{nopsscap}), rw, nil), - BzzAddr: network.NewBzzAddr(nopssaddr.Over(),nil), + BzzAddr: network.NewBzzAddr(nopssaddr.Over(), nil), }, kad) // add peers to kademlia and activate them @@ -1625,7 +1625,7 @@ func newServices(allowRaw bool) map[string]simulation.ServiceFunc { } return map[string]simulation.ServiceFunc{ "bzz": func(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Service, cleanup func(), err error) { - addr := network.NewAddr(ctx.Config.Node()) + addr := network.NewBzzAddrFromEnode(ctx.Config.Node()) bzzPrivateKey, err := simulation.BzzPrivateKeyFromConfig(ctx.Config) if err != nil { return nil, nil, err From 38d170c72a279fb082c8dceb706f2fb6730468d5 Mon Sep 17 00:00:00 2001 From: nolash Date: Wed, 4 Sep 2019 09:29:47 +0200 Subject: [PATCH 09/36] network, pot, pss: Rename RandomAddr to explicit RandomBzzAddr --- network/discovery_test.go | 4 ++-- network/hive_test.go | 2 +- network/kademlia_test.go | 6 ++--- network/network.go | 4 ++-- network/protocol_test.go | 2 +- network/retrieval/retrieve_test.go | 8 +++---- pot/address.go | 10 ++++---- pot/pot_test.go | 4 ++-- pss/client/doc.go | 2 +- pss/forwarding_test.go | 18 +++++++------- pss/prox_test.go | 2 +- pss/pss_test.go | 38 +++++++++++++++--------------- 12 files changed, 50 insertions(+), 50 deletions(-) diff --git a/network/discovery_test.go b/network/discovery_test.go index 7ff7e50c83..8a9876c442 100644 --- a/network/discovery_test.go +++ b/network/discovery_test.go @@ -106,7 +106,7 @@ func testInitialPeersMsg(t *testing.T, peerPO, peerDepth int) { sortPeers = testSortPeers pivotAddr := pot.NewAddressFromBytes(PrivateKeyToBzzKey(prvkey)) // generate control peers address at peerPO wrt pivot - peerAddr := pot.RandomAddressAt(pivotAddr, peerPO) + peerAddr := pot.RandomBzzAddressAt(pivotAddr, peerPO) // construct kademlia and hive to := NewKademlia(pivotAddr[:], NewKadParams()) hive := NewHive(NewHiveParams(), to, nil) @@ -116,7 +116,7 @@ func testInitialPeersMsg(t *testing.T, peerPO, peerDepth int) { connect := func(a pot.Address, po int) (addrs []*BzzAddr) { n := rand.Intn(maxPeersPerPO) for i := 0; i < n; i++ { - peer := newDiscPeer(pot.RandomAddressAt(a, po)) + peer := newDiscPeer(pot.RandomBzzAddressAt(a, po)) hive.On(peer) addrs = append(addrs, peer.BzzAddr) } diff --git a/network/hive_test.go b/network/hive_test.go index 7f01ffbf6e..6ebfb64903 100644 --- a/network/hive_test.go +++ b/network/hive_test.go @@ -154,7 +154,7 @@ func TestHiveStatePersistance(t *testing.T) { h1, cleanup1 := startHive(t, dir) peers := make(map[string]bool) for i := 0; i < peersCount; i++ { - raddr := RandomAddr().WithCapabilities(capability.NewCapabilities()) + raddr := RandomBzzAddr().WithCapabilities(capability.NewCapabilities()) h1.Register(raddr) peers[raddr.String()] = true } diff --git a/network/kademlia_test.go b/network/kademlia_test.go index de7cbcb722..3489bacd22 100644 --- a/network/kademlia_test.go +++ b/network/kademlia_test.go @@ -95,7 +95,7 @@ func (tk *testKademlia) Register(regs ...string) { // // TODO: Make test adapt to change in NeighbourhoodSize func TestNeighbourhoodDepth(t *testing.T) { - baseAddressBytes := RandomAddr().OAddr + baseAddressBytes := RandomBzzAddr().OAddr kad := NewKademlia(baseAddressBytes, NewKadParams()) baseAddress := pot.NewAddressFromBytes(baseAddressBytes) @@ -103,12 +103,12 @@ func TestNeighbourhoodDepth(t *testing.T) { // generate the peers var peers []*Peer for i := 0; i < 7; i++ { - addr := pot.RandomAddressAt(baseAddress, i) + addr := pot.RandomBzzAddressAt(baseAddress, i) peers = append(peers, newTestDiscoveryPeer(addr, kad)) } var sevenPeers []*Peer for i := 0; i < 2; i++ { - addr := pot.RandomAddressAt(baseAddress, 7) + addr := pot.RandomBzzAddressAt(baseAddress, 7) sevenPeers = append(sevenPeers, newTestDiscoveryPeer(addr, kad)) } diff --git a/network/network.go b/network/network.go index e9346918ef..5732c62010 100644 --- a/network/network.go +++ b/network/network.go @@ -64,8 +64,8 @@ func (a *BzzAddr) String() string { return fmt.Sprintf("%x <%s> cap:%s", a.OAddr, a.UAddr, a.capabilities) } -// RandomAddr is a utility method generating an address from a public key -func RandomAddr() *BzzAddr { +// RandomBzzAddr is a utility method generating an address from a public key +func RandomBzzAddr() *BzzAddr { key, err := crypto.GenerateKey() if err != nil { panic("unable to generate key") diff --git a/network/protocol_test.go b/network/protocol_test.go index edd95c48ae..8a8eb6e26e 100644 --- a/network/protocol_test.go +++ b/network/protocol_test.go @@ -90,7 +90,7 @@ func newBzzHandshakeMsg(version uint64, networkId uint64, addr *BzzAddr, lightNo func newBzzBaseTester(n int, prvkey *ecdsa.PrivateKey, spec *protocols.Spec, run func(*BzzPeer) error) (*bzzTester, error) { var addrs [][]byte for i := 0; i < n; i++ { - addr := pot.RandomAddress() + addr := pot.RandomBzzAddress() addrs = append(addrs, addr[:]) } pt, _, err := newBzzBaseTesterWithAddrs(prvkey, addrs, spec, run) diff --git a/network/retrieval/retrieve_test.go b/network/retrieval/retrieve_test.go index 0716d196eb..225e709a90 100644 --- a/network/retrieval/retrieve_test.go +++ b/network/retrieval/retrieve_test.go @@ -231,11 +231,11 @@ func setupTestDeliveryForwardingSimulation(t *testing.T) (sim *simulation.Simula func TestRequestFromPeers(t *testing.T) { dummyPeerID := enode.HexID("3431c3939e1ee2a6345e976a8234f9870152d64879f30bc272a074f6859e75e8") - addr := network.RandomAddr() + addr := network.RandomBzzAddr() to := network.NewKademlia(addr.OAddr, network.NewKadParams()) protocolsPeer := protocols.NewPeer(p2p.NewPeer(dummyPeerID, "dummy", []p2p.Cap{{Name: "bzz-retrieve", Version: 1}}), nil, nil) peer := network.NewPeer(&network.BzzPeer{ - BzzAddr: network.RandomAddr(), + BzzAddr: network.RandomBzzAddr(), LightNode: false, Peer: protocolsPeer, }, to) @@ -259,14 +259,14 @@ func TestRequestFromPeers(t *testing.T) { func TestRequestFromPeersWithLightNode(t *testing.T) { dummyPeerID := enode.HexID("3431c3939e1ee2a6345e976a8234f9870152d64879f30bc272a074f6859e75e8") - addr := network.RandomAddr() + addr := network.RandomBzzAddr() to := network.NewKademlia(addr.OAddr, network.NewKadParams()) protocolsPeer := protocols.NewPeer(p2p.NewPeer(dummyPeerID, "dummy", []p2p.Cap{{Name: "bzz-retrieve", Version: 1}}), nil, nil) // setting up a lightnode peer := network.NewPeer(&network.BzzPeer{ - BzzAddr: network.RandomAddr(), + BzzAddr: network.RandomBzzAddr(), LightNode: true, Peer: protocolsPeer, }, to) diff --git a/pot/address.go b/pot/address.go index d7bed6c46e..00e21eec71 100644 --- a/pot/address.go +++ b/pot/address.go @@ -134,10 +134,10 @@ func ProxCmp(a, x, y []byte) int { return 0 } -// RandomAddressAt (address, prox) generates a random address +// RandomBzzAddressAt (address, prox) generates a random address // at proximity order prox relative to address // if prox is negative a random address is generated -func RandomAddressAt(self Address, prox int) (addr Address) { +func RandomBzzAddressAt(self Address, prox int) (addr Address) { addr = self pos := -1 if prox >= 0 { @@ -159,9 +159,9 @@ func RandomAddressAt(self Address, prox int) (addr Address) { return } -// RandomAddress generates a random address -func RandomAddress() Address { - return RandomAddressAt(Address{}, -1) +// RandomBzzAddress generates a random address +func RandomBzzAddress() Address { + return RandomBzzAddressAt(Address{}, -1) } // NewAddressFromString creates a byte slice from a string in binary representation diff --git a/pot/pot_test.go b/pot/pot_test.go index 6189c7607a..39c6e0dc6a 100644 --- a/pot/pot_test.go +++ b/pot/pot_test.go @@ -56,12 +56,12 @@ func (a *testAddr) String() string { } func randomTestAddr(n int, i int) *testAddr { - v := RandomAddress().Bin()[:n] + v := RandomBzzAddress().Bin()[:n] return newTestAddr(v, i) } func randomtestAddr(n int, i int) *testAddr { - v := RandomAddress().Bin()[:n] + v := RandomBzzAddress().Bin()[:n] return newTestAddr(v, i) } diff --git a/pss/client/doc.go b/pss/client/doc.go index 8dd7ca7551..62ee6d275d 100644 --- a/pss/client/doc.go +++ b/pss/client/doc.go @@ -84,7 +84,7 @@ // os.Exit(1) // } // -// addr := pot.RandomAddress() // should be a real address, of course +// addr := pot.RandomBzzAddress() // should be a real address, of course // psc.AddPssPeer(addr, spec) // // // use the protocol for something diff --git a/pss/forwarding_test.go b/pss/forwarding_test.go index 9e9d0edbae..c7316e002b 100644 --- a/pss/forwarding_test.go +++ b/pss/forwarding_test.go @@ -41,15 +41,15 @@ func TestForwardBasic(t *testing.T) { const depth = 10 for i := 0; i <= depth; i++ { // add two peers for each proximity order - a := pot.RandomAddressAt(base, i) + a := pot.RandomBzzAddressAt(base, i) peerAddresses = append(peerAddresses, a) - a = pot.RandomAddressAt(base, i) + a = pot.RandomBzzAddressAt(base, i) peerAddresses = append(peerAddresses, a) } // skip one level, add one peer at one level deeper. // as a result, we will have an edge case of three peers in nearest neighbours' bin. - peerAddresses = append(peerAddresses, pot.RandomAddressAt(base, depth+2)) + peerAddresses = append(peerAddresses, pot.RandomBzzAddressAt(base, depth+2)) kad := network.NewKademlia(base[:], network.NewKadParams()) ps := createPss(t, kad) @@ -78,7 +78,7 @@ func TestForwardBasic(t *testing.T) { for i := 0; i < firstNearest; i++ { // send random messages with proximity orders, corresponding to PO of each bin, // with one peer being closer to the recipient address - a := pot.RandomAddressAt(peerAddresses[i], 64) + a := pot.RandomBzzAddressAt(peerAddresses[i], 64) c = testCase{ name: fmt.Sprintf("Send random to each PO, id: [%d]", i), recipient: a[:], @@ -93,7 +93,7 @@ func TestForwardBasic(t *testing.T) { // send random messages with proximity orders, corresponding to PO of each bin, // with random proximity relative to the recipient address po := i / 2 - a := pot.RandomAddressAt(base, po) + a := pot.RandomBzzAddressAt(base, po) c = testCase{ name: fmt.Sprintf("Send direct to known, id: [%d]", i), recipient: a[:], @@ -106,7 +106,7 @@ func TestForwardBasic(t *testing.T) { for i := firstNearest; i < len(peerAddresses); i++ { // recipient address falls into the nearest neighbours' bin - a := pot.RandomAddressAt(base, i) + a := pot.RandomBzzAddressAt(base, i) c = testCase{ name: fmt.Sprintf("recipient address falls into the nearest neighbours' bin, id: [%d]", i), recipient: a[:], @@ -118,7 +118,7 @@ func TestForwardBasic(t *testing.T) { } // send msg with proximity order much deeper than the deepest nearest neighbour - a2 := pot.RandomAddressAt(base, 77) + a2 := pot.RandomBzzAddressAt(base, 77) c = testCase{ name: "proximity order much deeper than the deepest nearest neighbour", recipient: a2[:], @@ -167,7 +167,7 @@ func TestForwardBasic(t *testing.T) { } // partial address with proximity order deeper than any of the nearest neighbour - a3 := pot.RandomAddressAt(base, part) + a3 := pot.RandomBzzAddressAt(base, part) c = testCase{ name: "partial address with proximity order deeper than any of the nearest neighbour", recipient: a3[:part], @@ -218,7 +218,7 @@ func TestForwardBasic(t *testing.T) { // send random messages with proximity orders, corresponding to PO of each bin, // with different numbers of failed attempts. // msg should be received by only one of the deeper peers. - a := pot.RandomAddressAt(base, po) + a := pot.RandomBzzAddressAt(base, po) c = testCase{ name: fmt.Sprintf("Send direct to known, id: [%d]", i), recipient: a[:], diff --git a/pss/prox_test.go b/pss/prox_test.go index f427559a8e..19421b16f4 100644 --- a/pss/prox_test.go +++ b/pss/prox_test.go @@ -136,7 +136,7 @@ func (td *testData) init(msgCount int) error { } for i := 0; i < msgCount; i++ { - msgAddr := pot.RandomAddress() // we choose message addresses randomly + msgAddr := pot.RandomBzzAddress() // we choose message addresses randomly td.recipientAddresses = append(td.recipientAddresses, msgAddr.Bytes()) smallestPo := 256 var target enode.ID diff --git a/pss/pss_test.go b/pss/pss_test.go index 55f0c0dfe1..f84df04039 100644 --- a/pss/pss_test.go +++ b/pss/pss_test.go @@ -108,7 +108,7 @@ func TestAPITopic(t *testing.T) { // matching of address hints; whether a message could be or is for the node func TestAddressMatch(t *testing.T) { - localaddr := network.RandomAddr().Over() + localaddr := network.RandomBzzAddr().Over() copy(localaddr[:8], []byte("deadbeef")) remoteaddr := []byte("feedbeef") kadparams := network.NewKadParams() @@ -160,7 +160,7 @@ func TestAddressMatch(t *testing.T) { func TestAddressMatchProx(t *testing.T) { // recipient node address - localAddr := network.RandomAddr().Over() + localAddr := network.RandomBzzAddr().Over() localPotAddr := pot.NewAddressFromBytes(localAddr) // set up kademlia @@ -189,7 +189,7 @@ func TestAddressMatchProx(t *testing.T) { rw := &p2p.MsgPipeRW{} ptpPeer := p2p.NewPeer(enode.ID{}, "362436 call me anytime", []p2p.Cap{}) protoPeer := protocols.NewPeer(ptpPeer, rw, &protocols.Spec{}) - peerAddr := pot.RandomAddressAt(localPotAddr, i) + peerAddr := pot.RandomBzzAddressAt(localPotAddr, i) bzzPeer := &network.BzzPeer{ Peer: protoPeer, BzzAddr: network.NewBzzAddr(peerAddr.Bytes(), []byte(fmt.Sprintf("%x", peerAddr[:]))), @@ -270,7 +270,7 @@ func TestAddressMatchProx(t *testing.T) { // test the distances var prevReceive int for i, distance := range remoteDistances { - remotePotAddr := pot.RandomAddressAt(localPotAddr, distance) + remotePotAddr := pot.RandomBzzAddressAt(localPotAddr, distance) remoteAddr := remotePotAddr.Bytes() var data [32]byte @@ -299,7 +299,7 @@ func TestAddressMatchProx(t *testing.T) { receives = 0 prevReceive = 0 for i, distance := range remoteDistances { - remotePotAddr := pot.RandomAddressAt(localPotAddr, distance) + remotePotAddr := pot.RandomBzzAddressAt(localPotAddr, distance) remoteAddr := remotePotAddr.Bytes() var data [32]byte @@ -323,7 +323,7 @@ func TestAddressMatchProx(t *testing.T) { receives = 0 for _, distance := range remoteDistances { - remotePotAddr := pot.RandomAddressAt(localPotAddr, distance) + remotePotAddr := pot.RandomBzzAddressAt(localPotAddr, distance) remoteAddr := remotePotAddr.Bytes() pssMsg := message.New(message.Flags{Raw: true}) @@ -478,8 +478,8 @@ func TestKeys(t *testing.T) { // set up peer with mock address, mapped to mocked publicaddress and with mocked symkey addr := make(PssAddress, 32) - copy(addr, network.RandomAddr().Over()) - outkey := network.RandomAddr().Over() + copy(addr, network.RandomBzzAddr().Over()) + outkey := network.RandomBzzAddr().Over() topicobj := message.NewTopic([]byte("foo:42")) ps.SetPeerPublicKey(&theirprivkey.PublicKey, topicobj, addr) outkeyid, err := ps.SetSymmetricKey(outkey, topicobj, addr, false) @@ -526,7 +526,7 @@ func TestGetPublickeyEntries(t *testing.T) { ps := newTestPss(privkey, nil, nil) defer ps.Stop() - peeraddr := network.RandomAddr().Over() + peeraddr := network.RandomBzzAddr().Over() topicaddr := make(map[message.Topic]PssAddress) topicaddr[message.Topic{0x13}] = peeraddr topicaddr[message.Topic{0x2a}] = peeraddr[:16] @@ -586,12 +586,12 @@ func TestPeerCapabilityMismatch(t *testing.T) { } // initialize kad - baseaddr := network.RandomAddr() + baseaddr := network.RandomBzzAddr() kad := network.NewKademlia((baseaddr).Over(), network.NewKadParams()) rw := &p2p.MsgPipeRW{} // one peer has a mismatching version of pss - wrongpssaddr := network.RandomAddr() + wrongpssaddr := network.RandomBzzAddr() wrongpsscap := p2p.Cap{ Name: protocolName, Version: 0, @@ -603,7 +603,7 @@ func TestPeerCapabilityMismatch(t *testing.T) { }, kad) // one peer doesn't even have pss (boo!) - nopssaddr := network.RandomAddr() + nopssaddr := network.RandomBzzAddr() nopsscap := p2p.Cap{ Name: "nopss", Version: 1, @@ -644,7 +644,7 @@ func TestRawAllow(t *testing.T) { if err != nil { t.Fatal(err) } - baseAddr := network.RandomAddr() + baseAddr := network.RandomBzzAddr() kad := network.NewKademlia((baseAddr).Over(), network.NewKadParams()) ps := newTestPss(privKey, kad, nil) defer ps.Stop() @@ -889,8 +889,8 @@ func testSendSym(t *testing.T) { log.Trace("rsub", "id", rsub) defer rsub.Unsubscribe() - lrecvkey := network.RandomAddr().Over() - rrecvkey := network.RandomAddr().Over() + lrecvkey := network.RandomBzzAddr().Over() + rrecvkey := network.RandomBzzAddr().Over() var lkeyids [2]string var rkeyids [2]string @@ -1370,7 +1370,7 @@ func benchmarkSymKeySend(b *testing.B) { rand.Read(msg) topic := message.NewTopic([]byte("foo")) to := make(PssAddress, 32) - copy(to[:], network.RandomAddr().Over()) + copy(to[:], network.RandomBzzAddr().Over()) symkeyid, err := ps.GenerateSymmetricKey(topic, to, true) if err != nil { b.Fatalf("could not generate symkey: %v", err) @@ -1412,7 +1412,7 @@ func benchmarkAsymKeySend(b *testing.B) { rand.Read(msg) topic := message.NewTopic([]byte("foo")) to := make(PssAddress, 32) - copy(to[:], network.RandomAddr().Over()) + copy(to[:], network.RandomBzzAddr().Over()) ps.SetPeerPublicKey(&privkey.PublicKey, topic, to) b.ResetTimer() for i := 0; i < b.N; i++ { @@ -1460,7 +1460,7 @@ func benchmarkSymkeyBruteforceChangeaddr(b *testing.B) { topic := message.NewTopic([]byte("foo")) for i := 0; i < int(keycount); i++ { to := make(PssAddress, 32) - copy(to[:], network.RandomAddr().Over()) + copy(to[:], network.RandomBzzAddr().Over()) keyid, err = ps.GenerateSymmetricKey(topic, to, true) if err != nil { b.Fatalf("cant generate symkey #%d: %v", i, err) @@ -1532,7 +1532,7 @@ func benchmarkSymkeyBruteforceSameaddr(b *testing.B) { defer ps.Stop() topic := message.NewTopic([]byte("foo")) for i := 0; i < int(keycount); i++ { - copy(addr[i], network.RandomAddr().Over()) + copy(addr[i], network.RandomBzzAddr().Over()) keyid, err = ps.GenerateSymmetricKey(topic, addr[i], true) if err != nil { b.Fatalf("cant generate symkey #%d: %v", i, err) From 75e70913162de87bcb838e7a728f774fbb8a1428 Mon Sep 17 00:00:00 2001 From: nolash Date: Wed, 4 Sep 2019 10:24:36 +0200 Subject: [PATCH 10/36] pss: Gofmt --- pss/forwarding_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pss/forwarding_test.go b/pss/forwarding_test.go index c7316e002b..738f7f4969 100644 --- a/pss/forwarding_test.go +++ b/pss/forwarding_test.go @@ -339,7 +339,7 @@ func newTestDiscoveryPeer(addr pot.Address, kad *network.Kademlia) *network.Peer p := p2p.NewPeer(enode.ID{}, "test", []p2p.Cap{}) pp := protocols.NewPeer(p, rw, &protocols.Spec{}) bp := &network.BzzPeer{ - Peer: pp, + Peer: pp, BzzAddr: network.NewBzzAddr(addr.Bytes(), []byte(fmt.Sprintf("%x", addr[:]))), } return network.NewPeer(bp, kad) From 7ef95032be31ac9743890406e62522b70d58ae56 Mon Sep 17 00:00:00 2001 From: nolash Date: Sun, 8 Sep 2019 14:40:37 +0200 Subject: [PATCH 11/36] network: Custom deserialize in BzzAddr for Capabilities --- network/discovery.go | 15 +++------------ network/discovery_test.go | 8 +------- network/kademlia.go | 2 +- network/network.go | 37 +++++++++++++++++++++++++++---------- network/protocol.go | 38 +++++++++++++++++++------------------- network/protocol_test.go | 14 +++++++------- 6 files changed, 58 insertions(+), 56 deletions(-) diff --git a/network/discovery.go b/network/discovery.go index 245def1dca..9e60c68fa0 100644 --- a/network/discovery.go +++ b/network/discovery.go @@ -21,7 +21,6 @@ import ( "fmt" "sync" - "github.com/ethersphere/swarm/network/capability" "github.com/ethersphere/swarm/pot" ) @@ -94,8 +93,7 @@ func (d *Peer) NotifyPeer(a *BzzAddr, po uint8) { return } resp := &peersMsg{ - Peers: []*BzzAddr{a}, - Capabilities: []*capability.Capabilities{a.capabilities}, + Peers: []*BzzAddr{a}, } go d.Send(context.TODO(), resp) } @@ -126,8 +124,7 @@ disconnected // used for communicating about known peers // relevant for bootstrapping connectivity and updating peersets type peersMsg struct { - Peers []*BzzAddr - Capabilities []*capability.Capabilities + Peers []*BzzAddr } // String pretty prints a peersMsg @@ -143,9 +140,6 @@ func (d *Peer) handlePeersMsg(msg *peersMsg) error { if len(msg.Peers) == 0 { return nil } - for i, cp := range msg.Capabilities { - msg.Peers[i].capabilities = cp - } for _, a := range msg.Peers { d.seen(a) NotifyPeer(a, d.kad) @@ -188,10 +182,7 @@ func (d *Peer) handleSubPeersMsg(msg *subPeersMsg) error { }) // if useful peers are found, send them over if len(peers) > 0 { - outMsg := &peersMsg{Peers: sortPeers(peers), Capabilities: []*capability.Capabilities{}} - for _, p := range peers { - outMsg.Capabilities = append(outMsg.Capabilities, p.capabilities) - } + outMsg := &peersMsg{Peers: sortPeers(peers)} go d.Send(context.TODO(), outMsg) } } diff --git a/network/discovery_test.go b/network/discovery_test.go index 8a9876c442..fcd975fb10 100644 --- a/network/discovery_test.go +++ b/network/discovery_test.go @@ -31,7 +31,6 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" - "github.com/ethersphere/swarm/network/capability" "github.com/ethersphere/swarm/p2p/protocols" p2ptest "github.com/ethersphere/swarm/p2p/testing" "github.com/ethersphere/swarm/pot" @@ -177,11 +176,6 @@ func testInitialPeersMsg(t *testing.T, peerPO, peerDepth int) { // 1. pivot sends to the control peer a `subPeersMsg` advertising its depth (ignored) // 2. peer sends to pivot a `subPeersMsg` advertising its own depth (arbitrarily chosen) // 3. pivot responds with `peersMsg` with the set of expected peers - var cps []*capability.Capabilities - for _, p := range expBzzAddrs { - cps = append(cps, p.capabilities) - } - err = s.TestExchanges( p2ptest.Exchange{ Label: "outgoing subPeersMsg", @@ -205,7 +199,7 @@ func testInitialPeersMsg(t *testing.T, peerPO, peerDepth int) { Expects: []p2ptest.Expect{ { Code: 0, - Msg: &peersMsg{Peers: testSortPeers(expBzzAddrs), Capabilities: cps}, + Msg: &peersMsg{Peers: testSortPeers(expBzzAddrs)}, Peer: peerID, Timeout: 100 * time.Millisecond, }, diff --git a/network/kademlia.go b/network/kademlia.go index aa34f65dee..3fdc92fca3 100644 --- a/network/kademlia.go +++ b/network/kademlia.go @@ -148,7 +148,7 @@ func (k *Kademlia) addToCapabilityIndex(p interface{}, connected bool) { eAddr = p.(*entry).BzzAddr } for s, idxItem := range k.capabilityIndex { - for _, vCap := range eAddr.capabilities.Caps { + for _, vCap := range eAddr.Capabilities.Caps { log.Debug("in add to capability compare", "idx", s, "peer", vCap, "idx", idxItem.Capability) if idxItem.Id != vCap.Id { continue diff --git a/network/network.go b/network/network.go index 5732c62010..70feafff87 100644 --- a/network/network.go +++ b/network/network.go @@ -8,6 +8,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/enr" + "github.com/ethereum/go-ethereum/rlp" "github.com/ethersphere/swarm/network/capability" ) @@ -15,14 +16,34 @@ import ( type BzzAddr struct { OAddr []byte UAddr []byte - capabilities *capability.Capabilities + Capabilities *capability.Capabilities +} + +func (b *BzzAddr) DecodeRLP(s *rlp.Stream) error { + _, err := s.List() + if err != nil { + return fmt.Errorf("list %v", err) + } + err = s.Decode(&b.OAddr) + if err != nil { + return fmt.Errorf("oaddr %v", err) + } + err = s.Decode(&b.UAddr) + if err != nil { + return fmt.Errorf("uaddr %v", err) + } + err = s.Decode(&b.Capabilities) + if err != nil { + return fmt.Errorf("capz %v", err) + } + return nil } func NewBzzAddr(oaddr []byte, uaddr []byte) *BzzAddr { return &BzzAddr{ OAddr: oaddr, UAddr: uaddr, - capabilities: capability.NewCapabilities(), + Capabilities: capability.NewCapabilities(), } } @@ -41,10 +62,6 @@ func (a *BzzAddr) Under() []byte { return a.UAddr } -func (a *BzzAddr) Capabilities() *capability.Capabilities { - return a.capabilities -} - // ID returns the node identifier in the underlay. func (a *BzzAddr) ID() enode.ID { n, err := enode.ParseV4(string(a.UAddr)) @@ -56,12 +73,12 @@ func (a *BzzAddr) ID() enode.ID { // Update updates the underlay address of a peer record func (a *BzzAddr) Update(na *BzzAddr) *BzzAddr { - return &BzzAddr{a.OAddr, na.UAddr, a.capabilities} + return &BzzAddr{a.OAddr, na.UAddr, a.Capabilities} } // String pretty prints the address func (a *BzzAddr) String() string { - return fmt.Sprintf("%x <%s> cap:%s", a.OAddr, a.UAddr, a.capabilities) + return fmt.Sprintf("%x <%s> cap:%s", a.OAddr, a.UAddr, a.Capabilities) } // RandomBzzAddr is a utility method generating an address from a public key @@ -75,11 +92,11 @@ func RandomBzzAddr() *BzzAddr { } func NewBzzAddrFromEnode(enod *enode.Node) *BzzAddr { - return &BzzAddr{OAddr: enod.ID().Bytes(), UAddr: []byte(enod.URLv4()), capabilities: capability.NewCapabilities()} + return &BzzAddr{OAddr: enod.ID().Bytes(), UAddr: []byte(enod.URLv4()), Capabilities: capability.NewCapabilities()} } func (b *BzzAddr) WithCapabilities(c *capability.Capabilities) *BzzAddr { - b.capabilities = c + b.Capabilities = c return b } diff --git a/network/protocol.go b/network/protocol.go index 45d086e791..14de0720ff 100644 --- a/network/protocol.go +++ b/network/protocol.go @@ -155,9 +155,9 @@ func NewBzz(config *BzzConfig, kad *Kademlia, store state.Store, streamerSpec, r // temporary soon-to-be-legacy light/full, as above if config.LightNode { - bzz.localAddr.capabilities.Add(newLightCapability()) + bzz.localAddr.Capabilities.Add(newLightCapability()) } else { - bzz.localAddr.capabilities.Add(newFullCapability()) + bzz.localAddr.Capabilities.Add(newFullCapability()) } return bzz @@ -263,7 +263,7 @@ func (b *Bzz) RunProtocol(spec *protocols.Spec, run func(*BzzPeer) error) func(* Peer: protocols.NewPeer(p, rw, spec), BzzAddr: handshake.peerAddr, lastActive: time.Now(), - LightNode: isLightCapability(handshake.Capabilities.Get(0)), // this is a temporary member kept until kademlia code accommodates Capabilities instead + LightNode: isLightCapability(handshake.Addr.Capabilities.Get(0)), // this is a temporary member kept until kademlia code accommodates Capabilities instead } log.Debug("peer created", "addr", handshake.peerAddr.String()) @@ -285,9 +285,9 @@ func (b *Bzz) performHandshake(p *protocols.Peer, handshake *HandshakeMsg) error handshake.err = err return err } - rsh.(*HandshakeMsg).Addr.capabilities = rsh.(*HandshakeMsg).Capabilities + rsh.(*HandshakeMsg).Addr.Capabilities = rsh.(*HandshakeMsg).Addr.Capabilities handshake.peerAddr = rsh.(*HandshakeMsg).Addr - handshake.Capabilities = rsh.(*HandshakeMsg).Capabilities + handshake.Addr.Capabilities = rsh.(*HandshakeMsg).Addr.Capabilities return nil } @@ -346,10 +346,10 @@ func (p *BzzPeer) ID() enode.ID { * Capabilities: the capabilities bitvector */ type HandshakeMsg struct { - Version uint64 - NetworkID uint64 - Addr *BzzAddr - Capabilities *capability.Capabilities + Version uint64 + NetworkID uint64 + Addr *BzzAddr + //Capabilities *capability.Capabilities // peerAddr is the address received in the peer handshake peerAddr *BzzAddr @@ -361,7 +361,7 @@ type HandshakeMsg struct { // String pretty prints the handshake func (bh *HandshakeMsg) String() string { - return fmt.Sprintf("Handshake: Version: %v, NetworkID: %v, Addr: %v, peerAddr: %v, caps: %s", bh.Version, bh.NetworkID, bh.Addr, bh.peerAddr, bh.Capabilities) + return fmt.Sprintf("Handshake: Version: %v, NetworkID: %v, Addr: %v, peerAddr: %v", bh.Version, bh.NetworkID, bh.Addr, bh.peerAddr) } // Perform initiates the handshake and validates the remote handshake message @@ -374,10 +374,10 @@ func (b *Bzz) checkHandshake(hs interface{}) error { return fmt.Errorf("version mismatch %d (!= %d)", rhs.Version, BzzSpec.Version) } // temporary check for valid capability settings, legacy full/light - if !isFullCapability(rhs.Capabilities.Get(0)) && !isLightCapability(rhs.Capabilities.Get(0)) { - return fmt.Errorf("invalid capabilities setting: %s", rhs.Capabilities) + if !isFullCapability(rhs.Addr.Capabilities.Get(0)) && !isLightCapability(rhs.Addr.Capabilities.Get(0)) { + return fmt.Errorf("invalid capabilities setting: %s", rhs.Addr.Capabilities) } - rhs.Addr.capabilities = hs.(*HandshakeMsg).Capabilities + rhs.Addr.Capabilities = hs.(*HandshakeMsg).Addr.Capabilities return nil } @@ -396,12 +396,12 @@ func (b *Bzz) GetOrCreateHandshake(peerID enode.ID) (*HandshakeMsg, bool) { handshake, found := b.handshakes[peerID] if !found { handshake = &HandshakeMsg{ - Version: uint64(BzzSpec.Version), - NetworkID: b.NetworkID, - Addr: b.localAddr, - Capabilities: b.localAddr.capabilities, - init: make(chan bool, 1), - done: make(chan struct{}), + Version: uint64(BzzSpec.Version), + NetworkID: b.NetworkID, + Addr: b.localAddr, + //Capabilities: b.localAddr.Capabilities, + init: make(chan bool, 1), + done: make(chan struct{}), } // when handhsake is first created for a remote peer // it is initialised with the init diff --git a/network/protocol_test.go b/network/protocol_test.go index 8a8eb6e26e..2cc7edf1e3 100644 --- a/network/protocol_test.go +++ b/network/protocol_test.go @@ -77,11 +77,11 @@ func newBzzHandshakeMsg(version uint64, networkId uint64, addr *BzzAddr, lightNo cap = newFullCapability() } capabilities.Add(cap) + addr.Capabilities = capabilities msg := &HandshakeMsg{ - Version: version, - NetworkID: networkId, - Addr: addr, - Capabilities: capabilities, + Version: version, + NetworkID: networkId, + Addr: addr, } return msg @@ -296,12 +296,12 @@ func TestBzzHandshakeInvalidCapabilities(t *testing.T) { node := s.Nodes[0] msg := newBzzHandshakeMsg(TestProtocolVersion, TestProtocolNetworkID, NewBzzAddrFromEnode(node), false) - cap := msg.Capabilities.Get(0) + cap := msg.Addr.Capabilities.Get(0) cap.Set(14) err = s.testHandshake( correctBzzHandshake(s.addr, lightNode), msg, - &p2ptest.Disconnect{Peer: node.ID(), Error: fmt.Errorf("Handshake error: Message handler error: (msg code 0): invalid capabilities setting: %s", msg.Capabilities)}, + &p2ptest.Disconnect{Peer: node.ID(), Error: fmt.Errorf("Handshake error: Message handler error: (msg code 0): invalid capabilities setting: %s", msg.Addr.Capabilities)}, ) if err != nil { @@ -373,7 +373,7 @@ func TestBzzHandshakeLightNode(t *testing.T) { select { case <-pt.bzz.handshakes[node.ID()].done: - for _, cp := range pt.bzz.handshakes[node.ID()].Capabilities.Caps { + for _, cp := range pt.bzz.handshakes[node.ID()].Addr.Capabilities.Caps { if cp.String() != nodeCapability.String() { t.Fatalf("peer LightNode flag is %v, should be %v", cp.String(), nodeCapability.String()) } From 5723f9f72a8c71f5ee440ef91793b289e2cfa6a4 Mon Sep 17 00:00:00 2001 From: nolash Date: Sun, 8 Sep 2019 15:24:53 +0200 Subject: [PATCH 12/36] network: Add serialization tests --- network/network.go | 8 ++++---- network/network_test.go | 23 +++++++++++++++++++++++ network/protocol.go | 38 ++++++++++++++++++++++++++++---------- network/protocol_test.go | 39 ++++++++++++++++++++++++++++++++++++++- 4 files changed, 93 insertions(+), 15 deletions(-) create mode 100644 network/network_test.go diff --git a/network/network.go b/network/network.go index 70feafff87..39f7d06fd4 100644 --- a/network/network.go +++ b/network/network.go @@ -22,19 +22,19 @@ type BzzAddr struct { func (b *BzzAddr) DecodeRLP(s *rlp.Stream) error { _, err := s.List() if err != nil { - return fmt.Errorf("list %v", err) + return fmt.Errorf("list --- %v", err) } err = s.Decode(&b.OAddr) if err != nil { - return fmt.Errorf("oaddr %v", err) + return fmt.Errorf("oaddr --- %v", err) } err = s.Decode(&b.UAddr) if err != nil { - return fmt.Errorf("uaddr %v", err) + return fmt.Errorf("uaddr --- %v", err) } err = s.Decode(&b.Capabilities) if err != nil { - return fmt.Errorf("capz %v", err) + return fmt.Errorf("capz --- %v", err) } return nil } diff --git a/network/network_test.go b/network/network_test.go new file mode 100644 index 0000000000..e8e75e060a --- /dev/null +++ b/network/network_test.go @@ -0,0 +1,23 @@ +package network + +import ( + "testing" + + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethersphere/swarm/network/capability" +) + +func TestBzzAddrRLPSerialization(t *testing.T) { + caps := capability.NewCapabilities() + caps.Add(lightCapability) + addr := RandomBzzAddr().WithCapabilities(caps) + b, err := rlp.EncodeToBytes(addr) + if err != nil { + t.Fatal(err) + } + var addrRecovered BzzAddr + err = rlp.DecodeBytes(b, &addrRecovered) + if err != nil { + t.Fatal(err) + } +} diff --git a/network/protocol.go b/network/protocol.go index 14de0720ff..65e5010326 100644 --- a/network/protocol.go +++ b/network/protocol.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" "github.com/ethersphere/swarm/log" "github.com/ethersphere/swarm/network/capability" @@ -171,8 +172,9 @@ func (b *Bzz) Stop() error { // UpdateLocalAddr updates underlayaddress of the running node func (b *Bzz) UpdateLocalAddr(byteaddr []byte) *BzzAddr { b.localAddr = b.localAddr.Update(&BzzAddr{ - UAddr: byteaddr, - OAddr: b.localAddr.OAddr, + UAddr: byteaddr, + OAddr: b.localAddr.OAddr, + Capabilities: b.localAddr.Capabilities, }) return b.localAddr @@ -263,7 +265,7 @@ func (b *Bzz) RunProtocol(spec *protocols.Spec, run func(*BzzPeer) error) func(* Peer: protocols.NewPeer(p, rw, spec), BzzAddr: handshake.peerAddr, lastActive: time.Now(), - LightNode: isLightCapability(handshake.Addr.Capabilities.Get(0)), // this is a temporary member kept until kademlia code accommodates Capabilities instead + LightNode: isLightCapability(handshake.peerAddr.Capabilities.Get(0)), // this is a temporary member kept until kademlia code accommodates Capabilities instead } log.Debug("peer created", "addr", handshake.peerAddr.String()) @@ -285,9 +287,7 @@ func (b *Bzz) performHandshake(p *protocols.Peer, handshake *HandshakeMsg) error handshake.err = err return err } - rsh.(*HandshakeMsg).Addr.Capabilities = rsh.(*HandshakeMsg).Addr.Capabilities handshake.peerAddr = rsh.(*HandshakeMsg).Addr - handshake.Addr.Capabilities = rsh.(*HandshakeMsg).Addr.Capabilities return nil } @@ -349,7 +349,6 @@ type HandshakeMsg struct { Version uint64 NetworkID uint64 Addr *BzzAddr - //Capabilities *capability.Capabilities // peerAddr is the address received in the peer handshake peerAddr *BzzAddr @@ -359,6 +358,26 @@ type HandshakeMsg struct { err error } +func (bh *HandshakeMsg) DecodeRLP(s *rlp.Stream) error { + _, err := s.List() + if err != nil { + return fmt.Errorf("list --- %v", err) + } + err = s.Decode(&bh.Version) + if err != nil { + return fmt.Errorf("version --- %v", err) + } + err = s.Decode(&bh.NetworkID) + if err != nil { + return fmt.Errorf("networkid --- %v", err) + } + err = s.Decode(&bh.Addr) + if err != nil { + return fmt.Errorf("addr --- %v", err) + } + return nil +} + // String pretty prints the handshake func (bh *HandshakeMsg) String() string { return fmt.Sprintf("Handshake: Version: %v, NetworkID: %v, Addr: %v, peerAddr: %v", bh.Version, bh.NetworkID, bh.Addr, bh.peerAddr) @@ -377,7 +396,7 @@ func (b *Bzz) checkHandshake(hs interface{}) error { if !isFullCapability(rhs.Addr.Capabilities.Get(0)) && !isLightCapability(rhs.Addr.Capabilities.Get(0)) { return fmt.Errorf("invalid capabilities setting: %s", rhs.Addr.Capabilities) } - rhs.Addr.Capabilities = hs.(*HandshakeMsg).Addr.Capabilities + //rhs.Addr.Capabilities = hs.(*HandshakeMsg).Addr.Capabilities return nil } @@ -399,9 +418,8 @@ func (b *Bzz) GetOrCreateHandshake(peerID enode.ID) (*HandshakeMsg, bool) { Version: uint64(BzzSpec.Version), NetworkID: b.NetworkID, Addr: b.localAddr, - //Capabilities: b.localAddr.Capabilities, - init: make(chan bool, 1), - done: make(chan struct{}), + init: make(chan bool, 1), + done: make(chan struct{}), } // when handhsake is first created for a remote peer // it is initialised with the init diff --git a/network/protocol_test.go b/network/protocol_test.go index 2cc7edf1e3..f710411daa 100644 --- a/network/protocol_test.go +++ b/network/protocol_test.go @@ -17,6 +17,7 @@ package network import ( + "bytes" "crypto/ecdsa" "fmt" "sync" @@ -28,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/enr" + "github.com/ethereum/go-ethereum/rlp" "github.com/ethersphere/swarm/network/capability" "github.com/ethersphere/swarm/p2p/protocols" p2ptest "github.com/ethersphere/swarm/p2p/testing" @@ -233,6 +235,41 @@ func correctBzzHandshake(addr *BzzAddr, lightNode bool) *HandshakeMsg { return newBzzHandshakeMsg(TestProtocolVersion, TestProtocolNetworkID, addr, lightNode) } +func TestBzzHandshakeRLPSerialization(t *testing.T) { + caps := capability.NewCapabilities() + caps.Add(fullCapability) + addr := RandomBzzAddr().WithCapabilities(caps) + msg := &HandshakeMsg{ + Version: 42, + NetworkID: 666, + Addr: addr, + } + b, err := rlp.EncodeToBytes(msg) + if err != nil { + t.Fatal(err) + } + var msgRecovered HandshakeMsg + err = rlp.DecodeBytes(b, &msgRecovered) + if err != nil { + t.Fatal(err) + } + if msg.Version != msgRecovered.Version { + t.Fatal("version mismatch") + } + if msg.NetworkID != msgRecovered.NetworkID { + t.Fatal("networkid mismatch") + } + if !bytes.Equal(msg.Addr.OAddr, msgRecovered.Addr.OAddr) { + t.Fatal("OAddr mismatch") + } + if !bytes.Equal(msg.Addr.UAddr, msgRecovered.Addr.UAddr) { + t.Fatal("UAddr mismatch") + } + if !fullCapability.IsSameAs(msgRecovered.Addr.Capabilities.Get(0)) { + t.Fatal("capabilities mismatch") + } +} + func TestBzzHandshakeNetworkIDMismatch(t *testing.T) { lightNode := false prvkey, err := crypto.GenerateKey() @@ -373,7 +410,7 @@ func TestBzzHandshakeLightNode(t *testing.T) { select { case <-pt.bzz.handshakes[node.ID()].done: - for _, cp := range pt.bzz.handshakes[node.ID()].Addr.Capabilities.Caps { + for _, cp := range pt.bzz.handshakes[node.ID()].peerAddr.Capabilities.Caps { if cp.String() != nodeCapability.String() { t.Fatalf("peer LightNode flag is %v, should be %v", cp.String(), nodeCapability.String()) } From 84222dacbd4e8f2fe8c7aba3a0f2a075fe9c5064 Mon Sep 17 00:00:00 2001 From: nolash Date: Mon, 9 Sep 2019 15:51:02 +0200 Subject: [PATCH 13/36] network: Temporary workaround for contingent list in BzzAddr deserialize --- network/network.go | 38 ++++++++++++++++++++++++++++++++++---- network/protocol.go | 21 --------------------- 2 files changed, 34 insertions(+), 25 deletions(-) diff --git a/network/network.go b/network/network.go index 39f7d06fd4..99ee97ebd7 100644 --- a/network/network.go +++ b/network/network.go @@ -3,6 +3,7 @@ package network import ( "crypto/ecdsa" "fmt" + "io" "net" "github.com/ethereum/go-ethereum/crypto" @@ -19,10 +20,33 @@ type BzzAddr struct { Capabilities *capability.Capabilities } +func (b *BzzAddr) EncodeRLP(w io.Writer) error { + err := rlp.Encode(w, b.OAddr) + if err != nil { + return err + } + err = rlp.Encode(w, b.UAddr) + if err != nil { + return err + } + y, err := rlp.EncodeToBytes(b.Capabilities) + if err != nil { + return err + } + err = rlp.Encode(w, y) + if err != nil { + return err + } + return nil +} + func (b *BzzAddr) DecodeRLP(s *rlp.Stream) error { - _, err := s.List() + knd, _, err := s.Kind() if err != nil { - return fmt.Errorf("list --- %v", err) + return err + } + if knd == rlp.List { + s.List() } err = s.Decode(&b.OAddr) if err != nil { @@ -32,9 +56,15 @@ func (b *BzzAddr) DecodeRLP(s *rlp.Stream) error { if err != nil { return fmt.Errorf("uaddr --- %v", err) } - err = s.Decode(&b.Capabilities) + + var y []byte + err = s.Decode(&y) + if err != nil { + return fmt.Errorf("capzbytes --- %v", err) + } + err = rlp.DecodeBytes(y, &b.Capabilities) if err != nil { - return fmt.Errorf("capz --- %v", err) + return fmt.Errorf("capzdecode --- %v", err) } return nil } diff --git a/network/protocol.go b/network/protocol.go index 65e5010326..71ea1b4e72 100644 --- a/network/protocol.go +++ b/network/protocol.go @@ -26,7 +26,6 @@ import ( "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" - "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" "github.com/ethersphere/swarm/log" "github.com/ethersphere/swarm/network/capability" @@ -358,26 +357,6 @@ type HandshakeMsg struct { err error } -func (bh *HandshakeMsg) DecodeRLP(s *rlp.Stream) error { - _, err := s.List() - if err != nil { - return fmt.Errorf("list --- %v", err) - } - err = s.Decode(&bh.Version) - if err != nil { - return fmt.Errorf("version --- %v", err) - } - err = s.Decode(&bh.NetworkID) - if err != nil { - return fmt.Errorf("networkid --- %v", err) - } - err = s.Decode(&bh.Addr) - if err != nil { - return fmt.Errorf("addr --- %v", err) - } - return nil -} - // String pretty prints the handshake func (bh *HandshakeMsg) String() string { return fmt.Sprintf("Handshake: Version: %v, NetworkID: %v, Addr: %v, peerAddr: %v", bh.Version, bh.NetworkID, bh.Addr, bh.peerAddr) From 70cb0cdc757fd314115771763ea8a20ab43ba0bd Mon Sep 17 00:00:00 2001 From: nolash Date: Tue, 10 Sep 2019 16:28:57 +0200 Subject: [PATCH 14/36] network: BzzAddr deserialize fail on empty object --- network/network.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/network/network.go b/network/network.go index 99ee97ebd7..fb4959f25b 100644 --- a/network/network.go +++ b/network/network.go @@ -41,13 +41,13 @@ func (b *BzzAddr) EncodeRLP(w io.Writer) error { } func (b *BzzAddr) DecodeRLP(s *rlp.Stream) error { - knd, _, err := s.Kind() + _, _, err := s.Kind() if err != nil { return err } - if knd == rlp.List { - s.List() - } + // if knd == rlp.List { + // s.List() + // } err = s.Decode(&b.OAddr) if err != nil { return fmt.Errorf("oaddr --- %v", err) From afc70b09cd87667ac8a466914c6235ecf4c13105 Mon Sep 17 00:00:00 2001 From: nolash Date: Wed, 11 Sep 2019 09:25:22 +0200 Subject: [PATCH 15/36] network, cmd: WIP debugging --- cmd/swarm/run_test.go | 5 ++++- network/discovery.go | 2 ++ network/network.go | 3 +++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/cmd/swarm/run_test.go b/cmd/swarm/run_test.go index d9b9ca1eb5..2cdf76fa98 100644 --- a/cmd/swarm/run_test.go +++ b/cmd/swarm/run_test.go @@ -131,7 +131,10 @@ func newTestCluster(t *testing.T, size int) *testCluster { } // connect the nodes together - for _, node := range cluster.Nodes { + for i, node := range cluster.Nodes { + if i == 0 { + continue + } if err := node.Client.Call(nil, "admin_addPeer", cluster.Nodes[0].Enode); err != nil { t.Fatal(err) } diff --git a/network/discovery.go b/network/discovery.go index 9e60c68fa0..4086895c46 100644 --- a/network/discovery.go +++ b/network/discovery.go @@ -21,6 +21,7 @@ import ( "fmt" "sync" + "github.com/ethersphere/swarm/log" "github.com/ethersphere/swarm/pot" ) @@ -95,6 +96,7 @@ func (d *Peer) NotifyPeer(a *BzzAddr, po uint8) { resp := &peersMsg{ Peers: []*BzzAddr{a}, } + log.Warn("notifypeer", "notify", resp) go d.Send(context.TODO(), resp) } diff --git a/network/network.go b/network/network.go index fb4959f25b..451652c6e6 100644 --- a/network/network.go +++ b/network/network.go @@ -7,6 +7,7 @@ import ( "net" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/enr" "github.com/ethereum/go-ethereum/rlp" @@ -43,6 +44,8 @@ func (b *BzzAddr) EncodeRLP(w io.Writer) error { func (b *BzzAddr) DecodeRLP(s *rlp.Stream) error { _, _, err := s.Kind() if err != nil { + log.Error("decoderlp bzzaddr err", "err", err) + panic(err) return err } // if knd == rlp.List { From b209db46ea6e87e366a97ce597263dd4faaf0239 Mon Sep 17 00:00:00 2001 From: nolash Date: Wed, 11 Sep 2019 17:16:58 +0200 Subject: [PATCH 16/36] network: Add deserialize in peersMsg to elimiate list error --- network/discovery.go | 22 ++++++++++++++++++++++ network/network.go | 32 +++++++------------------------- 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/network/discovery.go b/network/discovery.go index 4086895c46..941e1b859f 100644 --- a/network/discovery.go +++ b/network/discovery.go @@ -21,6 +21,7 @@ import ( "fmt" "sync" + "github.com/ethereum/go-ethereum/rlp" "github.com/ethersphere/swarm/log" "github.com/ethersphere/swarm/pot" ) @@ -129,6 +130,27 @@ type peersMsg struct { Peers []*BzzAddr } +// DecodeRLP implements rlp.Decoder interface +func (p *peersMsg) DecodeRLP(s *rlp.Stream) error { + _, err := s.List() + if err != nil { + return err + } + _, err = s.List() + if err != nil { + return err + } + for { + var addr BzzAddr + err = s.Decode(&addr) + if err != nil { + break + } + p.Peers = append(p.Peers, &addr) + } + return nil +} + // String pretty prints a peersMsg func (msg peersMsg) String() string { return fmt.Sprintf("%T: %v", msg, msg.Peers) diff --git a/network/network.go b/network/network.go index 451652c6e6..96721623f3 100644 --- a/network/network.go +++ b/network/network.go @@ -7,7 +7,6 @@ import ( "net" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/enr" "github.com/ethereum/go-ethereum/rlp" @@ -30,11 +29,7 @@ func (b *BzzAddr) EncodeRLP(w io.Writer) error { if err != nil { return err } - y, err := rlp.EncodeToBytes(b.Capabilities) - if err != nil { - return err - } - err = rlp.Encode(w, y) + err = rlp.Encode(w, b.Capabilities) if err != nil { return err } @@ -42,32 +37,19 @@ func (b *BzzAddr) EncodeRLP(w io.Writer) error { } func (b *BzzAddr) DecodeRLP(s *rlp.Stream) error { - _, _, err := s.Kind() - if err != nil { - log.Error("decoderlp bzzaddr err", "err", err) - panic(err) - return err - } - // if knd == rlp.List { - // s.List() - // } - err = s.Decode(&b.OAddr) + var err error + + b.OAddr, err = s.Bytes() if err != nil { return fmt.Errorf("oaddr --- %v", err) } - err = s.Decode(&b.UAddr) + b.UAddr, err = s.Bytes() if err != nil { return fmt.Errorf("uaddr --- %v", err) } - - var y []byte - err = s.Decode(&y) - if err != nil { - return fmt.Errorf("capzbytes --- %v", err) - } - err = rlp.DecodeBytes(y, &b.Capabilities) + err = s.Decode(&b.Capabilities) if err != nil { - return fmt.Errorf("capzdecode --- %v", err) + return fmt.Errorf("caps --- %v", err) } return nil } From b8149bbb4f6469b47b576ee6187c7d016b278202 Mon Sep 17 00:00:00 2001 From: nolash Date: Wed, 11 Sep 2019 17:25:37 +0200 Subject: [PATCH 17/36] network: Re-instate serialize capabilites to bytes in BzzAddr --- network/network.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/network/network.go b/network/network.go index 96721623f3..95c2cfa41b 100644 --- a/network/network.go +++ b/network/network.go @@ -29,7 +29,11 @@ func (b *BzzAddr) EncodeRLP(w io.Writer) error { if err != nil { return err } - err = rlp.Encode(w, b.Capabilities) + y, err := rlp.EncodeToBytes(b.Capabilities) + if err != nil { + return err + } + err = rlp.Encode(w, y) if err != nil { return err } @@ -47,7 +51,12 @@ func (b *BzzAddr) DecodeRLP(s *rlp.Stream) error { if err != nil { return fmt.Errorf("uaddr --- %v", err) } - err = s.Decode(&b.Capabilities) + + y, err := s.Bytes() + if err != nil { + return fmt.Errorf("capsbytes --- %v", err) + } + err = rlp.DecodeBytes(y, &b.Capabilities) if err != nil { return fmt.Errorf("caps --- %v", err) } From bd59236d762a4b9406d5edea05b2b8df3601205e Mon Sep 17 00:00:00 2001 From: nolash Date: Wed, 11 Sep 2019 19:22:49 +0200 Subject: [PATCH 18/36] pot: Revert inadvertent rename of RandomAddress* --- network/discovery_test.go | 4 ++-- network/kademlia_test.go | 4 ++-- network/protocol_test.go | 2 +- network/retrieval/retrieve_test.go | 4 ++-- network/stream/v2/common_test.go | 2 +- pot/address.go | 6 +++--- pss/forwarding_test.go | 18 +++++++++--------- pss/prox_test.go | 2 +- pss/pss_test.go | 8 ++++---- 9 files changed, 25 insertions(+), 25 deletions(-) diff --git a/network/discovery_test.go b/network/discovery_test.go index fcd975fb10..7f514c549a 100644 --- a/network/discovery_test.go +++ b/network/discovery_test.go @@ -105,7 +105,7 @@ func testInitialPeersMsg(t *testing.T, peerPO, peerDepth int) { sortPeers = testSortPeers pivotAddr := pot.NewAddressFromBytes(PrivateKeyToBzzKey(prvkey)) // generate control peers address at peerPO wrt pivot - peerAddr := pot.RandomBzzAddressAt(pivotAddr, peerPO) + peerAddr := pot.RandomAddressAt(pivotAddr, peerPO) // construct kademlia and hive to := NewKademlia(pivotAddr[:], NewKadParams()) hive := NewHive(NewHiveParams(), to, nil) @@ -115,7 +115,7 @@ func testInitialPeersMsg(t *testing.T, peerPO, peerDepth int) { connect := func(a pot.Address, po int) (addrs []*BzzAddr) { n := rand.Intn(maxPeersPerPO) for i := 0; i < n; i++ { - peer := newDiscPeer(pot.RandomBzzAddressAt(a, po)) + peer := newDiscPeer(pot.RandomAddressAt(a, po)) hive.On(peer) addrs = append(addrs, peer.BzzAddr) } diff --git a/network/kademlia_test.go b/network/kademlia_test.go index 3489bacd22..aa952e82cc 100644 --- a/network/kademlia_test.go +++ b/network/kademlia_test.go @@ -103,12 +103,12 @@ func TestNeighbourhoodDepth(t *testing.T) { // generate the peers var peers []*Peer for i := 0; i < 7; i++ { - addr := pot.RandomBzzAddressAt(baseAddress, i) + addr := pot.RandomAddressAt(baseAddress, i) peers = append(peers, newTestDiscoveryPeer(addr, kad)) } var sevenPeers []*Peer for i := 0; i < 2; i++ { - addr := pot.RandomBzzAddressAt(baseAddress, 7) + addr := pot.RandomAddressAt(baseAddress, 7) sevenPeers = append(sevenPeers, newTestDiscoveryPeer(addr, kad)) } diff --git a/network/protocol_test.go b/network/protocol_test.go index f710411daa..4d9363587e 100644 --- a/network/protocol_test.go +++ b/network/protocol_test.go @@ -92,7 +92,7 @@ func newBzzHandshakeMsg(version uint64, networkId uint64, addr *BzzAddr, lightNo func newBzzBaseTester(n int, prvkey *ecdsa.PrivateKey, spec *protocols.Spec, run func(*BzzPeer) error) (*bzzTester, error) { var addrs [][]byte for i := 0; i < n; i++ { - addr := pot.RandomBzzAddress() + addr := pot.RandomAddress() addrs = append(addrs, addr[:]) } pt, _, err := newBzzBaseTesterWithAddrs(prvkey, addrs, spec, run) diff --git a/network/retrieval/retrieve_test.go b/network/retrieval/retrieve_test.go index 225e709a90..1c810ea0c3 100644 --- a/network/retrieval/retrieve_test.go +++ b/network/retrieval/retrieve_test.go @@ -286,7 +286,7 @@ func TestRequestFromPeersWithLightNode(t *testing.T) { //TestHasPriceImplementation is to check that Retrieval implements protocols.Prices func TestHasPriceImplementation(t *testing.T) { - addr := network.RandomAddr() + addr := network.RandomBzzAddr() to := network.NewKademlia(addr.OAddr, network.NewKadParams()) r := New(to, nil, to.BaseAddr(), nil) @@ -437,7 +437,7 @@ func nodeConfigAtPo(t *testing.T, baseaddr []byte, po int) *adapters.NodeConfig t.Fatalf("unable to create enode: %v", err) } - n := network.NewAddr(nod) + n := network.NewBzzAddrFromEnode(nod) foundPo = chunk.Proximity(baseaddr, n.Over()) } diff --git a/network/stream/v2/common_test.go b/network/stream/v2/common_test.go index 1f57a17dec..5f844846db 100644 --- a/network/stream/v2/common_test.go +++ b/network/stream/v2/common_test.go @@ -110,7 +110,7 @@ func newSyncSimServiceFunc(o *SyncSimServiceOptions) func(ctx *adapters.ServiceC } return func(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Service, cleanup func(), err error) { n := ctx.Config.Node() - addr := network.NewAddr(n) + addr := network.NewBzzAddrFromEnode(n) localStore, localStoreCleanup, err := newTestLocalStore(n.ID(), addr, nil) if err != nil { diff --git a/pot/address.go b/pot/address.go index 00e21eec71..7bb9f7c0d1 100644 --- a/pot/address.go +++ b/pot/address.go @@ -137,7 +137,7 @@ func ProxCmp(a, x, y []byte) int { // RandomBzzAddressAt (address, prox) generates a random address // at proximity order prox relative to address // if prox is negative a random address is generated -func RandomBzzAddressAt(self Address, prox int) (addr Address) { +func RandomAddressAt(self Address, prox int) (addr Address) { addr = self pos := -1 if prox >= 0 { @@ -160,8 +160,8 @@ func RandomBzzAddressAt(self Address, prox int) (addr Address) { } // RandomBzzAddress generates a random address -func RandomBzzAddress() Address { - return RandomBzzAddressAt(Address{}, -1) +func RandomAddress() Address { + return RandomAddressAt(Address{}, -1) } // NewAddressFromString creates a byte slice from a string in binary representation diff --git a/pss/forwarding_test.go b/pss/forwarding_test.go index 738f7f4969..12dd7ea814 100644 --- a/pss/forwarding_test.go +++ b/pss/forwarding_test.go @@ -41,15 +41,15 @@ func TestForwardBasic(t *testing.T) { const depth = 10 for i := 0; i <= depth; i++ { // add two peers for each proximity order - a := pot.RandomBzzAddressAt(base, i) + a := pot.RandomAddressAt(base, i) peerAddresses = append(peerAddresses, a) - a = pot.RandomBzzAddressAt(base, i) + a = pot.RandomAddressAt(base, i) peerAddresses = append(peerAddresses, a) } // skip one level, add one peer at one level deeper. // as a result, we will have an edge case of three peers in nearest neighbours' bin. - peerAddresses = append(peerAddresses, pot.RandomBzzAddressAt(base, depth+2)) + peerAddresses = append(peerAddresses, pot.RandomAddressAt(base, depth+2)) kad := network.NewKademlia(base[:], network.NewKadParams()) ps := createPss(t, kad) @@ -78,7 +78,7 @@ func TestForwardBasic(t *testing.T) { for i := 0; i < firstNearest; i++ { // send random messages with proximity orders, corresponding to PO of each bin, // with one peer being closer to the recipient address - a := pot.RandomBzzAddressAt(peerAddresses[i], 64) + a := pot.RandomAddressAt(peerAddresses[i], 64) c = testCase{ name: fmt.Sprintf("Send random to each PO, id: [%d]", i), recipient: a[:], @@ -93,7 +93,7 @@ func TestForwardBasic(t *testing.T) { // send random messages with proximity orders, corresponding to PO of each bin, // with random proximity relative to the recipient address po := i / 2 - a := pot.RandomBzzAddressAt(base, po) + a := pot.RandomAddressAt(base, po) c = testCase{ name: fmt.Sprintf("Send direct to known, id: [%d]", i), recipient: a[:], @@ -106,7 +106,7 @@ func TestForwardBasic(t *testing.T) { for i := firstNearest; i < len(peerAddresses); i++ { // recipient address falls into the nearest neighbours' bin - a := pot.RandomBzzAddressAt(base, i) + a := pot.RandomAddressAt(base, i) c = testCase{ name: fmt.Sprintf("recipient address falls into the nearest neighbours' bin, id: [%d]", i), recipient: a[:], @@ -118,7 +118,7 @@ func TestForwardBasic(t *testing.T) { } // send msg with proximity order much deeper than the deepest nearest neighbour - a2 := pot.RandomBzzAddressAt(base, 77) + a2 := pot.RandomAddressAt(base, 77) c = testCase{ name: "proximity order much deeper than the deepest nearest neighbour", recipient: a2[:], @@ -167,7 +167,7 @@ func TestForwardBasic(t *testing.T) { } // partial address with proximity order deeper than any of the nearest neighbour - a3 := pot.RandomBzzAddressAt(base, part) + a3 := pot.RandomAddressAt(base, part) c = testCase{ name: "partial address with proximity order deeper than any of the nearest neighbour", recipient: a3[:part], @@ -218,7 +218,7 @@ func TestForwardBasic(t *testing.T) { // send random messages with proximity orders, corresponding to PO of each bin, // with different numbers of failed attempts. // msg should be received by only one of the deeper peers. - a := pot.RandomBzzAddressAt(base, po) + a := pot.RandomAddressAt(base, po) c = testCase{ name: fmt.Sprintf("Send direct to known, id: [%d]", i), recipient: a[:], diff --git a/pss/prox_test.go b/pss/prox_test.go index 19421b16f4..f427559a8e 100644 --- a/pss/prox_test.go +++ b/pss/prox_test.go @@ -136,7 +136,7 @@ func (td *testData) init(msgCount int) error { } for i := 0; i < msgCount; i++ { - msgAddr := pot.RandomBzzAddress() // we choose message addresses randomly + msgAddr := pot.RandomAddress() // we choose message addresses randomly td.recipientAddresses = append(td.recipientAddresses, msgAddr.Bytes()) smallestPo := 256 var target enode.ID diff --git a/pss/pss_test.go b/pss/pss_test.go index f84df04039..e6624a3f16 100644 --- a/pss/pss_test.go +++ b/pss/pss_test.go @@ -189,7 +189,7 @@ func TestAddressMatchProx(t *testing.T) { rw := &p2p.MsgPipeRW{} ptpPeer := p2p.NewPeer(enode.ID{}, "362436 call me anytime", []p2p.Cap{}) protoPeer := protocols.NewPeer(ptpPeer, rw, &protocols.Spec{}) - peerAddr := pot.RandomBzzAddressAt(localPotAddr, i) + peerAddr := pot.RandomAddressAt(localPotAddr, i) bzzPeer := &network.BzzPeer{ Peer: protoPeer, BzzAddr: network.NewBzzAddr(peerAddr.Bytes(), []byte(fmt.Sprintf("%x", peerAddr[:]))), @@ -270,7 +270,7 @@ func TestAddressMatchProx(t *testing.T) { // test the distances var prevReceive int for i, distance := range remoteDistances { - remotePotAddr := pot.RandomBzzAddressAt(localPotAddr, distance) + remotePotAddr := pot.RandomAddressAt(localPotAddr, distance) remoteAddr := remotePotAddr.Bytes() var data [32]byte @@ -299,7 +299,7 @@ func TestAddressMatchProx(t *testing.T) { receives = 0 prevReceive = 0 for i, distance := range remoteDistances { - remotePotAddr := pot.RandomBzzAddressAt(localPotAddr, distance) + remotePotAddr := pot.RandomAddressAt(localPotAddr, distance) remoteAddr := remotePotAddr.Bytes() var data [32]byte @@ -323,7 +323,7 @@ func TestAddressMatchProx(t *testing.T) { receives = 0 for _, distance := range remoteDistances { - remotePotAddr := pot.RandomBzzAddressAt(localPotAddr, distance) + remotePotAddr := pot.RandomAddressAt(localPotAddr, distance) remoteAddr := remotePotAddr.Bytes() pssMsg := message.New(message.Flags{Raw: true}) From 060331ba1c814b600c7a662159643ac3af2261b2 Mon Sep 17 00:00:00 2001 From: nolash Date: Thu, 12 Sep 2019 09:30:35 +0200 Subject: [PATCH 19/36] network/stream: Use new BzzAddr constructor in stream tests --- network/stream/v2/cursors_test.go | 7 ++----- network/stream/v2/syncing_test.go | 7 ++----- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/network/stream/v2/cursors_test.go b/network/stream/v2/cursors_test.go index 6b4315d53d..23bef7ef4a 100644 --- a/network/stream/v2/cursors_test.go +++ b/network/stream/v2/cursors_test.go @@ -550,11 +550,8 @@ func TestCorrectCursorsExchangeRace(t *testing.T) { protoPeer := protocols.NewPeer(ptpPeer, rw, &protocols.Spec{}) peerAddr := pot.RandomAddressAt(pivotAddr, i) bzzPeer := &network.BzzPeer{ - Peer: protoPeer, - BzzAddr: &network.BzzAddr{ - OAddr: peerAddr.Bytes(), - UAddr: []byte(fmt.Sprintf("%x", peerAddr[:])), - }, + Peer: protoPeer, + BzzAddr: network.NewBzzAddr(peerAddr.Bytes(), []byte(fmt.Sprintf("%x", peerAddr[:]))), } peer := network.NewPeer(bzzPeer, pivotKad) pivotKad.On(peer) diff --git a/network/stream/v2/syncing_test.go b/network/stream/v2/syncing_test.go index 2cac8310b9..e18b422821 100644 --- a/network/stream/v2/syncing_test.go +++ b/network/stream/v2/syncing_test.go @@ -699,11 +699,8 @@ func TestStarNetworkSyncWithBogusNodes(t *testing.T) { protoPeer := protocols.NewPeer(ptpPeer, rw, &protocols.Spec{}) peerAddr := pot.RandomAddressAt(pivotAddr, i) bzzPeer := &network.BzzPeer{ - Peer: protoPeer, - BzzAddr: &network.BzzAddr{ - OAddr: peerAddr.Bytes(), - UAddr: []byte(fmt.Sprintf("%x", peerAddr[:])), - }, + Peer: protoPeer, + BzzAddr: network.NewBzzAddr(peerAddr.Bytes(), []byte(fmt.Sprintf("%x", peerAddr[:]))), } peer := network.NewPeer(bzzPeer, pivotKad) pivotKad.On(peer) From 3d631e991582e7dead157f2968416c38dacd7c00 Mon Sep 17 00:00:00 2001 From: nolash Date: Thu, 12 Sep 2019 11:01:22 +0200 Subject: [PATCH 20/36] pot: Remove wrong rename for random address gen in pot test --- pot/pot_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pot/pot_test.go b/pot/pot_test.go index 39c6e0dc6a..6189c7607a 100644 --- a/pot/pot_test.go +++ b/pot/pot_test.go @@ -56,12 +56,12 @@ func (a *testAddr) String() string { } func randomTestAddr(n int, i int) *testAddr { - v := RandomBzzAddress().Bin()[:n] + v := RandomAddress().Bin()[:n] return newTestAddr(v, i) } func randomtestAddr(n int, i int) *testAddr { - v := RandomBzzAddress().Bin()[:n] + v := RandomAddress().Bin()[:n] return newTestAddr(v, i) } From 3d32505ed9e4813f4c0fa24c066ba5bae13d658b Mon Sep 17 00:00:00 2001 From: nolash Date: Thu, 12 Sep 2019 13:48:22 +0200 Subject: [PATCH 21/36] network: Remove comments, reinstate skipped hive test --- network/discovery_test.go | 3 +-- network/hive_test.go | 2 +- network/kademlia_test.go | 26 -------------------------- network/protocol.go | 1 - pss/forwarding_test.go | 1 - 5 files changed, 2 insertions(+), 31 deletions(-) diff --git a/network/discovery_test.go b/network/discovery_test.go index 7f514c549a..6e191aa56b 100644 --- a/network/discovery_test.go +++ b/network/discovery_test.go @@ -17,9 +17,8 @@ package network import ( - "crypto/ecdsa" - //crand "crypto/rand" "bytes" + "crypto/ecdsa" "encoding/binary" "fmt" "math/rand" diff --git a/network/hive_test.go b/network/hive_test.go index 6ebfb64903..919e5c0afb 100644 --- a/network/hive_test.go +++ b/network/hive_test.go @@ -110,7 +110,7 @@ func TestRegisterAndConnect(t *testing.T) { // Actual connectivity is not in scope for this test, as the peers loaded from state are not known to // the simulation; the test only verifies that the peers are known to the node func TestHiveStatePersistance(t *testing.T) { - t.Skip("persistence test does not retrieve capabilities properly") + // t.Skip("persistence test does not retrieve capabilities properly") dir, err := ioutil.TempDir("", "hive_test_store") if err != nil { t.Fatal(err) diff --git a/network/kademlia_test.go b/network/kademlia_test.go index aa952e82cc..4c568c21e8 100644 --- a/network/kademlia_test.go +++ b/network/kademlia_test.go @@ -472,32 +472,6 @@ func TestOffEffectingAddressBookNormalNode(t *testing.T) { } } -// a light node should not be in the address book -func TestOffEffectingAddressBookLightNode(t *testing.T) { - t.Skip("lightnode flag is now obsolete, this test must be changed") - tk := newTestKademlia(t, "00000000") - // light node peer added to kademlia - tk.Kademlia.On(tk.newTestKadPeer("01000000", true)) - // peer should not be in the address book - if tk.addrs.Size() != 0 { - t.Fatal("known peer addresses should contain 0 entry") - } - // peer should be among live connections - if tk.conns.Size() != 1 { - t.Fatal("live peers should contain 1 entry") - } - // remove peer from kademlia - tk.Kademlia.Off(tk.newTestKadPeer("01000000", true)) - // peer should not be in the address book - if tk.addrs.Size() != 0 { - t.Fatal("known peer addresses should contain 0 entry") - } - // peer should not be among live connections - if tk.conns.Size() != 0 { - t.Fatal("live peers should contain 0 entry") - } -} - func TestSuggestPeerRetries(t *testing.T) { tk := newTestKademlia(t, "00000000") tk.RetryInterval = int64(300 * time.Millisecond) // cycle diff --git a/network/protocol.go b/network/protocol.go index 71ea1b4e72..afe7e7f754 100644 --- a/network/protocol.go +++ b/network/protocol.go @@ -375,7 +375,6 @@ func (b *Bzz) checkHandshake(hs interface{}) error { if !isFullCapability(rhs.Addr.Capabilities.Get(0)) && !isLightCapability(rhs.Addr.Capabilities.Get(0)) { return fmt.Errorf("invalid capabilities setting: %s", rhs.Addr.Capabilities) } - //rhs.Addr.Capabilities = hs.(*HandshakeMsg).Addr.Capabilities return nil } diff --git a/pss/forwarding_test.go b/pss/forwarding_test.go index 12dd7ea814..ea56c489c4 100644 --- a/pss/forwarding_test.go +++ b/pss/forwarding_test.go @@ -319,7 +319,6 @@ func testForwardMsg(t *testing.T, ps *Pss, c *testCase) { func addPeers(kad *network.Kademlia, addresses []pot.Address) { for _, a := range addresses { p := newTestDiscoveryPeer(a, kad) - //p.Addr.SetCapabilities = capability.NewCapabilities() kad.On(p) } } From 83cca5e85f98ebb02648b9906a86bf758df3ab5a Mon Sep 17 00:00:00 2001 From: nolash Date: Fri, 13 Sep 2019 11:44:28 +0200 Subject: [PATCH 22/36] network: Add test for kademlia capability indices --- network/capability/capability.go | 2 +- network/kademlia.go | 2 +- network/kademlia_test.go | 129 +++++++++++++++++++++++++++++-- 3 files changed, 126 insertions(+), 7 deletions(-) diff --git a/network/capability/capability.go b/network/capability/capability.go index 6dd26ae19b..6f9415b9d6 100644 --- a/network/capability/capability.go +++ b/network/capability/capability.go @@ -188,7 +188,7 @@ func (c *Capabilities) DecodeRLP(s *rlp.Stream) error { // Match returns true if all bits set in the argument is also set in the receiver func (c *Capability) Match(capCompare *Capability) bool { - if len(c.Cap) != len(capCompare.Cap) { + if capCompare == nil || len(c.Cap) != len(capCompare.Cap) { return false } // on the first occurrence of false where query has true we can fail diff --git a/network/kademlia.go b/network/kademlia.go index 3fdc92fca3..f2143ca371 100644 --- a/network/kademlia.go +++ b/network/kademlia.go @@ -153,7 +153,7 @@ func (k *Kademlia) addToCapabilityIndex(p interface{}, connected bool) { if idxItem.Id != vCap.Id { continue } - if idxItem.Match(vCap) { + if vCap.Match(idxItem.Capability) { log.Debug("Added peer to capability index", "conn", connected, "s", s, "v", vCap, "p", p) if connected { k.capabilityIndex[s].conns, _, _ = pot.Add(idxItem.conns, ePeer, Pof) diff --git a/network/kademlia_test.go b/network/kademlia_test.go index 4c568c21e8..6e24fafd3e 100644 --- a/network/kademlia_test.go +++ b/network/kademlia_test.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethersphere/swarm/network/capability" "github.com/ethersphere/swarm/p2p/protocols" "github.com/ethersphere/swarm/pot" ) @@ -35,9 +36,14 @@ func init() { log.Root().SetHandler(h) } -func testKadPeerAddr(s string) *BzzAddr { +func testKadPeerAddr(s string, lightNode bool) *BzzAddr { a := pot.NewAddressFromString(s) - return NewBzzAddr(a, a) + bzzAddr := NewBzzAddr(a, a) + if lightNode { + bzzAddr.Capabilities.Add(lightCapability) + } + bzzAddr.Capabilities.Add(fullCapability) + return bzzAddr } func newTestKademliaParams() *KadParams { @@ -52,6 +58,11 @@ type testKademlia struct { t *testing.T } +type testKadAddrSpec struct { + bits string + lightNode bool +} + func newTestKademlia(t *testing.T, b string) *testKademlia { base := pot.NewAddressFromString(b) return &testKademlia{ @@ -61,7 +72,7 @@ func newTestKademlia(t *testing.T, b string) *testKademlia { } func (tk *testKademlia) newTestKadPeer(s string, lightNode bool) *Peer { - return NewPeer(&BzzPeer{BzzAddr: testKadPeerAddr(s), LightNode: lightNode}, tk.Kademlia) + return NewPeer(&BzzPeer{BzzAddr: testKadPeerAddr(s, lightNode), LightNode: lightNode}, tk.Kademlia) } func (tk *testKademlia) On(ons ...string) { @@ -77,9 +88,20 @@ func (tk *testKademlia) Off(offs ...string) { } func (tk *testKademlia) Register(regs ...string) { + var specs []testKadAddrSpec + for _, r := range regs { + specs = append(specs, testKadAddrSpec{ + bits: r, + lightNode: false, + }) + } + tk.RegisterWithCapability(specs...) +} + +func (tk *testKademlia) RegisterWithCapability(regs ...testKadAddrSpec) { var as []*BzzAddr - for _, s := range regs { - as = append(as, testKadPeerAddr(s)) + for _, r := range regs { + as = append(as, testKadPeerAddr(r.bits, r.lightNode)) } err := tk.Kademlia.Register(as...) if err != nil { @@ -208,6 +230,103 @@ func TestHighMinBinSize(t *testing.T) { } } +func TestCapabilitiesIndex(t *testing.T) { + kp := NewKadParams() + addr := RandomBzzAddr() + base := addr.OAddr + k := NewKademlia(base, kp) + + testMoreCapability := capability.NewCapability(42, 3) + testMoreCapability.Set(0) + testMoreCapability.Set(2) + k.RegisterCapabilityIndex("more", *testMoreCapability) + + testLessCapability := capability.NewCapability(42, 3) + testLessCapability.Set(2) + k.RegisterCapabilityIndex("less", *testLessCapability) + + testNoneCapability := capability.NewCapability(42, 3) + testNoneCapability.Set(1) + k.RegisterCapabilityIndex("none", *testNoneCapability) + + testOtherCapability := capability.NewCapability(666, 3) + testOtherCapability.Set(0) + testOtherCapability.Set(2) + k.RegisterCapabilityIndex("other", *testOtherCapability) + + moreAddr := RandomBzzAddr() + moreAddr.Capabilities.Add(testMoreCapability) + + lessAddr := RandomBzzAddr() + lessAddr.Capabilities.Add(testLessCapability) + + otherAddr := RandomBzzAddr() + otherAddr.Capabilities.Add(testOtherCapability) + + allAddr := RandomBzzAddr() + allAddr.Capabilities.Add(testOtherCapability) + allAddr.Capabilities.Add(testMoreCapability) + + k.Register(moreAddr, lessAddr, otherAddr, allAddr) + + var addrs []*BzzAddr + k.EachAddr(base, 255, func(a *BzzAddr, _ int) bool { + addrs = append(addrs, a) + return true + }) + if len(addrs) != 4 { + t.Fatalf("EachAddr expected 3 peers, got %d", len(addrs)) + } + + var c int + k.EachAddrFiltered(base, "more", 255, func(a *BzzAddr, _ int) bool { + c++ + cp := a.Capabilities.Get(42) + if !cp.Match(testMoreCapability) { + t.Fatalf("EachAddrFiltered 'more' capability mismatch, expected %v, got %v", testMoreCapability, cp) + } + return true + }) + if c != 2 { + t.Fatalf("EachAddrFiltered 'full' expected 2 peer, got %d", c) + } + + c = 0 + k.EachAddrFiltered(base, "less", 255, func(a *BzzAddr, _ int) bool { + c++ + return true + }) + if c != 3 { + t.Fatalf("EachAddrFiltered 'less' expected 2 peers, got %d", c) + } + + c = 0 + k.EachAddrFiltered(base, "none", 255, func(a *BzzAddr, _ int) bool { + c++ + return true + }) + if c != 0 { + t.Fatalf("EachAddrFiltered 'none' expected 0 peers, got %d", c) + } + + c = 0 + k.EachAddrFiltered(base, "other", 255, func(a *BzzAddr, _ int) bool { + c++ + cp := a.Capabilities.Get(666) + if !cp.Match(testOtherCapability) { + t.Fatalf("EachAddrFiltered 'other' capability mismatch, expected %v, got %v", testOtherCapability, cp) + } + cp = a.Capabilities.Get(42) + if cp != nil { + c++ + } + return true + }) + if c != 3 { + t.Fatalf("EachAddrFiltered 'other' expected 3 capability matches, got %d", c) + } +} + // TestHealthStrict tests the simplest definition of health // Which means whether we are connected to all neighbors we know of func TestHealthStrict(t *testing.T) { From 4cc3da168767d6d2a14ec24ce91d1b783726ae36 Mon Sep 17 00:00:00 2001 From: nolash Date: Fri, 13 Sep 2019 11:56:00 +0200 Subject: [PATCH 23/36] network: Add test for connections capabilities index filtering --- network/kademlia_test.go | 78 +++++++++++++++++++++++++++------------- 1 file changed, 54 insertions(+), 24 deletions(-) diff --git a/network/kademlia_test.go b/network/kademlia_test.go index 6e24fafd3e..6f86e156c1 100644 --- a/network/kademlia_test.go +++ b/network/kademlia_test.go @@ -36,13 +36,9 @@ func init() { log.Root().SetHandler(h) } -func testKadPeerAddr(s string, lightNode bool) *BzzAddr { +func testKadPeerAddr(s string) *BzzAddr { a := pot.NewAddressFromString(s) bzzAddr := NewBzzAddr(a, a) - if lightNode { - bzzAddr.Capabilities.Add(lightCapability) - } - bzzAddr.Capabilities.Add(fullCapability) return bzzAddr } @@ -58,11 +54,6 @@ type testKademlia struct { t *testing.T } -type testKadAddrSpec struct { - bits string - lightNode bool -} - func newTestKademlia(t *testing.T, b string) *testKademlia { base := pot.NewAddressFromString(b) return &testKademlia{ @@ -72,7 +63,7 @@ func newTestKademlia(t *testing.T, b string) *testKademlia { } func (tk *testKademlia) newTestKadPeer(s string, lightNode bool) *Peer { - return NewPeer(&BzzPeer{BzzAddr: testKadPeerAddr(s, lightNode), LightNode: lightNode}, tk.Kademlia) + return NewPeer(&BzzPeer{BzzAddr: testKadPeerAddr(s), LightNode: lightNode}, tk.Kademlia) } func (tk *testKademlia) On(ons ...string) { @@ -88,20 +79,9 @@ func (tk *testKademlia) Off(offs ...string) { } func (tk *testKademlia) Register(regs ...string) { - var specs []testKadAddrSpec - for _, r := range regs { - specs = append(specs, testKadAddrSpec{ - bits: r, - lightNode: false, - }) - } - tk.RegisterWithCapability(specs...) -} - -func (tk *testKademlia) RegisterWithCapability(regs ...testKadAddrSpec) { var as []*BzzAddr - for _, r := range regs { - as = append(as, testKadPeerAddr(r.bits, r.lightNode)) + for _, s := range regs { + as = append(as, testKadPeerAddr(s)) } err := tk.Kademlia.Register(as...) if err != nil { @@ -230,6 +210,7 @@ func TestHighMinBinSize(t *testing.T) { } } +// TestCapabilitiesIndex checks that capability indices contains only the peers that have the filters' capability bits set func TestCapabilitiesIndex(t *testing.T) { kp := NewKadParams() addr := RandomBzzAddr() @@ -325,6 +306,55 @@ func TestCapabilitiesIndex(t *testing.T) { if c != 3 { t.Fatalf("EachAddrFiltered 'other' expected 3 capability matches, got %d", c) } + + allBzzPeer := &BzzPeer{ + BzzAddr: allAddr, + } + allPeer := NewPeer(allBzzPeer, k) + k.On(allPeer) + + c = 0 + k.EachConn(base, 255, func(_ *Peer, _ int) bool { + c++ + return true + }) + if c != 1 { + t.Fatalf("EachConn expected 1 peer, got %d", c) + } + + c = 0 + k.EachConnFiltered(base, "other", 255, func(p *Peer, _ int) bool { + c++ + cp := p.Capabilities.Get(666) + if !cp.Match(testOtherCapability) { + t.Fatalf("EachConnFiltered 'other' missing capability %v", testOtherCapability) + } + cp = p.Capabilities.Get(42) + if !cp.Match(testMoreCapability) { + t.Fatalf("EachConnFiltered 'other' missing capability %v", testMoreCapability) + } + return true + }) + if c != 1 { + t.Fatalf("EachConnFiltered 'other' expected 1 peer, got %d", c) + } + + c = 0 + k.EachConnFiltered(base, "more", 255, func(p *Peer, _ int) bool { + c++ + cp := p.Capabilities.Get(666) + if !cp.Match(testOtherCapability) { + t.Fatalf("EachConnFiltered 'more' missing capability %v", testOtherCapability) + } + cp = p.Capabilities.Get(42) + if !cp.Match(testMoreCapability) { + t.Fatalf("EachConnFiltered 'more' missing capability %v", testMoreCapability) + } + return true + }) + if c != 1 { + t.Fatalf("EachConnFiltered 'more' expected 1 peer, got %d", c) + } } // TestHealthStrict tests the simplest definition of health From bedced017e07ef6fb08293cdb070db5e397ba996 Mon Sep 17 00:00:00 2001 From: nolash Date: Fri, 13 Sep 2019 12:19:50 +0200 Subject: [PATCH 24/36] network: Add comments to capabilities index test --- network/kademlia_test.go | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/network/kademlia_test.go b/network/kademlia_test.go index 6f86e156c1..cfb18f304e 100644 --- a/network/kademlia_test.go +++ b/network/kademlia_test.go @@ -38,8 +38,7 @@ func init() { func testKadPeerAddr(s string) *BzzAddr { a := pot.NewAddressFromString(s) - bzzAddr := NewBzzAddr(a, a) - return bzzAddr + return NewBzzAddr(a, a) } func newTestKademliaParams() *KadParams { @@ -217,19 +216,23 @@ func TestCapabilitiesIndex(t *testing.T) { base := addr.OAddr k := NewKademlia(base, kp) + // "more" matches "more" only testMoreCapability := capability.NewCapability(42, 3) testMoreCapability.Set(0) testMoreCapability.Set(2) k.RegisterCapabilityIndex("more", *testMoreCapability) + // "less" matches "more" and "less" testLessCapability := capability.NewCapability(42, 3) testLessCapability.Set(2) k.RegisterCapabilityIndex("less", *testLessCapability) + // "none" matches neither "more" nor "less" testNoneCapability := capability.NewCapability(42, 3) testNoneCapability.Set(1) k.RegisterCapabilityIndex("none", *testNoneCapability) + // "other" is a different capability array testOtherCapability := capability.NewCapability(666, 3) testOtherCapability.Set(0) testOtherCapability.Set(2) @@ -244,22 +247,27 @@ func TestCapabilitiesIndex(t *testing.T) { otherAddr := RandomBzzAddr() otherAddr.Capabilities.Add(testOtherCapability) + // all includes two different capability arrays allAddr := RandomBzzAddr() allAddr.Capabilities.Add(testOtherCapability) allAddr.Capabilities.Add(testMoreCapability) + // proceed to check the matches, first for the "known peers" pot + // first adding the different peer configurations to the kademli k.Register(moreAddr, lessAddr, otherAddr, allAddr) - var addrs []*BzzAddr - k.EachAddr(base, 255, func(a *BzzAddr, _ int) bool { - addrs = append(addrs, a) + // Call without filter should still return all peers + var c int + k.EachAddr(base, 255, func(_ *BzzAddr, _ int) bool { + c++ return true }) - if len(addrs) != 4 { - t.Fatalf("EachAddr expected 3 peers, got %d", len(addrs)) + if c != 4 { + t.Fatalf("EachAddr expected 3 peers, got %d", c) } - var c int + // Matches "more", "all" + c = 0 k.EachAddrFiltered(base, "more", 255, func(a *BzzAddr, _ int) bool { c++ cp := a.Capabilities.Get(42) @@ -272,6 +280,7 @@ func TestCapabilitiesIndex(t *testing.T) { t.Fatalf("EachAddrFiltered 'full' expected 2 peer, got %d", c) } + // Matches "more", "less", "all" c = 0 k.EachAddrFiltered(base, "less", 255, func(a *BzzAddr, _ int) bool { c++ @@ -281,6 +290,7 @@ func TestCapabilitiesIndex(t *testing.T) { t.Fatalf("EachAddrFiltered 'less' expected 2 peers, got %d", c) } + // No matches c = 0 k.EachAddrFiltered(base, "none", 255, func(a *BzzAddr, _ int) bool { c++ @@ -290,6 +300,8 @@ func TestCapabilitiesIndex(t *testing.T) { t.Fatalf("EachAddrFiltered 'none' expected 0 peers, got %d", c) } + // Matches "other", "all" + // Also checks that "all" has both capabilities c = 0 k.EachAddrFiltered(base, "other", 255, func(a *BzzAddr, _ int) bool { c++ @@ -307,12 +319,15 @@ func TestCapabilitiesIndex(t *testing.T) { t.Fatalf("EachAddrFiltered 'other' expected 3 capability matches, got %d", c) } + // Now check the connection pot index + // We add the "all" peer allBzzPeer := &BzzPeer{ BzzAddr: allAddr, } allPeer := NewPeer(allBzzPeer, k) k.On(allPeer) + // Call without filter should return the single connected peer c = 0 k.EachConn(base, 255, func(_ *Peer, _ int) bool { c++ @@ -322,6 +337,7 @@ func TestCapabilitiesIndex(t *testing.T) { t.Fatalf("EachConn expected 1 peer, got %d", c) } + // Check that the peer exists in the indices for both capability arrays c = 0 k.EachConnFiltered(base, "other", 255, func(p *Peer, _ int) bool { c++ From 0355266adf6c907e8052773e7bfc700a901b5bd3 Mon Sep 17 00:00:00 2001 From: nolash Date: Fri, 13 Sep 2019 12:59:30 +0200 Subject: [PATCH 25/36] newtork: Add comments, stub for capabilities index removal --- network/capability/capability.go | 1 + network/discovery.go | 3 +-- network/hive_test.go | 8 +++----- network/kademlia.go | 28 ++++++++++++++++++++++++---- network/network.go | 15 ++++++++++++++- network/network_test.go | 1 + network/protocol.go | 2 +- network/protocol_test.go | 1 + pot/address.go | 4 ++-- 9 files changed, 48 insertions(+), 15 deletions(-) diff --git a/network/capability/capability.go b/network/capability/capability.go index 6f9415b9d6..10f458f493 100644 --- a/network/capability/capability.go +++ b/network/capability/capability.go @@ -200,6 +200,7 @@ func (c *Capability) Match(capCompare *Capability) bool { return true } +// Match returns true if all bits set in all capability arguments are also set in the receiver's capabilities func (c *Capabilities) Match(capsCompare *Capabilities) bool { for _, capCompare := range capsCompare.Caps { diff --git a/network/discovery.go b/network/discovery.go index 941e1b859f..87a55e3eb6 100644 --- a/network/discovery.go +++ b/network/discovery.go @@ -206,8 +206,7 @@ func (d *Peer) handleSubPeersMsg(msg *subPeersMsg) error { }) // if useful peers are found, send them over if len(peers) > 0 { - outMsg := &peersMsg{Peers: sortPeers(peers)} - go d.Send(context.TODO(), outMsg) + go d.Send(context.TODO(), &peersMsg{Peers: sortPeers(peers)}) } } d.sentPeers = true diff --git a/network/hive_test.go b/network/hive_test.go index 919e5c0afb..670d1727f0 100644 --- a/network/hive_test.go +++ b/network/hive_test.go @@ -24,7 +24,6 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/p2p" - "github.com/ethersphere/swarm/network/capability" p2ptest "github.com/ethersphere/swarm/p2p/testing" "github.com/ethersphere/swarm/state" ) @@ -101,7 +100,7 @@ func TestRegisterAndConnect(t *testing.T) { } } -// TestHiveStatePersistance creates a protocol simulation with n peers for a node +// TestHiveStatePersistence creates a protocol simulation with n peers for a node // After protocols complete, the node is shut down and the state is stored. // Another simulation is created, where 0 nodes are created, but where the stored state is passed // The test succeeds if all the peers from the stored state are known after the protocols of the @@ -109,8 +108,7 @@ func TestRegisterAndConnect(t *testing.T) { // // Actual connectivity is not in scope for this test, as the peers loaded from state are not known to // the simulation; the test only verifies that the peers are known to the node -func TestHiveStatePersistance(t *testing.T) { - // t.Skip("persistence test does not retrieve capabilities properly") +func TestHiveStatePersistence(t *testing.T) { dir, err := ioutil.TempDir("", "hive_test_store") if err != nil { t.Fatal(err) @@ -154,7 +152,7 @@ func TestHiveStatePersistance(t *testing.T) { h1, cleanup1 := startHive(t, dir) peers := make(map[string]bool) for i := 0; i < peersCount; i++ { - raddr := RandomBzzAddr().WithCapabilities(capability.NewCapabilities()) + raddr := RandomBzzAddr() h1.Register(raddr) peers[raddr.String()] = true } diff --git a/network/kademlia.go b/network/kademlia.go index f2143ca371..7a48b1cda4 100644 --- a/network/kademlia.go +++ b/network/kademlia.go @@ -126,6 +126,9 @@ func NewKademlia(addr []byte, params *KadParams) *Kademlia { return k } +// RegisterCapabilityIndex adds an entry to the capability index of the kademlia +// The capability index is associated with the supplied string s +// Any peers matching any bits set in the capability in the index, will be added to the index (or removed on removal) func (k *Kademlia) RegisterCapabilityIndex(s string, c capability.Capability) error { if s == "" { return errors.New("Cannot add index with empty string key") @@ -137,7 +140,7 @@ func (k *Kademlia) RegisterCapabilityIndex(s string, c capability.Capability) er return nil } -// blindly add index to +// adds a peer to any capability indices it matches func (k *Kademlia) addToCapabilityIndex(p interface{}, connected bool) { var eAddr *BzzAddr var ePeer *Peer @@ -165,6 +168,22 @@ func (k *Kademlia) addToCapabilityIndex(p interface{}, connected bool) { } } +// removes a peer from any capability indices it matches +func (k *Kademlia) removeFromCapabilityIndex(p interface{}, disconnectOnly bool) { + var ok bool + var eAddr *BzzAddr + var ePeer *Peer + ePeer, ok = p.(*Peer) + if ok { + eAddr = ePeer.BzzAddr + } else if disconnectOnly { + return + } else { + eAddr = p.(*entry).BzzAddr + } + _ = eAddr +} + // entry represents a Kademlia table entry (an extension of BzzAddr) type entry struct { *BzzAddr @@ -485,9 +504,8 @@ func (k *Kademlia) Off(p *Peer) { k.setNeighbourhoodDepth() } -// EachConn is an iterator with args (base, po, f) applies f to each live peer -// that has proximity order po or less as measured from the base -// if base is nil, kademlia base address is used +// EachConnFiltered performs the same action as EachConn +// with the difference that it will only return peers that matches the specified capability index filter func (k *Kademlia) EachConnFiltered(base []byte, capKey string, o int, f func(*Peer, int) bool) error { k.lock.RLock() defer k.lock.RUnlock() @@ -523,6 +541,8 @@ func (k *Kademlia) eachConn(base []byte, db *pot.Pot, o int, f func(*Peer, int) }) } +// EachAddrFiltered performs the same action as EachAddr +// with the difference that it will only return peers that matches the specified capability index filter func (k *Kademlia) EachAddrFiltered(base []byte, capKey string, o int, f func(*BzzAddr, int) bool) error { k.lock.RLock() defer k.lock.RUnlock() diff --git a/network/network.go b/network/network.go index 95c2cfa41b..541d05b836 100644 --- a/network/network.go +++ b/network/network.go @@ -20,6 +20,7 @@ type BzzAddr struct { Capabilities *capability.Capabilities } +// EncodeRLP implements rlp.Encoder func (b *BzzAddr) EncodeRLP(w io.Writer) error { err := rlp.Encode(w, b.OAddr) if err != nil { @@ -40,6 +41,7 @@ func (b *BzzAddr) EncodeRLP(w io.Writer) error { return nil } +// EncodeRLP implements rlp.Decoder func (b *BzzAddr) DecodeRLP(s *rlp.Stream) error { var err error @@ -63,6 +65,8 @@ func (b *BzzAddr) DecodeRLP(s *rlp.Stream) error { return nil } +// NewBzzAddr creates a new BzzAddr with the specified byte values for over- and underlayaddresses +// It will contain an empty capabilities object func NewBzzAddr(oaddr []byte, uaddr []byte) *BzzAddr { return &BzzAddr{ OAddr: oaddr, @@ -105,7 +109,8 @@ func (a *BzzAddr) String() string { return fmt.Sprintf("%x <%s> cap:%s", a.OAddr, a.UAddr, a.Capabilities) } -// RandomBzzAddr is a utility method generating an address from a public key +// RandomBzzAddr is a utility method generating a private key and corresponding enode id +// It in turn calls NewBzzAddrFromEnode to generate a corresponding overlay address from enode func RandomBzzAddr() *BzzAddr { key, err := crypto.GenerateKey() if err != nil { @@ -115,20 +120,26 @@ func RandomBzzAddr() *BzzAddr { return NewBzzAddrFromEnode(node) } +// NewBzzAddrFromEnode creates a BzzAddr where the overlay address is the byte representation of the enode i +// It is only used for test purposes +// TODO: This method should be replaced by (optionally deterministic) generation of addresses using NewEnode and PrivateKeyToBzzKey func NewBzzAddrFromEnode(enod *enode.Node) *BzzAddr { return &BzzAddr{OAddr: enod.ID().Bytes(), UAddr: []byte(enod.URLv4()), Capabilities: capability.NewCapabilities()} } +// WithCapabilities is a chained constructor method to set the capabilities array for a BzzAddr func (b *BzzAddr) WithCapabilities(c *capability.Capabilities) *BzzAddr { b.Capabilities = c return b } +// PrivateKeyToBzzKey create a swarm overlay address from the given private key func PrivateKeyToBzzKey(prvKey *ecdsa.PrivateKey) []byte { pubkeyBytes := crypto.FromECDSAPub(&prvKey.PublicKey) return crypto.Keccak256Hash(pubkeyBytes).Bytes() } +// EnodeParams contains the parameters used to create new Enode Records type EnodeParams struct { PrivateKey *ecdsa.PrivateKey EnodeKey *ecdsa.PrivateKey @@ -136,6 +147,7 @@ type EnodeParams struct { Bootnode bool } +// NewEnodeRecord creates a new valid swarm node ENR record from the given parameters func NewEnodeRecord(params *EnodeParams) (*enr.Record, error) { if params.PrivateKey == nil { @@ -150,6 +162,7 @@ func NewEnodeRecord(params *EnodeParams) (*enr.Record, error) { return &record, nil } +// NewEnode creates a new enode object for the given parameters func NewEnode(params *EnodeParams) (*enode.Node, error) { record, err := NewEnodeRecord(params) if err != nil { diff --git a/network/network_test.go b/network/network_test.go index e8e75e060a..3638bbd04f 100644 --- a/network/network_test.go +++ b/network/network_test.go @@ -7,6 +7,7 @@ import ( "github.com/ethersphere/swarm/network/capability" ) +// TestBzzAddrRLPSerialzation verifies reverisibility of RLP serialization of BzzAddr func TestBzzAddrRLPSerialization(t *testing.T) { caps := capability.NewCapabilities() caps.Add(lightCapability) diff --git a/network/protocol.go b/network/protocol.go index afe7e7f754..3d4027c173 100644 --- a/network/protocol.go +++ b/network/protocol.go @@ -138,7 +138,7 @@ func NewBzz(config *BzzConfig, kad *Kademlia, store state.Store, streamerSpec, r bzz := &Bzz{ Hive: NewHive(config.HiveParams, kad, store), NetworkID: config.NetworkID, - localAddr: &BzzAddr{config.OverlayAddr, config.UnderlayAddr, capability.NewCapabilities()}, + localAddr: NewBzzAddr(config.OverlayAddr, config.UnderlayAddr), handshakes: make(map[enode.ID]*HandshakeMsg), streamerRun: streamerRun, streamerSpec: streamerSpec, diff --git a/network/protocol_test.go b/network/protocol_test.go index 4d9363587e..637667a60b 100644 --- a/network/protocol_test.go +++ b/network/protocol_test.go @@ -235,6 +235,7 @@ func correctBzzHandshake(addr *BzzAddr, lightNode bool) *HandshakeMsg { return newBzzHandshakeMsg(TestProtocolVersion, TestProtocolNetworkID, addr, lightNode) } +// TestBzzHandshakeRLPSerialization verifies the reversibility of RLP serialization of HandshakeMsg func TestBzzHandshakeRLPSerialization(t *testing.T) { caps := capability.NewCapabilities() caps.Add(fullCapability) diff --git a/pot/address.go b/pot/address.go index 7bb9f7c0d1..d7bed6c46e 100644 --- a/pot/address.go +++ b/pot/address.go @@ -134,7 +134,7 @@ func ProxCmp(a, x, y []byte) int { return 0 } -// RandomBzzAddressAt (address, prox) generates a random address +// RandomAddressAt (address, prox) generates a random address // at proximity order prox relative to address // if prox is negative a random address is generated func RandomAddressAt(self Address, prox int) (addr Address) { @@ -159,7 +159,7 @@ func RandomAddressAt(self Address, prox int) (addr Address) { return } -// RandomBzzAddress generates a random address +// RandomAddress generates a random address func RandomAddress() Address { return RandomAddressAt(Address{}, -1) } From 297eae39c8142d4c9cc2fe2b753390551ee02782 Mon Sep 17 00:00:00 2001 From: nolash Date: Fri, 13 Sep 2019 14:34:37 +0200 Subject: [PATCH 26/36] network: Complete remove from index method and corresponding test --- network/kademlia.go | 20 +- network/kademlia_test.go | 422 ++++++++++++++++++++++++--------------- pot/pot.go | 1 + 3 files changed, 278 insertions(+), 165 deletions(-) diff --git a/network/kademlia.go b/network/kademlia.go index 7a48b1cda4..eb45229861 100644 --- a/network/kademlia.go +++ b/network/kademlia.go @@ -181,7 +181,24 @@ func (k *Kademlia) removeFromCapabilityIndex(p interface{}, disconnectOnly bool) } else { eAddr = p.(*entry).BzzAddr } - _ = eAddr + for _, idxItem := range k.capabilityIndex { + if ok { + conns, _, found, _ := pot.Swap(idxItem.conns, ePeer, Pof, func(_ pot.Val) pot.Val { + return nil + }) + if found { + idxItem.conns = conns + } + } + if !disconnectOnly { + addrs, _, found, _ := pot.Swap(idxItem.addrs, eAddr, Pof, func(_ pot.Val) pot.Val { + return nil + }) + if found { + idxItem.addrs = addrs + } + } + } } // entry represents a Kademlia table entry (an extension of BzzAddr) @@ -501,6 +518,7 @@ func (k *Kademlia) Off(p *Peer) { // v cannot be nil, but no need to check return nil }) + k.removeFromCapabilityIndex(p, true) k.setNeighbourhoodDepth() } diff --git a/network/kademlia_test.go b/network/kademlia_test.go index cfb18f304e..82fe69d995 100644 --- a/network/kademlia_test.go +++ b/network/kademlia_test.go @@ -209,170 +209,6 @@ func TestHighMinBinSize(t *testing.T) { } } -// TestCapabilitiesIndex checks that capability indices contains only the peers that have the filters' capability bits set -func TestCapabilitiesIndex(t *testing.T) { - kp := NewKadParams() - addr := RandomBzzAddr() - base := addr.OAddr - k := NewKademlia(base, kp) - - // "more" matches "more" only - testMoreCapability := capability.NewCapability(42, 3) - testMoreCapability.Set(0) - testMoreCapability.Set(2) - k.RegisterCapabilityIndex("more", *testMoreCapability) - - // "less" matches "more" and "less" - testLessCapability := capability.NewCapability(42, 3) - testLessCapability.Set(2) - k.RegisterCapabilityIndex("less", *testLessCapability) - - // "none" matches neither "more" nor "less" - testNoneCapability := capability.NewCapability(42, 3) - testNoneCapability.Set(1) - k.RegisterCapabilityIndex("none", *testNoneCapability) - - // "other" is a different capability array - testOtherCapability := capability.NewCapability(666, 3) - testOtherCapability.Set(0) - testOtherCapability.Set(2) - k.RegisterCapabilityIndex("other", *testOtherCapability) - - moreAddr := RandomBzzAddr() - moreAddr.Capabilities.Add(testMoreCapability) - - lessAddr := RandomBzzAddr() - lessAddr.Capabilities.Add(testLessCapability) - - otherAddr := RandomBzzAddr() - otherAddr.Capabilities.Add(testOtherCapability) - - // all includes two different capability arrays - allAddr := RandomBzzAddr() - allAddr.Capabilities.Add(testOtherCapability) - allAddr.Capabilities.Add(testMoreCapability) - - // proceed to check the matches, first for the "known peers" pot - // first adding the different peer configurations to the kademli - k.Register(moreAddr, lessAddr, otherAddr, allAddr) - - // Call without filter should still return all peers - var c int - k.EachAddr(base, 255, func(_ *BzzAddr, _ int) bool { - c++ - return true - }) - if c != 4 { - t.Fatalf("EachAddr expected 3 peers, got %d", c) - } - - // Matches "more", "all" - c = 0 - k.EachAddrFiltered(base, "more", 255, func(a *BzzAddr, _ int) bool { - c++ - cp := a.Capabilities.Get(42) - if !cp.Match(testMoreCapability) { - t.Fatalf("EachAddrFiltered 'more' capability mismatch, expected %v, got %v", testMoreCapability, cp) - } - return true - }) - if c != 2 { - t.Fatalf("EachAddrFiltered 'full' expected 2 peer, got %d", c) - } - - // Matches "more", "less", "all" - c = 0 - k.EachAddrFiltered(base, "less", 255, func(a *BzzAddr, _ int) bool { - c++ - return true - }) - if c != 3 { - t.Fatalf("EachAddrFiltered 'less' expected 2 peers, got %d", c) - } - - // No matches - c = 0 - k.EachAddrFiltered(base, "none", 255, func(a *BzzAddr, _ int) bool { - c++ - return true - }) - if c != 0 { - t.Fatalf("EachAddrFiltered 'none' expected 0 peers, got %d", c) - } - - // Matches "other", "all" - // Also checks that "all" has both capabilities - c = 0 - k.EachAddrFiltered(base, "other", 255, func(a *BzzAddr, _ int) bool { - c++ - cp := a.Capabilities.Get(666) - if !cp.Match(testOtherCapability) { - t.Fatalf("EachAddrFiltered 'other' capability mismatch, expected %v, got %v", testOtherCapability, cp) - } - cp = a.Capabilities.Get(42) - if cp != nil { - c++ - } - return true - }) - if c != 3 { - t.Fatalf("EachAddrFiltered 'other' expected 3 capability matches, got %d", c) - } - - // Now check the connection pot index - // We add the "all" peer - allBzzPeer := &BzzPeer{ - BzzAddr: allAddr, - } - allPeer := NewPeer(allBzzPeer, k) - k.On(allPeer) - - // Call without filter should return the single connected peer - c = 0 - k.EachConn(base, 255, func(_ *Peer, _ int) bool { - c++ - return true - }) - if c != 1 { - t.Fatalf("EachConn expected 1 peer, got %d", c) - } - - // Check that the peer exists in the indices for both capability arrays - c = 0 - k.EachConnFiltered(base, "other", 255, func(p *Peer, _ int) bool { - c++ - cp := p.Capabilities.Get(666) - if !cp.Match(testOtherCapability) { - t.Fatalf("EachConnFiltered 'other' missing capability %v", testOtherCapability) - } - cp = p.Capabilities.Get(42) - if !cp.Match(testMoreCapability) { - t.Fatalf("EachConnFiltered 'other' missing capability %v", testMoreCapability) - } - return true - }) - if c != 1 { - t.Fatalf("EachConnFiltered 'other' expected 1 peer, got %d", c) - } - - c = 0 - k.EachConnFiltered(base, "more", 255, func(p *Peer, _ int) bool { - c++ - cp := p.Capabilities.Get(666) - if !cp.Match(testOtherCapability) { - t.Fatalf("EachConnFiltered 'more' missing capability %v", testOtherCapability) - } - cp = p.Capabilities.Get(42) - if !cp.Match(testMoreCapability) { - t.Fatalf("EachConnFiltered 'more' missing capability %v", testMoreCapability) - } - return true - }) - if c != 1 { - t.Fatalf("EachConnFiltered 'more' expected 1 peer, got %d", c) - } -} - // TestHealthStrict tests the simplest definition of health // Which means whether we are connected to all neighbors we know of func TestHealthStrict(t *testing.T) { @@ -807,3 +643,261 @@ func TestKademlia_SubscribeToNeighbourhoodDepthChange(t *testing.T) { } }) } + +// TestCapabilitiesIndex checks that capability indices contains only the peers that have the filters' capability bits set +func TestCapabilityIndex(t *testing.T) { + kp := NewKadParams() + addr := RandomBzzAddr() + base := addr.OAddr + k := NewKademlia(base, kp) + + // "more" matches "more" only + testMoreCapability := capability.NewCapability(42, 3) + testMoreCapability.Set(0) + testMoreCapability.Set(2) + k.RegisterCapabilityIndex("more", *testMoreCapability) + + // "less" matches "more" and "less" + testLessCapability := capability.NewCapability(42, 3) + testLessCapability.Set(2) + k.RegisterCapabilityIndex("less", *testLessCapability) + + // "none" matches neither "more" nor "less" + testNoneCapability := capability.NewCapability(42, 3) + testNoneCapability.Set(1) + k.RegisterCapabilityIndex("none", *testNoneCapability) + + // "other" is a different capability array + testOtherCapability := capability.NewCapability(666, 3) + testOtherCapability.Set(0) + testOtherCapability.Set(2) + k.RegisterCapabilityIndex("other", *testOtherCapability) + + moreAddr := RandomBzzAddr() + moreAddr.Capabilities.Add(testMoreCapability) + + lessAddr := RandomBzzAddr() + lessAddr.Capabilities.Add(testLessCapability) + + otherAddr := RandomBzzAddr() + otherAddr.Capabilities.Add(testOtherCapability) + + // all includes two different capability arrays + allAddr := RandomBzzAddr() + allAddr.Capabilities.Add(testOtherCapability) + allAddr.Capabilities.Add(testMoreCapability) + + // proceed to check the matches, first for the "known peers" pot + // first adding the different peer configurations to the kademli + k.Register(moreAddr, lessAddr, otherAddr, allAddr) + + // Call without filter should still return all peers + var c int + k.EachAddr(base, 255, func(_ *BzzAddr, _ int) bool { + c++ + return true + }) + if c != 4 { + t.Fatalf("EachAddr expected 3 peers, got %d", c) + } + + // Matches "more", "all" + c = 0 + k.EachAddrFiltered(base, "more", 255, func(a *BzzAddr, _ int) bool { + c++ + cp := a.Capabilities.Get(42) + if !cp.Match(testMoreCapability) { + t.Fatalf("EachAddrFiltered 'more' capability mismatch, expected %v, got %v", testMoreCapability, cp) + } + return true + }) + if c != 2 { + t.Fatalf("EachAddrFiltered 'full' expected 2 peer, got %d", c) + } + + // Matches "more", "less", "all" + c = 0 + k.EachAddrFiltered(base, "less", 255, func(a *BzzAddr, _ int) bool { + c++ + return true + }) + if c != 3 { + t.Fatalf("EachAddrFiltered 'less' expected 2 peers, got %d", c) + } + + // No matches + c = 0 + k.EachAddrFiltered(base, "none", 255, func(a *BzzAddr, _ int) bool { + c++ + return true + }) + if c != 0 { + t.Fatalf("EachAddrFiltered 'none' expected 0 peers, got %d", c) + } + + // Matches "other", "all" + // Also checks that "all" has both capabilities + c = 0 + k.EachAddrFiltered(base, "other", 255, func(a *BzzAddr, _ int) bool { + c++ + cp := a.Capabilities.Get(666) + if !cp.Match(testOtherCapability) { + t.Fatalf("EachAddrFiltered 'other' capability mismatch, expected %v, got %v", testOtherCapability, cp) + } + cp = a.Capabilities.Get(42) + if cp != nil { + c++ + } + return true + }) + if c != 3 { + t.Fatalf("EachAddrFiltered 'other' expected 3 capability matches, got %d", c) + } + + // Now check the connection pot index + // We add the "all" and "less" peers + allBzzPeer := &BzzPeer{ + BzzAddr: allAddr, + } + allPeer := NewPeer(allBzzPeer, k) + k.On(allPeer) + + lessBzzPeer := &BzzPeer{ + BzzAddr: lessAddr, + } + lessPeer := NewPeer(lessBzzPeer, k) + k.On(lessPeer) + + // Call without filter should return the single connected peer + c = 0 + k.EachConn(base, 255, func(_ *Peer, _ int) bool { + c++ + return true + }) + if c != 2 { + t.Fatalf("EachConn expected 2 peers, got %d", c) + } + + // Check that the "all" peer exists in the indices for both capability arrays + // first the "other" index ... + c = 0 + k.EachConnFiltered(base, "other", 255, func(p *Peer, _ int) bool { + c++ + cp := p.Capabilities.Get(666) + if !cp.Match(testOtherCapability) { + t.Fatalf("EachConnFiltered 'other' missing capability %v", testOtherCapability) + } + cp = p.Capabilities.Get(42) + if !cp.Match(testMoreCapability) { + t.Fatalf("EachConnFiltered 'other' missing capability %v", testMoreCapability) + } + return true + }) + if c != 1 { + t.Fatalf("EachConnFiltered 'other' expected 1 peer, got %d", c) + } + + // ...then the "more" index + c = 0 + k.EachConnFiltered(base, "more", 255, func(p *Peer, _ int) bool { + c++ + cp := p.Capabilities.Get(666) + if !cp.Match(testOtherCapability) { + t.Fatalf("EachConnFiltered 'more' missing capability %v", testOtherCapability) + } + cp = p.Capabilities.Get(42) + if !cp.Match(testMoreCapability) { + t.Fatalf("EachConnFiltered 'more' missing capability %v", testMoreCapability) + } + return true + }) + if c != 1 { + t.Fatalf("EachConnFiltered 'more' expected 1 peer, got %d", c) + } + + // Disconnect the "all" peer + k.Off(allPeer) + + // Check that the "all" is now removed from connections + c = 0 + k.EachConnFiltered(base, "other", 255, func(_ *Peer, _ int) bool { + c++ + return true + }) + if c != 0 { + t.Fatalf("EachConnFiltered 'other' expected 0 peers, got %d", c) + } + + // Check that there is still an "other" peer among known peers + // (the two matched peers will be "all" and "other") + c = 0 + k.EachAddrFiltered(base, "other", 255, func(_ *BzzAddr, _ int) bool { + c++ + return true + }) + if c != 2 { + t.Fatalf("EachAddrFiltered 'other' expected 2 peers, got %d", c) + } + + // Check that the "less" peer is still registered as connected + c = 0 + k.EachConnFiltered(base, "less", 255, func(p *Peer, _ int) bool { + c++ + cp := p.Capabilities.Get(42) + if !cp.Match(testLessCapability) { + t.Fatalf("EachConnFiltered 'less' missing capability %v", testLessCapability) + } + return true + }) + if c != 1 { + t.Fatalf("EachConnFiltered 'less' expected 1 peer, got %d", c) + } + + // Remove "less" from both connection and known peers (pruning) list + // TODO replace with the "prune" method when one is implemented + k.removeFromCapabilityIndex(lessPeer, false) + + // Check that the "less" peer is no longer registered as connected + c = 0 + k.EachConnFiltered(base, "less", 255, func(p *Peer, _ int) bool { + c++ + return true + }) + if c != 0 { + t.Fatalf("EachConnFiltered 'less' expected 0 peers, got %d", c) + } + + // check that the "less" peer is not known anymore + // (the two matched peers will be "all" and "more") + c = 0 + k.EachAddrFiltered(base, "less", 255, func(p *BzzAddr, _ int) bool { + c++ + cp := p.Capabilities.Get(42) + if !cp.Match(testMoreCapability) { + t.Fatalf("EachConnFiltered 'less' should now return only capability 'more': %v", testMoreCapability) + } + return true + }) + if c != 2 { + t.Fatalf("EachAddrFiltered 'less' expected 2 peer, got %d", c) + } + + // Remove "all" from known peers list (pruning only) + // TODO replace with the "prune" method when one is implemented + k.removeFromCapabilityIndex(allPeer, false) + + // check that the "less" peer is not known anymore + // (the two matched peers will be "all" and "more") + c = 0 + k.EachAddrFiltered(base, "more", 255, func(p *BzzAddr, _ int) bool { + c++ + cp := p.Capabilities.Get(666) + if cp != nil { + t.Fatalf("EachAddrFiltered 'more' should not contain a peer with capabilitiy %v", testOtherCapability) + } + return true + }) + if c != 1 { + t.Fatalf("EachAddrFiltered 'more' expected 1 peer, got %d", c) + } +} diff --git a/pot/pot.go b/pot/pot.go index 9579f43056..b2964e09e4 100644 --- a/pot/pot.go +++ b/pot/pot.go @@ -209,6 +209,7 @@ func remove(t *Pot, val Val, pof Pof) (r *Pot, po int, found bool) { // if f(v) returns v' <> v then v' is inserted into the Pot // if (v) == v the Pot is not changed // it panics if Pof(f(v), k) show that v' and v are not key-equal +// BUG if "default" empty pot is supplied (created with NewPot(nil, 0), quieried address NOT found, then returned pot will be a nil value func Swap(t *Pot, k Val, pof Pof, f func(v Val) Val) (r *Pot, po int, found bool, change bool) { var val Val if t.pin == nil { From a9b5fe8e13a9cae6e4cd7bd5feef446e9bbb5195 Mon Sep 17 00:00:00 2001 From: nolash Date: Fri, 13 Sep 2019 14:39:50 +0200 Subject: [PATCH 27/36] network: Speling --- network/kademlia_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/kademlia_test.go b/network/kademlia_test.go index 82fe69d995..2bd032a007 100644 --- a/network/kademlia_test.go +++ b/network/kademlia_test.go @@ -893,7 +893,7 @@ func TestCapabilityIndex(t *testing.T) { c++ cp := p.Capabilities.Get(666) if cp != nil { - t.Fatalf("EachAddrFiltered 'more' should not contain a peer with capabilitiy %v", testOtherCapability) + t.Fatalf("EachAddrFiltered 'more' should not contain a peer with capability %v", testOtherCapability) } return true }) From a876276cd78be12c6dc02778fe0aca8c3d2bc491 Mon Sep 17 00:00:00 2001 From: nolash Date: Fri, 13 Sep 2019 15:19:56 +0200 Subject: [PATCH 28/36] network: Same structure of detect conn peer in add to index as remove --- network/kademlia.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/network/kademlia.go b/network/kademlia.go index eb45229861..4a897ffd8c 100644 --- a/network/kademlia.go +++ b/network/kademlia.go @@ -141,24 +141,24 @@ func (k *Kademlia) RegisterCapabilityIndex(s string, c capability.Capability) er } // adds a peer to any capability indices it matches -func (k *Kademlia) addToCapabilityIndex(p interface{}, connected bool) { +func (k *Kademlia) addToCapabilityIndex(p interface{}) { + var ok bool var eAddr *BzzAddr var ePeer *Peer - if connected { - ePeer = p.(*Peer) + ePeer, ok = p.(*Peer) + if ok { eAddr = ePeer.BzzAddr } else { eAddr = p.(*entry).BzzAddr } for s, idxItem := range k.capabilityIndex { for _, vCap := range eAddr.Capabilities.Caps { - log.Debug("in add to capability compare", "idx", s, "peer", vCap, "idx", idxItem.Capability) if idxItem.Id != vCap.Id { continue } if vCap.Match(idxItem.Capability) { - log.Debug("Added peer to capability index", "conn", connected, "s", s, "v", vCap, "p", p) - if connected { + log.Trace("Added peer to capability index", "conn", ok, "s", s, "v", vCap, "p", p) + if ok { k.capabilityIndex[s].conns, _, _ = pot.Add(idxItem.conns, ePeer, Pof) } else { k.capabilityIndex[s].addrs, _, _ = pot.Add(idxItem.addrs, newEntry(eAddr), Pof) @@ -181,12 +181,13 @@ func (k *Kademlia) removeFromCapabilityIndex(p interface{}, disconnectOnly bool) } else { eAddr = p.(*entry).BzzAddr } - for _, idxItem := range k.capabilityIndex { + for s, idxItem := range k.capabilityIndex { if ok { conns, _, found, _ := pot.Swap(idxItem.conns, ePeer, Pof, func(_ pot.Val) pot.Val { return nil }) if found { + log.Trace("Removed peer from capability conns index", "s", s, "p", ePeer) idxItem.conns = conns } } @@ -195,6 +196,7 @@ func (k *Kademlia) removeFromCapabilityIndex(p interface{}, disconnectOnly bool) return nil }) if found { + log.Trace("Removed peer from capability addrs index", "s", s, "p", eAddr) idxItem.addrs = addrs } } @@ -278,7 +280,7 @@ func (k *Kademlia) Register(peers ...*BzzAddr) error { return v }) - k.addToCapabilityIndex(newEntry(p), false) + k.addToCapabilityIndex(newEntry(p)) size++ } @@ -418,7 +420,7 @@ func (k *Kademlia) On(p *Peer) (uint8, bool) { // found among live peers, do nothing return v }) - k.addToCapabilityIndex(p, true) + k.addToCapabilityIndex(p) if ins { a := newEntry(p.BzzAddr) a.conn = p From 82b065da408178ebc46ccfdb7cba46d04c7c9b5e Mon Sep 17 00:00:00 2001 From: nolash Date: Fri, 13 Sep 2019 15:28:27 +0200 Subject: [PATCH 29/36] network: Bump protocol versions --- network/protocol.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/network/protocol.go b/network/protocol.go index 3d4027c173..c8b46e50f4 100644 --- a/network/protocol.go +++ b/network/protocol.go @@ -56,7 +56,7 @@ var DefaultTestNetworkID = rand.Uint64() // BzzSpec is the spec of the generic swarm handshake var BzzSpec = &protocols.Spec{ Name: "bzz", - Version: 12, + Version: 13, MaxMsgSize: 10 * 1024 * 1024, Messages: []interface{}{ HandshakeMsg{}, @@ -66,7 +66,7 @@ var BzzSpec = &protocols.Spec{ // DiscoverySpec is the spec for the bzz discovery subprotocols var DiscoverySpec = &protocols.Spec{ Name: "hive", - Version: 10, + Version: 11, MaxMsgSize: 10 * 1024 * 1024, Messages: []interface{}{ peersMsg{}, From 2790fdf1b57624b24d21afe95243aa0cb2b4f1bb Mon Sep 17 00:00:00 2001 From: nolash Date: Fri, 13 Sep 2019 16:08:43 +0200 Subject: [PATCH 30/36] network: Update test protocol version in handshake/bzz protocol tests --- network/protocol_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/protocol_test.go b/network/protocol_test.go index 637667a60b..b0686a1eb0 100644 --- a/network/protocol_test.go +++ b/network/protocol_test.go @@ -38,7 +38,7 @@ import ( ) const ( - TestProtocolVersion = 12 + TestProtocolVersion = 13 ) var TestProtocolNetworkID = DefaultTestNetworkID From 7d05df631a96c56cee36f6a6ef0dfe259222187b Mon Sep 17 00:00:00 2001 From: nolash Date: Mon, 16 Sep 2019 18:15:03 +0200 Subject: [PATCH 31/36] network: Remove lightnode from BzzPeer and cryptic nil in eachAddr --- network/kademlia.go | 2 +- network/kademlia_test.go | 8 ++++---- network/protocol.go | 2 -- network/retrieval/retrieve.go | 5 ----- 4 files changed, 5 insertions(+), 12 deletions(-) diff --git a/network/kademlia.go b/network/kademlia.go index 4a897ffd8c..d16eac967a 100644 --- a/network/kademlia.go +++ b/network/kademlia.go @@ -960,7 +960,7 @@ func (k *Kademlia) knowNeighbours(addrs [][]byte) (got bool, n int, missing [][] pm := make(map[string]bool) depth := depthForPot(k.conns, k.NeighbourhoodSize, k.base) // create a map with all peers at depth and deeper known in the kademlia - k.eachAddr(nil, nil, 255, func(p *BzzAddr, po int) bool { + k.eachAddr(nil, k.addrs, 255, func(p *BzzAddr, po int) bool { // in order deepest to shallowest compared to the kademlia base address // all bins (except self) are included (0 <= bin <= 255) if po < depth { diff --git a/network/kademlia_test.go b/network/kademlia_test.go index 2bd032a007..4d2e6fe76f 100644 --- a/network/kademlia_test.go +++ b/network/kademlia_test.go @@ -61,19 +61,19 @@ func newTestKademlia(t *testing.T, b string) *testKademlia { } } -func (tk *testKademlia) newTestKadPeer(s string, lightNode bool) *Peer { - return NewPeer(&BzzPeer{BzzAddr: testKadPeerAddr(s), LightNode: lightNode}, tk.Kademlia) +func (tk *testKademlia) newTestKadPeer(s string) *Peer { + return NewPeer(&BzzPeer{BzzAddr: testKadPeerAddr(s)}, tk.Kademlia) } func (tk *testKademlia) On(ons ...string) { for _, s := range ons { - tk.Kademlia.On(tk.newTestKadPeer(s, false)) + tk.Kademlia.On(tk.newTestKadPeer(s)) } } func (tk *testKademlia) Off(offs ...string) { for _, s := range offs { - tk.Kademlia.Off(tk.newTestKadPeer(s, false)) + tk.Kademlia.Off(tk.newTestKadPeer(s)) } } diff --git a/network/protocol.go b/network/protocol.go index c8b46e50f4..6721c6249f 100644 --- a/network/protocol.go +++ b/network/protocol.go @@ -264,7 +264,6 @@ func (b *Bzz) RunProtocol(spec *protocols.Spec, run func(*BzzPeer) error) func(* Peer: protocols.NewPeer(p, rw, spec), BzzAddr: handshake.peerAddr, lastActive: time.Now(), - LightNode: isLightCapability(handshake.peerAddr.Capabilities.Get(0)), // this is a temporary member kept until kademlia code accommodates Capabilities instead } log.Debug("peer created", "addr", handshake.peerAddr.String()) @@ -321,7 +320,6 @@ type BzzPeer struct { *protocols.Peer // represents the connection for online peers *BzzAddr // remote address -> implements Addr interface = protocols.Peer lastActive time.Time // time is updated whenever mutexes are releasing - LightNode bool } func NewBzzPeer(p *protocols.Peer) *BzzPeer { diff --git a/network/retrieval/retrieve.go b/network/retrieval/retrieve.go index d2d35dd7dd..9ab21ca41f 100644 --- a/network/retrieval/retrieve.go +++ b/network/retrieval/retrieve.go @@ -237,11 +237,6 @@ func (r *Retrieval) findPeer(ctx context.Context, req *storage.Request) (retPeer return true } - // skip light nodes, even though they support `bzz-retrieve` protocol - if p.LightNode { - return true - } - // do not send request back to peer who asked us. maybe merge with SkipPeer at some point if bytes.Equal(req.Origin.Bytes(), id.Bytes()) { return true From 61bf42c19767485dbff0bcf9c50077ad8a3c3a53 Mon Sep 17 00:00:00 2001 From: nolash Date: Mon, 16 Sep 2019 18:18:48 +0200 Subject: [PATCH 32/36] network: Speling --- network/kademlia_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/kademlia_test.go b/network/kademlia_test.go index 4d2e6fe76f..2d0afe32ca 100644 --- a/network/kademlia_test.go +++ b/network/kademlia_test.go @@ -688,7 +688,7 @@ func TestCapabilityIndex(t *testing.T) { allAddr.Capabilities.Add(testMoreCapability) // proceed to check the matches, first for the "known peers" pot - // first adding the different peer configurations to the kademli + // first adding the different peer configurations to the kademlia k.Register(moreAddr, lessAddr, otherAddr, allAddr) // Call without filter should still return all peers From 7f240598fb7490e3eeb972c28cc536a1b8f47342 Mon Sep 17 00:00:00 2001 From: nolash Date: Mon, 16 Sep 2019 19:07:57 +0200 Subject: [PATCH 33/36] network: Remove retrieval light node test --- network/retrieval/retrieve_test.go | 34 ++---------------------------- 1 file changed, 2 insertions(+), 32 deletions(-) diff --git a/network/retrieval/retrieve_test.go b/network/retrieval/retrieve_test.go index 1c810ea0c3..acde870f82 100644 --- a/network/retrieval/retrieve_test.go +++ b/network/retrieval/retrieve_test.go @@ -235,9 +235,8 @@ func TestRequestFromPeers(t *testing.T) { to := network.NewKademlia(addr.OAddr, network.NewKadParams()) protocolsPeer := protocols.NewPeer(p2p.NewPeer(dummyPeerID, "dummy", []p2p.Cap{{Name: "bzz-retrieve", Version: 1}}), nil, nil) peer := network.NewPeer(&network.BzzPeer{ - BzzAddr: network.RandomBzzAddr(), - LightNode: false, - Peer: protocolsPeer, + BzzAddr: network.RandomBzzAddr(), + Peer: protocolsPeer, }, to) to.On(peer) @@ -255,35 +254,6 @@ func TestRequestFromPeers(t *testing.T) { } } -// RequestFromPeers should not return light nodes -func TestRequestFromPeersWithLightNode(t *testing.T) { - dummyPeerID := enode.HexID("3431c3939e1ee2a6345e976a8234f9870152d64879f30bc272a074f6859e75e8") - - addr := network.RandomBzzAddr() - to := network.NewKademlia(addr.OAddr, network.NewKadParams()) - - protocolsPeer := protocols.NewPeer(p2p.NewPeer(dummyPeerID, "dummy", []p2p.Cap{{Name: "bzz-retrieve", Version: 1}}), nil, nil) - - // setting up a lightnode - peer := network.NewPeer(&network.BzzPeer{ - BzzAddr: network.RandomBzzAddr(), - LightNode: true, - Peer: protocolsPeer, - }, to) - - to.On(peer) - - r := New(to, nil, to.BaseAddr(), nil) - req := storage.NewRequest(storage.Address(hash0[:])) - - // making a request which should return with "no peer found" - _, err := r.findPeer(context.Background(), req) - - if err != ErrNoPeerFound { - t.Fatalf("expected '%v', got %v", ErrNoPeerFound, err) - } -} - //TestHasPriceImplementation is to check that Retrieval implements protocols.Prices func TestHasPriceImplementation(t *testing.T) { addr := network.RandomBzzAddr() From cb71212c27be4a5787590d2c0a577d4652ddbc7a Mon Sep 17 00:00:00 2001 From: nolash Date: Mon, 16 Sep 2019 22:35:25 +0200 Subject: [PATCH 34/36] network: Split up capability indices test --- network/kademlia_test.go | 258 +++++++++++++++++++++++---------------- 1 file changed, 150 insertions(+), 108 deletions(-) diff --git a/network/kademlia_test.go b/network/kademlia_test.go index 2d0afe32ca..b751b61552 100644 --- a/network/kademlia_test.go +++ b/network/kademlia_test.go @@ -645,55 +645,80 @@ func TestKademlia_SubscribeToNeighbourhoodDepthChange(t *testing.T) { } // TestCapabilitiesIndex checks that capability indices contains only the peers that have the filters' capability bits set +// It tests the state of the indices after registering, connecting, disconnecting and removing peers +// +// It sets up peers with capability arrays 42:101, 42:001 and 666:101, and registers these capabilities as filters in the kademlia +// It also sets up a peer with both capability arrays 42:101 and 666:101 +// Lastly it registers a filter for the capability 42:010 in the kademlia which will match no peers +// +// The tests are split up to make them easier to read func TestCapabilityIndex(t *testing.T) { + t.Run("register", testCapabilityIndexRegister) + t.Run("connect", testCapabilityIndexConnect) + t.Run("disconnect", testCapabilityIndexDisconnect) + t.Run("remove", testCapabilityIndexRemove) +} + +// set up capabilities and peers for each individual test +func testCapabilityIndexHelper() (*Kademlia, map[string]*Peer, map[string]*capability.Capability) { + + bzzAddrs := make(map[string]*BzzAddr) + discPeers := make(map[string]*Peer) + caps := make(map[string]*capability.Capability) + kp := NewKadParams() addr := RandomBzzAddr() base := addr.OAddr k := NewKademlia(base, kp) - // "more" matches "more" only - testMoreCapability := capability.NewCapability(42, 3) - testMoreCapability.Set(0) - testMoreCapability.Set(2) - k.RegisterCapabilityIndex("more", *testMoreCapability) + caps["42:101"] = capability.NewCapability(42, 3) + caps["42:101"].Set(0) + caps["42:101"].Set(2) + k.RegisterCapabilityIndex("42:101", *caps["42:101"]) - // "less" matches "more" and "less" - testLessCapability := capability.NewCapability(42, 3) - testLessCapability.Set(2) - k.RegisterCapabilityIndex("less", *testLessCapability) + caps["42:001"] = capability.NewCapability(42, 3) + caps["42:001"].Set(2) + k.RegisterCapabilityIndex("42:001", *caps["42:001"]) - // "none" matches neither "more" nor "less" - testNoneCapability := capability.NewCapability(42, 3) - testNoneCapability.Set(1) - k.RegisterCapabilityIndex("none", *testNoneCapability) + caps["42:010"] = capability.NewCapability(42, 3) + caps["42:010"].Set(1) + k.RegisterCapabilityIndex("42:010", *caps["42:010"]) - // "other" is a different capability array - testOtherCapability := capability.NewCapability(666, 3) - testOtherCapability.Set(0) - testOtherCapability.Set(2) - k.RegisterCapabilityIndex("other", *testOtherCapability) + caps["666:101"] = capability.NewCapability(666, 3) + caps["666:101"].Set(0) + caps["666:101"].Set(2) + k.RegisterCapabilityIndex("666:101", *caps["666:101"]) - moreAddr := RandomBzzAddr() - moreAddr.Capabilities.Add(testMoreCapability) + bzzAddrs["42:101"] = RandomBzzAddr() + bzzAddrs["42:101"].Capabilities.Add(caps["42:101"]) + discPeers["42:101"] = NewPeer(&BzzPeer{BzzAddr: bzzAddrs["42:101"]}, k) - lessAddr := RandomBzzAddr() - lessAddr.Capabilities.Add(testLessCapability) + bzzAddrs["42:001"] = RandomBzzAddr() + bzzAddrs["42:001"].Capabilities.Add(caps["42:001"]) + discPeers["42:001"] = NewPeer(&BzzPeer{BzzAddr: bzzAddrs["42:001"]}, k) - otherAddr := RandomBzzAddr() - otherAddr.Capabilities.Add(testOtherCapability) + bzzAddrs["666:101"] = RandomBzzAddr() + bzzAddrs["666:101"].Capabilities.Add(caps["666:101"]) + discPeers["666:101"] = NewPeer(&BzzPeer{BzzAddr: bzzAddrs["666:101"]}, k) - // all includes two different capability arrays - allAddr := RandomBzzAddr() - allAddr.Capabilities.Add(testOtherCapability) - allAddr.Capabilities.Add(testMoreCapability) + bzzAddrs["42:101,666:101"] = RandomBzzAddr() + bzzAddrs["42:101,666:101"].Capabilities.Add(caps["666:101"]) + bzzAddrs["42:101,666:101"].Capabilities.Add(caps["42:101"]) + discPeers["42:101,666:101"] = NewPeer(&BzzPeer{BzzAddr: bzzAddrs["42:101,666:101"]}, k) - // proceed to check the matches, first for the "known peers" pot - // first adding the different peer configurations to the kademlia - k.Register(moreAddr, lessAddr, otherAddr, allAddr) + k.Register(bzzAddrs["42:101"], bzzAddrs["42:001"], bzzAddrs["666:101"], bzzAddrs["42:101,666:101"]) + + return k, discPeers, caps +} + +// test indices after registering peers +func testCapabilityIndexRegister(t *testing.T) { + + k, _, caps := testCapabilityIndexHelper() // Call without filter should still return all peers - var c int - k.EachAddr(base, 255, func(_ *BzzAddr, _ int) bool { + c := 0 + k.EachAddr(k.BaseAddr(), 255, func(_ *BzzAddr, _ int) bool { c++ return true }) @@ -701,13 +726,13 @@ func TestCapabilityIndex(t *testing.T) { t.Fatalf("EachAddr expected 3 peers, got %d", c) } - // Matches "more", "all" + // match capability 42:101 c = 0 - k.EachAddrFiltered(base, "more", 255, func(a *BzzAddr, _ int) bool { + k.EachAddrFiltered(k.BaseAddr(), "42:101", 255, func(a *BzzAddr, _ int) bool { c++ cp := a.Capabilities.Get(42) - if !cp.Match(testMoreCapability) { - t.Fatalf("EachAddrFiltered 'more' capability mismatch, expected %v, got %v", testMoreCapability, cp) + if !cp.Match(caps["42:101"]) { + t.Fatalf("EachAddrFiltered '42:101' capability mismatch, expected %v, got %v", caps["42:101"], cp) } return true }) @@ -715,34 +740,34 @@ func TestCapabilityIndex(t *testing.T) { t.Fatalf("EachAddrFiltered 'full' expected 2 peer, got %d", c) } - // Matches "more", "less", "all" + // Match capability 42:101 and 42:001 c = 0 - k.EachAddrFiltered(base, "less", 255, func(a *BzzAddr, _ int) bool { + k.EachAddrFiltered(k.BaseAddr(), "42:001", 255, func(a *BzzAddr, _ int) bool { c++ return true }) if c != 3 { - t.Fatalf("EachAddrFiltered 'less' expected 2 peers, got %d", c) + t.Fatalf("EachAddrFiltered '42:001' expected 2 peers, got %d", c) } - // No matches + // Match no capability c = 0 - k.EachAddrFiltered(base, "none", 255, func(a *BzzAddr, _ int) bool { + k.EachAddrFiltered(k.BaseAddr(), "42:010", 255, func(a *BzzAddr, _ int) bool { c++ return true }) if c != 0 { - t.Fatalf("EachAddrFiltered 'none' expected 0 peers, got %d", c) + t.Fatalf("EachAddrFiltered '42:010' expected 0 peers, got %d", c) } - // Matches "other", "all" - // Also checks that "all" has both capabilities + // Match 666:101 + // Also checks that one node has both 42:101 and 666:101 c = 0 - k.EachAddrFiltered(base, "other", 255, func(a *BzzAddr, _ int) bool { + k.EachAddrFiltered(k.BaseAddr(), "666:101", 255, func(a *BzzAddr, _ int) bool { c++ cp := a.Capabilities.Get(666) - if !cp.Match(testOtherCapability) { - t.Fatalf("EachAddrFiltered 'other' capability mismatch, expected %v, got %v", testOtherCapability, cp) + if !cp.Match(caps["666:101"]) { + t.Fatalf("EachAddrFiltered 'other' capability mismatch, expected %v, got %v", caps["666:101"], cp) } cp = a.Capabilities.Get(42) if cp != nil { @@ -753,24 +778,20 @@ func TestCapabilityIndex(t *testing.T) { if c != 3 { t.Fatalf("EachAddrFiltered 'other' expected 3 capability matches, got %d", c) } +} - // Now check the connection pot index - // We add the "all" and "less" peers - allBzzPeer := &BzzPeer{ - BzzAddr: allAddr, - } - allPeer := NewPeer(allBzzPeer, k) - k.On(allPeer) +// test indices after connecting peers +func testCapabilityIndexConnect(t *testing.T) { - lessBzzPeer := &BzzPeer{ - BzzAddr: lessAddr, - } - lessPeer := NewPeer(lessBzzPeer, k) - k.On(lessPeer) + k, discPeers, caps := testCapabilityIndexHelper() + + // Set 42:101 and 42:101,666:101 as connected + k.On(discPeers["42:001"]) + k.On(discPeers["42:101,666:101"]) // Call without filter should return the single connected peer - c = 0 - k.EachConn(base, 255, func(_ *Peer, _ int) bool { + c := 0 + k.EachConn(k.BaseAddr(), 255, func(_ *Peer, _ int) bool { c++ return true }) @@ -778,18 +799,18 @@ func TestCapabilityIndex(t *testing.T) { t.Fatalf("EachConn expected 2 peers, got %d", c) } - // Check that the "all" peer exists in the indices for both capability arrays - // first the "other" index ... + // Check that the "42:101,666:101" peer exists in the indices for both capability arrays + // first the "666:101" index ... c = 0 - k.EachConnFiltered(base, "other", 255, func(p *Peer, _ int) bool { + k.EachConnFiltered(k.BaseAddr(), "666:101", 255, func(p *Peer, _ int) bool { c++ cp := p.Capabilities.Get(666) - if !cp.Match(testOtherCapability) { - t.Fatalf("EachConnFiltered 'other' missing capability %v", testOtherCapability) + if !cp.Match(caps["666:101"]) { + t.Fatalf("EachConnFiltered '666:101' missing capability %v", caps["666:101"]) } cp = p.Capabilities.Get(42) - if !cp.Match(testMoreCapability) { - t.Fatalf("EachConnFiltered 'other' missing capability %v", testMoreCapability) + if !cp.Match(caps["42:101"]) { + t.Fatalf("EachConnFiltered '666:101' missing capability %v", caps["42:101"]) } return true }) @@ -797,107 +818,128 @@ func TestCapabilityIndex(t *testing.T) { t.Fatalf("EachConnFiltered 'other' expected 1 peer, got %d", c) } - // ...then the "more" index + // ... and in 42:101 c = 0 - k.EachConnFiltered(base, "more", 255, func(p *Peer, _ int) bool { + k.EachConnFiltered(k.BaseAddr(), "42:101", 255, func(p *Peer, _ int) bool { c++ cp := p.Capabilities.Get(666) - if !cp.Match(testOtherCapability) { - t.Fatalf("EachConnFiltered 'more' missing capability %v", testOtherCapability) + if !cp.Match(caps["666:101"]) { + t.Fatalf("EachConnFiltered '42:101' missing capability %v", caps["666:101"]) } cp = p.Capabilities.Get(42) - if !cp.Match(testMoreCapability) { - t.Fatalf("EachConnFiltered 'more' missing capability %v", testMoreCapability) + if !cp.Match(caps["42:101"]) { + t.Fatalf("EachConnFiltered '42:101' missing capability %v", caps["42:101"]) } return true }) if c != 1 { t.Fatalf("EachConnFiltered 'more' expected 1 peer, got %d", c) } +} - // Disconnect the "all" peer - k.Off(allPeer) +// test indices after disconnecting peers +func testCapabilityIndexDisconnect(t *testing.T) { - // Check that the "all" is now removed from connections - c = 0 - k.EachConnFiltered(base, "other", 255, func(_ *Peer, _ int) bool { + k, discPeers, caps := testCapabilityIndexHelper() + + // Set "42:101" and "42:101,666:101" as connected + // And then disconnect the "42:101,666:101" peer + k.On(discPeers["42:001"]) + k.On(discPeers["42:101,666:101"]) + k.Off(discPeers["42:101,666:101"]) + + // Check that the "42:101,666:101" is now removed from connections + c := 0 + k.EachConnFiltered(k.BaseAddr(), "666:101", 255, func(_ *Peer, _ int) bool { c++ return true }) if c != 0 { - t.Fatalf("EachConnFiltered 'other' expected 0 peers, got %d", c) + t.Fatalf("EachConnFiltered '666:101' expected 0 peers, got %d", c) } - // Check that there is still an "other" peer among known peers - // (the two matched peers will be "all" and "other") + // Check that there is still a "666:101" peer among known peers + // (the two matched peers will be "42:101,666:101" and "666:101") c = 0 - k.EachAddrFiltered(base, "other", 255, func(_ *BzzAddr, _ int) bool { + k.EachAddrFiltered(k.BaseAddr(), "666:101", 255, func(_ *BzzAddr, _ int) bool { c++ return true }) if c != 2 { - t.Fatalf("EachAddrFiltered 'other' expected 2 peers, got %d", c) + t.Fatalf("EachAddrFiltered '666:101' expected 2 peers, got %d", c) } - // Check that the "less" peer is still registered as connected + // Check that the "42:001" peer is still registered as connected c = 0 - k.EachConnFiltered(base, "less", 255, func(p *Peer, _ int) bool { + k.EachConnFiltered(k.BaseAddr(), "42:001", 255, func(p *Peer, _ int) bool { c++ cp := p.Capabilities.Get(42) - if !cp.Match(testLessCapability) { - t.Fatalf("EachConnFiltered 'less' missing capability %v", testLessCapability) + if !cp.Match(caps["42:001"]) { + t.Fatalf("EachConnFiltered '42:001' missing capability %v", caps["42:001"]) } return true }) if c != 1 { - t.Fatalf("EachConnFiltered 'less' expected 1 peer, got %d", c) + t.Fatalf("EachConnFiltered '42:001' expected 1 peer, got %d", c) } +} + +// test indices after (disconnecting and) removing peers +func testCapabilityIndexRemove(t *testing.T) { + + k, discPeers, caps := testCapabilityIndexHelper() + + // Set "42:101" and "42:101,666:101" as connected + // And then disconnect the "42:101,666:101" peer + k.On(discPeers["42:001"]) + k.On(discPeers["42:101,666:101"]) + k.Off(discPeers["42:101,666:101"]) // Remove "less" from both connection and known peers (pruning) list // TODO replace with the "prune" method when one is implemented - k.removeFromCapabilityIndex(lessPeer, false) + k.removeFromCapabilityIndex(discPeers["42:001"], false) - // Check that the "less" peer is no longer registered as connected - c = 0 - k.EachConnFiltered(base, "less", 255, func(p *Peer, _ int) bool { + // Check that the "42:001" peer is no longer registered as connected + c := 0 + k.EachConnFiltered(k.BaseAddr(), "42:001", 255, func(p *Peer, _ int) bool { c++ return true }) if c != 0 { - t.Fatalf("EachConnFiltered 'less' expected 0 peers, got %d", c) + t.Fatalf("EachConnFiltered '42:001' expected 0 peers, got %d", c) } - // check that the "less" peer is not known anymore - // (the two matched peers will be "all" and "more") + // check that the "42:001" peer is not known anymore + // (the two matched peers will be "42:101,666:101" and "42:101") c = 0 - k.EachAddrFiltered(base, "less", 255, func(p *BzzAddr, _ int) bool { + k.EachAddrFiltered(k.BaseAddr(), "42:001", 255, func(p *BzzAddr, _ int) bool { c++ cp := p.Capabilities.Get(42) - if !cp.Match(testMoreCapability) { - t.Fatalf("EachConnFiltered 'less' should now return only capability 'more': %v", testMoreCapability) + if !cp.Match(caps["42:101"]) { + t.Fatalf("EachConnFiltered '42:001' should now return only capability '42:101': %v", caps["42:101"]) } return true }) if c != 2 { - t.Fatalf("EachAddrFiltered 'less' expected 2 peer, got %d", c) + t.Fatalf("EachAddrFiltered '42:001' expected 2 peer, got %d", c) } - // Remove "all" from known peers list (pruning only) + // Remove "42:101,666:101" from known peers list (pruning only) // TODO replace with the "prune" method when one is implemented - k.removeFromCapabilityIndex(allPeer, false) + k.removeFromCapabilityIndex(discPeers["42:101,666:101"], false) - // check that the "less" peer is not known anymore - // (the two matched peers will be "all" and "more") + // check that the "42:101,666:101" peer is not known anymore + // (the only matched peer should now be "42:101") c = 0 - k.EachAddrFiltered(base, "more", 255, func(p *BzzAddr, _ int) bool { + k.EachAddrFiltered(k.BaseAddr(), "42:101", 255, func(p *BzzAddr, _ int) bool { c++ cp := p.Capabilities.Get(666) if cp != nil { - t.Fatalf("EachAddrFiltered 'more' should not contain a peer with capability %v", testOtherCapability) + t.Fatalf("EachAddrFiltered '42:101' should not contain a peer with capability %v", caps["666:101"]) } return true }) if c != 1 { - t.Fatalf("EachAddrFiltered 'more' expected 1 peer, got %d", c) + t.Fatalf("EachAddrFiltered '42:101' expected 1 peer, got %d", c) } } From f3cdb0d26498281a8cfe39bcc2f6d254b6b3720f Mon Sep 17 00:00:00 2001 From: nolash Date: Wed, 18 Sep 2019 00:30:05 +0200 Subject: [PATCH 35/36] network: Add test for deserialization result for bzzaddr --- network/network.go | 2 +- network/network_test.go | 18 ++++++++++++++++++ network/protocol_test.go | 15 ++++----------- 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/network/network.go b/network/network.go index 541d05b836..a0aae05141 100644 --- a/network/network.go +++ b/network/network.go @@ -41,7 +41,7 @@ func (b *BzzAddr) EncodeRLP(w io.Writer) error { return nil } -// EncodeRLP implements rlp.Decoder +// DecodeRLP implements rlp.Decoder func (b *BzzAddr) DecodeRLP(s *rlp.Stream) error { var err error diff --git a/network/network_test.go b/network/network_test.go index 3638bbd04f..15df554c58 100644 --- a/network/network_test.go +++ b/network/network_test.go @@ -1,6 +1,7 @@ package network import ( + "bytes" "testing" "github.com/ethereum/go-ethereum/rlp" @@ -21,4 +22,21 @@ func TestBzzAddrRLPSerialization(t *testing.T) { if err != nil { t.Fatal(err) } + if !addr.Match(&addrRecovered) { + t.Fatalf("bzzaddr mismatch, expected %v, got %v", addr, addrRecovered) + } +} + +// Match returns true if the passed BzzAddr is identical to the receiver +func (b *BzzAddr) Match(bcmp *BzzAddr) bool { + if !bytes.Equal(b.OAddr, bcmp.OAddr) { + return false + } + if !bytes.Equal(b.UAddr, bcmp.UAddr) { + return false + } + if !b.Capabilities.Match(bcmp.Capabilities) { + return false + } + return true } diff --git a/network/protocol_test.go b/network/protocol_test.go index b0686a1eb0..f91e1725ca 100644 --- a/network/protocol_test.go +++ b/network/protocol_test.go @@ -17,7 +17,6 @@ package network import ( - "bytes" "crypto/ecdsa" "fmt" "sync" @@ -255,19 +254,13 @@ func TestBzzHandshakeRLPSerialization(t *testing.T) { t.Fatal(err) } if msg.Version != msgRecovered.Version { - t.Fatal("version mismatch") + t.Fatalf("version mismatch, expected %v, got %v", msg.Version, msgRecovered.Version) } if msg.NetworkID != msgRecovered.NetworkID { - t.Fatal("networkid mismatch") + t.Fatalf("networkid mismatch, expected %v, got %v", msg.NetworkID, msgRecovered.NetworkID) } - if !bytes.Equal(msg.Addr.OAddr, msgRecovered.Addr.OAddr) { - t.Fatal("OAddr mismatch") - } - if !bytes.Equal(msg.Addr.UAddr, msgRecovered.Addr.UAddr) { - t.Fatal("UAddr mismatch") - } - if !fullCapability.IsSameAs(msgRecovered.Addr.Capabilities.Get(0)) { - t.Fatal("capabilities mismatch") + if !msg.Addr.Match(msgRecovered.Addr) { + t.Fatalf("bzzaddr mismatch, expected %v, got %v", msg.Addr, msgRecovered.Addr) } } From 44795f2a041a4f64d0c1f0d12f44b9c8940ac500 Mon Sep 17 00:00:00 2001 From: nolash Date: Wed, 18 Sep 2019 23:01:54 +0200 Subject: [PATCH 36/36] Travis travis travis