Mobile discovery improvements

go.mod

@@ -56,6 +56,7 @@ require (
 	github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef
 	github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208
 	golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550
+	golang.org/x/mobile v0.0.0-20200801112145-973feb4309de // indirect
 	golang.org/x/net v0.0.0-20190628185345-da137c7871d7 // indirect
 	golang.org/x/sync v0.0.0-20190423024810-112230192c58
 	golang.org/x/sys v0.0.0-20200519105757-fe76b779f299

go.sum

@@ -20,6 +20,7 @@ github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6L
 github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k=
 github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/OneOfOne/xxhash v1.2.5 h1:zl/OfRA6nftbBK9qTohYBJ5xvw6C/oNKizR7cZGl3cI=
 github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
@@ -192,10 +193,27 @@ github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 h1:1cngl9mPEoITZG8s8
 github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+mw0EzQ08zFqg7pK3FebNXpaMsRy2RT+Ees=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20200629153529-33b80540585f h1:9MxnlCHwn6IfUTinHBBzcBhmrX4OXfRmi954tWGKq+M=
+golang.org/x/mobile v0.0.0-20200629153529-33b80540585f/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4=
+golang.org/x/mobile v0.0.0-20200721161523-bcce01171201 h1:MP0kSyV1YErXydpuJrNIYtGzuv2B/NcGRvxG3UeI+TU=
+golang.org/x/mobile v0.0.0-20200721161523-bcce01171201/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4=
+golang.org/x/mobile v0.0.0-20200801112145-973feb4309de h1:OVJ6QQUBAesB8CZijKDSsXX7xYVtUhrkY0gwMfbi4p4=
+golang.org/x/mobile v0.0.0-20200801112145-973feb4309de/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd h1:ePuNC7PZ6O5BzgPn9bZayERXBdfZjUYoXEf5BTfDfh8=
+golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190628185345-da137c7871d7 h1:rTIdg5QFRR7XCaK4LCjBiPbx8j4DQRpdYMnGn/bJUEU=
 golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -216,6 +234,13 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69 h1:yBHHx+XZqXJBm6Exke3N7V9gnlsyXxoCPEb1yVenjfk=
+golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=

internal/web3ext/web3ext.go

@@ -858,6 +858,10 @@ web3._extend({
 		new web3._extend.Property({
 			name: 'gatewayFeeCache',
 			getter: 'les_gatewayFeeCache'
+		}),
+		new web3._extend.Property({
+			name: 'serverPoolEntries',
+			getter: 'les_serverPoolEntries'
 		})
 	]
 });

les/api.go

@@ -390,21 +390,21 @@ func (api *PrivateLightAPI) GetCheckpointContractAddress() (string, error) {
 	return api.backend.oracle.Contract().ContractAddr().Hex(), nil
 }
 
-//API should be for light clients of les protocol
-type LightClientAPI struct {
+// API should be for light clients of les protocol
+type PrivateLightClientAPI struct {
 	le *LightEthereum
 }
 
-func NewLightClientAPI(le *LightEthereum) *LightClientAPI {
-	return &LightClientAPI{le}
+func NewPrivateLightClientAPI(le *LightEthereum) *PrivateLightClientAPI {
+	return &PrivateLightClientAPI{le}
 }
 
-func (api *LightClientAPI) GatewayFeeCache() map[string]*GatewayFeeInformation {
+func (api *PrivateLightClientAPI) GatewayFeeCache() map[string]*GatewayFeeInformation {
 	return api.le.handler.gatewayFeeCache.getMap()
 }
 
 // RequestPeerGatewayFees updates cache by pulling gateway fee peer nodes
-func (api *LightClientAPI) RequestPeerGatewayFees() error {
+func (api *PrivateLightClientAPI) RequestPeerGatewayFees() error {
 	peerNodes := api.le.peers.AllPeers()
 	for _, peerNode := range peerNodes {
 		cost := peerNode.GetRequestCost(GetGatewayFeeMsg, int(1))
@@ -417,10 +417,14 @@ func (api *LightClientAPI) RequestPeerGatewayFees() error {
 }
 
 // SuggestGatewayFee suggests the best light server to choose based on different factors. Currently only minPeerGatewayFee.
-func (api *LightClientAPI) SuggestGatewayFee() (*GatewayFeeInformation, error) {
+func (api *PrivateLightClientAPI) SuggestGatewayFee() (*GatewayFeeInformation, error) {
 	bestGatewayFeeInfo, err := api.le.handler.gatewayFeeCache.MinPeerGatewayFee()
 	if err != nil {
 		return nil, err
 	}
 	return bestGatewayFeeInfo, nil
 }
+
+func (api *PrivateLightClientAPI) ServerPoolEntries() ([]*poolEntryInfo, error) {
+	return api.le.serverPool.Info(), nil
+}

les/client.go

@@ -234,8 +234,8 @@ func (s *LightEthereum) APIs() []rpc.API {
 		}, {
 			Namespace: "les",
 			Version:   "1.0",
-			Service:   NewLightClientAPI(s),
-			Public:    true,
+			Service:   NewPrivateLightClientAPI(s),
+			Public:    false,
 		}, {
 			Namespace: "eth",
 			Version:   "1.0",

les/serverpool.go

@@ -409,6 +409,7 @@ func (pool *serverPool) eventLoop() {
 			}
 			pool.knownQueue.setLatest(entry)
 			entry.shortRetry = shortRetryCnt
+			pool.saveNodes()
 			close(req.done)
 
 		case req := <-pool.disconnCh:
@@ -530,11 +531,9 @@ func parseTrustedNodes(trustedNodes []string) map[enode.ID]*enode.Node {
 // saveNodes saves known nodes and their statistics into the database. Nodes are
 // ordered from least to most recently connected.
 func (pool *serverPool) saveNodes() {
-	list := make([]*poolEntry, len(pool.knownQueue.queue))
-	for i := range list {
-		list[i] = pool.knownQueue.fetchOldest()
-	}
-	enc, err := rlp.EncodeToBytes(list)
+	nodes := pool.knownQueue.list()
+	log.Debug("Saving serverPool nodes", "length", len(nodes))
+	enc, err := rlp.EncodeToBytes(nodes)
 	if err == nil {
 		pool.db.Put(pool.dbKey, enc)
 	}
@@ -658,13 +657,47 @@ func (pool *serverPool) checkDialTimeout(entry *poolEntry) {
 	pool.setRetryDial(entry)
 }
 
+func (pool *serverPool) Info() []*poolEntryInfo {
+	entryInfos := make([]*poolEntryInfo, 0)
+	for _, entry := range pool.entries {
+		entryInfos = append(entryInfos, &poolEntryInfo{
+			Peered:             entry.peer != nil,
+			Node:               entry.node,
+			Known:              entry.known,
+			KnownSelected:      entry.knownSelected,
+			Trusted:            entry.trusted,
+			LastDiscoveredTime: entry.lastDiscovered,
+			RegisteredTime:     entry.regTime,
+			State:              entry.state.String(),
+		})
+	}
+	return entryInfos
+}
+
+type poolEntryState int
+
 const (
-	psNotConnected = iota
+	psNotConnected = poolEntryState(iota)
 	psDialed
 	psConnected
 	psRegistered
 )
 
+func (e poolEntryState) String() string {
+	switch e {
+	case psNotConnected:
+		return "NotConnected"
+	case psDialed:
+		return "Dialed"
+	case psConnected:
+		return "Connected"
+	case psRegistered:
+		return "Registered"
+	default:
+		return fmt.Sprintf("poolEntryState(%d)", e)
+	}
+}
+
 // poolEntry represents a server node and stores its current state and statistics.
 type poolEntry struct {
 	peer                  *peer
@@ -677,7 +710,7 @@ type poolEntry struct {
 	known, knownSelected, trusted bool
 	connectStats, delayStats      poolStats
 	responseStats, timeoutStats   poolStats
-	state                         int
+	state                         poolEntryState
 	regTime                       mclock.AbsTime
 	queueIdx                      int
 	removed                       bool
@@ -686,6 +719,17 @@ type poolEntry struct {
 	shortRetry   int
 }
 
+type poolEntryInfo struct {
+	Peered             bool           `json:"peered"`
+	Node               *enode.Node    `json:"node"`
+	Known              bool           `json:"known"`
+	KnownSelected      bool           `json:"knownSelected"`
+	Trusted            bool           `json:"trusted"`
+	LastDiscoveredTime mclock.AbsTime `json:"lastDiscoveredTime"`
+	RegisteredTime     mclock.AbsTime `json:"registeredTime"`
+	State              string         `json:"state"`
+}
+
 // poolEntryEnc is the RLP encoding of poolEntry.
 type poolEntryEnc struct {
 	Pubkey                     []byte
@@ -904,3 +948,24 @@ func (q *poolEntryQueue) setLatest(entry *poolEntry) {
 	q.queue[entry.queueIdx] = entry
 	q.newPtr++
 }
+
+// traverses the queue and returns it as an array ordered oldest -> newest
+func (q *poolEntryQueue) list() []*poolEntry {
+	if len(q.queue) == 0 {
+		return nil
+	}
+	list := make([]*poolEntry, len(q.queue))
+	ptr := q.oldPtr
+	for i := range list {
+	queueTraverser:
+		for {
+			if e := q.queue[ptr]; e != nil {
+				ptr++
+				list[i] = e
+				break queueTraverser
+			}
+			ptr++
+		}
+	}
+	return list
+}

mobile/geth.go

@@ -23,6 +23,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"path/filepath"
+	"strings"
 
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/eth"
@@ -62,6 +63,9 @@ type NodeConfig struct {
 	// set to zero, then only the configured static and trusted peers can connect.
 	MaxPeers int
 
+	// NoDiscovery indicates whether the node should not participate in p2p discovery
+	NoDiscovery bool
+
 	// EthereumEnabled specifies whether the node should run the Ethereum protocol.
 	EthereumEnabled bool
 
@@ -83,6 +87,29 @@ type NodeConfig struct {
 	// It has the form "nodename:secret@host:port"
 	EthereumNetStats string
 
+	// HTTPHost is the host interface on which to start the HTTP RPC server. If this
+	// field is empty, no HTTP API endpoint will be started.
+	HTTPHost string
+
+	// HTTPPort is the TCP port number on which to start the HTTP RPC server. The
+	// default zero value is/ valid and will pick a port number randomly (useful
+	// for ephemeral nodes).
+	HTTPPort int
+
+	// HTTPVirtualHosts is a comma separated list of virtual hostnames which are allowed on incoming requests.
+	// This is by default {'localhost'}. Using this prevents attacks like
+	// DNS rebinding, which bypasses SOP by simply masquerading as being within the same
+	// origin. These attacks do not utilize CORS, since they are not cross-domain.
+	// By explicitly checking the Host-header, the server will not allow requests
+	// made against the server with a malicious host domain.
+	// Requests using ip address directly are not affected
+	HTTPVirtualHosts string
+
+	// HTTPModules is a comma separated list of API modules to expose via the HTTP RPC interface.
+	// If the module list is empty, all RPC API endpoints designated public will be
+	// exposed.
+	HTTPModules string
+
 	// WhisperEnabled specifies whether the node should run the Whisper protocol.
 	WhisperEnabled bool
 
@@ -111,6 +138,7 @@ type NodeConfig struct {
 var defaultNodeConfig = &NodeConfig{
 	BootstrapNodes:        FoundationBootnodes(),
 	MaxPeers:              25,
+	NoDiscovery:           true,
 	EthereumEnabled:       true,
 	EthereumNetworkID:     1,
 	EthereumDatabaseCache: 16,
@@ -137,6 +165,16 @@ func NewNode(datadir string, config *NodeConfig) (stack *Node, _ error) {
 		config.BootstrapNodes = defaultNodeConfig.BootstrapNodes
 	}
 
+	// gomobile doesn't allow arrays to be passed in, so comma separated strings are used
+	var httpVirtualHosts []string
+	if config.HTTPVirtualHosts != "" {
+		httpVirtualHosts = strings.Split(config.HTTPVirtualHosts, ",")
+	}
+	var httpModules []string
+	if config.HTTPModules != "" {
+		httpModules = strings.Split(config.HTTPModules, ",")
+	}
+
 	if config.PprofAddress != "" {
 		debug.StartPProf(config.PprofAddress)
 	}
@@ -149,9 +187,13 @@ func NewNode(datadir string, config *NodeConfig) (stack *Node, _ error) {
 		KeyStoreDir:       filepath.Join(datadir, "keystore"), // Mobile should never use internal keystores!
 		UseLightweightKDF: config.UseLightweightKDF,
 		IPCPath:           config.IPCPath,
+		HTTPHost:          config.HTTPHost,
+		HTTPPort:          config.HTTPPort,
+		HTTPVirtualHosts:  httpVirtualHosts,
+		HTTPModules:       httpModules,
 		P2P: p2p.Config{
-			NoDiscovery:      true,
-			DiscoveryV5:      false,
+			NoDiscovery:      config.NoDiscovery,
+			DiscoveryV5:      !config.NoDiscovery,
 			BootstrapNodesV5: config.BootstrapNodes.nodes,
 			ListenAddr:       ":0",
 			NAT:              nat.Any(),