Merge pull request #15 from Alethio/batch-calls-receipts

update web3-go version and take advantage of batch calls

commands/helpers.go

@@ -13,11 +13,11 @@ import (
 func initLogging() {
 	logging := viper.GetString("logging")
 
-	if verbose && logging == "" {
+	if verbose {
 		logging = "*=debug"
 	}
 
-	if vverbose && logging == "" {
+	if vverbose {
 		logging = "*=trace"
 	}
 

commands/run.go

@@ -14,10 +14,11 @@ import (
 
 	"github.com/Alethio/memento/taskmanager"
 
-	"github.com/Alethio/memento/core"
-	"github.com/Alethio/memento/eth/bestblock"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+
+	"github.com/Alethio/memento/core"
+	"github.com/Alethio/memento/eth/bestblock"
 )
 
 var runCmd = &cobra.Command{

data/data.go

@@ -2,6 +2,7 @@ package data
 
 import (
 	"database/sql"
+	"strconv"
 
 	"github.com/Alethio/memento/metrics"
 
@@ -15,12 +16,23 @@ var log = logrus.WithField("module", "data")
 
 type FullBlock struct {
 	Block    types.Block
-	Receipts []types.Receipt
+	Receipts Receipts
 	Uncles   []types.Block
 
 	storables []Storable
 }
 
+type Receipts []types.Receipt
+
+func (a Receipts) Len() int      { return len(a) }
+func (a Receipts) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a Receipts) Less(i, j int) bool {
+	iIdx, _ := strconv.ParseInt(a[i].TransactionIndex, 0, 64)
+	jIdx, _ := strconv.ParseInt(a[j].TransactionIndex, 0, 64)
+
+	return iIdx < jIdx
+}
+
 // Storable
 // role: a Storable serves as a means of transforming raw data and inserting it into the database
 // input: raw Ethereum data + a database transaction

go.mod

@@ -3,7 +3,7 @@ module github.com/Alethio/memento
 require (
 	github.com/BurntSushi/toml v0.3.1 // indirect
 	github.com/alethio/ethmock v0.0.0-20190820104914-7b8327fb645e // indirect
-	github.com/alethio/web3-go v0.0.3
+	github.com/alethio/web3-go v0.0.6
 	github.com/davecgh/go-spew v1.1.1
 	github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484 // indirect
 	github.com/gin-contrib/cors v1.3.0
@@ -18,15 +18,15 @@ require (
 	github.com/onsi/ginkgo v1.9.0 // indirect
 	github.com/onsi/gomega v1.6.0 // indirect
 	github.com/parnurzeal/gorequest v0.2.15
-	github.com/pkg/errors v0.8.1 // indirect
+	github.com/pkg/errors v0.8.1
 	github.com/pressly/goose v2.6.0+incompatible
 	github.com/sirupsen/logrus v1.4.2
 	github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 // indirect
 	github.com/spf13/cobra v0.0.3
 	github.com/spf13/viper v1.3.2
 	github.com/stretchr/testify v1.4.0 // indirect
 	github.com/ugorji/go v1.1.7 // indirect
-	golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 // indirect
+	golang.org/x/net v0.0.0-20200226121028-0de0cce0169b // indirect
 	golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a // indirect
 	golang.org/x/text v0.3.2 // indirect
 	gopkg.in/yaml.v2 v2.2.4 // indirect

go.sum

@@ -4,8 +4,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/alethio/ethmock v0.0.0-20190607140831-ce4476424f5c/go.mod h1:Lpp9nEVo/nKuq0uf/Le6r1zjaiW4/tu0maG47HBE65A=
 github.com/alethio/ethmock v0.0.0-20190820104914-7b8327fb645e h1:3g/lGHjagEkqc3L/WiNY1QfpZqMUxtTfZ1uKMu+wMnc=
 github.com/alethio/ethmock v0.0.0-20190820104914-7b8327fb645e/go.mod h1:Lpp9nEVo/nKuq0uf/Le6r1zjaiW4/tu0maG47HBE65A=
-github.com/alethio/web3-go v0.0.3 h1:K7RVdrn6S8MKo0h0s/N8+6P9fKDcOtS/kK9w5dEXLYI=
-github.com/alethio/web3-go v0.0.3/go.mod h1:AsfM7UJMW47BDfa7LKPfaaSEkxyxYCsRtCftScgNn+c=
+github.com/alethio/web3-go v0.0.6 h1:ZEDJY57OKq9tCmdkxMyy8M11mthX33GmFCzZ8kYuMY0=
+github.com/alethio/web3-go v0.0.6/go.mod h1:tnrqWtLdde8ttsGwiiJ6VwmA9WQwI3sFbmyNjMQVT8M=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
@@ -49,9 +49,8 @@ github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwK
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
-github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -128,14 +127,12 @@ golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190313220215-9f648a60d977 h1:actzWV6iWn3GLqN8dZjzsB+CLt+gaV2+wsxroxiQI8I=
-golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 h1:fHDIZ2oxGnUZRN6WgWFCbYBjH9uqVPRCUVUDhs0wnbA=
-golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190412183630-56d357773e84 h1:IqXQ59gzdXv58Jmm2xn0tSOR9i6HqroaOFRQ3wR/dJQ=
-golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a h1:1n5lsVfiQW3yfsRGu98756EH1YthsFqr/5mxHduZW2A=
 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -153,6 +150,7 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384 h1:TFlARGu6Czu1z7q93HTxcP1P+/ZFC/IKythI5RzrnRg=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=

scraper/scraper.go

@@ -1,9 +1,14 @@
 package scraper
 
 import (
+	"sort"
 	"strconv"
+	"sync"
 	"time"
 
+	"github.com/alethio/web3-go/ethrpc/provider/httprpc"
+	"github.com/pkg/errors"
+
 	"github.com/Alethio/memento/data"
 
 	"github.com/alethio/web3-go/ethrpc"
@@ -24,10 +29,19 @@ type Scraper struct {
 }
 
 func New(config Config) (*Scraper, error) {
-	c, err := ethrpc.NewWithDefaults(config.NodeURL)
+	batchLoader, err := httprpc.NewBatchLoader(0, 4*time.Millisecond)
 	if err != nil {
-		log.Error(err)
-		return nil, err
+		return nil, errors.Wrap(err, "could not init batch loader")
+	}
+
+	provider, err := httprpc.NewWithLoader(config.NodeURL, batchLoader)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not init httprpc provider")
+	}
+	provider.SetHTTPTimeout(5000 * time.Millisecond)
+	c, err := ethrpc.New(provider)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not init ethrpc")
 	}
 
 	return &Scraper{
@@ -58,16 +72,35 @@ func (s *Scraper) Exec(block int64) (*data.FullBlock, error) {
 
 	log.Debug("getting receipts")
 	start = time.Now()
+
+	var wg sync.WaitGroup
+	var errs []error
+	var mu sync.Mutex
 	for _, tx := range dataBlock.Transactions {
-		dataReceipt, err := s.conn.GetTransactionReceipt(tx.Hash)
-		if err != nil {
-			log.Error(err)
-			return nil, err
-		}
+		wg.Add(1)
+		txCopy := tx
+
+		go func() {
+			defer wg.Done()
+
+			dataReceipt, err := s.conn.GetTransactionReceipt(txCopy.Hash)
+			if err != nil {
+				errs = append(errs, err)
+				return
+			}
 
-		b.Receipts = append(b.Receipts, dataReceipt)
+			mu.Lock()
+			b.Receipts = append(b.Receipts, dataReceipt)
+			mu.Unlock()
+		}()
 	}
+	wg.Wait()
+	sort.Sort(b.Receipts)
+
 	log.WithField("duration", time.Since(start)).Debugf("got %d receipts", len(b.Receipts))
+	if len(errs) > 0 {
+		return nil, errs[0]
+	}
 
 	if s.config.EnableUncles {
 		log.Debug("getting uncles")