Skip to content

Commit aa73bd2

Browse files
tybrittentybritten
committed
added ES retries using httpx
1 parent d85bf42 commit aa73bd2

File tree

4 files changed

+206
-13
lines changed

4 files changed

+206
-13
lines changed

‎go.mod

+13-12
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,25 @@
11
module github.com/nyaruka/rp-indexer
22

33
require (
4-
github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261
5-
github.com/davecgh/go-spew v1.1.0
4+
github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261 // indirect
5+
github.com/davecgh/go-spew v1.1.1 // indirect
66
github.com/evalphobia/logrus_sentry v0.4.5
7-
github.com/fatih/structs v1.0.0
7+
github.com/fatih/structs v1.0.0 // indirect
88
github.com/fortytw2/leaktest v1.3.0 // indirect
9-
github.com/getsentry/raven-go v0.0.0-20180405121644-d1470f50d3a3
9+
github.com/getsentry/raven-go v0.0.0-20180405121644-d1470f50d3a3 // indirect
10+
github.com/kylelemons/godebug v1.1.0 // indirect
1011
github.com/lib/pq v0.0.0-20180327071824-d34b9ff171c2
11-
github.com/mailru/easyjson v0.0.0-20180323154445-8b799c424f57
12-
github.com/naoina/go-stringutil v0.1.0
13-
github.com/naoina/toml v0.1.1
12+
github.com/mailru/easyjson v0.0.0-20180323154445-8b799c424f57 // indirect
13+
github.com/naoina/go-stringutil v0.1.0 // indirect
14+
github.com/naoina/toml v0.1.1 // indirect
1415
github.com/nyaruka/ezconf v0.2.1
16+
github.com/nyaruka/gocommon v1.3.0
1517
github.com/olivere/elastic v6.1.14+incompatible
16-
github.com/pkg/errors v0.8.0
17-
github.com/pmezard/go-difflib v1.0.0
18+
github.com/onsi/ginkgo v1.14.0 // indirect
19+
github.com/pkg/errors v0.9.1 // indirect
20+
github.com/pmezard/go-difflib v1.0.0 // indirect
1821
github.com/sirupsen/logrus v1.0.5
19-
github.com/stretchr/testify v1.2.1
20-
golang.org/x/crypto v0.0.0-20180322175230-88942b9c40a4
21-
golang.org/x/sys v0.0.0-20180326154331-13d03a9a82fb
22+
github.com/stretchr/testify v1.2.2
2223
gopkg.in/airbrake/gobrake.v2 v2.0.9 // indirect
2324
gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 // indirect
2425
)

‎go.sum

+75
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,34 @@ github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261 h1:6/yVvBsKeAw05
22
github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4=
33
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
44
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
5+
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
6+
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
57
github.com/evalphobia/logrus_sentry v0.4.5 h1:weRoBjojMYPp57TLDjPEkP58JVHHSiqNrxG+h3ODdPM=
68
github.com/evalphobia/logrus_sentry v0.4.5/go.mod h1:pKcp+vriitUqu9KiWj/VRFbRfFNUwz95/UkgG8a6MNc=
79
github.com/fatih/structs v1.0.0 h1:BrX964Rv5uQ3wwS+KRUAJCBBw5PQmgJfJ6v4yly5QwU=
810
github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
911
github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
1012
github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
13+
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
14+
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
15+
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
1116
github.com/getsentry/raven-go v0.0.0-20180405121644-d1470f50d3a3 h1:md1zEr2oSVWYNfQj+6TL/nmAFf5gY3Tp44lzskzK9QU=
1217
github.com/getsentry/raven-go v0.0.0-20180405121644-d1470f50d3a3/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
18+
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
19+
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
20+
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
21+
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
22+
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
23+
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
24+
github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
25+
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
26+
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
27+
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
28+
github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
29+
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
30+
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
31+
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
32+
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
1333
github.com/lib/pq v0.0.0-20180327071824-d34b9ff171c2 h1:hRGSmZu7j271trc9sneMrpOW7GN5ngLm8YUZIPzf394=
1434
github.com/lib/pq v0.0.0-20180327071824-d34b9ff171c2/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
1535
github.com/mailru/easyjson v0.0.0-20180323154445-8b799c424f57 h1:qhv1ir3dIyOFmFU+5KqG4dF3zSQTA4nn1DFhu2NQC44=
@@ -18,23 +38,78 @@ github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hz
1838
github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0=
1939
github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8=
2040
github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E=
41+
github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
42+
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
2143
github.com/nyaruka/ezconf v0.2.1 h1:TDXWoqjqYya1uhou1mAJZg7rgFYL98EB0Tb3+BWtUh0=
2244
github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDYYxgnQw=
45+
github.com/nyaruka/gocommon v1.3.0 h1:IqaPT4KQ2oVq/2Ivp/c+RVCs8v71+RzPU2VhMoRrgpU=
46+
github.com/nyaruka/gocommon v1.3.0/go.mod h1:w7lKxIkm/qLAoO9Y3aI1LV7EiYogn6+1C8MTEjxTC9M=
47+
github.com/nyaruka/phonenumbers v1.0.34/go.mod h1:GQ0cTHlrxPrhoLwyQ1blyN1hO794ygt6FTHWrFB5SSc=
2348
github.com/olivere/elastic v6.1.14+incompatible h1:X7PDDou5+WuNrh5WgtS5+gKzbUmSNXvF0mQZ++VsZYU=
2449
github.com/olivere/elastic v6.1.14+incompatible/go.mod h1:J+q1zQJTgAz9woqsbVRqGeB5G1iqDKVBWLNSYW8yfJ8=
50+
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
51+
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
52+
github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA=
53+
github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
54+
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
55+
github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE=
56+
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
2557
github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
2658
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
59+
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
60+
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
2761
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
2862
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
63+
github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ=
64+
github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
2965
github.com/sirupsen/logrus v1.0.5 h1:8c8b5uO0zS4X6RPl/sd1ENwSkIc0/H2PaHxE3udaE8I=
3066
github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
3167
github.com/stretchr/testify v1.2.1 h1:52QO5WkIUcHGIR7EnGagH88x1bUzqGXTC5/1bDTUQ7U=
3268
github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
69+
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
70+
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
3371
golang.org/x/crypto v0.0.0-20180322175230-88942b9c40a4 h1:AJCW0rhPjFKEAoValWpqnRKxX8YV0Xvqfw+dOexCTPc=
3472
golang.org/x/crypto v0.0.0-20180322175230-88942b9c40a4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
73+
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
74+
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
75+
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
76+
golang.org/x/net v0.0.0-20180921000356-2f5d2388922f h1:QM2QVxvDoW9PFSPp/zy9FgxJLfaWTZlS61KEPtBwacM=
77+
golang.org/x/net v0.0.0-20180921000356-2f5d2388922f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
78+
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7 h1:AeiKBIuRw3UomYXSbLy0Mc2dDLfdtbT/IVn4keq83P0=
79+
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
80+
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
3581
golang.org/x/sys v0.0.0-20180326154331-13d03a9a82fb h1:w1eDrzbtlRsu1SyjnZZFUBVuhRN2Rn6DlrHCx9s4ud0=
3682
golang.org/x/sys v0.0.0-20180326154331-13d03a9a82fb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
83+
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
84+
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
85+
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
86+
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
87+
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
88+
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
89+
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7NkNAQs+6Q8b9WEB/F4=
90+
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
91+
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
92+
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
93+
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
94+
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
95+
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
96+
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
97+
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
98+
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
99+
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
100+
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
101+
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
102+
google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
103+
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
37104
gopkg.in/airbrake/gobrake.v2 v2.0.9 h1:7z2uVWwn7oVeeugY1DtlPAy5H+KYgB1KeKTnqjNatLo=
38105
gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
106+
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
107+
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
108+
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
39109
gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 h1:OAj3g0cR6Dx/R07QgQe8wkA9RNjB2u4i700xBkIT4e0=
40110
gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
111+
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
112+
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
113+
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
114+
gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
115+
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

‎indexer.go

+34-1
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,41 @@ import (
1111
"strings"
1212
"time"
1313

14+
"github.com/nyaruka/gocommon/httpx"
1415
log "github.com/sirupsen/logrus"
1516
)
1617

1718
var batchSize = 500
1819

20+
func ElasticRetries(initialBackoff time.Duration, count int) *httpx.RetryConfig {
21+
backoffs := make([]time.Duration, count)
22+
backoffs[0] = initialBackoff
23+
for i := 1; i < count; i++ {
24+
backoffs[i] = backoffs[i-1] * 2
25+
}
26+
return &httpx.RetryConfig{Backoffs: backoffs, ShouldRetry: ShouldRetry}
27+
}
28+
func ShouldRetry(request *http.Request, response *http.Response, withDelay time.Duration) bool {
29+
30+
// 429 Too Many Requests is recoverable. Sometimes the server puts
31+
// a Retry-After response header to indicate when the server is
32+
// available to start processing request from client.
33+
if response.StatusCode == http.StatusTooManyRequests {
34+
return true
35+
}
36+
37+
// check for unexpected EOF
38+
bodyBytes, err := ioutil.ReadAll(response.Body)
39+
response.Body.Close()
40+
if err != nil {
41+
log.WithError(err).Error("error reading ES response, retrying")
42+
return true
43+
}
44+
45+
response.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBytes))
46+
return false
47+
}
48+
1949
// CreateNewIndex creates a new index for the passed in alias.
2050
//
2151
// Note that we do not create an index with the passed name, instead creating one
@@ -132,7 +162,10 @@ func CleanupIndexes(url string, alias string) error {
132162
func MakeJSONRequest(method string, url string, body string, jsonStruct interface{}) (*http.Response, error) {
133163
req, _ := http.NewRequest(method, url, bytes.NewReader([]byte(body)))
134164
req.Header.Add("Content-Type", "application/json")
135-
resp, err := http.DefaultClient.Do(req)
165+
retrycount := 5
166+
initialBackoff := 2 * time.Second
167+
retryConfig := ElasticRetries(initialBackoff, retrycount)
168+
resp, err := httpx.Do(http.DefaultClient, req, retryConfig, nil)
136169

137170
l := log.WithField("url", url).WithField("method", method).WithField("request", body)
138171
if err != nil {

‎indexer_test.go

+84
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"io/ioutil"
88
"log"
99
"net/http"
10+
"net/http/httptest"
1011
"os"
1112
"testing"
1213
"time"
@@ -295,3 +296,86 @@ func TestIndexing(t *testing.T) {
295296
assertQuery(t, client, indexName, elastic.NewMatchQuery("groups", "529bac39-550a-4d6f-817c-1833f3449007"), []int64{1})
296297

297298
}
299+
func TestRetryServer(t *testing.T) {
300+
responseCounter := 0
301+
responses := []func(w http.ResponseWriter, r *http.Request){
302+
func(w http.ResponseWriter, r *http.Request) {
303+
w.Header().Set("Content-Length", "5")
304+
},
305+
func(w http.ResponseWriter, r *http.Request) {
306+
w.Header().Set("Content-Length", "1")
307+
},
308+
func(w http.ResponseWriter, r *http.Request) {
309+
w.Header().Set("Content-Length", "1")
310+
},
311+
func(w http.ResponseWriter, r *http.Request) {
312+
resp := `{
313+
"took": 1,
314+
"timed_out": false,
315+
"_shards": {
316+
"total": 2,
317+
"successful": 2,
318+
"skipped": 0,
319+
"failed": 0
320+
},
321+
"hits": {
322+
"total": 1,
323+
"max_score": null,
324+
"hits": [
325+
{
326+
"_index": "rp_elastic_test_2020_08_14_1",
327+
"_type": "_doc",
328+
"_id": "1",
329+
"_score": null,
330+
"_routing": "1",
331+
"_source": {
332+
"id": 1,
333+
"org_id": 1,
334+
"uuid": "c7a2dd87-a80e-420b-8431-ca48d422e924",
335+
"name": null,
336+
"language": "eng",
337+
"is_stopped": false,
338+
"is_blocked": false,
339+
"is_active": true,
340+
"created_on": "2017-11-10T16:11:59.890662-05:00",
341+
"modified_on": "2017-11-10T16:11:59.890662-05:00",
342+
"last_seen_on": "2020-08-04T21:11:00-04:00",
343+
"modified_on_mu": 1.510348319890662e15,
344+
"urns": [
345+
{
346+
"scheme": "tel",
347+
"path": "+12067791111"
348+
},
349+
{
350+
"scheme": "tel",
351+
"path": "+12067792222"
352+
}
353+
],
354+
"fields": [
355+
{
356+
"text": "the rock",
357+
"field": "17103bb1-1b48-4b70-92f7-1f6b73bd3488"
358+
}
359+
],
360+
"groups": [
361+
"4ea0f313-2f62-4e57-bdf0-232b5191dd57",
362+
"529bac39-550a-4d6f-817c-1833f3449007"
363+
]
364+
},
365+
"sort": [1]
366+
}
367+
]
368+
}
369+
}`
370+
371+
w.Write([]byte(resp))
372+
},
373+
}
374+
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
375+
responses[responseCounter](w, r)
376+
responseCounter++
377+
}))
378+
defer ts.Close()
379+
FindPhysicalIndexes(ts.URL, "rp_elastic_test")
380+
require.Equal(t, responseCounter, 4)
381+
}

0 commit comments

Comments
 (0)