Skip to content

Commit ff7e63b

Browse files
Robi9Robi9
authored
Merge pull request #25 from Ilhasoft/update/go-version-main
Update go version and add retry modifications
2 parents 59bf3f4 + 8b9c331 commit ff7e63b

File tree

6 files changed

+72
-12
lines changed

6 files changed

+72
-12
lines changed

‎WENI-CHANGELOG.md

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
1.2.0-indexer-8.2.0
2+
----------
3+
* Update go version and add retry modifications

‎cmd/rp-indexer/main.go

+9
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"time"
99

1010
"github.com/evalphobia/logrus_sentry"
11+
"github.com/getsentry/sentry-go"
1112
_ "github.com/lib/pq"
1213
"github.com/nyaruka/ezconf"
1314
indexer "github.com/nyaruka/rp-indexer/v8"
@@ -47,6 +48,14 @@ func main() {
4748
log.Fatalf("invalid sentry DSN: '%s': %s", cfg.SentryDSN, err)
4849
}
4950
log.StandardLogger().Hooks.Add(hook)
51+
52+
err = sentry.Init(sentry.ClientOptions{
53+
Dsn: cfg.SentryDSN,
54+
})
55+
if err != nil {
56+
log.Fatalf("Sentry init failed: %v", err)
57+
}
58+
defer sentry.Flush(2 * time.Second)
5059
}
5160

5261
db, err := sql.Open("postgres", cfg.DB)

‎docker/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM golang:1.17.3-alpine3.14
1+
FROM golang:1.19-alpine3.18
22

33
WORKDIR /app
44

‎go.mod

+6-3
Original file line numberDiff line numberDiff line change
@@ -10,15 +10,18 @@ require (
1010
github.com/olivere/elastic/v7 v7.0.32
1111
github.com/pkg/errors v0.9.1
1212
github.com/sirupsen/logrus v1.9.0
13-
github.com/stretchr/testify v1.8.0
13+
github.com/stretchr/testify v1.8.2
1414
)
1515

16+
require golang.org/x/text v0.14.0 // indirect
17+
1618
require (
1719
github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d // indirect
1820
github.com/davecgh/go-spew v1.1.1 // indirect
1921
github.com/fatih/structs v1.1.0 // indirect
2022
github.com/gabriel-vasile/mimetype v1.4.1 // indirect
2123
github.com/getsentry/raven-go v0.2.0 // indirect
24+
github.com/getsentry/sentry-go v0.29.0
2225
github.com/go-chi/chi v4.1.2+incompatible // indirect
2326
github.com/josharian/intern v1.0.0 // indirect
2427
github.com/kylelemons/godebug v1.1.0 // indirect
@@ -28,7 +31,7 @@ require (
2831
github.com/nyaruka/librato v1.0.0 // indirect
2932
github.com/pmezard/go-difflib v1.0.0 // indirect
3033
github.com/shopspring/decimal v1.3.1 // indirect
31-
golang.org/x/net v0.7.0 // indirect
32-
golang.org/x/sys v0.5.0 // indirect
34+
golang.org/x/net v0.23.0 // indirect
35+
golang.org/x/sys v0.18.0 // indirect
3336
gopkg.in/yaml.v3 v3.0.1 // indirect
3437
)

‎go.sum

+14-7
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,12 @@ github.com/gabriel-vasile/mimetype v1.4.1 h1:TRWk7se+TOjCYgRth7+1/OYLNiRNIotknkF
1313
github.com/gabriel-vasile/mimetype v1.4.1/go.mod h1:05Vi0w3Y9c/lNvJOdmIwvrrAhX3rYhfQQCaf9VJcv7M=
1414
github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs=
1515
github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
16+
github.com/getsentry/sentry-go v0.29.0 h1:YtWluuCFg9OfcqnaujpY918N/AhCCwarIDWOYSBAjCA=
17+
github.com/getsentry/sentry-go v0.29.0/go.mod h1:jhPesDAL0Q0W2+2YEuVOvdWmVtdsr1+jtBrlDEVWwLY=
1618
github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec=
1719
github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
18-
github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
20+
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
21+
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
1922
github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g=
2023
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
2124
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
@@ -38,6 +41,7 @@ github.com/nyaruka/librato v1.0.0 h1:Vznj9WCeC1yZXbBYyYp40KnbmXLbEkjKmHesV/v2SR0
3841
github.com/nyaruka/librato v1.0.0/go.mod h1:pkRNLFhFurOz0QqBz6/DuTFhHHxAubWxs4Jx+J7yUgg=
3942
github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E=
4043
github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
44+
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
4145
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
4246
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
4347
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -50,24 +54,27 @@ github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVs
5054
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
5155
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
5256
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
57+
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
5358
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
5459
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
5560
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
5661
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
57-
github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
5862
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
63+
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
64+
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
5965
golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
60-
golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
61-
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
66+
golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
67+
golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
6268
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
6369
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
6470
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
6571
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
66-
golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
67-
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
72+
golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
73+
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
6874
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
6975
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
70-
golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
76+
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
77+
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
7178
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
7279
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
7380
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

‎utils/http.go

+39-1
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"net/http"
99
"time"
1010

11+
"github.com/getsentry/sentry-go"
1112
"github.com/nyaruka/gocommon/httpx"
1213
log "github.com/sirupsen/logrus"
1314
)
@@ -37,6 +38,11 @@ func shouldRetry(request *http.Request, response *http.Response, withDelay time.
3738
return true
3839
}
3940

41+
if response.StatusCode == http.StatusBadRequest {
42+
fmt.Println("Retry error 400")
43+
return true
44+
}
45+
4046
// check for unexpected EOF
4147
bodyBytes, err := ioutil.ReadAll(response.Body)
4248
response.Body.Close()
@@ -55,6 +61,18 @@ func MakeJSONRequest(method string, url string, body []byte, dest any) (*http.Re
5561

5662
req, _ := httpx.NewRequest(method, url, bytes.NewReader(body), map[string]string{"Content-Type": "application/json"})
5763
resp, err := httpx.Do(http.DefaultClient, req, retryConfig, nil)
64+
65+
originalSize := len(body)
66+
67+
l.WithField("request header", req.Header).WithField("response header", resp.Header)
68+
var formattedJSON bytes.Buffer
69+
if err := json.Indent(&formattedJSON, body, "", " "); err != nil {
70+
formattedJSON.Write(body)
71+
}
72+
formattedSize := formattedJSON.Len()
73+
74+
l.WithField("request", formattedJSON.String())
75+
5876
if err != nil {
5977
l.WithError(err).Error("error making request")
6078
return resp, err
@@ -73,7 +91,9 @@ func MakeJSONRequest(method string, url string, body []byte, dest any) (*http.Re
7391
// error if we got a non-200
7492
if resp.StatusCode != http.StatusOK {
7593
l.WithError(err).Error("error reaching ES")
76-
return resp, fmt.Errorf("received non-200 response %d: %s", resp.StatusCode, respBody)
94+
time.Sleep(time.Second * 5)
95+
sendFileToSentry(formattedJSON, originalSize, formattedSize, fmt.Errorf("received non 200 response %d: %s", resp.StatusCode, respBody), resp)
96+
return resp, fmt.Errorf("received non 200 response %d: %s", resp.StatusCode, respBody)
7797
}
7898

7999
if dest != nil {
@@ -86,3 +106,21 @@ func MakeJSONRequest(method string, url string, body []byte, dest any) (*http.Re
86106

87107
return resp, nil
88108
}
109+
110+
func sendFileToSentry(formattedJSON bytes.Buffer, originalSize int, formattedSize int, err error, resp *http.Response) {
111+
sentry.WithScope(func(scope *sentry.Scope) {
112+
scope.SetExtra("original_body_size", originalSize)
113+
scope.SetExtra("formatted_body_size", formattedSize)
114+
scope.SetExtra("response header", resp.Header)
115+
scope.SetExtra("response body", resp.Body)
116+
scope.SetExtra("status", resp.StatusCode)
117+
scope.AddAttachment(&sentry.Attachment{
118+
Filename: "request_body.json",
119+
ContentType: "application/json",
120+
Payload: formattedJSON.Bytes(),
121+
})
122+
123+
sentry.CaptureException(err)
124+
})
125+
sentry.Flush(6 * time.Second)
126+
}

0 commit comments

Comments
 (0)