Skip to content

Commit

Permalink
Added support for rate limit header, removed gitlab dep
Browse files Browse the repository at this point in the history
  • Loading branch information
@gboddin
gboddin committed Dec 18, 2021
1 parent 9c31572 commit 3ce735c
Show file tree
Hide file tree
Showing 3 changed files with 21 additions and 31 deletions.
16 changes: 12 additions & 4 deletions Client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import (
"fmt"
"github.com/LeakIX/l9format"
"github.com/gorilla/websocket"
"gitlab.nobody.run/tbi/core"
"io/ioutil"
"log"
"net/http"
Expand All @@ -15,7 +14,7 @@ import (
"time"
)

var LeakIXProxy = &core.ProxiedPlugin{}
var LeakIXProxy = &l9format.ServicePluginBase{}
var LeakIXHttpTranport = &http.Transport{
DialContext: LeakIXProxy.DialContext,
ResponseHeaderTimeout: 5 * time.Second,
Expand Down Expand Up @@ -85,6 +84,15 @@ func (sc *SearchResultsClient) GetSearchResults(scope string, query string, page
if err != nil {
return searchResults, err
}
if resp.StatusCode == 429 {
// parse wait header and try again
duration, err := time.ParseDuration(resp.Header.Get("x-limited-for"))
if err != nil {
panic(err)
}
time.Sleep(duration)
return sc.GetSearchResults(scope, query, page)
}
err = json.Unmarshal(body, &searchResults)
if err != nil {
return searchResults, err
Expand All @@ -101,8 +109,8 @@ func (sc *SearchResultsClient) GetChannel(scope string) (chan l9format.L9Event,
endpointUrl.Scheme = strings.Replace(endpointUrl.Scheme, "http", "ws", -1)
log.Println(endpointUrl.String())
wsConnection, _, err := websocket.DefaultDialer.Dial(endpointUrl.String()+"/ws/"+scope, map[string][]string{
"Origin": {endpointUrl.Host + ":" + endpointUrl.Port()},
"api-key":{sc.ApiKey},
"Origin": {endpointUrl.Host + ":" + endpointUrl.Port()},
"api-key": {sc.ApiKey},
})
if err != nil {
return nil, err
Expand Down
9 changes: 3 additions & 6 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,8 @@ module github.com/LeakIX/LeakIXClient
go 1.15

require (
github.com/LeakIX/l9format v1.3.0
github.com/Machiel/slugify v1.0.1 // indirect
github.com/PuerkitoBio/goquery v1.7.1 // indirect
github.com/andybalholm/cascadia v1.3.1 // indirect
github.com/LeakIX/l9format v1.3.1
github.com/PuerkitoBio/goquery v1.8.0 // indirect
github.com/gorilla/websocket v1.4.2
gitlab.nobody.run/tbi/core v0.0.0-20201124161030-42e21c6deb8e
golang.org/x/net v0.0.0-20210929193557-e81a3d93ecf6 // indirect
golang.org/x/net v0.0.0-20211216030914-fe4d6282115f // indirect
)
27 changes: 6 additions & 21 deletions go.sum
View file
Original file line number Diff line number Diff line change
@@ -1,40 +1,25 @@
github.com/LeakIX/l9format v1.3.0-beta.2 h1:2zwQryPYFvxju25vjL9TXLdActgVmjpDJ/zgc2Fb5hY=
github.com/LeakIX/l9format v1.3.0-beta.2/go.mod h1:eKQn32c5PgUM7806Un2v6WTSmJcdcixed+cRHsPEp0k=
github.com/LeakIX/l9format v1.3.0 h1:IrWuvYQ6ayEwf3oTTWG54D3oVfLApQDd0qyKdKK5mw8=
github.com/LeakIX/l9format v1.3.0/go.mod h1:eKQn32c5PgUM7806Un2v6WTSmJcdcixed+cRHsPEp0k=
github.com/Machiel/slugify v1.0.1 h1:EfWSlRWstMadsgzmiV7d0yVd2IFlagWH68Q+DcYCm4E=
github.com/Machiel/slugify v1.0.1/go.mod h1:fTFGn5uWEynW4CUMG7sWkYXOf1UgDxyTM3DbR6Qfg3k=
github.com/PuerkitoBio/goquery v1.6.1 h1:FgjbQZKl5HTmcn4sKBgvx8vv63nhyhIpv7lJpFGCWpk=
github.com/LeakIX/l9format v1.3.1 h1:IjkVAwzijtQGVar9py93IDGkW5pNiByPgK2xzupxwig=
github.com/LeakIX/l9format v1.3.1/go.mod h1:ra1Z8Zxia6ucdVubZmtuKDOoSVHLxkpT1Sklqbzg+JU=
github.com/PuerkitoBio/goquery v1.6.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
github.com/PuerkitoBio/goquery v1.7.1 h1:oE+T06D+1T7LNrn91B4aERsRIeCLJ/oPSa6xB9FPnz4=
github.com/PuerkitoBio/goquery v1.7.1/go.mod h1:XY0pP4kfraEmmV1O7Uf6XyjoslwsneBbgeDjLYuN8xY=
github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
github.com/andybalholm/cascadia v1.2.0 h1:vuRCkM5Ozh/BfmsaTm26kbjm0mIOM3yS5Ek/F5h18aE=
github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY=
github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
gitlab.nobody.run/tbi/core v0.0.0-20201124161030-42e21c6deb8e h1:0EEfNlVSZtV1q/CqAXpoPcAgyim7s04kAKVmRkhUOfY=
gitlab.nobody.run/tbi/core v0.0.0-20201124161030-42e21c6deb8e/go.mod h1:obLTFd7diUMk/cXeXJpgYdsj5OKn5c0EARFPatcZQCk=
gitlab.nobody.run/tbi/socksme v0.0.0-20201130024528-3ad02d4108c5 h1:OUBbWsRDwvD6Nu6rP8PN6xGxIxXpVPBPBE9iuvDQH90=
gitlab.nobody.run/tbi/socksme v0.0.0-20201130024528-3ad02d4108c5/go.mod h1:2R0aT7bhH9+xiSielZpKS/TL4nTki1jxOCCgXN8BzV8=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201216054612-986b41b23924 h1:QsnDpLLOKwHBBDa8nDws4DYNc/ryVW2vCpxCs09d4PY=
golang.org/x/net v0.0.0-20201216054612-986b41b23924/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0=
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210929193557-e81a3d93ecf6 h1:Z04ewVs7JhXaYkmDhBERPi41gnltfQpMWDnTnQbaCqk=
golang.org/x/net v0.0.0-20210929193557-e81a3d93ecf6/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211216030914-fe4d6282115f h1:hEYJvxw1lSnWIl8X9ofsYMklzaDs90JI2az5YMd4fPM=
golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

0 comments on commit 3ce735c

Please sign in to comment.