From c121130057834eba400bbdb8a62e6e452487540b Mon Sep 17 00:00:00 2001 From: Gregory Boddin Date: Sun, 3 Oct 2021 21:38:56 +0200 Subject: [PATCH] Preparing for v2 release, using l9format --- Client.go | 17 +++---- SearchResult.go | 25 ----------- build-archs.sh | 26 +++++------ cmd/leakix-ns/main.go | 102 ------------------------------------------ cmd/leakix/main.go | 3 +- example/main.go | 4 +- go.mod | 13 ++++++ go.sum | 40 +++++++++++++++++ 8 files changed, 79 insertions(+), 151 deletions(-) delete mode 100644 SearchResult.go delete mode 100644 cmd/leakix-ns/main.go create mode 100644 go.mod create mode 100644 go.sum diff --git a/Client.go b/Client.go index fadd33b..1940017 100644 --- a/Client.go +++ b/Client.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/LeakIX/l9format" "github.com/gorilla/websocket" "gitlab.nobody.run/tbi/core" "io/ioutil" @@ -28,7 +29,7 @@ var HttpClient = &http.Client{ type SearchResultsClient struct { Scope string Query string - SearchResults []SearchResult + SearchResults []l9format.L9Event Position int Page int ApiKey string @@ -46,7 +47,7 @@ func (sc *SearchResultsClient) GetEndpoint() string { } func (sc *SearchResultsClient) Next() bool { - var results []SearchResult + var results []l9format.L9Event if len(sc.SearchResults) > sc.Position { sc.Position++ return true @@ -64,14 +65,14 @@ func (sc *SearchResultsClient) Next() bool { return false } -func (sc *SearchResultsClient) SearchResult() SearchResult { +func (sc *SearchResultsClient) SearchResult() l9format.L9Event { return sc.SearchResults[sc.Position-1] } -func (sc *SearchResultsClient) GetSearchResults(scope string, query string, page int) ([]SearchResult, error) { +func (sc *SearchResultsClient) GetSearchResults(scope string, query string, page int) ([]l9format.L9Event, error) { url := fmt.Sprintf( "%s/search?scope=%s&q=%s&page=%d", sc.GetEndpoint(), url2.QueryEscape(scope), url2.QueryEscape(query), page) - var searchResults []SearchResult + var searchResults []l9format.L9Event req, _ := http.NewRequest("GET", url, nil) req.Header.Set("Accept", "application/json") req.Header.Set("api-key", sc.ApiKey) @@ -91,8 +92,8 @@ func (sc *SearchResultsClient) GetSearchResults(scope string, query string, page return searchResults, nil } -func (sc *SearchResultsClient) GetChannel(scope string) (chan SearchResult, error) { - channel := make(chan SearchResult) +func (sc *SearchResultsClient) GetChannel(scope string) (chan l9format.L9Event, error) { + channel := make(chan l9format.L9Event) endpointUrl, err := url2.Parse(sc.GetEndpoint()) if err != nil { return nil, errors.New("invalid endpoint") @@ -107,7 +108,7 @@ func (sc *SearchResultsClient) GetChannel(scope string) (chan SearchResult, erro return nil, err } go func() { - searchResult := SearchResult{} + searchResult := l9format.L9Event{} for { err := wsConnection.ReadJSON(&searchResult) if err != nil { diff --git a/SearchResult.go b/SearchResult.go deleted file mode 100644 index 4b26279..0000000 --- a/SearchResult.go +++ /dev/null @@ -1,25 +0,0 @@ -package LeakIXClient - -import ( - "gitlab.nobody.run/tbi/core" - "time" -) - -// Single leak occurence on a single service -type SearchResult struct { - Ip string `json:"ip"` - Port string `json:"port"` - Type string `json:"type"` - Time time.Time `json:"time"` - Date int64 `json:"date"` - Data string `json:"data"` - Headers map[string][]string `json:"headers"` - Plugin string `json:"plugin"` - Network core.Network `json:"network"` - GeoLocation core.GeoLocation `json:"geoip"` - Credentials []*core.HostServiceCredentials `json:"credentials"` - Software core.Software `json:"software"` - Reverse string `json:"reverse"` - Hostname string `json:"hostname"` - Dataset core.DatasetLeak `json:"dataset"` -} \ No newline at end of file diff --git a/build-archs.sh b/build-archs.sh index 914de9f..10a54ac 100644 --- a/build-archs.sh +++ b/build-archs.sh @@ -17,21 +17,21 @@ GOOS=netbsd GOARCH=386 go build -ldflags="-s -w" -o bin/leakix-ns-netbsd-32 ./cm GOOS=openbsd GOARCH=amd64 go build -ldflags="-s -w" -o bin/leakix-ns-openbsd-64 ./cmd/leakix-ns & GOOS=openbsd GOARCH=386 go build -ldflags="-s -w" -o bin/leakix-ns-openbsd-32 ./cmd/leakix-ns & -GOOS=linux GOARCH=386 go build -ldflags="-s -w" -o bin/leakix-linux-32 ./cmd/leakix & -GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o bin/leakix-linux-64 ./cmd/leakix & -GOOS=freebsd GOARCH=amd64 go build -ldflags="-s -w" -o bin/leakix-freebsd-64 ./cmd/leakix & -GOOS=freebsd GOARCH=386 go build -ldflags="-s -w" -o bin/leakix-freebsd-32 ./cmd/leakix & +CGO_ENABLED=0 GOOS=linux GOARCH=386 go build -ldflags="-s -w -extldflags '-static'" -o bin/leakix-linux-32 ./cmd/leakix & +CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w -extldflags '-static'" -o bin/leakix-linux-64 ./cmd/leakix & +CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 go build -ldflags="-s -w -extldflags '-static'" -o bin/leakix-freebsd-64 ./cmd/leakix & +CGO_ENABLED=0 GOOS=freebsd GOARCH=386 go build -ldflags="-s -w -extldflags '-static'" -o bin/leakix-freebsd-32 ./cmd/leakix & -GOOS=linux GOARCH=arm GOARM=7 go build -ldflags="-s -w" -o bin/leakix-linux-arm7 ./cmd/leakix & -GOOS=linux GOARCH=arm GOARM=6 go build -ldflags="-s -w" -o bin/leakix-linux-arm6 ./cmd/leakix & +CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=7 go build -ldflags="-s -w -extldflags '-static'" -o bin/leakix-linux-arm7 ./cmd/leakix & +CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=6 go build -ldflags="-s -w -extldflags '-static'" -o bin/leakix-linux-arm6 ./cmd/leakix & -GOOS=darwin GOARCH=amd64 go build -ldflags="-s -w" -o bin/leakix-osx ./cmd/leakix & +CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -ldflags="-s -w -extldflags '-static'" -o bin/leakix-osx ./cmd/leakix & -GOOS=windows GOARCH=amd64 go build -ldflags="-s -w" -o bin/leakix-win64.exe ./cmd/leakix & -GOOS=windows GOARCH=386 go build -ldflags="-s -w" -o bin/leakix-win32.exe ./cmd/leakix & +CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -ldflags="-s -w -extldflags '-static'" -o bin/leakix-win64.exe ./cmd/leakix & +CGO_ENABLED=0 GOOS=windows GOARCH=386 go build -ldflags="-s -w -extldflags '-static'" -o bin/leakix-win32.exe ./cmd/leakix & -GOOS=netbsd GOARCH=amd64 go build -ldflags="-s -w" -o bin/leakix-netbsd-64 ./cmd/leakix & -GOOS=netbsd GOARCH=386 go build -ldflags="-s -w" -o bin/leakix-netbsd-32 ./cmd/leakix & +CGO_ENABLED=0 GOOS=netbsd GOARCH=amd64 go build -ldflags="-s -w -extldflags '-static'" -o bin/leakix-netbsd-64 ./cmd/leakix & +CGO_ENABLED=0 GOOS=netbsd GOARCH=386 go build -ldflags="-s -w -extldflags '-static'" -o bin/leakix-netbsd-32 ./cmd/leakix & -GOOS=openbsd GOARCH=amd64 go build -ldflags="-s -w" -o bin/leakix-openbsd-64 ./cmd/leakix & -GOOS=openbsd GOARCH=386 go build -ldflags="-s -w" -o bin/leakix-openbsd-32 ./cmd/leakix & +CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 go build -ldflags="-s -w -extldflags '-static'" -o bin/leakix-openbsd-64 ./cmd/leakix & +CGO_ENABLED=0 GOOS=openbsd GOARCH=386 go build -ldflags="-s -w -extldflags '-static'" -o bin/leakix-openbsd-32 ./cmd/leakix & \ No newline at end of file diff --git a/cmd/leakix-ns/main.go b/cmd/leakix-ns/main.go deleted file mode 100644 index f618c49..0000000 --- a/cmd/leakix-ns/main.go +++ /dev/null @@ -1,102 +0,0 @@ -package main - -import ( - "encoding/json" - "flag" - "fmt" - "github.com/LeakIX/LeakIXClient" - "os" - "strings" -) - -func main() { - //Config our app - app := App{} - flag.StringVar(&app.Domain, "d", "", "Specify domain") - flag.BoolVar(&app.OutputJson, "j", false, "JSON mode, (excludes -t)") - flag.IntVar(&app.Limit, "l", 100, "Limit results output") - flag.StringVar(&app.ApiKey, "k", "", "API Key") - flag.Usage = func() { - fmt.Printf("Usage of leakix-ns: \n") - fmt.Printf(" ./leakix-ns -d -l 200\n\n") - flag.PrintDefaults() - } - flag.Parse() - if len(app.Domain) < 2 { - flag.Usage() - os.Exit(1) - } - app.Run() -} - -type App struct { - Domain string - OutputJson bool - Limit int - Searcher *LeakIXClient.SearchResultsClient - Reverse map[string][]LeakIXClient.SearchResult - Forward map[string][]LeakIXClient.SearchResult - ApiKey string -} - -func (app *App) Run() { - app.Searcher = &LeakIXClient.SearchResultsClient{ - Scope: "service", - Query: fmt.Sprintf("hostname:\"%s\" OR reverse:\"%s\" OR ip:\"%s\"", app.Domain, app.Domain, app.Domain), - ApiKey: app.ApiKey, - Endpoint: "https://leakix.net", - } - app.Reverse = make(map[string][]LeakIXClient.SearchResult) - app.Forward = make(map[string][]LeakIXClient.SearchResult) - count := 0 - for app.Searcher.Next() { - if !strings.Contains(app.Searcher.SearchResult().Reverse, app.Domain) && - !strings.Contains(app.Searcher.SearchResult().Hostname, app.Domain) && - !strings.Contains(app.Searcher.SearchResult().Ip, app.Domain) { - continue - } - count++ - if count > app.Limit { - break - } - if app.OutputJson { - jsonLine, _ := json.Marshal(app.Searcher.SearchResult()) - fmt.Println(string(jsonLine)) - continue - } - reverse := strings.TrimRight(app.Searcher.SearchResult().Reverse, ".") - ip := app.Searcher.SearchResult().Ip - hostname := app.Searcher.SearchResult().Hostname - - if hostname != ip && len(hostname) > 2 && (strings.Contains(hostname, app.Domain) || app.Domain == ip) { - app.Forward[hostname] = append(app.Forward[hostname], app.Searcher.SearchResult()) - } - if len(reverse) > 1 && (strings.Contains(reverse, app.Domain) || app.Domain == ip) { - app.Reverse[reverse] = append(app.Reverse[reverse], app.Searcher.SearchResult()) - } - } - if app.OutputJson { - os.Exit(0) - } - fmt.Println("PTR records :") - for reverseName, results := range app.Reverse { - for _, result := range results { - fmt.Printf("[%s] %s <- %s", result.Time.Format("02-01-2006 15:04"), reverseName, result.Ip) - if len(result.Hostname) > 1 && result.Hostname != result.Ip { - fmt.Printf(" -> %s", result.Hostname) - } - fmt.Println() - } - } - fmt.Println("Forward records :") - - for forwardName, results := range app.Forward { - for _, result := range results { - fmt.Printf("[%s] %s -> %s", result.Time.Format("02-01-2006 15:04"), forwardName, result.Ip) - if len(result.Reverse) > 1 { - fmt.Printf(" <- %s", result.Reverse) - } - fmt.Println() - } - } -} diff --git a/cmd/leakix/main.go b/cmd/leakix/main.go index 9c80f35..37f29d2 100644 --- a/cmd/leakix/main.go +++ b/cmd/leakix/main.go @@ -5,6 +5,7 @@ import ( "flag" "fmt" "github.com/LeakIX/LeakIXClient" + "github.com/LeakIX/l9format" "log" "os" "text/template" @@ -97,7 +98,7 @@ func LiveStream() { } } -func OutputSearchResult(searchResult LeakIXClient.SearchResult) { +func OutputSearchResult(searchResult l9format.L9Event) { if outputJson { jsonBody, err := json.Marshal(searchResult) if err != nil { diff --git a/example/main.go b/example/main.go index aaabc65..42f361c 100644 --- a/example/main.go +++ b/example/main.go @@ -8,12 +8,12 @@ func main(){ // Create a searcher LeakIXSearch := LeakIXClient.SearchResultsClient{ Scope: "leak", - Query: "protocol:kafka AND \"telecom_italia_data\"", + Query: "+protocol:elasticsearch +\"telecom_italia_data\"", } // Iterate, the lib will query further pages if needed for LeakIXSearch.Next() { // Use the result leak := LeakIXSearch.SearchResult() - fmt.Printf("%s:%s - Country:%s\n", leak.Ip, leak.Port, leak.GeoLocation.CountryName) + fmt.Printf("%s:%s - Country:%s\n", leak.Ip, leak.Port, leak.GeoIp.CountryName) } } diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..8fd319a --- /dev/null +++ b/go.mod @@ -0,0 +1,13 @@ +module github.com/LeakIX/LeakIXClient + +go 1.15 + +require ( + github.com/LeakIX/l9format v1.3.0 + github.com/Machiel/slugify v1.0.1 // indirect + github.com/PuerkitoBio/goquery v1.7.1 // indirect + github.com/andybalholm/cascadia v1.3.1 // indirect + github.com/gorilla/websocket v1.4.2 + gitlab.nobody.run/tbi/core v0.0.0-20201124161030-42e21c6deb8e + golang.org/x/net v0.0.0-20210929193557-e81a3d93ecf6 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..5537815 --- /dev/null +++ b/go.sum @@ -0,0 +1,40 @@ +github.com/LeakIX/l9format v1.3.0-beta.2 h1:2zwQryPYFvxju25vjL9TXLdActgVmjpDJ/zgc2Fb5hY= +github.com/LeakIX/l9format v1.3.0-beta.2/go.mod h1:eKQn32c5PgUM7806Un2v6WTSmJcdcixed+cRHsPEp0k= +github.com/LeakIX/l9format v1.3.0 h1:IrWuvYQ6ayEwf3oTTWG54D3oVfLApQDd0qyKdKK5mw8= +github.com/LeakIX/l9format v1.3.0/go.mod h1:eKQn32c5PgUM7806Un2v6WTSmJcdcixed+cRHsPEp0k= +github.com/Machiel/slugify v1.0.1 h1:EfWSlRWstMadsgzmiV7d0yVd2IFlagWH68Q+DcYCm4E= +github.com/Machiel/slugify v1.0.1/go.mod h1:fTFGn5uWEynW4CUMG7sWkYXOf1UgDxyTM3DbR6Qfg3k= +github.com/PuerkitoBio/goquery v1.6.1 h1:FgjbQZKl5HTmcn4sKBgvx8vv63nhyhIpv7lJpFGCWpk= +github.com/PuerkitoBio/goquery v1.6.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= +github.com/PuerkitoBio/goquery v1.7.1 h1:oE+T06D+1T7LNrn91B4aERsRIeCLJ/oPSa6xB9FPnz4= +github.com/PuerkitoBio/goquery v1.7.1/go.mod h1:XY0pP4kfraEmmV1O7Uf6XyjoslwsneBbgeDjLYuN8xY= +github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/andybalholm/cascadia v1.2.0 h1:vuRCkM5Ozh/BfmsaTm26kbjm0mIOM3yS5Ek/F5h18aE= +github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY= +github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= +github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +gitlab.nobody.run/tbi/core v0.0.0-20201124161030-42e21c6deb8e h1:0EEfNlVSZtV1q/CqAXpoPcAgyim7s04kAKVmRkhUOfY= +gitlab.nobody.run/tbi/core v0.0.0-20201124161030-42e21c6deb8e/go.mod h1:obLTFd7diUMk/cXeXJpgYdsj5OKn5c0EARFPatcZQCk= +gitlab.nobody.run/tbi/socksme v0.0.0-20201130024528-3ad02d4108c5 h1:OUBbWsRDwvD6Nu6rP8PN6xGxIxXpVPBPBE9iuvDQH90= +gitlab.nobody.run/tbi/socksme v0.0.0-20201130024528-3ad02d4108c5/go.mod h1:2R0aT7bhH9+xiSielZpKS/TL4nTki1jxOCCgXN8BzV8= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201216054612-986b41b23924 h1:QsnDpLLOKwHBBDa8nDws4DYNc/ryVW2vCpxCs09d4PY= +golang.org/x/net v0.0.0-20201216054612-986b41b23924/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210929193557-e81a3d93ecf6 h1:Z04ewVs7JhXaYkmDhBERPi41gnltfQpMWDnTnQbaCqk= +golang.org/x/net v0.0.0-20210929193557-e81a3d93ecf6/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=