From 2c964f832408bfb9c46710be3829ec2d6479fb25 Mon Sep 17 00:00:00 2001 From: Gregory Boddin Date: Fri, 14 Aug 2020 16:38:27 +0200 Subject: [PATCH] [alpha] Implement realtime stream from staging --- Client.go | 32 ++++++++++++++---- README.md | 32 ++++++++++++++++-- cmd/leakix/main.go | 84 ++++++++++++++++++++++++++++++++++++---------- 3 files changed, 121 insertions(+), 27 deletions(-) diff --git a/Client.go b/Client.go index 4901e31..bee518e 100644 --- a/Client.go +++ b/Client.go @@ -3,6 +3,7 @@ package LeakIXClient import ( "encoding/json" "fmt" + "github.com/gorilla/websocket" "gitlab.nobody.run/tbi/core" "io/ioutil" "log" @@ -25,7 +26,7 @@ var HttpClient = &http.Client{ type SearchResultsClient struct { Scope string Query string - SearchResults []*SearchResult + SearchResults []SearchResult Position int Page int } @@ -48,31 +49,48 @@ func (sc *SearchResultsClient) Next() bool { return false } -func (sc *SearchResultsClient) SearchResult() *SearchResult { +func (sc *SearchResultsClient) SearchResult() SearchResult { return sc.SearchResults[sc.Position-1] } -func GetSearchResults(scope string, query string, page int) ([]*SearchResult, error) { +func GetSearchResults(scope string, query string, page int) ([]SearchResult, error) { url := fmt.Sprintf( "https://leakix.net/search?scope=%s&q=%s&page=%d", url2.QueryEscape(scope), url2.QueryEscape(query), page) - var searchResults []*SearchResult + var searchResults []SearchResult req, _ := http.NewRequest("GET", url, nil) req.Header.Set("Accept", "application/json") resp, err := HttpClient.Do(req) if err != nil { - log.Println(err) return searchResults, err } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { - log.Println(err) return searchResults, err } err = json.Unmarshal(body, &searchResults) if err != nil { - log.Println(err) return searchResults, err } return searchResults, nil } + +func GetChannel(scope string) (chan SearchResult, error) { + channel := make(chan SearchResult) + wsConnection, _, err := websocket.DefaultDialer.Dial("wss://staging.leakix.net/ws/" + scope, map[string][]string{"Origin":{"staging.leakix.net:443"}}) + if err != nil { + return nil, err + } + go func() { + searchResult := SearchResult{} + for { + err := wsConnection.ReadJSON(&searchResult) + if err != nil { + log.Println("Error parsing websocket results. Is your scope correct?") + log.Fatal(err) + } + channel <- searchResult + } + }() + return channel, nil +} \ No newline at end of file diff --git a/README.md b/README.md index 4ad8868..faeaa89 100644 --- a/README.md +++ b/README.md @@ -7,17 +7,19 @@ This is a Go CLI & library making queries to LeakIX easier. ```sh $ leakix -h Usage of leakix: - ./leakix -q '*' -l 200 + -j JSON mode, (excludes -t) -l int Limit results output (default 100) -q string - Specify search query (default "*") + Search mode, specify search query (default "*") + -r Realtime mode, (excludes -q) -s string Specify scope (default "leak") -t string Specify output template (default "{{ .Ip }}:{{ .Port }}") +$ # Example query on the index $ leakix -l 2 -q "protocol:web AND plugin:GitConfigPlugin" -t "{{ .Ip }}:{{ .Port }} : {{ .Data }}" 178.62.217.44:80 : Found git deployment configuration [core] @@ -44,6 +46,16 @@ $ leakix -l 2 -q "protocol:web AND plugin:GitConfigPlugin" -t "{{ .Ip }}:{{ .Por [branch "staging"] remote = origin merge = refs/heads/staging + +$ # Stream results in realtime from the engine, no filtering +$ ./leakix -r -s services -l 0 +14.167.7.149:81 +54.249.38.136:9200 +23.65.39.190:80 +[2a01:4f8:10a:1b5a::2]:80 +23.225.38.43:3306 +210.16.68.51:80 +...keeps streaming... ``` ## Library usage @@ -55,7 +67,7 @@ import ( "github.com/LeakIX/LeakIXClient" ) -func main(){ +func DoSearch(){ // Create a searcher LeakIXSearch := LeakIXClient.SearchResultsClient{ Scope: "leak", @@ -69,4 +81,18 @@ func main(){ } } + +func LiveStream() { + // Get a channel from the websocket + serviceChannel, err := LeakIXClient.GetChannel("services") + if err != nil { + log.Println("Websocket connection error:") + log.Fatal(err) + } + for { + // Print everything received on the channel + service := <- serviceChannel + log.Println(service.Ip) + } +} ``` diff --git a/cmd/leakix/main.go b/cmd/leakix/main.go index 0d45cfd..4538ebe 100644 --- a/cmd/leakix/main.go +++ b/cmd/leakix/main.go @@ -1,52 +1,102 @@ package main import ( + "encoding/json" "flag" "fmt" "github.com/LeakIX/LeakIXClient" - "text/template" "log" "os" + "text/template" ) +// It's all sequential, Couldn't care sorry : +var outputJson bool +var scope string +var query string +var liveStream bool +var outputTemplate string +var limit int +var tmpl *template.Template +var err error + func main() { - var scope string + //Config our app + err = nil flag.StringVar(&scope, "s", "leak", "Specify scope") - var query string - flag.StringVar(&query, "q", "*", "Specify search query") - var outputTemplate string + flag.StringVar(&query, "q", "*", "Search mode, specify search query") + flag.BoolVar(&liveStream, "r", false, "Realtime mode, (excludes -q)") + flag.BoolVar(&outputJson, "j", false, "JSON mode, (excludes -t)") flag.StringVar(&outputTemplate, "t", "{{ .Ip }}:{{ .Port }}", "Specify output template") - var limit int flag.IntVar(&limit, "l", 100, "Limit results output") - flag.Usage = func() { fmt.Printf("Usage of leakix: \n") - fmt.Printf(" ./leakix -q '*' -l 200\n\n") + fmt.Printf(" ./leakix -q '*' -s leaks -l 200\n\n") + fmt.Printf(" ./leakix -r -s services -l 0\n\n") flag.PrintDefaults() } - flag.Parse() + tmpl, err = template.New("output").Parse(outputTemplate + "\n") + if err != nil { + log.Println("Template error :") + log.Fatal(err) + } + // Run the right thing + if liveStream { + LiveStream() + } else { + Search() + } + +} + +func Search() { searcher := LeakIXClient.SearchResultsClient{ Scope: scope, Query: query, } + count := 0 + for searcher.Next() { + count++ + OutputSearchResult(searcher.SearchResult()) + if count >= limit || count >= 10000 { + os.Exit(0) + } + } +} - tmpl, err := template.New("output").Parse(outputTemplate + "\n") +func LiveStream() { + count := 0 + serviceChannel, err := LeakIXClient.GetChannel(scope) if err != nil { - log.Println("Template error :") + log.Println("Websocket connection error:") log.Fatal(err) } - count := 0 - for searcher.Next() { + for { + service := <- serviceChannel count++ - err = tmpl.Execute(os.Stdout, searcher.SearchResult()) + OutputSearchResult(service) + if count >= limit && limit != 0 { + os.Exit(0) + } + } +} + +func OutputSearchResult(searchResult LeakIXClient.SearchResult) { + if outputJson { + jsonBody, err := json.Marshal(searchResult) if err != nil { - log.Println("Template error :") + log.Println("JSON error :") log.Fatal(err) } - if count >= limit { - os.Exit(0) + fmt.Println(string(jsonBody)) + } else { + err := tmpl.Execute(os.Stdout, searchResult) + if err != nil { + log.Println("Template error :") + log.Fatal(err) } } } +