Skip to content

Commit

Permalink
[alpha] Implement realtime stream from staging
Browse files Browse the repository at this point in the history
  • Loading branch information
gboddin committed Aug 14, 2020
1 parent 26bf4a4 commit 2c964f8
Show file tree
Hide file tree
Showing 3 changed files with 121 additions and 27 deletions.
32 changes: 25 additions & 7 deletions Client.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package LeakIXClient
import (
"encoding/json"
"fmt"
"github.com/gorilla/websocket"
"gitlab.nobody.run/tbi/core"
"io/ioutil"
"log"
Expand All @@ -25,7 +26,7 @@ var HttpClient = &http.Client{
type SearchResultsClient struct {
Scope string
Query string
SearchResults []*SearchResult
SearchResults []SearchResult
Position int
Page int
}
Expand All @@ -48,31 +49,48 @@ func (sc *SearchResultsClient) Next() bool {
return false
}

func (sc *SearchResultsClient) SearchResult() *SearchResult {
func (sc *SearchResultsClient) SearchResult() SearchResult {
return sc.SearchResults[sc.Position-1]
}

func GetSearchResults(scope string, query string, page int) ([]*SearchResult, error) {
func GetSearchResults(scope string, query string, page int) ([]SearchResult, error) {
url := fmt.Sprintf(
"https://leakix.net/search?scope=%s&q=%s&page=%d", url2.QueryEscape(scope), url2.QueryEscape(query), page)
var searchResults []*SearchResult
var searchResults []SearchResult
req, _ := http.NewRequest("GET", url, nil)
req.Header.Set("Accept", "application/json")
resp, err := HttpClient.Do(req)
if err != nil {
log.Println(err)
return searchResults, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Println(err)
return searchResults, err
}
err = json.Unmarshal(body, &searchResults)
if err != nil {
log.Println(err)
return searchResults, err
}
return searchResults, nil
}

func GetChannel(scope string) (chan SearchResult, error) {
channel := make(chan SearchResult)
wsConnection, _, err := websocket.DefaultDialer.Dial("wss://staging.leakix.net/ws/" + scope, map[string][]string{"Origin":{"staging.leakix.net:443"}})
if err != nil {
return nil, err
}
go func() {
searchResult := SearchResult{}
for {
err := wsConnection.ReadJSON(&searchResult)
if err != nil {
log.Println("Error parsing websocket results. Is your scope correct?")
log.Fatal(err)
}
channel <- searchResult
}
}()
return channel, nil
}
32 changes: 29 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,17 +7,19 @@ This is a Go CLI & library making queries to LeakIX easier.
```sh
$ leakix -h
Usage of leakix:
./leakix -q '*' -l 200

-j JSON mode, (excludes -t)
-l int
Limit results output (default 100)
-q string
Specify search query (default "*")
Search mode, specify search query (default "*")
-r Realtime mode, (excludes -q)
-s string
Specify scope (default "leak")
-t string
Specify output template (default "{{ .Ip }}:{{ .Port }}")

$ # Example query on the index
$ leakix -l 2 -q "protocol:web AND plugin:GitConfigPlugin" -t "{{ .Ip }}:{{ .Port }} : {{ .Data }}"
178.62.217.44:80 : Found git deployment configuration
[core]
Expand All @@ -44,6 +46,16 @@ $ leakix -l 2 -q "protocol:web AND plugin:GitConfigPlugin" -t "{{ .Ip }}:{{ .Por
[branch "staging"]
remote = origin
merge = refs/heads/staging

$ # Stream results in realtime from the engine, no filtering
$ ./leakix -r -s services -l 0
14.167.7.149:81
54.249.38.136:9200
23.65.39.190:80
[2a01:4f8:10a:1b5a::2]:80
23.225.38.43:3306
210.16.68.51:80
...keeps streaming...
```

## Library usage
Expand All @@ -55,7 +67,7 @@ import (
"github.com/LeakIX/LeakIXClient"
)

func main(){
func DoSearch(){
// Create a searcher
LeakIXSearch := LeakIXClient.SearchResultsClient{
Scope: "leak",
Expand All @@ -69,4 +81,18 @@ func main(){
}
}


func LiveStream() {
// Get a channel from the websocket
serviceChannel, err := LeakIXClient.GetChannel("services")
if err != nil {
log.Println("Websocket connection error:")
log.Fatal(err)
}
for {
// Print everything received on the channel
service := <- serviceChannel
log.Println(service.Ip)
}
}
```
84 changes: 67 additions & 17 deletions cmd/leakix/main.go
Original file line number Diff line number Diff line change
@@ -1,52 +1,102 @@
package main

import (
"encoding/json"
"flag"
"fmt"
"github.com/LeakIX/LeakIXClient"
"text/template"
"log"
"os"
"text/template"
)

// It's all sequential, Couldn't care sorry :
var outputJson bool
var scope string
var query string
var liveStream bool
var outputTemplate string
var limit int
var tmpl *template.Template
var err error

func main() {
var scope string
//Config our app
err = nil
flag.StringVar(&scope, "s", "leak", "Specify scope")
var query string
flag.StringVar(&query, "q", "*", "Specify search query")
var outputTemplate string
flag.StringVar(&query, "q", "*", "Search mode, specify search query")
flag.BoolVar(&liveStream, "r", false, "Realtime mode, (excludes -q)")
flag.BoolVar(&outputJson, "j", false, "JSON mode, (excludes -t)")
flag.StringVar(&outputTemplate, "t", "{{ .Ip }}:{{ .Port }}", "Specify output template")
var limit int
flag.IntVar(&limit, "l", 100, "Limit results output")

flag.Usage = func() {
fmt.Printf("Usage of leakix: \n")
fmt.Printf(" ./leakix -q '*' -l 200\n\n")
fmt.Printf(" ./leakix -q '*' -s leaks -l 200\n\n")
fmt.Printf(" ./leakix -r -s services -l 0\n\n")
flag.PrintDefaults()
}

flag.Parse()
tmpl, err = template.New("output").Parse(outputTemplate + "\n")
if err != nil {
log.Println("Template error :")
log.Fatal(err)
}

// Run the right thing
if liveStream {
LiveStream()
} else {
Search()
}

}

func Search() {
searcher := LeakIXClient.SearchResultsClient{
Scope: scope,
Query: query,
}
count := 0
for searcher.Next() {
count++
OutputSearchResult(searcher.SearchResult())
if count >= limit || count >= 10000 {
os.Exit(0)
}
}
}

tmpl, err := template.New("output").Parse(outputTemplate + "\n")
func LiveStream() {
count := 0
serviceChannel, err := LeakIXClient.GetChannel(scope)
if err != nil {
log.Println("Template error :")
log.Println("Websocket connection error:")
log.Fatal(err)
}
count := 0
for searcher.Next() {
for {
service := <- serviceChannel
count++
err = tmpl.Execute(os.Stdout, searcher.SearchResult())
OutputSearchResult(service)
if count >= limit && limit != 0 {
os.Exit(0)
}
}
}

func OutputSearchResult(searchResult LeakIXClient.SearchResult) {
if outputJson {
jsonBody, err := json.Marshal(searchResult)
if err != nil {
log.Println("Template error :")
log.Println("JSON error :")
log.Fatal(err)
}
if count >= limit {
os.Exit(0)
fmt.Println(string(jsonBody))
} else {
err := tmpl.Execute(os.Stdout, searchResult)
if err != nil {
log.Println("Template error :")
log.Fatal(err)
}
}
}

0 comments on commit 2c964f8

Please sign in to comment.