Preparing for v2 release, using l9format

Client.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/LeakIX/l9format"
 	"github.com/gorilla/websocket"
 	"gitlab.nobody.run/tbi/core"
 	"io/ioutil"
@@ -28,7 +29,7 @@ var HttpClient = &http.Client{
 type SearchResultsClient struct {
 	Scope         string
 	Query         string
-	SearchResults []SearchResult
+	SearchResults []l9format.L9Event
 	Position      int
 	Page          int
 	ApiKey        string
@@ -46,7 +47,7 @@ func (sc *SearchResultsClient) GetEndpoint() string {
 }
 
 func (sc *SearchResultsClient) Next() bool {
-	var results []SearchResult
+	var results []l9format.L9Event
 	if len(sc.SearchResults) > sc.Position {
 		sc.Position++
 		return true
@@ -64,14 +65,14 @@ func (sc *SearchResultsClient) Next() bool {
 	return false
 }
 
-func (sc *SearchResultsClient) SearchResult() SearchResult {
+func (sc *SearchResultsClient) SearchResult() l9format.L9Event {
 	return sc.SearchResults[sc.Position-1]
 }
 
-func (sc *SearchResultsClient) GetSearchResults(scope string, query string, page int) ([]SearchResult, error) {
+func (sc *SearchResultsClient) GetSearchResults(scope string, query string, page int) ([]l9format.L9Event, error) {
 	url := fmt.Sprintf(
 		"%s/search?scope=%s&q=%s&page=%d", sc.GetEndpoint(), url2.QueryEscape(scope), url2.QueryEscape(query), page)
-	var searchResults []SearchResult
+	var searchResults []l9format.L9Event
 	req, _ := http.NewRequest("GET", url, nil)
 	req.Header.Set("Accept", "application/json")
 	req.Header.Set("api-key", sc.ApiKey)
@@ -91,8 +92,8 @@ func (sc *SearchResultsClient) GetSearchResults(scope string, query string, page
 	return searchResults, nil
 }
 
-func (sc *SearchResultsClient) GetChannel(scope string) (chan SearchResult, error) {
-	channel := make(chan SearchResult)
+func (sc *SearchResultsClient) GetChannel(scope string) (chan l9format.L9Event, error) {
+	channel := make(chan l9format.L9Event)
 	endpointUrl, err := url2.Parse(sc.GetEndpoint())
 	if err != nil {
 		return nil, errors.New("invalid endpoint")
@@ -107,7 +108,7 @@ func (sc *SearchResultsClient) GetChannel(scope string) (chan SearchResult, erro
 		return nil, err
 	}
 	go func() {
-		searchResult := SearchResult{}
+		searchResult := l9format.L9Event{}
 		for {
 			err := wsConnection.ReadJSON(&searchResult)
 			if err != nil {

build-archs.sh

@@ -1,19 +1,19 @@
 
-GOOS=linux GOARCH=386 go build -ldflags="-s -w" -o bin/leakix-linux-32 ./cmd/leakix &
-GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o bin/leakix-linux-64 ./cmd/leakix &
-GOOS=freebsd GOARCH=amd64 go build -ldflags="-s -w" -o bin/leakix-freebsd-64 ./cmd/leakix &
-GOOS=freebsd GOARCH=386 go build -ldflags="-s -w" -o bin/leakix-freebsd-32 ./cmd/leakix &
+CGO_ENABLED=0 GOOS=linux GOARCH=386 go build -ldflags="-s -w  -extldflags '-static'" -o bin/leakix-linux-32 ./cmd/leakix &
+CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w -extldflags '-static'" -o bin/leakix-linux-64 ./cmd/leakix &
+CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 go build -ldflags="-s -w -extldflags '-static'" -o bin/leakix-freebsd-64 ./cmd/leakix &
+CGO_ENABLED=0 GOOS=freebsd GOARCH=386 go build -ldflags="-s -w -extldflags '-static'" -o bin/leakix-freebsd-32 ./cmd/leakix &
 
-GOOS=linux GOARCH=arm GOARM=7 go build -ldflags="-s -w" -o bin/leakix-linux-arm7 ./cmd/leakix &
-GOOS=linux GOARCH=arm GOARM=6 go build -ldflags="-s -w" -o bin/leakix-linux-arm6 ./cmd/leakix &
+CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=7 go build -ldflags="-s -w -extldflags '-static'" -o bin/leakix-linux-arm7 ./cmd/leakix &
+CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=6 go build -ldflags="-s -w -extldflags '-static'" -o bin/leakix-linux-arm6 ./cmd/leakix &
 
-GOOS=darwin GOARCH=amd64 go build -ldflags="-s -w" -o bin/leakix-osx ./cmd/leakix &
+CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -ldflags="-s -w -extldflags '-static'" -o bin/leakix-osx ./cmd/leakix &
 
-GOOS=windows GOARCH=amd64 go build -ldflags="-s -w" -o bin/leakix-win64.exe ./cmd/leakix &
-GOOS=windows GOARCH=386 go build -ldflags="-s -w" -o bin/leakix-win32.exe ./cmd/leakix &
+CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -ldflags="-s -w -extldflags '-static'" -o bin/leakix-win64.exe ./cmd/leakix &
+CGO_ENABLED=0 GOOS=windows GOARCH=386 go build -ldflags="-s -w -extldflags '-static'" -o bin/leakix-win32.exe ./cmd/leakix &
 
-GOOS=netbsd GOARCH=amd64 go build -ldflags="-s -w" -o bin/leakix-netbsd-64 ./cmd/leakix &
-GOOS=netbsd GOARCH=386 go build -ldflags="-s -w" -o bin/leakix-netbsd-32 ./cmd/leakix &
+CGO_ENABLED=0 GOOS=netbsd GOARCH=amd64 go build -ldflags="-s -w -extldflags '-static'" -o bin/leakix-netbsd-64 ./cmd/leakix &
+CGO_ENABLED=0 GOOS=netbsd GOARCH=386 go build -ldflags="-s -w -extldflags '-static'" -o bin/leakix-netbsd-32 ./cmd/leakix &
 
-GOOS=openbsd GOARCH=amd64 go build -ldflags="-s -w" -o bin/leakix-openbsd-64 ./cmd/leakix &
-GOOS=openbsd GOARCH=386 go build -ldflags="-s -w" -o bin/leakix-openbsd-32 ./cmd/leakix &
+CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 go build -ldflags="-s -w -extldflags '-static'" -o bin/leakix-openbsd-64 ./cmd/leakix &
+CGO_ENABLED=0 GOOS=openbsd GOARCH=386 go build -ldflags="-s -w -extldflags '-static'" -o bin/leakix-openbsd-32 ./cmd/leakix &

cmd/leakix/main.go

@@ -5,6 +5,7 @@ import (
 	"flag"
 	"fmt"
 	"github.com/LeakIX/LeakIXClient"
+	"github.com/LeakIX/l9format"
 	"log"
 	"os"
 	"text/template"
@@ -97,7 +98,7 @@ func LiveStream() {
 	}
 }
 
-func OutputSearchResult(searchResult LeakIXClient.SearchResult) {
+func OutputSearchResult(searchResult l9format.L9Event) {
 	if outputJson {
 		jsonBody, err := json.Marshal(searchResult)
 		if err != nil {

example/main.go

@@ -8,12 +8,12 @@ func main(){
 	// Create a searcher
 	LeakIXSearch := LeakIXClient.SearchResultsClient{
 		Scope: "leak",
-		Query: "protocol:kafka AND \"telecom_italia_data\"",
+		Query: "+protocol:elasticsearch  +\"telecom_italia_data\"",
 	}
 	// Iterate, the lib will query further pages if needed
 	for LeakIXSearch.Next() {
 		// Use the result
 		leak := LeakIXSearch.SearchResult()
-		fmt.Printf("%s:%s - Country:%s\n", leak.Ip, leak.Port, leak.GeoLocation.CountryName)
+		fmt.Printf("%s:%s - Country:%s\n", leak.Ip, leak.Port, leak.GeoIp.CountryName)
 	}
 }

go.mod

@@ -0,0 +1,13 @@
+module github.com/LeakIX/LeakIXClient
+
+go 1.15
+
+require (
+	github.com/LeakIX/l9format v1.3.0
+	github.com/Machiel/slugify v1.0.1 // indirect
+	github.com/PuerkitoBio/goquery v1.7.1 // indirect
+	github.com/andybalholm/cascadia v1.3.1 // indirect
+	github.com/gorilla/websocket v1.4.2
+	gitlab.nobody.run/tbi/core v0.0.0-20201124161030-42e21c6deb8e
+	golang.org/x/net v0.0.0-20210929193557-e81a3d93ecf6 // indirect
+)

go.sum

@@ -0,0 +1,40 @@
+github.com/LeakIX/l9format v1.3.0-beta.2 h1:2zwQryPYFvxju25vjL9TXLdActgVmjpDJ/zgc2Fb5hY=
+github.com/LeakIX/l9format v1.3.0-beta.2/go.mod h1:eKQn32c5PgUM7806Un2v6WTSmJcdcixed+cRHsPEp0k=
+github.com/LeakIX/l9format v1.3.0 h1:IrWuvYQ6ayEwf3oTTWG54D3oVfLApQDd0qyKdKK5mw8=
+github.com/LeakIX/l9format v1.3.0/go.mod h1:eKQn32c5PgUM7806Un2v6WTSmJcdcixed+cRHsPEp0k=
+github.com/Machiel/slugify v1.0.1 h1:EfWSlRWstMadsgzmiV7d0yVd2IFlagWH68Q+DcYCm4E=
+github.com/Machiel/slugify v1.0.1/go.mod h1:fTFGn5uWEynW4CUMG7sWkYXOf1UgDxyTM3DbR6Qfg3k=
+github.com/PuerkitoBio/goquery v1.6.1 h1:FgjbQZKl5HTmcn4sKBgvx8vv63nhyhIpv7lJpFGCWpk=
+github.com/PuerkitoBio/goquery v1.6.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
+github.com/PuerkitoBio/goquery v1.7.1 h1:oE+T06D+1T7LNrn91B4aERsRIeCLJ/oPSa6xB9FPnz4=
+github.com/PuerkitoBio/goquery v1.7.1/go.mod h1:XY0pP4kfraEmmV1O7Uf6XyjoslwsneBbgeDjLYuN8xY=
+github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/andybalholm/cascadia v1.2.0 h1:vuRCkM5Ozh/BfmsaTm26kbjm0mIOM3yS5Ek/F5h18aE=
+github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
+github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+gitlab.nobody.run/tbi/core v0.0.0-20201124161030-42e21c6deb8e h1:0EEfNlVSZtV1q/CqAXpoPcAgyim7s04kAKVmRkhUOfY=
+gitlab.nobody.run/tbi/core v0.0.0-20201124161030-42e21c6deb8e/go.mod h1:obLTFd7diUMk/cXeXJpgYdsj5OKn5c0EARFPatcZQCk=
+gitlab.nobody.run/tbi/socksme v0.0.0-20201130024528-3ad02d4108c5 h1:OUBbWsRDwvD6Nu6rP8PN6xGxIxXpVPBPBE9iuvDQH90=
+gitlab.nobody.run/tbi/socksme v0.0.0-20201130024528-3ad02d4108c5/go.mod h1:2R0aT7bhH9+xiSielZpKS/TL4nTki1jxOCCgXN8BzV8=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201216054612-986b41b23924 h1:QsnDpLLOKwHBBDa8nDws4DYNc/ryVW2vCpxCs09d4PY=
+golang.org/x/net v0.0.0-20201216054612-986b41b23924/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0=
+golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
+golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210929193557-e81a3d93ecf6 h1:Z04ewVs7JhXaYkmDhBERPi41gnltfQpMWDnTnQbaCqk=
+golang.org/x/net v0.0.0-20210929193557-e81a3d93ecf6/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=