Added SX transformer

LeakIX · May 17, 2021 · 9e496b6 · 9e496b6
1 parent fe6754a
commit 9e496b6
Show file tree

Hide file tree

Showing 8 changed files with 99 additions and 14 deletions.
diff --git a/README.md b/README.md
@@ -42,8 +42,10 @@ The following formats are supported
 | url      | Handles URL conversion |
 | human        | Human readable format (output only) |
 | dnsx | Handles DNS resolution from [dnsx](https://github.com/projectdiscovery/dnsx)'s json output
+| sx | Handles SX's JSON input/output [sx](https://github.com/v-byte-cpu/sx) |
 | tbicore | Handles [LeakIX API](https://leakix.net/api-documentation)'s schema |
 
+
 ## Installation Instructions
 
 ### From Binary

diff --git a/go.mod b/go.mod
@@ -3,7 +3,7 @@ module github.com/LeakIX/l9filter
 go 1.15
 
 require (
-	github.com/LeakIX/l9format v1.0.0-alpha.1
+	github.com/LeakIX/l9format v1.0.0-alpha.5
 	github.com/Machiel/slugify v1.0.1 // indirect
 	github.com/alecthomas/kong v0.2.12
 	github.com/miekg/dns v1.1.35 // indirect
@@ -12,4 +12,5 @@ require (
 	github.com/stretchr/testify v1.5.1 // indirect
 	gitlab.nobody.run/tbi/core v0.0.0-20201124161030-42e21c6deb8e
 	golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9 // indirect
+	golang.org/x/net v0.0.0-20210510120150-4163338589ed // indirect
 )
diff --git a/go.sum b/go.sum
@@ -4,6 +4,10 @@ github.com/LeakIX/l9format v0.8.3 h1:o5YCfwThuOONt86pmNXcJvB47QxifldLvHdXsFZkf7I
 github.com/LeakIX/l9format v0.8.3/go.mod h1:6x8VufxxIuUV6B9xeJAYdNI/bLHZFBphbryKEHvomxM=
 github.com/LeakIX/l9format v1.0.0-alpha.1 h1:QCDDH+AJy5V2VwNo/gxDAFx3wLpkDkk6qoLt2JCIYcc=
 github.com/LeakIX/l9format v1.0.0-alpha.1/go.mod h1:zAMhvA0dNLwo66UmcL6e9vxKo2JxPQEVMDwNcc9Il9w=
+github.com/LeakIX/l9format v1.0.0-alpha.3 h1:0tSTnhcVfr8kykk0WEGWisvGa1NGnkq3UZWJao+OuZ4=
+github.com/LeakIX/l9format v1.0.0-alpha.3/go.mod h1:eKQn32c5PgUM7806Un2v6WTSmJcdcixed+cRHsPEp0k=
+github.com/LeakIX/l9format v1.0.0-alpha.5 h1:TUA9u2NQJDf3AUr4HPPM1IfWbTuQ7epg9s/uqPImRVM=
+github.com/LeakIX/l9format v1.0.0-alpha.5/go.mod h1:eKQn32c5PgUM7806Un2v6WTSmJcdcixed+cRHsPEp0k=
 github.com/Machiel/slugify v1.0.1 h1:EfWSlRWstMadsgzmiV7d0yVd2IFlagWH68Q+DcYCm4E=
 github.com/Machiel/slugify v1.0.1/go.mod h1:fTFGn5uWEynW4CUMG7sWkYXOf1UgDxyTM3DbR6Qfg3k=
 github.com/PuerkitoBio/goquery v1.6.1 h1:FgjbQZKl5HTmcn4sKBgvx8vv63nhyhIpv7lJpFGCWpk=
@@ -52,6 +56,8 @@ golang.org/x/net v0.0.0-20201209123823-ac852fbbde11 h1:lwlPPsmjDKK0J6eG6xDWd5XPe
 golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0=
 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
+golang.org/x/net v0.0.0-20210510120150-4163338589ed h1:p9UgmWI9wKpfYmgaV/IZKGdXc5qEK45tDwwwDyjS26I=
+golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

diff --git a/transform.go b/transform.go
@@ -18,9 +18,11 @@ type TransformCommand struct {
 	InputTransformer  transformer.TransformerInterface `kong:"-"`
 	OutputTransformer transformer.TransformerInterface `kong:"-"`
 	LogWriter         io.Writer                        `kong:"-"`
+	L9Sources          []string						    `short:"l" default:"l9filter"`
 }
 
 func (cmd *TransformCommand) Run() error {
+	transformer.L9Sources = cmd.L9Sources
 	for _, trs := range transformer.Transformers {
 		if cmd.InputFormat == trs.Name() {
 			trs.SetReader(os.Stdin)

diff --git a/transformer/l9.go b/transformer/l9.go
@@ -36,6 +36,9 @@ func (t *JsonServiceTransformer) Encode(event l9format.L9Event) error {
 	if t.jsonEncoder == nil {
 		t.jsonEncoder = json.NewEncoder(t.Writer)
 	}
+	for _, source := range L9Sources {
+		event.EventPipeline = append([]string{source}, event.EventPipeline...)
+	}
 	return t.jsonEncoder.Encode(event)
 }
 

diff --git a/transformer/sx.go b/transformer/sx.go
@@ -0,0 +1,70 @@
+package transformer
+
+import (
+	"encoding/json"
+	"github.com/LeakIX/l9format"
+	"strconv"
+	"time"
+)
+
+type SxTransformer struct {
+	Transformer
+	jsonEncoder *json.Encoder
+	jsonDecoder *json.Decoder
+}
+
+func NewSxTransformer() TransformerInterface {
+	return &SxTransformer{}
+}
+
+func (t *SxTransformer) Decode(outputTransformer TransformerInterface) (err error) {
+	if t.jsonDecoder == nil {
+		t.jsonDecoder = json.NewDecoder(t.Reader)
+	}
+	sxLine := SxResult{}
+	err = t.jsonDecoder.Decode(&sxLine)
+	if err != nil {
+		return err
+	}
+	if len(sxLine.ScanType) < 1 {
+		sxLine.ScanType = "arpscan"
+	}
+	return outputTransformer.Encode(l9format.L9Event{
+		EventType:     sxLine.ScanType,
+		Ip:            sxLine.Ip,
+		Time:          time.Now(),
+		EventSource:   "sx-" + sxLine.ScanType,
+		EventPipeline: []string{"sx-" + sxLine.ScanType},
+		Port:          strconv.Itoa(sxLine.Port),
+		Vendor:        sxLine.Vendor,
+	})
+}
+
+func (t *SxTransformer) Encode(event l9format.L9Event) error {
+	if t.jsonEncoder == nil {
+		t.jsonEncoder = json.NewEncoder(t.Writer)
+	}
+	port, err := strconv.Atoi(event.Port)
+	if err != nil {
+		return err
+	}
+	return t.jsonEncoder.Encode(SxResult{
+		Ip:       event.Ip,
+		Mac:      event.Mac,
+		Vendor:   event.Vendor,
+		Port:     port,
+		ScanType: event.EventType,
+	})
+}
+
+func (t *SxTransformer) Name() string {
+	return "sx"
+}
+
+type SxResult struct {
+	Ip       string `json:"ip"`
+	Mac      string `json:"mac"`
+	Vendor   string `json:"vendor"`
+	Port     int    `json:"port"'`
+	ScanType string `json:"scan"`
+}
diff --git a/transformer/tbicore.go b/transformer/tbicore.go
@@ -61,17 +61,15 @@ func (t *TbiCoreTransformer) Decode(outputTransformer TransformerInterface) (err
 
 func (t *TbiCoreTransformer) decodeService(hostService *core.HostService) (l9format.L9Event, error) {
 	event := l9format.L9Event{
-		EventType:     "service",
-		EventSource:   "l9filter-tbi",
-		EventPipeline: []string{"l9filter-tbi"},
-		Ip:            hostService.Ip,
-		Host:          hostService.Hostname,
-		Reverse:       hostService.Reverse,
-		Port:          hostService.Port,
-		Transports:    []string{"tcp"},
-		Protocol:      hostService.Type,
-		Summary:       hostService.Data,
-		Time:          time.Unix(hostService.Date, 0),
+		EventType:  "service",
+		Ip:         hostService.Ip,
+		Host:       hostService.Hostname,
+		Reverse:    hostService.Reverse,
+		Port:       hostService.Port,
+		Transports: []string{"tcp"},
+		Protocol:   hostService.Type,
+		Summary:    hostService.Data,
+		Time:       time.Unix(hostService.Date, 0),
 		SSL: l9format.L9SSLEvent{
 			JARM:        hostService.Certificate.JARM,
 			CypherSuite: hostService.Certificate.CypherSuite,
@@ -128,8 +126,8 @@ func (t *TbiCoreTransformer) decodeService(hostService *core.HostService) (l9for
 func (t *TbiCoreTransformer) decodeLeak(hostServiceLeak *core.HostServiceLeak) (l9format.L9Event, error) {
 	event := l9format.L9Event{
 		EventType:     "leak",
-		EventSource:   "l9filter-tbi",
-		EventPipeline: []string{hostServiceLeak.Plugin, "l9filter-tbi"},
+		EventSource:   hostServiceLeak.Plugin,
+		EventPipeline: []string{hostServiceLeak.Plugin},
 		Ip:            hostServiceLeak.Ip,
 		Host:          hostServiceLeak.Hostname,
 		Reverse:       hostServiceLeak.Reverse,

diff --git a/transformer/transformer_interface.go b/transformer/transformer_interface.go
@@ -14,6 +14,7 @@ var Transformers = []TransformerInterface{
 	NewTbiCoreTransformer(),
 	NewNmapTransformer(),
 	NewMasscanTransformer(),
+	NewSxTransformer(),
 }
 
 type TransformerInterface interface {
@@ -24,6 +25,8 @@ type TransformerInterface interface {
 	SetWriter(writer io.Writer)
 }
 
+var L9Sources []string
+
 type Transformer struct {
 	Reader io.Reader
 	Writer io.Writer