Added event fingerprint, bugfix, upgrade to l9format v1.0.0

gboddin · gboddin · commit 843d78226345 · 2021-06-01T17:48:32.000+02:00
diff --git a/go.mod b/go.mod
@@ -3,7 +3,7 @@ module github.com/LeakIX/l9filter
 go 1.15
 
 require (
-	github.com/LeakIX/l9format v1.0.0-alpha.5
+	github.com/LeakIX/l9format v1.0.0
 	github.com/Machiel/slugify v1.0.1 // indirect
 	github.com/alecthomas/kong v0.2.12
 	github.com/miekg/dns v1.1.35 // indirect
@@ -12,5 +12,5 @@ require (
 	github.com/stretchr/testify v1.5.1 // indirect
 	gitlab.nobody.run/tbi/core v0.0.0-20201124161030-42e21c6deb8e
 	golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9 // indirect
-	golang.org/x/net v0.0.0-20210510120150-4163338589ed // indirect
+	golang.org/x/net v0.0.0-20210525063256-abc453219eb5 // indirect
 )
diff --git a/go.sum b/go.sum
@@ -8,6 +8,14 @@ github.com/LeakIX/l9format v1.0.0-alpha.3 h1:0tSTnhcVfr8kykk0WEGWisvGa1NGnkq3UZW
 github.com/LeakIX/l9format v1.0.0-alpha.3/go.mod h1:eKQn32c5PgUM7806Un2v6WTSmJcdcixed+cRHsPEp0k=
 github.com/LeakIX/l9format v1.0.0-alpha.5 h1:TUA9u2NQJDf3AUr4HPPM1IfWbTuQ7epg9s/uqPImRVM=
 github.com/LeakIX/l9format v1.0.0-alpha.5/go.mod h1:eKQn32c5PgUM7806Un2v6WTSmJcdcixed+cRHsPEp0k=
+github.com/LeakIX/l9format v1.0.0-alpha.6 h1:erFEoyjJhhmw986wqzv1buccMVVMM60CR32T+YtTYYs=
+github.com/LeakIX/l9format v1.0.0-alpha.6/go.mod h1:eKQn32c5PgUM7806Un2v6WTSmJcdcixed+cRHsPEp0k=
+github.com/LeakIX/l9format v1.0.0-alpha.7 h1:ow0iFvxC7u7bVLGPFBxABlfpHBRMoMuvB4JRf4jcomY=
+github.com/LeakIX/l9format v1.0.0-alpha.7/go.mod h1:eKQn32c5PgUM7806Un2v6WTSmJcdcixed+cRHsPEp0k=
+github.com/LeakIX/l9format v1.0.0-alpha.8 h1:9C3BdTtLEEWuALjSDDPmgis8knz2wKObUBN4EacQ3vs=
+github.com/LeakIX/l9format v1.0.0-alpha.8/go.mod h1:eKQn32c5PgUM7806Un2v6WTSmJcdcixed+cRHsPEp0k=
+github.com/LeakIX/l9format v1.0.0 h1:zHqGvnwvr5iuI55nCSQbmVlPdiEfiNrjn890MMlX2rk=
+github.com/LeakIX/l9format v1.0.0/go.mod h1:eKQn32c5PgUM7806Un2v6WTSmJcdcixed+cRHsPEp0k=
 github.com/Machiel/slugify v1.0.1 h1:EfWSlRWstMadsgzmiV7d0yVd2IFlagWH68Q+DcYCm4E=
 github.com/Machiel/slugify v1.0.1/go.mod h1:fTFGn5uWEynW4CUMG7sWkYXOf1UgDxyTM3DbR6Qfg3k=
 github.com/PuerkitoBio/goquery v1.6.1 h1:FgjbQZKl5HTmcn4sKBgvx8vv63nhyhIpv7lJpFGCWpk=
@@ -58,6 +66,8 @@ golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG0
 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
 golang.org/x/net v0.0.0-20210510120150-4163338589ed h1:p9UgmWI9wKpfYmgaV/IZKGdXc5qEK45tDwwwDyjS26I=
 golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210525063256-abc453219eb5 h1:wjuX4b5yYQnEQHzd+CBcrcC6OVR2J1CN6mUy0oSxIPo=
+golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
diff --git a/transformer/l9.go b/transformer/l9.go
@@ -39,6 +39,10 @@ func (t *JsonServiceTransformer) Encode(event l9format.L9Event) error {
 	for _, source := range L9Sources {
 		event.EventPipeline = append([]string{source}, event.EventPipeline...)
 	}
+	err := event.UpdateFingerprint()
+	if err != nil {
+		return err
+	}
 	return t.jsonEncoder.Encode(event)
 }
 
diff --git a/transformer/tbicore.go b/transformer/tbicore.go
@@ -29,13 +29,13 @@ func (t *TbiCoreTransformer) Decode(outputTransformer TransformerInterface) (err
 	if err != nil {
 		return err
 	}
-	checkEvent := &TbiCheckEvent{}
+	checkEvent := TbiCheckEvent{}
 	err = json.Unmarshal(jsonPayload, &checkEvent)
 	if err != nil {
 		return err
 	}
 
-	if len(checkEvent.Plugin) > 0 {
+	if  len(checkEvent.Plugin) > 0 {
 		hostServiceLeak := &core.HostServiceLeak{}
 		err = json.Unmarshal(jsonPayload, &hostServiceLeak)
 		if err != nil {
@@ -47,12 +47,12 @@ func (t *TbiCoreTransformer) Decode(outputTransformer TransformerInterface) (err
 		}
 		return outputTransformer.Encode(event)
 	}
-	hostService := &core.HostService{}
+	hostService := core.HostService{}
 	err = json.Unmarshal(jsonPayload, &hostService)
 	if err != nil {
 		return err
 	}
-	event, err := t.decodeService(hostService)
+	event, err := t.decodeService(&hostService)
 	if err != nil {
 		return err
 	}

Original file line number	Diff line number	Diff line change
`@@ -39,6 +39,10 @@ func (t *JsonServiceTransformer) Encode(event l9format.L9Event) error {`
`39`	`39`	`for _, source := range L9Sources {`
`40`	`40`	`event.EventPipeline = append([]string{source}, event.EventPipeline...)`
`41`	`41`	`}`
	`42`	`+ err := event.UpdateFingerprint()`
	`43`	`+ if err != nil {`
	`44`	`+ return err`
	`45`	`+ }`
`42`	`46`	`return t.jsonEncoder.Encode(event)`
`43`	`47`	`}`
`44`	`48`
Original file line number	Diff line number	Diff line change
`@@ -29,13 +29,13 @@ func (t *TbiCoreTransformer) Decode(outputTransformer TransformerInterface) (err`
`29`	`29`	`if err != nil {`
`30`	`30`	`return err`
`31`	`31`	`}`
`32`		`- checkEvent := &TbiCheckEvent{}`
	`32`	`+ checkEvent := TbiCheckEvent{}`
`33`	`33`	`err = json.Unmarshal(jsonPayload, &checkEvent)`
`34`	`34`	`if err != nil {`
`35`	`35`	`return err`
`36`	`36`	`}`
`37`	`37`
`38`		`- if len(checkEvent.Plugin) > 0 {`
	`38`	`+ if len(checkEvent.Plugin) > 0 {`
`39`	`39`	`hostServiceLeak := &core.HostServiceLeak{}`
`40`	`40`	`err = json.Unmarshal(jsonPayload, &hostServiceLeak)`
`41`	`41`	`if err != nil {`
`@@ -47,12 +47,12 @@ func (t *TbiCoreTransformer) Decode(outputTransformer TransformerInterface) (err`
`47`	`47`	`}`
`48`	`48`	`return outputTransformer.Encode(event)`
`49`	`49`	`}`
`50`		`- hostService := &core.HostService{}`
	`50`	`+ hostService := core.HostService{}`
`51`	`51`	`err = json.Unmarshal(jsonPayload, &hostService)`
`52`	`52`	`if err != nil {`
`53`	`53`	`return err`
`54`	`54`	`}`
`55`		`- event, err := t.decodeService(hostService)`
	`55`	`+ event, err := t.decodeService(&hostService)`
`56`	`56`	`if err != nil {`
`57`	`57`	`return err`
`58`	`58`	`}`