From cc31da11c414c1ab47b30f1a4fe248a99dbc0adf Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 22 Oct 2019 16:50:20 -0700 Subject: [PATCH 01/28] WIP: fingerprint processor --- libbeat/processors/fingerprint/config.go | 32 ++++++++++ libbeat/processors/fingerprint/fingerprint.go | 62 +++++++++++++++++++ .../fingerprint/fingerprint_test.go | 57 +++++++++++++++++ libbeat/processors/fingerprint/method.go | 48 ++++++++++++++ 4 files changed, 199 insertions(+) create mode 100644 libbeat/processors/fingerprint/config.go create mode 100644 libbeat/processors/fingerprint/fingerprint.go create mode 100644 libbeat/processors/fingerprint/fingerprint_test.go create mode 100644 libbeat/processors/fingerprint/method.go diff --git a/libbeat/processors/fingerprint/config.go b/libbeat/processors/fingerprint/config.go new file mode 100644 index 000000000000..bed896e9bced --- /dev/null +++ b/libbeat/processors/fingerprint/config.go @@ -0,0 +1,32 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package fingerprint + +// Config for fingerprint processor. +type Config struct { + Method Method `config:"method"` // Algorithm to use for fingerprinting + Fields []string `config:"fields" validate:"required"` // Source fields to compute fingerprint from + TargetField string `config:"target_field"` // Target field for the fingerprint +} + +func defaultConfig() Config { + return Config{ + Method: MethodSHA256, + TargetField: "fingerprint", + } +} diff --git a/libbeat/processors/fingerprint/fingerprint.go b/libbeat/processors/fingerprint/fingerprint.go new file mode 100644 index 000000000000..7388d91b529c --- /dev/null +++ b/libbeat/processors/fingerprint/fingerprint.go @@ -0,0 +1,62 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package fingerprint + +import ( + "fmt" + + "github.com/elastic/beats/libbeat/beat" + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/processors" + jsprocessor "github.com/elastic/beats/libbeat/processors/script/javascript/module/processor" + "github.com/pkg/errors" +) + +func init() { + processors.RegisterPlugin("fingerprint", New) + jsprocessor.RegisterPlugin("Fingerprint", New) +} + +const processorName = "fingerprint" + +type fingerprint struct { + config Config +} + +// New constructs a new fingerprint processor. +func New(cfg *common.Config) (processors.Processor, error) { + config := defaultConfig() + if err := cfg.Unpack(&config); err != nil { + return nil, errors.Wrapf(err, "failed to unpack %v processor configuration", processorName) + } + + p := &fingerprint{ + config: config, + } + + return p, nil +} + +// Run enriches the given event with fingerprint information +func (p *fingerprint) Run(event *beat.Event) (*beat.Event, error) { + return event, nil +} + +func (p *fingerprint) String() string { + return fmt.Sprintf("%v=[method=[%v]]", processorName, p.config.Method) +} diff --git a/libbeat/processors/fingerprint/fingerprint_test.go b/libbeat/processors/fingerprint/fingerprint_test.go new file mode 100644 index 000000000000..d12eff66d662 --- /dev/null +++ b/libbeat/processors/fingerprint/fingerprint_test.go @@ -0,0 +1,57 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package fingerprint + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/libbeat/beat" + "github.com/elastic/beats/libbeat/common" +) + +func TestMethodDefault(t *testing.T) { + TestMethodSHA256(t) +} + +func TestMethodSHA256(t *testing.T) { + testConfig, err := common.NewConfigFrom(common.MapStr{ + "fields": []string{"field1"}, + "method": "sha256", + }) + assert.NoError(t, err) + + p, err := New(testConfig) + assert.NoError(t, err) + + testEvent := &beat.Event{ + Fields: common.MapStr{ + "field1": "foo", + }, + Timestamp: time.Now(), + } + + newEvent, err := p.Run(testEvent) + assert.NoError(t, err) + + v, err := newEvent.GetValue("fingerprint") + assert.NoError(t, err) + assert.Equal(t, "a292b0c17d13b06dbd244226b72f9e38a87ad9a41fd1853fce3ed875b834fc62", v) +} diff --git a/libbeat/processors/fingerprint/method.go b/libbeat/processors/fingerprint/method.go new file mode 100644 index 000000000000..44c1498d381b --- /dev/null +++ b/libbeat/processors/fingerprint/method.go @@ -0,0 +1,48 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package fingerprint + +import ( + "errors" + "strings" +) + +var errMethodUnknown = errors.New("unknown method") + +type Method uint8 + +const ( + MethodSHA1 Method = iota + MethodSHA256 +) + +// Unpack creates the Method enumeration value from the given string +func (m *Method) Unpack(str string) error { + str = strings.ToLower(str) + + switch str { + case "sha1": + *m = MethodSHA1 + case "sha256": + *m = MethodSHA256 + default: + return errMethodUnknown + } + + return nil +} From ecc4772246ee1d89732dc1bb2a127117870003e9 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 22 Oct 2019 17:37:58 -0700 Subject: [PATCH 02/28] Implementing SHA256 fingerprinter --- libbeat/processors/fingerprint/fingerprint.go | 45 +++++++++++++++++++ .../fingerprint/fingerprint_test.go | 2 +- libbeat/processors/fingerprint/method.go | 27 +++++++++++ 3 files changed, 73 insertions(+), 1 deletion(-) diff --git a/libbeat/processors/fingerprint/fingerprint.go b/libbeat/processors/fingerprint/fingerprint.go index 7388d91b529c..2788ab514b71 100644 --- a/libbeat/processors/fingerprint/fingerprint.go +++ b/libbeat/processors/fingerprint/fingerprint.go @@ -19,6 +19,7 @@ package fingerprint import ( "fmt" + "strconv" "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" @@ -54,9 +55,53 @@ func New(cfg *common.Config) (processors.Processor, error) { // Run enriches the given event with fingerprint information func (p *fingerprint) Run(event *beat.Event) (*beat.Event, error) { + var str string + for _, k := range p.config.Fields { + v, err := event.Fields.GetValue(k) + if err != nil { + return nil, errors.Wrap(err, "failed to compute fingerprint") + } + + var s string + switch v := v.(type) { + case map[string]interface{}, []interface{}: + return nil, errors.Errorf("cannot compute fingerprint using non-scalar field: %v", k) + case string: + s = v + case int: + s = strconv.Itoa(v) + } + + str += fmt.Sprintf("|%v|%v", k, s) + } + str += "|" + + makeFingerprint, err := p.config.Method.factory() + if err != nil { + return nil, errors.Wrap(err, "failed to compute fingerprint") + } + + f, err := makeFingerprint(str) + if err != nil { + return nil, errors.Wrap(err, "failed to compute fingerprint") + } + + if _, err = event.PutValue(p.config.TargetField, f); err != nil { + return nil, errors.Wrap(err, "failed to compute fingerprint") + } + return event, nil } func (p *fingerprint) String() string { return fmt.Sprintf("%v=[method=[%v]]", processorName, p.config.Method) } + +func contains(haystack []string, needle string) bool { + for _, item := range haystack { + if item == needle { + return true + } + } + return false +} diff --git a/libbeat/processors/fingerprint/fingerprint_test.go b/libbeat/processors/fingerprint/fingerprint_test.go index d12eff66d662..33169500bfb1 100644 --- a/libbeat/processors/fingerprint/fingerprint_test.go +++ b/libbeat/processors/fingerprint/fingerprint_test.go @@ -53,5 +53,5 @@ func TestMethodSHA256(t *testing.T) { v, err := newEvent.GetValue("fingerprint") assert.NoError(t, err) - assert.Equal(t, "a292b0c17d13b06dbd244226b72f9e38a87ad9a41fd1853fce3ed875b834fc62", v) + assert.Equal(t, "4cf8b768ad20266c348d63a6d1ff5d6f6f9ed0f59f5c68ae031b78e3e04c5144", v) } diff --git a/libbeat/processors/fingerprint/method.go b/libbeat/processors/fingerprint/method.go index 44c1498d381b..05977c093861 100644 --- a/libbeat/processors/fingerprint/method.go +++ b/libbeat/processors/fingerprint/method.go @@ -18,7 +18,9 @@ package fingerprint import ( + "crypto/sha256" "errors" + "fmt" "strings" ) @@ -46,3 +48,28 @@ func (m *Method) Unpack(str string) error { return nil } + +type fingerprinter func(string) (string, error) + +func (m *Method) factory() (fingerprinter, error) { + var f fingerprinter + switch *m { + case MethodSHA1: + f = sha1Fingerprinter + case MethodSHA256: + f = sha256Fingerprinter + default: + return nil, errMethodUnknown + } + + return f, nil +} + +func sha1Fingerprinter(in string) (string, error) { + return in, nil +} + +func sha256Fingerprinter(in string) (string, error) { + return fmt.Sprintf("%x", sha256.Sum256([]byte(in))), nil + +} From 0d2af8c113f9266b85cbba5a2340fe111df2ece2 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 22 Oct 2019 17:48:43 -0700 Subject: [PATCH 03/28] Sort source fields --- libbeat/processors/fingerprint/fingerprint.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libbeat/processors/fingerprint/fingerprint.go b/libbeat/processors/fingerprint/fingerprint.go index 2788ab514b71..4118cd26f9c3 100644 --- a/libbeat/processors/fingerprint/fingerprint.go +++ b/libbeat/processors/fingerprint/fingerprint.go @@ -19,6 +19,7 @@ package fingerprint import ( "fmt" + "sort" "strconv" "github.com/elastic/beats/libbeat/beat" @@ -46,6 +47,8 @@ func New(cfg *common.Config) (processors.Processor, error) { return nil, errors.Wrapf(err, "failed to unpack %v processor configuration", processorName) } + sort.Strings(config.Fields) + p := &fingerprint{ config: config, } From 00a3575b9327ce271361b486447f42106a9913bf Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 22 Oct 2019 17:55:49 -0700 Subject: [PATCH 04/28] Refactoring --- libbeat/processors/fingerprint/fingerprint.go | 59 ++++++++++++------- .../fingerprint/fingerprint_test.go | 27 ++++++++- 2 files changed, 63 insertions(+), 23 deletions(-) diff --git a/libbeat/processors/fingerprint/fingerprint.go b/libbeat/processors/fingerprint/fingerprint.go index 4118cd26f9c3..9f7a5b3ad718 100644 --- a/libbeat/processors/fingerprint/fingerprint.go +++ b/libbeat/processors/fingerprint/fingerprint.go @@ -58,39 +58,23 @@ func New(cfg *common.Config) (processors.Processor, error) { // Run enriches the given event with fingerprint information func (p *fingerprint) Run(event *beat.Event) (*beat.Event, error) { - var str string - for _, k := range p.config.Fields { - v, err := event.Fields.GetValue(k) - if err != nil { - return nil, errors.Wrap(err, "failed to compute fingerprint") - } - - var s string - switch v := v.(type) { - case map[string]interface{}, []interface{}: - return nil, errors.Errorf("cannot compute fingerprint using non-scalar field: %v", k) - case string: - s = v - case int: - s = strconv.Itoa(v) - } - - str += fmt.Sprintf("|%v|%v", k, s) + str, err := makeSourceString(p.config.Fields, event.Fields) + if err != nil { + return nil, makeComputeFingerprintError(err) } - str += "|" makeFingerprint, err := p.config.Method.factory() if err != nil { - return nil, errors.Wrap(err, "failed to compute fingerprint") + return nil, makeComputeFingerprintError(err) } f, err := makeFingerprint(str) if err != nil { - return nil, errors.Wrap(err, "failed to compute fingerprint") + return nil, makeComputeFingerprintError(err) } if _, err = event.PutValue(p.config.TargetField, f); err != nil { - return nil, errors.Wrap(err, "failed to compute fingerprint") + return nil, makeComputeFingerprintError(err) } return event, nil @@ -100,6 +84,33 @@ func (p *fingerprint) String() string { return fmt.Sprintf("%v=[method=[%v]]", processorName, p.config.Method) } +func makeSourceString(sourceFields []string, eventFields common.MapStr) (string, error) { + var str string + for _, k := range sourceFields { + v, err := eventFields.GetValue(k) + if err == common.ErrKeyNotFound { + return "", errors.Wrapf(err, "failed to find field [%v] in event", k) + } + if err != nil { + return "", errors.Wrapf(err, "failed when finding field [%v] in event", k) + } + + var s string + switch v := v.(type) { + case map[string]interface{}, []interface{}: + return "", errors.Errorf("cannot compute fingerprint using non-scalar field [%v]", k) + case string: + s = v + case int: + s = strconv.Itoa(v) + } + + str += fmt.Sprintf("|%v|%v", k, s) + } + str += "|" + return str, nil +} + func contains(haystack []string, needle string) bool { for _, item := range haystack { if item == needle { @@ -108,3 +119,7 @@ func contains(haystack []string, needle string) bool { } return false } + +func makeComputeFingerprintError(err error) error { + return errors.Wrap(err, "failed to compute fingerprint") +} diff --git a/libbeat/processors/fingerprint/fingerprint_test.go b/libbeat/processors/fingerprint/fingerprint_test.go index 33169500bfb1..1dc5b32c9bc2 100644 --- a/libbeat/processors/fingerprint/fingerprint_test.go +++ b/libbeat/processors/fingerprint/fingerprint_test.go @@ -28,7 +28,27 @@ import ( ) func TestMethodDefault(t *testing.T) { - TestMethodSHA256(t) + testConfig, err := common.NewConfigFrom(common.MapStr{ + "fields": []string{"field1"}, + }) + assert.NoError(t, err) + + p, err := New(testConfig) + assert.NoError(t, err) + + testEvent := &beat.Event{ + Fields: common.MapStr{ + "field1": "foo", + }, + Timestamp: time.Now(), + } + + newEvent, err := p.Run(testEvent) + assert.NoError(t, err) + + v, err := newEvent.GetValue("fingerprint") + assert.NoError(t, err) + assert.Equal(t, "4cf8b768ad20266c348d63a6d1ff5d6f6f9ed0f59f5c68ae031b78e3e04c5144", v) } func TestMethodSHA256(t *testing.T) { @@ -55,3 +75,8 @@ func TestMethodSHA256(t *testing.T) { assert.NoError(t, err) assert.Equal(t, "4cf8b768ad20266c348d63a6d1ff5d6f6f9ed0f59f5c68ae031b78e3e04c5144", v) } + +// TODO: other fingerprinting methods +// TODO: Order of source fields doesn't matter +// TODO: Missing source fields? +// TODO: non-scalar fields From a6afbc9068cfb2335fdd639c53abc7b22b186a1b Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 22 Oct 2019 17:56:54 -0700 Subject: [PATCH 05/28] Add TODO --- libbeat/processors/fingerprint/fingerprint_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/libbeat/processors/fingerprint/fingerprint_test.go b/libbeat/processors/fingerprint/fingerprint_test.go index 1dc5b32c9bc2..c4f4821e0ed9 100644 --- a/libbeat/processors/fingerprint/fingerprint_test.go +++ b/libbeat/processors/fingerprint/fingerprint_test.go @@ -80,3 +80,4 @@ func TestMethodSHA256(t *testing.T) { // TODO: Order of source fields doesn't matter // TODO: Missing source fields? // TODO: non-scalar fields +// TODO: invalid fingerprinting method in config From 7521e118bee52ac45e4b39100f99bc8251b43446 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 28 Oct 2019 17:38:38 -0700 Subject: [PATCH 06/28] Convert time fields to UTC --- libbeat/processors/fingerprint/fingerprint.go | 15 +++++++-------- .../processors/fingerprint/fingerprint_test.go | 4 +++- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/libbeat/processors/fingerprint/fingerprint.go b/libbeat/processors/fingerprint/fingerprint.go index 9f7a5b3ad718..6a3fdab53312 100644 --- a/libbeat/processors/fingerprint/fingerprint.go +++ b/libbeat/processors/fingerprint/fingerprint.go @@ -20,7 +20,7 @@ package fingerprint import ( "fmt" "sort" - "strconv" + "time" "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" @@ -95,17 +95,16 @@ func makeSourceString(sourceFields []string, eventFields common.MapStr) (string, return "", errors.Wrapf(err, "failed when finding field [%v] in event", k) } - var s string - switch v := v.(type) { + i := v + switch vv := v.(type) { case map[string]interface{}, []interface{}: return "", errors.Errorf("cannot compute fingerprint using non-scalar field [%v]", k) - case string: - s = v - case int: - s = strconv.Itoa(v) + case time.Time: + // Ensure we consistently hash times in UTC. + i = vv.UTC() } - str += fmt.Sprintf("|%v|%v", k, s) + str += fmt.Sprintf("|%v|%v", k, i) } str += "|" return str, nil diff --git a/libbeat/processors/fingerprint/fingerprint_test.go b/libbeat/processors/fingerprint/fingerprint_test.go index c4f4821e0ed9..c9afcbb691ed 100644 --- a/libbeat/processors/fingerprint/fingerprint_test.go +++ b/libbeat/processors/fingerprint/fingerprint_test.go @@ -78,6 +78,8 @@ func TestMethodSHA256(t *testing.T) { // TODO: other fingerprinting methods // TODO: Order of source fields doesn't matter -// TODO: Missing source fields? +// TODO: Missing source fields // TODO: non-scalar fields +// TODO: hashing time fields // TODO: invalid fingerprinting method in config +// TODO: encoding From d8b5958dbe76e448f41d87d961f9a9cd6f854ada Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 28 Oct 2019 17:40:30 -0700 Subject: [PATCH 07/28] Removing unnecessary function --- libbeat/processors/fingerprint/fingerprint.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/libbeat/processors/fingerprint/fingerprint.go b/libbeat/processors/fingerprint/fingerprint.go index 6a3fdab53312..72dd134e202d 100644 --- a/libbeat/processors/fingerprint/fingerprint.go +++ b/libbeat/processors/fingerprint/fingerprint.go @@ -110,15 +110,6 @@ func makeSourceString(sourceFields []string, eventFields common.MapStr) (string, return str, nil } -func contains(haystack []string, needle string) bool { - for _, item := range haystack { - if item == needle { - return true - } - } - return false -} - func makeComputeFingerprintError(err error) error { return errors.Wrap(err, "failed to compute fingerprint") } From a6b615659ffe8a1ae33e460af2d0b9f094fb7500 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 28 Oct 2019 17:48:32 -0700 Subject: [PATCH 08/28] Adding SHA1 --- libbeat/processors/fingerprint/fingerprint.go | 14 +++++------ .../fingerprint/fingerprint_test.go | 25 +++++++++++++++++++ libbeat/processors/fingerprint/method.go | 11 ++++---- 3 files changed, 38 insertions(+), 12 deletions(-) diff --git a/libbeat/processors/fingerprint/fingerprint.go b/libbeat/processors/fingerprint/fingerprint.go index 72dd134e202d..ca2a6d8b486a 100644 --- a/libbeat/processors/fingerprint/fingerprint.go +++ b/libbeat/processors/fingerprint/fingerprint.go @@ -58,7 +58,7 @@ func New(cfg *common.Config) (processors.Processor, error) { // Run enriches the given event with fingerprint information func (p *fingerprint) Run(event *beat.Event) (*beat.Event, error) { - str, err := makeSourceString(p.config.Fields, event.Fields) + source, err := makeSource(p.config.Fields, event.Fields) if err != nil { return nil, makeComputeFingerprintError(err) } @@ -68,7 +68,7 @@ func (p *fingerprint) Run(event *beat.Event) (*beat.Event, error) { return nil, makeComputeFingerprintError(err) } - f, err := makeFingerprint(str) + f, err := makeFingerprint(source) if err != nil { return nil, makeComputeFingerprintError(err) } @@ -84,21 +84,21 @@ func (p *fingerprint) String() string { return fmt.Sprintf("%v=[method=[%v]]", processorName, p.config.Method) } -func makeSourceString(sourceFields []string, eventFields common.MapStr) (string, error) { +func makeSource(sourceFields []string, eventFields common.MapStr) ([]byte, error) { var str string for _, k := range sourceFields { v, err := eventFields.GetValue(k) if err == common.ErrKeyNotFound { - return "", errors.Wrapf(err, "failed to find field [%v] in event", k) + return nil, errors.Wrapf(err, "failed to find field [%v] in event", k) } if err != nil { - return "", errors.Wrapf(err, "failed when finding field [%v] in event", k) + return nil, errors.Wrapf(err, "failed when finding field [%v] in event", k) } i := v switch vv := v.(type) { case map[string]interface{}, []interface{}: - return "", errors.Errorf("cannot compute fingerprint using non-scalar field [%v]", k) + return nil, errors.Errorf("cannot compute fingerprint using non-scalar field [%v]", k) case time.Time: // Ensure we consistently hash times in UTC. i = vv.UTC() @@ -107,7 +107,7 @@ func makeSourceString(sourceFields []string, eventFields common.MapStr) (string, str += fmt.Sprintf("|%v|%v", k, i) } str += "|" - return str, nil + return []byte(str), nil } func makeComputeFingerprintError(err error) error { diff --git a/libbeat/processors/fingerprint/fingerprint_test.go b/libbeat/processors/fingerprint/fingerprint_test.go index c9afcbb691ed..400dae95c4fd 100644 --- a/libbeat/processors/fingerprint/fingerprint_test.go +++ b/libbeat/processors/fingerprint/fingerprint_test.go @@ -51,6 +51,31 @@ func TestMethodDefault(t *testing.T) { assert.Equal(t, "4cf8b768ad20266c348d63a6d1ff5d6f6f9ed0f59f5c68ae031b78e3e04c5144", v) } +func TestMethodSHA1(t *testing.T) { + testConfig, err := common.NewConfigFrom(common.MapStr{ + "fields": []string{"field1"}, + "method": "sha1", + }) + assert.NoError(t, err) + + p, err := New(testConfig) + assert.NoError(t, err) + + testEvent := &beat.Event{ + Fields: common.MapStr{ + "field1": "foo", + }, + Timestamp: time.Now(), + } + + newEvent, err := p.Run(testEvent) + assert.NoError(t, err) + + v, err := newEvent.GetValue("fingerprint") + assert.NoError(t, err) + assert.Equal(t, "46de5d8225e75aeedd559c953f100dca41612b18", v) +} + func TestMethodSHA256(t *testing.T) { testConfig, err := common.NewConfigFrom(common.MapStr{ "fields": []string{"field1"}, diff --git a/libbeat/processors/fingerprint/method.go b/libbeat/processors/fingerprint/method.go index 05977c093861..2096e647fbb0 100644 --- a/libbeat/processors/fingerprint/method.go +++ b/libbeat/processors/fingerprint/method.go @@ -18,6 +18,7 @@ package fingerprint import ( + "crypto/sha1" "crypto/sha256" "errors" "fmt" @@ -49,7 +50,7 @@ func (m *Method) Unpack(str string) error { return nil } -type fingerprinter func(string) (string, error) +type fingerprinter func([]byte) (string, error) func (m *Method) factory() (fingerprinter, error) { var f fingerprinter @@ -65,11 +66,11 @@ func (m *Method) factory() (fingerprinter, error) { return f, nil } -func sha1Fingerprinter(in string) (string, error) { - return in, nil +func sha1Fingerprinter(in []byte) (string, error) { + return fmt.Sprintf("%x", sha1.Sum(in)), nil } -func sha256Fingerprinter(in string) (string, error) { - return fmt.Sprintf("%x", sha256.Sum256([]byte(in))), nil +func sha256Fingerprinter(in []byte) (string, error) { + return fmt.Sprintf("%x", sha256.Sum256(in)), nil } From 85ef943207f676124ab8bcbd9d440a604ae21c37 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 28 Oct 2019 18:22:06 -0700 Subject: [PATCH 09/28] WIP: add encoding --- libbeat/processors/fingerprint/config.go | 10 ++-- libbeat/processors/fingerprint/encoder.go | 49 ++++++++++++++++++ libbeat/processors/fingerprint/fingerprint.go | 14 ++---- .../{method.go => fingerprinter.go} | 50 ++++--------------- 4 files changed, 70 insertions(+), 53 deletions(-) create mode 100644 libbeat/processors/fingerprint/encoder.go rename libbeat/processors/fingerprint/{method.go => fingerprinter.go} (57%) diff --git a/libbeat/processors/fingerprint/config.go b/libbeat/processors/fingerprint/config.go index bed896e9bced..69993eb16e78 100644 --- a/libbeat/processors/fingerprint/config.go +++ b/libbeat/processors/fingerprint/config.go @@ -19,14 +19,16 @@ package fingerprint // Config for fingerprint processor. type Config struct { - Method Method `config:"method"` // Algorithm to use for fingerprinting - Fields []string `config:"fields" validate:"required"` // Source fields to compute fingerprint from - TargetField string `config:"target_field"` // Target field for the fingerprint + Method fingerprinter `config:"method"` // Algorithm to use for fingerprinting + Fields []string `config:"fields" validate:"required"` // Source fields to compute fingerprint from + TargetField string `config:"target_field"` // Target field for the fingerprint + Encoding encoder `config:"encoding"` } func defaultConfig() Config { return Config{ - Method: MethodSHA256, + Method: methods["sha256"], TargetField: "fingerprint", + Encoding: encodings["hex"], } } diff --git a/libbeat/processors/fingerprint/encoder.go b/libbeat/processors/fingerprint/encoder.go new file mode 100644 index 000000000000..77c04bd2057c --- /dev/null +++ b/libbeat/processors/fingerprint/encoder.go @@ -0,0 +1,49 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package fingerprint + +import ( + "encoding/base32" + "encoding/base64" + "encoding/hex" + "errors" + "strings" +) + +var errEncoderUnknown = errors.New("unknown encoding method") + +type encoder func([]byte) string + +var encodings = map[string]encoder{ + "hex": hex.EncodeToString, + "base32": base32.StdEncoding.EncodeToString, + "base64": base64.StdEncoding.EncodeToString, +} + +// Unpack creates the Method enumeration value from the given string +func (e *encoder) Unpack(str string) error { + str = strings.ToLower(str) + + m, found := encodings[str] + if !found { + return errEncoderUnknown + } + + *e = m + return nil +} diff --git a/libbeat/processors/fingerprint/fingerprint.go b/libbeat/processors/fingerprint/fingerprint.go index ca2a6d8b486a..aa35d9076082 100644 --- a/libbeat/processors/fingerprint/fingerprint.go +++ b/libbeat/processors/fingerprint/fingerprint.go @@ -63,17 +63,11 @@ func (p *fingerprint) Run(event *beat.Event) (*beat.Event, error) { return nil, makeComputeFingerprintError(err) } - makeFingerprint, err := p.config.Method.factory() - if err != nil { - return nil, makeComputeFingerprintError(err) - } - - f, err := makeFingerprint(source) - if err != nil { - return nil, makeComputeFingerprintError(err) - } + f := p.config.Method() + s := f.Sum(source) + v := p.config.Encoding(s) - if _, err = event.PutValue(p.config.TargetField, f); err != nil { + if _, err = event.PutValue(p.config.TargetField, v); err != nil { return nil, makeComputeFingerprintError(err) } diff --git a/libbeat/processors/fingerprint/method.go b/libbeat/processors/fingerprint/fingerprinter.go similarity index 57% rename from libbeat/processors/fingerprint/method.go rename to libbeat/processors/fingerprint/fingerprinter.go index 2096e647fbb0..9a3f6d254e7c 100644 --- a/libbeat/processors/fingerprint/method.go +++ b/libbeat/processors/fingerprint/fingerprinter.go @@ -21,56 +21,28 @@ import ( "crypto/sha1" "crypto/sha256" "errors" - "fmt" + "hash" "strings" ) -var errMethodUnknown = errors.New("unknown method") +var errMethodUnknown = errors.New("unknown fingerprinting method") -type Method uint8 +type fingerprinter func() hash.Hash -const ( - MethodSHA1 Method = iota - MethodSHA256 -) +var methods = map[string]fingerprinter{ + "sha1": sha1.New, + "sha256": sha256.New, +} // Unpack creates the Method enumeration value from the given string -func (m *Method) Unpack(str string) error { +func (f *fingerprinter) Unpack(str string) error { str = strings.ToLower(str) - switch str { - case "sha1": - *m = MethodSHA1 - case "sha256": - *m = MethodSHA256 - default: + m, found := methods[str] + if !found { return errMethodUnknown } + *f = m return nil } - -type fingerprinter func([]byte) (string, error) - -func (m *Method) factory() (fingerprinter, error) { - var f fingerprinter - switch *m { - case MethodSHA1: - f = sha1Fingerprinter - case MethodSHA256: - f = sha256Fingerprinter - default: - return nil, errMethodUnknown - } - - return f, nil -} - -func sha1Fingerprinter(in []byte) (string, error) { - return fmt.Sprintf("%x", sha1.Sum(in)), nil -} - -func sha256Fingerprinter(in []byte) (string, error) { - return fmt.Sprintf("%x", sha256.Sum256(in)), nil - -} From 8cda4be97b6a0bed45323169d34104bd352191c0 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 29 Oct 2019 11:24:57 -0700 Subject: [PATCH 10/28] Cleanup --- libbeat/processors/fingerprint/config.go | 10 ++++---- .../fingerprint/{encoder.go => encode.go} | 12 ++++----- libbeat/processors/fingerprint/fingerprint.go | 25 ++++++++++++------- .../fingerprint/{fingerprinter.go => hash.go} | 14 +++++------ 4 files changed, 34 insertions(+), 27 deletions(-) rename libbeat/processors/fingerprint/{encoder.go => encode.go} (79%) rename libbeat/processors/fingerprint/{fingerprinter.go => hash.go} (75%) diff --git a/libbeat/processors/fingerprint/config.go b/libbeat/processors/fingerprint/config.go index 69993eb16e78..d8e9e741eb12 100644 --- a/libbeat/processors/fingerprint/config.go +++ b/libbeat/processors/fingerprint/config.go @@ -19,15 +19,15 @@ package fingerprint // Config for fingerprint processor. type Config struct { - Method fingerprinter `config:"method"` // Algorithm to use for fingerprinting - Fields []string `config:"fields" validate:"required"` // Source fields to compute fingerprint from - TargetField string `config:"target_field"` // Target field for the fingerprint - Encoding encoder `config:"encoding"` + Method hashMethod `config:"method"` // Hash function to use for fingerprinting + Fields []string `config:"fields" validate:"required"` // Source fields to compute fingerprint from + TargetField string `config:"target_field"` // Target field for the fingerprint + Encoding encodingMethod `config:"encoding"` // Encoding to use for target field value } func defaultConfig() Config { return Config{ - Method: methods["sha256"], + Method: hashes["sha256"], TargetField: "fingerprint", Encoding: encodings["hex"], } diff --git a/libbeat/processors/fingerprint/encoder.go b/libbeat/processors/fingerprint/encode.go similarity index 79% rename from libbeat/processors/fingerprint/encoder.go rename to libbeat/processors/fingerprint/encode.go index 77c04bd2057c..c611b4e0f625 100644 --- a/libbeat/processors/fingerprint/encoder.go +++ b/libbeat/processors/fingerprint/encode.go @@ -25,23 +25,23 @@ import ( "strings" ) -var errEncoderUnknown = errors.New("unknown encoding method") +var errEncodingMethodUnknown = errors.New("unknown encoding method") -type encoder func([]byte) string +type encodingMethod func([]byte) string -var encodings = map[string]encoder{ +var encodings = map[string]encodingMethod{ "hex": hex.EncodeToString, "base32": base32.StdEncoding.EncodeToString, "base64": base64.StdEncoding.EncodeToString, } -// Unpack creates the Method enumeration value from the given string -func (e *encoder) Unpack(str string) error { +// Unpack creates the encodingMethod from the given string +func (e *encodingMethod) Unpack(str string) error { str = strings.ToLower(str) m, found := encodings[str] if !found { - return errEncoderUnknown + return errEncodingMethodUnknown } *e = m diff --git a/libbeat/processors/fingerprint/fingerprint.go b/libbeat/processors/fingerprint/fingerprint.go index aa35d9076082..85cd887c202f 100644 --- a/libbeat/processors/fingerprint/fingerprint.go +++ b/libbeat/processors/fingerprint/fingerprint.go @@ -19,6 +19,7 @@ package fingerprint import ( "fmt" + "hash" "sort" "time" @@ -38,6 +39,7 @@ const processorName = "fingerprint" type fingerprint struct { config Config + hash hash.Hash } // New constructs a new fingerprint processor. @@ -51,6 +53,7 @@ func New(cfg *common.Config) (processors.Processor, error) { p := &fingerprint{ config: config, + hash: config.Method(), } return p, nil @@ -63,11 +66,15 @@ func (p *fingerprint) Run(event *beat.Event) (*beat.Event, error) { return nil, makeComputeFingerprintError(err) } - f := p.config.Method() - s := f.Sum(source) - v := p.config.Encoding(s) + hashFn := p.hash + hashFn.Reset() - if _, err = event.PutValue(p.config.TargetField, v); err != nil { + hashFn.Write([]byte(source)) + hash := hashFn.Sum(nil) + + encodedHash := p.config.Encoding(hash) + + if _, err = event.PutValue(p.config.TargetField, encodedHash); err != nil { return nil, makeComputeFingerprintError(err) } @@ -78,21 +85,21 @@ func (p *fingerprint) String() string { return fmt.Sprintf("%v=[method=[%v]]", processorName, p.config.Method) } -func makeSource(sourceFields []string, eventFields common.MapStr) ([]byte, error) { +func makeSource(sourceFields []string, eventFields common.MapStr) (string, error) { var str string for _, k := range sourceFields { v, err := eventFields.GetValue(k) if err == common.ErrKeyNotFound { - return nil, errors.Wrapf(err, "failed to find field [%v] in event", k) + return "", errors.Wrapf(err, "failed to find field [%v] in event", k) } if err != nil { - return nil, errors.Wrapf(err, "failed when finding field [%v] in event", k) + return "", errors.Wrapf(err, "failed when finding field [%v] in event", k) } i := v switch vv := v.(type) { case map[string]interface{}, []interface{}: - return nil, errors.Errorf("cannot compute fingerprint using non-scalar field [%v]", k) + return "", errors.Errorf("cannot compute fingerprint using non-scalar field [%v]", k) case time.Time: // Ensure we consistently hash times in UTC. i = vv.UTC() @@ -101,7 +108,7 @@ func makeSource(sourceFields []string, eventFields common.MapStr) ([]byte, error str += fmt.Sprintf("|%v|%v", k, i) } str += "|" - return []byte(str), nil + return str, nil } func makeComputeFingerprintError(err error) error { diff --git a/libbeat/processors/fingerprint/fingerprinter.go b/libbeat/processors/fingerprint/hash.go similarity index 75% rename from libbeat/processors/fingerprint/fingerprinter.go rename to libbeat/processors/fingerprint/hash.go index 9a3f6d254e7c..8c88b879640e 100644 --- a/libbeat/processors/fingerprint/fingerprinter.go +++ b/libbeat/processors/fingerprint/hash.go @@ -25,22 +25,22 @@ import ( "strings" ) -var errMethodUnknown = errors.New("unknown fingerprinting method") +var errFingerprintingMethodUnknown = errors.New("unknown fingerprinting method") -type fingerprinter func() hash.Hash +type hashMethod func() hash.Hash -var methods = map[string]fingerprinter{ +var hashes = map[string]hashMethod{ "sha1": sha1.New, "sha256": sha256.New, } -// Unpack creates the Method enumeration value from the given string -func (f *fingerprinter) Unpack(str string) error { +// Unpack creates the hashMethod from the given string +func (f *hashMethod) Unpack(str string) error { str = strings.ToLower(str) - m, found := methods[str] + m, found := hashes[str] if !found { - return errMethodUnknown + return errFingerprintingMethodUnknown } *f = m From 3c75d3b86999ea47b2c7c29bed5c51a738b85ffe Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 29 Oct 2019 11:36:56 -0700 Subject: [PATCH 11/28] Running mage fmt --- libbeat/processors/fingerprint/fingerprint.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libbeat/processors/fingerprint/fingerprint.go b/libbeat/processors/fingerprint/fingerprint.go index 85cd887c202f..73b2b9e0fbdc 100644 --- a/libbeat/processors/fingerprint/fingerprint.go +++ b/libbeat/processors/fingerprint/fingerprint.go @@ -23,11 +23,12 @@ import ( "sort" "time" + "github.com/pkg/errors" + "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/processors" jsprocessor "github.com/elastic/beats/libbeat/processors/script/javascript/module/processor" - "github.com/pkg/errors" ) func init() { From da29e8d62b21c07a95f76c56045120fb9eff19ff Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 29 Oct 2019 15:01:27 -0700 Subject: [PATCH 12/28] More methods + consolidating tests --- .../fingerprint/fingerprint_test.go | 123 ++++++++---------- libbeat/processors/fingerprint/hash.go | 5 + 2 files changed, 60 insertions(+), 68 deletions(-) diff --git a/libbeat/processors/fingerprint/fingerprint_test.go b/libbeat/processors/fingerprint/fingerprint_test.go index 400dae95c4fd..9691e0c25eaa 100644 --- a/libbeat/processors/fingerprint/fingerprint_test.go +++ b/libbeat/processors/fingerprint/fingerprint_test.go @@ -18,6 +18,7 @@ package fingerprint import ( + "fmt" "testing" "time" @@ -27,81 +28,67 @@ import ( "github.com/elastic/beats/libbeat/common" ) -func TestMethodDefault(t *testing.T) { - testConfig, err := common.NewConfigFrom(common.MapStr{ - "fields": []string{"field1"}, - }) - assert.NoError(t, err) - - p, err := New(testConfig) - assert.NoError(t, err) - - testEvent := &beat.Event{ - Fields: common.MapStr{ - "field1": "foo", +func TestHashMethods(t *testing.T) { + tests := []struct { + method string + expected string + }{ + { + "md5", + "3455d980d9c2a5a1c2c0b090a929aa3a", }, - Timestamp: time.Now(), - } - - newEvent, err := p.Run(testEvent) - assert.NoError(t, err) - - v, err := newEvent.GetValue("fingerprint") - assert.NoError(t, err) - assert.Equal(t, "4cf8b768ad20266c348d63a6d1ff5d6f6f9ed0f59f5c68ae031b78e3e04c5144", v) -} - -func TestMethodSHA1(t *testing.T) { - testConfig, err := common.NewConfigFrom(common.MapStr{ - "fields": []string{"field1"}, - "method": "sha1", - }) - assert.NoError(t, err) - - p, err := New(testConfig) - assert.NoError(t, err) - - testEvent := &beat.Event{ - Fields: common.MapStr{ - "field1": "foo", + { + "sha1", + "46de5d8225e75aeedd559c953f100dca41612b18", }, - Timestamp: time.Now(), - } - - newEvent, err := p.Run(testEvent) - assert.NoError(t, err) - - v, err := newEvent.GetValue("fingerprint") - assert.NoError(t, err) - assert.Equal(t, "46de5d8225e75aeedd559c953f100dca41612b18", v) -} - -func TestMethodSHA256(t *testing.T) { - testConfig, err := common.NewConfigFrom(common.MapStr{ - "fields": []string{"field1"}, - "method": "sha256", - }) - assert.NoError(t, err) - - p, err := New(testConfig) - assert.NoError(t, err) - - testEvent := &beat.Event{ - Fields: common.MapStr{ - "field1": "foo", + { + "sha256", + "4cf8b768ad20266c348d63a6d1ff5d6f6f9ed0f59f5c68ae031b78e3e04c5144", + }, + { + "sha384", + "251b4d77ceea8ad64bf5ed906b5760f9b758af3b30e8f9de5d0d70ec6a2745d25b1be00c5317dc7859256de2d416b179", + }, + { + "sha512", + "903a7f492a22015c89a8e00c40a85da814c2ff42c28cdf1a29495faa8a849eba00449921a75b12c9c212169f100ebf6b05ac8389a8fbfd61cba6026e86a6e2c1", }, - Timestamp: time.Now(), } - newEvent, err := p.Run(testEvent) - assert.NoError(t, err) - - v, err := newEvent.GetValue("fingerprint") - assert.NoError(t, err) - assert.Equal(t, "4cf8b768ad20266c348d63a6d1ff5d6f6f9ed0f59f5c68ae031b78e3e04c5144", v) + for _, test := range tests { + name := test.method + if name == "" { + name = "default" + } + + name = fmt.Sprintf("testing %v method", name) + t.Run(name, func(t *testing.T) { + testEvent := &beat.Event{ + Fields: common.MapStr{ + "field1": "foo", + }, + Timestamp: time.Now(), + } + + testConfig, err := common.NewConfigFrom(common.MapStr{ + "fields": []string{"field1"}, + "method": test.method, + }) + assert.NoError(t, err) + + p, err := New(testConfig) + assert.NoError(t, err) + + newEvent, err := p.Run(testEvent) + assert.NoError(t, err) + + v, err := newEvent.GetValue("fingerprint") + assert.NoError(t, err) + assert.Equal(t, test.expected, v) + }) + } } -// TODO: other fingerprinting methods // TODO: Order of source fields doesn't matter // TODO: Missing source fields // TODO: non-scalar fields diff --git a/libbeat/processors/fingerprint/hash.go b/libbeat/processors/fingerprint/hash.go index 8c88b879640e..e8f14a6f59f5 100644 --- a/libbeat/processors/fingerprint/hash.go +++ b/libbeat/processors/fingerprint/hash.go @@ -18,8 +18,10 @@ package fingerprint import ( + "crypto/md5" "crypto/sha1" "crypto/sha256" + "crypto/sha512" "errors" "hash" "strings" @@ -30,8 +32,11 @@ var errFingerprintingMethodUnknown = errors.New("unknown fingerprinting method") type hashMethod func() hash.Hash var hashes = map[string]hashMethod{ + "md5": md5.New, "sha1": sha1.New, "sha256": sha256.New, + "sha384": sha512.New384, + "sha512": sha512.New, } // Unpack creates the hashMethod from the given string From 52e5110cbd7f8f6ffdb1a3e61a75544928fdadf7 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 29 Oct 2019 19:17:49 -0700 Subject: [PATCH 13/28] Fleshing out tests --- libbeat/processors/fingerprint/encode.go | 10 +- libbeat/processors/fingerprint/fingerprint.go | 19 +- .../fingerprint/fingerprint_test.go | 291 ++++++++++++++++-- libbeat/processors/fingerprint/hash.go | 10 +- 4 files changed, 296 insertions(+), 34 deletions(-) diff --git a/libbeat/processors/fingerprint/encode.go b/libbeat/processors/fingerprint/encode.go index c611b4e0f625..0087e2e5d0f5 100644 --- a/libbeat/processors/fingerprint/encode.go +++ b/libbeat/processors/fingerprint/encode.go @@ -21,12 +21,10 @@ import ( "encoding/base32" "encoding/base64" "encoding/hex" - "errors" + "fmt" "strings" ) -var errEncodingMethodUnknown = errors.New("unknown encoding method") - type encodingMethod func([]byte) string var encodings = map[string]encodingMethod{ @@ -41,9 +39,13 @@ func (e *encodingMethod) Unpack(str string) error { m, found := encodings[str] if !found { - return errEncodingMethodUnknown + return makeUnknownEncodingError(str) } *e = m return nil } + +func makeUnknownEncodingError(encoding string) error { + return fmt.Errorf("invalid encoding [%s]", encoding) +} diff --git a/libbeat/processors/fingerprint/fingerprint.go b/libbeat/processors/fingerprint/fingerprint.go index 73b2b9e0fbdc..9728b919080e 100644 --- a/libbeat/processors/fingerprint/fingerprint.go +++ b/libbeat/processors/fingerprint/fingerprint.go @@ -38,8 +38,11 @@ func init() { const processorName = "fingerprint" +var errNoFields = errors.New("must specify at least one field") + type fingerprint struct { config Config + fields []string hash hash.Hash } @@ -55,6 +58,7 @@ func New(cfg *common.Config) (processors.Processor, error) { p := &fingerprint{ config: config, hash: config.Method(), + fields: unique(config.Fields), } return p, nil @@ -62,7 +66,7 @@ func New(cfg *common.Config) (processors.Processor, error) { // Run enriches the given event with fingerprint information func (p *fingerprint) Run(event *beat.Event) (*beat.Event, error) { - source, err := makeSource(p.config.Fields, event.Fields) + source, err := makeSource(p.fields, event.Fields) if err != nil { return nil, makeComputeFingerprintError(err) } @@ -115,3 +119,16 @@ func makeSource(sourceFields []string, eventFields common.MapStr) (string, error func makeComputeFingerprintError(err error) error { return errors.Wrap(err, "failed to compute fingerprint") } + +func unique(in []string) []string { + seen := map[string]bool{} + var out = make([]string, 0, len(in)) + for _, item := range in { + if _, found := seen[item]; !found { + seen[item] = true + out = append(out, item) + } + } + + return out +} diff --git a/libbeat/processors/fingerprint/fingerprint_test.go b/libbeat/processors/fingerprint/fingerprint_test.go index 9691e0c25eaa..ffddba5f5a40 100644 --- a/libbeat/processors/fingerprint/fingerprint_test.go +++ b/libbeat/processors/fingerprint/fingerprint_test.go @@ -29,49 +29,46 @@ import ( ) func TestHashMethods(t *testing.T) { + testEvent := &beat.Event{ + Fields: common.MapStr{ + "field1": "foo", + "field2": "bar", + "unused_field": "baz", + }, + Timestamp: time.Now(), + } + tests := []struct { method string expected string }{ { "md5", - "3455d980d9c2a5a1c2c0b090a929aa3a", + "4c45df4792f3ef850c928ec5f5232538", }, { "sha1", - "46de5d8225e75aeedd559c953f100dca41612b18", + "22f76427d626516d3f7a05785165b99617683b22", }, { "sha256", - "4cf8b768ad20266c348d63a6d1ff5d6f6f9ed0f59f5c68ae031b78e3e04c5144", + "1208288932231e313b369bae587ff574cd3016a408e52e7128d7bee752674003", }, { "sha384", - "251b4d77ceea8ad64bf5ed906b5760f9b758af3b30e8f9de5d0d70ec6a2745d25b1be00c5317dc7859256de2d416b179", + "295adfe0bc03908948e4b0b6a54f441767867e426dda590430459c8a147fbba242a38cba282adee78335b9e08877b86c", }, { "sha512", - "903a7f492a22015c89a8e00c40a85da814c2ff42c28cdf1a29495faa8a849eba00449921a75b12c9c212169f100ebf6b05ac8389a8fbfd61cba6026e86a6e2c1", + "f50ad51b63c92a0ed0c910527119b81806f3110f0afaa1dcb93506a78371ea761e50c0fc09b08c441d832dd2da1b45e5d8361adfb240e1fffc2695122a23e183", }, } for _, test := range tests { - name := test.method - if name == "" { - name = "default" - } - - name = fmt.Sprintf("testing %v method", name) + name := fmt.Sprintf("testing %v fingerprinting method", test.method) t.Run(name, func(t *testing.T) { - testEvent := &beat.Event{ - Fields: common.MapStr{ - "field1": "foo", - }, - Timestamp: time.Now(), - } - testConfig, err := common.NewConfigFrom(common.MapStr{ - "fields": []string{"field1"}, + "fields": []string{"field1", "field2"}, "method": test.method, }) assert.NoError(t, err) @@ -89,9 +86,253 @@ func TestHashMethods(t *testing.T) { } } -// TODO: Order of source fields doesn't matter -// TODO: Missing source fields -// TODO: non-scalar fields -// TODO: hashing time fields -// TODO: invalid fingerprinting method in config -// TODO: encoding +func TestSourceFields(t *testing.T) { + testEvent := &beat.Event{ + Fields: common.MapStr{ + "field1": "foo", + "field2": "bar", + "nested": common.MapStr{ + "field": "qux", + }, + "unused_field": "baz", + }, + Timestamp: time.Now(), + } + expectedFingerprint := "3d51237d384215a6e731f2cc67ead6d7d9a5138377897c8f542a915be3c25bcf" + + tests := []struct { + name string + fields []string + }{ + { + "order is insignificant", + []string{"field1", "nested.field"}, + }, + { + "order is insignificant", + []string{"nested.field", "field1"}, + }, + { + "duplicates are ignored", + []string{"nested.field", "field1", "nested.field"}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testConfig, err := common.NewConfigFrom(common.MapStr{ + "fields": test.fields, + "method": "sha256", + }) + assert.NoError(t, err) + + p, err := New(testConfig) + assert.NoError(t, err) + + newEvent, err := p.Run(testEvent) + assert.NoError(t, err) + + v, err := newEvent.GetValue("fingerprint") + assert.NoError(t, err) + assert.Equal(t, expectedFingerprint, v) + }) + } +} + +func TestEncoding(t *testing.T) { + testEvent := &beat.Event{ + Fields: common.MapStr{ + "field1": "foo", + "field2": "bar", + "nested": common.MapStr{ + "field": "qux", + }, + "unused_field": "baz", + }, + Timestamp: time.Now(), + } + + tests := []struct { + encoding string + expectedFingerprint string + }{ + { + "hex", + "8934ca639027aab1ee9f3944d4d6bd1e", + }, + { + "base32", + "RE2MUY4QE6VLD3U7HFCNJVV5DY======", + }, + { + "base64", + "iTTKY5AnqrHunzlE1Na9Hg==", + }, + } + + for _, test := range tests { + name := fmt.Sprintf("testing %v encoding", test.encoding) + t.Run(name, func(t *testing.T) { + testConfig, err := common.NewConfigFrom(common.MapStr{ + "fields": []string{"field2", "nested.field"}, + "method": "md5", + "encoding": test.encoding, + }) + assert.NoError(t, err) + + p, err := New(testConfig) + assert.NoError(t, err) + + newEvent, err := p.Run(testEvent) + assert.NoError(t, err) + + v, err := newEvent.GetValue("fingerprint") + assert.NoError(t, err) + assert.Equal(t, test.expectedFingerprint, v) + }) + } +} + +func TestConsistentHashingTimeFields(t *testing.T) { + tzUTC := time.UTC + tzMTV := time.FixedZone("Mountain View, California, USA", int((-8 * time.Hour).Seconds())) + tzBOM := time.FixedZone("Bombay, Maharashtra, India", int((5*time.Hour + 30*time.Minute).Seconds())) + + expectedFingerprint := "4534d56a673c2da41df32db5da87cf47e639e84fe82907f2c015c8dfcac5d4f5" + + tests := []struct { + name string + event common.MapStr + }{ + { + "time field in UTC", + common.MapStr{ + "timestamp": time.Date(2019, 10, 29, 0, 0, 0, 0, tzUTC), + }, + }, + { + "time field in Mountain View time", + common.MapStr{ + "timestamp": time.Date(2019, 10, 28, 16, 0, 0, 0, tzMTV), + }, + }, + { + "time field in Bombay time", + common.MapStr{ + "timestamp": time.Date(2019, 10, 29, 5, 30, 0, 0, tzBOM), + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testConfig, err := common.NewConfigFrom(common.MapStr{ + "fields": []string{"timestamp"}, + }) + assert.NoError(t, err) + + p, err := New(testConfig) + assert.NoError(t, err) + + testEvent := &beat.Event{ + Fields: test.event, + } + newEvent, err := p.Run(testEvent) + assert.NoError(t, err) + + v, err := newEvent.GetValue("fingerprint") + assert.NoError(t, err) + assert.Equal(t, expectedFingerprint, v) + }) + } +} + +func TestSourceFieldErrors(t *testing.T) { + testEvent := &beat.Event{ + Fields: common.MapStr{ + "field1": "foo", + "field2": "bar", + "complex_field": map[string]interface{}{ + "child": "qux", + }, + "unused_field": "baz", + }, + Timestamp: time.Now(), + } + + tests := []struct { + name string + fields []string + expectedErrMsg string + }{ + { + "missing field", + []string{"field1", "missing_field"}, + "failed to compute fingerprint: failed to find field [missing_field] in event: key not found", + }, + { + "non-scalar field", + []string{"field1", "complex_field"}, + "failed to compute fingerprint: cannot compute fingerprint using non-scalar field [complex_field]", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testConfig, err := common.NewConfigFrom(common.MapStr{ + "fields": test.fields, + "method": "sha256", + }) + assert.NoError(t, err) + + p, err := New(testConfig) + assert.NoError(t, err) + + _, err = p.Run(testEvent) + assert.EqualError(t, err, test.expectedErrMsg) + }) + } +} + +func TestInvalidConfig(t *testing.T) { + tests := []struct { + name string + config common.MapStr + expectedErrMsg string + }{ + { + "no fields", + common.MapStr{ + "fields": []string{}, + "method": "sha256", + }, + "failed to unpack fingerprint processor configuration: empty field accessing 'fields'", + }, + { + "invalid fingerprinting method", + common.MapStr{ + "fields": []string{"doesnt", "matter"}, + "method": "non_existent", + }, + "failed to unpack fingerprint processor configuration: invalid fingerprinting method [non_existent] accessing 'method'", + }, + { + "invalid encoding", + common.MapStr{ + "fields": []string{"doesnt", "matter"}, + "encoding": "non_existent", + }, + "failed to unpack fingerprint processor configuration: invalid encoding method [non_existent]", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testConfig, err := common.NewConfigFrom(test.config) + assert.NoError(t, err) + + _, err = New(testConfig) + assert.EqualError(t, err, test.expectedErrMsg) + }) + } +} diff --git a/libbeat/processors/fingerprint/hash.go b/libbeat/processors/fingerprint/hash.go index e8f14a6f59f5..6628f5b66644 100644 --- a/libbeat/processors/fingerprint/hash.go +++ b/libbeat/processors/fingerprint/hash.go @@ -22,13 +22,11 @@ import ( "crypto/sha1" "crypto/sha256" "crypto/sha512" - "errors" + "fmt" "hash" "strings" ) -var errFingerprintingMethodUnknown = errors.New("unknown fingerprinting method") - type hashMethod func() hash.Hash var hashes = map[string]hashMethod{ @@ -45,9 +43,13 @@ func (f *hashMethod) Unpack(str string) error { m, found := hashes[str] if !found { - return errFingerprintingMethodUnknown + return makeUnknownMethodError(str) } *f = m return nil } + +func makeUnknownMethodError(method string) error { + return fmt.Errorf("invalid fingerprinting method [%s]", method) +} From 92af70ce606eb00104718d6972678cb07c138015 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 29 Oct 2019 19:28:45 -0700 Subject: [PATCH 14/28] Adding test for target field --- .../fingerprint/fingerprint_test.go | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/libbeat/processors/fingerprint/fingerprint_test.go b/libbeat/processors/fingerprint/fingerprint_test.go index ffddba5f5a40..46a6e1d5ead5 100644 --- a/libbeat/processors/fingerprint/fingerprint_test.go +++ b/libbeat/processors/fingerprint/fingerprint_test.go @@ -247,6 +247,58 @@ func TestConsistentHashingTimeFields(t *testing.T) { } } +func TestTargetField(t *testing.T) { + testEvent := &beat.Event{ + Fields: common.MapStr{ + "field1": "foo", + "nested": common.MapStr{ + "field": "bar", + }, + "unused_field": "baz", + }, + Timestamp: time.Now(), + } + + expectedFingerprint := "4cf8b768ad20266c348d63a6d1ff5d6f6f9ed0f59f5c68ae031b78e3e04c5144" + + tests := []struct { + name string + targetField string + }{ + { + "root-level target field", + "target_field", + }, + { + "nested target field", + "nested.target_field", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testConfig, err := common.NewConfigFrom(common.MapStr{ + "fields": []string{"field1"}, + "target_field": test.targetField, + }) + assert.NoError(t, err) + + p, err := New(testConfig) + assert.NoError(t, err) + + newEvent, err := p.Run(testEvent) + assert.NoError(t, err) + + v, err := newEvent.GetValue(test.targetField) + assert.NoError(t, err) + assert.Equal(t, expectedFingerprint, v) + + _, err = newEvent.GetValue("fingerprint") + assert.EqualError(t, err, common.ErrKeyNotFound.Error()) + }) + } +} + func TestSourceFieldErrors(t *testing.T) { testEvent := &beat.Event{ Fields: common.MapStr{ From b1981e80eb429f526981673af0502ff43ff33851 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 29 Oct 2019 19:38:05 -0700 Subject: [PATCH 15/28] Adding documentation --- libbeat/docs/processors-using.asciidoc | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/libbeat/docs/processors-using.asciidoc b/libbeat/docs/processors-using.asciidoc index 3321f4a2fbb9..fe2e49ce247c 100644 --- a/libbeat/docs/processors-using.asciidoc +++ b/libbeat/docs/processors-using.asciidoc @@ -1184,6 +1184,28 @@ The following settings are supported: empty array (`[]`) or an empty object (`{}`) are considered empty values. Default is `false`. +[[fingerprint]] +=== Generate a fingerprint of an event + +The `fingerprint` processor generates a fingerprint of an event based on a +specified subset of its fields. + +[source,yaml] +----------------------------------------------------- +processors: + - fingerprint: + fields: ["field1", "field2", ...] +----------------------------------------------------- + +.Fingerprint options +[options="header"] +|====== +| `fields` | yes | | List of fields to use as the source for the fingerprint. | +| `target_field` | no | `fingerprint` | Field in which the generated fingerprint should be stored. | +| `method` | no | `sha256` | Algorithm to use for computing the fingerprint. Must be one of: `md5`, `sha1`, `sha256` (default), `sha384`, `sha512`. | +| `encoding` | no | `hex` | Encoding to use on the fingerprint value. Must be one of `hex` (default), `base32`, or `base64`. | +|====== + [[include-fields]] === Keep fields from events From 3a2825cdab04544e6601c3278c11d04703079438 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 29 Oct 2019 19:55:00 -0700 Subject: [PATCH 16/28] Adding CHANGELOG entry --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index a57cad8f5b82..75bb82ea39eb 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -314,6 +314,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add `keep_null` setting to allow Beats to publish null values in events. {issue}5522[5522] {pull}13928[13928] - Add shared_credential_file option in aws related config for specifying credential file directory. {issue}14157[14157] {pull}14178[14178] - GA the `script` processor. {pull}14325[14325] +- Add `fingerprint` processor. {issue}11173[11173] {pull}14205[14205] *Auditbeat* From 9a0be57ceaa67fd8404e69aa2d32f7968dd014db Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 30 Oct 2019 04:48:42 -0700 Subject: [PATCH 17/28] Fixing test --- libbeat/processors/fingerprint/fingerprint_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/processors/fingerprint/fingerprint_test.go b/libbeat/processors/fingerprint/fingerprint_test.go index 46a6e1d5ead5..919c5dc31a5b 100644 --- a/libbeat/processors/fingerprint/fingerprint_test.go +++ b/libbeat/processors/fingerprint/fingerprint_test.go @@ -374,7 +374,7 @@ func TestInvalidConfig(t *testing.T) { "fields": []string{"doesnt", "matter"}, "encoding": "non_existent", }, - "failed to unpack fingerprint processor configuration: invalid encoding method [non_existent]", + "failed to unpack fingerprint processor configuration: invalid encoding [non_existent] accessing 'encoding'", }, } From b2ecab65f63f81f738d6fd10890874491fbfbf71 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 30 Oct 2019 16:27:32 -0700 Subject: [PATCH 18/28] Converting tests to map --- .../fingerprint/fingerprint_test.go | 153 ++++++------------ 1 file changed, 48 insertions(+), 105 deletions(-) diff --git a/libbeat/processors/fingerprint/fingerprint_test.go b/libbeat/processors/fingerprint/fingerprint_test.go index 919c5dc31a5b..e09c91f30354 100644 --- a/libbeat/processors/fingerprint/fingerprint_test.go +++ b/libbeat/processors/fingerprint/fingerprint_test.go @@ -18,7 +18,6 @@ package fingerprint import ( - "fmt" "testing" "time" @@ -38,38 +37,21 @@ func TestHashMethods(t *testing.T) { Timestamp: time.Now(), } - tests := []struct { - method string + tests := map[string]struct { expected string }{ - { - "md5", - "4c45df4792f3ef850c928ec5f5232538", - }, - { - "sha1", - "22f76427d626516d3f7a05785165b99617683b22", - }, - { - "sha256", - "1208288932231e313b369bae587ff574cd3016a408e52e7128d7bee752674003", - }, - { - "sha384", - "295adfe0bc03908948e4b0b6a54f441767867e426dda590430459c8a147fbba242a38cba282adee78335b9e08877b86c", - }, - { - "sha512", - "f50ad51b63c92a0ed0c910527119b81806f3110f0afaa1dcb93506a78371ea761e50c0fc09b08c441d832dd2da1b45e5d8361adfb240e1fffc2695122a23e183", - }, + "md5": {"4c45df4792f3ef850c928ec5f5232538"}, + "sha1": {"22f76427d626516d3f7a05785165b99617683b22"}, + "sha256": {"1208288932231e313b369bae587ff574cd3016a408e52e7128d7bee752674003"}, + "sha384": {"295adfe0bc03908948e4b0b6a54f441767867e426dda590430459c8a147fbba242a38cba282adee78335b9e08877b86c"}, + "sha512": {"f50ad51b63c92a0ed0c910527119b81806f3110f0afaa1dcb93506a78371ea761e50c0fc09b08c441d832dd2da1b45e5d8361adfb240e1fffc2695122a23e183"}, } - for _, test := range tests { - name := fmt.Sprintf("testing %v fingerprinting method", test.method) - t.Run(name, func(t *testing.T) { + for method, test := range tests { + t.Run(method, func(t *testing.T) { testConfig, err := common.NewConfigFrom(common.MapStr{ "fields": []string{"field1", "field2"}, - "method": test.method, + "method": method, }) assert.NoError(t, err) @@ -100,26 +82,16 @@ func TestSourceFields(t *testing.T) { } expectedFingerprint := "3d51237d384215a6e731f2cc67ead6d7d9a5138377897c8f542a915be3c25bcf" - tests := []struct { - name string + tests := map[string]struct { fields []string }{ - { - "order is insignificant", - []string{"field1", "nested.field"}, - }, - { - "order is insignificant", - []string{"nested.field", "field1"}, - }, - { - "duplicates are ignored", - []string{"nested.field", "field1", "nested.field"}, - }, + "order_1": {[]string{"field1", "nested.field"}}, + "order_2": {[]string{"nested.field", "field1"}}, + "duplicates_ignored": {[]string{"nested.field", "field1", "nested.field"}}, } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { + for name, test := range tests { + t.Run(name, func(t *testing.T) { testConfig, err := common.NewConfigFrom(common.MapStr{ "fields": test.fields, "method": "sha256", @@ -152,31 +124,20 @@ func TestEncoding(t *testing.T) { Timestamp: time.Now(), } - tests := []struct { - encoding string + tests := map[string]struct { expectedFingerprint string }{ - { - "hex", - "8934ca639027aab1ee9f3944d4d6bd1e", - }, - { - "base32", - "RE2MUY4QE6VLD3U7HFCNJVV5DY======", - }, - { - "base64", - "iTTKY5AnqrHunzlE1Na9Hg==", - }, + "hex": {"8934ca639027aab1ee9f3944d4d6bd1e"}, + "base32": {"RE2MUY4QE6VLD3U7HFCNJVV5DY======"}, + "base64": {"iTTKY5AnqrHunzlE1Na9Hg=="}, } - for _, test := range tests { - name := fmt.Sprintf("testing %v encoding", test.encoding) - t.Run(name, func(t *testing.T) { + for encoding, test := range tests { + t.Run(encoding, func(t *testing.T) { testConfig, err := common.NewConfigFrom(common.MapStr{ "fields": []string{"field2", "nested.field"}, "method": "md5", - "encoding": test.encoding, + "encoding": encoding, }) assert.NoError(t, err) @@ -195,37 +156,33 @@ func TestEncoding(t *testing.T) { func TestConsistentHashingTimeFields(t *testing.T) { tzUTC := time.UTC - tzMTV := time.FixedZone("Mountain View, California, USA", int((-8 * time.Hour).Seconds())) - tzBOM := time.FixedZone("Bombay, Maharashtra, India", int((5*time.Hour + 30*time.Minute).Seconds())) + tzPST := time.FixedZone("Pacific Standard Time", int((-8 * time.Hour).Seconds())) + tzIST := time.FixedZone("Indian Standard Time", int((5*time.Hour + 30*time.Minute).Seconds())) expectedFingerprint := "4534d56a673c2da41df32db5da87cf47e639e84fe82907f2c015c8dfcac5d4f5" - tests := []struct { - name string + tests := map[string]struct { event common.MapStr }{ - { - "time field in UTC", + "UTC": { common.MapStr{ "timestamp": time.Date(2019, 10, 29, 0, 0, 0, 0, tzUTC), }, }, - { - "time field in Mountain View time", + "PST": { common.MapStr{ - "timestamp": time.Date(2019, 10, 28, 16, 0, 0, 0, tzMTV), + "timestamp": time.Date(2019, 10, 28, 16, 0, 0, 0, tzPST), }, }, - { - "time field in Bombay time", + "IST": { common.MapStr{ - "timestamp": time.Date(2019, 10, 29, 5, 30, 0, 0, tzBOM), + "timestamp": time.Date(2019, 10, 29, 5, 30, 0, 0, tzIST), }, }, } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { + for name, test := range tests { + t.Run(name, func(t *testing.T) { testConfig, err := common.NewConfigFrom(common.MapStr{ "fields": []string{"timestamp"}, }) @@ -261,22 +218,15 @@ func TestTargetField(t *testing.T) { expectedFingerprint := "4cf8b768ad20266c348d63a6d1ff5d6f6f9ed0f59f5c68ae031b78e3e04c5144" - tests := []struct { - name string + tests := map[string]struct { targetField string }{ - { - "root-level target field", - "target_field", - }, - { - "nested target field", - "nested.target_field", - }, + "root": {"target_field"}, + "nested": {"nested.target_field"}, } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { + for name, test := range tests { + t.Run(name, func(t *testing.T) { testConfig, err := common.NewConfigFrom(common.MapStr{ "fields": []string{"field1"}, "target_field": test.targetField, @@ -312,25 +262,22 @@ func TestSourceFieldErrors(t *testing.T) { Timestamp: time.Now(), } - tests := []struct { - name string + tests := map[string]struct { fields []string expectedErrMsg string }{ - { - "missing field", + "missing": { []string{"field1", "missing_field"}, "failed to compute fingerprint: failed to find field [missing_field] in event: key not found", }, - { - "non-scalar field", + "non-scalar": { []string{"field1", "complex_field"}, "failed to compute fingerprint: cannot compute fingerprint using non-scalar field [complex_field]", }, } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { + for name, test := range tests { + t.Run(name, func(t *testing.T) { testConfig, err := common.NewConfigFrom(common.MapStr{ "fields": test.fields, "method": "sha256", @@ -347,29 +294,25 @@ func TestSourceFieldErrors(t *testing.T) { } func TestInvalidConfig(t *testing.T) { - tests := []struct { - name string + tests := map[string]struct { config common.MapStr expectedErrMsg string }{ - { - "no fields", + "no fields": { common.MapStr{ "fields": []string{}, "method": "sha256", }, "failed to unpack fingerprint processor configuration: empty field accessing 'fields'", }, - { - "invalid fingerprinting method", + "invalid fingerprinting method": { common.MapStr{ "fields": []string{"doesnt", "matter"}, "method": "non_existent", }, "failed to unpack fingerprint processor configuration: invalid fingerprinting method [non_existent] accessing 'method'", }, - { - "invalid encoding", + "invalid encoding": { common.MapStr{ "fields": []string{"doesnt", "matter"}, "encoding": "non_existent", @@ -378,8 +321,8 @@ func TestInvalidConfig(t *testing.T) { }, } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { + for name, test := range tests { + t.Run(name, func(t *testing.T) { testConfig, err := common.NewConfigFrom(test.config) assert.NoError(t, err) From 217e3184929abee2b6d086a9c93e310f46c61dcc Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 30 Oct 2019 16:47:18 -0700 Subject: [PATCH 19/28] Isolating tests --- libbeat/processors/fingerprint/fingerprint.go | 2 +- .../fingerprint/fingerprint_test.go | 91 ++++++++++--------- 2 files changed, 49 insertions(+), 44 deletions(-) diff --git a/libbeat/processors/fingerprint/fingerprint.go b/libbeat/processors/fingerprint/fingerprint.go index 9728b919080e..bde918892aed 100644 --- a/libbeat/processors/fingerprint/fingerprint.go +++ b/libbeat/processors/fingerprint/fingerprint.go @@ -103,7 +103,7 @@ func makeSource(sourceFields []string, eventFields common.MapStr) (string, error i := v switch vv := v.(type) { - case map[string]interface{}, []interface{}: + case map[string]interface{}, []interface{}, common.MapStr: return "", errors.Errorf("cannot compute fingerprint using non-scalar field [%v]", k) case time.Time: // Ensure we consistently hash times in UTC. diff --git a/libbeat/processors/fingerprint/fingerprint_test.go b/libbeat/processors/fingerprint/fingerprint_test.go index e09c91f30354..efb6ec9c52cd 100644 --- a/libbeat/processors/fingerprint/fingerprint_test.go +++ b/libbeat/processors/fingerprint/fingerprint_test.go @@ -28,13 +28,10 @@ import ( ) func TestHashMethods(t *testing.T) { - testEvent := &beat.Event{ - Fields: common.MapStr{ - "field1": "foo", - "field2": "bar", - "unused_field": "baz", - }, - Timestamp: time.Now(), + testFields := common.MapStr{ + "field1": "foo", + "field2": "bar", + "unused_field": "baz", } tests := map[string]struct { @@ -58,6 +55,11 @@ func TestHashMethods(t *testing.T) { p, err := New(testConfig) assert.NoError(t, err) + testEvent := &beat.Event{ + Fields: testFields.Clone(), + Timestamp: time.Now(), + } + newEvent, err := p.Run(testEvent) assert.NoError(t, err) @@ -69,16 +71,13 @@ func TestHashMethods(t *testing.T) { } func TestSourceFields(t *testing.T) { - testEvent := &beat.Event{ - Fields: common.MapStr{ - "field1": "foo", - "field2": "bar", - "nested": common.MapStr{ - "field": "qux", - }, - "unused_field": "baz", + testFields := common.MapStr{ + "field1": "foo", + "field2": "bar", + "nested": common.MapStr{ + "field": "qux", }, - Timestamp: time.Now(), + "unused_field": "baz", } expectedFingerprint := "3d51237d384215a6e731f2cc67ead6d7d9a5138377897c8f542a915be3c25bcf" @@ -101,6 +100,10 @@ func TestSourceFields(t *testing.T) { p, err := New(testConfig) assert.NoError(t, err) + testEvent := &beat.Event{ + Fields: testFields.Clone(), + Timestamp: time.Now(), + } newEvent, err := p.Run(testEvent) assert.NoError(t, err) @@ -112,16 +115,13 @@ func TestSourceFields(t *testing.T) { } func TestEncoding(t *testing.T) { - testEvent := &beat.Event{ - Fields: common.MapStr{ - "field1": "foo", - "field2": "bar", - "nested": common.MapStr{ - "field": "qux", - }, - "unused_field": "baz", + testFields := common.MapStr{ + "field1": "foo", + "field2": "bar", + "nested": common.MapStr{ + "field": "qux", }, - Timestamp: time.Now(), + "unused_field": "baz", } tests := map[string]struct { @@ -144,6 +144,10 @@ func TestEncoding(t *testing.T) { p, err := New(testConfig) assert.NoError(t, err) + testEvent := &beat.Event{ + Fields: testFields.Clone(), + Timestamp: time.Now(), + } newEvent, err := p.Run(testEvent) assert.NoError(t, err) @@ -205,17 +209,13 @@ func TestConsistentHashingTimeFields(t *testing.T) { } func TestTargetField(t *testing.T) { - testEvent := &beat.Event{ - Fields: common.MapStr{ - "field1": "foo", - "nested": common.MapStr{ - "field": "bar", - }, - "unused_field": "baz", + testFields := common.MapStr{ + "field1": "foo", + "nested": common.MapStr{ + "field": "bar", }, - Timestamp: time.Now(), + "unused_field": "baz", } - expectedFingerprint := "4cf8b768ad20266c348d63a6d1ff5d6f6f9ed0f59f5c68ae031b78e3e04c5144" tests := map[string]struct { @@ -236,6 +236,10 @@ func TestTargetField(t *testing.T) { p, err := New(testConfig) assert.NoError(t, err) + testEvent := &beat.Event{ + Fields: testFields.Clone(), + Timestamp: time.Now(), + } newEvent, err := p.Run(testEvent) assert.NoError(t, err) @@ -250,16 +254,13 @@ func TestTargetField(t *testing.T) { } func TestSourceFieldErrors(t *testing.T) { - testEvent := &beat.Event{ - Fields: common.MapStr{ - "field1": "foo", - "field2": "bar", - "complex_field": map[string]interface{}{ - "child": "qux", - }, - "unused_field": "baz", + testFields := common.MapStr{ + "field1": "foo", + "field2": "bar", + "complex_field": map[string]interface{}{ + "child": "qux", }, - Timestamp: time.Now(), + "unused_field": "baz", } tests := map[string]struct { @@ -287,6 +288,10 @@ func TestSourceFieldErrors(t *testing.T) { p, err := New(testConfig) assert.NoError(t, err) + testEvent := &beat.Event{ + Fields: testFields.Clone(), + Timestamp: time.Now(), + } _, err = p.Run(testEvent) assert.EqualError(t, err, test.expectedErrMsg) }) From 6479e84065197b661d740ca4f4d27293c7757159 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 30 Oct 2019 16:54:24 -0700 Subject: [PATCH 20/28] Use io.Writer to stream in fields --- libbeat/processors/fingerprint/fingerprint.go | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/libbeat/processors/fingerprint/fingerprint.go b/libbeat/processors/fingerprint/fingerprint.go index bde918892aed..91cf9fa10f72 100644 --- a/libbeat/processors/fingerprint/fingerprint.go +++ b/libbeat/processors/fingerprint/fingerprint.go @@ -20,6 +20,7 @@ package fingerprint import ( "fmt" "hash" + "io" "sort" "time" @@ -66,17 +67,15 @@ func New(cfg *common.Config) (processors.Processor, error) { // Run enriches the given event with fingerprint information func (p *fingerprint) Run(event *beat.Event) (*beat.Event, error) { - source, err := makeSource(p.fields, event.Fields) + hashFn := p.hash + hashFn.Reset() + + err := writeFields(hashFn, p.fields, event.Fields) if err != nil { return nil, makeComputeFingerprintError(err) } - hashFn := p.hash - hashFn.Reset() - - hashFn.Write([]byte(source)) hash := hashFn.Sum(nil) - encodedHash := p.config.Encoding(hash) if _, err = event.PutValue(p.config.TargetField, encodedHash); err != nil { @@ -90,30 +89,30 @@ func (p *fingerprint) String() string { return fmt.Sprintf("%v=[method=[%v]]", processorName, p.config.Method) } -func makeSource(sourceFields []string, eventFields common.MapStr) (string, error) { - var str string +func writeFields(to io.Writer, sourceFields []string, eventFields common.MapStr) error { for _, k := range sourceFields { v, err := eventFields.GetValue(k) if err == common.ErrKeyNotFound { - return "", errors.Wrapf(err, "failed to find field [%v] in event", k) + return errors.Wrapf(err, "failed to find field [%v] in event", k) } if err != nil { - return "", errors.Wrapf(err, "failed when finding field [%v] in event", k) + return errors.Wrapf(err, "failed when finding field [%v] in event", k) } i := v switch vv := v.(type) { case map[string]interface{}, []interface{}, common.MapStr: - return "", errors.Errorf("cannot compute fingerprint using non-scalar field [%v]", k) + return errors.Errorf("cannot compute fingerprint using non-scalar field [%v]", k) case time.Time: // Ensure we consistently hash times in UTC. i = vv.UTC() } - str += fmt.Sprintf("|%v|%v", k, i) + io.WriteString(to, fmt.Sprintf("|%v|%v", k, i)) } - str += "|" - return str, nil + + io.WriteString(to, "|") + return nil } func makeComputeFingerprintError(err error) error { From 661f891f542dbbd7d074463e32e32c5853cd4d78 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 30 Oct 2019 17:20:00 -0700 Subject: [PATCH 21/28] Implement ignore_missing setting --- libbeat/processors/fingerprint/config.go | 16 ++++--- libbeat/processors/fingerprint/fingerprint.go | 14 +++--- .../fingerprint/fingerprint_test.go | 47 +++++++++++++++++++ 3 files changed, 63 insertions(+), 14 deletions(-) diff --git a/libbeat/processors/fingerprint/config.go b/libbeat/processors/fingerprint/config.go index d8e9e741eb12..dc36b6bceffb 100644 --- a/libbeat/processors/fingerprint/config.go +++ b/libbeat/processors/fingerprint/config.go @@ -19,16 +19,18 @@ package fingerprint // Config for fingerprint processor. type Config struct { - Method hashMethod `config:"method"` // Hash function to use for fingerprinting - Fields []string `config:"fields" validate:"required"` // Source fields to compute fingerprint from - TargetField string `config:"target_field"` // Target field for the fingerprint - Encoding encodingMethod `config:"encoding"` // Encoding to use for target field value + Method hashMethod `config:"method"` // Hash function to use for fingerprinting + Fields []string `config:"fields" validate:"required"` // Source fields to compute fingerprint from + TargetField string `config:"target_field"` // Target field for the fingerprint + Encoding encodingMethod `config:"encoding"` // Encoding to use for target field value + IgnoreMissing bool `config:"ignore_missing"` // Ignore missing fields? } func defaultConfig() Config { return Config{ - Method: hashes["sha256"], - TargetField: "fingerprint", - Encoding: encodings["hex"], + Method: hashes["sha256"], + TargetField: "fingerprint", + Encoding: encodings["hex"], + IgnoreMissing: false, } } diff --git a/libbeat/processors/fingerprint/fingerprint.go b/libbeat/processors/fingerprint/fingerprint.go index 91cf9fa10f72..1e4b12e911d7 100644 --- a/libbeat/processors/fingerprint/fingerprint.go +++ b/libbeat/processors/fingerprint/fingerprint.go @@ -70,7 +70,7 @@ func (p *fingerprint) Run(event *beat.Event) (*beat.Event, error) { hashFn := p.hash hashFn.Reset() - err := writeFields(hashFn, p.fields, event.Fields) + err := p.writeFields(hashFn, event.Fields) if err != nil { return nil, makeComputeFingerprintError(err) } @@ -89,14 +89,14 @@ func (p *fingerprint) String() string { return fmt.Sprintf("%v=[method=[%v]]", processorName, p.config.Method) } -func writeFields(to io.Writer, sourceFields []string, eventFields common.MapStr) error { - for _, k := range sourceFields { +func (p *fingerprint) writeFields(to io.Writer, eventFields common.MapStr) error { + for _, k := range p.fields { v, err := eventFields.GetValue(k) - if err == common.ErrKeyNotFound { - return errors.Wrapf(err, "failed to find field [%v] in event", k) - } if err != nil { - return errors.Wrapf(err, "failed when finding field [%v] in event", k) + if p.config.IgnoreMissing { + continue + } + return errors.Wrapf(err, "failed to find field [%v] in event", k) } i := v diff --git a/libbeat/processors/fingerprint/fingerprint_test.go b/libbeat/processors/fingerprint/fingerprint_test.go index efb6ec9c52cd..98d5a56ff6e9 100644 --- a/libbeat/processors/fingerprint/fingerprint_test.go +++ b/libbeat/processors/fingerprint/fingerprint_test.go @@ -18,6 +18,7 @@ package fingerprint import ( + "strconv" "testing" "time" @@ -336,3 +337,49 @@ func TestInvalidConfig(t *testing.T) { }) } } + +func TestIgnoreMissing(t *testing.T) { + testFields := common.MapStr{ + "field1": "foo", + } + + tests := map[string]struct { + assertErr assert.ErrorAssertionFunc + expectedFingerprint string + }{ + "true": { + assert.NoError, + "4cf8b768ad20266c348d63a6d1ff5d6f6f9ed0f59f5c68ae031b78e3e04c5144", + }, + "false": { + assertErr: assert.Error, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + ignoreMissing, _ := strconv.ParseBool(name) + testConfig, err := common.NewConfigFrom(common.MapStr{ + "fields": []string{"field1", "missing_field"}, + "ignore_missing": ignoreMissing, + }) + assert.NoError(t, err) + + p, err := New(testConfig) + assert.NoError(t, err) + + testEvent := &beat.Event{ + Fields: testFields.Clone(), + Timestamp: time.Now(), + } + newEvent, err := p.Run(testEvent) + test.assertErr(t, err) + + if err == nil { + v, err := newEvent.GetValue("fingerprint") + assert.NoError(t, err) + assert.Equal(t, test.expectedFingerprint, v) + } + }) + } +} From ce27088a39f15c07a23fb0c7553fadcc2f4aad28 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 30 Oct 2019 17:27:39 -0700 Subject: [PATCH 22/28] Replace table with definition list --- libbeat/docs/processors-using.asciidoc | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/libbeat/docs/processors-using.asciidoc b/libbeat/docs/processors-using.asciidoc index fe2e49ce247c..d7a202c196ed 100644 --- a/libbeat/docs/processors-using.asciidoc +++ b/libbeat/docs/processors-using.asciidoc @@ -1197,14 +1197,12 @@ processors: fields: ["field1", "field2", ...] ----------------------------------------------------- -.Fingerprint options -[options="header"] -|====== -| `fields` | yes | | List of fields to use as the source for the fingerprint. | -| `target_field` | no | `fingerprint` | Field in which the generated fingerprint should be stored. | -| `method` | no | `sha256` | Algorithm to use for computing the fingerprint. Must be one of: `md5`, `sha1`, `sha256` (default), `sha384`, `sha512`. | -| `encoding` | no | `hex` | Encoding to use on the fingerprint value. Must be one of `hex` (default), `base32`, or `base64`. | -|====== +The following settings are supported: + +`fields`:: List of fields to use as the source for the fingerprint. +`target_field`:: (Optional) Field in which the generated fingerprint should be stored. Default is `fingerprint`. +`method`:: (Optional) Algorithm to use for computing the fingerprint. Must be one of: `md5`, `sha1`, `sha256`, `sha384`, `sha512`. Default is `sha256`. +`encoding`:: (Optional) Encoding to use on the fingerprint value. Must be one of `hex`, `base32`, or `base64`. Default is `hex`. [[include-fields]] === Keep fields from events From 2d17110fa585e40bb74486e38f60495c609fb2d2 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 30 Oct 2019 17:30:30 -0700 Subject: [PATCH 23/28] Adding `ignore_missing` to doc --- libbeat/docs/processors-using.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/libbeat/docs/processors-using.asciidoc b/libbeat/docs/processors-using.asciidoc index d7a202c196ed..97a866f8c15a 100644 --- a/libbeat/docs/processors-using.asciidoc +++ b/libbeat/docs/processors-using.asciidoc @@ -1200,6 +1200,7 @@ processors: The following settings are supported: `fields`:: List of fields to use as the source for the fingerprint. +`ignore_missing`:: (Optional) Whether to ignore missing fields. Default is `false`. `target_field`:: (Optional) Field in which the generated fingerprint should be stored. Default is `fingerprint`. `method`:: (Optional) Algorithm to use for computing the fingerprint. Must be one of: `md5`, `sha1`, `sha256`, `sha384`, `sha512`. Default is `sha256`. `encoding`:: (Optional) Encoding to use on the fingerprint value. Must be one of `hex`, `base32`, or `base64`. Default is `hex`. From ba390ad742f48f72de5ba0719e77c856b70edfc9 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 31 Oct 2019 15:09:42 -0700 Subject: [PATCH 24/28] using io.Fprintf --- libbeat/processors/fingerprint/fingerprint.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/processors/fingerprint/fingerprint.go b/libbeat/processors/fingerprint/fingerprint.go index 1e4b12e911d7..655787e66026 100644 --- a/libbeat/processors/fingerprint/fingerprint.go +++ b/libbeat/processors/fingerprint/fingerprint.go @@ -108,7 +108,7 @@ func (p *fingerprint) writeFields(to io.Writer, eventFields common.MapStr) error i = vv.UTC() } - io.WriteString(to, fmt.Sprintf("|%v|%v", k, i)) + fmt.Fprintf(to, "|%v|%v", k, i) } io.WriteString(to, "|") From de73d0da0f0662fcd0970c9039c12f9df1d5355d Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 31 Oct 2019 15:39:23 -0700 Subject: [PATCH 25/28] Use common.StringSet --- libbeat/processors/fingerprint/fingerprint.go | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/libbeat/processors/fingerprint/fingerprint.go b/libbeat/processors/fingerprint/fingerprint.go index 655787e66026..21188b6e25a7 100644 --- a/libbeat/processors/fingerprint/fingerprint.go +++ b/libbeat/processors/fingerprint/fingerprint.go @@ -21,7 +21,6 @@ import ( "fmt" "hash" "io" - "sort" "time" "github.com/pkg/errors" @@ -54,12 +53,12 @@ func New(cfg *common.Config) (processors.Processor, error) { return nil, errors.Wrapf(err, "failed to unpack %v processor configuration", processorName) } - sort.Strings(config.Fields) + fields := common.MakeStringSet(config.Fields...) p := &fingerprint{ config: config, hash: config.Method(), - fields: unique(config.Fields), + fields: fields.ToSlice(), } return p, nil @@ -118,16 +117,3 @@ func (p *fingerprint) writeFields(to io.Writer, eventFields common.MapStr) error func makeComputeFingerprintError(err error) error { return errors.Wrap(err, "failed to compute fingerprint") } - -func unique(in []string) []string { - seen := map[string]bool{} - var out = make([]string, 0, len(in)) - for _, item := range in { - if _, found := seen[item]; !found { - seen[item] = true - out = append(out, item) - } - } - - return out -} From a6be2ab5120c9dc2d3f075ae9be562dfb258d6f2 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 31 Oct 2019 18:15:58 -0700 Subject: [PATCH 26/28] Adding typed errors --- libbeat/processors/fingerprint/encode.go | 7 +-- libbeat/processors/fingerprint/errors.go | 43 +++++++++++++++++++ libbeat/processors/fingerprint/fingerprint.go | 12 ++---- .../fingerprint/fingerprint_test.go | 15 ++----- libbeat/processors/fingerprint/hash.go | 7 +-- 5 files changed, 52 insertions(+), 32 deletions(-) create mode 100644 libbeat/processors/fingerprint/errors.go diff --git a/libbeat/processors/fingerprint/encode.go b/libbeat/processors/fingerprint/encode.go index 0087e2e5d0f5..843c7bd5d293 100644 --- a/libbeat/processors/fingerprint/encode.go +++ b/libbeat/processors/fingerprint/encode.go @@ -21,7 +21,6 @@ import ( "encoding/base32" "encoding/base64" "encoding/hex" - "fmt" "strings" ) @@ -39,13 +38,9 @@ func (e *encodingMethod) Unpack(str string) error { m, found := encodings[str] if !found { - return makeUnknownEncodingError(str) + return makeErrUnknownEncoding(str) } *e = m return nil } - -func makeUnknownEncodingError(encoding string) error { - return fmt.Errorf("invalid encoding [%s]", encoding) -} diff --git a/libbeat/processors/fingerprint/errors.go b/libbeat/processors/fingerprint/errors.go new file mode 100644 index 000000000000..1a88b5dfa176 --- /dev/null +++ b/libbeat/processors/fingerprint/errors.go @@ -0,0 +1,43 @@ +package fingerprint + +import ( + "errors" + "fmt" +) + +var errNoFields = errors.New("must specify at least one field") + +type ( + errUnknownEncoding struct{ encoding string } + errUnknownMethod struct{ method string } + errConfigUnpack struct{ cause error } + errComputeFingerprint struct{ cause error } +) + +func makeErrUnknownEncoding(encoding string) errUnknownEncoding { + return errUnknownEncoding{encoding} +} +func (e errUnknownEncoding) Error() string { + return fmt.Sprintf("invalid encoding [%s]", e.encoding) +} + +func makeErrUnknownMethod(method string) errUnknownMethod { + return errUnknownMethod{method} +} +func (e errUnknownMethod) Error() string { + return fmt.Sprintf("invalid fingerprinting method [%s]", e.method) +} + +func makeErrConfigUnpack(cause error) errConfigUnpack { + return errConfigUnpack{cause} +} +func (e errConfigUnpack) Error() string { + return fmt.Sprintf("failed to unpack %v processor configuration: %v", processorName, e.cause) +} + +func makeErrComputeFingerprint(cause error) errComputeFingerprint { + return errComputeFingerprint{cause} +} +func (e errComputeFingerprint) Error() string { + return fmt.Sprintf("failed to compute fingerprint: %v", e.cause) +} diff --git a/libbeat/processors/fingerprint/fingerprint.go b/libbeat/processors/fingerprint/fingerprint.go index 21188b6e25a7..ecd6e1151a55 100644 --- a/libbeat/processors/fingerprint/fingerprint.go +++ b/libbeat/processors/fingerprint/fingerprint.go @@ -38,8 +38,6 @@ func init() { const processorName = "fingerprint" -var errNoFields = errors.New("must specify at least one field") - type fingerprint struct { config Config fields []string @@ -50,7 +48,7 @@ type fingerprint struct { func New(cfg *common.Config) (processors.Processor, error) { config := defaultConfig() if err := cfg.Unpack(&config); err != nil { - return nil, errors.Wrapf(err, "failed to unpack %v processor configuration", processorName) + return nil, makeErrConfigUnpack(err) } fields := common.MakeStringSet(config.Fields...) @@ -71,14 +69,14 @@ func (p *fingerprint) Run(event *beat.Event) (*beat.Event, error) { err := p.writeFields(hashFn, event.Fields) if err != nil { - return nil, makeComputeFingerprintError(err) + return nil, makeErrComputeFingerprint(err) } hash := hashFn.Sum(nil) encodedHash := p.config.Encoding(hash) if _, err = event.PutValue(p.config.TargetField, encodedHash); err != nil { - return nil, makeComputeFingerprintError(err) + return nil, makeErrComputeFingerprint(err) } return event, nil @@ -113,7 +111,3 @@ func (p *fingerprint) writeFields(to io.Writer, eventFields common.MapStr) error io.WriteString(to, "|") return nil } - -func makeComputeFingerprintError(err error) error { - return errors.Wrap(err, "failed to compute fingerprint") -} diff --git a/libbeat/processors/fingerprint/fingerprint_test.go b/libbeat/processors/fingerprint/fingerprint_test.go index 98d5a56ff6e9..95e13669a090 100644 --- a/libbeat/processors/fingerprint/fingerprint_test.go +++ b/libbeat/processors/fingerprint/fingerprint_test.go @@ -265,16 +265,13 @@ func TestSourceFieldErrors(t *testing.T) { } tests := map[string]struct { - fields []string - expectedErrMsg string + fields []string }{ "missing": { []string{"field1", "missing_field"}, - "failed to compute fingerprint: failed to find field [missing_field] in event: key not found", }, "non-scalar": { []string{"field1", "complex_field"}, - "failed to compute fingerprint: cannot compute fingerprint using non-scalar field [complex_field]", }, } @@ -294,36 +291,32 @@ func TestSourceFieldErrors(t *testing.T) { Timestamp: time.Now(), } _, err = p.Run(testEvent) - assert.EqualError(t, err, test.expectedErrMsg) + assert.IsType(t, errComputeFingerprint{}, err) }) } } func TestInvalidConfig(t *testing.T) { tests := map[string]struct { - config common.MapStr - expectedErrMsg string + config common.MapStr }{ "no fields": { common.MapStr{ "fields": []string{}, "method": "sha256", }, - "failed to unpack fingerprint processor configuration: empty field accessing 'fields'", }, "invalid fingerprinting method": { common.MapStr{ "fields": []string{"doesnt", "matter"}, "method": "non_existent", }, - "failed to unpack fingerprint processor configuration: invalid fingerprinting method [non_existent] accessing 'method'", }, "invalid encoding": { common.MapStr{ "fields": []string{"doesnt", "matter"}, "encoding": "non_existent", }, - "failed to unpack fingerprint processor configuration: invalid encoding [non_existent] accessing 'encoding'", }, } @@ -333,7 +326,7 @@ func TestInvalidConfig(t *testing.T) { assert.NoError(t, err) _, err = New(testConfig) - assert.EqualError(t, err, test.expectedErrMsg) + assert.IsType(t, errConfigUnpack{}, err) }) } } diff --git a/libbeat/processors/fingerprint/hash.go b/libbeat/processors/fingerprint/hash.go index 6628f5b66644..50ff51894ea5 100644 --- a/libbeat/processors/fingerprint/hash.go +++ b/libbeat/processors/fingerprint/hash.go @@ -22,7 +22,6 @@ import ( "crypto/sha1" "crypto/sha256" "crypto/sha512" - "fmt" "hash" "strings" ) @@ -43,13 +42,9 @@ func (f *hashMethod) Unpack(str string) error { m, found := hashes[str] if !found { - return makeUnknownMethodError(str) + return makeErrUnknownMethod(str) } *f = m return nil } - -func makeUnknownMethodError(method string) error { - return fmt.Errorf("invalid fingerprinting method [%s]", method) -} From c65da9a2bbdfffaec0fb783d2e733f53729fd911 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 31 Oct 2019 18:22:31 -0700 Subject: [PATCH 27/28] Adding more typed errors --- libbeat/processors/fingerprint/errors.go | 19 +++++++++++++++++++ libbeat/processors/fingerprint/fingerprint.go | 6 ++---- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/libbeat/processors/fingerprint/errors.go b/libbeat/processors/fingerprint/errors.go index 1a88b5dfa176..3dc190907658 100644 --- a/libbeat/processors/fingerprint/errors.go +++ b/libbeat/processors/fingerprint/errors.go @@ -12,6 +12,11 @@ type ( errUnknownMethod struct{ method string } errConfigUnpack struct{ cause error } errComputeFingerprint struct{ cause error } + errMissingField struct { + field string + cause error + } + errNonScalarField struct{ field string } ) func makeErrUnknownEncoding(encoding string) errUnknownEncoding { @@ -41,3 +46,17 @@ func makeErrComputeFingerprint(cause error) errComputeFingerprint { func (e errComputeFingerprint) Error() string { return fmt.Sprintf("failed to compute fingerprint: %v", e.cause) } + +func makeErrMissingField(field string, cause error) errMissingField { + return errMissingField{field, cause} +} +func (e errMissingField) Error() string { + return fmt.Sprintf("failed to find field [%v] in event: %v", e.field, e.cause) +} + +func makeErrNonScalarField(field string) errNonScalarField { + return errNonScalarField{field} +} +func (e errNonScalarField) Error() string { + return fmt.Sprintf("cannot compute fingerprint using non-scalar field [%v]", e.field) +} diff --git a/libbeat/processors/fingerprint/fingerprint.go b/libbeat/processors/fingerprint/fingerprint.go index ecd6e1151a55..b0f877dfeb32 100644 --- a/libbeat/processors/fingerprint/fingerprint.go +++ b/libbeat/processors/fingerprint/fingerprint.go @@ -23,8 +23,6 @@ import ( "io" "time" - "github.com/pkg/errors" - "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/processors" @@ -93,13 +91,13 @@ func (p *fingerprint) writeFields(to io.Writer, eventFields common.MapStr) error if p.config.IgnoreMissing { continue } - return errors.Wrapf(err, "failed to find field [%v] in event", k) + return makeErrMissingField(k, err) } i := v switch vv := v.(type) { case map[string]interface{}, []interface{}, common.MapStr: - return errors.Errorf("cannot compute fingerprint using non-scalar field [%v]", k) + return makeErrNonScalarField(k) case time.Time: // Ensure we consistently hash times in UTC. i = vv.UTC() From 5f577f4f0a20059eba9db3ce5ba602511dc0494e Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 31 Oct 2019 18:24:40 -0700 Subject: [PATCH 28/28] Adding license header --- libbeat/processors/fingerprint/errors.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/libbeat/processors/fingerprint/errors.go b/libbeat/processors/fingerprint/errors.go index 3dc190907658..e025015883b6 100644 --- a/libbeat/processors/fingerprint/errors.go +++ b/libbeat/processors/fingerprint/errors.go @@ -1,3 +1,20 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package fingerprint import (