Skip to content

Commit 1795efd

Browse files
authored
[7.8] Followup to 12606 (#18316) (#18521)
* Followup to 12606 (#18316) * Adding developer CHANGELOG entry * Refactoring: extracting helper method * Adding unit tests * Consolidate event metadata field constants * Use events.GetMetaStringValue * Implement op_type values as enum * Add doc strings * Deference event pointer * Renaming op type consts and breaking them out into own block * Renaming type * Using stringer * Using go idiom instead of if-else * Adding default op type * Empty string for default * Store op type enum, not string, in event metadata * Using events.GetMetaStringValue * Updating dev CHANGELOG entry * Allow for op_type metadata field to be set as either string or enum * No need for .String() * Handle missing key case gracefully * Update unit tests * Update developer CHANGELOG entry * Fixing up CHANGELOG
1 parent 7933e01 commit 1795efd

File tree

17 files changed

+311
-73
lines changed

17 files changed

+311
-73
lines changed

CHANGELOG-developer.next.asciidoc

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,3 +79,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
7979
- Add support for MODULE environment variable in `mage goIntegTest` in metricbeat to run integration tests for a single module. {pull}17147[17147]
8080
- Add support for a `TEST_TAGS` environment variable to add tags for tests selection following go build tags semantics, this environment variable is used by mage test targets to add build tags. Python tests can also be tagged with a decorator (`@beat.tag('sometag')`). {pull}16937[16937] {pull}17075[17075]
8181
- Add fields validation for histogram subfields. {pull}17759[17759]
82+
- Events intended for the Elasticsearch output can now take an `op_type` metadata field of type events.OpType or string to indicate the `op_type` to use for bulk indexing. {pull}12606[12606]

filebeat/channel/connector_test.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
"github.com/stretchr/testify/assert"
2626

2727
"github.com/elastic/beats/v7/libbeat/beat"
28+
"github.com/elastic/beats/v7/libbeat/beat/events"
2829
"github.com/elastic/beats/v7/libbeat/common"
2930
"github.com/elastic/beats/v7/libbeat/processors"
3031
"github.com/elastic/beats/v7/libbeat/processors/actions"
@@ -183,7 +184,7 @@ func (p *setRawIndex) Run(event *beat.Event) (*beat.Event, error) {
183184
if event.Meta == nil {
184185
event.Meta = common.MapStr{}
185186
}
186-
event.Meta["raw_index"] = p.indexStr
187+
event.Meta[events.FieldMetaRawIndex] = p.indexStr
187188
return event, nil
188189
}
189190

journalbeat/input/input_test.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
"github.com/stretchr/testify/assert"
2626

2727
"github.com/elastic/beats/v7/libbeat/beat"
28+
"github.com/elastic/beats/v7/libbeat/beat/events"
2829
"github.com/elastic/beats/v7/libbeat/common"
2930
"github.com/elastic/beats/v7/libbeat/processors"
3031
_ "github.com/elastic/beats/v7/libbeat/processors/actions"
@@ -138,7 +139,7 @@ func (p *setRawIndex) Run(event *beat.Event) (*beat.Event, error) {
138139
if event.Meta == nil {
139140
event.Meta = common.MapStr{}
140141
}
141-
event.Meta["raw_index"] = p.indexStr
142+
event.Meta[events.FieldMetaRawIndex] = p.indexStr
142143
return event, nil
143144
}
144145

libbeat/beat/event.go

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -54,19 +54,6 @@ func (e *Event) SetID(id string) {
5454
e.Meta["_id"] = id
5555
}
5656

57-
func (e *Event) GetMetaStringValue(key string) (string, error) {
58-
tmp, err := e.Meta.GetValue(key)
59-
if err != nil {
60-
return "", err
61-
}
62-
63-
if s, ok := tmp.(string); ok {
64-
return s, nil
65-
}
66-
67-
return "", nil
68-
}
69-
7057
func (e *Event) GetValue(key string) (interface{}, error) {
7158
if key == "@timestamp" {
7259
return e.Timestamp, nil

libbeat/beat/events/optype.go

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
// Licensed to Elasticsearch B.V. under one or more contributor
2+
// license agreements. See the NOTICE file distributed with
3+
// this work for additional information regarding copyright
4+
// ownership. Elasticsearch B.V. licenses this file to you under
5+
// the Apache License, Version 2.0 (the "License"); you may
6+
// not use this file except in compliance with the License.
7+
// You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package events
19+
20+
type OpType int
21+
22+
//go:generate stringer -linecomment -type OpType
23+
const (
24+
OpTypeDefault OpType = iota //
25+
OpTypeCreate //create
26+
OpTypeIndex // index
27+
OpTypeDelete // delete
28+
)

libbeat/beat/events/optype_string.go

Lines changed: 43 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

libbeat/beat/events/util.go

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
// Licensed to Elasticsearch B.V. under one or more contributor
2+
// license agreements. See the NOTICE file distributed with
3+
// this work for additional information regarding copyright
4+
// ownership. Elasticsearch B.V. licenses this file to you under
5+
// the Apache License, Version 2.0 (the "License"); you may
6+
// not use this file except in compliance with the License.
7+
// You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package events
19+
20+
import "github.com/elastic/beats/v7/libbeat/beat"
21+
22+
const (
23+
// FieldMetaID defines the ID for the event. Also see FieldMetaOpType.
24+
FieldMetaID = "_id"
25+
26+
// FieldMetaAlias defines the index alias to use for the event. If set, it takes
27+
// precedence over values defined using FieldMetaIndex or FieldMetaRawIndex.
28+
FieldMetaAlias = "alias"
29+
30+
// FieldMetaIndex defines the base index name to use for the event. The value is suffixed
31+
// with a Y-m-d value based on the event's timestamp. If set, it takes precedence over the
32+
// value defined using FieldMetaRawIndex.
33+
FieldMetaIndex = "index"
34+
35+
// FieldMetaRawIndex defines the raw index name to use for the event. It is used as-is, without
36+
// any additional manipulation.
37+
FieldMetaRawIndex = "raw_index"
38+
39+
// FieldMetaPipeline defines the ingest node pipeline to use for this event.
40+
FieldMetaPipeline = "pipeline"
41+
42+
// FieldMetaOpType defines the metadata key name for event operation type to use with the Elasticsearch
43+
// Bulk API encoding of the event. The key's value can be an empty string, `create`, `index`, or `delete`.
44+
// If empty, `create` will be used if FieldMetaID is set; otherwise `index` will be used.
45+
FieldMetaOpType = "op_type"
46+
)
47+
48+
// GetMetaStringValue returns the value of the given event metadata string field
49+
func GetMetaStringValue(e beat.Event, key string) (string, error) {
50+
tmp, err := e.Meta.GetValue(key)
51+
if err != nil {
52+
return "", err
53+
}
54+
55+
if s, ok := tmp.(string); ok {
56+
return s, nil
57+
}
58+
59+
return "", nil
60+
}
61+
62+
// GetOpType returns the event's op_type, if set
63+
func GetOpType(e beat.Event) OpType {
64+
tmp, err := e.Meta.GetValue(FieldMetaOpType)
65+
if err != nil {
66+
return OpTypeDefault
67+
}
68+
69+
switch v := tmp.(type) {
70+
case OpType:
71+
return v
72+
case string:
73+
switch v {
74+
case "create":
75+
return OpTypeCreate
76+
case "index":
77+
return OpTypeIndex
78+
case "delete":
79+
return OpTypeDelete
80+
}
81+
}
82+
83+
return OpTypeDefault
84+
}

libbeat/beat/events/util_test.go

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
// Licensed to Elasticsearch B.V. under one or more contributor
2+
// license agreements. See the NOTICE file distributed with
3+
// this work for additional information regarding copyright
4+
// ownership. Elasticsearch B.V. licenses this file to you under
5+
// the Apache License, Version 2.0 (the "License"); you may
6+
// not use this file except in compliance with the License.
7+
// You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package events
19+
20+
import (
21+
"testing"
22+
23+
"github.com/stretchr/testify/require"
24+
25+
"github.com/elastic/beats/v7/libbeat/beat"
26+
"github.com/elastic/beats/v7/libbeat/common"
27+
)
28+
29+
func TestGetMetaStringValue(t *testing.T) {
30+
tests := map[string]struct {
31+
event beat.Event
32+
metaFieldPath string
33+
expectedValue string
34+
expectedErr error
35+
}{
36+
"nonexistent_field": {
37+
beat.Event{
38+
Meta: common.MapStr{
39+
"foo": "bar",
40+
},
41+
},
42+
"nonexistent",
43+
"",
44+
common.ErrKeyNotFound,
45+
},
46+
"root": {
47+
beat.Event{
48+
Meta: common.MapStr{
49+
"foo": "bar",
50+
"baz": "hello",
51+
},
52+
},
53+
"baz",
54+
"hello",
55+
nil,
56+
},
57+
"nested": {
58+
beat.Event{
59+
Meta: common.MapStr{
60+
"foo": "bar",
61+
"baz": common.MapStr{
62+
"qux": "hello",
63+
},
64+
},
65+
},
66+
"baz.qux",
67+
"hello",
68+
nil,
69+
},
70+
"non_string": {
71+
beat.Event{
72+
Meta: common.MapStr{
73+
"foo": "bar",
74+
"baz": 17,
75+
},
76+
},
77+
"baz",
78+
"",
79+
nil,
80+
},
81+
}
82+
83+
for name, test := range tests {
84+
t.Run(name, func(t *testing.T) {
85+
value, err := GetMetaStringValue(test.event, test.metaFieldPath)
86+
require.Equal(t, test.expectedValue, value)
87+
require.Equal(t, test.expectedErr, err)
88+
})
89+
}
90+
}

libbeat/idxmgmt/std.go

Lines changed: 9 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
"fmt"
2323

2424
"github.com/elastic/beats/v7/libbeat/beat"
25+
"github.com/elastic/beats/v7/libbeat/beat/events"
2526
"github.com/elastic/beats/v7/libbeat/common"
2627
"github.com/elastic/beats/v7/libbeat/common/atomic"
2728
"github.com/elastic/beats/v7/libbeat/idxmgmt/ilm"
@@ -352,28 +353,22 @@ func getEventCustomIndex(evt *beat.Event, beatInfo beat.Info) string {
352353
return ""
353354
}
354355

355-
if tmp := evt.Meta["alias"]; tmp != nil {
356-
if alias, ok := tmp.(string); ok {
357-
return alias
358-
}
356+
if alias, err := events.GetMetaStringValue(*evt, events.FieldMetaAlias); err == nil {
357+
return alias
359358
}
360359

361-
if tmp := evt.Meta["index"]; tmp != nil {
362-
if idx, ok := tmp.(string); ok {
363-
ts := evt.Timestamp.UTC()
364-
return fmt.Sprintf("%s-%d.%02d.%02d",
365-
idx, ts.Year(), ts.Month(), ts.Day())
366-
}
360+
if idx, err := events.GetMetaStringValue(*evt, events.FieldMetaIndex); err == nil {
361+
ts := evt.Timestamp.UTC()
362+
return fmt.Sprintf("%s-%d.%02d.%02d",
363+
idx, ts.Year(), ts.Month(), ts.Day())
367364
}
368365

369366
// This is functionally identical to Meta["alias"], returning the overriding
370367
// metadata as the index name if present. It is currently used by Filebeat
371368
// to send the index for particular inputs to formatted string templates,
372369
// which are then expanded by a processor to the "raw_index" field.
373-
if tmp := evt.Meta["raw_index"]; tmp != nil {
374-
if idx, ok := tmp.(string); ok {
375-
return idx
376-
}
370+
if idx, err := events.GetMetaStringValue(*evt, events.FieldMetaRawIndex); err == nil {
371+
return idx
377372
}
378373

379374
return ""

libbeat/monitoring/report/elasticsearch/client.go

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import (
2828

2929
"github.com/pkg/errors"
3030

31+
"github.com/elastic/beats/v7/libbeat/beat/events"
3132
"github.com/elastic/beats/v7/libbeat/common"
3233
"github.com/elastic/beats/v7/libbeat/esleg/eslegclient"
3334
"github.com/elastic/beats/v7/libbeat/logp"
@@ -200,14 +201,14 @@ func (c *publishClient) publishBulk(ctx context.Context, event publisher.Event,
200201
meta["_type"] = "doc"
201202
}
202203

203-
action := common.MapStr{}
204-
var opType string
204+
opType := events.OpTypeCreate
205205
if esVersion.LessThan(createDocPrivAvailableESVersion) {
206-
opType = "index"
207-
} else {
208-
opType = "create"
206+
opType = events.OpTypeIndex
207+
}
208+
209+
action := common.MapStr{
210+
opType.String(): meta,
209211
}
210-
action[opType] = meta
211212

212213
event.Content.Fields.Put("timestamp", event.Content.Timestamp)
213214

0 commit comments

Comments
 (0)