Skip to content

Commit 7bb31e6

Browse files
kaiyan-shengkaiyan-sheng
authored
Add replace_fields config option in add_host_metadata for replacing host fields (#20490)
* add replace_host_fields config param
1 parent 6244316 commit 7bb31e6

File tree

5 files changed

+250
-1
lines changed

5 files changed

+250
-1
lines changed

‎CHANGELOG.next.asciidoc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -386,7 +386,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
386386
- Set index.max_docvalue_fields_search in index template to increase value to 200 fields. {issue}20215[20215]
387387
- Add leader election for Kubernetes autodiscover. {pull}20281[20281]
388388
- Add capability of enriching process metadata with contianer id also for non-privileged containers in `add_process_metadata` processor. {pull}19767[19767]
389-
389+
- Add replace_fields config option in add_host_metadata for replacing host fields. {pull}20490[20490] {issue}20464[20464]
390390

391391
*Auditbeat*
392392

‎libbeat/processors/add_host_metadata/add_host_metadata.go‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,11 @@ func New(cfg *common.Config) (processors.Processor, error) {
8181

8282
// Run enriches the given event with the host meta data
8383
func (p *addHostMetadata) Run(event *beat.Event) (*beat.Event, error) {
84+
// check replace_host_fields field
85+
if !p.config.ReplaceFields && skipAddingHostMetadata(event) {
86+
return event, nil
87+
}
88+
8489
err := p.loadData()
8590
if err != nil {
8691
return nil, err
@@ -146,3 +151,22 @@ func (p *addHostMetadata) String() string {
146151
return fmt.Sprintf("%v=[netinfo.enabled=[%v], cache.ttl=[%v]]",
147152
processorName, p.config.NetInfoEnabled, p.config.CacheTTL)
148153
}
154+
155+
func skipAddingHostMetadata(event *beat.Event) bool {
156+
// If host fields exist(besides host.name added by libbeat) in event, skip add_host_metadata.
157+
hostFields, err := event.Fields.GetValue("host")
158+
159+
// Don't skip if there are no fields
160+
if err != nil || hostFields == nil {
161+
return false
162+
}
163+
164+
hostFieldsMap := hostFields.(common.MapStr)
165+
// or if "name" is the only field, don't skip
166+
hasName, _ := hostFieldsMap.HasKey("name")
167+
if hasName && len(hostFieldsMap) == 1 {
168+
return false
169+
}
170+
171+
return true
172+
}

‎libbeat/processors/add_host_metadata/add_host_metadata_test.go‎

Lines changed: 215 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,11 @@ import (
3131
"github.com/elastic/go-sysinfo/types"
3232
)
3333

34+
var (
35+
hostName = "testHost"
36+
hostID = "9C7FAB7B"
37+
)
38+
3439
func TestConfigDefault(t *testing.T) {
3540
event := &beat.Event{
3641
Fields: common.MapStr{},
@@ -196,3 +201,213 @@ func TestConfigGeoDisabled(t *testing.T) {
196201
assert.Error(t, err)
197202
assert.Equal(t, nil, eventGeoField)
198203
}
204+
205+
func TestEventWithReplaceFieldsFalse(t *testing.T) {
206+
cfg := map[string]interface{}{}
207+
cfg["replace_fields"] = false
208+
testConfig, err := common.NewConfigFrom(cfg)
209+
assert.NoError(t, err)
210+
211+
p, err := New(testConfig)
212+
switch runtime.GOOS {
213+
case "windows", "darwin", "linux":
214+
assert.NoError(t, err)
215+
default:
216+
assert.IsType(t, types.ErrNotImplemented, err)
217+
return
218+
}
219+
220+
cases := []struct {
221+
title string
222+
event beat.Event
223+
hostLengthLargerThanOne bool
224+
hostLengthEqualsToOne bool
225+
expectedHostFieldLength int
226+
}{
227+
{
228+
"replace_fields=false with only host.name",
229+
beat.Event{
230+
Fields: common.MapStr{
231+
"host": common.MapStr{
232+
"name": hostName,
233+
},
234+
},
235+
},
236+
true,
237+
false,
238+
-1,
239+
},
240+
{
241+
"replace_fields=false with only host.id",
242+
beat.Event{
243+
Fields: common.MapStr{
244+
"host": common.MapStr{
245+
"id": hostID,
246+
},
247+
},
248+
},
249+
false,
250+
true,
251+
1,
252+
},
253+
{
254+
"replace_fields=false with host.name and host.id",
255+
beat.Event{
256+
Fields: common.MapStr{
257+
"host": common.MapStr{
258+
"name": hostName,
259+
"id": hostID,
260+
},
261+
},
262+
},
263+
true,
264+
false,
265+
2,
266+
},
267+
}
268+
269+
for _, c := range cases {
270+
t.Run(c.title, func(t *testing.T) {
271+
newEvent, err := p.Run(&c.event)
272+
assert.NoError(t, err)
273+
274+
v, err := newEvent.GetValue("host")
275+
assert.NoError(t, err)
276+
assert.Equal(t, c.hostLengthLargerThanOne, len(v.(common.MapStr)) > 1)
277+
assert.Equal(t, c.hostLengthEqualsToOne, len(v.(common.MapStr)) == 1)
278+
if c.expectedHostFieldLength != -1 {
279+
assert.Equal(t, c.expectedHostFieldLength, len(v.(common.MapStr)))
280+
}
281+
})
282+
}
283+
}
284+
285+
func TestEventWithReplaceFieldsTrue(t *testing.T) {
286+
cfg := map[string]interface{}{}
287+
cfg["replace_fields"] = true
288+
testConfig, err := common.NewConfigFrom(cfg)
289+
assert.NoError(t, err)
290+
291+
p, err := New(testConfig)
292+
switch runtime.GOOS {
293+
case "windows", "darwin", "linux":
294+
assert.NoError(t, err)
295+
default:
296+
assert.IsType(t, types.ErrNotImplemented, err)
297+
return
298+
}
299+
300+
cases := []struct {
301+
title string
302+
event beat.Event
303+
hostLengthLargerThanOne bool
304+
hostLengthEqualsToOne bool
305+
}{
306+
{
307+
"replace_fields=true with host.name",
308+
beat.Event{
309+
Fields: common.MapStr{
310+
"host": common.MapStr{
311+
"name": hostName,
312+
},
313+
},
314+
},
315+
true,
316+
false,
317+
},
318+
{
319+
"replace_fields=true with host.id",
320+
beat.Event{
321+
Fields: common.MapStr{
322+
"host": common.MapStr{
323+
"id": hostID,
324+
},
325+
},
326+
},
327+
true,
328+
false,
329+
},
330+
{
331+
"replace_fields=true with host.name and host.id",
332+
beat.Event{
333+
Fields: common.MapStr{
334+
"host": common.MapStr{
335+
"name": hostName,
336+
"id": hostID,
337+
},
338+
},
339+
},
340+
true,
341+
false,
342+
},
343+
}
344+
345+
for _, c := range cases {
346+
t.Run(c.title, func(t *testing.T) {
347+
newEvent, err := p.Run(&c.event)
348+
assert.NoError(t, err)
349+
350+
v, err := newEvent.GetValue("host")
351+
assert.NoError(t, err)
352+
assert.Equal(t, c.hostLengthLargerThanOne, len(v.(common.MapStr)) > 1)
353+
assert.Equal(t, c.hostLengthEqualsToOne, len(v.(common.MapStr)) == 1)
354+
})
355+
}
356+
}
357+
358+
func TestSkipAddingHostMetadata(t *testing.T) {
359+
cases := []struct {
360+
title string
361+
event beat.Event
362+
expectedSkip bool
363+
}{
364+
{
365+
"event only with host.name",
366+
beat.Event{
367+
Fields: common.MapStr{
368+
"host": common.MapStr{
369+
"name": hostName,
370+
},
371+
},
372+
},
373+
false,
374+
},
375+
{
376+
"event only with host.id",
377+
beat.Event{
378+
Fields: common.MapStr{
379+
"host": common.MapStr{
380+
"id": hostID,
381+
},
382+
},
383+
},
384+
true,
385+
},
386+
{
387+
"event with host.name and host.id",
388+
beat.Event{
389+
Fields: common.MapStr{
390+
"host": common.MapStr{
391+
"name": hostName,
392+
"id": hostID,
393+
},
394+
},
395+
},
396+
true,
397+
},
398+
{
399+
"event without host field",
400+
beat.Event{
401+
Fields: common.MapStr{},
402+
},
403+
false,
404+
},
405+
}
406+
407+
for _, c := range cases {
408+
t.Run(c.title, func(t *testing.T) {
409+
skip := skipAddingHostMetadata(&c.event)
410+
assert.Equal(t, c.expectedSkip, skip)
411+
})
412+
}
413+
}

‎libbeat/processors/add_host_metadata/config.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,13 @@ type Config struct {
2929
CacheTTL time.Duration `config:"cache.ttl"`
3030
Geo *util.GeoConfig `config:"geo"`
3131
Name string `config:"name"`
32+
ReplaceFields bool `config:"replace_fields"` // replace existing host fields with add_host_metadata
3233
}
3334

3435
func defaultConfig() Config {
3536
return Config{
3637
NetInfoEnabled: true,
3738
CacheTTL: 5 * time.Minute,
39+
ReplaceFields: true,
3840
}
3941
}

‎libbeat/processors/add_host_metadata/docs/add_host_metadata.asciidoc‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,8 @@ It has the following settings:
4242

4343
`geo.region_iso_code`:: (Optional) ISO region code.
4444

45+
`replace_fields`:: (Optional) Default true. If set to false, original host
46+
fields from the event will not be replaced by host fields from `add_host_metadata`.
4547

4648
The `add_host_metadata` processor annotates each event with relevant metadata from the host machine.
4749
The fields added to the event look like the following:
@@ -75,3 +77,9 @@ The fields added to the event look like the following:
7577
}
7678
}
7779
-------------------------------------------------------------------------------
80+
81+
Note: `add_host_metadata` processor will overwrite host fields if `host.*`
82+
fields already exist in the event from Beats by default with `replace_fields`
83+
equals to `true`.
84+
Please use `add_observer_metadata` if the beat is being used to monitor external
85+
systems.

0 commit comments

Comments
 (0)