Skip to content

Commit a01ef2f

Browse files
authored
Merge branch 'master' into ilm-createalias-exists
2 parents e45cba9 + cb085d0 commit a01ef2f

File tree

29 files changed

+961
-851
lines changed

29 files changed

+961
-851
lines changed

CHANGELOG.next.asciidoc

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -239,6 +239,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
239239
- Fix out of date FreeBSD vagrantbox. {pull}25652[25652]
240240
- Fix handling of `file_selectors` in aws-s3 input. {pull}25792[25792]
241241
- Fix ILM alias creation when write alias exists and initial index does not exist {pull}26143[26143]
242+
- Include date separator in the filename prefix of `dateRotator` to make sure nothing gets purged accidentally {pull}26176[26176]
242243

243244
*Auditbeat*
244245

@@ -260,6 +261,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
260261
- system/socket: Fixed start failure when run under config reloader. {issue}20851[20851] {pull}21693[21693]
261262
- system/socket: Having some CPUs unavailable to Auditbeat could cause startup errors or event loss. {pull}22827[22827]
262263
- Note incompatibility of system/socket on ARM. {pull}23381[23381]
264+
- auditd: Fix kernel deadlock when netlink congestion causes "no buffer space available" errors. {issue}26031[26031] {pull}26032[26032]
263265

264266
*Filebeat*
265267

@@ -383,6 +385,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
383385
- Fix `fortinet.firewall.addr` when its a string, not an IP address. {issue}25585[25585] {pull}25608[25608]
384386
- Fix incorrect field name appending to `related.hash` in `threatintel.abusechmalware` ingest pipeline. {issue}25151[25151] {pull}25674[25674]
385387
- Fix `kibana.log` pipeline when `event.duration` calculation becomes a Long. {issue}24556[24556] {pull}25675[25675]
388+
- o365: Avoid mapping exception for `Parameters` and `ExtendedProperties` fields of string type. {pull}26164[26164]
386389

387390
*Heartbeat*
388391

@@ -809,6 +812,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
809812
- Add fingerprint processor to generate fixed ids for `google_workspace` events. {pull}25841[25841]
810813
- Update PanOS module to parse HIP Match logs. {issue}24350[24350] {pull}25686[25686]
811814
- Enhance GCP module to populate orchestrator.* fields for GKE / K8S logs {pull}25368[25368]
815+
- http_endpoint: Support multiple documents in a single request by POSTing an array or NDJSON format. {pull}25764[25764]
812816

813817
*Heartbeat*
814818

@@ -941,6 +945,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
941945
- Add additional network metrics to docker/network {pull}25354[25354]
942946
- Migrate ec2 metricsets to use cloudwatch input. {pull}25924[25924]
943947
- Reduce number of requests done by kubernetes metricsets to kubelet. {pull}25782[25782]
948+
- Migrate rds metricsets to use cloudwatch input. {pull}26077[26077]
944949
- Migrate sqs metricsets to use cloudwatch input. {pull}26117[26117]
945950

946951
*Packetbeat*

auditbeat/module/auditd/audit_linux.go

Lines changed: 45 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,8 @@ const (
5151

5252
lostEventsUpdateInterval = time.Second * 15
5353
maxDefaultStreamBufferConsumers = 4
54+
55+
setPIDMaxRetries = 5
5456
)
5557

5658
type backpressureStrategy uint8
@@ -137,10 +139,32 @@ func newAuditClient(c *Config, log *logp.Logger) (*libaudit.AuditClient, error)
137139
return libaudit.NewAuditClient(nil)
138140
}
139141

142+
func closeAuditClient(client *libaudit.AuditClient) error {
143+
discard := func(bytes []byte) ([]syscall.NetlinkMessage, error) {
144+
return nil, nil
145+
}
146+
// Drain the netlink channel in parallel to Close() to prevent a deadlock.
147+
// This goroutine will terminate once receive from netlink errors (EBADF,
148+
// EBADFD, or any other error). This happens because the fd is closed.
149+
go func() {
150+
for {
151+
_, err := client.Netlink.Receive(true, discard)
152+
switch err {
153+
case nil, syscall.EINTR:
154+
case syscall.EAGAIN:
155+
time.Sleep(50 * time.Millisecond)
156+
default:
157+
return
158+
}
159+
}
160+
}()
161+
return client.Close()
162+
}
163+
140164
// Run initializes the audit client and receives audit messages from the
141165
// kernel until the reporter's done channel is closed.
142166
func (ms *MetricSet) Run(reporter mb.PushReporterV2) {
143-
defer ms.client.Close()
167+
defer closeAuditClient(ms.client)
144168

145169
if err := ms.addRules(reporter); err != nil {
146170
reporter.Error(err)
@@ -164,7 +188,7 @@ func (ms *MetricSet) Run(reporter mb.PushReporterV2) {
164188
go func() {
165189
defer func() { // Close the most recently allocated "client" instance.
166190
if client != nil {
167-
client.Close()
191+
closeAuditClient(client)
168192
}
169193
}()
170194
timer := time.NewTicker(lostEventsUpdateInterval)
@@ -178,7 +202,7 @@ func (ms *MetricSet) Run(reporter mb.PushReporterV2) {
178202
ms.updateKernelLostMetric(status.Lost)
179203
} else {
180204
ms.log.Error("get status request failed:", err)
181-
if err = client.Close(); err != nil {
205+
if err = closeAuditClient(client); err != nil {
182206
ms.log.Errorw("Error closing audit monitoring client", "error", err)
183207
}
184208
client, err = libaudit.NewAuditClient(nil)
@@ -233,7 +257,7 @@ func (ms *MetricSet) addRules(reporter mb.PushReporterV2) error {
233257
if err != nil {
234258
return errors.Wrap(err, "failed to create audit client for adding rules")
235259
}
236-
defer client.Close()
260+
defer closeAuditClient(client)
237261

238262
// Don't attempt to change configuration if audit rules are locked (enabled == 2).
239263
// Will result in EPERM.
@@ -350,10 +374,12 @@ func (ms *MetricSet) initClient() error {
350374
return errors.Wrap(err, "failed to enable auditing in the kernel")
351375
}
352376
}
377+
353378
if err := ms.client.WaitForPendingACKs(); err != nil {
354379
return errors.Wrap(err, "failed to wait for ACKs")
355380
}
356-
if err := ms.client.SetPID(libaudit.WaitForReply); err != nil {
381+
382+
if err := ms.setPID(setPIDMaxRetries); err != nil {
357383
if errno, ok := err.(syscall.Errno); ok && errno == syscall.EEXIST && status.PID != 0 {
358384
return fmt.Errorf("failed to set audit PID. An audit process is already running (PID %d)", status.PID)
359385
}
@@ -362,6 +388,20 @@ func (ms *MetricSet) initClient() error {
362388
return nil
363389
}
364390

391+
func (ms *MetricSet) setPID(retries int) (err error) {
392+
if err = ms.client.SetPID(libaudit.WaitForReply); err == nil || errors.Cause(err) != syscall.ENOBUFS || retries == 0 {
393+
return err
394+
}
395+
// At this point the netlink channel is congested (ENOBUFS).
396+
// Drain and close the client, then retry with a new client.
397+
closeAuditClient(ms.client)
398+
if ms.client, err = newAuditClient(&ms.config, ms.log); err != nil {
399+
return errors.Wrapf(err, "failed to recover from ENOBUFS")
400+
}
401+
ms.log.Info("Recovering from ENOBUFS ...")
402+
return ms.setPID(retries - 1)
403+
}
404+
365405
func (ms *MetricSet) updateKernelLostMetric(lost uint32) {
366406
if !ms.kernelLost.enabled {
367407
return

heartbeat/_meta/fields.common.yml

Lines changed: 52 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,52 @@
174174
type: text
175175
- name: stack
176176
type: text
177+
- name: browser
178+
type: group
179+
fields:
180+
- name: experience
181+
type: group
182+
fields:
183+
- name: name
184+
type: keyword
185+
- name: type
186+
type: text
187+
description: >
188+
denotes the 'mark' event
189+
- name: start
190+
type: long
191+
description: >
192+
offset of time relative to journey start in milliseconds
193+
- name: user_timing
194+
type: group
195+
fields:
196+
- name: name
197+
type: keyword
198+
- name: type
199+
type: text
200+
description: >
201+
could be one of mark or measure event types.
202+
- name: start
203+
type: long
204+
description: >
205+
offset of time relative to journey start in milliseconds
206+
- name: end
207+
type: long
208+
description: >
209+
offset of time relative to journey start in milliseconds
210+
- name: layout_shift
211+
type: group
212+
fields:
213+
- name: name
214+
type: keyword
215+
- name: score
216+
type: integer
217+
- name: exists
218+
type: boolean
219+
description: >
220+
flag that indicates if there was any layout shift events
221+
present on the page.
222+
177223
- key: http
178224
title: "HTTP monitor"
179225
description:
@@ -379,12 +425,12 @@
379425
type: group
380426
description: Detailed x509 certificate metadata
381427
fields:
382-
- name: version_number
383-
type: keyword
384-
ignore_above: 1024
385-
description: Version of x509 format.
386-
example: 3
387-
default_field: false
428+
- name: version_number
429+
type: keyword
430+
ignore_above: 1024
431+
description: Version of x509 format.
432+
example: 3
433+
default_field: false
388434

389435
- key: icmp
390436
title: "ICMP"

heartbeat/docs/fields.asciidoc

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10499,6 +10499,98 @@ type: text
1049910499
1050010500
--
1050110501
10502+
10503+
10504+
*`synthetics.browser.experience.name`*::
10505+
+
10506+
--
10507+
type: keyword
10508+
10509+
--
10510+
10511+
*`synthetics.browser.experience.type`*::
10512+
+
10513+
--
10514+
denotes the 'mark' event
10515+
10516+
10517+
type: text
10518+
10519+
--
10520+
10521+
*`synthetics.browser.experience.start`*::
10522+
+
10523+
--
10524+
offset of time relative to journey start in milliseconds
10525+
10526+
10527+
type: long
10528+
10529+
--
10530+
10531+
10532+
*`synthetics.browser.user_timing.name`*::
10533+
+
10534+
--
10535+
type: keyword
10536+
10537+
--
10538+
10539+
*`synthetics.browser.user_timing.type`*::
10540+
+
10541+
--
10542+
could be one of mark or measure event types.
10543+
10544+
10545+
type: text
10546+
10547+
--
10548+
10549+
*`synthetics.browser.user_timing.start`*::
10550+
+
10551+
--
10552+
offset of time relative to journey start in milliseconds
10553+
10554+
10555+
type: long
10556+
10557+
--
10558+
10559+
*`synthetics.browser.user_timing.end`*::
10560+
+
10561+
--
10562+
offset of time relative to journey start in milliseconds
10563+
10564+
10565+
type: long
10566+
10567+
--
10568+
10569+
10570+
*`synthetics.browser.layout_shift.name`*::
10571+
+
10572+
--
10573+
type: keyword
10574+
10575+
--
10576+
10577+
*`synthetics.browser.layout_shift.score`*::
10578+
+
10579+
--
10580+
type: integer
10581+
10582+
--
10583+
10584+
*`synthetics.browser.layout_shift.exists`*::
10585+
+
10586+
--
10587+
flag that indicates if there was any layout shift events present on the page.
10588+
10589+
10590+
type: boolean
10591+
10592+
--
10593+
1050210594
[[exported-fields-tcp]]
1050310595
== TCP layer fields
1050410596

heartbeat/include/fields.go

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

libbeat/common/file/rotator.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -435,11 +435,11 @@ func newRotater(log Logger, s SuffixType, filename string, maxBackups uint, inte
435435
func newDateRotater(log Logger, filename string) rotater {
436436
d := &dateRotator{
437437
log: log,
438-
filenamePrefix: filename,
438+
filenamePrefix: filename + "-",
439439
format: "20060102150405",
440440
}
441441

442-
d.currentFilename = d.filenamePrefix + "-" + time.Now().Format(d.format)
442+
d.currentFilename = d.filenamePrefix + time.Now().Format(d.format)
443443
files, err := filepath.Glob(d.filenamePrefix + "*")
444444
if err != nil {
445445
return d
@@ -467,7 +467,7 @@ func (d *dateRotator) Rotate(reason rotateReason, rotateTime time.Time) error {
467467
d.log.Debugw("Rotating file", "filename", d.currentFilename, "reason", reason)
468468
}
469469

470-
d.currentFilename = d.filenamePrefix + "-" + rotateTime.Format(d.format)
470+
d.currentFilename = d.filenamePrefix + rotateTime.Format(d.format)
471471
return nil
472472
}
473473

@@ -493,7 +493,7 @@ func (d *dateRotator) SortModTimeLogs(strings []string) {
493493
}
494494

495495
func (d *dateRotator) OrderLog(filename string) time.Time {
496-
ts, err := time.Parse(d.format, filepath.Base(filename))
496+
ts, err := time.Parse(d.filenamePrefix+d.format, filepath.Base(filename))
497497
if err != nil {
498498
return time.Time{}
499499
}

metricbeat/docs/modules/aws/rds.asciidoc

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ This file is generated! See scripts/mage/docs_collector.go
88

99
include::../../../../x-pack/metricbeat/module/aws/rds/_meta/docs.asciidoc[]
1010

11+
This is a default metricset. If the host module is unconfigured, this metricset is enabled by default.
1112

1213
==== Fields
1314

x-pack/elastic-agent/CHANGELOG.next.asciidoc

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@
6868
- Handle case where policy doesn't contain Fleet connection information {pull}25707[25707]
6969
- Fix fleet-server.yml spec to not overwrite existing keys {pull}25741[25741]
7070
- Agent sends wrong log level to Endpoint {issue}25583[25583]
71+
- Change timestamp in elatic-agent-json.log to use UTC {issue}25391[25391]
7172

7273
==== New features
7374

0 commit comments

Comments
 (0)