From 82579fa0ffcf24e5e8d43e98ad6a72785094d10f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Fri, 29 Mar 2019 10:50:18 +0100 Subject: [PATCH 1/5] copy bytes instead of slicing so the underlying big array can be freed --- libbeat/reader/readfile/limit.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/libbeat/reader/readfile/limit.go b/libbeat/reader/readfile/limit.go index aff5fac67345..6cf72ad1c68b 100644 --- a/libbeat/reader/readfile/limit.go +++ b/libbeat/reader/readfile/limit.go @@ -18,6 +18,8 @@ package readfile import ( + "fmt" + "github.com/elastic/beats/libbeat/reader" ) @@ -37,7 +39,10 @@ func NewLimitReader(r reader.Reader, maxBytes int) *LimitReader { func (r *LimitReader) Next() (reader.Message, error) { message, err := r.reader.Next() if len(message.Content) > r.maxBytes { - message.Content = message.Content[:r.maxBytes] + n = copy(message.Content, message.Content[:r.maxBytes]) + if n != r.MaxBytes { + return event, fmt.Errorf("unexpected number of bytes were copied, %d instead of limit %d", n, r.MaxBytes) + } message.AddFlagsWithKey("log.flags", "truncated") } return message, err From a1d9fa5ab958f9bce852ed4b9d871005ce06aec7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Fri, 5 Apr 2019 16:48:49 +0200 Subject: [PATCH 2/5] make it work --- libbeat/reader/readfile/limit.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libbeat/reader/readfile/limit.go b/libbeat/reader/readfile/limit.go index 6cf72ad1c68b..d068be05aa39 100644 --- a/libbeat/reader/readfile/limit.go +++ b/libbeat/reader/readfile/limit.go @@ -39,10 +39,12 @@ func NewLimitReader(r reader.Reader, maxBytes int) *LimitReader { func (r *LimitReader) Next() (reader.Message, error) { message, err := r.reader.Next() if len(message.Content) > r.maxBytes { - n = copy(message.Content, message.Content[:r.maxBytes]) + tmp := make([]byte, r.maxBytes) + n = copy(tmp, message.Content) if n != r.MaxBytes { return event, fmt.Errorf("unexpected number of bytes were copied, %d instead of limit %d", n, r.MaxBytes) } + message.Content = tmp message.AddFlagsWithKey("log.flags", "truncated") } return message, err From 6921dc21acad799dd0a2dd0c9694e7f53da7eae2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Mon, 15 Apr 2019 11:55:53 +0200 Subject: [PATCH 3/5] add changelog entry --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index a3fba8f8ed10..d9fed927c375 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -73,6 +73,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix goroutine leak happening when harvesters are dynamically stopped. {pull}11263[11263] - Fix `add_docker_metadata` source matching, using `log.file.path` field now. {pull}11577[11577] - Add missing Kubernetes metadata fields to Filebeat CoreDNS module, and fix a documentation error. {pull}11591[11591] +- LimitReader copies bytes instead of slicing so the underlying big array can be freed. {pull}11524[11524] *Heartbeat* From cb6718af9583c33e9cc826fe5c0d539912e85aaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Mon, 15 Apr 2019 12:43:57 +0200 Subject: [PATCH 4/5] rly fix it --- libbeat/reader/readfile/limit.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libbeat/reader/readfile/limit.go b/libbeat/reader/readfile/limit.go index d068be05aa39..403dea958476 100644 --- a/libbeat/reader/readfile/limit.go +++ b/libbeat/reader/readfile/limit.go @@ -40,9 +40,9 @@ func (r *LimitReader) Next() (reader.Message, error) { message, err := r.reader.Next() if len(message.Content) > r.maxBytes { tmp := make([]byte, r.maxBytes) - n = copy(tmp, message.Content) - if n != r.MaxBytes { - return event, fmt.Errorf("unexpected number of bytes were copied, %d instead of limit %d", n, r.MaxBytes) + n := copy(tmp, message.Content) + if n != r.maxBytes { + return message, fmt.Errorf("unexpected number of bytes were copied, %d instead of limit %d", n, r.maxBytes) } message.Content = tmp message.AddFlagsWithKey("log.flags", "truncated") From 7fd28241a3e3f0f29d66d31edaa7c46a46d72790 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Tue, 16 Apr 2019 16:50:26 +0200 Subject: [PATCH 5/5] reword changelog entry --- CHANGELOG.next.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index d9fed927c375..f6840a747848 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -73,7 +73,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix goroutine leak happening when harvesters are dynamically stopped. {pull}11263[11263] - Fix `add_docker_metadata` source matching, using `log.file.path` field now. {pull}11577[11577] - Add missing Kubernetes metadata fields to Filebeat CoreDNS module, and fix a documentation error. {pull}11591[11591] -- LimitReader copies bytes instead of slicing so the underlying big array can be freed. {pull}11524[11524] +- Reduce memory usage if long lines are truncated to fit `max_bytes` limit. The line buffer is copied into a smaller buffer now. This allows the runtime to release unused memory earlier. {pull}11524[11524] *Heartbeat*