Automatic merge from master to 6.x branch

CHANGELOG.asciidoc

@@ -49,6 +49,7 @@ https://github.com/elastic/beats/compare/v6.2.3...master[Check the HEAD diff]
 - Change http/server metricset to put events by default under http.server and prefix config options with server.. {pull}7100[7100]
 - Disable dedotting in docker module configuration. This will change the out-of-the-box behaviour, but not the one of already configured instances. {pull}7485[7485]
 - Fix typo in etcd/self metricset fields from *.bandwithrate to *.bandwidthrate. {pull}7456[7456]
+- Changed the definition of the `system.cpu.total.pct` and `system.cpu.total.norm.cou` fields to exclude the IOWait time. {pull}7691[7691]
 
 *Packetbeat*
 
@@ -131,6 +132,7 @@ https://github.com/elastic/beats/compare/v6.2.3...master[Check the HEAD diff]
 - Do not report Metricbeat container host as hostname in Kubernetes deployment. {issue}7199[7199]
 - Ensure metadata updates don't replace existing pod metrics. {pull}7573[7573]
 - Fix kubernetes pct fields reporting. {pull}7677[7677]
+- Add support for new `kube_node_status_condition` in Kubernetes `state_node`. {pull}7699[7699]
 
 *Packetbeat*
 
@@ -317,6 +319,8 @@ https://github.com/elastic/beats/compare/v6.2.3...master[Check the HEAD diff]
 - Release raid and socket metricset from system module as GA. {pull}7658[7658]
 - Release elasticsearch module and all its metricsets as beta. {pull}7662[7662]
 - Release munin and traefik module as beta. {pull}7660[7660]
+- Add envoyproxy module. {pull}7569[7569]
+- Release prometheus collector metricset as GA. {pull}7660[7660]
 
 *Packetbeat*
 

auditbeat/docs/modules/auditd.asciidoc

@@ -67,6 +67,55 @@ from listening to audit messages:
 systemctl mask systemd-journald-audit.socket
 -----
 
+[float]
+=== Inspect the kernel audit system status
+
+{beatname_uc} provides useful commands to query the state of the audit system
+in the Linux kernel.
+
+* See the list of installed audit rules:
++
+[source,shell]
+-----
+auditbeat show auditd-rules
+-----
++
+Prints the list of loaded rules, similar to `auditctl -l`:
++
+[source,shell]
+-----
+-a never,exit -S all -F pid=26253
+-a always,exit -F arch=b32 -S all -F key=32bit-abi
+-a always,exit -F arch=b64 -S execve,execveat -F key=exec
+-a always,exit -F arch=b64 -S connect,accept,bind -F key=external-access
+-w /etc/group -p wa -k identity
+-w /etc/passwd -p wa -k identity
+-w /etc/gshadow -p wa -k identity
+-a always,exit -F arch=b64 -S open,truncate,ftruncate,creat,openat,open_by_handle_at -F exit=-EACCES -F key=access
+-a always,exit -F arch=b64 -S open,truncate,ftruncate,creat,openat,open_by_handle_at -F exit=-EPERM -F key=access
+-----
+
+* See the status of the audit system:
++
+[source,shell]
+-----
+auditbeat show auditd-status
+-----
++
+Prints the status of the kernel audit system, similar to `auditctl -s`:
++
+[source,shell]
+-----
+enabled 1
+failure 0
+pid 0
+rate_limit 0
+backlog_limit 8192
+lost 14407
+backlog 0
+backlog_wait_time 0
+features 0xf
+-----
 
 [float]
 === Configuration options
@@ -79,10 +128,11 @@ following example shows all configuration options with their default values.
 - module: auditd
   resolve_ids: true
   failure_mode: silent
-  backlog_limit: 8196
+  backlog_limit: 8192
   rate_limit: 0
   include_raw_message: false
   include_warnings: false
+  backpressure_strategy: auto
 ----
 
 *`socket_type`*:: This optional setting controls the type of
@@ -146,6 +196,28 @@ loaded after the rules declared in `audit_rules` are loaded. Wildcards are
 supported and will expand in lexicographical order. The format is the same as
 that of the `audit_rules` field.
 
+*`backpressure_strategy`*:: Specifies the strategy that {beatname_uc} uses to
+prevent backpressure from propagating to the kernel and impacting audited
+processes.
++
+--
+The possible values are:
+
+- `auto` (default): {beatname_uc} uses the `kernel` strategy, if supported, or
+falls back to the `userspace` strategy.
+- `kernel`: {beatname_uc} sets the `backlog_wait_time` in the kernel's
+audit framework to 0. This causes events to be discarded in the kernel if
+the audit backlog queue fills to capacity. Requires a 3.14 kernel or
+newer.
+- `userspace`: {beatname_uc} drops events when there is backpressure
+from the publishing pipeline. If no `rate_limit` is set, {beatname_uc} sets a rate
+limit of 5000. Users should test their setup and adjust the `rate_limit`
+option accordingly.
+- `both`: {beatname_uc} uses the `kernel` and `userspace` strategies at the same
+time.
+- `none`: No backpressure mitigation measures are enabled.
+--
+
 [float]
 === Audit rules
 

auditbeat/module/auditd/_meta/docs.asciidoc

@@ -62,6 +62,55 @@ from listening to audit messages:
 systemctl mask systemd-journald-audit.socket
 -----
 
+[float]
+=== Inspect the kernel audit system status
+
+{beatname_uc} provides useful commands to query the state of the audit system
+in the Linux kernel.
+
+* See the list of installed audit rules:
++
+[source,shell]
+-----
+auditbeat show auditd-rules
+-----
++
+Prints the list of loaded rules, similar to `auditctl -l`:
++
+[source,shell]
+-----
+-a never,exit -S all -F pid=26253
+-a always,exit -F arch=b32 -S all -F key=32bit-abi
+-a always,exit -F arch=b64 -S execve,execveat -F key=exec
+-a always,exit -F arch=b64 -S connect,accept,bind -F key=external-access
+-w /etc/group -p wa -k identity
+-w /etc/passwd -p wa -k identity
+-w /etc/gshadow -p wa -k identity
+-a always,exit -F arch=b64 -S open,truncate,ftruncate,creat,openat,open_by_handle_at -F exit=-EACCES -F key=access
+-a always,exit -F arch=b64 -S open,truncate,ftruncate,creat,openat,open_by_handle_at -F exit=-EPERM -F key=access
+-----
+
+* See the status of the audit system:
++
+[source,shell]
+-----
+auditbeat show auditd-status
+-----
++
+Prints the status of the kernel audit system, similar to `auditctl -s`:
++
+[source,shell]
+-----
+enabled 1
+failure 0
+pid 0
+rate_limit 0
+backlog_limit 8192
+lost 14407
+backlog 0
+backlog_wait_time 0
+features 0xf
+-----
 
 [float]
 === Configuration options
@@ -74,10 +123,11 @@ following example shows all configuration options with their default values.
 - module: auditd
   resolve_ids: true
   failure_mode: silent
-  backlog_limit: 8196
+  backlog_limit: 8192
   rate_limit: 0
   include_raw_message: false
   include_warnings: false
+  backpressure_strategy: auto
 ----
 
 *`socket_type`*:: This optional setting controls the type of
@@ -141,6 +191,28 @@ loaded after the rules declared in `audit_rules` are loaded. Wildcards are
 supported and will expand in lexicographical order. The format is the same as
 that of the `audit_rules` field.
 
+*`backpressure_strategy`*:: Specifies the strategy that {beatname_uc} uses to
+prevent backpressure from propagating to the kernel and impacting audited
+processes.
++
+--
+The possible values are:
+
+- `auto` (default): {beatname_uc} uses the `kernel` strategy, if supported, or
+falls back to the `userspace` strategy.
+- `kernel`: {beatname_uc} sets the `backlog_wait_time` in the kernel's
+audit framework to 0. This causes events to be discarded in the kernel if
+the audit backlog queue fills to capacity. Requires a 3.14 kernel or
+newer.
+- `userspace`: {beatname_uc} drops events when there is backpressure
+from the publishing pipeline. If no `rate_limit` is set, {beatname_uc} sets a rate
+limit of 5000. Users should test their setup and adjust the `rate_limit`
+option accordingly.
+- `both`: {beatname_uc} uses the `kernel` and `userspace` strategies at the same
+time.
+- `none`: No backpressure mitigation measures are enabled.
+--
+
 [float]
 === Audit rules
 

filebeat/module/icinga/startup/test/test.log-expected.json

@@ -1,24 +1,24 @@
 [
     {
-        "@timestamp": "2018-07-24T10:26:47.908Z", 
-        "fileset.module": "icinga", 
-        "fileset.name": "startup", 
-        "icinga.startup.facility": "cli", 
-        "icinga.startup.message": "Icinga application loader (version: r2.6.3-1)", 
-        "icinga.startup.severity": "information", 
-        "input.type": "log", 
-        "offset": 0, 
+        "@timestamp": "2018-07-23T11:50:38.896Z",
+        "fileset.module": "icinga",
+        "fileset.name": "startup",
+        "icinga.startup.facility": "cli",
+        "icinga.startup.message": "Icinga application loader (version: r2.6.3-1)",
+        "icinga.startup.severity": "information",
+        "input.type": "log",
+        "offset": 0,
         "prospector.type": "log"
-    }, 
+    },
     {
-        "@timestamp": "2018-07-24T10:26:47.908Z", 
-        "fileset.module": "icinga", 
-        "fileset.name": "startup", 
-        "icinga.startup.facility": "cli", 
-        "icinga.startup.message": "Loading configuration file(s).", 
-        "icinga.startup.severity": "information", 
-        "input.type": "log", 
-        "offset": 63, 
+        "@timestamp": "2018-07-23T11:50:38.896Z",
+        "fileset.module": "icinga",
+        "fileset.name": "startup",
+        "icinga.startup.facility": "cli",
+        "icinga.startup.message": "Loading configuration file(s).",
+        "icinga.startup.severity": "information",
+        "input.type": "log",
+        "offset": 63,
         "prospector.type": "log"
     }
-]
+]

filebeat/module/iis/access/test/test.log-expected.json

@@ -8,6 +8,7 @@
         "iis.access.geoip.country_iso_code": "DE", 
         "iis.access.geoip.location.lat": 52.5167, 
         "iis.access.geoip.location.lon": 13.4, 
+        "iis.access.geoip.region_iso_code": "DE-BE", 
         "iis.access.geoip.region_name": "Land Berlin", 
         "iis.access.method": "GET", 
         "iis.access.port": "80", 
@@ -73,6 +74,7 @@
         "iis.access.geoip.country_iso_code": "DE", 
         "iis.access.geoip.location.lat": 52.5167, 
         "iis.access.geoip.location.lon": 13.4, 
+        "iis.access.geoip.region_iso_code": "DE-BE", 
         "iis.access.geoip.region_name": "Land Berlin", 
         "iis.access.hostname": "example.com", 
         "iis.access.http_version": "1.1", 

filebeat/module/iis/error/test/test.log-expected.json

@@ -26,6 +26,7 @@
         "iis.error.geoip.country_iso_code": "DE", 
         "iis.error.geoip.location.lat": 52.5167, 
         "iis.error.geoip.location.lon": 13.4, 
+        "iis.error.geoip.region_iso_code": "DE-BE", 
         "iis.error.geoip.region_name": "Land Berlin", 
         "iis.error.http_version": "1.1", 
         "iis.error.method": "GET", 
@@ -50,6 +51,7 @@
         "iis.error.geoip.country_iso_code": "DE", 
         "iis.error.geoip.location.lat": 52.5167, 
         "iis.error.geoip.location.lon": 13.4, 
+        "iis.error.geoip.region_iso_code": "DE-BE", 
         "iis.error.geoip.region_name": "Land Berlin", 
         "iis.error.http_version": "2.0", 
         "iis.error.method": "GET", 
@@ -74,6 +76,7 @@
         "iis.error.geoip.country_iso_code": "DE", 
         "iis.error.geoip.location.lat": 52.5167, 
         "iis.error.geoip.location.lon": 13.4, 
+        "iis.error.geoip.region_iso_code": "DE-BE", 
         "iis.error.geoip.region_name": "Land Berlin", 
         "iis.error.queue_name": "-", 
         "iis.error.reason_phrase": "Timer_MinBytesPerSecond", 

filebeat/module/nginx/access/test/test.log-expected.json

@@ -65,6 +65,7 @@
         "nginx.access.geoip.country_iso_code": "DE", 
         "nginx.access.geoip.location.lat": 52.5167, 
         "nginx.access.geoip.location.lon": 13.4, 
+        "nginx.access.geoip.region_iso_code": "DE-BE", 
         "nginx.access.geoip.region_name": "Land Berlin", 
         "nginx.access.http_version": "1.1", 
         "nginx.access.method": "GET", 
@@ -100,6 +101,7 @@
         "nginx.access.geoip.country_iso_code": "DE", 
         "nginx.access.geoip.location.lat": 52.5167, 
         "nginx.access.geoip.location.lon": 13.4, 
+        "nginx.access.geoip.region_iso_code": "DE-BE", 
         "nginx.access.geoip.region_name": "Land Berlin", 
         "nginx.access.http_version": "1.1", 
         "nginx.access.method": "GET", 
@@ -133,6 +135,7 @@
         "nginx.access.geoip.country_iso_code": "US", 
         "nginx.access.geoip.location.lat": 39.772, 
         "nginx.access.geoip.location.lon": -89.6859, 
+        "nginx.access.geoip.region_iso_code": "US-IL", 
         "nginx.access.geoip.region_name": "Illinois", 
         "nginx.access.http_version": "1.1", 
         "nginx.access.method": "GET", 

filebeat/module/system/auth/test/test.log-expected.json

@@ -61,6 +61,7 @@
         "system.auth.ssh.geoip.country_iso_code": "CN", 
         "system.auth.ssh.geoip.location.lat": 22.5333, 
         "system.auth.ssh.geoip.location.lon": 114.1333, 
+        "system.auth.ssh.geoip.region_iso_code": "CN-44", 
         "system.auth.ssh.geoip.region_name": "Guangdong", 
         "system.auth.ssh.ip": "116.31.116.24", 
         "system.auth.ssh.method": "password", 

filebeat/module/traefik/access/test/test.log-expected.json

@@ -35,6 +35,7 @@
         "traefik.access.geoip.country_iso_code": "DE", 
         "traefik.access.geoip.location.lat": 52.5167, 
         "traefik.access.geoip.location.lon": 13.4, 
+        "traefik.access.geoip.region_iso_code": "DE-BE", 
         "traefik.access.geoip.region_name": "Land Berlin", 
         "traefik.access.http_version": "1.1", 
         "traefik.access.method": "GET", 

libbeat/common/field.go

@@ -52,6 +52,7 @@ type Field struct {
 	DocValues             *bool       `config:"doc_values"`
 	CopyTo                string      `config:"copy_to"`
 	IgnoreAbove           int         `config:"ignore_above"`
+	AliasPath             string      `config:"path"`
 
 	// Kibana specific
 	Analyzed     *bool  `config:"analyzed"`

libbeat/metric/system/cpu/cpu.go

@@ -116,7 +116,9 @@ func cpuPercentages(s0, s1 *sigar.Cpu, numCPU int) Percentages {
 	}
 
 	calculateTotalPct := func() float64 {
-		return common.Round(float64(numCPU)-calculatePct(s0.Idle, s1.Idle), common.DefaultDecimalPlacesCount)
+		// IOWait time is excluded from the total as per #7627.
+		idle := calculatePct(s0.Idle, s1.Idle) + calculatePct(s0.Wait, s1.Wait)
+		return common.Round(float64(numCPU)-idle, common.DefaultDecimalPlacesCount)
 	}
 
 	return Percentages{

libbeat/template/processor.go

@@ -62,6 +62,8 @@ func (p *Processor) Process(fields common.Fields, path string, output common.Map
 			mapping = p.object(&field)
 		case "array":
 			mapping = p.array(&field)
+		case "alias":
+			mapping = p.alias(&field)
 		case "group":
 			var newPath string
 			if path == "" {
@@ -242,6 +244,13 @@ func (p *Processor) array(f *common.Field) common.MapStr {
 	return properties
 }
 
+func (p *Processor) alias(f *common.Field) common.MapStr {
+	properties := getDefaultProperties(f)
+	properties["type"] = "alias"
+	properties["path"] = f.AliasPath
+	return properties
+}
+
 func (p *Processor) object(f *common.Field) common.MapStr {
 	dynProperties := getDefaultProperties(f)
 

libbeat/template/processor_test.go

@@ -86,6 +86,10 @@ func TestProcessor(t *testing.T) {
 			output:   p.array(&common.Field{Type: "array", Index: &falseVar, ObjectType: "keyword"}),
 			expected: common.MapStr{"index": false, "type": "keyword"},
 		},
+		{
+			output:   p.alias(&common.Field{Type: "alias", AliasPath: "a.b"}),
+			expected: common.MapStr{"path": "a.b", "type": "alias"},
+		},
 		{
 			output: p.object(&common.Field{Type: "object", Enabled: &falseVar}),
 			expected: common.MapStr{