Migration of metricset.* fields to ECS

CHANGELOG.asciidoc

@@ -129,6 +129,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha1...master[Check the HEAD d
 *Journalbeat*
 
 *Metricbeat*
+- event.duration is now in nano and not microseconds anymore. {pull}8941[8941]
 
 *Packetbeat*
 

dev-tools/ecs-migration.yml

@@ -375,7 +375,6 @@
 - from: nginx.access.method
   to: http.request.method
   alias: true
-  copy_to: false
 
 - from: nginx.access.http_version
   to: http.version
@@ -413,3 +412,29 @@
 - from: source.hostname
   to: source.domain
   alias: true
+
+# Metricbeat base fields
+- from: metricset.name
+  to: event.dataset
+  alias: false
+  comment: Data structure of field changed, old field will stay around
+  breaking: true
+
+- from: metricset.module
+  to: event.module
+  alias: true
+
+- from: metricset.rrt
+  to: event.duration
+  alias: false
+  comment: Unit changed, old field will stay around if needed
+  breaking: true
+
+- from: metricset.host
+  to: service.address
+  alias: true
+
+- from: metricset.namespace
+  to: event.dataset
+  alias: false
+  comment: No alias mapping as field did not always exist

metricbeat/_meta/fields.common.yml

@@ -7,27 +7,18 @@
     - name: metricset.module
       description: >
         The name of the module that generated the event.
+      type: alias
+      path: event.module
 
     - name: metricset.name
       description: >
         The name of the metricset that generated the event.
 
-    - name: metricset.host
+    - name: service.address
       description: >
-        Hostname of the machine from which the metricset was collected. This
+        Connection address of the machine from which the metricset was collected. This
         field may not be present when the data was collected locally.
 
-    - name: metricset.rtt
-      type: long
-      required: true
-      description: >
-        Event round trip time in microseconds.
-
-    - name: metricset.namespace
-      type: keyword
-      description: >
-        Namespace of dynamic metricsets.
-
     - name: type
       required: true
       example: metricsets

metricbeat/docs/fields.asciidoc

@@ -1781,6 +1781,10 @@ Contains common fields available in all event types.
 *`metricset.module`*::
 +
 --
+type: alias
+
+alias to: event.module
+
 The name of the module that generated the event.
 
 
@@ -1794,32 +1798,10 @@ The name of the metricset that generated the event.
 
 --
 
-*`metricset.host`*::
+*`service.address`*::
 +
 --
-Hostname of the machine from which the metricset was collected. This field may not be present when the data was collected locally.
-
-
---
-
-*`metricset.rtt`*::
-+
---
-type: long
-
-required: True
-
-Event round trip time in microseconds.
-
-
---
-
-*`metricset.namespace`*::
-+
---
-type: keyword
-
-Namespace of dynamic metricsets.
+Connection address of the machine from which the metricset was collected. This field may not be present when the data was collected locally.
 
 
 --

metricbeat/mb/event.go

@@ -18,6 +18,7 @@
 package mb
 
 import (
+	"fmt"
 	"time"
 
 	"github.com/elastic/beats/libbeat/beat"
@@ -116,27 +117,32 @@ func (e *Event) BeatEvent(module, metricSet string, modifiers ...EventModifier)
 //       "rtt": 115
 //     }
 func AddMetricSetInfo(module, metricset string, event *Event) {
-	info := common.MapStr{
-		"name":   metricset,
-		"module": module,
+
+	if event.Namespace == "" {
+		event.Namespace = fmt.Sprintf("%s.%s", module, metricset)
+	}
+
+	e := common.MapStr{
+		"event": common.MapStr{
+			"dataset": event.Namespace,
+			"module":  module,
+		},
+		// TODO: This should only be sent if migration layer is enabled
+		"metricset": common.MapStr{
+			"name": metricset,
+		},
 	}
 	if event.Host != "" {
-		info["host"] = event.Host
+		e.Put("service.address", event.Host)
 	}
 	if event.Took > 0 {
-		info["rtt"] = event.Took / time.Microsecond
-	}
-	if event.Namespace != "" {
-		info["namespace"] = event.Namespace
-	}
-	info = common.MapStr{
-		"metricset": info,
+		e.Put("event.duration", event.Took/time.Nanosecond)
 	}
 
 	if event.RootFields == nil {
-		event.RootFields = info
+		event.RootFields = e
 	} else {
-		event.RootFields.DeepUpdate(info)
+		event.RootFields.DeepUpdate(e)
 	}
 }
 

metricbeat/mb/event_test.go

@@ -186,11 +186,16 @@ func TestAddMetricSetInfo(t *testing.T) {
 		AddMetricSetInfo(moduleName, metricSetName, &e)
 
 		assert.Equal(t, common.MapStr{
+			"event": common.MapStr{
+				"module":   moduleName,
+				"dataset":  moduleName + "." + metricSetName,
+				"duration": time.Duration(500000000),
+			},
+			"service": common.MapStr{
+				"address": host,
+			},
 			"metricset": common.MapStr{
-				"host":   host,
-				"module": moduleName,
-				"name":   metricSetName,
-				"rtt":    time.Duration(500000),
+				"name": metricSetName,
 			},
 		}, e.RootFields)
 	})
@@ -201,9 +206,12 @@ func TestAddMetricSetInfo(t *testing.T) {
 		AddMetricSetInfo(moduleName, metricSetName, &e)
 
 		assert.Equal(t, common.MapStr{
+			"event": common.MapStr{
+				"module":  moduleName,
+				"dataset": moduleName + "." + metricSetName,
+			},
 			"metricset": common.MapStr{
-				"module": moduleName,
-				"name":   metricSetName,
+				"name": metricSetName,
 			},
 		}, e.RootFields)
 	})

metricbeat/mb/module/example_test.go

@@ -64,7 +64,7 @@ func ExampleWrapper() {
 	go func() {
 		defer wg.Done()
 		for event := range output {
-			event.Fields.Put("metricset.rtt", 111)
+			event.Fields.Put("event.duration", 111)
 
 			output, err := encodeEvent(event)
 			if err == nil {
@@ -90,15 +90,18 @@ func ExampleWrapper() {
 	//     "version": "1.2.3"
 	//   },
 	//   "@timestamp": "2016-05-10T23:27:58.485Z",
+	//   "event": {
+	//     "dataset": "fake.eventfetcher",
+	//     "duration": 111,
+	//     "module": "fake"
+	//   },
 	//   "fake": {
 	//     "eventfetcher": {
 	//       "metric": 1
 	//     }
 	//   },
 	//   "metricset": {
-	//     "module": "fake",
-	//     "name": "eventfetcher",
-	//     "rtt": 111
+	//     "name": "eventfetcher"
 	//   },
 	//   "service": {
 	//     "type": "fake"

metricbeat/module/docker/image/_meta/data.json

@@ -1,29 +1,48 @@
 {
     "@timestamp": "2017-10-12T08:05:34.853Z",
-    "beat": {
+    "agent": {
         "hostname": "host.example.com",
         "name": "host.example.com"
     },
     "docker": {
         "image": {
-            "created": "2017-12-07T06:18:10.000Z",
+            "created": "2018-11-27T14:54:24.000Z",
             "id": {
-                "current": "sha256:8af9af305016ff6501ad0605e390e0a3b535265c0d0cf43375ea8149b794d1d3",
-                "parent": "sha256:d082cc35db8aef622e2a3e659cca62962158e72fbad5c8edd9f664007ae048f1"
+                "current": "sha256:bd0649cfa954c27054c5b4c67a2091e5d03f41a441e78b6971328cd3936a997f",
+                "parent": "sha256:a75037d73967907c74a2e5e219ae1eb10e9a35bb055a926eead2920866f75764"
+            },
+            "labels": {
+                "description": "Filebeat sends log files to Logstash or directly to Elasticsearch.",
+                "license": "ASL 2.0",
+                "org_label-schema_build-date": "20181006",
+                "org_label-schema_license": "GPLv2",
+                "org_label-schema_name": "filebeat",
+                "org_label-schema_schema-version": "1.0",
+                "org_label-schema_url": "https://www.elastic.co/products/beats/filebeat",
+                "org_label-schema_vcs-ref": "b9806f16b79dce1223119de94251cb1900f45e7f",
+                "org_label-schema_vcs-url": "github.com/elastic/beats",
+                "org_label-schema_vendor": "Elastic",
+                "org_label-schema_version": "7.0.0"
             },
             "size": {
-                "regular": 799052371,
-                "virtual": 799052371
+                "regular": 280257780,
+                "virtual": 280257780
             },
             "tags": [
-                "metricbeatsnapshotnoxpack700alpha1d71419298b58ed8b0a5b60a6d1e4a476ffaf80a8_beat:latest"
+                "docker.elastic.co/beats/filebeat-oss:7.0.0"
             ]
         }
     },
+    "event": {
+        "dataset": "docker.image",
+        "duration": 115000,
+        "module": "docker"
+    },
     "metricset": {
-        "host": "unix:///var/run/docker.sock",
-        "module": "docker",
-        "name": "image",
-        "rtt": 115
+        "name": "image"
+    },
+    "service": {
+        "type": "docker",
+        "uri": "/var/run/docker.sock"
     }
 }

metricbeat/module/elasticsearch/index/index.go

@@ -31,6 +31,7 @@ import (
 func init() {
 	mb.Registry.MustAddMetricSet(elasticsearch.ModuleName, "index", New,
 		mb.WithHostParser(elasticsearch.HostParser),
+		mb.WithNamespace("elasticsearch.index"),
 	)
 }
 

metricbeat/module/elasticsearch/test_elasticsearch.py

@@ -42,7 +42,7 @@ def test_metricsets(self, metricset):
         es = Elasticsearch(self.get_hosts())
         es.indices.create(index='test-index', ignore=400)
         self.check_metricset("elasticsearch", metricset, self.get_hosts(), self.FIELDS +
-                             ["service.name"], extras={"index_recovery.active_only": "false"})
+                             ["service"], extras={"index_recovery.active_only": "false"})
 
     def get_hosts(self):
         return [os.getenv('ES_HOST', 'localhost') + ':' +

metricbeat/module/system/core/_meta/data.json

@@ -1,48 +1,54 @@
 {
     "@timestamp": "2017-10-12T08:05:34.853Z",
-    "beat": {
+    "agent": {
         "hostname": "host.example.com",
         "name": "host.example.com"
     },
+    "event": {
+        "dataset": "system.core",
+        "duration": 115000,
+        "module": "system"
+    },
     "metricset": {
-        "module": "system",
-        "name": "core",
-        "rtt": 115
+        "name": "core"
+    },
+    "service": {
+        "type": "system"
     },
     "system": {
         "core": {
             "id": 1,
             "idle": {
-                "pct": 0.3333,
-                "ticks": 51795238
+                "pct": 0.2157,
+                "ticks": 19266526
             },
             "iowait": {
                 "pct": 0,
-                "ticks": 42254
+                "ticks": 0
             },
             "irq": {
                 "pct": 0,
                 "ticks": 0
             },
             "nice": {
                 "pct": 0,
-                "ticks": 15
+                "ticks": 0
             },
             "softirq": {
                 "pct": 0,
-                "ticks": 1469
+                "ticks": 0
             },
             "steal": {
                 "pct": 0,
                 "ticks": 0
             },
             "system": {
-                "pct": 0.25,
-                "ticks": 109372
+                "pct": 0.3137,
+                "ticks": 781976
             },
             "user": {
-                "pct": 0.4167,
-                "ticks": 185023
+                "pct": 0.4706,
+                "ticks": 566036
             }
         }
     }