model: ECS for 6.x

_beats/dev-tools/ecs-migration.yml

@@ -15,11 +15,6 @@
 
 # Beat fields
 
-- from: beat.hostname
-  to: host.hostname
-  alias6: true
-  alias: true
-
 - from: beat.timezone
   to: event.timezone
   alias6: true

_beats/libbeat/_meta/fields.common.yml

@@ -66,9 +66,6 @@
     Alias fields for compatibility with 7.x.
   fields:
     # Common Beats fields
-    - name: host.hostname
-      type: alias
-      path: beat.hostname
     - name: event.timezone
       type: alias
       path: beat.timezone

_beats/libbeat/processors/add_host_metadata/_meta/fields.yml

@@ -15,14 +15,6 @@
           type: keyword
           description: >
             Unique host id.
-        - name: architecture
-          type: keyword
-          description: >
-            Host architecture (e.g. x86_64, arm, ppc, mips).
-        - name: os.platform
-          type: keyword
-          description: >
-            OS platform (e.g. centos, ubuntu, windows).
         - name: os.version
           type: keyword
           description: >
@@ -31,12 +23,7 @@
           type: keyword
           description: >
             OS family (e.g. redhat, debian, freebsd, windows).
-        - name: ip
-          type: ip
-          description: >
-            List of IP-addresses.
         - name: mac
           type: keyword
           description: >
             List of hardware-addresses, usually MAC-addresses.
-

_beats/libbeat/scripts/generate_fields_docs.py

@@ -1,6 +1,8 @@
-import yaml
-import os
 import argparse
+from collections import OrderedDict
+import os
+
+import yaml
 
 
 def document_fields(output, section, sections, path):
@@ -66,8 +68,8 @@ def document_field(output, field, field_path):
         if not field["index"]:
             output.write("{}\n\n".format("Field is not indexed."))
 
-    if "enable" in field:
-        if not field["enable"]:
+    if "enabled" in field:
+        if not field["enabled"]:
             output.write("{}\n\n".format("Object is not enabled."))
 
     if "multi_fields" in field:
@@ -103,6 +105,21 @@ def fields_to_asciidoc(input, output, beat):
         print("fields.yml file is empty. fields.asciidoc cannot be generated.")
         return
 
+    # deduplicate fields, last one wins
+    for section in docs:
+        if not section.get("fields"):
+            continue
+        fields = OrderedDict()
+        for field in section["fields"]:
+            name = field["name"]
+            if name in fields:
+                assert field["type"] == fields[name]["type"], 'field "{}" redefined with different type "{}"'.format(
+                    name, field["type"])
+                fields[name].update(field)
+            else:
+                fields[name] = field
+        section["fields"] = list(fields.values())
+
     # Create sections from available fields
     sections = {}
     for v in docs:

_meta/ecs-migration.yml

@@ -0,0 +1,140 @@
+# The ECS migration file contains the information about all the fields which are migrated to ECS in 7.0.
+# The goal of the file is to potentially have scripts on top of this information to convert visualisations and templates
+# based on this information in an automated way and to keep track of all changes which were applied.
+#
+# The format of the file is as following:
+#
+# - from: source-field-in-6.x
+#   to: target-filed-in-ECS
+#   # Alias field is useful for fields where there is a 1-1 mapping from old to new
+#   alias: true-if-alias-is-required-in-6x (default is true)
+#   # Copy to is useful for fields where multiple fields map to the same ECS field
+#   copy_to: true-if-field-should-be-copied-to-target-in-6x (default is false)
+
+- from: beat.hostname
+  to: observer.hostname
+
+- from: beat.name
+  to: observer.type
+
+- from: beat.version
+  to: observer.version
+
+- from: context.service.agent.name
+  to: agent.name
+
+- from: context.service.agent.version
+  to: agent.version
+
+- from: context.system.architecture
+  to: host.architecture
+
+- from: context.system.hostname
+  to: host.hostname
+
+- from: context.system.ip
+  to: host.ip
+
+- from: context.system.platform
+  to: host.os.platform
+
+- from: context.request.method
+  to: http.request.method
+
+- from: context.request.http_version
+  to: http.version
+
+- from: context.process.pid
+  to: process.pid
+
+- from: context.process.ppid
+  to: process.ppid
+
+- from: context.process.title
+  to: process.title
+
+  # not in ECS
+- from: context.service.environment
+  to: service.environment
+
+  # not in ECS
+- from: context.service.framework.name
+  to: service.framework.name
+
+  # not in ECS
+- from: context.service.framework.version
+  to: service.framework.version
+
+  # not in ECS
+- from: context.service.language.name
+  to: service.language.name
+
+  # not in ECS
+- from: context.service.language.version
+  to: service.language.version
+
+- from: context.service.name
+  to: service.name
+
+  # not in ECS
+- from: context.service.runtime.name
+  to: service.runtime.name
+
+  # not in ECS
+- from: context.service.runtime.version
+  to: service.runtime.version
+
+- from: context.service.version
+  to: service.version
+
+- from: context.request.url.full
+  to: url.full
+
+- from: context.request.url.hash
+  to: url.fragment
+
+- from: context.request.url.hostname
+  to: url.domain
+
+- from: context.request.url.pathname
+  to: url.path
+
+- from: context.request.url.port
+  to: url.port
+  alias: false
+  copy_to: true
+
+- from: context.request.url.raw
+  to: url.original
+
+- from: context.request.url.search
+  to: url.query
+
+- from: context.request.url.protocol
+  to: url.scheme
+  alias: false
+  copy_to: true
+
+- from: context.response.finished
+  to: http.response.finished
+
+- from: context.response.status_code
+  to: http.response.status_code
+
+- from: context.user.email
+  to: user.email
+
+- from: context.user.id
+  to: user.id
+
+- from: context.user.username
+  to: user.name
+
+- from: context.user.ip
+  to: client.ip
+
+- from: context.user.user-agent
+  to: user_agent.original.text
+
+- from: listening
+  to: observer.listening