Beats: Set default_field=false in multi_fields

generated/beats/fields.ecs.yml

@@ -133,6 +133,7 @@
       - name: text
         type: text
         norms: false
+        default_field: false
       description: Organization name.
       example: Google LLC
   - name: client
@@ -179,6 +180,7 @@
       - name: text
         type: text
         norms: false
+        default_field: false
       description: Organization name.
       example: Google LLC
     - name: bytes
@@ -325,6 +327,7 @@
       - name: text
         type: text
         norms: false
+        default_field: false
       description: User's full name, if available.
       example: Albert Einstein
     - name: user.group.domain
@@ -366,6 +369,7 @@
       - name: text
         type: text
         norms: false
+        default_field: false
       description: Short name or login of the user.
       example: albert
   - name: cloud
@@ -497,6 +501,7 @@
       - name: text
         type: text
         norms: false
+        default_field: false
       description: Organization name.
       example: Google LLC
     - name: bytes
@@ -642,6 +647,7 @@
       - name: text
         type: text
         norms: false
+        default_field: false
       description: User's full name, if available.
       example: Albert Einstein
     - name: user.group.domain
@@ -683,6 +689,7 @@
       - name: text
         type: text
         norms: false
+        default_field: false
       description: Short name or login of the user.
       example: albert
   - name: dns
@@ -908,6 +915,7 @@
       - name: text
         type: text
         norms: false
+        default_field: false
       description: The stack trace of this error in plain text.
     - name: type
       level: extended
@@ -1298,6 +1306,7 @@
       - name: text
         type: text
         norms: false
+        default_field: false
       description: Full path to the file, including the file name. It should include
         the drive letter, when appropriate.
       example: /home/alice/example.png
@@ -1316,6 +1325,7 @@
       - name: text
         type: text
         norms: false
+        default_field: false
       description: Target path for symlinks.
     - name: type
       level: extended
@@ -1574,6 +1584,7 @@
       - name: text
         type: text
         norms: false
+        default_field: false
       description: Operating system name, including the version or code name.
       example: Mac OS Mojave
     - name: os.kernel
@@ -1590,6 +1601,7 @@
       - name: text
         type: text
         norms: false
+        default_field: false
       description: Operating system name, without the version.
       example: Mac OS X
     - name: os.platform
@@ -1638,6 +1650,7 @@
       - name: text
         type: text
         norms: false
+        default_field: false
       description: User's full name, if available.
       example: Albert Einstein
     - name: user.group.domain
@@ -1679,6 +1692,7 @@
       - name: text
         type: text
         norms: false
+        default_field: false
       description: Short name or login of the user.
       example: albert
   - name: http
@@ -1702,6 +1716,7 @@
       - name: text
         type: text
         norms: false
+        default_field: false
       description: The full HTTP request body.
       example: Hello world
     - name: request.bytes
@@ -1739,6 +1754,7 @@
       - name: text
         type: text
         norms: false
+        default_field: false
       description: The full HTTP response body.
       example: Hello world
     - name: response.bytes
@@ -2094,6 +2110,7 @@
       - name: text
         type: text
         norms: false
+        default_field: false
       description: Operating system name, including the version or code name.
       example: Mac OS Mojave
     - name: os.kernel
@@ -2110,6 +2127,7 @@
       - name: text
         type: text
         norms: false
+        default_field: false
       description: Operating system name, without the version.
       example: Mac OS X
     - name: os.platform
@@ -2178,6 +2196,7 @@
       - name: text
         type: text
         norms: false
+        default_field: false
       description: Organization name.
   - name: os
     title: Operating System
@@ -2199,6 +2218,7 @@
       - name: text
         type: text
         norms: false
+        default_field: false
       description: Operating system name, including the version or code name.
       example: Mac OS Mojave
     - name: kernel
@@ -2215,6 +2235,7 @@
       - name: text
         type: text
         norms: false
+        default_field: false
       description: Operating system name, without the version.
       example: Mac OS X
     - name: platform
@@ -2382,6 +2403,7 @@
       - name: text
         type: text
         norms: false
+        default_field: false
       description: Absolute path to the process executable.
       example: /usr/bin/ssh
     - name: exit_code
@@ -2421,6 +2443,7 @@
       - name: text
         type: text
         norms: false
+        default_field: false
       description: 'Process name.
 
         Sometimes called program name or similar.'
@@ -2607,6 +2630,7 @@
       - name: text
         type: text
         norms: false
+        default_field: false
       description: 'Process title.
 
         The proctitle, some times the same as process name. Can also be different:
@@ -2624,6 +2648,7 @@
       - name: text
         type: text
         norms: false
+        default_field: false
       description: The working directory of the process.
       example: /home/alice
   - name: registry
@@ -2848,6 +2873,7 @@
       - name: text
         type: text
         norms: false
+        default_field: false
       description: Organization name.
       example: Google LLC
     - name: bytes
@@ -2994,6 +3020,7 @@
       - name: text
         type: text
         norms: false
+        default_field: false
       description: User's full name, if available.
       example: Albert Einstein
     - name: user.group.domain
@@ -3035,6 +3062,7 @@
       - name: text
         type: text
         norms: false
+        default_field: false
       description: Short name or login of the user.
       example: albert
   - name: service
@@ -3156,6 +3184,7 @@
       - name: text
         type: text
         norms: false
+        default_field: false
       description: Organization name.
       example: Google LLC
     - name: bytes
@@ -3302,6 +3331,7 @@
       - name: text
         type: text
         norms: false
+        default_field: false
       description: User's full name, if available.
       example: Albert Einstein
     - name: user.group.domain
@@ -3343,6 +3373,7 @@
       - name: text
         type: text
         norms: false
+        default_field: false
       description: Short name or login of the user.
       example: albert
   - name: threat
@@ -3407,6 +3438,7 @@
       - name: text
         type: text
         norms: false
+        default_field: false
       description: The name of technique used by this tactic. You can use the Mitre
         ATT&CK Matrix Tactic categorization, for example. (ex. https://attack.mitre.org/techniques/T1499/
         )
@@ -3734,6 +3766,7 @@
       - name: text
         type: text
         norms: false
+        default_field: false
       description: If full URLs are important to your use case, they should be stored
         in `url.full`, whether this field is reconstructed or present in the event
         source.
@@ -3746,6 +3779,7 @@
       - name: text
         type: text
         norms: false
+        default_field: false
       description: 'Unmodified original url as seen in the event source.
 
         Note that in network monitoring, the observed URL may be a full URL, whereas
@@ -3847,6 +3881,7 @@
       - name: text
         type: text
         norms: false
+        default_field: false
       description: User's full name, if available.
       example: Albert Einstein
     - name: group.domain
@@ -3888,6 +3923,7 @@
       - name: text
         type: text
         norms: false
+        default_field: false
       description: Short name or login of the user.
       example: albert
   - name: user_agent
@@ -3935,6 +3971,7 @@
       - name: text
         type: text
         norms: false
+        default_field: false
       description: Operating system name, including the version or code name.
       example: Mac OS Mojave
     - name: os.kernel
@@ -3951,6 +3988,7 @@
       - name: text
         type: text
         norms: false
+        default_field: false
       description: Operating system name, without the version.
       example: Mac OS X
     - name: os.platform

scripts/generators/beats.py

@@ -36,7 +36,7 @@ def fieldset_field_array(source_fields, df_whitelist):
                     'ignore_above', 'multi_fields', 'format', 'input_format',
                     'output_format', 'output_precision', 'description',
                     'example']
-    multi_fields_allowed_keys = ['name', 'type', 'norms']
+    multi_fields_allowed_keys = ['name', 'type', 'norms', 'default_field']
 
     fields = []
     for nested_field_name in source_fields:
@@ -46,6 +46,10 @@ def fieldset_field_array(source_fields, df_whitelist):
         cleaned_multi_fields = []
         if 'multi_fields' in ecs_field:
             for mf in ecs_field['multi_fields']:
+                # Set default_field if necessary. Avoid adding the key if the parent
+                # field already is marked with default_field: false.
+                if not mf['flat_name'] in df_whitelist and ecs_field['flat_name'] in df_whitelist:
+                    mf['default_field'] = False
                 cleaned_multi_fields.append(
                     ecs_helpers.dict_copy_keys_ordered(mf, multi_fields_allowed_keys))
             beats_field['multi_fields'] = cleaned_multi_fields

scripts/generators/beats_default_fields_whitelist.yml

@@ -392,6 +392,7 @@ user.name: null
 user_agent.device.name: null
 user_agent.name: null
 user_agent.original: null
+user_agent.original.text: null
 user_agent.os.family: null
 user_agent.os.full: null
 user_agent.os.kernel: null