From 5e5b7bb227253aeba1140b0e43d63f926f40c8a0 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 7 Mar 2019 11:05:59 -0500 Subject: [PATCH 01/11] Sanity check for the presence of reusable fields in flat fields --- scripts/tests/test_ecs_spec.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/scripts/tests/test_ecs_spec.py b/scripts/tests/test_ecs_spec.py index 8fa27f910d..46fc93b177 100644 --- a/scripts/tests/test_ecs_spec.py +++ b/scripts/tests/test_ecs_spec.py @@ -1,12 +1,14 @@ import unittest from scripts import schema_reader +(nested, flat) = schema_reader.load_ecs() class TestEcsSpec(unittest.TestCase): """Sanity check for things that should be true in the ECS spec.""" def setUp(self): - (nested, flat) = schema_reader.load_ecs() + global nested + global flat self.ecs_nested = nested self.ecs_fields = flat @@ -16,6 +18,30 @@ def test_base_flat_name(self): self.ecs_nested['base']['fields']['@timestamp']['flat_name'], '@timestamp') + def test_flat_includes_reusable_fields(self): + all_keys = sorted(self.ecs_fields.keys()) + + # geo + self.assertIn('client.geo.name', all_keys) + self.assertIn('destination.geo.name', all_keys) + self.assertIn('host.geo.name', all_keys) + self.assertIn('observer.geo.name', all_keys) + self.assertIn('server.geo.name', all_keys) + self.assertIn('source.geo.name', all_keys) + + # group + self.assertIn('user.group.name', all_keys) + + # user + self.assertIn('client.user.id', all_keys) + self.assertIn('destination.user.id', all_keys) + self.assertIn('server.user.id', all_keys) + self.assertIn('source.user.id', all_keys) + + # os + self.assertIn('host.os.name', all_keys) + self.assertIn('observer.os.name', all_keys) + self.assertIn('user_agent.os.name', all_keys) if __name__ == '__main__': unittest.main() From a77b7dbf3176548f016dcca1892d7d68ea805af5 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 7 Mar 2019 11:34:18 -0500 Subject: [PATCH 02/11] Represent reusable fields in the nested structure as well... This also fixes a bug where the `group` fieldset was not actually being nested in all places where user is nested (e.g. `source.user.group.*`). --- scripts/schema_reader.py | 21 +++++++++++++---- scripts/tests/test_ecs_spec.py | 42 ++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 4 deletions(-) diff --git a/scripts/schema_reader.py b/scripts/schema_reader.py index a2127ca3c0..960f03c162 100644 --- a/scripts/schema_reader.py +++ b/scripts/schema_reader.py @@ -109,8 +109,12 @@ def field_set_multi_field_defaults(parent_field): mf['flat_name'] = parent_field['flat_name'] + '.' + mf['name'] -def duplicate_reusable_fieldsets(schema, fields_flat): - """Copies reusable field definitions to their expected places in the flattened schema only""" +def duplicate_reusable_fieldsets(schema, fields_flat, fields_nested): + """Copies reusable field definitions to their expected places""" + # Note: across this schema reader, functions are modifying dictionaries passed + # as arguments, which is usually a risk of unintended side effects. + # Here it simplifies the nesting of 'group' under 'user', + # which is in turn reusable in a few places. if 'reusable' in schema: for new_nesting in schema['reusable']['expected']: @@ -123,14 +127,19 @@ def duplicate_reusable_fieldsets(schema, fields_flat): destination_name = new_nesting + '.' + field['flat_name'] copied_field['flat_name'] = destination_name copied_field['original_fieldset'] = schema['name'] + fields_flat[destination_name] = copied_field + # Nested: use original flat name under the destination fieldset + fields_nested[new_nesting]['fields'][field['flat_name']] = copied_field + # Main def finalize_schemas(fields_nested, fields_flat): for schema_name in fields_nested: schema = fields_nested[schema_name] + schema_cleanup_values(schema) for (name, field) in schema['fields'].items(): @@ -138,8 +147,12 @@ def finalize_schemas(fields_nested, fields_flat): fields_flat[field['flat_name']] = field - # TODO duplicate in nested too? - duplicate_reusable_fieldsets(schema, fields_flat) + # This happens as a second pass, so that all fieldsets have their + # fields array replaced with a fields dictionary. + for schema_name in fields_nested: + schema = fields_nested[schema_name] + + duplicate_reusable_fieldsets(schema, fields_flat, fields_nested) def load_ecs(): diff --git a/scripts/tests/test_ecs_spec.py b/scripts/tests/test_ecs_spec.py index 46fc93b177..b69e5e11fe 100644 --- a/scripts/tests/test_ecs_spec.py +++ b/scripts/tests/test_ecs_spec.py @@ -3,6 +3,7 @@ (nested, flat) = schema_reader.load_ecs() + class TestEcsSpec(unittest.TestCase): """Sanity check for things that should be true in the ECS spec.""" @@ -31,6 +32,10 @@ def test_flat_includes_reusable_fields(self): # group self.assertIn('user.group.name', all_keys) + self.assertIn('client.user.group.id', all_keys) + self.assertIn('destination.user.group.id', all_keys) + self.assertIn('server.user.group.id', all_keys) + self.assertIn('source.user.group.id', all_keys) # user self.assertIn('client.user.id', all_keys) @@ -43,5 +48,42 @@ def test_flat_includes_reusable_fields(self): self.assertIn('observer.os.name', all_keys) self.assertIn('user_agent.os.name', all_keys) + def test_nested_includes_reusable_fields(self): + client_keys = sorted(self.ecs_nested['client']['fields'].keys()) + destination_keys = sorted(self.ecs_nested['destination']['fields'].keys()) + host_keys = sorted(self.ecs_nested['host']['fields'].keys()) + observer_keys = sorted(self.ecs_nested['observer']['fields'].keys()) + server_keys = sorted(self.ecs_nested['server']['fields'].keys()) + source_keys = sorted(self.ecs_nested['source']['fields'].keys()) + user_agent_keys = sorted(self.ecs_nested['user_agent']['fields'].keys()) + user_keys = sorted(self.ecs_nested['user']['fields'].keys()) + + # geo + self.assertIn('geo.name', client_keys) + self.assertIn('geo.name', destination_keys) + self.assertIn('geo.name', host_keys) + self.assertIn('geo.name', observer_keys) + self.assertIn('geo.name', server_keys) + self.assertIn('geo.name', source_keys) + + # group + self.assertIn('group.name', user_keys) + self.assertIn('user.group.id', client_keys) + self.assertIn('user.group.id', destination_keys) + self.assertIn('user.group.id', server_keys) + self.assertIn('user.group.id', source_keys) + + # user + self.assertIn('user.id', client_keys) + self.assertIn('user.id', destination_keys) + self.assertIn('user.id', server_keys) + self.assertIn('user.id', source_keys) + + # os + self.assertIn('os.name', host_keys) + self.assertIn('os.name', observer_keys) + self.assertIn('os.name', user_agent_keys) + + if __name__ == '__main__': unittest.main() From 5f1dad69addfbe36f9ca9212a1c3aa6e08747ceb Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 7 Mar 2019 11:35:38 -0500 Subject: [PATCH 03/11] Generated files from previous few commits --- generated/csv/fields.csv | 10 + generated/ecs/fields_flat.yml | 90 ++ generated/ecs/fields_nested.yml | 1161 +++++++++++++++++++++-- generated/elasticsearch/6/template.json | 60 ++ generated/elasticsearch/7/template.json | 60 ++ 5 files changed, 1327 insertions(+), 54 deletions(-) diff --git a/generated/csv/fields.csv b/generated/csv/fields.csv index bd543b44fb..c955eced51 100644 --- a/generated/csv/fields.csv +++ b/generated/csv/fields.csv @@ -25,6 +25,8 @@ client.packets,long,core,12,1.1.0-dev client.port,long,core,,1.1.0-dev client.user.email,keyword,extended,,1.1.0-dev client.user.full_name,keyword,extended,Albert Einstein,1.1.0-dev +client.user.group.id,keyword,extended,,1.1.0-dev +client.user.group.name,keyword,extended,,1.1.0-dev client.user.hash,keyword,extended,,1.1.0-dev client.user.id,keyword,core,,1.1.0-dev client.user.name,keyword,core,albert,1.1.0-dev @@ -58,6 +60,8 @@ destination.packets,long,core,12,1.1.0-dev destination.port,long,core,,1.1.0-dev destination.user.email,keyword,extended,,1.1.0-dev destination.user.full_name,keyword,extended,Albert Einstein,1.1.0-dev +destination.user.group.id,keyword,extended,,1.1.0-dev +destination.user.group.name,keyword,extended,,1.1.0-dev destination.user.hash,keyword,extended,,1.1.0-dev destination.user.id,keyword,core,,1.1.0-dev destination.user.name,keyword,core,albert,1.1.0-dev @@ -130,6 +134,8 @@ host.os.version,keyword,extended,10.14.1,1.1.0-dev host.type,keyword,core,,1.1.0-dev host.user.email,keyword,extended,,1.1.0-dev host.user.full_name,keyword,extended,Albert Einstein,1.1.0-dev +host.user.group.id,keyword,extended,,1.1.0-dev +host.user.group.name,keyword,extended,,1.1.0-dev host.user.hash,keyword,extended,,1.1.0-dev host.user.id,keyword,core,,1.1.0-dev host.user.name,keyword,core,albert,1.1.0-dev @@ -212,6 +218,8 @@ server.packets,long,core,12,1.1.0-dev server.port,long,core,,1.1.0-dev server.user.email,keyword,extended,,1.1.0-dev server.user.full_name,keyword,extended,Albert Einstein,1.1.0-dev +server.user.group.id,keyword,extended,,1.1.0-dev +server.user.group.name,keyword,extended,,1.1.0-dev server.user.hash,keyword,extended,,1.1.0-dev server.user.id,keyword,core,,1.1.0-dev server.user.name,keyword,core,albert,1.1.0-dev @@ -238,6 +246,8 @@ source.packets,long,core,12,1.1.0-dev source.port,long,core,,1.1.0-dev source.user.email,keyword,extended,,1.1.0-dev source.user.full_name,keyword,extended,Albert Einstein,1.1.0-dev +source.user.group.id,keyword,extended,,1.1.0-dev +source.user.group.name,keyword,extended,,1.1.0-dev source.user.hash,keyword,extended,,1.1.0-dev source.user.id,keyword,core,,1.1.0-dev source.user.name,keyword,core,albert,1.1.0-dev diff --git a/generated/ecs/fields_flat.yml b/generated/ecs/fields_flat.yml index d82ee97028..905d06d9a5 100644 --- a/generated/ecs/fields_flat.yml +++ b/generated/ecs/fields_flat.yml @@ -239,6 +239,24 @@ client.user.full_name: original_fieldset: user short: User's full name, if available. type: keyword +client.user.group.id: + description: Unique identifier for the group on the system/platform. + flat_name: client.user.group.id + ignore_above: 1024 + level: extended + name: id + original_fieldset: user + short: Unique identifier for the group on the system/platform. + type: keyword +client.user.group.name: + description: Name of the group. + flat_name: client.user.group.name + ignore_above: 1024 + level: extended + name: name + original_fieldset: user + short: Name of the group. + type: keyword client.user.hash: description: 'Unique user hash to correlate information for a user in anonymized form. @@ -551,6 +569,24 @@ destination.user.full_name: original_fieldset: user short: User's full name, if available. type: keyword +destination.user.group.id: + description: Unique identifier for the group on the system/platform. + flat_name: destination.user.group.id + ignore_above: 1024 + level: extended + name: id + original_fieldset: user + short: Unique identifier for the group on the system/platform. + type: keyword +destination.user.group.name: + description: Name of the group. + flat_name: destination.user.group.name + ignore_above: 1024 + level: extended + name: name + original_fieldset: user + short: Name of the group. + type: keyword destination.user.hash: description: 'Unique user hash to correlate information for a user in anonymized form. @@ -1275,6 +1311,24 @@ host.user.full_name: original_fieldset: user short: User's full name, if available. type: keyword +host.user.group.id: + description: Unique identifier for the group on the system/platform. + flat_name: host.user.group.id + ignore_above: 1024 + level: extended + name: id + original_fieldset: user + short: Unique identifier for the group on the system/platform. + type: keyword +host.user.group.name: + description: Name of the group. + flat_name: host.user.group.name + ignore_above: 1024 + level: extended + name: name + original_fieldset: user + short: Name of the group. + type: keyword host.user.hash: description: 'Unique user hash to correlate information for a user in anonymized form. @@ -2118,6 +2172,24 @@ server.user.full_name: original_fieldset: user short: User's full name, if available. type: keyword +server.user.group.id: + description: Unique identifier for the group on the system/platform. + flat_name: server.user.group.id + ignore_above: 1024 + level: extended + name: id + original_fieldset: user + short: Unique identifier for the group on the system/platform. + type: keyword +server.user.group.name: + description: Name of the group. + flat_name: server.user.group.name + ignore_above: 1024 + level: extended + name: name + original_fieldset: user + short: Name of the group. + type: keyword server.user.hash: description: 'Unique user hash to correlate information for a user in anonymized form. @@ -2395,6 +2467,24 @@ source.user.full_name: original_fieldset: user short: User's full name, if available. type: keyword +source.user.group.id: + description: Unique identifier for the group on the system/platform. + flat_name: source.user.group.id + ignore_above: 1024 + level: extended + name: id + original_fieldset: user + short: Unique identifier for the group on the system/platform. + type: keyword +source.user.group.name: + description: Name of the group. + flat_name: source.user.group.name + ignore_above: 1024 + level: extended + name: name + original_fieldset: user + short: Name of the group. + type: keyword source.user.hash: description: 'Unique user hash to correlate information for a user in anonymized form. diff --git a/generated/ecs/fields_nested.yml b/generated/ecs/fields_nested.yml index ff0c0add8e..92331173d5 100644 --- a/generated/ecs/fields_nested.yml +++ b/generated/ecs/fields_nested.yml @@ -190,6 +190,91 @@ client: name: domain short: Client domain. type: keyword + geo.city_name: + description: City name. + example: Montreal + flat_name: client.geo.city_name + ignore_above: 1024 + level: core + name: city_name + original_fieldset: geo + short: City name. + type: keyword + geo.continent_name: + description: Name of the continent. + example: North America + flat_name: client.geo.continent_name + ignore_above: 1024 + level: core + name: continent_name + original_fieldset: geo + short: Name of the continent. + type: keyword + geo.country_iso_code: + description: Country ISO code. + example: CA + flat_name: client.geo.country_iso_code + ignore_above: 1024 + level: core + name: country_iso_code + original_fieldset: geo + short: Country ISO code. + type: keyword + geo.country_name: + description: Country name. + example: Canada + flat_name: client.geo.country_name + ignore_above: 1024 + level: core + name: country_name + original_fieldset: geo + short: Country name. + type: keyword + geo.location: + description: Longitude and latitude. + example: '{ "lon": -73.614830, "lat": 45.505918 }' + flat_name: client.geo.location + level: core + name: location + original_fieldset: geo + short: Longitude and latitude. + type: geo_point + geo.name: + description: 'User-defined description of a location, at the level of granularity + they care about. + + Could be the name of their data centers, the floor number, if this describes + a local physical entity, city names. + + Not typically used in automated geolocation.' + example: boston-dc + flat_name: client.geo.name + ignore_above: 1024 + level: extended + name: name + original_fieldset: geo + short: User-defined description of a location. + type: keyword + geo.region_iso_code: + description: Region ISO code. + example: CA-QC + flat_name: client.geo.region_iso_code + ignore_above: 1024 + level: core + name: region_iso_code + original_fieldset: geo + short: Region ISO code. + type: keyword + geo.region_name: + description: Region name. + example: Quebec + flat_name: client.geo.region_name + ignore_above: 1024 + level: core + name: region_name + original_fieldset: geo + short: Region name. + type: keyword ip: description: 'IP address of the client. @@ -222,6 +307,75 @@ client: name: port short: Port of the client. type: long + user.email: + description: User email address. + flat_name: client.user.email + ignore_above: 1024 + level: extended + name: email + original_fieldset: user + short: User email address. + type: keyword + user.full_name: + description: User's full name, if available. + example: Albert Einstein + flat_name: client.user.full_name + ignore_above: 1024 + level: extended + name: full_name + original_fieldset: user + short: User's full name, if available. + type: keyword + user.group.id: + description: Unique identifier for the group on the system/platform. + flat_name: client.user.group.id + ignore_above: 1024 + level: extended + name: id + original_fieldset: user + short: Unique identifier for the group on the system/platform. + type: keyword + user.group.name: + description: Name of the group. + flat_name: client.user.group.name + ignore_above: 1024 + level: extended + name: name + original_fieldset: user + short: Name of the group. + type: keyword + user.hash: + description: 'Unique user hash to correlate information for a user in anonymized + form. + + Useful if `user.id` or `user.name` contain confidential information and cannot + be used.' + flat_name: client.user.hash + ignore_above: 1024 + level: extended + name: hash + original_fieldset: user + short: Unique user hash to correlate information for a user in anonymized form. + type: keyword + user.id: + description: One or multiple unique identifiers of the user. + flat_name: client.user.id + ignore_above: 1024 + level: core + name: id + original_fieldset: user + short: One or multiple unique identifiers of the user. + type: keyword + user.name: + description: Short name or login of the user. + example: albert + flat_name: client.user.name + ignore_above: 1024 + level: core + name: name + original_fieldset: user + short: Short name or login of the user. + type: keyword group: 2 name: client prefix: client. @@ -404,6 +558,91 @@ destination: name: domain short: Destination domain. type: keyword + geo.city_name: + description: City name. + example: Montreal + flat_name: destination.geo.city_name + ignore_above: 1024 + level: core + name: city_name + original_fieldset: geo + short: City name. + type: keyword + geo.continent_name: + description: Name of the continent. + example: North America + flat_name: destination.geo.continent_name + ignore_above: 1024 + level: core + name: continent_name + original_fieldset: geo + short: Name of the continent. + type: keyword + geo.country_iso_code: + description: Country ISO code. + example: CA + flat_name: destination.geo.country_iso_code + ignore_above: 1024 + level: core + name: country_iso_code + original_fieldset: geo + short: Country ISO code. + type: keyword + geo.country_name: + description: Country name. + example: Canada + flat_name: destination.geo.country_name + ignore_above: 1024 + level: core + name: country_name + original_fieldset: geo + short: Country name. + type: keyword + geo.location: + description: Longitude and latitude. + example: '{ "lon": -73.614830, "lat": 45.505918 }' + flat_name: destination.geo.location + level: core + name: location + original_fieldset: geo + short: Longitude and latitude. + type: geo_point + geo.name: + description: 'User-defined description of a location, at the level of granularity + they care about. + + Could be the name of their data centers, the floor number, if this describes + a local physical entity, city names. + + Not typically used in automated geolocation.' + example: boston-dc + flat_name: destination.geo.name + ignore_above: 1024 + level: extended + name: name + original_fieldset: geo + short: User-defined description of a location. + type: keyword + geo.region_iso_code: + description: Region ISO code. + example: CA-QC + flat_name: destination.geo.region_iso_code + ignore_above: 1024 + level: core + name: region_iso_code + original_fieldset: geo + short: Region ISO code. + type: keyword + geo.region_name: + description: Region name. + example: Quebec + flat_name: destination.geo.region_name + ignore_above: 1024 + level: core + name: region_name + original_fieldset: geo + short: Region name. + type: keyword ip: description: 'IP address of the destination. @@ -436,6 +675,75 @@ destination: name: port short: Port of the destination. type: long + user.email: + description: User email address. + flat_name: destination.user.email + ignore_above: 1024 + level: extended + name: email + original_fieldset: user + short: User email address. + type: keyword + user.full_name: + description: User's full name, if available. + example: Albert Einstein + flat_name: destination.user.full_name + ignore_above: 1024 + level: extended + name: full_name + original_fieldset: user + short: User's full name, if available. + type: keyword + user.group.id: + description: Unique identifier for the group on the system/platform. + flat_name: destination.user.group.id + ignore_above: 1024 + level: extended + name: id + original_fieldset: user + short: Unique identifier for the group on the system/platform. + type: keyword + user.group.name: + description: Name of the group. + flat_name: destination.user.group.name + ignore_above: 1024 + level: extended + name: name + original_fieldset: user + short: Name of the group. + type: keyword + user.hash: + description: 'Unique user hash to correlate information for a user in anonymized + form. + + Useful if `user.id` or `user.name` contain confidential information and cannot + be used.' + flat_name: destination.user.hash + ignore_above: 1024 + level: extended + name: hash + original_fieldset: user + short: Unique user hash to correlate information for a user in anonymized form. + type: keyword + user.id: + description: One or multiple unique identifiers of the user. + flat_name: destination.user.id + ignore_above: 1024 + level: core + name: id + original_fieldset: user + short: One or multiple unique identifiers of the user. + type: keyword + user.name: + description: Short name or login of the user. + example: albert + flat_name: destination.user.name + ignore_above: 1024 + level: core + name: name + original_fieldset: user + short: Short name or login of the user. + type: keyword group: 2 name: destination prefix: destination. @@ -1008,42 +1316,127 @@ host: name: architecture short: Operating system architecture. type: keyword - hostname: - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - flat_name: host.hostname + geo.city_name: + description: City name. + example: Montreal + flat_name: host.geo.city_name ignore_above: 1024 level: core - name: hostname - short: Hostname of the host. + name: city_name + original_fieldset: geo + short: City name. type: keyword - id: - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - flat_name: host.id + geo.continent_name: + description: Name of the continent. + example: North America + flat_name: host.geo.continent_name ignore_above: 1024 level: core - name: id - short: Unique host id. + name: continent_name + original_fieldset: geo + short: Name of the continent. type: keyword - ip: - description: Host ip address. - flat_name: host.ip - level: core - name: ip - short: Host ip address. - type: ip - mac: - description: Host mac address. - flat_name: host.mac + geo.country_iso_code: + description: Country ISO code. + example: CA + flat_name: host.geo.country_iso_code ignore_above: 1024 level: core - name: mac - short: Host mac address. + name: country_iso_code + original_fieldset: geo + short: Country ISO code. + type: keyword + geo.country_name: + description: Country name. + example: Canada + flat_name: host.geo.country_name + ignore_above: 1024 + level: core + name: country_name + original_fieldset: geo + short: Country name. + type: keyword + geo.location: + description: Longitude and latitude. + example: '{ "lon": -73.614830, "lat": 45.505918 }' + flat_name: host.geo.location + level: core + name: location + original_fieldset: geo + short: Longitude and latitude. + type: geo_point + geo.name: + description: 'User-defined description of a location, at the level of granularity + they care about. + + Could be the name of their data centers, the floor number, if this describes + a local physical entity, city names. + + Not typically used in automated geolocation.' + example: boston-dc + flat_name: host.geo.name + ignore_above: 1024 + level: extended + name: name + original_fieldset: geo + short: User-defined description of a location. + type: keyword + geo.region_iso_code: + description: Region ISO code. + example: CA-QC + flat_name: host.geo.region_iso_code + ignore_above: 1024 + level: core + name: region_iso_code + original_fieldset: geo + short: Region ISO code. + type: keyword + geo.region_name: + description: Region name. + example: Quebec + flat_name: host.geo.region_name + ignore_above: 1024 + level: core + name: region_name + original_fieldset: geo + short: Region name. + type: keyword + hostname: + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + flat_name: host.hostname + ignore_above: 1024 + level: core + name: hostname + short: Hostname of the host. + type: keyword + id: + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + flat_name: host.id + ignore_above: 1024 + level: core + name: id + short: Unique host id. + type: keyword + ip: + description: Host ip address. + flat_name: host.ip + level: core + name: ip + short: Host ip address. + type: ip + mac: + description: Host mac address. + flat_name: host.mac + ignore_above: 1024 + level: core + name: mac + short: Host mac address. type: keyword name: description: 'Name of the host. @@ -1057,6 +1450,66 @@ host: name: name short: Name of the host. type: keyword + os.family: + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + flat_name: host.os.family + ignore_above: 1024 + level: extended + name: family + original_fieldset: os + short: OS family (such as redhat, debian, freebsd, windows). + type: keyword + os.full: + description: Operating system name, including the version or code name. + example: Mac OS Mojave + flat_name: host.os.full + ignore_above: 1024 + level: extended + name: full + original_fieldset: os + short: Operating system name, including the version or code name. + type: keyword + os.kernel: + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + flat_name: host.os.kernel + ignore_above: 1024 + level: extended + name: kernel + original_fieldset: os + short: Operating system kernel version as a raw string. + type: keyword + os.name: + description: Operating system name, without the version. + example: Mac OS X + flat_name: host.os.name + ignore_above: 1024 + level: extended + name: name + original_fieldset: os + short: Operating system name, without the version. + type: keyword + os.platform: + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + flat_name: host.os.platform + ignore_above: 1024 + level: extended + name: platform + original_fieldset: os + short: Operating system platform (such centos, ubuntu, windows). + type: keyword + os.version: + description: Operating system version as a raw string. + example: 10.14.1 + flat_name: host.os.version + ignore_above: 1024 + level: extended + name: version + original_fieldset: os + short: Operating system version as a raw string. + type: keyword type: description: 'Type of host. @@ -1069,6 +1522,75 @@ host: name: type short: Type of host. type: keyword + user.email: + description: User email address. + flat_name: host.user.email + ignore_above: 1024 + level: extended + name: email + original_fieldset: user + short: User email address. + type: keyword + user.full_name: + description: User's full name, if available. + example: Albert Einstein + flat_name: host.user.full_name + ignore_above: 1024 + level: extended + name: full_name + original_fieldset: user + short: User's full name, if available. + type: keyword + user.group.id: + description: Unique identifier for the group on the system/platform. + flat_name: host.user.group.id + ignore_above: 1024 + level: extended + name: id + original_fieldset: user + short: Unique identifier for the group on the system/platform. + type: keyword + user.group.name: + description: Name of the group. + flat_name: host.user.group.name + ignore_above: 1024 + level: extended + name: name + original_fieldset: user + short: Name of the group. + type: keyword + user.hash: + description: 'Unique user hash to correlate information for a user in anonymized + form. + + Useful if `user.id` or `user.name` contain confidential information and cannot + be used.' + flat_name: host.user.hash + ignore_above: 1024 + level: extended + name: hash + original_fieldset: user + short: Unique user hash to correlate information for a user in anonymized form. + type: keyword + user.id: + description: One or multiple unique identifiers of the user. + flat_name: host.user.id + ignore_above: 1024 + level: core + name: id + original_fieldset: user + short: One or multiple unique identifiers of the user. + type: keyword + user.name: + description: Short name or login of the user. + example: albert + flat_name: host.user.name + ignore_above: 1024 + level: core + name: name + original_fieldset: user + short: Short name or login of the user. + type: keyword group: 2 name: host prefix: host. @@ -1371,6 +1893,91 @@ observer: and ETL components used in processing events or metrics are not considered observers in ECS.' fields: + geo.city_name: + description: City name. + example: Montreal + flat_name: observer.geo.city_name + ignore_above: 1024 + level: core + name: city_name + original_fieldset: geo + short: City name. + type: keyword + geo.continent_name: + description: Name of the continent. + example: North America + flat_name: observer.geo.continent_name + ignore_above: 1024 + level: core + name: continent_name + original_fieldset: geo + short: Name of the continent. + type: keyword + geo.country_iso_code: + description: Country ISO code. + example: CA + flat_name: observer.geo.country_iso_code + ignore_above: 1024 + level: core + name: country_iso_code + original_fieldset: geo + short: Country ISO code. + type: keyword + geo.country_name: + description: Country name. + example: Canada + flat_name: observer.geo.country_name + ignore_above: 1024 + level: core + name: country_name + original_fieldset: geo + short: Country name. + type: keyword + geo.location: + description: Longitude and latitude. + example: '{ "lon": -73.614830, "lat": 45.505918 }' + flat_name: observer.geo.location + level: core + name: location + original_fieldset: geo + short: Longitude and latitude. + type: geo_point + geo.name: + description: 'User-defined description of a location, at the level of granularity + they care about. + + Could be the name of their data centers, the floor number, if this describes + a local physical entity, city names. + + Not typically used in automated geolocation.' + example: boston-dc + flat_name: observer.geo.name + ignore_above: 1024 + level: extended + name: name + original_fieldset: geo + short: User-defined description of a location. + type: keyword + geo.region_iso_code: + description: Region ISO code. + example: CA-QC + flat_name: observer.geo.region_iso_code + ignore_above: 1024 + level: core + name: region_iso_code + original_fieldset: geo + short: Region ISO code. + type: keyword + geo.region_name: + description: Region name. + example: Quebec + flat_name: observer.geo.region_name + ignore_above: 1024 + level: core + name: region_name + original_fieldset: geo + short: Region name. + type: keyword hostname: description: Hostname of the observer. flat_name: observer.hostname @@ -1394,6 +2001,66 @@ observer: name: mac short: MAC address of the observer type: keyword + os.family: + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + flat_name: observer.os.family + ignore_above: 1024 + level: extended + name: family + original_fieldset: os + short: OS family (such as redhat, debian, freebsd, windows). + type: keyword + os.full: + description: Operating system name, including the version or code name. + example: Mac OS Mojave + flat_name: observer.os.full + ignore_above: 1024 + level: extended + name: full + original_fieldset: os + short: Operating system name, including the version or code name. + type: keyword + os.kernel: + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + flat_name: observer.os.kernel + ignore_above: 1024 + level: extended + name: kernel + original_fieldset: os + short: Operating system kernel version as a raw string. + type: keyword + os.name: + description: Operating system name, without the version. + example: Mac OS X + flat_name: observer.os.name + ignore_above: 1024 + level: extended + name: name + original_fieldset: os + short: Operating system name, without the version. + type: keyword + os.platform: + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + flat_name: observer.os.platform + ignore_above: 1024 + level: extended + name: platform + original_fieldset: os + short: Operating system platform (such centos, ubuntu, windows). + type: keyword + os.version: + description: Operating system version as a raw string. + example: 10.14.1 + flat_name: observer.os.version + ignore_above: 1024 + level: extended + name: version + original_fieldset: os + short: Operating system version as a raw string. + type: keyword serial_number: description: Observer serial number. flat_name: observer.serial_number @@ -1670,39 +2337,124 @@ server: in conjunction with client fields. Server fields are generally not populated for packet-level events. - Client / server representations can add semantic context to an exchange, which - is helpful to visualize the data in certain situations. If your context falls - in that category, you should still ensure that source and destination are filled - appropriately.' - fields: - address: - description: 'Some event server addresses are defined ambiguously. The event - will sometimes list an IP, a domain or a unix socket. You should always store - the raw address in the `.address` field. + Client / server representations can add semantic context to an exchange, which + is helpful to visualize the data in certain situations. If your context falls + in that category, you should still ensure that source and destination are filled + appropriately.' + fields: + address: + description: 'Some event server addresses are defined ambiguously. The event + will sometimes list an IP, a domain or a unix socket. You should always store + the raw address in the `.address` field. + + Then it should be duplicated to `.ip` or `.domain`, depending on which one + it is.' + flat_name: server.address + ignore_above: 1024 + level: extended + name: address + short: Server network address. + type: keyword + bytes: + description: Bytes sent from the server to the client. + example: 184 + flat_name: server.bytes + level: core + name: bytes + short: Bytes sent from the server to the client. + type: long + domain: + description: Server domain. + flat_name: server.domain + ignore_above: 1024 + level: core + name: domain + short: Server domain. + type: keyword + geo.city_name: + description: City name. + example: Montreal + flat_name: server.geo.city_name + ignore_above: 1024 + level: core + name: city_name + original_fieldset: geo + short: City name. + type: keyword + geo.continent_name: + description: Name of the continent. + example: North America + flat_name: server.geo.continent_name + ignore_above: 1024 + level: core + name: continent_name + original_fieldset: geo + short: Name of the continent. + type: keyword + geo.country_iso_code: + description: Country ISO code. + example: CA + flat_name: server.geo.country_iso_code + ignore_above: 1024 + level: core + name: country_iso_code + original_fieldset: geo + short: Country ISO code. + type: keyword + geo.country_name: + description: Country name. + example: Canada + flat_name: server.geo.country_name + ignore_above: 1024 + level: core + name: country_name + original_fieldset: geo + short: Country name. + type: keyword + geo.location: + description: Longitude and latitude. + example: '{ "lon": -73.614830, "lat": 45.505918 }' + flat_name: server.geo.location + level: core + name: location + original_fieldset: geo + short: Longitude and latitude. + type: geo_point + geo.name: + description: 'User-defined description of a location, at the level of granularity + they care about. + + Could be the name of their data centers, the floor number, if this describes + a local physical entity, city names. - Then it should be duplicated to `.ip` or `.domain`, depending on which one - it is.' - flat_name: server.address + Not typically used in automated geolocation.' + example: boston-dc + flat_name: server.geo.name ignore_above: 1024 level: extended - name: address - short: Server network address. + name: name + original_fieldset: geo + short: User-defined description of a location. type: keyword - bytes: - description: Bytes sent from the server to the client. - example: 184 - flat_name: server.bytes + geo.region_iso_code: + description: Region ISO code. + example: CA-QC + flat_name: server.geo.region_iso_code + ignore_above: 1024 level: core - name: bytes - short: Bytes sent from the server to the client. - type: long - domain: - description: Server domain. - flat_name: server.domain + name: region_iso_code + original_fieldset: geo + short: Region ISO code. + type: keyword + geo.region_name: + description: Region name. + example: Quebec + flat_name: server.geo.region_name ignore_above: 1024 level: core - name: domain - short: Server domain. + name: region_name + original_fieldset: geo + short: Region name. type: keyword ip: description: 'IP address of the server. @@ -1736,6 +2488,75 @@ server: name: port short: Port of the server. type: long + user.email: + description: User email address. + flat_name: server.user.email + ignore_above: 1024 + level: extended + name: email + original_fieldset: user + short: User email address. + type: keyword + user.full_name: + description: User's full name, if available. + example: Albert Einstein + flat_name: server.user.full_name + ignore_above: 1024 + level: extended + name: full_name + original_fieldset: user + short: User's full name, if available. + type: keyword + user.group.id: + description: Unique identifier for the group on the system/platform. + flat_name: server.user.group.id + ignore_above: 1024 + level: extended + name: id + original_fieldset: user + short: Unique identifier for the group on the system/platform. + type: keyword + user.group.name: + description: Name of the group. + flat_name: server.user.group.name + ignore_above: 1024 + level: extended + name: name + original_fieldset: user + short: Name of the group. + type: keyword + user.hash: + description: 'Unique user hash to correlate information for a user in anonymized + form. + + Useful if `user.id` or `user.name` contain confidential information and cannot + be used.' + flat_name: server.user.hash + ignore_above: 1024 + level: extended + name: hash + original_fieldset: user + short: Unique user hash to correlate information for a user in anonymized form. + type: keyword + user.id: + description: One or multiple unique identifiers of the user. + flat_name: server.user.id + ignore_above: 1024 + level: core + name: id + original_fieldset: user + short: One or multiple unique identifiers of the user. + type: keyword + user.name: + description: Short name or login of the user. + example: albert + flat_name: server.user.name + ignore_above: 1024 + level: core + name: name + original_fieldset: user + short: Short name or login of the user. + type: keyword group: 2 name: server prefix: server. @@ -1868,6 +2689,91 @@ source: name: domain short: Source domain. type: keyword + geo.city_name: + description: City name. + example: Montreal + flat_name: source.geo.city_name + ignore_above: 1024 + level: core + name: city_name + original_fieldset: geo + short: City name. + type: keyword + geo.continent_name: + description: Name of the continent. + example: North America + flat_name: source.geo.continent_name + ignore_above: 1024 + level: core + name: continent_name + original_fieldset: geo + short: Name of the continent. + type: keyword + geo.country_iso_code: + description: Country ISO code. + example: CA + flat_name: source.geo.country_iso_code + ignore_above: 1024 + level: core + name: country_iso_code + original_fieldset: geo + short: Country ISO code. + type: keyword + geo.country_name: + description: Country name. + example: Canada + flat_name: source.geo.country_name + ignore_above: 1024 + level: core + name: country_name + original_fieldset: geo + short: Country name. + type: keyword + geo.location: + description: Longitude and latitude. + example: '{ "lon": -73.614830, "lat": 45.505918 }' + flat_name: source.geo.location + level: core + name: location + original_fieldset: geo + short: Longitude and latitude. + type: geo_point + geo.name: + description: 'User-defined description of a location, at the level of granularity + they care about. + + Could be the name of their data centers, the floor number, if this describes + a local physical entity, city names. + + Not typically used in automated geolocation.' + example: boston-dc + flat_name: source.geo.name + ignore_above: 1024 + level: extended + name: name + original_fieldset: geo + short: User-defined description of a location. + type: keyword + geo.region_iso_code: + description: Region ISO code. + example: CA-QC + flat_name: source.geo.region_iso_code + ignore_above: 1024 + level: core + name: region_iso_code + original_fieldset: geo + short: Region ISO code. + type: keyword + geo.region_name: + description: Region name. + example: Quebec + flat_name: source.geo.region_name + ignore_above: 1024 + level: core + name: region_name + original_fieldset: geo + short: Region name. + type: keyword ip: description: 'IP address of the source. @@ -1900,6 +2806,75 @@ source: name: port short: Port of the source. type: long + user.email: + description: User email address. + flat_name: source.user.email + ignore_above: 1024 + level: extended + name: email + original_fieldset: user + short: User email address. + type: keyword + user.full_name: + description: User's full name, if available. + example: Albert Einstein + flat_name: source.user.full_name + ignore_above: 1024 + level: extended + name: full_name + original_fieldset: user + short: User's full name, if available. + type: keyword + user.group.id: + description: Unique identifier for the group on the system/platform. + flat_name: source.user.group.id + ignore_above: 1024 + level: extended + name: id + original_fieldset: user + short: Unique identifier for the group on the system/platform. + type: keyword + user.group.name: + description: Name of the group. + flat_name: source.user.group.name + ignore_above: 1024 + level: extended + name: name + original_fieldset: user + short: Name of the group. + type: keyword + user.hash: + description: 'Unique user hash to correlate information for a user in anonymized + form. + + Useful if `user.id` or `user.name` contain confidential information and cannot + be used.' + flat_name: source.user.hash + ignore_above: 1024 + level: extended + name: hash + original_fieldset: user + short: Unique user hash to correlate information for a user in anonymized form. + type: keyword + user.id: + description: One or multiple unique identifiers of the user. + flat_name: source.user.id + ignore_above: 1024 + level: core + name: id + original_fieldset: user + short: One or multiple unique identifiers of the user. + type: keyword + user.name: + description: Short name or login of the user. + example: albert + flat_name: source.user.name + ignore_above: 1024 + level: core + name: name + original_fieldset: user + short: Short name or login of the user. + type: keyword group: 2 name: source prefix: source. @@ -2044,6 +3019,24 @@ user: name: full_name short: User's full name, if available. type: keyword + group.id: + description: Unique identifier for the group on the system/platform. + flat_name: user.group.id + ignore_above: 1024 + level: extended + name: id + original_fieldset: group + short: Unique identifier for the group on the system/platform. + type: keyword + group.name: + description: Name of the group. + flat_name: user.group.name + ignore_above: 1024 + level: extended + name: name + original_fieldset: group + short: Name of the group. + type: keyword hash: description: 'Unique user hash to correlate information for a user in anonymized form. @@ -2120,6 +3113,66 @@ user_agent: name: original short: Unparsed version of the user_agent. type: keyword + os.family: + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + flat_name: user_agent.os.family + ignore_above: 1024 + level: extended + name: family + original_fieldset: os + short: OS family (such as redhat, debian, freebsd, windows). + type: keyword + os.full: + description: Operating system name, including the version or code name. + example: Mac OS Mojave + flat_name: user_agent.os.full + ignore_above: 1024 + level: extended + name: full + original_fieldset: os + short: Operating system name, including the version or code name. + type: keyword + os.kernel: + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + flat_name: user_agent.os.kernel + ignore_above: 1024 + level: extended + name: kernel + original_fieldset: os + short: Operating system kernel version as a raw string. + type: keyword + os.name: + description: Operating system name, without the version. + example: Mac OS X + flat_name: user_agent.os.name + ignore_above: 1024 + level: extended + name: name + original_fieldset: os + short: Operating system name, without the version. + type: keyword + os.platform: + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + flat_name: user_agent.os.platform + ignore_above: 1024 + level: extended + name: platform + original_fieldset: os + short: Operating system platform (such centos, ubuntu, windows). + type: keyword + os.version: + description: Operating system version as a raw string. + example: 10.14.1 + flat_name: user_agent.os.version + ignore_above: 1024 + level: extended + name: version + original_fieldset: os + short: Operating system version as a raw string. + type: keyword version: description: Version of the user agent. example: 12.0 diff --git a/generated/elasticsearch/6/template.json b/generated/elasticsearch/6/template.json index 68bee03e61..bcd9d3a3c0 100644 --- a/generated/elasticsearch/6/template.json +++ b/generated/elasticsearch/6/template.json @@ -118,6 +118,18 @@ "ignore_above": 1024, "type": "keyword" }, + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "hash": { "ignore_above": 1024, "type": "keyword" @@ -280,6 +292,18 @@ "ignore_above": 1024, "type": "keyword" }, + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "hash": { "ignore_above": 1024, "type": "keyword" @@ -597,6 +621,18 @@ "ignore_above": 1024, "type": "keyword" }, + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "hash": { "ignore_above": 1024, "type": "keyword" @@ -986,6 +1022,18 @@ "ignore_above": 1024, "type": "keyword" }, + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "hash": { "ignore_above": 1024, "type": "keyword" @@ -1101,6 +1149,18 @@ "ignore_above": 1024, "type": "keyword" }, + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "hash": { "ignore_above": 1024, "type": "keyword" diff --git a/generated/elasticsearch/7/template.json b/generated/elasticsearch/7/template.json index 6647298cce..b7dbf70306 100644 --- a/generated/elasticsearch/7/template.json +++ b/generated/elasticsearch/7/template.json @@ -117,6 +117,18 @@ "ignore_above": 1024, "type": "keyword" }, + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "hash": { "ignore_above": 1024, "type": "keyword" @@ -279,6 +291,18 @@ "ignore_above": 1024, "type": "keyword" }, + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "hash": { "ignore_above": 1024, "type": "keyword" @@ -596,6 +620,18 @@ "ignore_above": 1024, "type": "keyword" }, + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "hash": { "ignore_above": 1024, "type": "keyword" @@ -985,6 +1021,18 @@ "ignore_above": 1024, "type": "keyword" }, + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "hash": { "ignore_above": 1024, "type": "keyword" @@ -1100,6 +1148,18 @@ "ignore_above": 1024, "type": "keyword" }, + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "hash": { "ignore_above": 1024, "type": "keyword" From f146f113c688b72f660771a3ebe61244469b6a3a Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 7 Mar 2019 17:01:31 -0500 Subject: [PATCH 04/11] First draft of a fully generated fields.ecs.yml for Beats --- generated/beats/fields.ecs.yml | 1893 ++++++++++++++++++++++++++++++++ scripts/generator.py | 2 + scripts/generators/beats.py | 80 ++ 3 files changed, 1975 insertions(+) create mode 100644 generated/beats/fields.ecs.yml create mode 100644 scripts/generators/beats.py diff --git a/generated/beats/fields.ecs.yml b/generated/beats/fields.ecs.yml new file mode 100644 index 0000000000..5bf7322ec0 --- /dev/null +++ b/generated/beats/fields.ecs.yml @@ -0,0 +1,1893 @@ +- key: ecs + title: ECS + description: ECS Fields. + fields: + - name: '@timestamp' + level: core + required: true + type: date + description: 'Date/time when the event originated. + + This is the date/time extracted from the event, typically representing when + the event was generated by the source. + + If the event source has no original timestamp, this value is typically populated + by the first time the event was received by the pipeline. + + Required field for all events.' + example: '2016-05-23T08:05:34.853Z' + - name: labels + level: core + type: object + object_type: keyword + description: 'Custom key/value pairs. + + Can be used to add meta information to events. Should not contain nested objects. + All values are stored as keyword. + + Example: `docker` and `k8s` labels.' + example: + application: foo-bar + env: production + - name: message + level: core + type: text + description: 'For log events the message field contains the log message, optimized + for viewing in a log viewer. + + For structured logs without an original message field, other fields can be concatenated + to form a human-readable summary of the event. + + If multiple messages exist, they can be combined into one message.' + example: Hello World + - name: tags + level: core + type: keyword + description: List of keywords used to tag each event. + example: '["production", "env2"]' + - name: agent + title: Agent + group: 2 + description: 'The agent fields contain the data about the software entity, if + any, that collects, detects, or observes events on a host, or takes measurements + on a host. + + Examples include Beats. Agents may also run on observers. ECS agent.* fields + shall be populated with details of the agent running on the host or observer + where the event happened or the measurement was taken.' + type: group + fields: + - name: ephemeral_id + level: extended + type: keyword + description: 'Ephemeral identifier of this agent (if one exists). + + This id normally changes across restarts, but `agent.id` does not.' + example: 8a4f500f + - name: version + level: core + type: keyword + description: Version of the agent. + example: 6.0.0-rc2 + - name: type + level: core + type: keyword + description: 'Type of the agent. + + The agent type stays always the same and should be given by the agent used. + In case of Filebeat the agent would always be Filebeat also if two Filebeat + instances are run on the same machine.' + example: filebeat + - name: name + level: core + type: keyword + description: 'Custom name of the agent. + + This is a name that can be given to an agent. This can be helpful if for example + two Filebeat instances are running on the same host but a human readable separation + is needed on which Filebeat instance data is coming from. + + If no name is given, the name is often left empty.' + example: foo + - name: id + level: core + type: keyword + description: 'Unique identifier of this agent (if one exists). + + Example: For Beats this would be beat.id.' + example: 8a4f500d + - name: client + title: Client + group: 2 + description: 'A client is defined as the initiator of a network connection for + events regarding sessions, connections, or bidirectional flow records. + + For TCP events, the client is the initiator of the TCP connection that sends + the SYN packet(s). For other protocols, the client is generally the initiator + or requestor in the network transaction. Some systems use the term "originator" + to refer the client in TCP connections. The client fields describe details about + the system acting as the client in the network event. Client fields are usually + populated in conjunction with server fields. Client fields are generally not + populated for packet-level events. + + Client / server representations can add semantic context to an exchange, which + is helpful to visualize the data in certain situations. If your context falls + in that category, you should still ensure that source and destination are filled + appropriately.' + type: group + fields: + - name: domain + level: core + type: keyword + description: Client domain. + - name: region_name + level: core + type: keyword + description: Region name. + example: Quebec + - name: ip + level: core + type: ip + description: 'IP address of the client. + + Can be one or multiple IPv4 or IPv6 addresses.' + - name: continent_name + level: core + type: keyword + description: Name of the continent. + example: North America + - name: country_iso_code + level: core + type: keyword + description: Country ISO code. + example: CA + - name: hash + level: extended + type: keyword + description: 'Unique user hash to correlate information for a user in anonymized + form. + + Useful if `user.id` or `user.name` contain confidential information and cannot + be used.' + - name: id + level: core + type: keyword + description: One or multiple unique identifiers of the user. + - name: port + level: core + type: long + description: Port of the client. + - name: country_name + level: core + type: keyword + description: Country name. + example: Canada + - name: email + level: extended + type: keyword + description: User email address. + - name: name + level: extended + type: keyword + description: Name of the group. + - name: name + level: core + type: keyword + description: Short name or login of the user. + example: albert + - name: full_name + level: extended + type: keyword + description: User's full name, if available. + example: Albert Einstein + - name: city_name + level: core + type: keyword + description: City name. + example: Montreal + - name: name + level: extended + type: keyword + description: 'User-defined description of a location, at the level of granularity + they care about. + + Could be the name of their data centers, the floor number, if this describes + a local physical entity, city names. + + Not typically used in automated geolocation.' + example: boston-dc + - name: id + level: extended + type: keyword + description: Unique identifier for the group on the system/platform. + - name: address + level: extended + type: keyword + description: 'Some event client addresses are defined ambiguously. The event + will sometimes list an IP, a domain or a unix socket. You should always store + the raw address in the `.address` field. + + Then it should be duplicated to `.ip` or `.domain`, depending on which one + it is.' + - name: region_iso_code + level: core + type: keyword + description: Region ISO code. + example: CA-QC + - name: location + level: core + type: geo_point + description: Longitude and latitude. + example: '{ "lon": -73.614830, "lat": 45.505918 }' + - name: bytes + level: core + type: long + description: Bytes sent from the client to the server. + example: 184 + - name: packets + level: core + type: long + description: Packets sent from the client to the server. + example: 12 + - name: mac + level: core + type: keyword + description: MAC address of the client. + - name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming + from. + type: group + fields: + - name: availability_zone + level: extended + type: keyword + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.name + level: extended + type: keyword + description: Instance name of the host machine. + - name: region + level: extended + type: keyword + description: Region in which this host is running. + example: us-east-1 + - name: machine.type + level: extended + type: keyword + description: Machine type of the host machine. + example: t2.medium + - name: account.id + level: extended + type: keyword + description: 'The cloud account or organization id used to identify different + entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: provider + level: extended + type: keyword + description: Name of the cloud provider. Example values are aws, azure, gcp, + or digitalocean. + example: aws + - name: instance.id + level: extended + type: keyword + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific + container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: name + level: extended + type: keyword + description: Container name. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: image.tag + level: extended + type: keyword + description: Container image tag. + - name: runtime + level: extended + type: keyword + description: Runtime managing this container. + example: docker + - name: image.name + level: extended + type: keyword + description: Name of the image the container was built on. + - name: id + level: core + type: keyword + description: Unique container id. + - name: destination + title: Destination + group: 2 + description: 'Destination fields describe details about the destination of a packet/event. + + Destination fields are usually populated in conjunction with source fields.' + type: group + fields: + - name: domain + level: core + type: keyword + description: Destination domain. + - name: region_name + level: core + type: keyword + description: Region name. + example: Quebec + - name: ip + level: core + type: ip + description: 'IP address of the destination. + + Can be one or multiple IPv4 or IPv6 addresses.' + - name: continent_name + level: core + type: keyword + description: Name of the continent. + example: North America + - name: country_iso_code + level: core + type: keyword + description: Country ISO code. + example: CA + - name: hash + level: extended + type: keyword + description: 'Unique user hash to correlate information for a user in anonymized + form. + + Useful if `user.id` or `user.name` contain confidential information and cannot + be used.' + - name: id + level: core + type: keyword + description: One or multiple unique identifiers of the user. + - name: port + level: core + type: long + description: Port of the destination. + - name: country_name + level: core + type: keyword + description: Country name. + example: Canada + - name: email + level: extended + type: keyword + description: User email address. + - name: name + level: extended + type: keyword + description: Name of the group. + - name: name + level: core + type: keyword + description: Short name or login of the user. + example: albert + - name: full_name + level: extended + type: keyword + description: User's full name, if available. + example: Albert Einstein + - name: city_name + level: core + type: keyword + description: City name. + example: Montreal + - name: name + level: extended + type: keyword + description: 'User-defined description of a location, at the level of granularity + they care about. + + Could be the name of their data centers, the floor number, if this describes + a local physical entity, city names. + + Not typically used in automated geolocation.' + example: boston-dc + - name: id + level: extended + type: keyword + description: Unique identifier for the group on the system/platform. + - name: address + level: extended + type: keyword + description: 'Some event destination addresses are defined ambiguously. The + event will sometimes list an IP, a domain or a unix socket. You should always + store the raw address in the `.address` field. + + Then it should be duplicated to `.ip` or `.domain`, depending on which one + it is.' + - name: region_iso_code + level: core + type: keyword + description: Region ISO code. + example: CA-QC + - name: location + level: core + type: geo_point + description: Longitude and latitude. + example: '{ "lon": -73.614830, "lat": 45.505918 }' + - name: bytes + level: core + type: long + description: Bytes sent from the destination to the source. + example: 184 + - name: packets + level: core + type: long + description: Packets sent from the destination to the source. + example: 12 + - name: mac + level: core + type: keyword + description: MAC address of the destination. + - name: ecs + title: ECS + group: 2 + description: Meta-information specific to ECS. + type: group + fields: + - name: version + level: core + required: true + type: keyword + description: 'ECS version this event conforms to. `ecs.version` is a required + field and must exist in all events. + + When querying across multiple indices -- which may conform to slightly different + ECS versions -- this field lets integrations adjust to the schema version + of the events.' + example: 1.0.0 + - name: error + title: Error + group: 2 + description: 'These fields can represent errors of any kind. + + Use them for errors that happen while fetching events or in cases where the + event itself contains an error.' + type: group + fields: + - name: message + level: core + type: text + description: Error message. + - name: code + level: core + type: keyword + description: Error code describing the error. + - name: id + level: core + type: keyword + description: Unique identifier for the error. + - name: event + title: Event + group: 2 + description: 'The event fields are used for context information about the log + or metric event itself. + + A log is defined as an event containing details of something that happened. + Log events must include the time at which the thing happened. Examples of log + events include a process starting on a host, a network packet being sent from + a source to a destination, or a network connection between a client and a server + being initiated or closed. A metric is defined as an event containing one or + more numerical or categorical measurements and the time at which the measurement + was taken. Examples of metric events include memory pressure measured on a host, + or vulnerabilities measured on a scanned host.' + type: group + fields: + - name: category + level: core + type: keyword + description: 'Event category. + + This contains high-level information about the contents of the event. It is + more generic than `event.action`, in the sense that typically a category contains + multiple actions. Warning: In future versions of ECS, we plan to provide a + list of acceptable values for this field, please use with caution.' + example: user-management + - name: risk_score + level: core + type: float + description: Risk score or priority of the event (e.g. security solutions). + Use your system's original value here. + - name: kind + level: extended + type: keyword + description: 'The kind of the event. + + This gives information about what type of information the event contains, + without being specific to the contents of the event. Examples are `event`, + `state`, `alarm`. Warning: In future versions of ECS, we plan to provide a + list of acceptable values for this field, please use with caution.' + example: state + - name: hash + level: extended + type: keyword + description: Hash (perhaps logstash fingerprint) of raw field to be able to + demonstrate log integrity. + example: 123456789012345678901234567890ABCD + - name: severity + level: core + type: long + description: Severity describes the original severity of the event. What the + different severity values mean can very different between use cases. It's + up to the implementer to make sure severities are consistent across events. + example: '7' + - name: created + level: core + type: date + description: 'event.created contains the date/time when the event was first + read by an agent, or by your pipeline. + + This field is distinct from @timestamp in that @timestamp typically contain + the time extracted from the original event. + + In most situations, these two timestamps will be slightly different. The difference + can be used to calculate the delay between your source generating an event, + and the time when your agent first processed it. This can be used to monitor + your agent''s or pipeline''s ability to keep up with your event source. + + In case the two timestamps are identical, @timestamp should be used.' + - name: original + level: core + type: keyword + description: 'Raw text message of entire event. Used to demonstrate log integrity. + + This field is not indexed and doc_values are disabled. It cannot be searched, + but it can be retrieved from `_source`.' + example: Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100| + worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232 + - name: risk_score_norm + level: extended + type: float + description: 'Normalized risk score or priority of the event, on a scale of + 0 to 100. + + This is mainly useful if you use more than one system that assigns risk scores, + and you want to see a normalized value across all systems.' + - name: module + level: core + type: keyword + description: 'Name of the module this data is coming from. + + This information is coming from the modules used in Beats or Logstash.' + example: mysql + - name: dataset + level: core + type: keyword + description: 'Name of the dataset. + + The concept of a `dataset` (fileset / metricset) is used in Beats as a subset + of modules. It contains the information which is currently stored in metricset.name + and metricset.module or fileset.name.' + example: stats + - name: end + level: extended + type: date + description: event.end contains the date when the event ended or when the activity + was last observed. + - name: duration + level: core + type: long + description: 'Duration of the event in nanoseconds. + + If event.start and event.end are known this value should be the difference + between the end and start time.' + - name: start + level: extended + type: date + description: event.start contains the date when the event started or when the + activity was first observed. + - name: action + level: core + type: keyword + description: 'The action captured by the event. + + This describes the information in the event. It is more specific than `event.category`. + Examples are `group-add`, `process-started`, `file-created`. The value is + normally defined by the implementer.' + example: user-password-change + - name: timezone + level: extended + type: keyword + description: 'This field should be populated when the event''s timestamp does + not include timezone information already (e.g. default Syslog timestamps). + It''s optional otherwise. + + Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), + abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00").' + - name: outcome + level: extended + type: keyword + description: 'The outcome of the event. + + If the event describes an action, this fields contains the outcome of that + action. Examples outcomes are `success` and `failure`. Warning: In future + versions of ECS, we plan to provide a list of acceptable values for this field, + please use with caution.' + example: success + - name: type + level: core + type: keyword + description: 'Reserved for future usage. + + Please avoid using this field for user data.' + - name: id + level: core + type: keyword + description: Unique ID to describe the event. + example: 8a4f500d + - name: file + title: File + group: 2 + description: 'A file is defined as a set of information that has been created + on, or has existed on a filesystem. + + File objects can be associated with host events, network events, and/or file + events (e.g., those produced by File Integrity Monitoring [FIM] products or + services). File fields provide details about the affected file associated with + the event or metric.' + type: group + fields: + - name: group + level: extended + type: keyword + description: Primary group name of the file. + - name: uid + level: extended + type: keyword + description: The user ID (UID) or security identifier (SID) of the file owner. + - name: extension + level: extended + type: keyword + description: 'File extension. + + This should allow easy filtering by file extensions.' + example: png + - name: type + level: extended + type: keyword + description: File type (file, dir, or symlink). + - name: ctime + level: extended + type: date + description: Last time file metadata changed. + - name: target_path + level: extended + type: keyword + description: Target path for symlinks. + - name: owner + level: extended + type: keyword + description: File owner's username. + - name: gid + level: extended + type: keyword + description: Primary group ID (GID) of the file. + - name: mode + level: extended + type: keyword + description: Mode of the file in octal representation. + example: 416 + - name: mtime + level: extended + type: date + description: Last time file content was modified. + - name: device + level: extended + type: keyword + description: Device that is the source of the file. + - name: path + level: extended + type: keyword + description: Path to the file. + - name: inode + level: extended + type: keyword + description: Inode representing the file in the filesystem. + - name: size + level: extended + type: long + description: File size in bytes (field is only added when `type` is `file`). + - name: geo + title: Geo + group: 2 + description: 'Geo fields can carry data about a specific location related to an + event. + + This geolocation information can be derived from techniques such as Geo IP, + or be user-supplied.' + type: group + fields: + - name: region_name + level: core + type: keyword + description: Region name. + example: Quebec + - name: country_iso_code + level: core + type: keyword + description: Country ISO code. + example: CA + - name: name + level: extended + type: keyword + description: 'User-defined description of a location, at the level of granularity + they care about. + + Could be the name of their data centers, the floor number, if this describes + a local physical entity, city names. + + Not typically used in automated geolocation.' + example: boston-dc + - name: country_name + level: core + type: keyword + description: Country name. + example: Canada + - name: city_name + level: core + type: keyword + description: City name. + example: Montreal + - name: location + level: core + type: geo_point + description: Longitude and latitude. + example: '{ "lon": -73.614830, "lat": 45.505918 }' + - name: continent_name + level: core + type: keyword + description: Name of the continent. + example: North America + - name: region_iso_code + level: core + type: keyword + description: Region ISO code. + example: CA-QC + - name: group + title: Group + group: 2 + description: The group fields are meant to represent groups that are relevant + to the event. + type: group + fields: + - name: id + level: extended + type: keyword + description: Unique identifier for the group on the system/platform. + - name: name + level: extended + type: keyword + description: Name of the group. + - name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the + event happened, or from which the measurement was taken. Host types include + hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: region_name + level: core + type: keyword + description: Region name. + example: Quebec + - name: ip + level: core + type: ip + description: Host ip address. + - name: continent_name + level: core + type: keyword + description: Name of the continent. + example: North America + - name: country_iso_code + level: core + type: keyword + description: Country ISO code. + example: CA + - name: hash + level: extended + type: keyword + description: 'Unique user hash to correlate information for a user in anonymized + form. + + Useful if `user.id` or `user.name` contain confidential information and cannot + be used.' + - name: family + level: extended + type: keyword + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: id + level: core + type: keyword + description: One or multiple unique identifiers of the user. + - name: name + level: extended + type: keyword + description: Operating system name, without the version. + example: Mac OS X + - name: id + level: core + type: keyword + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: kernel + level: extended + type: keyword + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: country_name + level: core + type: keyword + description: Country name. + example: Canada + - name: hostname + level: core + type: keyword + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: email + level: extended + type: keyword + description: User email address. + - name: name + level: extended + type: keyword + description: Name of the group. + - name: name + level: core + type: keyword + description: Short name or login of the user. + example: albert + - name: type + level: core + type: keyword + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, + this could be the container, for example, or other information meaningful + in your environment.' + - name: full_name + level: extended + type: keyword + description: User's full name, if available. + example: Albert Einstein + - name: city_name + level: core + type: keyword + description: City name. + example: Montreal + - name: version + level: extended + type: keyword + description: Operating system version as a raw string. + example: 10.14.1 + - name: name + level: extended + type: keyword + description: 'User-defined description of a location, at the level of granularity + they care about. + + Could be the name of their data centers, the floor number, if this describes + a local physical entity, city names. + + Not typically used in automated geolocation.' + example: boston-dc + - name: mac + level: core + type: keyword + description: Host mac address. + - name: platform + level: extended + type: keyword + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: name + level: core + type: keyword + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified + domain name, or a name specified by the user. The sender decides which value + to use.' + - name: region_iso_code + level: core + type: keyword + description: Region ISO code. + example: CA-QC + - name: location + level: core + type: geo_point + description: Longitude and latitude. + example: '{ "lon": -73.614830, "lat": 45.505918 }' + - name: id + level: extended + type: keyword + description: Unique identifier for the group on the system/platform. + - name: architecture + level: core + type: keyword + description: Operating system architecture. + example: x86_64 + - name: full + level: extended + type: keyword + description: Operating system name, including the version or code name. + example: Mac OS Mojave + - name: http + title: HTTP + group: 2 + description: Fields related to HTTP activity. Use the `url` field set to store + the url of the request. + type: group + fields: + - name: request.body.bytes + level: extended + type: long + description: Size in bytes of the request body. + example: 887 + - name: request.method + level: extended + type: keyword + description: 'HTTP request method. + + The field value must be normalized to lowercase for querying. See the documentation + section "Implementing ECS".' + example: get, post, put + - name: request.bytes + level: extended + type: long + description: Total size in bytes of the request (body and headers). + example: 1437 + - name: response.bytes + level: extended + type: long + description: Total size in bytes of the response (body and headers). + example: 1437 + - name: request.body.content + level: extended + type: keyword + description: The full HTTP request body. + example: Hello world + - name: response.body.bytes + level: extended + type: long + description: Size in bytes of the response body. + example: 887 + - name: version + level: extended + type: keyword + description: HTTP version. + example: 1.1 + - name: response.status_code + level: extended + type: long + description: HTTP response status code. + example: 404 + - name: request.referrer + level: extended + type: keyword + description: Referrer for this HTTP request. + example: https://blog.example.com/ + - name: response.body.content + level: extended + type: keyword + description: The full HTTP response body. + example: Hello world + - name: log + title: Log + group: 2 + description: Fields which are specific to log events. + type: group + fields: + - name: original + level: core + type: keyword + description: 'This is the original log message and contains the full log message + before splitting it up in multiple parts. + + In contrast to the `message` field which can contain an extracted part of + the log message, this field contains the original, full log message. It can + have already some modifications applied like encoding or new lines removed + to clean up the log message. + + This field is not indexed and doc_values are disabled so it can''t be queried + but the value can be retrieved from `_source`.' + example: Sep 19 08:26:10 localhost My log + - name: level + level: core + type: keyword + description: 'Original log level of the log event. + + Some examples are `warn`, `error`, `i`.' + example: err + - name: network + title: Network + group: 2 + description: 'The network is defined as the communication path over which a host + or network event happens. + + The network.* fields should be populated with details about the network activity + associated with an event.' + type: group + fields: + - name: direction + level: core + type: keyword + description: "Direction of the network traffic.\nRecommended values are:\n \ + \ * inbound\n * outbound\n * internal\n * external\n * unknown\n\nWhen\ + \ mapping events from a host-based monitoring context, populate this field\ + \ from the host's point of view.\nWhen mapping events from a network or perimeter-based\ + \ monitoring context, populate this field from the point of view of your network\ + \ perimeter." + example: inbound + - name: packets + level: core + type: long + description: 'Total packets transferred in both directions. + + If `source.packets` and `destination.packets` are known, `network.packets` + is their sum.' + example: 24 + - name: protocol + level: core + type: keyword + description: 'L7 Network protocol name. ex. http, lumberjack, transport protocol. + + The field value must be normalized to lowercase for querying. See the documentation + section "Implementing ECS".' + example: http + - name: name + level: extended + type: keyword + description: Name given by operators to sections of their network. + example: Guest Wifi + - name: bytes + level: core + type: long + description: 'Total bytes transferred in both directions. + + If `source.bytes` and `destination.bytes` are known, `network.bytes` is their + sum.' + example: 368 + - name: iana_number + level: extended + type: keyword + description: IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). + Standardized list of protocols. This aligns well with NetFlow and sFlow related + logs which use the IANA Protocol Number. + example: 6 + - name: community_id + level: extended + type: keyword + description: 'A hash of source and destination IPs and ports, as well as the + protocol used in a communication. This is a tool-agnostic standard to identify + flows. + + Learn more at https://github.com/corelight/community-id-spec.' + example: 1:hO+sN4H+MG5MY/8hIrXPqc4ZQz0= + - name: application + level: extended + type: keyword + description: 'A name given to an application level protocol. This can be arbitrarily + assigned for things like microservices, but also apply to things like skype, + icq, facebook, twitter. This would be used in situations where the vendor + or service can be decoded such as from the source/dest IP owners, ports, or + wire format. + + The field value must be normalized to lowercase for querying. See the documentation + section "Implementing ECS".' + example: aim + - name: forwarded_ip + level: core + type: ip + description: Host IP address when the source IP address is the proxy. + example: 192.1.1.2 + - name: type + level: core + type: keyword + description: 'In the OSI Model this would be the Network Layer. ipv4, ipv6, + ipsec, pim, etc + + The field value must be normalized to lowercase for querying. See the documentation + section "Implementing ECS".' + example: ipv4 + - name: transport + level: core + type: keyword + description: 'Same as network.iana_number, but instead using the Keyword name + of the transport layer (udp, tcp, ipv6-icmp, etc.) + + The field value must be normalized to lowercase for querying. See the documentation + section "Implementing ECS".' + example: tcp + - name: observer + title: Observer + group: 2 + description: 'An observer is defined as a special network, security, or application + device used to detect, observe, or create network, security, or application-related + events and metrics. + + This could be a custom hardware appliance or a server that has been configured + to run special network, security, or application software. Examples include + firewalls, intrusion detection/prevention systems, network monitoring sensors, + web application firewalls, data loss prevention systems, and APM servers. The + observer.* fields shall be populated with details of the system, if any, that + detects, observes and/or creates a network, security, or application event or + metric. Message queues and ETL components used in processing events or metrics + are not considered observers in ECS.' + type: group + fields: + - name: region_name + level: core + type: keyword + description: Region name. + example: Quebec + - name: continent_name + level: core + type: keyword + description: Name of the continent. + example: North America + - name: vendor + level: core + type: keyword + description: observer vendor information. + - name: city_name + level: core + type: keyword + description: City name. + example: Montreal + - name: country_name + level: core + type: keyword + description: Country name. + example: Canada + - name: country_iso_code + level: core + type: keyword + description: Country ISO code. + example: CA + - name: ip + level: core + type: ip + description: IP address of the observer. + - name: location + level: core + type: geo_point + description: Longitude and latitude. + example: '{ "lon": -73.614830, "lat": 45.505918 }' + - name: hostname + level: core + type: keyword + description: Hostname of the observer. + - name: kernel + level: extended + type: keyword + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: name + level: extended + type: keyword + description: 'User-defined description of a location, at the level of granularity + they care about. + + Could be the name of their data centers, the floor number, if this describes + a local physical entity, city names. + + Not typically used in automated geolocation.' + example: boston-dc + - name: region_iso_code + level: core + type: keyword + description: Region ISO code. + example: CA-QC + - name: mac + level: core + type: keyword + description: MAC address of the observer + - name: version + level: core + type: keyword + description: Observer version. + - name: family + level: extended + type: keyword + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: full + level: extended + type: keyword + description: Operating system name, including the version or code name. + example: Mac OS Mojave + - name: serial_number + level: extended + type: keyword + description: Observer serial number. + - name: name + level: extended + type: keyword + description: Operating system name, without the version. + example: Mac OS X + - name: type + level: core + type: keyword + description: 'The type of the observer the data is coming from. + + There is no predefined list of observer types. Some examples are `forwarder`, + `firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`.' + example: firewall + - name: version + level: extended + type: keyword + description: Operating system version as a raw string. + example: 10.14.1 + - name: platform + level: extended + type: keyword + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: organization + title: Organization + group: 2 + description: 'The organization fields enrich data with information about the company + or entity the data is associated with. + + These fields help you arrange or filter data stored in an index by one or multiple + organizations.' + type: group + fields: + - name: name + level: extended + type: keyword + description: Organization name. + - name: id + level: extended + type: keyword + description: Unique identifier for the organization. + - name: os + title: Operating System + group: 2 + description: The OS fields contain information about the operating system. + type: group + fields: + - name: kernel + level: extended + type: keyword + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: full + level: extended + type: keyword + description: Operating system name, including the version or code name. + example: Mac OS Mojave + - name: name + level: extended + type: keyword + description: Operating system name, without the version. + example: Mac OS X + - name: family + level: extended + type: keyword + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: platform + level: extended + type: keyword + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: version + level: extended + type: keyword + description: Operating system version as a raw string. + example: 10.14.1 + - name: process + title: Process + group: 2 + description: 'These fields contain information about a process. + + These fields can help you correlate metrics information with a process id/name + from a log message. The `process.pid` often stays in the metric itself and + is copied to the global field for correlation.' + type: group + fields: + - name: executable + level: extended + type: keyword + description: Absolute path to the process executable. + example: /usr/bin/ssh + - name: thread.id + level: extended + type: long + description: Thread ID. + example: 4242 + - name: name + level: extended + type: keyword + description: 'Process name. + + Sometimes called program name or similar.' + example: ssh + - name: title + level: extended + type: keyword + description: 'Process title. + + The proctitle, some times the same as process name. Can also be different: + for example a browser setting its title to the web page currently opened.' + - name: args + level: extended + type: keyword + description: 'Array of process arguments. + + May be filtered to protect sensitive information.' + example: + - ssh + - -l + - user + - 10.0.0.16 + - name: pid + level: core + type: long + description: Process id. + - name: start + level: extended + type: date + description: The time the process started. + example: '2016-05-23T08:05:34.853Z' + - name: working_directory + level: extended + type: keyword + description: The working directory of the process. + example: /home/alice + - name: ppid + level: extended + type: long + description: Process parent id. + - name: related + title: Related + group: 2 + description: 'This field set is meant to facilitate pivoting around a piece of + data. + + Some pieces of information can be seen in many places in an ECS event. To facilitate + searching for them, store an array of all seen values to their corresponding + field in `related.`. + + A concrete example is IP addresses, which can be under host, observer, source, + destination, client, server, and network.forwarded_ip. If you append all IPs + to `related.ip`, you can then search for a given IP trivially, no matter where + it appeared, by querying `related.ip:a.b.c.d`.' + type: group + fields: + - name: ip + level: extended + type: ip + description: All of the IPs seen on your event. + - name: server + title: Server + group: 2 + description: 'A Server is defined as the responder in a network connection for + events regarding sessions, connections, or bidirectional flow records. + + For TCP events, the server is the receiver of the initial SYN packet(s) of the + TCP connection. For other protocols, the server is generally the responder in + the network transaction. Some systems actually use the term "responder" to refer + the server in TCP connections. The server fields describe details about the + system acting as the server in the network event. Server fields are usually + populated in conjunction with client fields. Server fields are generally not + populated for packet-level events. + + Client / server representations can add semantic context to an exchange, which + is helpful to visualize the data in certain situations. If your context falls + in that category, you should still ensure that source and destination are filled + appropriately.' + type: group + fields: + - name: domain + level: core + type: keyword + description: Server domain. + - name: region_name + level: core + type: keyword + description: Region name. + example: Quebec + - name: ip + level: core + type: ip + description: 'IP address of the server. + + Can be one or multiple IPv4 or IPv6 addresses.' + - name: continent_name + level: core + type: keyword + description: Name of the continent. + example: North America + - name: country_iso_code + level: core + type: keyword + description: Country ISO code. + example: CA + - name: hash + level: extended + type: keyword + description: 'Unique user hash to correlate information for a user in anonymized + form. + + Useful if `user.id` or `user.name` contain confidential information and cannot + be used.' + - name: id + level: core + type: keyword + description: One or multiple unique identifiers of the user. + - name: port + level: core + type: long + description: Port of the server. + - name: country_name + level: core + type: keyword + description: Country name. + example: Canada + - name: email + level: extended + type: keyword + description: User email address. + - name: name + level: extended + type: keyword + description: Name of the group. + - name: name + level: core + type: keyword + description: Short name or login of the user. + example: albert + - name: full_name + level: extended + type: keyword + description: User's full name, if available. + example: Albert Einstein + - name: city_name + level: core + type: keyword + description: City name. + example: Montreal + - name: name + level: extended + type: keyword + description: 'User-defined description of a location, at the level of granularity + they care about. + + Could be the name of their data centers, the floor number, if this describes + a local physical entity, city names. + + Not typically used in automated geolocation.' + example: boston-dc + - name: id + level: extended + type: keyword + description: Unique identifier for the group on the system/platform. + - name: address + level: extended + type: keyword + description: 'Some event server addresses are defined ambiguously. The event + will sometimes list an IP, a domain or a unix socket. You should always store + the raw address in the `.address` field. + + Then it should be duplicated to `.ip` or `.domain`, depending on which one + it is.' + - name: region_iso_code + level: core + type: keyword + description: Region ISO code. + example: CA-QC + - name: location + level: core + type: geo_point + description: Longitude and latitude. + example: '{ "lon": -73.614830, "lat": 45.505918 }' + - name: bytes + level: core + type: long + description: Bytes sent from the server to the client. + example: 184 + - name: packets + level: core + type: long + description: Packets sent from the server to the client. + example: 12 + - name: mac + level: core + type: keyword + description: MAC address of the server. + - name: service + title: Service + group: 2 + description: 'The service fields describe the service for or from which the data + was collected. + + These fields help you find and correlate logs for a specific service and version.' + type: group + fields: + - name: ephemeral_id + level: extended + type: keyword + description: 'Ephemeral identifier of this service (if one exists). + + This id normally changes across restarts, but `service.id` does not.' + example: 8a4f500f + - name: name + level: core + type: keyword + description: 'Name of the service data is collected from. + + The name of the service is normally user given. This allows if two instances + of the same service are running on the same machine they can be differentiated + by the `service.name`. + + Also it allows for distributed services that run on multiple hosts to correlate + the related instances based on the name. + + In the case of Elasticsearch the service.name could contain the cluster name. + For Beats the service.name is by default a copy of the `service.type` field + if no name is specified.' + example: elasticsearch-metrics + - name: state + level: core + type: keyword + description: Current state of the service. + - name: version + level: core + type: keyword + description: 'Version of the service the data was collected from. + + This allows to look at a data set only for a specific version of a service.' + example: 3.2.4 + - name: type + level: core + type: keyword + description: 'The type of the service data is collected from. + + The type can be used to group and correlate logs and metrics from one service + type. + + Example: If logs or metrics are collected from Elasticsearch, `service.type` + would be `elasticsearch`.' + example: elasticsearch + - name: id + level: core + type: keyword + description: 'Unique identifier of the running service. + + This id should uniquely identify this service. This makes it possible to correlate + logs and metrics for one specific service. + + Example: If you are experiencing issues with one redis instance, you can filter + on that id to see metrics and logs for that single instance.' + example: d37e5ebfe0ae6c4972dbe9f0174a1637bb8247f6 + - name: source + title: Source + group: 2 + description: 'Source fields describe details about the source of a packet/event. + + Source fields are usually populated in conjunction with destination fields.' + type: group + fields: + - name: domain + level: core + type: keyword + description: Source domain. + - name: region_name + level: core + type: keyword + description: Region name. + example: Quebec + - name: ip + level: core + type: ip + description: 'IP address of the source. + + Can be one or multiple IPv4 or IPv6 addresses.' + - name: continent_name + level: core + type: keyword + description: Name of the continent. + example: North America + - name: country_iso_code + level: core + type: keyword + description: Country ISO code. + example: CA + - name: hash + level: extended + type: keyword + description: 'Unique user hash to correlate information for a user in anonymized + form. + + Useful if `user.id` or `user.name` contain confidential information and cannot + be used.' + - name: id + level: core + type: keyword + description: One or multiple unique identifiers of the user. + - name: port + level: core + type: long + description: Port of the source. + - name: country_name + level: core + type: keyword + description: Country name. + example: Canada + - name: email + level: extended + type: keyword + description: User email address. + - name: name + level: extended + type: keyword + description: Name of the group. + - name: name + level: core + type: keyword + description: Short name or login of the user. + example: albert + - name: full_name + level: extended + type: keyword + description: User's full name, if available. + example: Albert Einstein + - name: city_name + level: core + type: keyword + description: City name. + example: Montreal + - name: name + level: extended + type: keyword + description: 'User-defined description of a location, at the level of granularity + they care about. + + Could be the name of their data centers, the floor number, if this describes + a local physical entity, city names. + + Not typically used in automated geolocation.' + example: boston-dc + - name: id + level: extended + type: keyword + description: Unique identifier for the group on the system/platform. + - name: address + level: extended + type: keyword + description: 'Some event source addresses are defined ambiguously. The event + will sometimes list an IP, a domain or a unix socket. You should always store + the raw address in the `.address` field. + + Then it should be duplicated to `.ip` or `.domain`, depending on which one + it is.' + - name: region_iso_code + level: core + type: keyword + description: Region ISO code. + example: CA-QC + - name: location + level: core + type: geo_point + description: Longitude and latitude. + example: '{ "lon": -73.614830, "lat": 45.505918 }' + - name: bytes + level: core + type: long + description: Bytes sent from the source to the destination. + example: 184 + - name: packets + level: core + type: long + description: Packets sent from the source to the destination. + example: 12 + - name: mac + level: core + type: keyword + description: MAC address of the source. + - name: url + title: URL + group: 2 + description: URL fields provide support for complete or partial URLs, and supports + the breaking down into scheme, domain, path, and so on. + type: group + fields: + - name: username + level: extended + type: keyword + description: Username of the request. + - name: domain + level: extended + type: keyword + description: 'Domain of the url, such as "www.elastic.co". + + In some cases a URL may refer to an IP and/or port directly, without a domain + name. In this case, the IP address would go to the `domain` field.' + example: www.elastic.co + - name: full + level: extended + type: keyword + description: If full URLs are important to your use case, they should be stored + in `url.full`, whether this field is reconstructed or present in the event + source. + example: https://www.elastic.co:443/search?q=elasticsearch#top + - name: fragment + level: extended + type: keyword + description: 'Portion of the url after the `#`, such as "top". + + The `#` is not part of the fragment.' + - name: port + level: extended + type: long + description: Port of the request, such as 443. + example: 443 + - name: password + level: extended + type: keyword + description: Password of the request. + - name: query + level: extended + type: keyword + description: 'The query field describes the query string of the request, such + as "q=elasticsearch". + + The `?` is excluded from the query string. If a URL contains no `?`, there + is no query field. If there is a `?` but no query, the query field exists + with an empty string. The `exists` query can be used to differentiate between + the two cases.' + - name: path + level: extended + type: keyword + description: Path of the request, such as "/search". + - name: scheme + level: extended + type: keyword + description: 'Scheme of the request, such as "https". + + Note: The `:` is not part of the scheme.' + example: https + - name: original + level: extended + type: keyword + description: 'Unmodified original url as seen in the event source. + + Note that in network monitoring, the observed URL may be a full URL, whereas + in access logs, the URL is often just represented as a path. + + This field is meant to represent the URL as it was observed, complete or not.' + example: https://www.elastic.co:443/search?q=elasticsearch#top or /search?q=elasticsearch + - name: user + title: User + group: 2 + description: 'The user fields describe information about the user that is relevant + to the event. + + Fields can have one entry or multiple entries. If a user has more than one id, + provide an array that includes all of them.' + type: group + fields: + - name: hash + level: extended + type: keyword + description: 'Unique user hash to correlate information for a user in anonymized + form. + + Useful if `user.id` or `user.name` contain confidential information and cannot + be used.' + - name: name + level: core + type: keyword + description: Short name or login of the user. + example: albert + - name: id + level: extended + type: keyword + description: Unique identifier for the group on the system/platform. + - name: id + level: core + type: keyword + description: One or multiple unique identifiers of the user. + - name: full_name + level: extended + type: keyword + description: User's full name, if available. + example: Albert Einstein + - name: email + level: extended + type: keyword + description: User email address. + - name: name + level: extended + type: keyword + description: Name of the group. + - name: user_agent + title: User agent + group: 2 + description: 'The user_agent fields normally come from a browser request. + + They often show up in web service logs coming from the parsed user agent string.' + type: group + fields: + - name: kernel + level: extended + type: keyword + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: name + level: extended + type: keyword + description: Name of the user agent. + example: Safari + - name: version + level: extended + type: keyword + description: Operating system version as a raw string. + example: 10.14.1 + - name: version + level: extended + type: keyword + description: Version of the user agent. + example: 12.0 + - name: family + level: extended + type: keyword + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: device.name + level: extended + type: keyword + description: Name of the device. + example: iPhone + - name: name + level: extended + type: keyword + description: Operating system name, without the version. + example: Mac OS X + - name: full + level: extended + type: keyword + description: Operating system name, including the version or code name. + example: Mac OS Mojave + - name: original + level: extended + type: keyword + description: Unparsed version of the user_agent. + example: Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15 + (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1 + - name: platform + level: extended + type: keyword + description: Operating system platform (such centos, ubuntu, windows). + example: darwin diff --git a/scripts/generator.py b/scripts/generator.py index 2204c58a06..64d8073532 100644 --- a/scripts/generator.py +++ b/scripts/generator.py @@ -4,6 +4,7 @@ from generators import intermediate_files from generators import csv_generator from generators import es_template +from generators import beats def main(): @@ -20,6 +21,7 @@ def main(): csv_generator.generate(ecs_flat, ecs_version) es_template.generate(ecs_flat, ecs_version) + beats.generate(ecs_nested, ecs_version) def argument_parser(): diff --git a/scripts/generators/beats.py b/scripts/generators/beats.py new file mode 100644 index 0000000000..b65a962be4 --- /dev/null +++ b/scripts/generators/beats.py @@ -0,0 +1,80 @@ +import yaml +from collections import OrderedDict + + +def generate(ecs_nested, ecs_version): + # base first + beats_fields = fieldset_field_array(ecs_nested['base']['fields']) + + allowed_fieldset_keys = ['name', 'title', 'group', 'description', 'type'] + # other fieldsets + for fieldset_name in sorted(ecs_nested): + if 'base' == fieldset_name: + continue + fieldset = ecs_nested[fieldset_name] + + beats_field = dict_copy_keys_ordered(fieldset, allowed_fieldset_keys) + beats_field['fields'] = fieldset_field_array(fieldset['fields']) + beats_fields.append(beats_field) + + beats_file = OrderedDict() + beats_file['key'] = 'ecs' + beats_file['title'] = 'ECS' + beats_file['description'] = 'ECS Fields.' + beats_file['fields'] = beats_fields + + with open('generated/beats/fields.ecs.yml', 'w') as outfile: + yaml.dump([beats_file], outfile, default_flow_style=False) + + +def fieldset_field_array(source_fields): + allowed_keys = ['name', 'level', 'required', 'type', 'object_type', + 'multi_fields', 'format', 'description', 'example'] + fields = [] + for field_name in source_fields: + field = source_fields[field_name] + fields.append(dict_copy_keys_ordered(field, allowed_keys)) + return fields + +# Helpers + + +def dict_copy_keys_ordered(dict, copied_keys): + ordered_dict = OrderedDict() + for key in copied_keys: + if key in dict: + ordered_dict[key] = dict[key] + return ordered_dict + + +def indent(text, indent, char=' '): + padding = indent * char + return ''.join(padding + line for line in text.splitlines(True)) + +# Rendering + + +def represent_ordereddict(dumper, data): + value = [] + for item_key, item_value in data.items(): + node_key = dumper.represent_data(item_key) + node_value = dumper.represent_data(item_value) + value.append((node_key, node_value)) + return yaml.nodes.MappingNode(u'tag:yaml.org,2002:map', value) + + +yaml.add_representer(OrderedDict, represent_ordereddict) + +# Templates + + +def file_header(): + return ''' +# WARNING! Do not edit this file directly, it was generated by the ECS project. +# Please visit https://github.com/elastic/ecs to suggest changes to ECS fields. + +- key: ecs + title: ECS + description: ECS fields. + fields: +'''.lstrip() From 6e676dae3d0b34c177b101a24fa24fe0d852028e Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 7 Mar 2019 20:29:55 -0500 Subject: [PATCH 05/11] Add the 'generated file' warning to the output file --- generated/beats/fields.ecs.yml | 3 +++ scripts/generators/beats.py | 5 +---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/generated/beats/fields.ecs.yml b/generated/beats/fields.ecs.yml index 5bf7322ec0..2836c17d20 100644 --- a/generated/beats/fields.ecs.yml +++ b/generated/beats/fields.ecs.yml @@ -1,3 +1,6 @@ +# WARNING! Do not edit this file directly, it was generated by the ECS project. +# Please visit https://github.com/elastic/ecs to suggest changes to ECS fields. + - key: ecs title: ECS description: ECS Fields. diff --git a/scripts/generators/beats.py b/scripts/generators/beats.py index b65a962be4..afa7084411 100644 --- a/scripts/generators/beats.py +++ b/scripts/generators/beats.py @@ -24,6 +24,7 @@ def generate(ecs_nested, ecs_version): beats_file['fields'] = beats_fields with open('generated/beats/fields.ecs.yml', 'w') as outfile: + outfile.write(file_header()) yaml.dump([beats_file], outfile, default_flow_style=False) @@ -73,8 +74,4 @@ def file_header(): # WARNING! Do not edit this file directly, it was generated by the ECS project. # Please visit https://github.com/elastic/ecs to suggest changes to ECS fields. -- key: ecs - title: ECS - description: ECS fields. - fields: '''.lstrip() From 859a171eb165a34d84e90e42ec372329d58496d4 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 7 Mar 2019 20:32:37 -0500 Subject: [PATCH 06/11] Rename yaml marshaller for ordered dictionaries --- scripts/generators/beats.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/generators/beats.py b/scripts/generators/beats.py index afa7084411..f9b83eb2ce 100644 --- a/scripts/generators/beats.py +++ b/scripts/generators/beats.py @@ -55,7 +55,7 @@ def indent(text, indent, char=' '): # Rendering -def represent_ordereddict(dumper, data): +def yaml_ordereddict(dumper, data): value = [] for item_key, item_value in data.items(): node_key = dumper.represent_data(item_key) @@ -64,7 +64,7 @@ def represent_ordereddict(dumper, data): return yaml.nodes.MappingNode(u'tag:yaml.org,2002:map', value) -yaml.add_representer(OrderedDict, represent_ordereddict) +yaml.add_representer(OrderedDict, yaml_ordereddict) # Templates From 3cb227ba121fdb0824a860bd76539d37b71c51da Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 7 Mar 2019 20:36:41 -0500 Subject: [PATCH 07/11] Output footnote to beats fields --- generated/beats/fields.ecs.yml | 9 +++++++++ scripts/generators/beats.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/generated/beats/fields.ecs.yml b/generated/beats/fields.ecs.yml index 2836c17d20..b5d9149d47 100644 --- a/generated/beats/fields.ecs.yml +++ b/generated/beats/fields.ecs.yml @@ -58,6 +58,10 @@ Examples include Beats. Agents may also run on observers. ECS agent.* fields shall be populated with details of the agent running on the host or observer where the event happened or the measurement was taken.' + footnote: 'Examples: In the case of Beats for logs, the agent.name is filebeat. + For APM, it is the agent running in the app/service. The agent information does + not change if data is sent through queuing systems like Kafka, Redis, or processing + systems such as Logstash or APM Server.' type: group fields: - name: ephemeral_id @@ -241,6 +245,11 @@ group: 2 description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data + from its host, the cloud info contains the data about this machine. If Metricbeat + runs on a remote machine outside the cloud and fetches data from a service running + in the cloud, the field contains cloud data from the machine the service is + running on.' type: group fields: - name: availability_zone diff --git a/scripts/generators/beats.py b/scripts/generators/beats.py index f9b83eb2ce..47bfcca0aa 100644 --- a/scripts/generators/beats.py +++ b/scripts/generators/beats.py @@ -6,7 +6,7 @@ def generate(ecs_nested, ecs_version): # base first beats_fields = fieldset_field_array(ecs_nested['base']['fields']) - allowed_fieldset_keys = ['name', 'title', 'group', 'description', 'type'] + allowed_fieldset_keys = ['name', 'title', 'group', 'description', 'footnote', 'type'] # other fieldsets for fieldset_name in sorted(ecs_nested): if 'base' == fieldset_name: From e4082f97f88fd0ac310794f42e0d60a0842bdc37 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 7 Mar 2019 21:55:07 -0500 Subject: [PATCH 08/11] Fix nested field names in beats file, output ignore_above for keyword --- generated/beats/fields.ecs.yml | 431 +++++++++++++++++++++++++-------- scripts/generators/beats.py | 11 +- scripts/schema_reader.py | 1 - 3 files changed, 335 insertions(+), 108 deletions(-) diff --git a/generated/beats/fields.ecs.yml b/generated/beats/fields.ecs.yml index b5d9149d47..4f4c9f7715 100644 --- a/generated/beats/fields.ecs.yml +++ b/generated/beats/fields.ecs.yml @@ -46,6 +46,7 @@ - name: tags level: core type: keyword + ignore_above: 1024 description: List of keywords used to tag each event. example: '["production", "env2"]' - name: agent @@ -67,6 +68,7 @@ - name: ephemeral_id level: extended type: keyword + ignore_above: 1024 description: 'Ephemeral identifier of this agent (if one exists). This id normally changes across restarts, but `agent.id` does not.' @@ -74,11 +76,13 @@ - name: version level: core type: keyword + ignore_above: 1024 description: Version of the agent. example: 6.0.0-rc2 - name: type level: core type: keyword + ignore_above: 1024 description: 'Type of the agent. The agent type stays always the same and should be given by the agent used. @@ -88,6 +92,7 @@ - name: name level: core type: keyword + ignore_above: 1024 description: 'Custom name of the agent. This is a name that can be given to an agent. This can be helpful if for example @@ -99,6 +104,7 @@ - name: id level: core type: keyword + ignore_above: 1024 description: 'Unique identifier of this agent (if one exists). Example: For Beats this would be beat.id.' @@ -126,10 +132,12 @@ - name: domain level: core type: keyword + ignore_above: 1024 description: Client domain. - - name: region_name + - name: geo.region_name level: core type: keyword + ignore_above: 1024 description: Region name. example: Quebec - name: ip @@ -138,63 +146,74 @@ description: 'IP address of the client. Can be one or multiple IPv4 or IPv6 addresses.' - - name: continent_name + - name: geo.continent_name level: core type: keyword + ignore_above: 1024 description: Name of the continent. example: North America - - name: country_iso_code + - name: geo.country_iso_code level: core type: keyword + ignore_above: 1024 description: Country ISO code. example: CA - - name: hash + - name: user.hash level: extended type: keyword + ignore_above: 1024 description: 'Unique user hash to correlate information for a user in anonymized form. Useful if `user.id` or `user.name` contain confidential information and cannot be used.' - - name: id + - name: user.id level: core type: keyword + ignore_above: 1024 description: One or multiple unique identifiers of the user. - name: port level: core type: long description: Port of the client. - - name: country_name + - name: geo.country_name level: core type: keyword + ignore_above: 1024 description: Country name. example: Canada - - name: email + - name: user.email level: extended type: keyword + ignore_above: 1024 description: User email address. - - name: name + - name: user.group.name level: extended type: keyword + ignore_above: 1024 description: Name of the group. - - name: name + - name: user.name level: core type: keyword + ignore_above: 1024 description: Short name or login of the user. example: albert - - name: full_name + - name: user.full_name level: extended type: keyword + ignore_above: 1024 description: User's full name, if available. example: Albert Einstein - - name: city_name + - name: geo.city_name level: core type: keyword + ignore_above: 1024 description: City name. example: Montreal - - name: name + - name: geo.name level: extended type: keyword + ignore_above: 1024 description: 'User-defined description of a location, at the level of granularity they care about. @@ -203,25 +222,28 @@ Not typically used in automated geolocation.' example: boston-dc - - name: id + - name: user.group.id level: extended type: keyword + ignore_above: 1024 description: Unique identifier for the group on the system/platform. - name: address level: extended type: keyword + ignore_above: 1024 description: 'Some event client addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.' - - name: region_iso_code + - name: geo.region_iso_code level: core type: keyword + ignore_above: 1024 description: Region ISO code. example: CA-QC - - name: location + - name: geo.location level: core type: geo_point description: Longitude and latitude. @@ -239,6 +261,7 @@ - name: mac level: core type: keyword + ignore_above: 1024 description: MAC address of the client. - name: cloud title: Cloud @@ -255,25 +278,30 @@ - name: availability_zone level: extended type: keyword + ignore_above: 1024 description: Availability zone in which this host is running. example: us-east-1c - name: instance.name level: extended type: keyword + ignore_above: 1024 description: Instance name of the host machine. - name: region level: extended type: keyword + ignore_above: 1024 description: Region in which this host is running. example: us-east-1 - name: machine.type level: extended type: keyword + ignore_above: 1024 description: Machine type of the host machine. example: t2.medium - name: account.id level: extended type: keyword + ignore_above: 1024 description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. @@ -282,12 +310,14 @@ - name: provider level: extended type: keyword + ignore_above: 1024 description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. example: aws - name: instance.id level: extended type: keyword + ignore_above: 1024 description: Instance ID of the host machine. example: i-1234567890abcdef0 - name: container @@ -302,6 +332,7 @@ - name: name level: extended type: keyword + ignore_above: 1024 description: Container name. - name: labels level: extended @@ -311,19 +342,23 @@ - name: image.tag level: extended type: keyword + ignore_above: 1024 description: Container image tag. - name: runtime level: extended type: keyword + ignore_above: 1024 description: Runtime managing this container. example: docker - name: image.name level: extended type: keyword + ignore_above: 1024 description: Name of the image the container was built on. - name: id level: core type: keyword + ignore_above: 1024 description: Unique container id. - name: destination title: Destination @@ -336,10 +371,12 @@ - name: domain level: core type: keyword + ignore_above: 1024 description: Destination domain. - - name: region_name + - name: geo.region_name level: core type: keyword + ignore_above: 1024 description: Region name. example: Quebec - name: ip @@ -348,63 +385,74 @@ description: 'IP address of the destination. Can be one or multiple IPv4 or IPv6 addresses.' - - name: continent_name + - name: geo.continent_name level: core type: keyword + ignore_above: 1024 description: Name of the continent. example: North America - - name: country_iso_code + - name: geo.country_iso_code level: core type: keyword + ignore_above: 1024 description: Country ISO code. example: CA - - name: hash + - name: user.hash level: extended type: keyword + ignore_above: 1024 description: 'Unique user hash to correlate information for a user in anonymized form. Useful if `user.id` or `user.name` contain confidential information and cannot be used.' - - name: id + - name: user.id level: core type: keyword + ignore_above: 1024 description: One or multiple unique identifiers of the user. - name: port level: core type: long description: Port of the destination. - - name: country_name + - name: geo.country_name level: core type: keyword + ignore_above: 1024 description: Country name. example: Canada - - name: email + - name: user.email level: extended type: keyword + ignore_above: 1024 description: User email address. - - name: name + - name: user.group.name level: extended type: keyword + ignore_above: 1024 description: Name of the group. - - name: name + - name: user.name level: core type: keyword + ignore_above: 1024 description: Short name or login of the user. example: albert - - name: full_name + - name: user.full_name level: extended type: keyword + ignore_above: 1024 description: User's full name, if available. example: Albert Einstein - - name: city_name + - name: geo.city_name level: core type: keyword + ignore_above: 1024 description: City name. example: Montreal - - name: name + - name: geo.name level: extended type: keyword + ignore_above: 1024 description: 'User-defined description of a location, at the level of granularity they care about. @@ -413,25 +461,28 @@ Not typically used in automated geolocation.' example: boston-dc - - name: id + - name: user.group.id level: extended type: keyword + ignore_above: 1024 description: Unique identifier for the group on the system/platform. - name: address level: extended type: keyword + ignore_above: 1024 description: 'Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.' - - name: region_iso_code + - name: geo.region_iso_code level: core type: keyword + ignore_above: 1024 description: Region ISO code. example: CA-QC - - name: location + - name: geo.location level: core type: geo_point description: Longitude and latitude. @@ -449,6 +500,7 @@ - name: mac level: core type: keyword + ignore_above: 1024 description: MAC address of the destination. - name: ecs title: ECS @@ -460,6 +512,7 @@ level: core required: true type: keyword + ignore_above: 1024 description: 'ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. @@ -483,10 +536,12 @@ - name: code level: core type: keyword + ignore_above: 1024 description: Error code describing the error. - name: id level: core type: keyword + ignore_above: 1024 description: Unique identifier for the error. - name: event title: Event @@ -507,6 +562,7 @@ - name: category level: core type: keyword + ignore_above: 1024 description: 'Event category. This contains high-level information about the contents of the event. It is @@ -522,6 +578,7 @@ - name: kind level: extended type: keyword + ignore_above: 1024 description: 'The kind of the event. This gives information about what type of information the event contains, @@ -532,6 +589,7 @@ - name: hash level: extended type: keyword + ignore_above: 1024 description: Hash (perhaps logstash fingerprint) of raw field to be able to demonstrate log integrity. example: 123456789012345678901234567890ABCD @@ -560,6 +618,7 @@ - name: original level: core type: keyword + ignore_above: 1024 description: 'Raw text message of entire event. Used to demonstrate log integrity. This field is not indexed and doc_values are disabled. It cannot be searched, @@ -577,6 +636,7 @@ - name: module level: core type: keyword + ignore_above: 1024 description: 'Name of the module this data is coming from. This information is coming from the modules used in Beats or Logstash.' @@ -584,6 +644,7 @@ - name: dataset level: core type: keyword + ignore_above: 1024 description: 'Name of the dataset. The concept of a `dataset` (fileset / metricset) is used in Beats as a subset @@ -610,6 +671,7 @@ - name: action level: core type: keyword + ignore_above: 1024 description: 'The action captured by the event. This describes the information in the event. It is more specific than `event.category`. @@ -619,6 +681,7 @@ - name: timezone level: extended type: keyword + ignore_above: 1024 description: 'This field should be populated when the event''s timestamp does not include timezone information already (e.g. default Syslog timestamps). It''s optional otherwise. @@ -628,6 +691,7 @@ - name: outcome level: extended type: keyword + ignore_above: 1024 description: 'The outcome of the event. If the event describes an action, this fields contains the outcome of that @@ -638,12 +702,14 @@ - name: type level: core type: keyword + ignore_above: 1024 description: 'Reserved for future usage. Please avoid using this field for user data.' - name: id level: core type: keyword + ignore_above: 1024 description: Unique ID to describe the event. example: 8a4f500d - name: file @@ -661,14 +727,17 @@ - name: group level: extended type: keyword + ignore_above: 1024 description: Primary group name of the file. - name: uid level: extended type: keyword + ignore_above: 1024 description: The user ID (UID) or security identifier (SID) of the file owner. - name: extension level: extended type: keyword + ignore_above: 1024 description: 'File extension. This should allow easy filtering by file extensions.' @@ -676,6 +745,7 @@ - name: type level: extended type: keyword + ignore_above: 1024 description: File type (file, dir, or symlink). - name: ctime level: extended @@ -684,18 +754,22 @@ - name: target_path level: extended type: keyword + ignore_above: 1024 description: Target path for symlinks. - name: owner level: extended type: keyword + ignore_above: 1024 description: File owner's username. - name: gid level: extended type: keyword + ignore_above: 1024 description: Primary group ID (GID) of the file. - name: mode level: extended type: keyword + ignore_above: 1024 description: Mode of the file in octal representation. example: 416 - name: mtime @@ -705,14 +779,17 @@ - name: device level: extended type: keyword + ignore_above: 1024 description: Device that is the source of the file. - name: path level: extended type: keyword + ignore_above: 1024 description: Path to the file. - name: inode level: extended type: keyword + ignore_above: 1024 description: Inode representing the file in the filesystem. - name: size level: extended @@ -731,16 +808,19 @@ - name: region_name level: core type: keyword + ignore_above: 1024 description: Region name. example: Quebec - name: country_iso_code level: core type: keyword + ignore_above: 1024 description: Country ISO code. example: CA - name: name level: extended type: keyword + ignore_above: 1024 description: 'User-defined description of a location, at the level of granularity they care about. @@ -752,11 +832,13 @@ - name: country_name level: core type: keyword + ignore_above: 1024 description: Country name. example: Canada - name: city_name level: core type: keyword + ignore_above: 1024 description: City name. example: Montreal - name: location @@ -767,11 +849,13 @@ - name: continent_name level: core type: keyword + ignore_above: 1024 description: Name of the continent. example: North America - name: region_iso_code level: core type: keyword + ignore_above: 1024 description: Region ISO code. example: CA-QC - name: group @@ -784,10 +868,12 @@ - name: id level: extended type: keyword + ignore_above: 1024 description: Unique identifier for the group on the system/platform. - name: name level: extended type: keyword + ignore_above: 1024 description: Name of the group. - name: host title: Host @@ -799,110 +885,129 @@ hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: region_name + - name: geo.region_name level: core type: keyword + ignore_above: 1024 description: Region name. example: Quebec - name: ip level: core type: ip description: Host ip address. - - name: continent_name + - name: geo.continent_name level: core type: keyword + ignore_above: 1024 description: Name of the continent. example: North America - - name: country_iso_code + - name: geo.country_iso_code level: core type: keyword + ignore_above: 1024 description: Country ISO code. example: CA - - name: hash + - name: user.hash level: extended type: keyword + ignore_above: 1024 description: 'Unique user hash to correlate information for a user in anonymized form. Useful if `user.id` or `user.name` contain confidential information and cannot be used.' - - name: family + - name: os.family level: extended type: keyword + ignore_above: 1024 description: OS family (such as redhat, debian, freebsd, windows). example: debian - - name: id + - name: user.id level: core type: keyword + ignore_above: 1024 description: One or multiple unique identifiers of the user. - - name: name + - name: os.name level: extended type: keyword + ignore_above: 1024 description: Operating system name, without the version. example: Mac OS X - name: id level: core type: keyword + ignore_above: 1024 description: 'Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: kernel + - name: os.kernel level: extended type: keyword + ignore_above: 1024 description: Operating system kernel version as a raw string. example: 4.4.0-112-generic - - name: country_name + - name: geo.country_name level: core type: keyword + ignore_above: 1024 description: Country name. example: Canada - name: hostname level: core type: keyword + ignore_above: 1024 description: 'Hostname of the host. It normally contains what the `hostname` command returns on the host machine.' - - name: email + - name: user.email level: extended type: keyword + ignore_above: 1024 description: User email address. - - name: name + - name: user.group.name level: extended type: keyword + ignore_above: 1024 description: Name of the group. - - name: name + - name: user.name level: core type: keyword + ignore_above: 1024 description: Short name or login of the user. example: albert - name: type level: core type: keyword + ignore_above: 1024 description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - - name: full_name + - name: user.full_name level: extended type: keyword + ignore_above: 1024 description: User's full name, if available. example: Albert Einstein - - name: city_name + - name: geo.city_name level: core type: keyword + ignore_above: 1024 description: City name. example: Montreal - - name: version + - name: os.version level: extended type: keyword + ignore_above: 1024 description: Operating system version as a raw string. example: 10.14.1 - - name: name + - name: geo.name level: extended type: keyword + ignore_above: 1024 description: 'User-defined description of a location, at the level of granularity they care about. @@ -914,42 +1019,49 @@ - name: mac level: core type: keyword + ignore_above: 1024 description: Host mac address. - - name: platform + - name: os.platform level: extended type: keyword + ignore_above: 1024 description: Operating system platform (such centos, ubuntu, windows). example: darwin - name: name level: core type: keyword + ignore_above: 1024 description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: region_iso_code + - name: geo.region_iso_code level: core type: keyword + ignore_above: 1024 description: Region ISO code. example: CA-QC - - name: location + - name: geo.location level: core type: geo_point description: Longitude and latitude. example: '{ "lon": -73.614830, "lat": 45.505918 }' - - name: id + - name: user.group.id level: extended type: keyword + ignore_above: 1024 description: Unique identifier for the group on the system/platform. - name: architecture level: core type: keyword + ignore_above: 1024 description: Operating system architecture. example: x86_64 - - name: full + - name: os.full level: extended type: keyword + ignore_above: 1024 description: Operating system name, including the version or code name. example: Mac OS Mojave - name: http @@ -967,6 +1079,7 @@ - name: request.method level: extended type: keyword + ignore_above: 1024 description: 'HTTP request method. The field value must be normalized to lowercase for querying. See the documentation @@ -985,6 +1098,7 @@ - name: request.body.content level: extended type: keyword + ignore_above: 1024 description: The full HTTP request body. example: Hello world - name: response.body.bytes @@ -995,6 +1109,7 @@ - name: version level: extended type: keyword + ignore_above: 1024 description: HTTP version. example: 1.1 - name: response.status_code @@ -1005,11 +1120,13 @@ - name: request.referrer level: extended type: keyword + ignore_above: 1024 description: Referrer for this HTTP request. example: https://blog.example.com/ - name: response.body.content level: extended type: keyword + ignore_above: 1024 description: The full HTTP response body. example: Hello world - name: log @@ -1021,6 +1138,7 @@ - name: original level: core type: keyword + ignore_above: 1024 description: 'This is the original log message and contains the full log message before splitting it up in multiple parts. @@ -1035,6 +1153,7 @@ - name: level level: core type: keyword + ignore_above: 1024 description: 'Original log level of the log event. Some examples are `warn`, `error`, `i`.' @@ -1052,6 +1171,7 @@ - name: direction level: core type: keyword + ignore_above: 1024 description: "Direction of the network traffic.\nRecommended values are:\n \ \ * inbound\n * outbound\n * internal\n * external\n * unknown\n\nWhen\ \ mapping events from a host-based monitoring context, populate this field\ @@ -1070,6 +1190,7 @@ - name: protocol level: core type: keyword + ignore_above: 1024 description: 'L7 Network protocol name. ex. http, lumberjack, transport protocol. The field value must be normalized to lowercase for querying. See the documentation @@ -1078,6 +1199,7 @@ - name: name level: extended type: keyword + ignore_above: 1024 description: Name given by operators to sections of their network. example: Guest Wifi - name: bytes @@ -1091,6 +1213,7 @@ - name: iana_number level: extended type: keyword + ignore_above: 1024 description: IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). Standardized list of protocols. This aligns well with NetFlow and sFlow related logs which use the IANA Protocol Number. @@ -1098,6 +1221,7 @@ - name: community_id level: extended type: keyword + ignore_above: 1024 description: 'A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows. @@ -1107,6 +1231,7 @@ - name: application level: extended type: keyword + ignore_above: 1024 description: 'A name given to an application level protocol. This can be arbitrarily assigned for things like microservices, but also apply to things like skype, icq, facebook, twitter. This would be used in situations where the vendor @@ -1124,6 +1249,7 @@ - name: type level: core type: keyword + ignore_above: 1024 description: 'In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc @@ -1133,6 +1259,7 @@ - name: transport level: core type: keyword + ignore_above: 1024 description: 'Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) @@ -1156,40 +1283,46 @@ are not considered observers in ECS.' type: group fields: - - name: region_name + - name: geo.region_name level: core type: keyword + ignore_above: 1024 description: Region name. example: Quebec - - name: continent_name + - name: geo.continent_name level: core type: keyword + ignore_above: 1024 description: Name of the continent. example: North America - name: vendor level: core type: keyword + ignore_above: 1024 description: observer vendor information. - - name: city_name + - name: geo.city_name level: core type: keyword + ignore_above: 1024 description: City name. example: Montreal - - name: country_name + - name: geo.country_name level: core type: keyword + ignore_above: 1024 description: Country name. example: Canada - - name: country_iso_code + - name: geo.country_iso_code level: core type: keyword + ignore_above: 1024 description: Country ISO code. example: CA - name: ip level: core type: ip description: IP address of the observer. - - name: location + - name: geo.location level: core type: geo_point description: Longitude and latitude. @@ -1197,15 +1330,18 @@ - name: hostname level: core type: keyword + ignore_above: 1024 description: Hostname of the observer. - - name: kernel + - name: os.kernel level: extended type: keyword + ignore_above: 1024 description: Operating system kernel version as a raw string. example: 4.4.0-112-generic - - name: name + - name: geo.name level: extended type: keyword + ignore_above: 1024 description: 'User-defined description of a location, at the level of granularity they care about. @@ -1214,54 +1350,64 @@ Not typically used in automated geolocation.' example: boston-dc - - name: region_iso_code + - name: geo.region_iso_code level: core type: keyword + ignore_above: 1024 description: Region ISO code. example: CA-QC - name: mac level: core type: keyword + ignore_above: 1024 description: MAC address of the observer - name: version level: core type: keyword + ignore_above: 1024 description: Observer version. - - name: family + - name: os.family level: extended type: keyword + ignore_above: 1024 description: OS family (such as redhat, debian, freebsd, windows). example: debian - - name: full + - name: os.full level: extended type: keyword + ignore_above: 1024 description: Operating system name, including the version or code name. example: Mac OS Mojave - name: serial_number level: extended type: keyword + ignore_above: 1024 description: Observer serial number. - - name: name + - name: os.name level: extended type: keyword + ignore_above: 1024 description: Operating system name, without the version. example: Mac OS X - name: type level: core type: keyword + ignore_above: 1024 description: 'The type of the observer the data is coming from. There is no predefined list of observer types. Some examples are `forwarder`, `firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`.' example: firewall - - name: version + - name: os.version level: extended type: keyword + ignore_above: 1024 description: Operating system version as a raw string. example: 10.14.1 - - name: platform + - name: os.platform level: extended type: keyword + ignore_above: 1024 description: Operating system platform (such centos, ubuntu, windows). example: darwin - name: organization @@ -1277,10 +1423,12 @@ - name: name level: extended type: keyword + ignore_above: 1024 description: Organization name. - name: id level: extended type: keyword + ignore_above: 1024 description: Unique identifier for the organization. - name: os title: Operating System @@ -1291,31 +1439,37 @@ - name: kernel level: extended type: keyword + ignore_above: 1024 description: Operating system kernel version as a raw string. example: 4.4.0-112-generic - name: full level: extended type: keyword + ignore_above: 1024 description: Operating system name, including the version or code name. example: Mac OS Mojave - name: name level: extended type: keyword + ignore_above: 1024 description: Operating system name, without the version. example: Mac OS X - name: family level: extended type: keyword + ignore_above: 1024 description: OS family (such as redhat, debian, freebsd, windows). example: debian - name: platform level: extended type: keyword + ignore_above: 1024 description: Operating system platform (such centos, ubuntu, windows). example: darwin - name: version level: extended type: keyword + ignore_above: 1024 description: Operating system version as a raw string. example: 10.14.1 - name: process @@ -1331,6 +1485,7 @@ - name: executable level: extended type: keyword + ignore_above: 1024 description: Absolute path to the process executable. example: /usr/bin/ssh - name: thread.id @@ -1341,6 +1496,7 @@ - name: name level: extended type: keyword + ignore_above: 1024 description: 'Process name. Sometimes called program name or similar.' @@ -1348,6 +1504,7 @@ - name: title level: extended type: keyword + ignore_above: 1024 description: 'Process title. The proctitle, some times the same as process name. Can also be different: @@ -1355,6 +1512,7 @@ - name: args level: extended type: keyword + ignore_above: 1024 description: 'Array of process arguments. May be filtered to protect sensitive information.' @@ -1375,6 +1533,7 @@ - name: working_directory level: extended type: keyword + ignore_above: 1024 description: The working directory of the process. example: /home/alice - name: ppid @@ -1424,10 +1583,12 @@ - name: domain level: core type: keyword + ignore_above: 1024 description: Server domain. - - name: region_name + - name: geo.region_name level: core type: keyword + ignore_above: 1024 description: Region name. example: Quebec - name: ip @@ -1436,63 +1597,74 @@ description: 'IP address of the server. Can be one or multiple IPv4 or IPv6 addresses.' - - name: continent_name + - name: geo.continent_name level: core type: keyword + ignore_above: 1024 description: Name of the continent. example: North America - - name: country_iso_code + - name: geo.country_iso_code level: core type: keyword + ignore_above: 1024 description: Country ISO code. example: CA - - name: hash + - name: user.hash level: extended type: keyword + ignore_above: 1024 description: 'Unique user hash to correlate information for a user in anonymized form. Useful if `user.id` or `user.name` contain confidential information and cannot be used.' - - name: id + - name: user.id level: core type: keyword + ignore_above: 1024 description: One or multiple unique identifiers of the user. - name: port level: core type: long description: Port of the server. - - name: country_name + - name: geo.country_name level: core type: keyword + ignore_above: 1024 description: Country name. example: Canada - - name: email + - name: user.email level: extended type: keyword + ignore_above: 1024 description: User email address. - - name: name + - name: user.group.name level: extended type: keyword + ignore_above: 1024 description: Name of the group. - - name: name + - name: user.name level: core type: keyword + ignore_above: 1024 description: Short name or login of the user. example: albert - - name: full_name + - name: user.full_name level: extended type: keyword + ignore_above: 1024 description: User's full name, if available. example: Albert Einstein - - name: city_name + - name: geo.city_name level: core type: keyword + ignore_above: 1024 description: City name. example: Montreal - - name: name + - name: geo.name level: extended type: keyword + ignore_above: 1024 description: 'User-defined description of a location, at the level of granularity they care about. @@ -1501,25 +1673,28 @@ Not typically used in automated geolocation.' example: boston-dc - - name: id + - name: user.group.id level: extended type: keyword + ignore_above: 1024 description: Unique identifier for the group on the system/platform. - name: address level: extended type: keyword + ignore_above: 1024 description: 'Some event server addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.' - - name: region_iso_code + - name: geo.region_iso_code level: core type: keyword + ignore_above: 1024 description: Region ISO code. example: CA-QC - - name: location + - name: geo.location level: core type: geo_point description: Longitude and latitude. @@ -1537,6 +1712,7 @@ - name: mac level: core type: keyword + ignore_above: 1024 description: MAC address of the server. - name: service title: Service @@ -1550,6 +1726,7 @@ - name: ephemeral_id level: extended type: keyword + ignore_above: 1024 description: 'Ephemeral identifier of this service (if one exists). This id normally changes across restarts, but `service.id` does not.' @@ -1557,6 +1734,7 @@ - name: name level: core type: keyword + ignore_above: 1024 description: 'Name of the service data is collected from. The name of the service is normally user given. This allows if two instances @@ -1573,10 +1751,12 @@ - name: state level: core type: keyword + ignore_above: 1024 description: Current state of the service. - name: version level: core type: keyword + ignore_above: 1024 description: 'Version of the service the data was collected from. This allows to look at a data set only for a specific version of a service.' @@ -1584,6 +1764,7 @@ - name: type level: core type: keyword + ignore_above: 1024 description: 'The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service @@ -1595,6 +1776,7 @@ - name: id level: core type: keyword + ignore_above: 1024 description: 'Unique identifier of the running service. This id should uniquely identify this service. This makes it possible to correlate @@ -1614,10 +1796,12 @@ - name: domain level: core type: keyword + ignore_above: 1024 description: Source domain. - - name: region_name + - name: geo.region_name level: core type: keyword + ignore_above: 1024 description: Region name. example: Quebec - name: ip @@ -1626,63 +1810,74 @@ description: 'IP address of the source. Can be one or multiple IPv4 or IPv6 addresses.' - - name: continent_name + - name: geo.continent_name level: core type: keyword + ignore_above: 1024 description: Name of the continent. example: North America - - name: country_iso_code + - name: geo.country_iso_code level: core type: keyword + ignore_above: 1024 description: Country ISO code. example: CA - - name: hash + - name: user.hash level: extended type: keyword + ignore_above: 1024 description: 'Unique user hash to correlate information for a user in anonymized form. Useful if `user.id` or `user.name` contain confidential information and cannot be used.' - - name: id + - name: user.id level: core type: keyword + ignore_above: 1024 description: One or multiple unique identifiers of the user. - name: port level: core type: long description: Port of the source. - - name: country_name + - name: geo.country_name level: core type: keyword + ignore_above: 1024 description: Country name. example: Canada - - name: email + - name: user.email level: extended type: keyword + ignore_above: 1024 description: User email address. - - name: name + - name: user.group.name level: extended type: keyword + ignore_above: 1024 description: Name of the group. - - name: name + - name: user.name level: core type: keyword + ignore_above: 1024 description: Short name or login of the user. example: albert - - name: full_name + - name: user.full_name level: extended type: keyword + ignore_above: 1024 description: User's full name, if available. example: Albert Einstein - - name: city_name + - name: geo.city_name level: core type: keyword + ignore_above: 1024 description: City name. example: Montreal - - name: name + - name: geo.name level: extended type: keyword + ignore_above: 1024 description: 'User-defined description of a location, at the level of granularity they care about. @@ -1691,25 +1886,28 @@ Not typically used in automated geolocation.' example: boston-dc - - name: id + - name: user.group.id level: extended type: keyword + ignore_above: 1024 description: Unique identifier for the group on the system/platform. - name: address level: extended type: keyword + ignore_above: 1024 description: 'Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.' - - name: region_iso_code + - name: geo.region_iso_code level: core type: keyword + ignore_above: 1024 description: Region ISO code. example: CA-QC - - name: location + - name: geo.location level: core type: geo_point description: Longitude and latitude. @@ -1727,6 +1925,7 @@ - name: mac level: core type: keyword + ignore_above: 1024 description: MAC address of the source. - name: url title: URL @@ -1738,10 +1937,12 @@ - name: username level: extended type: keyword + ignore_above: 1024 description: Username of the request. - name: domain level: extended type: keyword + ignore_above: 1024 description: 'Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain @@ -1750,6 +1951,7 @@ - name: full level: extended type: keyword + ignore_above: 1024 description: If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. @@ -1757,6 +1959,7 @@ - name: fragment level: extended type: keyword + ignore_above: 1024 description: 'Portion of the url after the `#`, such as "top". The `#` is not part of the fragment.' @@ -1768,10 +1971,12 @@ - name: password level: extended type: keyword + ignore_above: 1024 description: Password of the request. - name: query level: extended type: keyword + ignore_above: 1024 description: 'The query field describes the query string of the request, such as "q=elasticsearch". @@ -1782,10 +1987,12 @@ - name: path level: extended type: keyword + ignore_above: 1024 description: Path of the request, such as "/search". - name: scheme level: extended type: keyword + ignore_above: 1024 description: 'Scheme of the request, such as "https". Note: The `:` is not part of the scheme.' @@ -1793,6 +2000,7 @@ - name: original level: extended type: keyword + ignore_above: 1024 description: 'Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas @@ -1813,6 +2021,7 @@ - name: hash level: extended type: keyword + ignore_above: 1024 description: 'Unique user hash to correlate information for a user in anonymized form. @@ -1821,28 +2030,34 @@ - name: name level: core type: keyword + ignore_above: 1024 description: Short name or login of the user. example: albert - - name: id + - name: group.id level: extended type: keyword + ignore_above: 1024 description: Unique identifier for the group on the system/platform. - name: id level: core type: keyword + ignore_above: 1024 description: One or multiple unique identifiers of the user. - name: full_name level: extended type: keyword + ignore_above: 1024 description: User's full name, if available. example: Albert Einstein - name: email level: extended type: keyword + ignore_above: 1024 description: User email address. - - name: name + - name: group.name level: extended type: keyword + ignore_above: 1024 description: Name of the group. - name: user_agent title: User agent @@ -1852,54 +2067,64 @@ They often show up in web service logs coming from the parsed user agent string.' type: group fields: - - name: kernel + - name: os.kernel level: extended type: keyword + ignore_above: 1024 description: Operating system kernel version as a raw string. example: 4.4.0-112-generic - name: name level: extended type: keyword + ignore_above: 1024 description: Name of the user agent. example: Safari - - name: version + - name: os.version level: extended type: keyword + ignore_above: 1024 description: Operating system version as a raw string. example: 10.14.1 - name: version level: extended type: keyword + ignore_above: 1024 description: Version of the user agent. example: 12.0 - - name: family + - name: os.family level: extended type: keyword + ignore_above: 1024 description: OS family (such as redhat, debian, freebsd, windows). example: debian - name: device.name level: extended type: keyword + ignore_above: 1024 description: Name of the device. example: iPhone - - name: name + - name: os.name level: extended type: keyword + ignore_above: 1024 description: Operating system name, without the version. example: Mac OS X - - name: full + - name: os.full level: extended type: keyword + ignore_above: 1024 description: Operating system name, including the version or code name. example: Mac OS Mojave - name: original level: extended type: keyword + ignore_above: 1024 description: Unparsed version of the user_agent. example: Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1 - - name: platform + - name: os.platform level: extended type: keyword + ignore_above: 1024 description: Operating system platform (such centos, ubuntu, windows). example: darwin diff --git a/scripts/generators/beats.py b/scripts/generators/beats.py index 47bfcca0aa..68f2c9a591 100644 --- a/scripts/generators/beats.py +++ b/scripts/generators/beats.py @@ -30,11 +30,14 @@ def generate(ecs_nested, ecs_version): def fieldset_field_array(source_fields): allowed_keys = ['name', 'level', 'required', 'type', 'object_type', - 'multi_fields', 'format', 'description', 'example'] + 'ignore_above', 'multi_fields', 'format', + 'description', 'example'] fields = [] - for field_name in source_fields: - field = source_fields[field_name] - fields.append(dict_copy_keys_ordered(field, allowed_keys)) + for nested_field_name in source_fields: + ecs_field = source_fields[nested_field_name] + beats_field = dict_copy_keys_ordered(ecs_field, allowed_keys) + beats_field['name'] = nested_field_name + fields.append(beats_field) return fields # Helpers diff --git a/scripts/schema_reader.py b/scripts/schema_reader.py index 960f03c162..66d2a4b5b6 100644 --- a/scripts/schema_reader.py +++ b/scripts/schema_reader.py @@ -35,7 +35,6 @@ def dict_clean_string_values(dict): value = dict[key] if isinstance(value, basestring): dict[key] = value.strip() - # TODO: Remove trailing \n? def dict_set_default(dict, key, default): From 30ad1758389b3720c947bc537ff929323e8e15af Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Fri, 8 Mar 2019 06:37:33 -0500 Subject: [PATCH 09/11] Include ECS version in Beats file warning --- generated/beats/fields.ecs.yml | 3 ++- scripts/generators/beats.py | 14 ++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/generated/beats/fields.ecs.yml b/generated/beats/fields.ecs.yml index 4f4c9f7715..eafe1e2899 100644 --- a/generated/beats/fields.ecs.yml +++ b/generated/beats/fields.ecs.yml @@ -1,4 +1,5 @@ -# WARNING! Do not edit this file directly, it was generated by the ECS project. +# WARNING! Do not edit this file directly, it was generated by the ECS project, +# based on ECS version 1.1.0-dev. # Please visit https://github.com/elastic/ecs to suggest changes to ECS fields. - key: ecs diff --git a/scripts/generators/beats.py b/scripts/generators/beats.py index 68f2c9a591..d43c680a51 100644 --- a/scripts/generators/beats.py +++ b/scripts/generators/beats.py @@ -23,9 +23,7 @@ def generate(ecs_nested, ecs_version): beats_file['description'] = 'ECS Fields.' beats_file['fields'] = beats_fields - with open('generated/beats/fields.ecs.yml', 'w') as outfile: - outfile.write(file_header()) - yaml.dump([beats_file], outfile, default_flow_style=False) + write_beats_yaml(beats_file, ecs_version) def fieldset_field_array(source_fields): @@ -43,6 +41,13 @@ def fieldset_field_array(source_fields): # Helpers +def write_beats_yaml(beats_file, ecs_version): + + with open('generated/beats/fields.ecs.yml', 'w') as outfile: + outfile.write(file_header().format(version=ecs_version)) + yaml.dump([beats_file], outfile, default_flow_style=False) + + def dict_copy_keys_ordered(dict, copied_keys): ordered_dict = OrderedDict() for key in copied_keys: @@ -74,7 +79,8 @@ def yaml_ordereddict(dumper, data): def file_header(): return ''' -# WARNING! Do not edit this file directly, it was generated by the ECS project. +# WARNING! Do not edit this file directly, it was generated by the ECS project, +# based on ECS version {version}. # Please visit https://github.com/elastic/ecs to suggest changes to ECS fields. '''.lstrip() From 6f6efb711ce367eb9b69a08a2ed82514120e1a84 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Fri, 8 Mar 2019 06:42:11 -0500 Subject: [PATCH 10/11] Changelog --- CHANGELOG.next.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.md b/CHANGELOG.next.md index 50a46cf6b2..8390424769 100644 --- a/CHANGELOG.next.md +++ b/CHANGELOG.next.md @@ -10,6 +10,7 @@ * New generator that supports reusable fields, for files based on ECS. It generates schema.csv, Elasticsearch 6 and 7 templates, and field documentation for the main website. #336 +* Generator for the Beats fields.ecs.yml file. #379 ### Improvements From 1cf391c047a97e3a4a25572a15702dbdda0a15f5 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Fri, 8 Mar 2019 14:01:37 -0500 Subject: [PATCH 11/11] Sort the field arrays in beats file --- generated/beats/fields.ecs.yml | 1744 ++++++++++++++++---------------- scripts/generators/beats.py | 2 +- 2 files changed, 873 insertions(+), 873 deletions(-) diff --git a/generated/beats/fields.ecs.yml b/generated/beats/fields.ecs.yml index eafe1e2899..85ddeb6bf8 100644 --- a/generated/beats/fields.ecs.yml +++ b/generated/beats/fields.ecs.yml @@ -74,22 +74,14 @@ This id normally changes across restarts, but `agent.id` does not.' example: 8a4f500f - - name: version - level: core - type: keyword - ignore_above: 1024 - description: Version of the agent. - example: 6.0.0-rc2 - - name: type + - name: id level: core type: keyword ignore_above: 1024 - description: 'Type of the agent. + description: 'Unique identifier of this agent (if one exists). - The agent type stays always the same and should be given by the agent used. - In case of Filebeat the agent would always be Filebeat also if two Filebeat - instances are run on the same machine.' - example: filebeat + Example: For Beats this would be beat.id.' + example: 8a4f500d - name: name level: core type: keyword @@ -102,14 +94,22 @@ If no name is given, the name is often left empty.' example: foo - - name: id + - name: type level: core type: keyword ignore_above: 1024 - description: 'Unique identifier of this agent (if one exists). + description: 'Type of the agent. - Example: For Beats this would be beat.id.' - example: 8a4f500d + The agent type stays always the same and should be given by the agent used. + In case of Filebeat the agent would always be Filebeat also if two Filebeat + instances are run on the same machine.' + example: filebeat + - name: version + level: core + type: keyword + ignore_above: 1024 + description: Version of the agent. + example: 6.0.0-rc2 - name: client title: Client group: 2 @@ -130,23 +130,32 @@ appropriately.' type: group fields: + - name: address + level: extended + type: keyword + ignore_above: 1024 + description: 'Some event client addresses are defined ambiguously. The event + will sometimes list an IP, a domain or a unix socket. You should always store + the raw address in the `.address` field. + + Then it should be duplicated to `.ip` or `.domain`, depending on which one + it is.' + - name: bytes + level: core + type: long + description: Bytes sent from the client to the server. + example: 184 - name: domain level: core type: keyword ignore_above: 1024 description: Client domain. - - name: geo.region_name + - name: geo.city_name level: core type: keyword ignore_above: 1024 - description: Region name. - example: Quebec - - name: ip - level: core - type: ip - description: 'IP address of the client. - - Can be one or multiple IPv4 or IPv6 addresses.' + description: City name. + example: Montreal - name: geo.continent_name level: core type: keyword @@ -159,111 +168,102 @@ ignore_above: 1024 description: Country ISO code. example: CA - - name: user.hash + - name: geo.country_name + level: core + type: keyword + ignore_above: 1024 + description: Country name. + example: Canada + - name: geo.location + level: core + type: geo_point + description: Longitude and latitude. + example: '{ "lon": -73.614830, "lat": 45.505918 }' + - name: geo.name level: extended type: keyword ignore_above: 1024 - description: 'Unique user hash to correlate information for a user in anonymized - form. + description: 'User-defined description of a location, at the level of granularity + they care about. - Useful if `user.id` or `user.name` contain confidential information and cannot - be used.' - - name: user.id + Could be the name of their data centers, the floor number, if this describes + a local physical entity, city names. + + Not typically used in automated geolocation.' + example: boston-dc + - name: geo.region_iso_code level: core type: keyword ignore_above: 1024 - description: One or multiple unique identifiers of the user. - - name: port + description: Region ISO code. + example: CA-QC + - name: geo.region_name level: core - type: long - description: Port of the client. - - name: geo.country_name + type: keyword + ignore_above: 1024 + description: Region name. + example: Quebec + - name: ip + level: core + type: ip + description: 'IP address of the client. + + Can be one or multiple IPv4 or IPv6 addresses.' + - name: mac level: core type: keyword ignore_above: 1024 - description: Country name. - example: Canada + description: MAC address of the client. + - name: packets + level: core + type: long + description: Packets sent from the client to the server. + example: 12 + - name: port + level: core + type: long + description: Port of the client. - name: user.email level: extended type: keyword ignore_above: 1024 description: User email address. - - name: user.group.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the group. - - name: user.name - level: core - type: keyword - ignore_above: 1024 - description: Short name or login of the user. - example: albert - name: user.full_name level: extended type: keyword ignore_above: 1024 description: User's full name, if available. example: Albert Einstein - - name: geo.city_name - level: core - type: keyword - ignore_above: 1024 - description: City name. - example: Montreal - - name: geo.name + - name: user.group.id level: extended type: keyword ignore_above: 1024 - description: 'User-defined description of a location, at the level of granularity - they care about. - - Could be the name of their data centers, the floor number, if this describes - a local physical entity, city names. - - Not typically used in automated geolocation.' - example: boston-dc - - name: user.group.id + description: Unique identifier for the group on the system/platform. + - name: user.group.name level: extended type: keyword ignore_above: 1024 - description: Unique identifier for the group on the system/platform. - - name: address + description: Name of the group. + - name: user.hash level: extended type: keyword ignore_above: 1024 - description: 'Some event client addresses are defined ambiguously. The event - will sometimes list an IP, a domain or a unix socket. You should always store - the raw address in the `.address` field. + description: 'Unique user hash to correlate information for a user in anonymized + form. - Then it should be duplicated to `.ip` or `.domain`, depending on which one - it is.' - - name: geo.region_iso_code + Useful if `user.id` or `user.name` contain confidential information and cannot + be used.' + - name: user.id level: core type: keyword ignore_above: 1024 - description: Region ISO code. - example: CA-QC - - name: geo.location - level: core - type: geo_point - description: Longitude and latitude. - example: '{ "lon": -73.614830, "lat": 45.505918 }' - - name: bytes - level: core - type: long - description: Bytes sent from the client to the server. - example: 184 - - name: packets - level: core - type: long - description: Packets sent from the client to the server. - example: 12 - - name: mac + description: One or multiple unique identifiers of the user. + - name: user.name level: core type: keyword ignore_above: 1024 - description: MAC address of the client. + description: Short name or login of the user. + example: albert - name: cloud title: Cloud group: 2 @@ -276,38 +276,38 @@ running on.' type: group fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different + entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 - name: availability_zone level: extended type: keyword ignore_above: 1024 description: Availability zone in which this host is running. example: us-east-1c - - name: instance.name + - name: instance.id level: extended type: keyword ignore_above: 1024 - description: Instance name of the host machine. - - name: region + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name level: extended type: keyword ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 + description: Instance name of the host machine. - name: machine.type level: extended type: keyword ignore_above: 1024 description: Machine type of the host machine. example: t2.medium - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different - entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - name: provider level: extended type: keyword @@ -315,12 +315,12 @@ description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. example: aws - - name: instance.id + - name: region level: extended type: keyword ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 + description: Region in which this host is running. + example: us-east-1 - name: container title: Container group: 2 @@ -330,37 +330,37 @@ These fields help correlate data based containers from any runtime.' type: group fields: - - name: name + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name level: extended type: keyword ignore_above: 1024 - description: Container name. + description: Name of the image the container was built on. + - name: image.tag + level: extended + type: keyword + ignore_above: 1024 + description: Container image tag. - name: labels level: extended type: object object_type: keyword description: Image labels. - - name: image.tag + - name: name level: extended type: keyword ignore_above: 1024 - description: Container image tag. + description: Container name. - name: runtime level: extended type: keyword ignore_above: 1024 description: Runtime managing this container. example: docker - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - name: destination title: Destination group: 2 @@ -369,23 +369,32 @@ Destination fields are usually populated in conjunction with source fields.' type: group fields: - - name: domain - level: core - type: keyword - ignore_above: 1024 - description: Destination domain. - - name: geo.region_name - level: core + - name: address + level: extended type: keyword ignore_above: 1024 - description: Region name. - example: Quebec - - name: ip - level: core - type: ip - description: 'IP address of the destination. + description: 'Some event destination addresses are defined ambiguously. The + event will sometimes list an IP, a domain or a unix socket. You should always + store the raw address in the `.address` field. - Can be one or multiple IPv4 or IPv6 addresses.' + Then it should be duplicated to `.ip` or `.domain`, depending on which one + it is.' + - name: bytes + level: core + type: long + description: Bytes sent from the destination to the source. + example: 184 + - name: domain + level: core + type: keyword + ignore_above: 1024 + description: Destination domain. + - name: geo.city_name + level: core + type: keyword + ignore_above: 1024 + description: City name. + example: Montreal - name: geo.continent_name level: core type: keyword @@ -398,111 +407,102 @@ ignore_above: 1024 description: Country ISO code. example: CA - - name: user.hash + - name: geo.country_name + level: core + type: keyword + ignore_above: 1024 + description: Country name. + example: Canada + - name: geo.location + level: core + type: geo_point + description: Longitude and latitude. + example: '{ "lon": -73.614830, "lat": 45.505918 }' + - name: geo.name level: extended type: keyword ignore_above: 1024 - description: 'Unique user hash to correlate information for a user in anonymized - form. + description: 'User-defined description of a location, at the level of granularity + they care about. - Useful if `user.id` or `user.name` contain confidential information and cannot - be used.' - - name: user.id + Could be the name of their data centers, the floor number, if this describes + a local physical entity, city names. + + Not typically used in automated geolocation.' + example: boston-dc + - name: geo.region_iso_code level: core type: keyword ignore_above: 1024 - description: One or multiple unique identifiers of the user. - - name: port + description: Region ISO code. + example: CA-QC + - name: geo.region_name level: core - type: long - description: Port of the destination. - - name: geo.country_name + type: keyword + ignore_above: 1024 + description: Region name. + example: Quebec + - name: ip + level: core + type: ip + description: 'IP address of the destination. + + Can be one or multiple IPv4 or IPv6 addresses.' + - name: mac level: core type: keyword ignore_above: 1024 - description: Country name. - example: Canada + description: MAC address of the destination. + - name: packets + level: core + type: long + description: Packets sent from the destination to the source. + example: 12 + - name: port + level: core + type: long + description: Port of the destination. - name: user.email level: extended type: keyword ignore_above: 1024 description: User email address. - - name: user.group.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the group. - - name: user.name - level: core - type: keyword - ignore_above: 1024 - description: Short name or login of the user. - example: albert - name: user.full_name level: extended type: keyword ignore_above: 1024 description: User's full name, if available. example: Albert Einstein - - name: geo.city_name - level: core - type: keyword - ignore_above: 1024 - description: City name. - example: Montreal - - name: geo.name + - name: user.group.id level: extended type: keyword ignore_above: 1024 - description: 'User-defined description of a location, at the level of granularity - they care about. - - Could be the name of their data centers, the floor number, if this describes - a local physical entity, city names. - - Not typically used in automated geolocation.' - example: boston-dc - - name: user.group.id + description: Unique identifier for the group on the system/platform. + - name: user.group.name level: extended type: keyword ignore_above: 1024 - description: Unique identifier for the group on the system/platform. - - name: address + description: Name of the group. + - name: user.hash level: extended type: keyword ignore_above: 1024 - description: 'Some event destination addresses are defined ambiguously. The - event will sometimes list an IP, a domain or a unix socket. You should always - store the raw address in the `.address` field. + description: 'Unique user hash to correlate information for a user in anonymized + form. - Then it should be duplicated to `.ip` or `.domain`, depending on which one - it is.' - - name: geo.region_iso_code + Useful if `user.id` or `user.name` contain confidential information and cannot + be used.' + - name: user.id level: core type: keyword ignore_above: 1024 - description: Region ISO code. - example: CA-QC - - name: geo.location - level: core - type: geo_point - description: Longitude and latitude. - example: '{ "lon": -73.614830, "lat": 45.505918 }' - - name: bytes - level: core - type: long - description: Bytes sent from the destination to the source. - example: 184 - - name: packets - level: core - type: long - description: Packets sent from the destination to the source. - example: 12 - - name: mac + description: One or multiple unique identifiers of the user. + - name: user.name level: core type: keyword ignore_above: 1024 - description: MAC address of the destination. + description: Short name or login of the user. + example: albert - name: ecs title: ECS group: 2 @@ -530,10 +530,6 @@ event itself contains an error.' type: group fields: - - name: message - level: core - type: text - description: Error message. - name: code level: core type: keyword @@ -544,6 +540,10 @@ type: keyword ignore_above: 1024 description: Unique identifier for the error. + - name: message + level: core + type: text + description: Error message. - name: event title: Event group: 2 @@ -560,6 +560,16 @@ or vulnerabilities measured on a scanned host.' type: group fields: + - name: action + level: core + type: keyword + ignore_above: 1024 + description: 'The action captured by the event. + + This describes the information in the event. It is more specific than `event.category`. + Examples are `group-add`, `process-started`, `file-created`. The value is + normally defined by the implementer.' + example: user-password-change - name: category level: core type: keyword @@ -571,36 +581,6 @@ multiple actions. Warning: In future versions of ECS, we plan to provide a list of acceptable values for this field, please use with caution.' example: user-management - - name: risk_score - level: core - type: float - description: Risk score or priority of the event (e.g. security solutions). - Use your system's original value here. - - name: kind - level: extended - type: keyword - ignore_above: 1024 - description: 'The kind of the event. - - This gives information about what type of information the event contains, - without being specific to the contents of the event. Examples are `event`, - `state`, `alarm`. Warning: In future versions of ECS, we plan to provide a - list of acceptable values for this field, please use with caution.' - example: state - - name: hash - level: extended - type: keyword - ignore_above: 1024 - description: Hash (perhaps logstash fingerprint) of raw field to be able to - demonstrate log integrity. - example: 123456789012345678901234567890ABCD - - name: severity - level: core - type: long - description: Severity describes the original severity of the event. What the - different severity values mean can very different between use cases. It's - up to the implementer to make sure severities are consistent across events. - example: '7' - name: created level: core type: date @@ -616,32 +596,6 @@ your agent''s or pipeline''s ability to keep up with your event source. In case the two timestamps are identical, @timestamp should be used.' - - name: original - level: core - type: keyword - ignore_above: 1024 - description: 'Raw text message of entire event. Used to demonstrate log integrity. - - This field is not indexed and doc_values are disabled. It cannot be searched, - but it can be retrieved from `_source`.' - example: Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100| - worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232 - - name: risk_score_norm - level: extended - type: float - description: 'Normalized risk score or priority of the event, on a scale of - 0 to 100. - - This is mainly useful if you use more than one system that assigns risk scores, - and you want to see a normalized value across all systems.' - - name: module - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the module this data is coming from. - - This information is coming from the modules used in Beats or Logstash.' - example: mysql - name: dataset level: core type: keyword @@ -652,11 +606,6 @@ of modules. It contains the information which is currently stored in metricset.name and metricset.module or fileset.name.' example: stats - - name: end - level: extended - type: date - description: event.end contains the date when the event ended or when the activity - was last observed. - name: duration level: core type: long @@ -664,31 +613,53 @@ If event.start and event.end are known this value should be the difference between the end and start time.' - - name: start + - name: end level: extended type: date - description: event.start contains the date when the event started or when the - activity was first observed. - - name: action + description: event.end contains the date when the event ended or when the activity + was last observed. + - name: hash + level: extended + type: keyword + ignore_above: 1024 + description: Hash (perhaps logstash fingerprint) of raw field to be able to + demonstrate log integrity. + example: 123456789012345678901234567890ABCD + - name: id level: core type: keyword ignore_above: 1024 - description: 'The action captured by the event. - - This describes the information in the event. It is more specific than `event.category`. - Examples are `group-add`, `process-started`, `file-created`. The value is - normally defined by the implementer.' - example: user-password-change - - name: timezone + description: Unique ID to describe the event. + example: 8a4f500d + - name: kind level: extended type: keyword ignore_above: 1024 - description: 'This field should be populated when the event''s timestamp does - not include timezone information already (e.g. default Syslog timestamps). - It''s optional otherwise. + description: 'The kind of the event. - Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), - abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00").' + This gives information about what type of information the event contains, + without being specific to the contents of the event. Examples are `event`, + `state`, `alarm`. Warning: In future versions of ECS, we plan to provide a + list of acceptable values for this field, please use with caution.' + example: state + - name: module + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the module this data is coming from. + + This information is coming from the modules used in Beats or Logstash.' + example: mysql + - name: original + level: core + type: keyword + ignore_above: 1024 + description: 'Raw text message of entire event. Used to demonstrate log integrity. + + This field is not indexed and doc_values are disabled. It cannot be searched, + but it can be retrieved from `_source`.' + example: Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100| + worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232 - name: outcome level: extended type: keyword @@ -700,19 +671,48 @@ versions of ECS, we plan to provide a list of acceptable values for this field, please use with caution.' example: success - - name: type + - name: risk_score + level: core + type: float + description: Risk score or priority of the event (e.g. security solutions). + Use your system's original value here. + - name: risk_score_norm + level: extended + type: float + description: 'Normalized risk score or priority of the event, on a scale of + 0 to 100. + + This is mainly useful if you use more than one system that assigns risk scores, + and you want to see a normalized value across all systems.' + - name: severity level: core + type: long + description: Severity describes the original severity of the event. What the + different severity values mean can very different between use cases. It's + up to the implementer to make sure severities are consistent across events. + example: '7' + - name: start + level: extended + type: date + description: event.start contains the date when the event started or when the + activity was first observed. + - name: timezone + level: extended type: keyword ignore_above: 1024 - description: 'Reserved for future usage. + description: 'This field should be populated when the event''s timestamp does + not include timezone information already (e.g. default Syslog timestamps). + It''s optional otherwise. - Please avoid using this field for user data.' - - name: id + Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), + abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00").' + - name: type level: core type: keyword ignore_above: 1024 - description: Unique ID to describe the event. - example: 8a4f500d + description: 'Reserved for future usage. + + Please avoid using this field for user data.' - name: file title: File group: 2 @@ -725,16 +725,15 @@ the event or metric.' type: group fields: - - name: group + - name: ctime level: extended - type: keyword - ignore_above: 1024 - description: Primary group name of the file. - - name: uid + type: date + description: Last time file metadata changed. + - name: device level: extended type: keyword ignore_above: 1024 - description: The user ID (UID) or security identifier (SID) of the file owner. + description: Device that is the source of the file. - name: extension level: extended type: keyword @@ -743,30 +742,21 @@ This should allow easy filtering by file extensions.' example: png - - name: type - level: extended - type: keyword - ignore_above: 1024 - description: File type (file, dir, or symlink). - - name: ctime - level: extended - type: date - description: Last time file metadata changed. - - name: target_path + - name: gid level: extended type: keyword ignore_above: 1024 - description: Target path for symlinks. - - name: owner + description: Primary group ID (GID) of the file. + - name: group level: extended type: keyword ignore_above: 1024 - description: File owner's username. - - name: gid + description: Primary group name of the file. + - name: inode level: extended type: keyword ignore_above: 1024 - description: Primary group ID (GID) of the file. + description: Inode representing the file in the filesystem. - name: mode level: extended type: keyword @@ -777,25 +767,35 @@ level: extended type: date description: Last time file content was modified. - - name: device + - name: owner level: extended type: keyword ignore_above: 1024 - description: Device that is the source of the file. + description: File owner's username. - name: path level: extended type: keyword ignore_above: 1024 description: Path to the file. - - name: inode - level: extended - type: keyword - ignore_above: 1024 - description: Inode representing the file in the filesystem. - name: size level: extended type: long description: File size in bytes (field is only added when `type` is `file`). + - name: target_path + level: extended + type: keyword + ignore_above: 1024 + description: Target path for symlinks. + - name: type + level: extended + type: keyword + ignore_above: 1024 + description: File type (file, dir, or symlink). + - name: uid + level: extended + type: keyword + ignore_above: 1024 + description: The user ID (UID) or security identifier (SID) of the file owner. - name: geo title: Geo group: 2 @@ -806,59 +806,59 @@ or be user-supplied.' type: group fields: - - name: region_name + - name: city_name level: core type: keyword ignore_above: 1024 - description: Region name. - example: Quebec + description: City name. + example: Montreal + - name: continent_name + level: core + type: keyword + ignore_above: 1024 + description: Name of the continent. + example: North America - name: country_iso_code level: core type: keyword ignore_above: 1024 description: Country ISO code. example: CA - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: 'User-defined description of a location, at the level of granularity - they care about. - - Could be the name of their data centers, the floor number, if this describes - a local physical entity, city names. - - Not typically used in automated geolocation.' - example: boston-dc - name: country_name level: core type: keyword ignore_above: 1024 description: Country name. example: Canada - - name: city_name - level: core - type: keyword - ignore_above: 1024 - description: City name. - example: Montreal - name: location level: core type: geo_point description: Longitude and latitude. example: '{ "lon": -73.614830, "lat": 45.505918 }' - - name: continent_name - level: core + - name: name + level: extended type: keyword ignore_above: 1024 - description: Name of the continent. - example: North America + description: 'User-defined description of a location, at the level of granularity + they care about. + + Could be the name of their data centers, the floor number, if this describes + a local physical entity, city names. + + Not typically used in automated geolocation.' + example: boston-dc - name: region_iso_code level: core type: keyword ignore_above: 1024 description: Region ISO code. example: CA-QC + - name: region_name + level: core + type: keyword + ignore_above: 1024 + description: Region name. + example: Quebec - name: group title: Group group: 2 @@ -886,16 +886,18 @@ hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: geo.region_name + - name: architecture level: core type: keyword ignore_above: 1024 - description: Region name. - example: Quebec - - name: ip + description: Operating system architecture. + example: x86_64 + - name: geo.city_name level: core - type: ip - description: Host ip address. + type: keyword + ignore_above: 1024 + description: City name. + example: Montreal - name: geo.continent_name level: core type: keyword @@ -908,32 +910,48 @@ ignore_above: 1024 description: Country ISO code. example: CA - - name: user.hash + - name: geo.country_name + level: core + type: keyword + ignore_above: 1024 + description: Country name. + example: Canada + - name: geo.location + level: core + type: geo_point + description: Longitude and latitude. + example: '{ "lon": -73.614830, "lat": 45.505918 }' + - name: geo.name level: extended type: keyword ignore_above: 1024 - description: 'Unique user hash to correlate information for a user in anonymized - form. + description: 'User-defined description of a location, at the level of granularity + they care about. - Useful if `user.id` or `user.name` contain confidential information and cannot - be used.' - - name: os.family - level: extended + Could be the name of their data centers, the floor number, if this describes + a local physical entity, city names. + + Not typically used in automated geolocation.' + example: boston-dc + - name: geo.region_iso_code + level: core type: keyword ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: user.id + description: Region ISO code. + example: CA-QC + - name: geo.region_name level: core type: keyword ignore_above: 1024 - description: One or multiple unique identifiers of the user. - - name: os.name - level: extended + description: Region name. + example: Quebec + - name: hostname + level: core type: keyword ignore_above: 1024 - description: Operating system name, without the version. - example: Mac OS X + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' - name: id level: core type: keyword @@ -943,128 +961,110 @@ As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: geo.country_name + - name: ip + level: core + type: ip + description: Host ip address. + - name: mac level: core type: keyword ignore_above: 1024 - description: Country name. - example: Canada - - name: hostname + description: Host mac address. + - name: name level: core type: keyword ignore_above: 1024 - description: 'Hostname of the host. + description: 'Name of the host. - It normally contains what the `hostname` command returns on the host machine.' - - name: user.email + It can contain what `hostname` returns on Unix systems, the fully qualified + domain name, or a name specified by the user. The sender decides which value + to use.' + - name: os.family level: extended type: keyword ignore_above: 1024 - description: User email address. - - name: user.group.name + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.full level: extended type: keyword ignore_above: 1024 - description: Name of the group. - - name: user.name - level: core - type: keyword - ignore_above: 1024 - description: Short name or login of the user. - example: albert - - name: type - level: core + description: Operating system name, including the version or code name. + example: Mac OS Mojave + - name: os.kernel + level: extended type: keyword ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, - this could be the container, for example, or other information meaningful - in your environment.' - - name: user.full_name + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name level: extended type: keyword ignore_above: 1024 - description: User's full name, if available. - example: Albert Einstein - - name: geo.city_name - level: core + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended type: keyword ignore_above: 1024 - description: City name. - example: Montreal + description: Operating system platform (such centos, ubuntu, windows). + example: darwin - name: os.version level: extended type: keyword ignore_above: 1024 description: Operating system version as a raw string. example: 10.14.1 - - name: geo.name - level: extended + - name: type + level: core type: keyword ignore_above: 1024 - description: 'User-defined description of a location, at the level of granularity - they care about. - - Could be the name of their data centers, the floor number, if this describes - a local physical entity, city names. + description: 'Type of host. - Not typically used in automated geolocation.' - example: boston-dc - - name: mac - level: core + For Cloud providers this can be the machine type like `t2.medium`. If vm, + this could be the container, for example, or other information meaningful + in your environment.' + - name: user.email + level: extended type: keyword ignore_above: 1024 - description: Host mac address. - - name: os.platform + description: User email address. + - name: user.full_name level: extended type: keyword ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: name - level: core + description: User's full name, if available. + example: Albert Einstein + - name: user.group.id + level: extended type: keyword ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified - domain name, or a name specified by the user. The sender decides which value - to use.' - - name: geo.region_iso_code - level: core + description: Unique identifier for the group on the system/platform. + - name: user.group.name + level: extended type: keyword ignore_above: 1024 - description: Region ISO code. - example: CA-QC - - name: geo.location - level: core - type: geo_point - description: Longitude and latitude. - example: '{ "lon": -73.614830, "lat": 45.505918 }' - - name: user.group.id + description: Name of the group. + - name: user.hash level: extended type: keyword ignore_above: 1024 - description: Unique identifier for the group on the system/platform. - - name: architecture + description: 'Unique user hash to correlate information for a user in anonymized + form. + + Useful if `user.id` or `user.name` contain confidential information and cannot + be used.' + - name: user.id level: core type: keyword ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: os.full - level: extended + description: One or multiple unique identifiers of the user. + - name: user.name + level: core type: keyword ignore_above: 1024 - description: Operating system name, including the version or code name. - example: Mac OS Mojave + description: Short name or login of the user. + example: albert - name: http title: HTTP group: 2 @@ -1077,65 +1077,73 @@ type: long description: Size in bytes of the request body. example: 887 - - name: request.method + - name: request.body.content level: extended type: keyword ignore_above: 1024 - description: 'HTTP request method. - - The field value must be normalized to lowercase for querying. See the documentation - section "Implementing ECS".' - example: get, post, put + description: The full HTTP request body. + example: Hello world - name: request.bytes level: extended type: long description: Total size in bytes of the request (body and headers). example: 1437 - - name: response.bytes + - name: request.method level: extended - type: long - description: Total size in bytes of the response (body and headers). - example: 1437 - - name: request.body.content + type: keyword + ignore_above: 1024 + description: 'HTTP request method. + + The field value must be normalized to lowercase for querying. See the documentation + section "Implementing ECS".' + example: get, post, put + - name: request.referrer level: extended type: keyword ignore_above: 1024 - description: The full HTTP request body. - example: Hello world + description: Referrer for this HTTP request. + example: https://blog.example.com/ - name: response.body.bytes level: extended type: long description: Size in bytes of the response body. example: 887 - - name: version + - name: response.body.content level: extended type: keyword ignore_above: 1024 - description: HTTP version. - example: 1.1 + description: The full HTTP response body. + example: Hello world + - name: response.bytes + level: extended + type: long + description: Total size in bytes of the response (body and headers). + example: 1437 - name: response.status_code level: extended type: long description: HTTP response status code. example: 404 - - name: request.referrer - level: extended - type: keyword - ignore_above: 1024 - description: Referrer for this HTTP request. - example: https://blog.example.com/ - - name: response.body.content + - name: version level: extended type: keyword ignore_above: 1024 - description: The full HTTP response body. - example: Hello world + description: HTTP version. + example: 1.1 - name: log title: Log group: 2 description: Fields which are specific to log events. type: group fields: + - name: level + level: core + type: keyword + ignore_above: 1024 + description: 'Original log level of the log event. + + Some examples are `warn`, `error`, `i`.' + example: err - name: original level: core type: keyword @@ -1151,58 +1159,29 @@ This field is not indexed and doc_values are disabled so it can''t be queried but the value can be retrieved from `_source`.' example: Sep 19 08:26:10 localhost My log - - name: level - level: core - type: keyword - ignore_above: 1024 - description: 'Original log level of the log event. - - Some examples are `warn`, `error`, `i`.' - example: err - name: network title: Network group: 2 - description: 'The network is defined as the communication path over which a host - or network event happens. - - The network.* fields should be populated with details about the network activity - associated with an event.' - type: group - fields: - - name: direction - level: core - type: keyword - ignore_above: 1024 - description: "Direction of the network traffic.\nRecommended values are:\n \ - \ * inbound\n * outbound\n * internal\n * external\n * unknown\n\nWhen\ - \ mapping events from a host-based monitoring context, populate this field\ - \ from the host's point of view.\nWhen mapping events from a network or perimeter-based\ - \ monitoring context, populate this field from the point of view of your network\ - \ perimeter." - example: inbound - - name: packets - level: core - type: long - description: 'Total packets transferred in both directions. - - If `source.packets` and `destination.packets` are known, `network.packets` - is their sum.' - example: 24 - - name: protocol - level: core + description: 'The network is defined as the communication path over which a host + or network event happens. + + The network.* fields should be populated with details about the network activity + associated with an event.' + type: group + fields: + - name: application + level: extended type: keyword ignore_above: 1024 - description: 'L7 Network protocol name. ex. http, lumberjack, transport protocol. + description: 'A name given to an application level protocol. This can be arbitrarily + assigned for things like microservices, but also apply to things like skype, + icq, facebook, twitter. This would be used in situations where the vendor + or service can be decoded such as from the source/dest IP owners, ports, or + wire format. The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS".' - example: http - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Name given by operators to sections of their network. - example: Guest Wifi + example: aim - name: bytes level: core type: long @@ -1211,14 +1190,6 @@ If `source.bytes` and `destination.bytes` are known, `network.bytes` is their sum.' example: 368 - - name: iana_number - level: extended - type: keyword - ignore_above: 1024 - description: IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). - Standardized list of protocols. This aligns well with NetFlow and sFlow related - logs which use the IANA Protocol Number. - example: 6 - name: community_id level: extended type: keyword @@ -1229,34 +1200,53 @@ Learn more at https://github.com/corelight/community-id-spec.' example: 1:hO+sN4H+MG5MY/8hIrXPqc4ZQz0= - - name: application - level: extended + - name: direction + level: core type: keyword ignore_above: 1024 - description: 'A name given to an application level protocol. This can be arbitrarily - assigned for things like microservices, but also apply to things like skype, - icq, facebook, twitter. This would be used in situations where the vendor - or service can be decoded such as from the source/dest IP owners, ports, or - wire format. - - The field value must be normalized to lowercase for querying. See the documentation - section "Implementing ECS".' - example: aim + description: "Direction of the network traffic.\nRecommended values are:\n \ + \ * inbound\n * outbound\n * internal\n * external\n * unknown\n\nWhen\ + \ mapping events from a host-based monitoring context, populate this field\ + \ from the host's point of view.\nWhen mapping events from a network or perimeter-based\ + \ monitoring context, populate this field from the point of view of your network\ + \ perimeter." + example: inbound - name: forwarded_ip level: core type: ip description: Host IP address when the source IP address is the proxy. example: 192.1.1.2 - - name: type + - name: iana_number + level: extended + type: keyword + ignore_above: 1024 + description: IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). + Standardized list of protocols. This aligns well with NetFlow and sFlow related + logs which use the IANA Protocol Number. + example: 6 + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Name given by operators to sections of their network. + example: Guest Wifi + - name: packets + level: core + type: long + description: 'Total packets transferred in both directions. + + If `source.packets` and `destination.packets` are known, `network.packets` + is their sum.' + example: 24 + - name: protocol level: core type: keyword ignore_above: 1024 - description: 'In the OSI Model this would be the Network Layer. ipv4, ipv6, - ipsec, pim, etc + description: 'L7 Network protocol name. ex. http, lumberjack, transport protocol. The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS".' - example: ipv4 + example: http - name: transport level: core type: keyword @@ -1267,6 +1257,16 @@ The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS".' example: tcp + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'In the OSI Model this would be the Network Layer. ipv4, ipv6, + ipsec, pim, etc + + The field value must be normalized to lowercase for querying. See the documentation + section "Implementing ECS".' + example: ipv4 - name: observer title: Observer group: 2 @@ -1284,61 +1284,35 @@ are not considered observers in ECS.' type: group fields: - - name: geo.region_name + - name: geo.city_name level: core type: keyword ignore_above: 1024 - description: Region name. - example: Quebec + description: City name. + example: Montreal - name: geo.continent_name level: core type: keyword ignore_above: 1024 description: Name of the continent. example: North America - - name: vendor - level: core - type: keyword - ignore_above: 1024 - description: observer vendor information. - - name: geo.city_name + - name: geo.country_iso_code level: core type: keyword ignore_above: 1024 - description: City name. - example: Montreal + description: Country ISO code. + example: CA - name: geo.country_name level: core type: keyword ignore_above: 1024 description: Country name. example: Canada - - name: geo.country_iso_code - level: core - type: keyword - ignore_above: 1024 - description: Country ISO code. - example: CA - - name: ip - level: core - type: ip - description: IP address of the observer. - name: geo.location level: core type: geo_point description: Longitude and latitude. example: '{ "lon": -73.614830, "lat": 45.505918 }' - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: Hostname of the observer. - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - name: geo.name level: extended type: keyword @@ -1357,16 +1331,26 @@ ignore_above: 1024 description: Region ISO code. example: CA-QC - - name: mac + - name: geo.region_name level: core type: keyword ignore_above: 1024 - description: MAC address of the observer - - name: version + description: Region name. + example: Quebec + - name: hostname level: core type: keyword ignore_above: 1024 - description: Observer version. + description: Hostname of the observer. + - name: ip + level: core + type: ip + description: IP address of the observer. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: MAC address of the observer - name: os.family level: extended type: keyword @@ -1379,17 +1363,35 @@ ignore_above: 1024 description: Operating system name, including the version or code name. example: Mac OS Mojave - - name: serial_number + - name: os.kernel level: extended type: keyword ignore_above: 1024 - description: Observer serial number. + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic - name: os.name level: extended type: keyword ignore_above: 1024 description: Operating system name, without the version. example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: serial_number + level: extended + type: keyword + ignore_above: 1024 + description: Observer serial number. - name: type level: core type: keyword @@ -1399,18 +1401,16 @@ There is no predefined list of observer types. Some examples are `forwarder`, `firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`.' example: firewall - - name: os.version - level: extended + - name: vendor + level: core type: keyword ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: os.platform - level: extended + description: observer vendor information. + - name: version + level: core type: keyword ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin + description: Observer version. - name: organization title: Organization group: 2 @@ -1421,46 +1421,46 @@ organizations.' type: group fields: - - name: name + - name: id level: extended type: keyword ignore_above: 1024 - description: Organization name. - - name: id + description: Unique identifier for the organization. + - name: name level: extended type: keyword ignore_above: 1024 - description: Unique identifier for the organization. + description: Organization name. - name: os title: Operating System group: 2 description: The OS fields contain information about the operating system. type: group fields: - - name: kernel + - name: family level: extended type: keyword ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic + description: OS family (such as redhat, debian, freebsd, windows). + example: debian - name: full level: extended type: keyword ignore_above: 1024 description: Operating system name, including the version or code name. example: Mac OS Mojave - - name: name + - name: kernel level: extended type: keyword ignore_above: 1024 - description: Operating system name, without the version. - example: Mac OS X - - name: family + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: name level: extended type: keyword ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian + description: Operating system name, without the version. + example: Mac OS X - name: platform level: extended type: keyword @@ -1483,17 +1483,24 @@ is copied to the global field for correlation.' type: group fields: + - name: args + level: extended + type: keyword + ignore_above: 1024 + description: 'Array of process arguments. + + May be filtered to protect sensitive information.' + example: + - ssh + - -l + - user + - 10.0.0.16 - name: executable level: extended type: keyword ignore_above: 1024 description: Absolute path to the process executable. example: /usr/bin/ssh - - name: thread.id - level: extended - type: long - description: Thread ID. - example: 4242 - name: name level: extended type: keyword @@ -1502,45 +1509,38 @@ Sometimes called program name or similar.' example: ssh - - name: title - level: extended - type: keyword - ignore_above: 1024 - description: 'Process title. - - The proctitle, some times the same as process name. Can also be different: - for example a browser setting its title to the web page currently opened.' - - name: args - level: extended - type: keyword - ignore_above: 1024 - description: 'Array of process arguments. - - May be filtered to protect sensitive information.' - example: - - ssh - - -l - - user - - 10.0.0.16 - name: pid level: core type: long description: Process id. + - name: ppid + level: extended + type: long + description: Process parent id. - name: start level: extended - type: date - description: The time the process started. - example: '2016-05-23T08:05:34.853Z' + type: date + description: The time the process started. + example: '2016-05-23T08:05:34.853Z' + - name: thread.id + level: extended + type: long + description: Thread ID. + example: 4242 + - name: title + level: extended + type: keyword + ignore_above: 1024 + description: 'Process title. + + The proctitle, some times the same as process name. Can also be different: + for example a browser setting its title to the web page currently opened.' - name: working_directory level: extended type: keyword ignore_above: 1024 description: The working directory of the process. example: /home/alice - - name: ppid - level: extended - type: long - description: Process parent id. - name: related title: Related group: 2 @@ -1581,23 +1581,32 @@ appropriately.' type: group fields: + - name: address + level: extended + type: keyword + ignore_above: 1024 + description: 'Some event server addresses are defined ambiguously. The event + will sometimes list an IP, a domain or a unix socket. You should always store + the raw address in the `.address` field. + + Then it should be duplicated to `.ip` or `.domain`, depending on which one + it is.' + - name: bytes + level: core + type: long + description: Bytes sent from the server to the client. + example: 184 - name: domain level: core type: keyword ignore_above: 1024 description: Server domain. - - name: geo.region_name + - name: geo.city_name level: core type: keyword ignore_above: 1024 - description: Region name. - example: Quebec - - name: ip - level: core - type: ip - description: 'IP address of the server. - - Can be one or multiple IPv4 or IPv6 addresses.' + description: City name. + example: Montreal - name: geo.continent_name level: core type: keyword @@ -1610,111 +1619,102 @@ ignore_above: 1024 description: Country ISO code. example: CA - - name: user.hash + - name: geo.country_name + level: core + type: keyword + ignore_above: 1024 + description: Country name. + example: Canada + - name: geo.location + level: core + type: geo_point + description: Longitude and latitude. + example: '{ "lon": -73.614830, "lat": 45.505918 }' + - name: geo.name level: extended type: keyword ignore_above: 1024 - description: 'Unique user hash to correlate information for a user in anonymized - form. + description: 'User-defined description of a location, at the level of granularity + they care about. - Useful if `user.id` or `user.name` contain confidential information and cannot - be used.' - - name: user.id + Could be the name of their data centers, the floor number, if this describes + a local physical entity, city names. + + Not typically used in automated geolocation.' + example: boston-dc + - name: geo.region_iso_code level: core type: keyword ignore_above: 1024 - description: One or multiple unique identifiers of the user. - - name: port + description: Region ISO code. + example: CA-QC + - name: geo.region_name level: core - type: long - description: Port of the server. - - name: geo.country_name + type: keyword + ignore_above: 1024 + description: Region name. + example: Quebec + - name: ip + level: core + type: ip + description: 'IP address of the server. + + Can be one or multiple IPv4 or IPv6 addresses.' + - name: mac level: core type: keyword ignore_above: 1024 - description: Country name. - example: Canada + description: MAC address of the server. + - name: packets + level: core + type: long + description: Packets sent from the server to the client. + example: 12 + - name: port + level: core + type: long + description: Port of the server. - name: user.email level: extended type: keyword ignore_above: 1024 description: User email address. - - name: user.group.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the group. - - name: user.name - level: core - type: keyword - ignore_above: 1024 - description: Short name or login of the user. - example: albert - name: user.full_name level: extended type: keyword ignore_above: 1024 description: User's full name, if available. example: Albert Einstein - - name: geo.city_name - level: core - type: keyword - ignore_above: 1024 - description: City name. - example: Montreal - - name: geo.name + - name: user.group.id level: extended type: keyword ignore_above: 1024 - description: 'User-defined description of a location, at the level of granularity - they care about. - - Could be the name of their data centers, the floor number, if this describes - a local physical entity, city names. - - Not typically used in automated geolocation.' - example: boston-dc - - name: user.group.id + description: Unique identifier for the group on the system/platform. + - name: user.group.name level: extended type: keyword ignore_above: 1024 - description: Unique identifier for the group on the system/platform. - - name: address + description: Name of the group. + - name: user.hash level: extended type: keyword ignore_above: 1024 - description: 'Some event server addresses are defined ambiguously. The event - will sometimes list an IP, a domain or a unix socket. You should always store - the raw address in the `.address` field. + description: 'Unique user hash to correlate information for a user in anonymized + form. - Then it should be duplicated to `.ip` or `.domain`, depending on which one - it is.' - - name: geo.region_iso_code + Useful if `user.id` or `user.name` contain confidential information and cannot + be used.' + - name: user.id level: core type: keyword ignore_above: 1024 - description: Region ISO code. - example: CA-QC - - name: geo.location - level: core - type: geo_point - description: Longitude and latitude. - example: '{ "lon": -73.614830, "lat": 45.505918 }' - - name: bytes - level: core - type: long - description: Bytes sent from the server to the client. - example: 184 - - name: packets - level: core - type: long - description: Packets sent from the server to the client. - example: 12 - - name: mac + description: One or multiple unique identifiers of the user. + - name: user.name level: core type: keyword ignore_above: 1024 - description: MAC address of the server. + description: Short name or login of the user. + example: albert - name: service title: Service group: 2 @@ -1732,6 +1732,18 @@ This id normally changes across restarts, but `service.id` does not.' example: 8a4f500f + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique identifier of the running service. + + This id should uniquely identify this service. This makes it possible to correlate + logs and metrics for one specific service. + + Example: If you are experiencing issues with one redis instance, you can filter + on that id to see metrics and logs for that single instance.' + example: d37e5ebfe0ae6c4972dbe9f0174a1637bb8247f6 - name: name level: core type: keyword @@ -1754,14 +1766,6 @@ type: keyword ignore_above: 1024 description: Current state of the service. - - name: version - level: core - type: keyword - ignore_above: 1024 - description: 'Version of the service the data was collected from. - - This allows to look at a data set only for a specific version of a service.' - example: 3.2.4 - name: type level: core type: keyword @@ -1774,18 +1778,14 @@ Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`.' example: elasticsearch - - name: id + - name: version level: core type: keyword ignore_above: 1024 - description: 'Unique identifier of the running service. - - This id should uniquely identify this service. This makes it possible to correlate - logs and metrics for one specific service. + description: 'Version of the service the data was collected from. - Example: If you are experiencing issues with one redis instance, you can filter - on that id to see metrics and logs for that single instance.' - example: d37e5ebfe0ae6c4972dbe9f0174a1637bb8247f6 + This allows to look at a data set only for a specific version of a service.' + example: 3.2.4 - name: source title: Source group: 2 @@ -1794,11 +1794,73 @@ Source fields are usually populated in conjunction with destination fields.' type: group fields: + - name: address + level: extended + type: keyword + ignore_above: 1024 + description: 'Some event source addresses are defined ambiguously. The event + will sometimes list an IP, a domain or a unix socket. You should always store + the raw address in the `.address` field. + + Then it should be duplicated to `.ip` or `.domain`, depending on which one + it is.' + - name: bytes + level: core + type: long + description: Bytes sent from the source to the destination. + example: 184 - name: domain level: core type: keyword ignore_above: 1024 - description: Source domain. + description: Source domain. + - name: geo.city_name + level: core + type: keyword + ignore_above: 1024 + description: City name. + example: Montreal + - name: geo.continent_name + level: core + type: keyword + ignore_above: 1024 + description: Name of the continent. + example: North America + - name: geo.country_iso_code + level: core + type: keyword + ignore_above: 1024 + description: Country ISO code. + example: CA + - name: geo.country_name + level: core + type: keyword + ignore_above: 1024 + description: Country name. + example: Canada + - name: geo.location + level: core + type: geo_point + description: Longitude and latitude. + example: '{ "lon": -73.614830, "lat": 45.505918 }' + - name: geo.name + level: extended + type: keyword + ignore_above: 1024 + description: 'User-defined description of a location, at the level of granularity + they care about. + + Could be the name of their data centers, the floor number, if this describes + a local physical entity, city names. + + Not typically used in automated geolocation.' + example: boston-dc + - name: geo.region_iso_code + level: core + type: keyword + ignore_above: 1024 + description: Region ISO code. + example: CA-QC - name: geo.region_name level: core type: keyword @@ -1811,123 +1873,61 @@ description: 'IP address of the source. Can be one or multiple IPv4 or IPv6 addresses.' - - name: geo.continent_name - level: core - type: keyword - ignore_above: 1024 - description: Name of the continent. - example: North America - - name: geo.country_iso_code + - name: mac level: core type: keyword ignore_above: 1024 - description: Country ISO code. - example: CA - - name: user.hash - level: extended - type: keyword - ignore_above: 1024 - description: 'Unique user hash to correlate information for a user in anonymized - form. - - Useful if `user.id` or `user.name` contain confidential information and cannot - be used.' - - name: user.id + description: MAC address of the source. + - name: packets level: core - type: keyword - ignore_above: 1024 - description: One or multiple unique identifiers of the user. + type: long + description: Packets sent from the source to the destination. + example: 12 - name: port level: core type: long description: Port of the source. - - name: geo.country_name - level: core - type: keyword - ignore_above: 1024 - description: Country name. - example: Canada - name: user.email level: extended type: keyword ignore_above: 1024 description: User email address. - - name: user.group.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the group. - - name: user.name - level: core - type: keyword - ignore_above: 1024 - description: Short name or login of the user. - example: albert - name: user.full_name level: extended type: keyword ignore_above: 1024 description: User's full name, if available. example: Albert Einstein - - name: geo.city_name - level: core - type: keyword - ignore_above: 1024 - description: City name. - example: Montreal - - name: geo.name + - name: user.group.id level: extended type: keyword ignore_above: 1024 - description: 'User-defined description of a location, at the level of granularity - they care about. - - Could be the name of their data centers, the floor number, if this describes - a local physical entity, city names. - - Not typically used in automated geolocation.' - example: boston-dc - - name: user.group.id + description: Unique identifier for the group on the system/platform. + - name: user.group.name level: extended type: keyword ignore_above: 1024 - description: Unique identifier for the group on the system/platform. - - name: address + description: Name of the group. + - name: user.hash level: extended type: keyword ignore_above: 1024 - description: 'Some event source addresses are defined ambiguously. The event - will sometimes list an IP, a domain or a unix socket. You should always store - the raw address in the `.address` field. + description: 'Unique user hash to correlate information for a user in anonymized + form. - Then it should be duplicated to `.ip` or `.domain`, depending on which one - it is.' - - name: geo.region_iso_code + Useful if `user.id` or `user.name` contain confidential information and cannot + be used.' + - name: user.id level: core type: keyword ignore_above: 1024 - description: Region ISO code. - example: CA-QC - - name: geo.location - level: core - type: geo_point - description: Longitude and latitude. - example: '{ "lon": -73.614830, "lat": 45.505918 }' - - name: bytes - level: core - type: long - description: Bytes sent from the source to the destination. - example: 184 - - name: packets - level: core - type: long - description: Packets sent from the source to the destination. - example: 12 - - name: mac + description: One or multiple unique identifiers of the user. + - name: user.name level: core type: keyword ignore_above: 1024 - description: MAC address of the source. + description: Short name or login of the user. + example: albert - name: url title: URL group: 2 @@ -1935,11 +1935,6 @@ the breaking down into scheme, domain, path, and so on. type: group fields: - - name: username - level: extended - type: keyword - ignore_above: 1024 - description: Username of the request. - name: domain level: extended type: keyword @@ -1949,6 +1944,13 @@ In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field.' example: www.elastic.co + - name: fragment + level: extended + type: keyword + ignore_above: 1024 + description: 'Portion of the url after the `#`, such as "top". + + The `#` is not part of the fragment.' - name: full level: extended type: keyword @@ -1957,23 +1959,32 @@ in `url.full`, whether this field is reconstructed or present in the event source. example: https://www.elastic.co:443/search?q=elasticsearch#top - - name: fragment + - name: original level: extended type: keyword ignore_above: 1024 - description: 'Portion of the url after the `#`, such as "top". + description: 'Unmodified original url as seen in the event source. - The `#` is not part of the fragment.' - - name: port - level: extended - type: long - description: Port of the request, such as 443. - example: 443 + Note that in network monitoring, the observed URL may be a full URL, whereas + in access logs, the URL is often just represented as a path. + + This field is meant to represent the URL as it was observed, complete or not.' + example: https://www.elastic.co:443/search?q=elasticsearch#top or /search?q=elasticsearch - name: password level: extended type: keyword ignore_above: 1024 description: Password of the request. + - name: path + level: extended + type: keyword + ignore_above: 1024 + description: Path of the request, such as "/search". + - name: port + level: extended + type: long + description: Port of the request, such as 443. + example: 443 - name: query level: extended type: keyword @@ -1985,11 +1996,6 @@ is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases.' - - name: path - level: extended - type: keyword - ignore_above: 1024 - description: Path of the request, such as "/search". - name: scheme level: extended type: keyword @@ -1998,17 +2004,11 @@ Note: The `:` is not part of the scheme.' example: https - - name: original + - name: username level: extended type: keyword ignore_above: 1024 - description: 'Unmodified original url as seen in the event source. - - Note that in network monitoring, the observed URL may be a full URL, whereas - in access logs, the URL is often just represented as a path. - - This field is meant to represent the URL as it was observed, complete or not.' - example: https://www.elastic.co:443/search?q=elasticsearch#top or /search?q=elasticsearch + description: Username of the request. - name: user title: User group: 2 @@ -2019,47 +2019,47 @@ provide an array that includes all of them.' type: group fields: - - name: hash + - name: email level: extended type: keyword ignore_above: 1024 - description: 'Unique user hash to correlate information for a user in anonymized - form. - - Useful if `user.id` or `user.name` contain confidential information and cannot - be used.' - - name: name - level: core + description: User email address. + - name: full_name + level: extended type: keyword ignore_above: 1024 - description: Short name or login of the user. - example: albert + description: User's full name, if available. + example: Albert Einstein - name: group.id level: extended type: keyword ignore_above: 1024 description: Unique identifier for the group on the system/platform. - - name: id - level: core + - name: group.name + level: extended type: keyword ignore_above: 1024 - description: One or multiple unique identifiers of the user. - - name: full_name + description: Name of the group. + - name: hash level: extended type: keyword ignore_above: 1024 - description: User's full name, if available. - example: Albert Einstein - - name: email - level: extended + description: 'Unique user hash to correlate information for a user in anonymized + form. + + Useful if `user.id` or `user.name` contain confidential information and cannot + be used.' + - name: id + level: core type: keyword ignore_above: 1024 - description: User email address. - - name: group.name - level: extended + description: One or multiple unique identifiers of the user. + - name: name + level: core type: keyword ignore_above: 1024 - description: Name of the group. + description: Short name or login of the user. + example: albert - name: user_agent title: User agent group: 2 @@ -2068,64 +2068,64 @@ They often show up in web service logs coming from the parsed user agent string.' type: group fields: - - name: os.kernel + - name: device.name level: extended type: keyword ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic + description: Name of the device. + example: iPhone - name: name level: extended type: keyword ignore_above: 1024 description: Name of the user agent. example: Safari - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: version + - name: original level: extended type: keyword ignore_above: 1024 - description: Version of the user agent. - example: 12.0 + description: Unparsed version of the user_agent. + example: Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15 + (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1 - name: os.family level: extended type: keyword ignore_above: 1024 description: OS family (such as redhat, debian, freebsd, windows). example: debian - - name: device.name + - name: os.full level: extended type: keyword ignore_above: 1024 - description: Name of the device. - example: iPhone + description: Operating system name, including the version or code name. + example: Mac OS Mojave + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic - name: os.name level: extended type: keyword ignore_above: 1024 description: Operating system name, without the version. example: Mac OS X - - name: os.full + - name: os.platform level: extended type: keyword ignore_above: 1024 - description: Operating system name, including the version or code name. - example: Mac OS Mojave - - name: original + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version level: extended type: keyword ignore_above: 1024 - description: Unparsed version of the user_agent. - example: Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15 - (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1 - - name: os.platform + description: Operating system version as a raw string. + example: 10.14.1 + - name: version level: extended type: keyword ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin + description: Version of the user agent. + example: 12.0 diff --git a/scripts/generators/beats.py b/scripts/generators/beats.py index d43c680a51..0c4a14fcb5 100644 --- a/scripts/generators/beats.py +++ b/scripts/generators/beats.py @@ -36,7 +36,7 @@ def fieldset_field_array(source_fields): beats_field = dict_copy_keys_ordered(ecs_field, allowed_keys) beats_field['name'] = nested_field_name fields.append(beats_field) - return fields + return sorted(fields, lambda x, y: cmp(x['name'], y['name'])) # Helpers