diff --git a/CHANGELOG.next.md b/CHANGELOG.next.md index e27747606a..88c48595f4 100644 --- a/CHANGELOG.next.md +++ b/CHANGELOG.next.md @@ -18,6 +18,8 @@ Thanks, you're awesome :-) --> #### Improvements +* Temporary workaround for Beats templates' `default_field` growing too big. #687 + #### Deprecated diff --git a/code/go/ecs/host.go b/code/go/ecs/host.go index 44e52b6c8a..96925dcc33 100644 --- a/code/go/ecs/host.go +++ b/code/go/ecs/host.go @@ -61,7 +61,7 @@ type Host struct { // Name of the domain of which the host is a member. // For example, on Windows this could be the host's Active Directory domain - // or NetBIOS domain name. For Linux this could be the domain of the - // host's LDAP provider. + // or NetBIOS domain name. For Linux this could be the domain of the host's + // LDAP provider. Domain string `ecs:"domain"` } diff --git a/code/go/ecs/rule.go b/code/go/ecs/rule.go index 708c922fbd..2e2f1eec3a 100644 --- a/code/go/ecs/rule.go +++ b/code/go/ecs/rule.go @@ -22,7 +22,7 @@ package ecs // Rule fields are used to capture the specifics of any observer or agent rules // that generate alerts or other notable events. // Examples of data sources that would populate the rule fields include: -// network admission control platforms, network or host IDS/IPS, network +// network admission control platforms, network or host IDS/IPS, network // firewalls, web application firewalls, url filters, endpoint detection and // response (EDR) systems, etc. type Rule struct { diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc index e8034c4b5c..beab45f6cf 100644 --- a/docs/field-details.asciidoc +++ b/docs/field-details.asciidoc @@ -2001,9 +2001,9 @@ example: `x86_64` // =============================================================== | host.domain -| Name of the domain of which the host is a member. +| Name of the domain of which the host is a member. -For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. +For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. type: keyword @@ -3671,7 +3671,7 @@ type: keyword Rule fields are used to capture the specifics of any observer or agent rules that generate alerts or other notable events. -Examples of data sources that would populate the rule fields include: network admission control platforms, network or host IDS/IPS, network firewalls, web application firewalls, url filters, endpoint detection and response (EDR) systems, etc. +Examples of data sources that would populate the rule fields include: network admission control platforms, network or host IDS/IPS, network firewalls, web application firewalls, url filters, endpoint detection and response (EDR) systems, etc. ==== Rule Field Details diff --git a/generated/beats/fields.ecs.yml b/generated/beats/fields.ecs.yml index 4813781558..05bffab79c 100644 --- a/generated/beats/fields.ecs.yml +++ b/generated/beats/fields.ecs.yml @@ -1036,6 +1036,7 @@ In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` < `event.created` < `event.ingested`.' example: 2016-05-23 08:05:35.101000 + default_field: false - name: kind level: core type: keyword @@ -1186,6 +1187,7 @@ that are expected in this field: archive, compressed, directory, encrypted, execute, hidden, read, readonly, system, write.' example: '["readonly", "system"]' + default_field: false - name: created level: extended type: date @@ -1221,6 +1223,7 @@ The value should be uppercase, and not include the colon.' example: C + default_field: false - name: extension level: extended type: keyword @@ -1463,10 +1466,13 @@ level: extended type: keyword ignore_above: 1024 - description: "Name of the domain of which the host is a member. \nFor example,\ - \ on Windows this could be the host's Active Directory domain or NetBIOS domain\ - \ name. For Linux this could be the domain of the host's LDAP provider." + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain + or NetBIOS domain name. For Linux this could be the domain of the host''s + LDAP provider.' example: CONTOSO + default_field: false - name: geo.city_name level: core type: keyword @@ -2246,6 +2252,7 @@ For example use the commit SHA of a non-released package.' example: 36f4f7e89dd61b0988b12ee000b98966867710cd + default_field: false - name: checksum level: extended type: keyword @@ -2297,6 +2304,7 @@ description: Home page or reference URL of the software in this package, if available. example: https://golang.org + default_field: false - name: size level: extended type: long @@ -2312,6 +2320,7 @@ This should contain the package file type, rather than the package manager name. Examples: rpm, dpkg, brew, npm, gem, nupkg, jar.' example: rpm + default_field: false - name: version level: extended type: keyword @@ -2350,6 +2359,7 @@ many arguments were provided to start a process. More arguments may be an indication of suspicious activity.' example: 4 + default_field: false - name: command_line level: extended type: keyword @@ -2363,6 +2373,7 @@ Some arguments may be filtered to protect sensitive information.' example: /usr/bin/ssh -l user 10.0.0.16 + default_field: false - name: executable level: extended type: keyword @@ -2381,6 +2392,7 @@ The field should be absent if there is no exit code for the event (e.g. process start).' example: 137 + default_field: false - name: hash.md5 level: extended type: keyword @@ -2425,6 +2437,7 @@ - -l - user - 10.0.0.16 + default_field: false - name: parent.args_count level: extended type: long @@ -2434,6 +2447,7 @@ many arguments were provided to start a process. More arguments may be an indication of suspicious activity.' example: 4 + default_field: false - name: parent.command_line level: extended type: keyword @@ -2447,6 +2461,7 @@ Some arguments may be filtered to protect sensitive information.' example: /usr/bin/ssh -l user 10.0.0.16 + default_field: false - name: parent.executable level: extended type: keyword @@ -2457,6 +2472,7 @@ norms: false description: Absolute path to the process executable. example: /usr/bin/ssh + default_field: false - name: parent.exit_code level: extended type: long @@ -2465,6 +2481,7 @@ The field should be absent if there is no exit code for the event (e.g. process start).' example: 137 + default_field: false - name: parent.name level: extended type: keyword @@ -2477,40 +2494,47 @@ Sometimes called program name or similar.' example: ssh + default_field: false - name: parent.pgid level: extended type: long format: string description: Identifier of the group of processes the process belongs to. + default_field: false - name: parent.pid level: core type: long format: string description: Process id. example: 4242 + default_field: false - name: parent.ppid level: extended type: long format: string description: Parent process' pid. example: 4241 + default_field: false - name: parent.start level: extended type: date description: The time the process started. example: '2016-05-23T08:05:34.853Z' + default_field: false - name: parent.thread.id level: extended type: long format: string description: Thread ID. example: 4242 + default_field: false - name: parent.thread.name level: extended type: keyword ignore_above: 1024 description: Thread name. example: thread-0 + default_field: false - name: parent.title level: extended type: keyword @@ -2523,11 +2547,13 @@ The proctitle, some times the same as process name. Can also be different: for example a browser setting its title to the web page currently opened.' + default_field: false - name: parent.uptime level: extended type: long description: Seconds the process has been up. example: 1325 + default_field: false - name: parent.working_directory level: extended type: keyword @@ -2538,6 +2564,7 @@ norms: false description: The working directory of the process. example: /home/alice + default_field: false - name: pgid level: extended type: long @@ -2615,6 +2642,7 @@ corresponds to the data pointed by `lp_data`. This is optional but provides better recoverability and should be populated for REG_BINARY encoded values.' example: ZQBuAC0AVQBTAAAAZQBuAAAAAAA= + default_field: false - name: data.strings level: core type: keyword @@ -2627,24 +2655,28 @@ variable length. For numeric data, such as REG_DWORD and REG_QWORD, this should be populated with the decimal representation (e.g `"1"`).' example: '["C:\rta\red_ttp\bin\myapp.exe"]' + default_field: false - name: data.type level: core type: keyword ignore_above: 1024 description: Standard registry type for encoding contents example: REG_SZ + default_field: false - name: hive level: core type: keyword ignore_above: 1024 description: Abbreviated name for the hive. example: HKLM + default_field: false - name: key level: core type: keyword ignore_above: 1024 description: Hive-relative path of keys. example: SOFTWARE\Microsoft\Windows NT\CurrentVersion\Image File Execution Options\winword.exe + default_field: false - name: path level: core type: keyword @@ -2652,12 +2684,14 @@ description: Full path, including hive, key and value example: HKLM\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Image File Execution Options\winword.exe\Debugger + default_field: false - name: value level: core type: keyword ignore_above: 1024 description: Name of the value written. example: Debugger + default_field: false - name: related title: Related group: 2 @@ -2683,6 +2717,7 @@ type: keyword ignore_above: 1024 description: All the user names seen on your event. + default_field: false - name: rule title: Rule group: 2 @@ -2690,7 +2725,7 @@ agent rules that generate alerts or other notable events. Examples of data sources that would populate the rule fields include: network - admission control platforms, network or host IDS/IPS, network firewalls, web + admission control platforms, network or host IDS/IPS, network firewalls, web application firewalls, url filters, endpoint detection and response (EDR) systems, etc.' type: group @@ -2702,12 +2737,14 @@ description: A categorization value keyword used by the entity using the rule for detection of this event. example: Attempted Information Leak + default_field: false - name: description level: extended type: keyword ignore_above: 1024 description: The description of the rule generating the event. example: Block requests to public DNS over HTTPS / TLS protocols + default_field: false - name: id level: extended type: keyword @@ -2715,12 +2752,14 @@ description: A rule ID that is unique within the scope of an agent, observer, or other entity using the rule for detection of this event. example: 101 + default_field: false - name: name level: extended type: keyword ignore_above: 1024 description: The name of the rule or signature generating the event. example: BLOCK_DNS_over_TLS + default_field: false - name: reference level: extended type: keyword @@ -2732,6 +2771,7 @@ not available, it can also be a link to a more general page describing this type of alert.' example: https://en.wikipedia.org/wiki/DNS_over_TLS + default_field: false - name: ruleset level: extended type: keyword @@ -2739,6 +2779,7 @@ description: Name of the ruleset, policy, group, or parent category in which the rule used to generate this event is a member. example: Standard_Protocol_Filters + default_field: false - name: uuid level: extended type: keyword @@ -2747,12 +2788,14 @@ agents, observers, or other entities using the rule for detection of this event. example: 1100110011 + default_field: false - name: version level: extended type: keyword ignore_above: 1024 description: The version / revision of the rule being used for analysis. example: 1.1 + default_field: false - name: server title: Server group: 2 @@ -3382,6 +3425,7 @@ ignore_above: 1024 description: String indicating the cipher used during the current connection. example: TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 + default_field: false - name: client.certificate level: extended type: keyword @@ -3390,6 +3434,7 @@ is usually mutually-exclusive of `client.certificate_chain` since this value also exists in that list. example: MII... + default_field: false - name: client.certificate_chain level: extended type: keyword @@ -3400,6 +3445,7 @@ example: - MII... - MII... + default_field: false - name: client.hash.md5 level: extended type: keyword @@ -3408,6 +3454,7 @@ of certificate offered by the client. For consistency with other hash values, this value should be formatted as an uppercase hash. example: 0F76C7F2C55BFD7D8E8B8F4BFBF0C9EC + default_field: false - name: client.hash.sha1 level: extended type: keyword @@ -3416,6 +3463,7 @@ of certificate offered by the client. For consistency with other hash values, this value should be formatted as an uppercase hash. example: 9E393D93138888D288266C2D915214D1D1CCEB2A + default_field: false - name: client.hash.sha256 level: extended type: keyword @@ -3424,6 +3472,7 @@ version of certificate offered by the client. For consistency with other hash values, this value should be formatted as an uppercase hash. example: 0687F666A054EF17A08E2F2162EAB4CBC0D265E1D7875BE74BF3C712CA92DAF0 + default_field: false - name: client.issuer level: extended type: keyword @@ -3431,6 +3480,7 @@ description: Distinguished name of subject of the issuer of the x.509 certificate presented by the client. example: CN=MyDomain Root CA, OU=Infrastructure Team, DC=mydomain, DC=com + default_field: false - name: client.ja3 level: extended type: keyword @@ -3438,18 +3488,21 @@ description: A hash that identifies clients based on how they perform an SSL/TLS handshake. example: d4e5b18d6b55c71272893221c96ba240 + default_field: false - name: client.not_after level: extended type: date description: Date/Time indicating when client certificate is no longer considered valid. example: '2021-01-01T00:00:00.000Z' + default_field: false - name: client.not_before level: extended type: date description: Date/Time indicating when client certificate is first considered valid. example: '1970-01-01T00:00:00.000Z' + default_field: false - name: client.server_name level: extended type: keyword @@ -3458,6 +3511,7 @@ the client is attempting to connect. When this value is available, it should get copied to `destination.domain`. example: www.elastic.co + default_field: false - name: client.subject level: extended type: keyword @@ -3465,6 +3519,7 @@ description: Distinguished name of subject of the x.509 certificate presented by the client. example: CN=myclient, OU=Documentation Team, DC=mydomain, DC=com + default_field: false - name: client.supported_ciphers level: extended type: keyword @@ -3474,17 +3529,20 @@ - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - '...' + default_field: false - name: curve level: extended type: keyword ignore_above: 1024 description: String indicating the curve used for the given cipher, when applicable. example: secp256r1 + default_field: false - name: established level: extended type: boolean description: Boolean flag indicating if the TLS negotiation was successful and transitioned to an encrypted tunnel. + default_field: false - name: next_protocol level: extended type: keyword @@ -3493,11 +3551,13 @@ the IANA registry (https://www.iana.org/assignments/tls-extensiontype-values/tls-extensiontype-values.xhtml#alpn-protocol-ids), this string should be lower case. example: http/1.1 + default_field: false - name: resumed level: extended type: boolean description: Boolean flag indicating if this TLS connection was resumed from an existing TLS negotiation. + default_field: false - name: server.certificate level: extended type: keyword @@ -3506,6 +3566,7 @@ is usually mutually-exclusive of `server.certificate_chain` since this value also exists in that list. example: MII... + default_field: false - name: server.certificate_chain level: extended type: keyword @@ -3516,6 +3577,7 @@ example: - MII... - MII... + default_field: false - name: server.hash.md5 level: extended type: keyword @@ -3524,6 +3586,7 @@ of certificate offered by the server. For consistency with other hash values, this value should be formatted as an uppercase hash. example: 0F76C7F2C55BFD7D8E8B8F4BFBF0C9EC + default_field: false - name: server.hash.sha1 level: extended type: keyword @@ -3532,6 +3595,7 @@ of certificate offered by the server. For consistency with other hash values, this value should be formatted as an uppercase hash. example: 9E393D93138888D288266C2D915214D1D1CCEB2A + default_field: false - name: server.hash.sha256 level: extended type: keyword @@ -3540,6 +3604,7 @@ version of certificate offered by the server. For consistency with other hash values, this value should be formatted as an uppercase hash. example: 0687F666A054EF17A08E2F2162EAB4CBC0D265E1D7875BE74BF3C712CA92DAF0 + default_field: false - name: server.issuer level: extended type: keyword @@ -3547,6 +3612,7 @@ description: Subject of the issuer of the x.509 certificate presented by the server. example: CN=MyDomain Root CA, OU=Infrastructure Team, DC=mydomain, DC=com + default_field: false - name: server.ja3s level: extended type: keyword @@ -3554,36 +3620,42 @@ description: A hash that identifies servers based on how they perform an SSL/TLS handshake. example: 394441ab65754e2207b1e1b457b3641d + default_field: false - name: server.not_after level: extended type: date description: Timestamp indicating when server certificate is no longer considered valid. example: '2021-01-01T00:00:00.000Z' + default_field: false - name: server.not_before level: extended type: date description: Timestamp indicating when server certificate is first considered valid. example: '1970-01-01T00:00:00.000Z' + default_field: false - name: server.subject level: extended type: keyword ignore_above: 1024 description: Subject of the x.509 certificate presented by the server. example: CN=www.mydomain.com, OU=Infrastructure Team, DC=mydomain, DC=com + default_field: false - name: version level: extended type: keyword ignore_above: 1024 description: Numeric part of the version parsed from the original string. example: '1.2' + default_field: false - name: version_protocol level: extended type: keyword ignore_above: 1024 description: Normalized lowercase protocol name parsed from original string. example: tls + default_field: false - name: tracing title: Tracing group: 2 @@ -3909,6 +3981,7 @@ This field must be an array.' example: '["Firewall"]' + default_field: false - name: classification level: extended type: keyword @@ -3916,6 +3989,7 @@ description: The classification of the vulnerability scoring system. For example (https://www.first.org/cvss/) example: CVSS + default_field: false - name: description level: extended type: keyword @@ -3928,6 +4002,7 @@ of the vulnerability. For example (https://cve.mitre.org/about/faqs.html#cve_entry_descriptions_created[Common Vulnerabilities and Exposure CVE description]) example: In macOS before 2.12.6, there is a vulnerability in the RPC... + default_field: false - name: enumeration level: extended type: keyword @@ -3935,6 +4010,7 @@ description: The type of identifier used for this vulnerability. For example (https://cve.mitre.org/about/) example: CVE + default_field: false - name: id level: extended type: keyword @@ -3944,6 +4020,7 @@ example (https://cve.mitre.org/about/faqs.html#what_is_cve_id)[Common Vulnerabilities and Exposure CVE ID] example: CVE-2019-00001 + default_field: false - name: reference level: extended type: keyword @@ -3951,18 +4028,21 @@ description: A resource that provides additional information, context, and mitigations for the identified vulnerability. example: https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-6111 + default_field: false - name: report_id level: extended type: keyword ignore_above: 1024 description: The report or scan identification number. example: 20191018.0001 + default_field: false - name: scanner.vendor level: extended type: keyword ignore_above: 1024 description: The name of the vulnerability scanner vendor. example: Tenable + default_field: false - name: score.base level: extended type: float @@ -3972,6 +4052,7 @@ complexity, privileges, and user interaction), impact metrics (confidentiality, integrity, and availability), and scope. For example (https://www.first.org/cvss/specification-document)' example: 5.5 + default_field: false - name: score.environmental level: extended type: float @@ -3980,6 +4061,7 @@ Environmental scores cover an assessment for any modified Base metrics, confidentiality, integrity, and availability requirements. For example (https://www.first.org/cvss/specification-document)' example: 5.5 + default_field: false - name: score.temporal level: extended type: float @@ -3987,6 +4069,7 @@ Temporal scores cover an assessment for code maturity, remediation level, and confidence. For example (https://www.first.org/cvss/specification-document)' + default_field: false - name: score.version level: extended type: keyword @@ -4000,6 +4083,7 @@ organization, whose mission is to help computer security incident response teams across the world. For example (https://nvd.nist.gov/vuln-metrics/cvss)' example: 2.0 + default_field: false - name: severity level: extended type: keyword @@ -4007,3 +4091,4 @@ description: The severity of the vulnerability can help with metrics and internal prioritization regarding remediation. For example (https://nvd.nist.gov/vuln-metrics/cvss) example: Critical + default_field: false diff --git a/generated/ecs/ecs_flat.yml b/generated/ecs/ecs_flat.yml index 5216b4029e..3779dfe3e7 100644 --- a/generated/ecs/ecs_flat.yml +++ b/generated/ecs/ecs_flat.yml @@ -2415,9 +2415,10 @@ host.architecture: type: keyword host.domain: dashed_name: host-domain - description: "Name of the domain of which the host is a member. \nFor example, on\ - \ Windows this could be the host's Active Directory domain or NetBIOS domain name.\ - \ For Linux this could be the domain of the host's LDAP provider." + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS + domain name. For Linux this could be the domain of the host''s LDAP provider.' example: CONTOSO flat_name: host.domain ignore_above: 1024 diff --git a/generated/ecs/ecs_nested.yml b/generated/ecs/ecs_nested.yml index 10bc0b8708..57cb4d8a90 100644 --- a/generated/ecs/ecs_nested.yml +++ b/generated/ecs/ecs_nested.yml @@ -2722,9 +2722,11 @@ host: type: keyword domain: dashed_name: host-domain - description: "Name of the domain of which the host is a member. \nFor example,\ - \ on Windows this could be the host's Active Directory domain or NetBIOS domain\ - \ name. For Linux this could be the domain of the host's LDAP provider." + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain + or NetBIOS domain name. For Linux this could be the domain of the host''s + LDAP provider.' example: CONTOSO flat_name: host.domain ignore_above: 1024 @@ -4819,9 +4821,8 @@ rule: rules that generate alerts or other notable events. Examples of data sources that would populate the rule fields include: network - admission control platforms, network or host IDS/IPS, network firewalls, web - application firewalls, url filters, endpoint detection and response (EDR) systems, - etc.' + admission control platforms, network or host IDS/IPS, network firewalls, web application + firewalls, url filters, endpoint detection and response (EDR) systems, etc.' fields: category: dashed_name: rule-category diff --git a/schemas/host.yml b/schemas/host.yml index c4eb751bf3..bdfe42fc5c 100644 --- a/schemas/host.yml +++ b/schemas/host.yml @@ -85,9 +85,9 @@ type: keyword short: Name of the directory the group is a member of. description: > - Name of the domain of which the host is a member. + Name of the domain of which the host is a member. - For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. + For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. example: CONTOSO diff --git a/schemas/rule.yml b/schemas/rule.yml index bd668fd60f..34e901839f 100644 --- a/schemas/rule.yml +++ b/schemas/rule.yml @@ -6,9 +6,9 @@ description: > Rule fields are used to capture the specifics of any observer or agent rules that generate alerts or other notable events. - Examples of data sources that would populate the rule fields include: network admission control platforms, network or - host IDS/IPS, network firewalls, web application firewalls, url filters, endpoint detection and response (EDR) systems, etc. - + Examples of data sources that would populate the rule fields include: network admission control platforms, network or + host IDS/IPS, network firewalls, web application firewalls, url filters, endpoint detection and response (EDR) systems, etc. + type: group fields: @@ -19,7 +19,7 @@ description: > A rule ID that is unique within the scope of an agent, observer, or other entity using the rule for detection of this event. - example: 101 + example: 101 - name: uuid level: extended diff --git a/scripts/generators/beats.py b/scripts/generators/beats.py index fcbc420bda..2926fe5570 100644 --- a/scripts/generators/beats.py +++ b/scripts/generators/beats.py @@ -5,8 +5,11 @@ def generate(ecs_nested, ecs_version): + # Load temporary whitelist for default_fields workaround. + df_whitelist = ecs_helpers.yaml_load('scripts/generators/beats_default_fields_whitelist.yml') + # base first - beats_fields = fieldset_field_array(ecs_nested['base']['fields']) + beats_fields = fieldset_field_array(ecs_nested['base']['fields'], df_whitelist) allowed_fieldset_keys = ['name', 'title', 'group', 'description', 'footnote', 'type'] # other fieldsets @@ -16,7 +19,7 @@ def generate(ecs_nested, ecs_version): fieldset = ecs_nested[fieldset_name] beats_field = ecs_helpers.dict_copy_keys_ordered(fieldset, allowed_fieldset_keys) - beats_field['fields'] = fieldset_field_array(fieldset['fields']) + beats_field['fields'] = fieldset_field_array(fieldset['fields'], df_whitelist) beats_fields.append(beats_field) beats_file = OrderedDict() @@ -28,10 +31,11 @@ def generate(ecs_nested, ecs_version): write_beats_yaml(beats_file, ecs_version) -def fieldset_field_array(source_fields): +def fieldset_field_array(source_fields, df_whitelist): allowed_keys = ['name', 'level', 'required', 'type', 'object_type', 'ignore_above', 'multi_fields', 'format', 'input_format', - 'output_format', 'output_precision', 'description', 'example'] + 'output_format', 'output_precision', 'description', + 'example'] multi_fields_allowed_keys = ['name', 'type', 'norms'] fields = [] @@ -47,6 +51,10 @@ def fieldset_field_array(source_fields): beats_field['multi_fields'] = cleaned_multi_fields beats_field['name'] = nested_field_name + + if not ecs_field['flat_name'] in df_whitelist: + beats_field['default_field'] = False + fields.append(beats_field) return sorted(fields, lambda x, y: cmp(x['name'], y['name'])) diff --git a/scripts/generators/beats_default_fields_whitelist.yml b/scripts/generators/beats_default_fields_whitelist.yml new file mode 100644 index 0000000000..a7fa30b9c8 --- /dev/null +++ b/scripts/generators/beats_default_fields_whitelist.yml @@ -0,0 +1,401 @@ +--- +!!set +# Note: other fields can be inserted as needed + +# ECS 1.2 fields +'@timestamp': null +agent.ephemeral_id: null +agent.id: null +agent.name: null +agent.type: null +agent.version: null +as.number: null +as.organization.name: null +client.address: null +client.as.number: null +client.as.organization.name: null +client.bytes: null +client.domain: null +client.geo.city_name: null +client.geo.continent_name: null +client.geo.country_iso_code: null +client.geo.country_name: null +client.geo.location: null +client.geo.name: null +client.geo.region_iso_code: null +client.geo.region_name: null +client.ip: null +client.mac: null +client.nat.ip: null +client.nat.port: null +client.packets: null +client.port: null +client.registered_domain: null +client.top_level_domain: null +client.user.domain: null +client.user.email: null +client.user.full_name: null +client.user.group.domain: null +client.user.group.id: null +client.user.group.name: null +client.user.hash: null +client.user.id: null +client.user.name: null +cloud.account.id: null +cloud.availability_zone: null +cloud.instance.id: null +cloud.instance.name: null +cloud.machine.type: null +cloud.provider: null +cloud.region: null +container.id: null +container.image.name: null +container.image.tag: null +container.labels: null +container.name: null +container.runtime: null +destination.address: null +destination.as.number: null +destination.as.organization.name: null +destination.bytes: null +destination.domain: null +destination.geo.city_name: null +destination.geo.continent_name: null +destination.geo.country_iso_code: null +destination.geo.country_name: null +destination.geo.location: null +destination.geo.name: null +destination.geo.region_iso_code: null +destination.geo.region_name: null +destination.ip: null +destination.mac: null +destination.nat.ip: null +destination.nat.port: null +destination.packets: null +destination.port: null +destination.registered_domain: null +destination.top_level_domain: null +destination.user.domain: null +destination.user.email: null +destination.user.full_name: null +destination.user.group.domain: null +destination.user.group.id: null +destination.user.group.name: null +destination.user.hash: null +destination.user.id: null +destination.user.name: null +dns.answers: null +dns.answers.class: null +dns.answers.data: null +dns.answers.name: null +dns.answers.ttl: null +dns.answers.type: null +dns.header_flags: null +dns.id: null +dns.op_code: null +dns.question.class: null +dns.question.name: null +dns.question.registered_domain: null +dns.question.subdomain: null +dns.question.top_level_domain: null +dns.question.type: null +dns.resolved_ip: null +dns.response_code: null +dns.type: null +ecs.version: null +error.code: null +error.id: null +error.message: null +error.stack_trace: null +error.type: null +event.action: null +event.category: null +event.code: null +event.created: null +event.dataset: null +event.duration: null +event.end: null +event.hash: null +event.id: null +event.kind: null +event.module: null +event.original: null +event.outcome: null +event.provider: null +event.risk_score: null +event.risk_score_norm: null +event.sequence: null +event.severity: null +event.start: null +event.timezone: null +event.type: null +file.accessed: null +file.created: null +file.ctime: null +file.device: null +file.directory: null +file.extension: null +file.gid: null +file.group: null +file.hash.md5: null +file.hash.sha1: null +file.hash.sha256: null +file.hash.sha512: null +file.inode: null +file.mode: null +file.mtime: null +file.name: null +file.owner: null +file.path: null +file.size: null +file.target_path: null +file.type: null +file.uid: null +geo.city_name: null +geo.continent_name: null +geo.country_iso_code: null +geo.country_name: null +geo.location: null +geo.name: null +geo.region_iso_code: null +geo.region_name: null +group.domain: null +group.id: null +group.name: null +hash.md5: null +hash.sha1: null +hash.sha256: null +hash.sha512: null +host.architecture: null +host.geo.city_name: null +host.geo.continent_name: null +host.geo.country_iso_code: null +host.geo.country_name: null +host.geo.location: null +host.geo.name: null +host.geo.region_iso_code: null +host.geo.region_name: null +host.hostname: null +host.id: null +host.ip: null +host.mac: null +host.name: null +host.os.family: null +host.os.full: null +host.os.kernel: null +host.os.name: null +host.os.platform: null +host.os.version: null +host.type: null +host.uptime: null +host.user.domain: null +host.user.email: null +host.user.full_name: null +host.user.group.domain: null +host.user.group.id: null +host.user.group.name: null +host.user.hash: null +host.user.id: null +host.user.name: null +http.request.body.bytes: null +http.request.body.content: null +http.request.bytes: null +http.request.method: null +http.request.referrer: null +http.response.body.bytes: null +http.response.body.content: null +http.response.bytes: null +http.response.status_code: null +http.version: null +labels: null +log.level: null +log.logger: null +log.origin.file.line: null +log.origin.file.name: null +log.origin.function: null +log.original: null +log.syslog: null +log.syslog.facility.code: null +log.syslog.facility.name: null +log.syslog.priority: null +log.syslog.severity.code: null +log.syslog.severity.name: null +message: null +network.application: null +network.bytes: null +network.community_id: null +network.direction: null +network.forwarded_ip: null +network.iana_number: null +network.name: null +network.packets: null +network.protocol: null +network.transport: null +network.type: null +observer.geo.city_name: null +observer.geo.continent_name: null +observer.geo.country_iso_code: null +observer.geo.country_name: null +observer.geo.location: null +observer.geo.name: null +observer.geo.region_iso_code: null +observer.geo.region_name: null +observer.hostname: null +observer.ip: null +observer.mac: null +observer.name: null +observer.os.family: null +observer.os.full: null +observer.os.kernel: null +observer.os.name: null +observer.os.platform: null +observer.os.version: null +observer.product: null +observer.serial_number: null +observer.type: null +observer.vendor: null +observer.version: null +organization.id: null +organization.name: null +os.family: null +os.full: null +os.kernel: null +os.name: null +os.platform: null +os.version: null +package.architecture: null +package.checksum: null +package.description: null +package.install_scope: null +package.installed: null +package.license: null +package.name: null +package.path: null +package.size: null +package.version: null +process.args: null +process.executable: null +process.hash.md5: null +process.hash.sha1: null +process.hash.sha256: null +process.hash.sha512: null +process.name: null +process.pgid: null +process.pid: null +process.ppid: null +process.start: null +process.thread.id: null +process.thread.name: null +process.title: null +process.uptime: null +process.working_directory: null +related.ip: null +server.address: null +server.as.number: null +server.as.organization.name: null +server.bytes: null +server.domain: null +server.geo.city_name: null +server.geo.continent_name: null +server.geo.country_iso_code: null +server.geo.country_name: null +server.geo.location: null +server.geo.name: null +server.geo.region_iso_code: null +server.geo.region_name: null +server.ip: null +server.mac: null +server.nat.ip: null +server.nat.port: null +server.packets: null +server.port: null +server.registered_domain: null +server.top_level_domain: null +server.user.domain: null +server.user.email: null +server.user.full_name: null +server.user.group.domain: null +server.user.group.id: null +server.user.group.name: null +server.user.hash: null +server.user.id: null +server.user.name: null +service.ephemeral_id: null +service.id: null +service.name: null +service.node.name: null +service.state: null +service.type: null +service.version: null +source.address: null +source.as.number: null +source.as.organization.name: null +source.bytes: null +source.domain: null +source.geo.city_name: null +source.geo.continent_name: null +source.geo.country_iso_code: null +source.geo.country_name: null +source.geo.location: null +source.geo.name: null +source.geo.region_iso_code: null +source.geo.region_name: null +source.ip: null +source.mac: null +source.nat.ip: null +source.nat.port: null +source.packets: null +source.port: null +source.registered_domain: null +source.top_level_domain: null +source.user.domain: null +source.user.email: null +source.user.full_name: null +source.user.group.domain: null +source.user.group.id: null +source.user.group.name: null +source.user.hash: null +source.user.id: null +source.user.name: null +tags: null +threat.framework: null +threat.tactic.id: null +threat.tactic.name: null +threat.tactic.reference: null +threat.technique.id: null +threat.technique.name: null +threat.technique.reference: null +trace.id: null +transaction.id: null +url.domain: null +url.extension: null +url.fragment: null +url.full: null +url.original: null +url.password: null +url.path: null +url.port: null +url.query: null +url.registered_domain: null +url.scheme: null +url.top_level_domain: null +url.username: null +user.domain: null +user.email: null +user.full_name: null +user.group.domain: null +user.group.id: null +user.group.name: null +user.hash: null +user.id: null +user.name: null +user_agent.device.name: null +user_agent.name: null +user_agent.original: null +user_agent.os.family: null +user_agent.os.full: null +user_agent.os.kernel: null +user_agent.os.name: null +user_agent.os.platform: null +user_agent.os.version: null +user_agent.version: null diff --git a/scripts/generators/ecs_helpers.py b/scripts/generators/ecs_helpers.py index d04c3e282d..ff2f33baa1 100644 --- a/scripts/generators/ecs_helpers.py +++ b/scripts/generators/ecs_helpers.py @@ -61,6 +61,14 @@ def yaml_ordereddict(dumper, data): yaml.add_representer(OrderedDict, yaml_ordereddict) + +def dict_rename_keys(dict, renames): + for key, value in dict.iteritems(): + if key in renames: + del dict[key] + dict[renames[key]] = value + + # File helpers @@ -70,6 +78,11 @@ def yaml_dump(filename, data, preamble=None): outfile.write(preamble) yaml.dump(data, outfile, default_flow_style=False) + +def yaml_load(filename): + with open(filename) as f: + return yaml.load(f.read()) + # List helpers