From a8a5a70ea635dc30d9ad46fa3c2cc1e20f3f5c81 Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Mon, 23 Dec 2019 12:36:36 +0100 Subject: [PATCH] Workaround for Beats issue with default_field growing too big (#687) This is so that Beats' default_fields don't go above 1024 field limit. See also https://github.com/elastic/beats/issues/14262 --- CHANGELOG.next.md | 2 + code/go/ecs/host.go | 4 +- docs/field-details.asciidoc | 4 +- generated/beats/fields.ecs.yml | 73 +++- generated/ecs/ecs_flat.yml | 7 +- generated/ecs/ecs_nested.yml | 8 +- schemas/host.yml | 4 +- scripts/generators/beats.py | 15 +- .../beats_default_fields_whitelist.yml | 401 ++++++++++++++++++ scripts/generators/ecs_helpers.py | 13 + 10 files changed, 513 insertions(+), 18 deletions(-) create mode 100644 scripts/generators/beats_default_fields_whitelist.yml diff --git a/CHANGELOG.next.md b/CHANGELOG.next.md index e27747606a..88c48595f4 100644 --- a/CHANGELOG.next.md +++ b/CHANGELOG.next.md @@ -18,6 +18,8 @@ Thanks, you're awesome :-) --> #### Improvements +* Temporary workaround for Beats templates' `default_field` growing too big. #687 + #### Deprecated diff --git a/code/go/ecs/host.go b/code/go/ecs/host.go index 44e52b6c8a..96925dcc33 100644 --- a/code/go/ecs/host.go +++ b/code/go/ecs/host.go @@ -61,7 +61,7 @@ type Host struct { // Name of the domain of which the host is a member. // For example, on Windows this could be the host's Active Directory domain - // or NetBIOS domain name. For Linux this could be the domain of the - // host's LDAP provider. + // or NetBIOS domain name. For Linux this could be the domain of the host's + // LDAP provider. Domain string `ecs:"domain"` } diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc index 6c0717455b..bcfcd84714 100644 --- a/docs/field-details.asciidoc +++ b/docs/field-details.asciidoc @@ -1917,9 +1917,9 @@ example: `x86_64` // =============================================================== | host.domain -| Name of the domain of which the host is a member. +| Name of the domain of which the host is a member. -For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. +For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. type: keyword diff --git a/generated/beats/fields.ecs.yml b/generated/beats/fields.ecs.yml index 2cf4d5b127..7a25d6871c 100644 --- a/generated/beats/fields.ecs.yml +++ b/generated/beats/fields.ecs.yml @@ -1000,6 +1000,7 @@ In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` < `event.created` < `event.ingested`.' example: 2016-05-23 08:05:35.101000 + default_field: false - name: kind level: extended type: keyword @@ -1387,10 +1388,13 @@ level: extended type: keyword ignore_above: 1024 - description: "Name of the domain of which the host is a member. \nFor example,\ - \ on Windows this could be the host's Active Directory domain or NetBIOS domain\ - \ name. For Linux this could be the domain of the host's LDAP provider." + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain + or NetBIOS domain name. For Linux this could be the domain of the host''s + LDAP provider.' example: CONTOSO + default_field: false - name: geo.city_name level: core type: keyword @@ -2126,6 +2130,7 @@ For example use the commit SHA of a non-released package.' example: 36f4f7e89dd61b0988b12ee000b98966867710cd + default_field: false - name: checksum level: extended type: keyword @@ -2177,6 +2182,7 @@ description: Home page or reference URL of the software in this package, if available. example: https://golang.org + default_field: false - name: size level: extended type: long @@ -2192,6 +2198,7 @@ This should contain the package file type, rather than the package manager name. Examples: rpm, dpkg, brew, npm, gem, nupkg, jar.' example: rpm + default_field: false - name: version level: extended type: keyword @@ -2230,6 +2237,7 @@ many arguments were provided to start a process. More arguments may be an indication of suspicious activity.' example: 4 + default_field: false - name: command_line level: extended type: keyword @@ -2239,6 +2247,7 @@ Some arguments may be filtered to protect sensitive information.' example: /usr/bin/ssh -l user 10.0.0.16 + default_field: false - name: executable level: extended type: keyword @@ -2253,6 +2262,7 @@ The field should be absent if there is no exit code for the event (e.g. process start).' example: 137 + default_field: false - name: hash.md5 level: extended type: keyword @@ -2293,6 +2303,7 @@ - -l - user - 10.0.0.16 + default_field: false - name: parent.args_count level: extended type: long @@ -2302,6 +2313,7 @@ many arguments were provided to start a process. More arguments may be an indication of suspicious activity.' example: 4 + default_field: false - name: parent.command_line level: extended type: keyword @@ -2311,12 +2323,14 @@ Some arguments may be filtered to protect sensitive information.' example: /usr/bin/ssh -l user 10.0.0.16 + default_field: false - name: parent.executable level: extended type: keyword ignore_above: 1024 description: Absolute path to the process executable. example: /usr/bin/ssh + default_field: false - name: parent.exit_code level: extended type: long @@ -2325,6 +2339,7 @@ The field should be absent if there is no exit code for the event (e.g. process start).' example: 137 + default_field: false - name: parent.name level: extended type: keyword @@ -2333,40 +2348,47 @@ Sometimes called program name or similar.' example: ssh + default_field: false - name: parent.pgid level: extended type: long format: string description: Identifier of the group of processes the process belongs to. + default_field: false - name: parent.pid level: core type: long format: string description: Process id. example: 4242 + default_field: false - name: parent.ppid level: extended type: long format: string description: Parent process' pid. example: 4241 + default_field: false - name: parent.start level: extended type: date description: The time the process started. example: '2016-05-23T08:05:34.853Z' + default_field: false - name: parent.thread.id level: extended type: long format: string description: Thread ID. example: 4242 + default_field: false - name: parent.thread.name level: extended type: keyword ignore_above: 1024 description: Thread name. example: thread-0 + default_field: false - name: parent.title level: extended type: keyword @@ -2375,17 +2397,20 @@ The proctitle, some times the same as process name. Can also be different: for example a browser setting its title to the web page currently opened.' + default_field: false - name: parent.uptime level: extended type: long description: Seconds the process has been up. example: 1325 + default_field: false - name: parent.working_directory level: extended type: keyword ignore_above: 1024 description: The working directory of the process. example: /home/alice + default_field: false - name: pgid level: extended type: long @@ -3060,6 +3085,7 @@ ignore_above: 1024 description: String indicating the cipher used during the current connection. example: TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 + default_field: false - name: client.certificate level: extended type: keyword @@ -3068,6 +3094,7 @@ is usually mutually-exclusive of `client.certificate_chain` since this value also exists in that list. example: MII... + default_field: false - name: client.certificate_chain level: extended type: keyword @@ -3078,6 +3105,7 @@ example: - MII... - MII... + default_field: false - name: client.hash.md5 level: extended type: keyword @@ -3086,6 +3114,7 @@ of certificate offered by the client. For consistency with other hash values, this value should be formatted as an uppercase hash. example: 0F76C7F2C55BFD7D8E8B8F4BFBF0C9EC + default_field: false - name: client.hash.sha1 level: extended type: keyword @@ -3094,6 +3123,7 @@ of certificate offered by the client. For consistency with other hash values, this value should be formatted as an uppercase hash. example: 9E393D93138888D288266C2D915214D1D1CCEB2A + default_field: false - name: client.hash.sha256 level: extended type: keyword @@ -3102,6 +3132,7 @@ version of certificate offered by the client. For consistency with other hash values, this value should be formatted as an uppercase hash. example: 0687F666A054EF17A08E2F2162EAB4CBC0D265E1D7875BE74BF3C712CA92DAF0 + default_field: false - name: client.issuer level: extended type: keyword @@ -3109,6 +3140,7 @@ description: Distinguished name of subject of the issuer of the x.509 certificate presented by the client. example: CN=MyDomain Root CA, OU=Infrastructure Team, DC=mydomain, DC=com + default_field: false - name: client.ja3 level: extended type: keyword @@ -3116,18 +3148,21 @@ description: A hash that identifies clients based on how they perform an SSL/TLS handshake. example: d4e5b18d6b55c71272893221c96ba240 + default_field: false - name: client.not_after level: extended type: date description: Date/Time indicating when client certificate is no longer considered valid. example: '2021-01-01T00:00:00.000Z' + default_field: false - name: client.not_before level: extended type: date description: Date/Time indicating when client certificate is first considered valid. example: '1970-01-01T00:00:00.000Z' + default_field: false - name: client.server_name level: extended type: keyword @@ -3136,6 +3171,7 @@ the client is attempting to connect. When this value is available, it should get copied to `destination.domain`. example: www.elastic.co + default_field: false - name: client.subject level: extended type: keyword @@ -3143,6 +3179,7 @@ description: Distinguished name of subject of the x.509 certificate presented by the client. example: CN=myclient, OU=Documentation Team, DC=mydomain, DC=com + default_field: false - name: client.supported_ciphers level: extended type: keyword @@ -3152,17 +3189,20 @@ - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - '...' + default_field: false - name: curve level: extended type: keyword ignore_above: 1024 description: String indicating the curve used for the given cipher, when applicable. example: secp256r1 + default_field: false - name: established level: extended type: boolean description: Boolean flag indicating if the TLS negotiation was successful and transitioned to an encrypted tunnel. + default_field: false - name: next_protocol level: extended type: keyword @@ -3171,11 +3211,13 @@ the IANA registry (https://www.iana.org/assignments/tls-extensiontype-values/tls-extensiontype-values.xhtml#alpn-protocol-ids), this string should be lower case. example: http/1.1 + default_field: false - name: resumed level: extended type: boolean description: Boolean flag indicating if this TLS connection was resumed from an existing TLS negotiation. + default_field: false - name: server.certificate level: extended type: keyword @@ -3184,6 +3226,7 @@ is usually mutually-exclusive of `server.certificate_chain` since this value also exists in that list. example: MII... + default_field: false - name: server.certificate_chain level: extended type: keyword @@ -3194,6 +3237,7 @@ example: - MII... - MII... + default_field: false - name: server.hash.md5 level: extended type: keyword @@ -3202,6 +3246,7 @@ of certificate offered by the server. For consistency with other hash values, this value should be formatted as an uppercase hash. example: 0F76C7F2C55BFD7D8E8B8F4BFBF0C9EC + default_field: false - name: server.hash.sha1 level: extended type: keyword @@ -3210,6 +3255,7 @@ of certificate offered by the server. For consistency with other hash values, this value should be formatted as an uppercase hash. example: 9E393D93138888D288266C2D915214D1D1CCEB2A + default_field: false - name: server.hash.sha256 level: extended type: keyword @@ -3218,6 +3264,7 @@ version of certificate offered by the server. For consistency with other hash values, this value should be formatted as an uppercase hash. example: 0687F666A054EF17A08E2F2162EAB4CBC0D265E1D7875BE74BF3C712CA92DAF0 + default_field: false - name: server.issuer level: extended type: keyword @@ -3225,6 +3272,7 @@ description: Subject of the issuer of the x.509 certificate presented by the server. example: CN=MyDomain Root CA, OU=Infrastructure Team, DC=mydomain, DC=com + default_field: false - name: server.ja3s level: extended type: keyword @@ -3232,36 +3280,42 @@ description: A hash that identifies servers based on how they perform an SSL/TLS handshake. example: 394441ab65754e2207b1e1b457b3641d + default_field: false - name: server.not_after level: extended type: date description: Timestamp indicating when server certificate is no longer considered valid. example: '2021-01-01T00:00:00.000Z' + default_field: false - name: server.not_before level: extended type: date description: Timestamp indicating when server certificate is first considered valid. example: '1970-01-01T00:00:00.000Z' + default_field: false - name: server.subject level: extended type: keyword ignore_above: 1024 description: Subject of the x.509 certificate presented by the server. example: CN=www.mydomain.com, OU=Infrastructure Team, DC=mydomain, DC=com + default_field: false - name: version level: extended type: keyword ignore_above: 1024 description: Numeric part of the version parsed from the original string. example: '1.2' + default_field: false - name: version_protocol level: extended type: keyword ignore_above: 1024 description: Normalized lowercase protocol name parsed from original string. example: tls + default_field: false - name: tracing title: Tracing group: 2 @@ -3559,6 +3613,7 @@ This field must be an array.' example: '["Firewall"]' + default_field: false - name: classification level: extended type: keyword @@ -3566,6 +3621,7 @@ description: The classification of the vulnerability scoring system. For example (https://www.first.org/cvss/) example: CVSS + default_field: false - name: description level: extended type: keyword @@ -3574,6 +3630,7 @@ of the vulnerability. For example (https://cve.mitre.org/about/faqs.html#cve_entry_descriptions_created[Common Vulnerabilities and Exposure CVE description]) example: In macOS before 2.12.6, there is a vulnerability in the RPC... + default_field: false - name: enumeration level: extended type: keyword @@ -3581,6 +3638,7 @@ description: The type of identifier used for this vulnerability. For example (https://cve.mitre.org/about/) example: CVE + default_field: false - name: id level: extended type: keyword @@ -3590,6 +3648,7 @@ example (https://cve.mitre.org/about/faqs.html#what_is_cve_id)[Common Vulnerabilities and Exposure CVE ID] example: CVE-2019-00001 + default_field: false - name: reference level: extended type: keyword @@ -3597,18 +3656,21 @@ description: A resource that provides additional information, context, and mitigations for the identified vulnerability. example: https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-6111 + default_field: false - name: report_id level: extended type: keyword ignore_above: 1024 description: The report or scan identification number. example: 20191018.0001 + default_field: false - name: scanner.vendor level: extended type: keyword ignore_above: 1024 description: The name of the vulnerability scanner vendor. example: Tenable + default_field: false - name: score.base level: extended type: float @@ -3618,6 +3680,7 @@ complexity, privileges, and user interaction), impact metrics (confidentiality, integrity, and availability), and scope. For example (https://www.first.org/cvss/specification-document)' example: 5.5 + default_field: false - name: score.environmental level: extended type: float @@ -3626,6 +3689,7 @@ Environmental scores cover an assessment for any modified Base metrics, confidentiality, integrity, and availability requirements. For example (https://www.first.org/cvss/specification-document)' example: 5.5 + default_field: false - name: score.temporal level: extended type: float @@ -3633,6 +3697,7 @@ Temporal scores cover an assessment for code maturity, remediation level, and confidence. For example (https://www.first.org/cvss/specification-document)' + default_field: false - name: score.version level: extended type: keyword @@ -3646,6 +3711,7 @@ organization, whose mission is to help computer security incident response teams across the world. For example (https://nvd.nist.gov/vuln-metrics/cvss)' example: 2.0 + default_field: false - name: severity level: extended type: keyword @@ -3653,3 +3719,4 @@ description: The severity of the vulnerability can help with metrics and internal prioritization regarding remediation. For example (https://nvd.nist.gov/vuln-metrics/cvss) example: Critical + default_field: false diff --git a/generated/ecs/ecs_flat.yml b/generated/ecs/ecs_flat.yml index c7152704d1..4202c5c505 100644 --- a/generated/ecs/ecs_flat.yml +++ b/generated/ecs/ecs_flat.yml @@ -1894,9 +1894,10 @@ host.architecture: short: Operating system architecture. type: keyword host.domain: - description: "Name of the domain of which the host is a member. \nFor example, on\ - \ Windows this could be the host's Active Directory domain or NetBIOS domain name.\ - \ For Linux this could be the domain of the host's LDAP provider." + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS + domain name. For Linux this could be the domain of the host''s LDAP provider.' example: CONTOSO flat_name: host.domain ignore_above: 1024 diff --git a/generated/ecs/ecs_nested.yml b/generated/ecs/ecs_nested.yml index e55adf013d..1989c4d388 100644 --- a/generated/ecs/ecs_nested.yml +++ b/generated/ecs/ecs_nested.yml @@ -2191,9 +2191,11 @@ host: short: Operating system architecture. type: keyword domain: - description: "Name of the domain of which the host is a member. \nFor example,\ - \ on Windows this could be the host's Active Directory domain or NetBIOS domain\ - \ name. For Linux this could be the domain of the host's LDAP provider." + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain + or NetBIOS domain name. For Linux this could be the domain of the host''s + LDAP provider.' example: CONTOSO flat_name: host.domain ignore_above: 1024 diff --git a/schemas/host.yml b/schemas/host.yml index c4eb751bf3..bdfe42fc5c 100644 --- a/schemas/host.yml +++ b/schemas/host.yml @@ -85,9 +85,9 @@ type: keyword short: Name of the directory the group is a member of. description: > - Name of the domain of which the host is a member. + Name of the domain of which the host is a member. - For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. + For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. example: CONTOSO diff --git a/scripts/generators/beats.py b/scripts/generators/beats.py index 54791df696..9fe1c87668 100644 --- a/scripts/generators/beats.py +++ b/scripts/generators/beats.py @@ -5,8 +5,11 @@ def generate(ecs_nested, ecs_version): + # Load temporary whitelist for default_fields workaround. + df_whitelist = ecs_helpers.yaml_load('scripts/generators/beats_default_fields_whitelist.yml') + # base first - beats_fields = fieldset_field_array(ecs_nested['base']['fields']) + beats_fields = fieldset_field_array(ecs_nested['base']['fields'], df_whitelist) allowed_fieldset_keys = ['name', 'title', 'group', 'description', 'footnote', 'type'] # other fieldsets @@ -16,7 +19,7 @@ def generate(ecs_nested, ecs_version): fieldset = ecs_nested[fieldset_name] beats_field = ecs_helpers.dict_copy_keys_ordered(fieldset, allowed_fieldset_keys) - beats_field['fields'] = fieldset_field_array(fieldset['fields']) + beats_field['fields'] = fieldset_field_array(fieldset['fields'], df_whitelist) beats_fields.append(beats_field) beats_file = OrderedDict() @@ -28,15 +31,21 @@ def generate(ecs_nested, ecs_version): write_beats_yaml(beats_file, ecs_version) -def fieldset_field_array(source_fields): +def fieldset_field_array(source_fields, df_whitelist): allowed_keys = ['name', 'level', 'required', 'type', 'object_type', 'ignore_above', 'multi_fields', 'format', 'input_format', 'output_format', 'output_precision', 'description', 'example'] + multi_fields_allowed_keys = ['name', 'type', 'norms'] + fields = [] for nested_field_name in source_fields: ecs_field = source_fields[nested_field_name] beats_field = ecs_helpers.dict_copy_keys_ordered(ecs_field, allowed_keys) beats_field['name'] = nested_field_name + + if not ecs_field['flat_name'] in df_whitelist: + beats_field['default_field'] = False + fields.append(beats_field) return sorted(fields, lambda x, y: cmp(x['name'], y['name'])) diff --git a/scripts/generators/beats_default_fields_whitelist.yml b/scripts/generators/beats_default_fields_whitelist.yml new file mode 100644 index 0000000000..a7fa30b9c8 --- /dev/null +++ b/scripts/generators/beats_default_fields_whitelist.yml @@ -0,0 +1,401 @@ +--- +!!set +# Note: other fields can be inserted as needed + +# ECS 1.2 fields +'@timestamp': null +agent.ephemeral_id: null +agent.id: null +agent.name: null +agent.type: null +agent.version: null +as.number: null +as.organization.name: null +client.address: null +client.as.number: null +client.as.organization.name: null +client.bytes: null +client.domain: null +client.geo.city_name: null +client.geo.continent_name: null +client.geo.country_iso_code: null +client.geo.country_name: null +client.geo.location: null +client.geo.name: null +client.geo.region_iso_code: null +client.geo.region_name: null +client.ip: null +client.mac: null +client.nat.ip: null +client.nat.port: null +client.packets: null +client.port: null +client.registered_domain: null +client.top_level_domain: null +client.user.domain: null +client.user.email: null +client.user.full_name: null +client.user.group.domain: null +client.user.group.id: null +client.user.group.name: null +client.user.hash: null +client.user.id: null +client.user.name: null +cloud.account.id: null +cloud.availability_zone: null +cloud.instance.id: null +cloud.instance.name: null +cloud.machine.type: null +cloud.provider: null +cloud.region: null +container.id: null +container.image.name: null +container.image.tag: null +container.labels: null +container.name: null +container.runtime: null +destination.address: null +destination.as.number: null +destination.as.organization.name: null +destination.bytes: null +destination.domain: null +destination.geo.city_name: null +destination.geo.continent_name: null +destination.geo.country_iso_code: null +destination.geo.country_name: null +destination.geo.location: null +destination.geo.name: null +destination.geo.region_iso_code: null +destination.geo.region_name: null +destination.ip: null +destination.mac: null +destination.nat.ip: null +destination.nat.port: null +destination.packets: null +destination.port: null +destination.registered_domain: null +destination.top_level_domain: null +destination.user.domain: null +destination.user.email: null +destination.user.full_name: null +destination.user.group.domain: null +destination.user.group.id: null +destination.user.group.name: null +destination.user.hash: null +destination.user.id: null +destination.user.name: null +dns.answers: null +dns.answers.class: null +dns.answers.data: null +dns.answers.name: null +dns.answers.ttl: null +dns.answers.type: null +dns.header_flags: null +dns.id: null +dns.op_code: null +dns.question.class: null +dns.question.name: null +dns.question.registered_domain: null +dns.question.subdomain: null +dns.question.top_level_domain: null +dns.question.type: null +dns.resolved_ip: null +dns.response_code: null +dns.type: null +ecs.version: null +error.code: null +error.id: null +error.message: null +error.stack_trace: null +error.type: null +event.action: null +event.category: null +event.code: null +event.created: null +event.dataset: null +event.duration: null +event.end: null +event.hash: null +event.id: null +event.kind: null +event.module: null +event.original: null +event.outcome: null +event.provider: null +event.risk_score: null +event.risk_score_norm: null +event.sequence: null +event.severity: null +event.start: null +event.timezone: null +event.type: null +file.accessed: null +file.created: null +file.ctime: null +file.device: null +file.directory: null +file.extension: null +file.gid: null +file.group: null +file.hash.md5: null +file.hash.sha1: null +file.hash.sha256: null +file.hash.sha512: null +file.inode: null +file.mode: null +file.mtime: null +file.name: null +file.owner: null +file.path: null +file.size: null +file.target_path: null +file.type: null +file.uid: null +geo.city_name: null +geo.continent_name: null +geo.country_iso_code: null +geo.country_name: null +geo.location: null +geo.name: null +geo.region_iso_code: null +geo.region_name: null +group.domain: null +group.id: null +group.name: null +hash.md5: null +hash.sha1: null +hash.sha256: null +hash.sha512: null +host.architecture: null +host.geo.city_name: null +host.geo.continent_name: null +host.geo.country_iso_code: null +host.geo.country_name: null +host.geo.location: null +host.geo.name: null +host.geo.region_iso_code: null +host.geo.region_name: null +host.hostname: null +host.id: null +host.ip: null +host.mac: null +host.name: null +host.os.family: null +host.os.full: null +host.os.kernel: null +host.os.name: null +host.os.platform: null +host.os.version: null +host.type: null +host.uptime: null +host.user.domain: null +host.user.email: null +host.user.full_name: null +host.user.group.domain: null +host.user.group.id: null +host.user.group.name: null +host.user.hash: null +host.user.id: null +host.user.name: null +http.request.body.bytes: null +http.request.body.content: null +http.request.bytes: null +http.request.method: null +http.request.referrer: null +http.response.body.bytes: null +http.response.body.content: null +http.response.bytes: null +http.response.status_code: null +http.version: null +labels: null +log.level: null +log.logger: null +log.origin.file.line: null +log.origin.file.name: null +log.origin.function: null +log.original: null +log.syslog: null +log.syslog.facility.code: null +log.syslog.facility.name: null +log.syslog.priority: null +log.syslog.severity.code: null +log.syslog.severity.name: null +message: null +network.application: null +network.bytes: null +network.community_id: null +network.direction: null +network.forwarded_ip: null +network.iana_number: null +network.name: null +network.packets: null +network.protocol: null +network.transport: null +network.type: null +observer.geo.city_name: null +observer.geo.continent_name: null +observer.geo.country_iso_code: null +observer.geo.country_name: null +observer.geo.location: null +observer.geo.name: null +observer.geo.region_iso_code: null +observer.geo.region_name: null +observer.hostname: null +observer.ip: null +observer.mac: null +observer.name: null +observer.os.family: null +observer.os.full: null +observer.os.kernel: null +observer.os.name: null +observer.os.platform: null +observer.os.version: null +observer.product: null +observer.serial_number: null +observer.type: null +observer.vendor: null +observer.version: null +organization.id: null +organization.name: null +os.family: null +os.full: null +os.kernel: null +os.name: null +os.platform: null +os.version: null +package.architecture: null +package.checksum: null +package.description: null +package.install_scope: null +package.installed: null +package.license: null +package.name: null +package.path: null +package.size: null +package.version: null +process.args: null +process.executable: null +process.hash.md5: null +process.hash.sha1: null +process.hash.sha256: null +process.hash.sha512: null +process.name: null +process.pgid: null +process.pid: null +process.ppid: null +process.start: null +process.thread.id: null +process.thread.name: null +process.title: null +process.uptime: null +process.working_directory: null +related.ip: null +server.address: null +server.as.number: null +server.as.organization.name: null +server.bytes: null +server.domain: null +server.geo.city_name: null +server.geo.continent_name: null +server.geo.country_iso_code: null +server.geo.country_name: null +server.geo.location: null +server.geo.name: null +server.geo.region_iso_code: null +server.geo.region_name: null +server.ip: null +server.mac: null +server.nat.ip: null +server.nat.port: null +server.packets: null +server.port: null +server.registered_domain: null +server.top_level_domain: null +server.user.domain: null +server.user.email: null +server.user.full_name: null +server.user.group.domain: null +server.user.group.id: null +server.user.group.name: null +server.user.hash: null +server.user.id: null +server.user.name: null +service.ephemeral_id: null +service.id: null +service.name: null +service.node.name: null +service.state: null +service.type: null +service.version: null +source.address: null +source.as.number: null +source.as.organization.name: null +source.bytes: null +source.domain: null +source.geo.city_name: null +source.geo.continent_name: null +source.geo.country_iso_code: null +source.geo.country_name: null +source.geo.location: null +source.geo.name: null +source.geo.region_iso_code: null +source.geo.region_name: null +source.ip: null +source.mac: null +source.nat.ip: null +source.nat.port: null +source.packets: null +source.port: null +source.registered_domain: null +source.top_level_domain: null +source.user.domain: null +source.user.email: null +source.user.full_name: null +source.user.group.domain: null +source.user.group.id: null +source.user.group.name: null +source.user.hash: null +source.user.id: null +source.user.name: null +tags: null +threat.framework: null +threat.tactic.id: null +threat.tactic.name: null +threat.tactic.reference: null +threat.technique.id: null +threat.technique.name: null +threat.technique.reference: null +trace.id: null +transaction.id: null +url.domain: null +url.extension: null +url.fragment: null +url.full: null +url.original: null +url.password: null +url.path: null +url.port: null +url.query: null +url.registered_domain: null +url.scheme: null +url.top_level_domain: null +url.username: null +user.domain: null +user.email: null +user.full_name: null +user.group.domain: null +user.group.id: null +user.group.name: null +user.hash: null +user.id: null +user.name: null +user_agent.device.name: null +user_agent.name: null +user_agent.original: null +user_agent.os.family: null +user_agent.os.full: null +user_agent.os.kernel: null +user_agent.os.name: null +user_agent.os.platform: null +user_agent.os.version: null +user_agent.version: null diff --git a/scripts/generators/ecs_helpers.py b/scripts/generators/ecs_helpers.py index f64e07ef9b..6c46dfe0de 100644 --- a/scripts/generators/ecs_helpers.py +++ b/scripts/generators/ecs_helpers.py @@ -61,6 +61,14 @@ def yaml_ordereddict(dumper, data): yaml.add_representer(OrderedDict, yaml_ordereddict) + +def dict_rename_keys(dict, renames): + for key, value in dict.iteritems(): + if key in renames: + del dict[key] + dict[renames[key]] = value + + # File helpers @@ -69,3 +77,8 @@ def yaml_dump(filename, data, preamble=None): if preamble: outfile.write(preamble) yaml.dump(data, outfile, default_flow_style=False) + + +def yaml_load(filename): + with open(filename) as f: + return yaml.load(f.read())