diff --git a/CHANGELOG.next.md b/CHANGELOG.next.md index 50a46cf6b2..8390424769 100644 --- a/CHANGELOG.next.md +++ b/CHANGELOG.next.md @@ -10,6 +10,7 @@ * New generator that supports reusable fields, for files based on ECS. It generates schema.csv, Elasticsearch 6 and 7 templates, and field documentation for the main website. #336 +* Generator for the Beats fields.ecs.yml file. #379 ### Improvements diff --git a/generated/beats/fields.ecs.yml b/generated/beats/fields.ecs.yml new file mode 100644 index 0000000000..2ecff3ead8 --- /dev/null +++ b/generated/beats/fields.ecs.yml @@ -0,0 +1,2131 @@ +# WARNING! Do not edit this file directly, it was generated by the ECS project, +# based on ECS version 1.0.0. +# Please visit https://github.com/elastic/ecs to suggest changes to ECS fields. + +- key: ecs + title: ECS + description: ECS Fields. + fields: + - name: '@timestamp' + level: core + required: true + type: date + description: 'Date/time when the event originated. + + This is the date/time extracted from the event, typically representing when + the event was generated by the source. + + If the event source has no original timestamp, this value is typically populated + by the first time the event was received by the pipeline. + + Required field for all events.' + example: '2016-05-23T08:05:34.853Z' + - name: labels + level: core + type: object + object_type: keyword + description: 'Custom key/value pairs. + + Can be used to add meta information to events. Should not contain nested objects. + All values are stored as keyword. + + Example: `docker` and `k8s` labels.' + example: + application: foo-bar + env: production + - name: message + level: core + type: text + description: 'For log events the message field contains the log message, optimized + for viewing in a log viewer. + + For structured logs without an original message field, other fields can be concatenated + to form a human-readable summary of the event. + + If multiple messages exist, they can be combined into one message.' + example: Hello World + - name: tags + level: core + type: keyword + ignore_above: 1024 + description: List of keywords used to tag each event. + example: '["production", "env2"]' + - name: agent + title: Agent + group: 2 + description: 'The agent fields contain the data about the software entity, if + any, that collects, detects, or observes events on a host, or takes measurements + on a host. + + Examples include Beats. Agents may also run on observers. ECS agent.* fields + shall be populated with details of the agent running on the host or observer + where the event happened or the measurement was taken.' + footnote: 'Examples: In the case of Beats for logs, the agent.name is filebeat. + For APM, it is the agent running in the app/service. The agent information does + not change if data is sent through queuing systems like Kafka, Redis, or processing + systems such as Logstash or APM Server.' + type: group + fields: + - name: ephemeral_id + level: extended + type: keyword + ignore_above: 1024 + description: 'Ephemeral identifier of this agent (if one exists). + + This id normally changes across restarts, but `agent.id` does not.' + example: 8a4f500f + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique identifier of this agent (if one exists). + + Example: For Beats this would be beat.id.' + example: 8a4f500d + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Custom name of the agent. + + This is a name that can be given to an agent. This can be helpful if for example + two Filebeat instances are running on the same host but a human readable separation + is needed on which Filebeat instance data is coming from. + + If no name is given, the name is often left empty.' + example: foo + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of the agent. + + The agent type stays always the same and should be given by the agent used. + In case of Filebeat the agent would always be Filebeat also if two Filebeat + instances are run on the same machine.' + example: filebeat + - name: version + level: core + type: keyword + ignore_above: 1024 + description: Version of the agent. + example: 6.0.0-rc2 + - name: client + title: Client + group: 2 + description: 'A client is defined as the initiator of a network connection for + events regarding sessions, connections, or bidirectional flow records. + + For TCP events, the client is the initiator of the TCP connection that sends + the SYN packet(s). For other protocols, the client is generally the initiator + or requestor in the network transaction. Some systems use the term "originator" + to refer the client in TCP connections. The client fields describe details about + the system acting as the client in the network event. Client fields are usually + populated in conjunction with server fields. Client fields are generally not + populated for packet-level events. + + Client / server representations can add semantic context to an exchange, which + is helpful to visualize the data in certain situations. If your context falls + in that category, you should still ensure that source and destination are filled + appropriately.' + type: group + fields: + - name: address + level: extended + type: keyword + ignore_above: 1024 + description: 'Some event client addresses are defined ambiguously. The event + will sometimes list an IP, a domain or a unix socket. You should always store + the raw address in the `.address` field. + + Then it should be duplicated to `.ip` or `.domain`, depending on which one + it is.' + - name: bytes + level: core + type: long + description: Bytes sent from the client to the server. + example: 184 + - name: domain + level: core + type: keyword + ignore_above: 1024 + description: Client domain. + - name: geo.city_name + level: core + type: keyword + ignore_above: 1024 + description: City name. + example: Montreal + - name: geo.continent_name + level: core + type: keyword + ignore_above: 1024 + description: Name of the continent. + example: North America + - name: geo.country_iso_code + level: core + type: keyword + ignore_above: 1024 + description: Country ISO code. + example: CA + - name: geo.country_name + level: core + type: keyword + ignore_above: 1024 + description: Country name. + example: Canada + - name: geo.location + level: core + type: geo_point + description: Longitude and latitude. + example: '{ "lon": -73.614830, "lat": 45.505918 }' + - name: geo.name + level: extended + type: keyword + ignore_above: 1024 + description: 'User-defined description of a location, at the level of granularity + they care about. + + Could be the name of their data centers, the floor number, if this describes + a local physical entity, city names. + + Not typically used in automated geolocation.' + example: boston-dc + - name: geo.region_iso_code + level: core + type: keyword + ignore_above: 1024 + description: Region ISO code. + example: CA-QC + - name: geo.region_name + level: core + type: keyword + ignore_above: 1024 + description: Region name. + example: Quebec + - name: ip + level: core + type: ip + description: 'IP address of the client. + + Can be one or multiple IPv4 or IPv6 addresses.' + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: MAC address of the client. + - name: packets + level: core + type: long + description: Packets sent from the client to the server. + example: 12 + - name: port + level: core + type: long + description: Port of the client. + - name: user.email + level: extended + type: keyword + ignore_above: 1024 + description: User email address. + - name: user.full_name + level: extended + type: keyword + ignore_above: 1024 + description: User's full name, if available. + example: Albert Einstein + - name: user.group.id + level: extended + type: keyword + ignore_above: 1024 + description: Unique identifier for the group on the system/platform. + - name: user.group.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the group. + - name: user.hash + level: extended + type: keyword + ignore_above: 1024 + description: 'Unique user hash to correlate information for a user in anonymized + form. + + Useful if `user.id` or `user.name` contain confidential information and cannot + be used.' + - name: user.id + level: core + type: keyword + ignore_above: 1024 + description: One or multiple unique identifiers of the user. + - name: user.name + level: core + type: keyword + ignore_above: 1024 + description: Short name or login of the user. + example: albert + - name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming + from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data + from its host, the cloud info contains the data about this machine. If Metricbeat + runs on a remote machine outside the cloud and fetches data from a service running + in the cloud, the field contains cloud data from the machine the service is + running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different + entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, + or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific + container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: image.tag + level: extended + type: keyword + ignore_above: 1024 + description: Container image tag. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. + - name: runtime + level: extended + type: keyword + ignore_above: 1024 + description: Runtime managing this container. + example: docker + - name: destination + title: Destination + group: 2 + description: 'Destination fields describe details about the destination of a packet/event. + + Destination fields are usually populated in conjunction with source fields.' + type: group + fields: + - name: address + level: extended + type: keyword + ignore_above: 1024 + description: 'Some event destination addresses are defined ambiguously. The + event will sometimes list an IP, a domain or a unix socket. You should always + store the raw address in the `.address` field. + + Then it should be duplicated to `.ip` or `.domain`, depending on which one + it is.' + - name: bytes + level: core + type: long + description: Bytes sent from the destination to the source. + example: 184 + - name: domain + level: core + type: keyword + ignore_above: 1024 + description: Destination domain. + - name: geo.city_name + level: core + type: keyword + ignore_above: 1024 + description: City name. + example: Montreal + - name: geo.continent_name + level: core + type: keyword + ignore_above: 1024 + description: Name of the continent. + example: North America + - name: geo.country_iso_code + level: core + type: keyword + ignore_above: 1024 + description: Country ISO code. + example: CA + - name: geo.country_name + level: core + type: keyword + ignore_above: 1024 + description: Country name. + example: Canada + - name: geo.location + level: core + type: geo_point + description: Longitude and latitude. + example: '{ "lon": -73.614830, "lat": 45.505918 }' + - name: geo.name + level: extended + type: keyword + ignore_above: 1024 + description: 'User-defined description of a location, at the level of granularity + they care about. + + Could be the name of their data centers, the floor number, if this describes + a local physical entity, city names. + + Not typically used in automated geolocation.' + example: boston-dc + - name: geo.region_iso_code + level: core + type: keyword + ignore_above: 1024 + description: Region ISO code. + example: CA-QC + - name: geo.region_name + level: core + type: keyword + ignore_above: 1024 + description: Region name. + example: Quebec + - name: ip + level: core + type: ip + description: 'IP address of the destination. + + Can be one or multiple IPv4 or IPv6 addresses.' + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: MAC address of the destination. + - name: packets + level: core + type: long + description: Packets sent from the destination to the source. + example: 12 + - name: port + level: core + type: long + description: Port of the destination. + - name: user.email + level: extended + type: keyword + ignore_above: 1024 + description: User email address. + - name: user.full_name + level: extended + type: keyword + ignore_above: 1024 + description: User's full name, if available. + example: Albert Einstein + - name: user.group.id + level: extended + type: keyword + ignore_above: 1024 + description: Unique identifier for the group on the system/platform. + - name: user.group.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the group. + - name: user.hash + level: extended + type: keyword + ignore_above: 1024 + description: 'Unique user hash to correlate information for a user in anonymized + form. + + Useful if `user.id` or `user.name` contain confidential information and cannot + be used.' + - name: user.id + level: core + type: keyword + ignore_above: 1024 + description: One or multiple unique identifiers of the user. + - name: user.name + level: core + type: keyword + ignore_above: 1024 + description: Short name or login of the user. + example: albert + - name: ecs + title: ECS + group: 2 + description: Meta-information specific to ECS. + type: group + fields: + - name: version + level: core + required: true + type: keyword + ignore_above: 1024 + description: 'ECS version this event conforms to. `ecs.version` is a required + field and must exist in all events. + + When querying across multiple indices -- which may conform to slightly different + ECS versions -- this field lets integrations adjust to the schema version + of the events.' + example: 1.0.0 + - name: error + title: Error + group: 2 + description: 'These fields can represent errors of any kind. + + Use them for errors that happen while fetching events or in cases where the + event itself contains an error.' + type: group + fields: + - name: code + level: core + type: keyword + ignore_above: 1024 + description: Error code describing the error. + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique identifier for the error. + - name: message + level: core + type: text + description: Error message. + - name: event + title: Event + group: 2 + description: 'The event fields are used for context information about the log + or metric event itself. + + A log is defined as an event containing details of something that happened. + Log events must include the time at which the thing happened. Examples of log + events include a process starting on a host, a network packet being sent from + a source to a destination, or a network connection between a client and a server + being initiated or closed. A metric is defined as an event containing one or + more numerical or categorical measurements and the time at which the measurement + was taken. Examples of metric events include memory pressure measured on a host, + or vulnerabilities measured on a scanned host.' + type: group + fields: + - name: action + level: core + type: keyword + ignore_above: 1024 + description: 'The action captured by the event. + + This describes the information in the event. It is more specific than `event.category`. + Examples are `group-add`, `process-started`, `file-created`. The value is + normally defined by the implementer.' + example: user-password-change + - name: category + level: core + type: keyword + ignore_above: 1024 + description: 'Event category. + + This contains high-level information about the contents of the event. It is + more generic than `event.action`, in the sense that typically a category contains + multiple actions. Warning: In future versions of ECS, we plan to provide a + list of acceptable values for this field, please use with caution.' + example: user-management + - name: created + level: core + type: date + description: 'event.created contains the date/time when the event was first + read by an agent, or by your pipeline. + + This field is distinct from @timestamp in that @timestamp typically contain + the time extracted from the original event. + + In most situations, these two timestamps will be slightly different. The difference + can be used to calculate the delay between your source generating an event, + and the time when your agent first processed it. This can be used to monitor + your agent''s or pipeline''s ability to keep up with your event source. + + In case the two timestamps are identical, @timestamp should be used.' + - name: dataset + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the dataset. + + The concept of a `dataset` (fileset / metricset) is used in Beats as a subset + of modules. It contains the information which is currently stored in metricset.name + and metricset.module or fileset.name.' + example: stats + - name: duration + level: core + type: long + description: 'Duration of the event in nanoseconds. + + If event.start and event.end are known this value should be the difference + between the end and start time.' + - name: end + level: extended + type: date + description: event.end contains the date when the event ended or when the activity + was last observed. + - name: hash + level: extended + type: keyword + ignore_above: 1024 + description: Hash (perhaps logstash fingerprint) of raw field to be able to + demonstrate log integrity. + example: 123456789012345678901234567890ABCD + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique ID to describe the event. + example: 8a4f500d + - name: kind + level: extended + type: keyword + ignore_above: 1024 + description: 'The kind of the event. + + This gives information about what type of information the event contains, + without being specific to the contents of the event. Examples are `event`, + `state`, `alarm`. Warning: In future versions of ECS, we plan to provide a + list of acceptable values for this field, please use with caution.' + example: state + - name: module + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the module this data is coming from. + + This information is coming from the modules used in Beats or Logstash.' + example: mysql + - name: original + level: core + type: keyword + ignore_above: 1024 + description: 'Raw text message of entire event. Used to demonstrate log integrity. + + This field is not indexed and doc_values are disabled. It cannot be searched, + but it can be retrieved from `_source`.' + example: Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100| + worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232 + - name: outcome + level: extended + type: keyword + ignore_above: 1024 + description: 'The outcome of the event. + + If the event describes an action, this fields contains the outcome of that + action. Examples outcomes are `success` and `failure`. Warning: In future + versions of ECS, we plan to provide a list of acceptable values for this field, + please use with caution.' + example: success + - name: risk_score + level: core + type: float + description: Risk score or priority of the event (e.g. security solutions). + Use your system's original value here. + - name: risk_score_norm + level: extended + type: float + description: 'Normalized risk score or priority of the event, on a scale of + 0 to 100. + + This is mainly useful if you use more than one system that assigns risk scores, + and you want to see a normalized value across all systems.' + - name: severity + level: core + type: long + description: Severity describes the original severity of the event. What the + different severity values mean can very different between use cases. It's + up to the implementer to make sure severities are consistent across events. + example: '7' + - name: start + level: extended + type: date + description: event.start contains the date when the event started or when the + activity was first observed. + - name: timezone + level: extended + type: keyword + ignore_above: 1024 + description: 'This field should be populated when the event''s timestamp does + not include timezone information already (e.g. default Syslog timestamps). + It''s optional otherwise. + + Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), + abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00").' + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Reserved for future usage. + + Please avoid using this field for user data.' + - name: file + title: File + group: 2 + description: 'A file is defined as a set of information that has been created + on, or has existed on a filesystem. + + File objects can be associated with host events, network events, and/or file + events (e.g., those produced by File Integrity Monitoring [FIM] products or + services). File fields provide details about the affected file associated with + the event or metric.' + type: group + fields: + - name: ctime + level: extended + type: date + description: Last time file metadata changed. + - name: device + level: extended + type: keyword + ignore_above: 1024 + description: Device that is the source of the file. + - name: extension + level: extended + type: keyword + ignore_above: 1024 + description: 'File extension. + + This should allow easy filtering by file extensions.' + example: png + - name: gid + level: extended + type: keyword + ignore_above: 1024 + description: Primary group ID (GID) of the file. + - name: group + level: extended + type: keyword + ignore_above: 1024 + description: Primary group name of the file. + - name: inode + level: extended + type: keyword + ignore_above: 1024 + description: Inode representing the file in the filesystem. + - name: mode + level: extended + type: keyword + ignore_above: 1024 + description: Mode of the file in octal representation. + example: 416 + - name: mtime + level: extended + type: date + description: Last time file content was modified. + - name: owner + level: extended + type: keyword + ignore_above: 1024 + description: File owner's username. + - name: path + level: extended + type: keyword + ignore_above: 1024 + description: Path to the file. + - name: size + level: extended + type: long + description: File size in bytes (field is only added when `type` is `file`). + - name: target_path + level: extended + type: keyword + ignore_above: 1024 + description: Target path for symlinks. + - name: type + level: extended + type: keyword + ignore_above: 1024 + description: File type (file, dir, or symlink). + - name: uid + level: extended + type: keyword + ignore_above: 1024 + description: The user ID (UID) or security identifier (SID) of the file owner. + - name: geo + title: Geo + group: 2 + description: 'Geo fields can carry data about a specific location related to an + event. + + This geolocation information can be derived from techniques such as Geo IP, + or be user-supplied.' + type: group + fields: + - name: city_name + level: core + type: keyword + ignore_above: 1024 + description: City name. + example: Montreal + - name: continent_name + level: core + type: keyword + ignore_above: 1024 + description: Name of the continent. + example: North America + - name: country_iso_code + level: core + type: keyword + ignore_above: 1024 + description: Country ISO code. + example: CA + - name: country_name + level: core + type: keyword + ignore_above: 1024 + description: Country name. + example: Canada + - name: location + level: core + type: geo_point + description: Longitude and latitude. + example: '{ "lon": -73.614830, "lat": 45.505918 }' + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: 'User-defined description of a location, at the level of granularity + they care about. + + Could be the name of their data centers, the floor number, if this describes + a local physical entity, city names. + + Not typically used in automated geolocation.' + example: boston-dc + - name: region_iso_code + level: core + type: keyword + ignore_above: 1024 + description: Region ISO code. + example: CA-QC + - name: region_name + level: core + type: keyword + ignore_above: 1024 + description: Region name. + example: Quebec + - name: group + title: Group + group: 2 + description: The group fields are meant to represent groups that are relevant + to the event. + type: group + fields: + - name: id + level: extended + type: keyword + ignore_above: 1024 + description: Unique identifier for the group on the system/platform. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the group. + - name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the + event happened, or from which the measurement was taken. Host types include + hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: geo.city_name + level: core + type: keyword + ignore_above: 1024 + description: City name. + example: Montreal + - name: geo.continent_name + level: core + type: keyword + ignore_above: 1024 + description: Name of the continent. + example: North America + - name: geo.country_iso_code + level: core + type: keyword + ignore_above: 1024 + description: Country ISO code. + example: CA + - name: geo.country_name + level: core + type: keyword + ignore_above: 1024 + description: Country name. + example: Canada + - name: geo.location + level: core + type: geo_point + description: Longitude and latitude. + example: '{ "lon": -73.614830, "lat": 45.505918 }' + - name: geo.name + level: extended + type: keyword + ignore_above: 1024 + description: 'User-defined description of a location, at the level of granularity + they care about. + + Could be the name of their data centers, the floor number, if this describes + a local physical entity, city names. + + Not typically used in automated geolocation.' + example: boston-dc + - name: geo.region_iso_code + level: core + type: keyword + ignore_above: 1024 + description: Region ISO code. + example: CA-QC + - name: geo.region_name + level: core + type: keyword + ignore_above: 1024 + description: Region name. + example: Quebec + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip address. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac address. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified + domain name, or a name specified by the user. The sender decides which value + to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.full + level: extended + type: keyword + ignore_above: 1024 + description: Operating system name, including the version or code name. + example: Mac OS Mojave + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, + this could be the container, for example, or other information meaningful + in your environment.' + - name: user.email + level: extended + type: keyword + ignore_above: 1024 + description: User email address. + - name: user.full_name + level: extended + type: keyword + ignore_above: 1024 + description: User's full name, if available. + example: Albert Einstein + - name: user.group.id + level: extended + type: keyword + ignore_above: 1024 + description: Unique identifier for the group on the system/platform. + - name: user.group.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the group. + - name: user.hash + level: extended + type: keyword + ignore_above: 1024 + description: 'Unique user hash to correlate information for a user in anonymized + form. + + Useful if `user.id` or `user.name` contain confidential information and cannot + be used.' + - name: user.id + level: core + type: keyword + ignore_above: 1024 + description: One or multiple unique identifiers of the user. + - name: user.name + level: core + type: keyword + ignore_above: 1024 + description: Short name or login of the user. + example: albert + - name: http + title: HTTP + group: 2 + description: Fields related to HTTP activity. Use the `url` field set to store + the url of the request. + type: group + fields: + - name: request.body.bytes + level: extended + type: long + description: Size in bytes of the request body. + example: 887 + - name: request.body.content + level: extended + type: keyword + ignore_above: 1024 + description: The full HTTP request body. + example: Hello world + - name: request.bytes + level: extended + type: long + description: Total size in bytes of the request (body and headers). + example: 1437 + - name: request.method + level: extended + type: keyword + ignore_above: 1024 + description: 'HTTP request method. + + The field value must be normalized to lowercase for querying. See the documentation + section "Implementing ECS".' + example: get, post, put + - name: request.referrer + level: extended + type: keyword + ignore_above: 1024 + description: Referrer for this HTTP request. + example: https://blog.example.com/ + - name: response.body.bytes + level: extended + type: long + description: Size in bytes of the response body. + example: 887 + - name: response.body.content + level: extended + type: keyword + ignore_above: 1024 + description: The full HTTP response body. + example: Hello world + - name: response.bytes + level: extended + type: long + description: Total size in bytes of the response (body and headers). + example: 1437 + - name: response.status_code + level: extended + type: long + description: HTTP response status code. + example: 404 + - name: version + level: extended + type: keyword + ignore_above: 1024 + description: HTTP version. + example: 1.1 + - name: log + title: Log + group: 2 + description: Fields which are specific to log events. + type: group + fields: + - name: level + level: core + type: keyword + ignore_above: 1024 + description: 'Original log level of the log event. + + Some examples are `warn`, `error`, `i`.' + example: err + - name: original + level: core + type: keyword + ignore_above: 1024 + description: 'This is the original log message and contains the full log message + before splitting it up in multiple parts. + + In contrast to the `message` field which can contain an extracted part of + the log message, this field contains the original, full log message. It can + have already some modifications applied like encoding or new lines removed + to clean up the log message. + + This field is not indexed and doc_values are disabled so it can''t be queried + but the value can be retrieved from `_source`.' + example: Sep 19 08:26:10 localhost My log + - name: network + title: Network + group: 2 + description: 'The network is defined as the communication path over which a host + or network event happens. + + The network.* fields should be populated with details about the network activity + associated with an event.' + type: group + fields: + - name: application + level: extended + type: keyword + ignore_above: 1024 + description: 'A name given to an application level protocol. This can be arbitrarily + assigned for things like microservices, but also apply to things like skype, + icq, facebook, twitter. This would be used in situations where the vendor + or service can be decoded such as from the source/dest IP owners, ports, or + wire format. + + The field value must be normalized to lowercase for querying. See the documentation + section "Implementing ECS".' + example: aim + - name: bytes + level: core + type: long + description: 'Total bytes transferred in both directions. + + If `source.bytes` and `destination.bytes` are known, `network.bytes` is their + sum.' + example: 368 + - name: community_id + level: extended + type: keyword + ignore_above: 1024 + description: 'A hash of source and destination IPs and ports, as well as the + protocol used in a communication. This is a tool-agnostic standard to identify + flows. + + Learn more at https://github.com/corelight/community-id-spec.' + example: 1:hO+sN4H+MG5MY/8hIrXPqc4ZQz0= + - name: direction + level: core + type: keyword + ignore_above: 1024 + description: "Direction of the network traffic.\nRecommended values are:\n \ + \ * inbound\n * outbound\n * internal\n * external\n * unknown\n\nWhen\ + \ mapping events from a host-based monitoring context, populate this field\ + \ from the host's point of view.\nWhen mapping events from a network or perimeter-based\ + \ monitoring context, populate this field from the point of view of your network\ + \ perimeter." + example: inbound + - name: forwarded_ip + level: core + type: ip + description: Host IP address when the source IP address is the proxy. + example: 192.1.1.2 + - name: iana_number + level: extended + type: keyword + ignore_above: 1024 + description: IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). + Standardized list of protocols. This aligns well with NetFlow and sFlow related + logs which use the IANA Protocol Number. + example: 6 + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Name given by operators to sections of their network. + example: Guest Wifi + - name: packets + level: core + type: long + description: 'Total packets transferred in both directions. + + If `source.packets` and `destination.packets` are known, `network.packets` + is their sum.' + example: 24 + - name: protocol + level: core + type: keyword + ignore_above: 1024 + description: 'L7 Network protocol name. ex. http, lumberjack, transport protocol. + + The field value must be normalized to lowercase for querying. See the documentation + section "Implementing ECS".' + example: http + - name: transport + level: core + type: keyword + ignore_above: 1024 + description: 'Same as network.iana_number, but instead using the Keyword name + of the transport layer (udp, tcp, ipv6-icmp, etc.) + + The field value must be normalized to lowercase for querying. See the documentation + section "Implementing ECS".' + example: tcp + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'In the OSI Model this would be the Network Layer. ipv4, ipv6, + ipsec, pim, etc + + The field value must be normalized to lowercase for querying. See the documentation + section "Implementing ECS".' + example: ipv4 + - name: observer + title: Observer + group: 2 + description: 'An observer is defined as a special network, security, or application + device used to detect, observe, or create network, security, or application-related + events and metrics. + + This could be a custom hardware appliance or a server that has been configured + to run special network, security, or application software. Examples include + firewalls, intrusion detection/prevention systems, network monitoring sensors, + web application firewalls, data loss prevention systems, and APM servers. The + observer.* fields shall be populated with details of the system, if any, that + detects, observes and/or creates a network, security, or application event or + metric. Message queues and ETL components used in processing events or metrics + are not considered observers in ECS.' + type: group + fields: + - name: geo.city_name + level: core + type: keyword + ignore_above: 1024 + description: City name. + example: Montreal + - name: geo.continent_name + level: core + type: keyword + ignore_above: 1024 + description: Name of the continent. + example: North America + - name: geo.country_iso_code + level: core + type: keyword + ignore_above: 1024 + description: Country ISO code. + example: CA + - name: geo.country_name + level: core + type: keyword + ignore_above: 1024 + description: Country name. + example: Canada + - name: geo.location + level: core + type: geo_point + description: Longitude and latitude. + example: '{ "lon": -73.614830, "lat": 45.505918 }' + - name: geo.name + level: extended + type: keyword + ignore_above: 1024 + description: 'User-defined description of a location, at the level of granularity + they care about. + + Could be the name of their data centers, the floor number, if this describes + a local physical entity, city names. + + Not typically used in automated geolocation.' + example: boston-dc + - name: geo.region_iso_code + level: core + type: keyword + ignore_above: 1024 + description: Region ISO code. + example: CA-QC + - name: geo.region_name + level: core + type: keyword + ignore_above: 1024 + description: Region name. + example: Quebec + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: Hostname of the observer. + - name: ip + level: core + type: ip + description: IP address of the observer. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: MAC address of the observer + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.full + level: extended + type: keyword + ignore_above: 1024 + description: Operating system name, including the version or code name. + example: Mac OS Mojave + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: serial_number + level: extended + type: keyword + ignore_above: 1024 + description: Observer serial number. + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'The type of the observer the data is coming from. + + There is no predefined list of observer types. Some examples are `forwarder`, + `firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`.' + example: firewall + - name: vendor + level: core + type: keyword + ignore_above: 1024 + description: observer vendor information. + - name: version + level: core + type: keyword + ignore_above: 1024 + description: Observer version. + - name: organization + title: Organization + group: 2 + description: 'The organization fields enrich data with information about the company + or entity the data is associated with. + + These fields help you arrange or filter data stored in an index by one or multiple + organizations.' + type: group + fields: + - name: id + level: extended + type: keyword + ignore_above: 1024 + description: Unique identifier for the organization. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Organization name. + - name: os + title: Operating System + group: 2 + description: The OS fields contain information about the operating system. + type: group + fields: + - name: family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: full + level: extended + type: keyword + ignore_above: 1024 + description: Operating system name, including the version or code name. + example: Mac OS Mojave + - name: kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Operating system name, without the version. + example: Mac OS X + - name: platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: process + title: Process + group: 2 + description: 'These fields contain information about a process. + + These fields can help you correlate metrics information with a process id/name + from a log message. The `process.pid` often stays in the metric itself and + is copied to the global field for correlation.' + type: group + fields: + - name: args + level: extended + type: keyword + ignore_above: 1024 + description: 'Array of process arguments. + + May be filtered to protect sensitive information.' + example: + - ssh + - -l + - user + - 10.0.0.16 + - name: executable + level: extended + type: keyword + ignore_above: 1024 + description: Absolute path to the process executable. + example: /usr/bin/ssh + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: 'Process name. + + Sometimes called program name or similar.' + example: ssh + - name: pid + level: core + type: long + description: Process id. + - name: ppid + level: extended + type: long + description: Process parent id. + - name: start + level: extended + type: date + description: The time the process started. + example: '2016-05-23T08:05:34.853Z' + - name: thread.id + level: extended + type: long + description: Thread ID. + example: 4242 + - name: title + level: extended + type: keyword + ignore_above: 1024 + description: 'Process title. + + The proctitle, some times the same as process name. Can also be different: + for example a browser setting its title to the web page currently opened.' + - name: working_directory + level: extended + type: keyword + ignore_above: 1024 + description: The working directory of the process. + example: /home/alice + - name: related + title: Related + group: 2 + description: 'This field set is meant to facilitate pivoting around a piece of + data. + + Some pieces of information can be seen in many places in an ECS event. To facilitate + searching for them, store an array of all seen values to their corresponding + field in `related.`. + + A concrete example is IP addresses, which can be under host, observer, source, + destination, client, server, and network.forwarded_ip. If you append all IPs + to `related.ip`, you can then search for a given IP trivially, no matter where + it appeared, by querying `related.ip:a.b.c.d`.' + type: group + fields: + - name: ip + level: extended + type: ip + description: All of the IPs seen on your event. + - name: server + title: Server + group: 2 + description: 'A Server is defined as the responder in a network connection for + events regarding sessions, connections, or bidirectional flow records. + + For TCP events, the server is the receiver of the initial SYN packet(s) of the + TCP connection. For other protocols, the server is generally the responder in + the network transaction. Some systems actually use the term "responder" to refer + the server in TCP connections. The server fields describe details about the + system acting as the server in the network event. Server fields are usually + populated in conjunction with client fields. Server fields are generally not + populated for packet-level events. + + Client / server representations can add semantic context to an exchange, which + is helpful to visualize the data in certain situations. If your context falls + in that category, you should still ensure that source and destination are filled + appropriately.' + type: group + fields: + - name: address + level: extended + type: keyword + ignore_above: 1024 + description: 'Some event server addresses are defined ambiguously. The event + will sometimes list an IP, a domain or a unix socket. You should always store + the raw address in the `.address` field. + + Then it should be duplicated to `.ip` or `.domain`, depending on which one + it is.' + - name: bytes + level: core + type: long + description: Bytes sent from the server to the client. + example: 184 + - name: domain + level: core + type: keyword + ignore_above: 1024 + description: Server domain. + - name: geo.city_name + level: core + type: keyword + ignore_above: 1024 + description: City name. + example: Montreal + - name: geo.continent_name + level: core + type: keyword + ignore_above: 1024 + description: Name of the continent. + example: North America + - name: geo.country_iso_code + level: core + type: keyword + ignore_above: 1024 + description: Country ISO code. + example: CA + - name: geo.country_name + level: core + type: keyword + ignore_above: 1024 + description: Country name. + example: Canada + - name: geo.location + level: core + type: geo_point + description: Longitude and latitude. + example: '{ "lon": -73.614830, "lat": 45.505918 }' + - name: geo.name + level: extended + type: keyword + ignore_above: 1024 + description: 'User-defined description of a location, at the level of granularity + they care about. + + Could be the name of their data centers, the floor number, if this describes + a local physical entity, city names. + + Not typically used in automated geolocation.' + example: boston-dc + - name: geo.region_iso_code + level: core + type: keyword + ignore_above: 1024 + description: Region ISO code. + example: CA-QC + - name: geo.region_name + level: core + type: keyword + ignore_above: 1024 + description: Region name. + example: Quebec + - name: ip + level: core + type: ip + description: 'IP address of the server. + + Can be one or multiple IPv4 or IPv6 addresses.' + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: MAC address of the server. + - name: packets + level: core + type: long + description: Packets sent from the server to the client. + example: 12 + - name: port + level: core + type: long + description: Port of the server. + - name: user.email + level: extended + type: keyword + ignore_above: 1024 + description: User email address. + - name: user.full_name + level: extended + type: keyword + ignore_above: 1024 + description: User's full name, if available. + example: Albert Einstein + - name: user.group.id + level: extended + type: keyword + ignore_above: 1024 + description: Unique identifier for the group on the system/platform. + - name: user.group.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the group. + - name: user.hash + level: extended + type: keyword + ignore_above: 1024 + description: 'Unique user hash to correlate information for a user in anonymized + form. + + Useful if `user.id` or `user.name` contain confidential information and cannot + be used.' + - name: user.id + level: core + type: keyword + ignore_above: 1024 + description: One or multiple unique identifiers of the user. + - name: user.name + level: core + type: keyword + ignore_above: 1024 + description: Short name or login of the user. + example: albert + - name: service + title: Service + group: 2 + description: 'The service fields describe the service for or from which the data + was collected. + + These fields help you find and correlate logs for a specific service and version.' + type: group + fields: + - name: ephemeral_id + level: extended + type: keyword + ignore_above: 1024 + description: 'Ephemeral identifier of this service (if one exists). + + This id normally changes across restarts, but `service.id` does not.' + example: 8a4f500f + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique identifier of the running service. + + This id should uniquely identify this service. This makes it possible to correlate + logs and metrics for one specific service. + + Example: If you are experiencing issues with one redis instance, you can filter + on that id to see metrics and logs for that single instance.' + example: d37e5ebfe0ae6c4972dbe9f0174a1637bb8247f6 + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the service data is collected from. + + The name of the service is normally user given. This allows if two instances + of the same service are running on the same machine they can be differentiated + by the `service.name`. + + Also it allows for distributed services that run on multiple hosts to correlate + the related instances based on the name. + + In the case of Elasticsearch the service.name could contain the cluster name. + For Beats the service.name is by default a copy of the `service.type` field + if no name is specified.' + example: elasticsearch-metrics + - name: state + level: core + type: keyword + ignore_above: 1024 + description: Current state of the service. + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'The type of the service data is collected from. + + The type can be used to group and correlate logs and metrics from one service + type. + + Example: If logs or metrics are collected from Elasticsearch, `service.type` + would be `elasticsearch`.' + example: elasticsearch + - name: version + level: core + type: keyword + ignore_above: 1024 + description: 'Version of the service the data was collected from. + + This allows to look at a data set only for a specific version of a service.' + example: 3.2.4 + - name: source + title: Source + group: 2 + description: 'Source fields describe details about the source of a packet/event. + + Source fields are usually populated in conjunction with destination fields.' + type: group + fields: + - name: address + level: extended + type: keyword + ignore_above: 1024 + description: 'Some event source addresses are defined ambiguously. The event + will sometimes list an IP, a domain or a unix socket. You should always store + the raw address in the `.address` field. + + Then it should be duplicated to `.ip` or `.domain`, depending on which one + it is.' + - name: bytes + level: core + type: long + description: Bytes sent from the source to the destination. + example: 184 + - name: domain + level: core + type: keyword + ignore_above: 1024 + description: Source domain. + - name: geo.city_name + level: core + type: keyword + ignore_above: 1024 + description: City name. + example: Montreal + - name: geo.continent_name + level: core + type: keyword + ignore_above: 1024 + description: Name of the continent. + example: North America + - name: geo.country_iso_code + level: core + type: keyword + ignore_above: 1024 + description: Country ISO code. + example: CA + - name: geo.country_name + level: core + type: keyword + ignore_above: 1024 + description: Country name. + example: Canada + - name: geo.location + level: core + type: geo_point + description: Longitude and latitude. + example: '{ "lon": -73.614830, "lat": 45.505918 }' + - name: geo.name + level: extended + type: keyword + ignore_above: 1024 + description: 'User-defined description of a location, at the level of granularity + they care about. + + Could be the name of their data centers, the floor number, if this describes + a local physical entity, city names. + + Not typically used in automated geolocation.' + example: boston-dc + - name: geo.region_iso_code + level: core + type: keyword + ignore_above: 1024 + description: Region ISO code. + example: CA-QC + - name: geo.region_name + level: core + type: keyword + ignore_above: 1024 + description: Region name. + example: Quebec + - name: ip + level: core + type: ip + description: 'IP address of the source. + + Can be one or multiple IPv4 or IPv6 addresses.' + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: MAC address of the source. + - name: packets + level: core + type: long + description: Packets sent from the source to the destination. + example: 12 + - name: port + level: core + type: long + description: Port of the source. + - name: user.email + level: extended + type: keyword + ignore_above: 1024 + description: User email address. + - name: user.full_name + level: extended + type: keyword + ignore_above: 1024 + description: User's full name, if available. + example: Albert Einstein + - name: user.group.id + level: extended + type: keyword + ignore_above: 1024 + description: Unique identifier for the group on the system/platform. + - name: user.group.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the group. + - name: user.hash + level: extended + type: keyword + ignore_above: 1024 + description: 'Unique user hash to correlate information for a user in anonymized + form. + + Useful if `user.id` or `user.name` contain confidential information and cannot + be used.' + - name: user.id + level: core + type: keyword + ignore_above: 1024 + description: One or multiple unique identifiers of the user. + - name: user.name + level: core + type: keyword + ignore_above: 1024 + description: Short name or login of the user. + example: albert + - name: url + title: URL + group: 2 + description: URL fields provide support for complete or partial URLs, and supports + the breaking down into scheme, domain, path, and so on. + type: group + fields: + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Domain of the url, such as "www.elastic.co". + + In some cases a URL may refer to an IP and/or port directly, without a domain + name. In this case, the IP address would go to the `domain` field.' + example: www.elastic.co + - name: fragment + level: extended + type: keyword + ignore_above: 1024 + description: 'Portion of the url after the `#`, such as "top". + + The `#` is not part of the fragment.' + - name: full + level: extended + type: keyword + ignore_above: 1024 + description: If full URLs are important to your use case, they should be stored + in `url.full`, whether this field is reconstructed or present in the event + source. + example: https://www.elastic.co:443/search?q=elasticsearch#top + - name: original + level: extended + type: keyword + ignore_above: 1024 + description: 'Unmodified original url as seen in the event source. + + Note that in network monitoring, the observed URL may be a full URL, whereas + in access logs, the URL is often just represented as a path. + + This field is meant to represent the URL as it was observed, complete or not.' + example: https://www.elastic.co:443/search?q=elasticsearch#top or /search?q=elasticsearch + - name: password + level: extended + type: keyword + ignore_above: 1024 + description: Password of the request. + - name: path + level: extended + type: keyword + ignore_above: 1024 + description: Path of the request, such as "/search". + - name: port + level: extended + type: long + description: Port of the request, such as 443. + example: 443 + - name: query + level: extended + type: keyword + ignore_above: 1024 + description: 'The query field describes the query string of the request, such + as "q=elasticsearch". + + The `?` is excluded from the query string. If a URL contains no `?`, there + is no query field. If there is a `?` but no query, the query field exists + with an empty string. The `exists` query can be used to differentiate between + the two cases.' + - name: scheme + level: extended + type: keyword + ignore_above: 1024 + description: 'Scheme of the request, such as "https". + + Note: The `:` is not part of the scheme.' + example: https + - name: username + level: extended + type: keyword + ignore_above: 1024 + description: Username of the request. + - name: user + title: User + group: 2 + description: 'The user fields describe information about the user that is relevant + to the event. + + Fields can have one entry or multiple entries. If a user has more than one id, + provide an array that includes all of them.' + type: group + fields: + - name: email + level: extended + type: keyword + ignore_above: 1024 + description: User email address. + - name: full_name + level: extended + type: keyword + ignore_above: 1024 + description: User's full name, if available. + example: Albert Einstein + - name: group.id + level: extended + type: keyword + ignore_above: 1024 + description: Unique identifier for the group on the system/platform. + - name: group.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the group. + - name: hash + level: extended + type: keyword + ignore_above: 1024 + description: 'Unique user hash to correlate information for a user in anonymized + form. + + Useful if `user.id` or `user.name` contain confidential information and cannot + be used.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: One or multiple unique identifiers of the user. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: Short name or login of the user. + example: albert + - name: user_agent + title: User agent + group: 2 + description: 'The user_agent fields normally come from a browser request. + + They often show up in web service logs coming from the parsed user agent string.' + type: group + fields: + - name: device.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the device. + example: iPhone + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the user agent. + example: Safari + - name: original + level: extended + type: keyword + ignore_above: 1024 + description: Unparsed version of the user_agent. + example: Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15 + (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1 + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.full + level: extended + type: keyword + ignore_above: 1024 + description: Operating system name, including the version or code name. + example: Mac OS Mojave + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: version + level: extended + type: keyword + ignore_above: 1024 + description: Version of the user agent. + example: 12.0 diff --git a/generated/csv/fields.csv b/generated/csv/fields.csv index 5a1c13f700..5385796b26 100644 --- a/generated/csv/fields.csv +++ b/generated/csv/fields.csv @@ -25,6 +25,8 @@ client.packets,long,core,12,1.0.0 client.port,long,core,,1.0.0 client.user.email,keyword,extended,,1.0.0 client.user.full_name,keyword,extended,Albert Einstein,1.0.0 +client.user.group.id,keyword,extended,,1.0.0 +client.user.group.name,keyword,extended,,1.0.0 client.user.hash,keyword,extended,,1.0.0 client.user.id,keyword,core,,1.0.0 client.user.name,keyword,core,albert,1.0.0 @@ -58,6 +60,8 @@ destination.packets,long,core,12,1.0.0 destination.port,long,core,,1.0.0 destination.user.email,keyword,extended,,1.0.0 destination.user.full_name,keyword,extended,Albert Einstein,1.0.0 +destination.user.group.id,keyword,extended,,1.0.0 +destination.user.group.name,keyword,extended,,1.0.0 destination.user.hash,keyword,extended,,1.0.0 destination.user.id,keyword,core,,1.0.0 destination.user.name,keyword,core,albert,1.0.0 @@ -130,6 +134,8 @@ host.os.version,keyword,extended,10.14.1,1.0.0 host.type,keyword,core,,1.0.0 host.user.email,keyword,extended,,1.0.0 host.user.full_name,keyword,extended,Albert Einstein,1.0.0 +host.user.group.id,keyword,extended,,1.0.0 +host.user.group.name,keyword,extended,,1.0.0 host.user.hash,keyword,extended,,1.0.0 host.user.id,keyword,core,,1.0.0 host.user.name,keyword,core,albert,1.0.0 @@ -212,6 +218,8 @@ server.packets,long,core,12,1.0.0 server.port,long,core,,1.0.0 server.user.email,keyword,extended,,1.0.0 server.user.full_name,keyword,extended,Albert Einstein,1.0.0 +server.user.group.id,keyword,extended,,1.0.0 +server.user.group.name,keyword,extended,,1.0.0 server.user.hash,keyword,extended,,1.0.0 server.user.id,keyword,core,,1.0.0 server.user.name,keyword,core,albert,1.0.0 @@ -238,6 +246,8 @@ source.packets,long,core,12,1.0.0 source.port,long,core,,1.0.0 source.user.email,keyword,extended,,1.0.0 source.user.full_name,keyword,extended,Albert Einstein,1.0.0 +source.user.group.id,keyword,extended,,1.0.0 +source.user.group.name,keyword,extended,,1.0.0 source.user.hash,keyword,extended,,1.0.0 source.user.id,keyword,core,,1.0.0 source.user.name,keyword,core,albert,1.0.0 diff --git a/generated/ecs/fields_flat.yml b/generated/ecs/fields_flat.yml index d82ee97028..905d06d9a5 100644 --- a/generated/ecs/fields_flat.yml +++ b/generated/ecs/fields_flat.yml @@ -239,6 +239,24 @@ client.user.full_name: original_fieldset: user short: User's full name, if available. type: keyword +client.user.group.id: + description: Unique identifier for the group on the system/platform. + flat_name: client.user.group.id + ignore_above: 1024 + level: extended + name: id + original_fieldset: user + short: Unique identifier for the group on the system/platform. + type: keyword +client.user.group.name: + description: Name of the group. + flat_name: client.user.group.name + ignore_above: 1024 + level: extended + name: name + original_fieldset: user + short: Name of the group. + type: keyword client.user.hash: description: 'Unique user hash to correlate information for a user in anonymized form. @@ -551,6 +569,24 @@ destination.user.full_name: original_fieldset: user short: User's full name, if available. type: keyword +destination.user.group.id: + description: Unique identifier for the group on the system/platform. + flat_name: destination.user.group.id + ignore_above: 1024 + level: extended + name: id + original_fieldset: user + short: Unique identifier for the group on the system/platform. + type: keyword +destination.user.group.name: + description: Name of the group. + flat_name: destination.user.group.name + ignore_above: 1024 + level: extended + name: name + original_fieldset: user + short: Name of the group. + type: keyword destination.user.hash: description: 'Unique user hash to correlate information for a user in anonymized form. @@ -1275,6 +1311,24 @@ host.user.full_name: original_fieldset: user short: User's full name, if available. type: keyword +host.user.group.id: + description: Unique identifier for the group on the system/platform. + flat_name: host.user.group.id + ignore_above: 1024 + level: extended + name: id + original_fieldset: user + short: Unique identifier for the group on the system/platform. + type: keyword +host.user.group.name: + description: Name of the group. + flat_name: host.user.group.name + ignore_above: 1024 + level: extended + name: name + original_fieldset: user + short: Name of the group. + type: keyword host.user.hash: description: 'Unique user hash to correlate information for a user in anonymized form. @@ -2118,6 +2172,24 @@ server.user.full_name: original_fieldset: user short: User's full name, if available. type: keyword +server.user.group.id: + description: Unique identifier for the group on the system/platform. + flat_name: server.user.group.id + ignore_above: 1024 + level: extended + name: id + original_fieldset: user + short: Unique identifier for the group on the system/platform. + type: keyword +server.user.group.name: + description: Name of the group. + flat_name: server.user.group.name + ignore_above: 1024 + level: extended + name: name + original_fieldset: user + short: Name of the group. + type: keyword server.user.hash: description: 'Unique user hash to correlate information for a user in anonymized form. @@ -2395,6 +2467,24 @@ source.user.full_name: original_fieldset: user short: User's full name, if available. type: keyword +source.user.group.id: + description: Unique identifier for the group on the system/platform. + flat_name: source.user.group.id + ignore_above: 1024 + level: extended + name: id + original_fieldset: user + short: Unique identifier for the group on the system/platform. + type: keyword +source.user.group.name: + description: Name of the group. + flat_name: source.user.group.name + ignore_above: 1024 + level: extended + name: name + original_fieldset: user + short: Name of the group. + type: keyword source.user.hash: description: 'Unique user hash to correlate information for a user in anonymized form. diff --git a/generated/ecs/fields_nested.yml b/generated/ecs/fields_nested.yml index ff0c0add8e..92331173d5 100644 --- a/generated/ecs/fields_nested.yml +++ b/generated/ecs/fields_nested.yml @@ -190,6 +190,91 @@ client: name: domain short: Client domain. type: keyword + geo.city_name: + description: City name. + example: Montreal + flat_name: client.geo.city_name + ignore_above: 1024 + level: core + name: city_name + original_fieldset: geo + short: City name. + type: keyword + geo.continent_name: + description: Name of the continent. + example: North America + flat_name: client.geo.continent_name + ignore_above: 1024 + level: core + name: continent_name + original_fieldset: geo + short: Name of the continent. + type: keyword + geo.country_iso_code: + description: Country ISO code. + example: CA + flat_name: client.geo.country_iso_code + ignore_above: 1024 + level: core + name: country_iso_code + original_fieldset: geo + short: Country ISO code. + type: keyword + geo.country_name: + description: Country name. + example: Canada + flat_name: client.geo.country_name + ignore_above: 1024 + level: core + name: country_name + original_fieldset: geo + short: Country name. + type: keyword + geo.location: + description: Longitude and latitude. + example: '{ "lon": -73.614830, "lat": 45.505918 }' + flat_name: client.geo.location + level: core + name: location + original_fieldset: geo + short: Longitude and latitude. + type: geo_point + geo.name: + description: 'User-defined description of a location, at the level of granularity + they care about. + + Could be the name of their data centers, the floor number, if this describes + a local physical entity, city names. + + Not typically used in automated geolocation.' + example: boston-dc + flat_name: client.geo.name + ignore_above: 1024 + level: extended + name: name + original_fieldset: geo + short: User-defined description of a location. + type: keyword + geo.region_iso_code: + description: Region ISO code. + example: CA-QC + flat_name: client.geo.region_iso_code + ignore_above: 1024 + level: core + name: region_iso_code + original_fieldset: geo + short: Region ISO code. + type: keyword + geo.region_name: + description: Region name. + example: Quebec + flat_name: client.geo.region_name + ignore_above: 1024 + level: core + name: region_name + original_fieldset: geo + short: Region name. + type: keyword ip: description: 'IP address of the client. @@ -222,6 +307,75 @@ client: name: port short: Port of the client. type: long + user.email: + description: User email address. + flat_name: client.user.email + ignore_above: 1024 + level: extended + name: email + original_fieldset: user + short: User email address. + type: keyword + user.full_name: + description: User's full name, if available. + example: Albert Einstein + flat_name: client.user.full_name + ignore_above: 1024 + level: extended + name: full_name + original_fieldset: user + short: User's full name, if available. + type: keyword + user.group.id: + description: Unique identifier for the group on the system/platform. + flat_name: client.user.group.id + ignore_above: 1024 + level: extended + name: id + original_fieldset: user + short: Unique identifier for the group on the system/platform. + type: keyword + user.group.name: + description: Name of the group. + flat_name: client.user.group.name + ignore_above: 1024 + level: extended + name: name + original_fieldset: user + short: Name of the group. + type: keyword + user.hash: + description: 'Unique user hash to correlate information for a user in anonymized + form. + + Useful if `user.id` or `user.name` contain confidential information and cannot + be used.' + flat_name: client.user.hash + ignore_above: 1024 + level: extended + name: hash + original_fieldset: user + short: Unique user hash to correlate information for a user in anonymized form. + type: keyword + user.id: + description: One or multiple unique identifiers of the user. + flat_name: client.user.id + ignore_above: 1024 + level: core + name: id + original_fieldset: user + short: One or multiple unique identifiers of the user. + type: keyword + user.name: + description: Short name or login of the user. + example: albert + flat_name: client.user.name + ignore_above: 1024 + level: core + name: name + original_fieldset: user + short: Short name or login of the user. + type: keyword group: 2 name: client prefix: client. @@ -404,6 +558,91 @@ destination: name: domain short: Destination domain. type: keyword + geo.city_name: + description: City name. + example: Montreal + flat_name: destination.geo.city_name + ignore_above: 1024 + level: core + name: city_name + original_fieldset: geo + short: City name. + type: keyword + geo.continent_name: + description: Name of the continent. + example: North America + flat_name: destination.geo.continent_name + ignore_above: 1024 + level: core + name: continent_name + original_fieldset: geo + short: Name of the continent. + type: keyword + geo.country_iso_code: + description: Country ISO code. + example: CA + flat_name: destination.geo.country_iso_code + ignore_above: 1024 + level: core + name: country_iso_code + original_fieldset: geo + short: Country ISO code. + type: keyword + geo.country_name: + description: Country name. + example: Canada + flat_name: destination.geo.country_name + ignore_above: 1024 + level: core + name: country_name + original_fieldset: geo + short: Country name. + type: keyword + geo.location: + description: Longitude and latitude. + example: '{ "lon": -73.614830, "lat": 45.505918 }' + flat_name: destination.geo.location + level: core + name: location + original_fieldset: geo + short: Longitude and latitude. + type: geo_point + geo.name: + description: 'User-defined description of a location, at the level of granularity + they care about. + + Could be the name of their data centers, the floor number, if this describes + a local physical entity, city names. + + Not typically used in automated geolocation.' + example: boston-dc + flat_name: destination.geo.name + ignore_above: 1024 + level: extended + name: name + original_fieldset: geo + short: User-defined description of a location. + type: keyword + geo.region_iso_code: + description: Region ISO code. + example: CA-QC + flat_name: destination.geo.region_iso_code + ignore_above: 1024 + level: core + name: region_iso_code + original_fieldset: geo + short: Region ISO code. + type: keyword + geo.region_name: + description: Region name. + example: Quebec + flat_name: destination.geo.region_name + ignore_above: 1024 + level: core + name: region_name + original_fieldset: geo + short: Region name. + type: keyword ip: description: 'IP address of the destination. @@ -436,6 +675,75 @@ destination: name: port short: Port of the destination. type: long + user.email: + description: User email address. + flat_name: destination.user.email + ignore_above: 1024 + level: extended + name: email + original_fieldset: user + short: User email address. + type: keyword + user.full_name: + description: User's full name, if available. + example: Albert Einstein + flat_name: destination.user.full_name + ignore_above: 1024 + level: extended + name: full_name + original_fieldset: user + short: User's full name, if available. + type: keyword + user.group.id: + description: Unique identifier for the group on the system/platform. + flat_name: destination.user.group.id + ignore_above: 1024 + level: extended + name: id + original_fieldset: user + short: Unique identifier for the group on the system/platform. + type: keyword + user.group.name: + description: Name of the group. + flat_name: destination.user.group.name + ignore_above: 1024 + level: extended + name: name + original_fieldset: user + short: Name of the group. + type: keyword + user.hash: + description: 'Unique user hash to correlate information for a user in anonymized + form. + + Useful if `user.id` or `user.name` contain confidential information and cannot + be used.' + flat_name: destination.user.hash + ignore_above: 1024 + level: extended + name: hash + original_fieldset: user + short: Unique user hash to correlate information for a user in anonymized form. + type: keyword + user.id: + description: One or multiple unique identifiers of the user. + flat_name: destination.user.id + ignore_above: 1024 + level: core + name: id + original_fieldset: user + short: One or multiple unique identifiers of the user. + type: keyword + user.name: + description: Short name or login of the user. + example: albert + flat_name: destination.user.name + ignore_above: 1024 + level: core + name: name + original_fieldset: user + short: Short name or login of the user. + type: keyword group: 2 name: destination prefix: destination. @@ -1008,42 +1316,127 @@ host: name: architecture short: Operating system architecture. type: keyword - hostname: - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - flat_name: host.hostname + geo.city_name: + description: City name. + example: Montreal + flat_name: host.geo.city_name ignore_above: 1024 level: core - name: hostname - short: Hostname of the host. + name: city_name + original_fieldset: geo + short: City name. type: keyword - id: - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - flat_name: host.id + geo.continent_name: + description: Name of the continent. + example: North America + flat_name: host.geo.continent_name ignore_above: 1024 level: core - name: id - short: Unique host id. + name: continent_name + original_fieldset: geo + short: Name of the continent. type: keyword - ip: - description: Host ip address. - flat_name: host.ip - level: core - name: ip - short: Host ip address. - type: ip - mac: - description: Host mac address. - flat_name: host.mac + geo.country_iso_code: + description: Country ISO code. + example: CA + flat_name: host.geo.country_iso_code ignore_above: 1024 level: core - name: mac - short: Host mac address. + name: country_iso_code + original_fieldset: geo + short: Country ISO code. + type: keyword + geo.country_name: + description: Country name. + example: Canada + flat_name: host.geo.country_name + ignore_above: 1024 + level: core + name: country_name + original_fieldset: geo + short: Country name. + type: keyword + geo.location: + description: Longitude and latitude. + example: '{ "lon": -73.614830, "lat": 45.505918 }' + flat_name: host.geo.location + level: core + name: location + original_fieldset: geo + short: Longitude and latitude. + type: geo_point + geo.name: + description: 'User-defined description of a location, at the level of granularity + they care about. + + Could be the name of their data centers, the floor number, if this describes + a local physical entity, city names. + + Not typically used in automated geolocation.' + example: boston-dc + flat_name: host.geo.name + ignore_above: 1024 + level: extended + name: name + original_fieldset: geo + short: User-defined description of a location. + type: keyword + geo.region_iso_code: + description: Region ISO code. + example: CA-QC + flat_name: host.geo.region_iso_code + ignore_above: 1024 + level: core + name: region_iso_code + original_fieldset: geo + short: Region ISO code. + type: keyword + geo.region_name: + description: Region name. + example: Quebec + flat_name: host.geo.region_name + ignore_above: 1024 + level: core + name: region_name + original_fieldset: geo + short: Region name. + type: keyword + hostname: + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + flat_name: host.hostname + ignore_above: 1024 + level: core + name: hostname + short: Hostname of the host. + type: keyword + id: + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + flat_name: host.id + ignore_above: 1024 + level: core + name: id + short: Unique host id. + type: keyword + ip: + description: Host ip address. + flat_name: host.ip + level: core + name: ip + short: Host ip address. + type: ip + mac: + description: Host mac address. + flat_name: host.mac + ignore_above: 1024 + level: core + name: mac + short: Host mac address. type: keyword name: description: 'Name of the host. @@ -1057,6 +1450,66 @@ host: name: name short: Name of the host. type: keyword + os.family: + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + flat_name: host.os.family + ignore_above: 1024 + level: extended + name: family + original_fieldset: os + short: OS family (such as redhat, debian, freebsd, windows). + type: keyword + os.full: + description: Operating system name, including the version or code name. + example: Mac OS Mojave + flat_name: host.os.full + ignore_above: 1024 + level: extended + name: full + original_fieldset: os + short: Operating system name, including the version or code name. + type: keyword + os.kernel: + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + flat_name: host.os.kernel + ignore_above: 1024 + level: extended + name: kernel + original_fieldset: os + short: Operating system kernel version as a raw string. + type: keyword + os.name: + description: Operating system name, without the version. + example: Mac OS X + flat_name: host.os.name + ignore_above: 1024 + level: extended + name: name + original_fieldset: os + short: Operating system name, without the version. + type: keyword + os.platform: + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + flat_name: host.os.platform + ignore_above: 1024 + level: extended + name: platform + original_fieldset: os + short: Operating system platform (such centos, ubuntu, windows). + type: keyword + os.version: + description: Operating system version as a raw string. + example: 10.14.1 + flat_name: host.os.version + ignore_above: 1024 + level: extended + name: version + original_fieldset: os + short: Operating system version as a raw string. + type: keyword type: description: 'Type of host. @@ -1069,6 +1522,75 @@ host: name: type short: Type of host. type: keyword + user.email: + description: User email address. + flat_name: host.user.email + ignore_above: 1024 + level: extended + name: email + original_fieldset: user + short: User email address. + type: keyword + user.full_name: + description: User's full name, if available. + example: Albert Einstein + flat_name: host.user.full_name + ignore_above: 1024 + level: extended + name: full_name + original_fieldset: user + short: User's full name, if available. + type: keyword + user.group.id: + description: Unique identifier for the group on the system/platform. + flat_name: host.user.group.id + ignore_above: 1024 + level: extended + name: id + original_fieldset: user + short: Unique identifier for the group on the system/platform. + type: keyword + user.group.name: + description: Name of the group. + flat_name: host.user.group.name + ignore_above: 1024 + level: extended + name: name + original_fieldset: user + short: Name of the group. + type: keyword + user.hash: + description: 'Unique user hash to correlate information for a user in anonymized + form. + + Useful if `user.id` or `user.name` contain confidential information and cannot + be used.' + flat_name: host.user.hash + ignore_above: 1024 + level: extended + name: hash + original_fieldset: user + short: Unique user hash to correlate information for a user in anonymized form. + type: keyword + user.id: + description: One or multiple unique identifiers of the user. + flat_name: host.user.id + ignore_above: 1024 + level: core + name: id + original_fieldset: user + short: One or multiple unique identifiers of the user. + type: keyword + user.name: + description: Short name or login of the user. + example: albert + flat_name: host.user.name + ignore_above: 1024 + level: core + name: name + original_fieldset: user + short: Short name or login of the user. + type: keyword group: 2 name: host prefix: host. @@ -1371,6 +1893,91 @@ observer: and ETL components used in processing events or metrics are not considered observers in ECS.' fields: + geo.city_name: + description: City name. + example: Montreal + flat_name: observer.geo.city_name + ignore_above: 1024 + level: core + name: city_name + original_fieldset: geo + short: City name. + type: keyword + geo.continent_name: + description: Name of the continent. + example: North America + flat_name: observer.geo.continent_name + ignore_above: 1024 + level: core + name: continent_name + original_fieldset: geo + short: Name of the continent. + type: keyword + geo.country_iso_code: + description: Country ISO code. + example: CA + flat_name: observer.geo.country_iso_code + ignore_above: 1024 + level: core + name: country_iso_code + original_fieldset: geo + short: Country ISO code. + type: keyword + geo.country_name: + description: Country name. + example: Canada + flat_name: observer.geo.country_name + ignore_above: 1024 + level: core + name: country_name + original_fieldset: geo + short: Country name. + type: keyword + geo.location: + description: Longitude and latitude. + example: '{ "lon": -73.614830, "lat": 45.505918 }' + flat_name: observer.geo.location + level: core + name: location + original_fieldset: geo + short: Longitude and latitude. + type: geo_point + geo.name: + description: 'User-defined description of a location, at the level of granularity + they care about. + + Could be the name of their data centers, the floor number, if this describes + a local physical entity, city names. + + Not typically used in automated geolocation.' + example: boston-dc + flat_name: observer.geo.name + ignore_above: 1024 + level: extended + name: name + original_fieldset: geo + short: User-defined description of a location. + type: keyword + geo.region_iso_code: + description: Region ISO code. + example: CA-QC + flat_name: observer.geo.region_iso_code + ignore_above: 1024 + level: core + name: region_iso_code + original_fieldset: geo + short: Region ISO code. + type: keyword + geo.region_name: + description: Region name. + example: Quebec + flat_name: observer.geo.region_name + ignore_above: 1024 + level: core + name: region_name + original_fieldset: geo + short: Region name. + type: keyword hostname: description: Hostname of the observer. flat_name: observer.hostname @@ -1394,6 +2001,66 @@ observer: name: mac short: MAC address of the observer type: keyword + os.family: + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + flat_name: observer.os.family + ignore_above: 1024 + level: extended + name: family + original_fieldset: os + short: OS family (such as redhat, debian, freebsd, windows). + type: keyword + os.full: + description: Operating system name, including the version or code name. + example: Mac OS Mojave + flat_name: observer.os.full + ignore_above: 1024 + level: extended + name: full + original_fieldset: os + short: Operating system name, including the version or code name. + type: keyword + os.kernel: + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + flat_name: observer.os.kernel + ignore_above: 1024 + level: extended + name: kernel + original_fieldset: os + short: Operating system kernel version as a raw string. + type: keyword + os.name: + description: Operating system name, without the version. + example: Mac OS X + flat_name: observer.os.name + ignore_above: 1024 + level: extended + name: name + original_fieldset: os + short: Operating system name, without the version. + type: keyword + os.platform: + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + flat_name: observer.os.platform + ignore_above: 1024 + level: extended + name: platform + original_fieldset: os + short: Operating system platform (such centos, ubuntu, windows). + type: keyword + os.version: + description: Operating system version as a raw string. + example: 10.14.1 + flat_name: observer.os.version + ignore_above: 1024 + level: extended + name: version + original_fieldset: os + short: Operating system version as a raw string. + type: keyword serial_number: description: Observer serial number. flat_name: observer.serial_number @@ -1670,39 +2337,124 @@ server: in conjunction with client fields. Server fields are generally not populated for packet-level events. - Client / server representations can add semantic context to an exchange, which - is helpful to visualize the data in certain situations. If your context falls - in that category, you should still ensure that source and destination are filled - appropriately.' - fields: - address: - description: 'Some event server addresses are defined ambiguously. The event - will sometimes list an IP, a domain or a unix socket. You should always store - the raw address in the `.address` field. + Client / server representations can add semantic context to an exchange, which + is helpful to visualize the data in certain situations. If your context falls + in that category, you should still ensure that source and destination are filled + appropriately.' + fields: + address: + description: 'Some event server addresses are defined ambiguously. The event + will sometimes list an IP, a domain or a unix socket. You should always store + the raw address in the `.address` field. + + Then it should be duplicated to `.ip` or `.domain`, depending on which one + it is.' + flat_name: server.address + ignore_above: 1024 + level: extended + name: address + short: Server network address. + type: keyword + bytes: + description: Bytes sent from the server to the client. + example: 184 + flat_name: server.bytes + level: core + name: bytes + short: Bytes sent from the server to the client. + type: long + domain: + description: Server domain. + flat_name: server.domain + ignore_above: 1024 + level: core + name: domain + short: Server domain. + type: keyword + geo.city_name: + description: City name. + example: Montreal + flat_name: server.geo.city_name + ignore_above: 1024 + level: core + name: city_name + original_fieldset: geo + short: City name. + type: keyword + geo.continent_name: + description: Name of the continent. + example: North America + flat_name: server.geo.continent_name + ignore_above: 1024 + level: core + name: continent_name + original_fieldset: geo + short: Name of the continent. + type: keyword + geo.country_iso_code: + description: Country ISO code. + example: CA + flat_name: server.geo.country_iso_code + ignore_above: 1024 + level: core + name: country_iso_code + original_fieldset: geo + short: Country ISO code. + type: keyword + geo.country_name: + description: Country name. + example: Canada + flat_name: server.geo.country_name + ignore_above: 1024 + level: core + name: country_name + original_fieldset: geo + short: Country name. + type: keyword + geo.location: + description: Longitude and latitude. + example: '{ "lon": -73.614830, "lat": 45.505918 }' + flat_name: server.geo.location + level: core + name: location + original_fieldset: geo + short: Longitude and latitude. + type: geo_point + geo.name: + description: 'User-defined description of a location, at the level of granularity + they care about. + + Could be the name of their data centers, the floor number, if this describes + a local physical entity, city names. - Then it should be duplicated to `.ip` or `.domain`, depending on which one - it is.' - flat_name: server.address + Not typically used in automated geolocation.' + example: boston-dc + flat_name: server.geo.name ignore_above: 1024 level: extended - name: address - short: Server network address. + name: name + original_fieldset: geo + short: User-defined description of a location. type: keyword - bytes: - description: Bytes sent from the server to the client. - example: 184 - flat_name: server.bytes + geo.region_iso_code: + description: Region ISO code. + example: CA-QC + flat_name: server.geo.region_iso_code + ignore_above: 1024 level: core - name: bytes - short: Bytes sent from the server to the client. - type: long - domain: - description: Server domain. - flat_name: server.domain + name: region_iso_code + original_fieldset: geo + short: Region ISO code. + type: keyword + geo.region_name: + description: Region name. + example: Quebec + flat_name: server.geo.region_name ignore_above: 1024 level: core - name: domain - short: Server domain. + name: region_name + original_fieldset: geo + short: Region name. type: keyword ip: description: 'IP address of the server. @@ -1736,6 +2488,75 @@ server: name: port short: Port of the server. type: long + user.email: + description: User email address. + flat_name: server.user.email + ignore_above: 1024 + level: extended + name: email + original_fieldset: user + short: User email address. + type: keyword + user.full_name: + description: User's full name, if available. + example: Albert Einstein + flat_name: server.user.full_name + ignore_above: 1024 + level: extended + name: full_name + original_fieldset: user + short: User's full name, if available. + type: keyword + user.group.id: + description: Unique identifier for the group on the system/platform. + flat_name: server.user.group.id + ignore_above: 1024 + level: extended + name: id + original_fieldset: user + short: Unique identifier for the group on the system/platform. + type: keyword + user.group.name: + description: Name of the group. + flat_name: server.user.group.name + ignore_above: 1024 + level: extended + name: name + original_fieldset: user + short: Name of the group. + type: keyword + user.hash: + description: 'Unique user hash to correlate information for a user in anonymized + form. + + Useful if `user.id` or `user.name` contain confidential information and cannot + be used.' + flat_name: server.user.hash + ignore_above: 1024 + level: extended + name: hash + original_fieldset: user + short: Unique user hash to correlate information for a user in anonymized form. + type: keyword + user.id: + description: One or multiple unique identifiers of the user. + flat_name: server.user.id + ignore_above: 1024 + level: core + name: id + original_fieldset: user + short: One or multiple unique identifiers of the user. + type: keyword + user.name: + description: Short name or login of the user. + example: albert + flat_name: server.user.name + ignore_above: 1024 + level: core + name: name + original_fieldset: user + short: Short name or login of the user. + type: keyword group: 2 name: server prefix: server. @@ -1868,6 +2689,91 @@ source: name: domain short: Source domain. type: keyword + geo.city_name: + description: City name. + example: Montreal + flat_name: source.geo.city_name + ignore_above: 1024 + level: core + name: city_name + original_fieldset: geo + short: City name. + type: keyword + geo.continent_name: + description: Name of the continent. + example: North America + flat_name: source.geo.continent_name + ignore_above: 1024 + level: core + name: continent_name + original_fieldset: geo + short: Name of the continent. + type: keyword + geo.country_iso_code: + description: Country ISO code. + example: CA + flat_name: source.geo.country_iso_code + ignore_above: 1024 + level: core + name: country_iso_code + original_fieldset: geo + short: Country ISO code. + type: keyword + geo.country_name: + description: Country name. + example: Canada + flat_name: source.geo.country_name + ignore_above: 1024 + level: core + name: country_name + original_fieldset: geo + short: Country name. + type: keyword + geo.location: + description: Longitude and latitude. + example: '{ "lon": -73.614830, "lat": 45.505918 }' + flat_name: source.geo.location + level: core + name: location + original_fieldset: geo + short: Longitude and latitude. + type: geo_point + geo.name: + description: 'User-defined description of a location, at the level of granularity + they care about. + + Could be the name of their data centers, the floor number, if this describes + a local physical entity, city names. + + Not typically used in automated geolocation.' + example: boston-dc + flat_name: source.geo.name + ignore_above: 1024 + level: extended + name: name + original_fieldset: geo + short: User-defined description of a location. + type: keyword + geo.region_iso_code: + description: Region ISO code. + example: CA-QC + flat_name: source.geo.region_iso_code + ignore_above: 1024 + level: core + name: region_iso_code + original_fieldset: geo + short: Region ISO code. + type: keyword + geo.region_name: + description: Region name. + example: Quebec + flat_name: source.geo.region_name + ignore_above: 1024 + level: core + name: region_name + original_fieldset: geo + short: Region name. + type: keyword ip: description: 'IP address of the source. @@ -1900,6 +2806,75 @@ source: name: port short: Port of the source. type: long + user.email: + description: User email address. + flat_name: source.user.email + ignore_above: 1024 + level: extended + name: email + original_fieldset: user + short: User email address. + type: keyword + user.full_name: + description: User's full name, if available. + example: Albert Einstein + flat_name: source.user.full_name + ignore_above: 1024 + level: extended + name: full_name + original_fieldset: user + short: User's full name, if available. + type: keyword + user.group.id: + description: Unique identifier for the group on the system/platform. + flat_name: source.user.group.id + ignore_above: 1024 + level: extended + name: id + original_fieldset: user + short: Unique identifier for the group on the system/platform. + type: keyword + user.group.name: + description: Name of the group. + flat_name: source.user.group.name + ignore_above: 1024 + level: extended + name: name + original_fieldset: user + short: Name of the group. + type: keyword + user.hash: + description: 'Unique user hash to correlate information for a user in anonymized + form. + + Useful if `user.id` or `user.name` contain confidential information and cannot + be used.' + flat_name: source.user.hash + ignore_above: 1024 + level: extended + name: hash + original_fieldset: user + short: Unique user hash to correlate information for a user in anonymized form. + type: keyword + user.id: + description: One or multiple unique identifiers of the user. + flat_name: source.user.id + ignore_above: 1024 + level: core + name: id + original_fieldset: user + short: One or multiple unique identifiers of the user. + type: keyword + user.name: + description: Short name or login of the user. + example: albert + flat_name: source.user.name + ignore_above: 1024 + level: core + name: name + original_fieldset: user + short: Short name or login of the user. + type: keyword group: 2 name: source prefix: source. @@ -2044,6 +3019,24 @@ user: name: full_name short: User's full name, if available. type: keyword + group.id: + description: Unique identifier for the group on the system/platform. + flat_name: user.group.id + ignore_above: 1024 + level: extended + name: id + original_fieldset: group + short: Unique identifier for the group on the system/platform. + type: keyword + group.name: + description: Name of the group. + flat_name: user.group.name + ignore_above: 1024 + level: extended + name: name + original_fieldset: group + short: Name of the group. + type: keyword hash: description: 'Unique user hash to correlate information for a user in anonymized form. @@ -2120,6 +3113,66 @@ user_agent: name: original short: Unparsed version of the user_agent. type: keyword + os.family: + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + flat_name: user_agent.os.family + ignore_above: 1024 + level: extended + name: family + original_fieldset: os + short: OS family (such as redhat, debian, freebsd, windows). + type: keyword + os.full: + description: Operating system name, including the version or code name. + example: Mac OS Mojave + flat_name: user_agent.os.full + ignore_above: 1024 + level: extended + name: full + original_fieldset: os + short: Operating system name, including the version or code name. + type: keyword + os.kernel: + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + flat_name: user_agent.os.kernel + ignore_above: 1024 + level: extended + name: kernel + original_fieldset: os + short: Operating system kernel version as a raw string. + type: keyword + os.name: + description: Operating system name, without the version. + example: Mac OS X + flat_name: user_agent.os.name + ignore_above: 1024 + level: extended + name: name + original_fieldset: os + short: Operating system name, without the version. + type: keyword + os.platform: + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + flat_name: user_agent.os.platform + ignore_above: 1024 + level: extended + name: platform + original_fieldset: os + short: Operating system platform (such centos, ubuntu, windows). + type: keyword + os.version: + description: Operating system version as a raw string. + example: 10.14.1 + flat_name: user_agent.os.version + ignore_above: 1024 + level: extended + name: version + original_fieldset: os + short: Operating system version as a raw string. + type: keyword version: description: Version of the user agent. example: 12.0 diff --git a/generated/elasticsearch/6/template.json b/generated/elasticsearch/6/template.json index 5455ba61a7..b665b57c69 100644 --- a/generated/elasticsearch/6/template.json +++ b/generated/elasticsearch/6/template.json @@ -118,6 +118,18 @@ "ignore_above": 1024, "type": "keyword" }, + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "hash": { "ignore_above": 1024, "type": "keyword" @@ -280,6 +292,18 @@ "ignore_above": 1024, "type": "keyword" }, + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "hash": { "ignore_above": 1024, "type": "keyword" @@ -597,6 +621,18 @@ "ignore_above": 1024, "type": "keyword" }, + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "hash": { "ignore_above": 1024, "type": "keyword" @@ -986,6 +1022,18 @@ "ignore_above": 1024, "type": "keyword" }, + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "hash": { "ignore_above": 1024, "type": "keyword" @@ -1101,6 +1149,18 @@ "ignore_above": 1024, "type": "keyword" }, + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "hash": { "ignore_above": 1024, "type": "keyword" diff --git a/generated/elasticsearch/7/template.json b/generated/elasticsearch/7/template.json index 6a0c1d32e4..e40bacb977 100644 --- a/generated/elasticsearch/7/template.json +++ b/generated/elasticsearch/7/template.json @@ -117,6 +117,18 @@ "ignore_above": 1024, "type": "keyword" }, + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "hash": { "ignore_above": 1024, "type": "keyword" @@ -279,6 +291,18 @@ "ignore_above": 1024, "type": "keyword" }, + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "hash": { "ignore_above": 1024, "type": "keyword" @@ -596,6 +620,18 @@ "ignore_above": 1024, "type": "keyword" }, + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "hash": { "ignore_above": 1024, "type": "keyword" @@ -985,6 +1021,18 @@ "ignore_above": 1024, "type": "keyword" }, + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "hash": { "ignore_above": 1024, "type": "keyword" @@ -1100,6 +1148,18 @@ "ignore_above": 1024, "type": "keyword" }, + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "hash": { "ignore_above": 1024, "type": "keyword" diff --git a/scripts/generator.py b/scripts/generator.py index 2204c58a06..64d8073532 100644 --- a/scripts/generator.py +++ b/scripts/generator.py @@ -4,6 +4,7 @@ from generators import intermediate_files from generators import csv_generator from generators import es_template +from generators import beats def main(): @@ -20,6 +21,7 @@ def main(): csv_generator.generate(ecs_flat, ecs_version) es_template.generate(ecs_flat, ecs_version) + beats.generate(ecs_nested, ecs_version) def argument_parser(): diff --git a/scripts/generators/beats.py b/scripts/generators/beats.py new file mode 100644 index 0000000000..0c4a14fcb5 --- /dev/null +++ b/scripts/generators/beats.py @@ -0,0 +1,86 @@ +import yaml +from collections import OrderedDict + + +def generate(ecs_nested, ecs_version): + # base first + beats_fields = fieldset_field_array(ecs_nested['base']['fields']) + + allowed_fieldset_keys = ['name', 'title', 'group', 'description', 'footnote', 'type'] + # other fieldsets + for fieldset_name in sorted(ecs_nested): + if 'base' == fieldset_name: + continue + fieldset = ecs_nested[fieldset_name] + + beats_field = dict_copy_keys_ordered(fieldset, allowed_fieldset_keys) + beats_field['fields'] = fieldset_field_array(fieldset['fields']) + beats_fields.append(beats_field) + + beats_file = OrderedDict() + beats_file['key'] = 'ecs' + beats_file['title'] = 'ECS' + beats_file['description'] = 'ECS Fields.' + beats_file['fields'] = beats_fields + + write_beats_yaml(beats_file, ecs_version) + + +def fieldset_field_array(source_fields): + allowed_keys = ['name', 'level', 'required', 'type', 'object_type', + 'ignore_above', 'multi_fields', 'format', + 'description', 'example'] + fields = [] + for nested_field_name in source_fields: + ecs_field = source_fields[nested_field_name] + beats_field = dict_copy_keys_ordered(ecs_field, allowed_keys) + beats_field['name'] = nested_field_name + fields.append(beats_field) + return sorted(fields, lambda x, y: cmp(x['name'], y['name'])) + +# Helpers + + +def write_beats_yaml(beats_file, ecs_version): + + with open('generated/beats/fields.ecs.yml', 'w') as outfile: + outfile.write(file_header().format(version=ecs_version)) + yaml.dump([beats_file], outfile, default_flow_style=False) + + +def dict_copy_keys_ordered(dict, copied_keys): + ordered_dict = OrderedDict() + for key in copied_keys: + if key in dict: + ordered_dict[key] = dict[key] + return ordered_dict + + +def indent(text, indent, char=' '): + padding = indent * char + return ''.join(padding + line for line in text.splitlines(True)) + +# Rendering + + +def yaml_ordereddict(dumper, data): + value = [] + for item_key, item_value in data.items(): + node_key = dumper.represent_data(item_key) + node_value = dumper.represent_data(item_value) + value.append((node_key, node_value)) + return yaml.nodes.MappingNode(u'tag:yaml.org,2002:map', value) + + +yaml.add_representer(OrderedDict, yaml_ordereddict) + +# Templates + + +def file_header(): + return ''' +# WARNING! Do not edit this file directly, it was generated by the ECS project, +# based on ECS version {version}. +# Please visit https://github.com/elastic/ecs to suggest changes to ECS fields. + +'''.lstrip() diff --git a/scripts/schema_reader.py b/scripts/schema_reader.py index a2127ca3c0..66d2a4b5b6 100644 --- a/scripts/schema_reader.py +++ b/scripts/schema_reader.py @@ -35,7 +35,6 @@ def dict_clean_string_values(dict): value = dict[key] if isinstance(value, basestring): dict[key] = value.strip() - # TODO: Remove trailing \n? def dict_set_default(dict, key, default): @@ -109,8 +108,12 @@ def field_set_multi_field_defaults(parent_field): mf['flat_name'] = parent_field['flat_name'] + '.' + mf['name'] -def duplicate_reusable_fieldsets(schema, fields_flat): - """Copies reusable field definitions to their expected places in the flattened schema only""" +def duplicate_reusable_fieldsets(schema, fields_flat, fields_nested): + """Copies reusable field definitions to their expected places""" + # Note: across this schema reader, functions are modifying dictionaries passed + # as arguments, which is usually a risk of unintended side effects. + # Here it simplifies the nesting of 'group' under 'user', + # which is in turn reusable in a few places. if 'reusable' in schema: for new_nesting in schema['reusable']['expected']: @@ -123,14 +126,19 @@ def duplicate_reusable_fieldsets(schema, fields_flat): destination_name = new_nesting + '.' + field['flat_name'] copied_field['flat_name'] = destination_name copied_field['original_fieldset'] = schema['name'] + fields_flat[destination_name] = copied_field + # Nested: use original flat name under the destination fieldset + fields_nested[new_nesting]['fields'][field['flat_name']] = copied_field + # Main def finalize_schemas(fields_nested, fields_flat): for schema_name in fields_nested: schema = fields_nested[schema_name] + schema_cleanup_values(schema) for (name, field) in schema['fields'].items(): @@ -138,8 +146,12 @@ def finalize_schemas(fields_nested, fields_flat): fields_flat[field['flat_name']] = field - # TODO duplicate in nested too? - duplicate_reusable_fieldsets(schema, fields_flat) + # This happens as a second pass, so that all fieldsets have their + # fields array replaced with a fields dictionary. + for schema_name in fields_nested: + schema = fields_nested[schema_name] + + duplicate_reusable_fieldsets(schema, fields_flat, fields_nested) def load_ecs(): diff --git a/scripts/tests/test_ecs_spec.py b/scripts/tests/test_ecs_spec.py index 8fa27f910d..b69e5e11fe 100644 --- a/scripts/tests/test_ecs_spec.py +++ b/scripts/tests/test_ecs_spec.py @@ -1,12 +1,15 @@ import unittest from scripts import schema_reader +(nested, flat) = schema_reader.load_ecs() + class TestEcsSpec(unittest.TestCase): """Sanity check for things that should be true in the ECS spec.""" def setUp(self): - (nested, flat) = schema_reader.load_ecs() + global nested + global flat self.ecs_nested = nested self.ecs_fields = flat @@ -16,6 +19,71 @@ def test_base_flat_name(self): self.ecs_nested['base']['fields']['@timestamp']['flat_name'], '@timestamp') + def test_flat_includes_reusable_fields(self): + all_keys = sorted(self.ecs_fields.keys()) + + # geo + self.assertIn('client.geo.name', all_keys) + self.assertIn('destination.geo.name', all_keys) + self.assertIn('host.geo.name', all_keys) + self.assertIn('observer.geo.name', all_keys) + self.assertIn('server.geo.name', all_keys) + self.assertIn('source.geo.name', all_keys) + + # group + self.assertIn('user.group.name', all_keys) + self.assertIn('client.user.group.id', all_keys) + self.assertIn('destination.user.group.id', all_keys) + self.assertIn('server.user.group.id', all_keys) + self.assertIn('source.user.group.id', all_keys) + + # user + self.assertIn('client.user.id', all_keys) + self.assertIn('destination.user.id', all_keys) + self.assertIn('server.user.id', all_keys) + self.assertIn('source.user.id', all_keys) + + # os + self.assertIn('host.os.name', all_keys) + self.assertIn('observer.os.name', all_keys) + self.assertIn('user_agent.os.name', all_keys) + + def test_nested_includes_reusable_fields(self): + client_keys = sorted(self.ecs_nested['client']['fields'].keys()) + destination_keys = sorted(self.ecs_nested['destination']['fields'].keys()) + host_keys = sorted(self.ecs_nested['host']['fields'].keys()) + observer_keys = sorted(self.ecs_nested['observer']['fields'].keys()) + server_keys = sorted(self.ecs_nested['server']['fields'].keys()) + source_keys = sorted(self.ecs_nested['source']['fields'].keys()) + user_agent_keys = sorted(self.ecs_nested['user_agent']['fields'].keys()) + user_keys = sorted(self.ecs_nested['user']['fields'].keys()) + + # geo + self.assertIn('geo.name', client_keys) + self.assertIn('geo.name', destination_keys) + self.assertIn('geo.name', host_keys) + self.assertIn('geo.name', observer_keys) + self.assertIn('geo.name', server_keys) + self.assertIn('geo.name', source_keys) + + # group + self.assertIn('group.name', user_keys) + self.assertIn('user.group.id', client_keys) + self.assertIn('user.group.id', destination_keys) + self.assertIn('user.group.id', server_keys) + self.assertIn('user.group.id', source_keys) + + # user + self.assertIn('user.id', client_keys) + self.assertIn('user.id', destination_keys) + self.assertIn('user.id', server_keys) + self.assertIn('user.id', source_keys) + + # os + self.assertIn('os.name', host_keys) + self.assertIn('os.name', observer_keys) + self.assertIn('os.name', user_agent_keys) + if __name__ == '__main__': unittest.main()