diff --git a/CHANGELOG.next.md b/CHANGELOG.next.md index fa1ddfc8ef..0f8976c556 100644 --- a/CHANGELOG.next.md +++ b/CHANGELOG.next.md @@ -40,6 +40,8 @@ Thanks, you're awesome :-) --> had `reusable.top_level:false`. This PR affects `ecs_flat.yml`, the csv file and the sample Elasticsearch templates. #495, #813 * Removed the `order` attribute from the `ecs_nested.yml` and `ecs_flat.yml` files. #811 +* In `ecs_nested.yml`, the array of strings that used to be in `reusable.expected` + has been replaced by an array of objects with 3 keys: 'as', 'at' and 'full'. #820 #### Bugfixes @@ -58,6 +60,8 @@ Thanks, you're awesome :-) --> * Add support for reusing offical fieldsets in custom schemas. #751 * Add full path names to reused fieldsets in `nestings` array in `ecs_nested.yml`. #803 * Allow shorthand notation for including all subfields in subsets. #805 +* Add ability to nest field sets as another name. #820 +* Add ability to nest field sets within themselves (e.g. `process.parent.*`). #820 #### Deprecated diff --git a/code/go/ecs/process.go b/code/go/ecs/process.go index 568a3cb584..6817fe6939 100644 --- a/code/go/ecs/process.go +++ b/code/go/ecs/process.go @@ -31,9 +31,6 @@ type Process struct { // Process id. PID int64 `ecs:"pid"` - // Process id. - ParentPID int64 `ecs:"parent.pid"` - // Unique identifier for the process. // The implementation of this is specified by the data source, but some // examples of what could be used here are a process-generated UUID, Sysmon @@ -44,122 +41,58 @@ type Process struct { // across multiple monitored hosts. EntityID string `ecs:"entity_id"` - // Unique identifier for the process. - // The implementation of this is specified by the data source, but some - // examples of what could be used here are a process-generated UUID, Sysmon - // Process GUIDs, or a hash of some uniquely identifying components of a - // process. - // Constructing a globally unique identifier is a common practice to - // mitigate PID reuse as well as to identify a specific process over time, - // across multiple monitored hosts. - ParentEntityID string `ecs:"parent.entity_id"` - // Process name. // Sometimes called program name or similar. Name string `ecs:"name"` - // Process name. - // Sometimes called program name or similar. - ParentName string `ecs:"parent.name"` - // Parent process' pid. PPID int64 `ecs:"ppid"` - // Parent process' pid. - ParentPPID int64 `ecs:"parent.ppid"` - // Identifier of the group of processes the process belongs to. PGID int64 `ecs:"pgid"` - // Identifier of the group of processes the process belongs to. - ParentPGID int64 `ecs:"parent.pgid"` - // Full command line that started the process, including the absolute path // to the executable, and all arguments. // Some arguments may be filtered to protect sensitive information. CommandLine string `ecs:"command_line"` - // Full command line that started the process, including the absolute path - // to the executable, and all arguments. - // Some arguments may be filtered to protect sensitive information. - ParentCommandLine string `ecs:"parent.command_line"` - // Array of process arguments, starting with the absolute path to the // executable. // May be filtered to protect sensitive information. Args []string `ecs:"args"` - // Array of process arguments. - // May be filtered to protect sensitive information. - ParentArgs string `ecs:"parent.args"` - // Length of the process.args array. // This field can be useful for querying or performing bucket analysis on // how many arguments were provided to start a process. More arguments may // be an indication of suspicious activity. ArgsCount int64 `ecs:"args_count"` - // Length of the process.args array. - // This field can be useful for querying or performing bucket analysis on - // how many arguments were provided to start a process. More arguments may - // be an indication of suspicious activity. - ParentArgsCount int64 `ecs:"parent.args_count"` - // Absolute path to the process executable. Executable string `ecs:"executable"` - // Absolute path to the process executable. - ParentExecutable string `ecs:"parent.executable"` - // Process title. // The proctitle, some times the same as process name. Can also be // different: for example a browser setting its title to the web page // currently opened. Title string `ecs:"title"` - // Process title. - // The proctitle, some times the same as process name. Can also be - // different: for example a browser setting its title to the web page - // currently opened. - ParentTitle string `ecs:"parent.title"` - // Thread ID. ThreadID int64 `ecs:"thread.id"` - // Thread ID. - ParentThreadID int64 `ecs:"parent.thread.id"` - // Thread name. ThreadName string `ecs:"thread.name"` - // Thread name. - ParentThreadName string `ecs:"parent.thread.name"` - // The time the process started. Start time.Time `ecs:"start"` - // The time the process started. - ParentStart time.Time `ecs:"parent.start"` - // Seconds the process has been up. Uptime int64 `ecs:"uptime"` - // Seconds the process has been up. - ParentUptime int64 `ecs:"parent.uptime"` - // The working directory of the process. WorkingDirectory string `ecs:"working_directory"` - // The working directory of the process. - ParentWorkingDirectory string `ecs:"parent.working_directory"` - // The exit code of the process, if this is a termination event. // The field should be absent if there is no exit code for the event (e.g. // process start). ExitCode int64 `ecs:"exit_code"` - - // The exit code of the process, if this is a termination event. - // The field should be absent if there is no exit code for the event (e.g. - // process start). - ParentExitCode int64 `ecs:"parent.exit_code"` } diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc index 731a56ae31..6504b7c6db 100644 --- a/docs/field-details.asciidoc +++ b/docs/field-details.asciidoc @@ -241,7 +241,7 @@ example: `Google LLC` The `as` fields are expected to be nested at: `client.as`, `destination.as`, `server.as`, `source.as`. -Note also that the `as` fields are not expected to be used directly at the top level. +Note also that the `as` fields are not expected to be used directly at the root of the events. @@ -657,9 +657,9 @@ example: `true` ==== Field Reuse -The `code_signature` fields are expected to be nested at: `dll.code_signature`, `file.code_signature`, `process.code_signature`, `process.parent.code_signature`. +The `code_signature` fields are expected to be nested at: `dll.code_signature`, `file.code_signature`, `process.code_signature`. -Note also that the `code_signature` fields are not expected to be used directly at the top level. +Note also that the `code_signature` fields are not expected to be used directly at the root of the events. @@ -2371,7 +2371,7 @@ example: `Quebec` The `geo` fields are expected to be nested at: `client.geo`, `destination.geo`, `host.geo`, `observer.geo`, `server.geo`, `source.geo`. -Note also that the `geo` fields are not expected to be used directly at the top level. +Note also that the `geo` fields are not expected to be used directly at the root of the events. @@ -2436,7 +2436,7 @@ type: keyword The `group` fields are expected to be nested at: `user.group`. -Note also that the `group` fields may be used directly at the top level. +Note also that the `group` fields may be used directly at the root of the events. @@ -2512,9 +2512,9 @@ type: keyword ==== Field Reuse -The `hash` fields are expected to be nested at: `dll.hash`, `file.hash`, `process.hash`, `process.parent.hash`. +The `hash` fields are expected to be nested at: `dll.hash`, `file.hash`, `process.hash`. -Note also that the `hash` fields are not expected to be used directly at the top level. +Note also that the `hash` fields are not expected to be used directly at the root of the events. @@ -2929,7 +2929,7 @@ example: `eth0` The `interface` fields are expected to be nested at: `observer.egress.interface`, `observer.ingress.interface`. -Note also that the `interface` fields are not expected to be used directly at the top level. +Note also that the `interface` fields are not expected to be used directly at the root of the events. @@ -3785,7 +3785,7 @@ example: `10.14.1` The `os` fields are expected to be nested at: `host.os`, `observer.os`, `user_agent.os`. -Note also that the `os` fields are not expected to be used directly at the top level. +Note also that the `os` fields are not expected to be used directly at the root of the events. @@ -4092,7 +4092,7 @@ example: `Microsoft® Windows® Operating System` The `pe` fields are expected to be nested at: `dll.pe`, `file.pe`, `process.pe`. -Note also that the `pe` fields are not expected to be used directly at the top level. +Note also that the `pe` fields are not expected to be used directly at the root of the events. @@ -4238,263 +4238,6 @@ example: `ssh` // =============================================================== -| process.parent.args -| Array of process arguments. - -May be filtered to protect sensitive information. - -type: keyword - - -Note: this field should contain an array of values. - - - -example: `['ssh', '-l', 'user', '10.0.0.16']` - -| extended - -// =============================================================== - -| process.parent.args_count -| Length of the process.args array. - -This field can be useful for querying or performing bucket analysis on how many arguments were provided to start a process. More arguments may be an indication of suspicious activity. - -type: long - - - -example: `4` - -| extended - -// =============================================================== - -| process.parent.command_line -| Full command line that started the process, including the absolute path to the executable, and all arguments. - -Some arguments may be filtered to protect sensitive information. - -type: keyword - -Multi-fields: - -* process.parent.command_line.text (type: text) - - - - - -example: `/usr/bin/ssh -l user 10.0.0.16` - -| extended - -// =============================================================== - -| process.parent.entity_id -| Unique identifier for the process. - -The implementation of this is specified by the data source, but some examples of what could be used here are a process-generated UUID, Sysmon Process GUIDs, or a hash of some uniquely identifying components of a process. - -Constructing a globally unique identifier is a common practice to mitigate PID reuse as well as to identify a specific process over time, across multiple monitored hosts. - -type: keyword - - - -example: `c2c455d9f99375d` - -| extended - -// =============================================================== - -| process.parent.executable -| Absolute path to the process executable. - -type: keyword - -Multi-fields: - -* process.parent.executable.text (type: text) - - - - - -example: `/usr/bin/ssh` - -| extended - -// =============================================================== - -| process.parent.exit_code -| The exit code of the process, if this is a termination event. - -The field should be absent if there is no exit code for the event (e.g. process start). - -type: long - - - -example: `137` - -| extended - -// =============================================================== - -| process.parent.name -| Process name. - -Sometimes called program name or similar. - -type: keyword - -Multi-fields: - -* process.parent.name.text (type: text) - - - - - -example: `ssh` - -| extended - -// =============================================================== - -| process.parent.pgid -| Identifier of the group of processes the process belongs to. - -type: long - - - - - -| extended - -// =============================================================== - -| process.parent.pid -| Process id. - -type: long - - - -example: `4242` - -| core - -// =============================================================== - -| process.parent.ppid -| Parent process' pid. - -type: long - - - -example: `4241` - -| extended - -// =============================================================== - -| process.parent.start -| The time the process started. - -type: date - - - -example: `2016-05-23T08:05:34.853Z` - -| extended - -// =============================================================== - -| process.parent.thread.id -| Thread ID. - -type: long - - - -example: `4242` - -| extended - -// =============================================================== - -| process.parent.thread.name -| Thread name. - -type: keyword - - - -example: `thread-0` - -| extended - -// =============================================================== - -| process.parent.title -| Process title. - -The proctitle, some times the same as process name. Can also be different: for example a browser setting its title to the web page currently opened. - -type: keyword - -Multi-fields: - -* process.parent.title.text (type: text) - - - - - - - -| extended - -// =============================================================== - -| process.parent.uptime -| Seconds the process has been up. - -type: long - - - -example: `1325` - -| extended - -// =============================================================== - -| process.parent.working_directory -| The working directory of the process. - -type: keyword - -Multi-fields: - -* process.parent.working_directory.text (type: text) - - - - - -example: `/home/alice` - -| extended - -// =============================================================== - | process.pgid | Identifier of the group of processes the process belongs to. @@ -4630,6 +4373,10 @@ example: `/home/alice` ==== Field Reuse +The `process` fields are expected to be nested at: `process.parent`. + +Note also that the `process` fields may be used directly at the root of the events. + @@ -4655,6 +4402,12 @@ example: `/home/alice` // =============================================================== +| <> +| These fields contain information about a process. + +// =============================================================== + + | <> | These fields contain information about binary code signatures. @@ -4667,6 +4420,12 @@ example: `/home/alice` // =============================================================== +| <> +| These fields contain Windows Portable Executable (PE) metadata. + +// =============================================================== + + | <> | These fields contain Windows Portable Executable (PE) metadata. @@ -6474,7 +6233,7 @@ example: `albert` The `user` fields are expected to be nested at: `client.user`, `destination.user`, `host.user`, `server.user`, `source.user`. -Note also that the `user` fields may be used directly at the top level. +Note also that the `user` fields may be used directly at the root of the events. @@ -6644,9 +6403,9 @@ example: `outside` ==== Field Reuse -The `vlan` fields are expected to be nested at: `network.vlan`, `network.inner.vlan`, `observer.egress.vlan`, `observer.ingress.vlan`. +The `vlan` fields are expected to be nested at: `network.inner.vlan`, `network.vlan`, `observer.egress.vlan`, `observer.ingress.vlan`. -Note also that the `vlan` fields are not expected to be used directly at the top level. +Note also that the `vlan` fields are not expected to be used directly at the root of the events. diff --git a/generated/beats/fields.ecs.yml b/generated/beats/fields.ecs.yml index 0497bc3396..c5c4ff16f2 100644 --- a/generated/beats/fields.ecs.yml +++ b/generated/beats/fields.ecs.yml @@ -3106,11 +3106,12 @@ level: extended type: keyword ignore_above: 1024 - description: 'Array of process arguments. + description: 'Array of process arguments, starting with the absolute path to + the executable. May be filtered to protect sensitive information.' example: - - ssh + - /usr/bin/ssh - -l - user - 10.0.0.16 @@ -3253,6 +3254,59 @@ Sometimes called program name or similar.' example: ssh default_field: false + - name: parent.pe.architecture + level: extended + type: keyword + ignore_above: 1024 + description: CPU architecture target for the file. + example: x64 + default_field: false + - name: parent.pe.company + level: extended + type: keyword + ignore_above: 1024 + description: Internal company name of the file, provided at compile-time. + example: Microsoft Corporation + default_field: false + - name: parent.pe.description + level: extended + type: keyword + ignore_above: 1024 + description: Internal description of the file, provided at compile-time. + example: Paint + default_field: false + - name: parent.pe.file_version + level: extended + type: keyword + ignore_above: 1024 + description: Internal version of the file, provided at compile-time. + example: 6.3.9600.17415 + default_field: false + - name: parent.pe.imphash + level: extended + type: keyword + ignore_above: 1024 + description: 'A hash of the imports in a PE file. An imphash -- or import hash + -- can be used to fingerprint binaries even after recompilation or other code-level + transformations have occurred, which would change more traditional hash values. + + Learn more at https://www.fireeye.com/blog/threat-research/2014/01/tracking-malware-import-hashing.html.' + example: 0c6803c4e922103c4dca5963aad36ddf + default_field: false + - name: parent.pe.original_file_name + level: extended + type: keyword + ignore_above: 1024 + description: Internal name of the file, provided at compile-time. + example: MSPAINT.EXE + default_field: false + - name: parent.pe.product + level: extended + type: keyword + ignore_above: 1024 + description: Internal product name of the file, provided at compile-time. + example: "Microsoft\xAE Windows\xAE Operating System" + default_field: false - name: parent.pgid level: extended type: long diff --git a/generated/csv/fields.csv b/generated/csv/fields.csv index eb4e205e39..19e2a8e51e 100644 --- a/generated/csv/fields.csv +++ b/generated/csv/fields.csv @@ -348,7 +348,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 1.6.0-dev,true,process,process.hash.sha512,keyword,extended,,,SHA512 hash. 1.6.0-dev,true,process,process.name,keyword,extended,,ssh,Process name. 1.6.0-dev,true,process,process.name.text,text,extended,,ssh,Process name. -1.6.0-dev,true,process,process.parent.args,keyword,extended,array,"['ssh', '-l', 'user', '10.0.0.16']",Array of process arguments. +1.6.0-dev,true,process,process.parent.args,keyword,extended,array,"['/usr/bin/ssh', '-l', 'user', '10.0.0.16']",Array of process arguments. 1.6.0-dev,true,process,process.parent.args_count,long,extended,,4,Length of the process.args array. 1.6.0-dev,true,process,process.parent.code_signature.exists,boolean,core,,true,Boolean to capture if a signature is present. 1.6.0-dev,true,process,process.parent.code_signature.status,keyword,extended,,ERROR_UNTRUSTED_ROOT,Additional information about the certificate status. @@ -367,6 +367,13 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 1.6.0-dev,true,process,process.parent.hash.sha512,keyword,extended,,,SHA512 hash. 1.6.0-dev,true,process,process.parent.name,keyword,extended,,ssh,Process name. 1.6.0-dev,true,process,process.parent.name.text,text,extended,,ssh,Process name. +1.6.0-dev,true,process,process.parent.pe.architecture,keyword,extended,,x64,CPU architecture target for the file. +1.6.0-dev,true,process,process.parent.pe.company,keyword,extended,,Microsoft Corporation,"Internal company name of the file, provided at compile-time." +1.6.0-dev,true,process,process.parent.pe.description,keyword,extended,,Paint,"Internal description of the file, provided at compile-time." +1.6.0-dev,true,process,process.parent.pe.file_version,keyword,extended,,6.3.9600.17415,Process name. +1.6.0-dev,true,process,process.parent.pe.imphash,keyword,extended,,0c6803c4e922103c4dca5963aad36ddf,A hash of the imports in a PE file. +1.6.0-dev,true,process,process.parent.pe.original_file_name,keyword,extended,,MSPAINT.EXE,"Internal name of the file, provided at compile-time." +1.6.0-dev,true,process,process.parent.pe.product,keyword,extended,,Microsoft® Windows® Operating System,"Internal product name of the file, provided at compile-time." 1.6.0-dev,true,process,process.parent.pgid,long,extended,,,Identifier of the group of processes the process belongs to. 1.6.0-dev,true,process,process.parent.pid,long,core,,4242,Process id. 1.6.0-dev,true,process,process.parent.ppid,long,extended,,4241,Parent process' pid. diff --git a/generated/ecs/ecs_flat.yml b/generated/ecs/ecs_flat.yml index 82e881126e..7e970e8e52 100644 --- a/generated/ecs/ecs_flat.yml +++ b/generated/ecs/ecs_flat.yml @@ -4655,20 +4655,22 @@ process.name: type: keyword process.parent.args: dashed_name: process-parent-args - description: 'Array of process arguments. + description: 'Array of process arguments, starting with the absolute path to the + executable. May be filtered to protect sensitive information.' example: - - ssh + - /usr/bin/ssh - -l - user - 10.0.0.16 flat_name: process.parent.args ignore_above: 1024 level: extended - name: parent.args + name: args normalize: - array + original_fieldset: parent short: Array of process arguments. type: keyword process.parent.args_count: @@ -4681,8 +4683,9 @@ process.parent.args_count: example: 4 flat_name: process.parent.args_count level: extended - name: parent.args_count + name: args_count normalize: [] + original_fieldset: parent short: Length of the process.args array. type: long process.parent.code_signature.exists: @@ -4768,8 +4771,9 @@ process.parent.command_line: name: text norms: false type: text - name: parent.command_line + name: command_line normalize: [] + original_fieldset: parent short: Full command line that started the process. type: keyword process.parent.entity_id: @@ -4787,8 +4791,9 @@ process.parent.entity_id: flat_name: process.parent.entity_id ignore_above: 1024 level: extended - name: parent.entity_id + name: entity_id normalize: [] + original_fieldset: parent short: Unique identifier for the process. type: keyword process.parent.executable: @@ -4803,8 +4808,9 @@ process.parent.executable: name: text norms: false type: text - name: parent.executable + name: executable normalize: [] + original_fieldset: parent short: Absolute path to the process executable. type: keyword process.parent.exit_code: @@ -4816,8 +4822,9 @@ process.parent.exit_code: example: 137 flat_name: process.parent.exit_code level: extended - name: parent.exit_code + name: exit_code normalize: [] + original_fieldset: parent short: The exit code of the process. type: long process.parent.hash.md5: @@ -4878,18 +4885,108 @@ process.parent.name: name: text norms: false type: text - name: parent.name + name: name + normalize: [] + original_fieldset: parent + short: Process name. + type: keyword +process.parent.pe.architecture: + dashed_name: process-parent-pe-architecture + description: CPU architecture target for the file. + example: x64 + flat_name: process.parent.pe.architecture + ignore_above: 1024 + level: extended + name: architecture + normalize: [] + original_fieldset: pe + short: CPU architecture target for the file. + type: keyword +process.parent.pe.company: + dashed_name: process-parent-pe-company + description: Internal company name of the file, provided at compile-time. + example: Microsoft Corporation + flat_name: process.parent.pe.company + ignore_above: 1024 + level: extended + name: company + normalize: [] + original_fieldset: pe + short: Internal company name of the file, provided at compile-time. + type: keyword +process.parent.pe.description: + dashed_name: process-parent-pe-description + description: Internal description of the file, provided at compile-time. + example: Paint + flat_name: process.parent.pe.description + ignore_above: 1024 + level: extended + name: description + normalize: [] + original_fieldset: pe + short: Internal description of the file, provided at compile-time. + type: keyword +process.parent.pe.file_version: + dashed_name: process-parent-pe-file-version + description: Internal version of the file, provided at compile-time. + example: 6.3.9600.17415 + flat_name: process.parent.pe.file_version + ignore_above: 1024 + level: extended + name: file_version normalize: [] + original_fieldset: pe short: Process name. type: keyword +process.parent.pe.imphash: + dashed_name: process-parent-pe-imphash + description: 'A hash of the imports in a PE file. An imphash -- or import hash -- + can be used to fingerprint binaries even after recompilation or other code-level + transformations have occurred, which would change more traditional hash values. + + Learn more at https://www.fireeye.com/blog/threat-research/2014/01/tracking-malware-import-hashing.html.' + example: 0c6803c4e922103c4dca5963aad36ddf + flat_name: process.parent.pe.imphash + ignore_above: 1024 + level: extended + name: imphash + normalize: [] + original_fieldset: pe + short: A hash of the imports in a PE file. + type: keyword +process.parent.pe.original_file_name: + dashed_name: process-parent-pe-original-file-name + description: Internal name of the file, provided at compile-time. + example: MSPAINT.EXE + flat_name: process.parent.pe.original_file_name + ignore_above: 1024 + level: extended + name: original_file_name + normalize: [] + original_fieldset: pe + short: Internal name of the file, provided at compile-time. + type: keyword +process.parent.pe.product: + dashed_name: process-parent-pe-product + description: Internal product name of the file, provided at compile-time. + example: "Microsoft\xAE Windows\xAE Operating System" + flat_name: process.parent.pe.product + ignore_above: 1024 + level: extended + name: product + normalize: [] + original_fieldset: pe + short: Internal product name of the file, provided at compile-time. + type: keyword process.parent.pgid: dashed_name: process-parent-pgid description: Identifier of the group of processes the process belongs to. flat_name: process.parent.pgid format: string level: extended - name: parent.pgid + name: pgid normalize: [] + original_fieldset: parent short: Identifier of the group of processes the process belongs to. type: long process.parent.pid: @@ -4899,8 +4996,9 @@ process.parent.pid: flat_name: process.parent.pid format: string level: core - name: parent.pid + name: pid normalize: [] + original_fieldset: parent short: Process id. type: long process.parent.ppid: @@ -4910,8 +5008,9 @@ process.parent.ppid: flat_name: process.parent.ppid format: string level: extended - name: parent.ppid + name: ppid normalize: [] + original_fieldset: parent short: Parent process' pid. type: long process.parent.start: @@ -4920,8 +5019,9 @@ process.parent.start: example: '2016-05-23T08:05:34.853Z' flat_name: process.parent.start level: extended - name: parent.start + name: start normalize: [] + original_fieldset: parent short: The time the process started. type: date process.parent.thread.id: @@ -4931,8 +5031,9 @@ process.parent.thread.id: flat_name: process.parent.thread.id format: string level: extended - name: parent.thread.id + name: thread.id normalize: [] + original_fieldset: parent short: Thread ID. type: long process.parent.thread.name: @@ -4942,8 +5043,9 @@ process.parent.thread.name: flat_name: process.parent.thread.name ignore_above: 1024 level: extended - name: parent.thread.name + name: thread.name normalize: [] + original_fieldset: parent short: Thread name. type: keyword process.parent.title: @@ -4960,8 +5062,9 @@ process.parent.title: name: text norms: false type: text - name: parent.title + name: title normalize: [] + original_fieldset: parent short: Process title. type: keyword process.parent.uptime: @@ -4970,8 +5073,9 @@ process.parent.uptime: example: 1325 flat_name: process.parent.uptime level: extended - name: parent.uptime + name: uptime normalize: [] + original_fieldset: parent short: Seconds the process has been up. type: long process.parent.working_directory: @@ -4986,8 +5090,9 @@ process.parent.working_directory: name: text norms: false type: text - name: parent.working_directory + name: working_directory normalize: [] + original_fieldset: parent short: The working directory of the process. type: keyword process.pe.architecture: diff --git a/generated/ecs/ecs_nested.yml b/generated/ecs/ecs_nested.yml index c0afc2336e..6741bb81de 100644 --- a/generated/ecs/ecs_nested.yml +++ b/generated/ecs/ecs_nested.yml @@ -140,10 +140,18 @@ as: prefix: as. reusable: expected: - - client - - destination - - server - - source + - as: as + at: client + full: client.as + - as: as + at: destination + full: destination.as + - as: as + at: server + full: server.as + - as: as + at: source + full: source.as top_level: false short: Fields describing an Autonomous System (Internet routing prefix). title: Autonomous System @@ -801,10 +809,15 @@ code_signature: prefix: code_signature. reusable: expected: - - file - - process - - process.parent - - dll + - as: code_signature + at: file + full: file.code_signature + - as: code_signature + at: process + full: process.code_signature + - as: code_signature + at: dll + full: dll.code_signature top_level: false short: These fields contain information about binary code signatures. title: Code Signature @@ -3289,12 +3302,24 @@ geo: prefix: geo. reusable: expected: - - client - - destination - - observer - - host - - server - - source + - as: geo + at: client + full: client.geo + - as: geo + at: destination + full: destination.geo + - as: geo + at: observer + full: observer.geo + - as: geo + at: host + full: host.geo + - as: geo + at: server + full: server.geo + - as: geo + at: source + full: source.geo top_level: false short: Fields describing a location. title: Geo @@ -3340,7 +3365,9 @@ group: prefix: group. reusable: expected: - - user + - as: group + at: user + full: user.group top_level: true short: User's group relevant to the event. title: Group @@ -3397,10 +3424,15 @@ hash: prefix: hash. reusable: expected: - - file - - process - - process.parent - - dll + - as: hash + at: file + full: file.hash + - as: hash + at: process + full: process.hash + - as: hash + at: dll + full: dll.hash top_level: false short: Hashes, usually file hashes. title: Hash @@ -4016,8 +4048,12 @@ interface: prefix: interface. reusable: expected: - - observer.ingress - - observer.egress + - as: interface + at: observer.ingress + full: observer.ingress.interface + - as: interface + at: observer.egress + full: observer.egress.interface top_level: false short: Fields to describe observer interface information. title: Interface @@ -5065,9 +5101,15 @@ os: prefix: os. reusable: expected: - - observer - - host - - user_agent + - as: os + at: observer + full: observer.os + - as: os + at: host + full: host.os + - as: os + at: user_agent + full: user_agent.os top_level: false short: OS fields contain information about the operating system. title: Operating System @@ -5324,9 +5366,15 @@ pe: prefix: pe. reusable: expected: - - file - - dll - - process + - as: pe + at: file + full: file.pe + - as: pe + at: dll + full: dll.pe + - as: pe + at: process + full: process.pe top_level: false short: These fields contain Windows Portable Executable (PE) metadata. title: PE Header @@ -5570,20 +5618,22 @@ process: type: keyword parent.args: dashed_name: process-parent-args - description: 'Array of process arguments. + description: 'Array of process arguments, starting with the absolute path to + the executable. May be filtered to protect sensitive information.' example: - - ssh + - /usr/bin/ssh - -l - user - 10.0.0.16 flat_name: process.parent.args ignore_above: 1024 level: extended - name: parent.args + name: args normalize: - array + original_fieldset: parent short: Array of process arguments. type: keyword parent.args_count: @@ -5596,8 +5646,9 @@ process: example: 4 flat_name: process.parent.args_count level: extended - name: parent.args_count + name: args_count normalize: [] + original_fieldset: parent short: Length of the process.args array. type: long parent.code_signature.exists: @@ -5683,8 +5734,9 @@ process: name: text norms: false type: text - name: parent.command_line + name: command_line normalize: [] + original_fieldset: parent short: Full command line that started the process. type: keyword parent.entity_id: @@ -5702,8 +5754,9 @@ process: flat_name: process.parent.entity_id ignore_above: 1024 level: extended - name: parent.entity_id + name: entity_id normalize: [] + original_fieldset: parent short: Unique identifier for the process. type: keyword parent.executable: @@ -5718,8 +5771,9 @@ process: name: text norms: false type: text - name: parent.executable + name: executable normalize: [] + original_fieldset: parent short: Absolute path to the process executable. type: keyword parent.exit_code: @@ -5731,8 +5785,9 @@ process: example: 137 flat_name: process.parent.exit_code level: extended - name: parent.exit_code + name: exit_code normalize: [] + original_fieldset: parent short: The exit code of the process. type: long parent.hash.md5: @@ -5793,18 +5848,108 @@ process: name: text norms: false type: text - name: parent.name + name: name normalize: [] + original_fieldset: parent short: Process name. type: keyword + parent.pe.architecture: + dashed_name: process-parent-pe-architecture + description: CPU architecture target for the file. + example: x64 + flat_name: process.parent.pe.architecture + ignore_above: 1024 + level: extended + name: architecture + normalize: [] + original_fieldset: pe + short: CPU architecture target for the file. + type: keyword + parent.pe.company: + dashed_name: process-parent-pe-company + description: Internal company name of the file, provided at compile-time. + example: Microsoft Corporation + flat_name: process.parent.pe.company + ignore_above: 1024 + level: extended + name: company + normalize: [] + original_fieldset: pe + short: Internal company name of the file, provided at compile-time. + type: keyword + parent.pe.description: + dashed_name: process-parent-pe-description + description: Internal description of the file, provided at compile-time. + example: Paint + flat_name: process.parent.pe.description + ignore_above: 1024 + level: extended + name: description + normalize: [] + original_fieldset: pe + short: Internal description of the file, provided at compile-time. + type: keyword + parent.pe.file_version: + dashed_name: process-parent-pe-file-version + description: Internal version of the file, provided at compile-time. + example: 6.3.9600.17415 + flat_name: process.parent.pe.file_version + ignore_above: 1024 + level: extended + name: file_version + normalize: [] + original_fieldset: pe + short: Process name. + type: keyword + parent.pe.imphash: + dashed_name: process-parent-pe-imphash + description: 'A hash of the imports in a PE file. An imphash -- or import hash + -- can be used to fingerprint binaries even after recompilation or other code-level + transformations have occurred, which would change more traditional hash values. + + Learn more at https://www.fireeye.com/blog/threat-research/2014/01/tracking-malware-import-hashing.html.' + example: 0c6803c4e922103c4dca5963aad36ddf + flat_name: process.parent.pe.imphash + ignore_above: 1024 + level: extended + name: imphash + normalize: [] + original_fieldset: pe + short: A hash of the imports in a PE file. + type: keyword + parent.pe.original_file_name: + dashed_name: process-parent-pe-original-file-name + description: Internal name of the file, provided at compile-time. + example: MSPAINT.EXE + flat_name: process.parent.pe.original_file_name + ignore_above: 1024 + level: extended + name: original_file_name + normalize: [] + original_fieldset: pe + short: Internal name of the file, provided at compile-time. + type: keyword + parent.pe.product: + dashed_name: process-parent-pe-product + description: Internal product name of the file, provided at compile-time. + example: "Microsoft\xAE Windows\xAE Operating System" + flat_name: process.parent.pe.product + ignore_above: 1024 + level: extended + name: product + normalize: [] + original_fieldset: pe + short: Internal product name of the file, provided at compile-time. + type: keyword parent.pgid: dashed_name: process-parent-pgid description: Identifier of the group of processes the process belongs to. flat_name: process.parent.pgid format: string level: extended - name: parent.pgid + name: pgid normalize: [] + original_fieldset: parent short: Identifier of the group of processes the process belongs to. type: long parent.pid: @@ -5814,8 +5959,9 @@ process: flat_name: process.parent.pid format: string level: core - name: parent.pid + name: pid normalize: [] + original_fieldset: parent short: Process id. type: long parent.ppid: @@ -5825,8 +5971,9 @@ process: flat_name: process.parent.ppid format: string level: extended - name: parent.ppid + name: ppid normalize: [] + original_fieldset: parent short: Parent process' pid. type: long parent.start: @@ -5835,8 +5982,9 @@ process: example: '2016-05-23T08:05:34.853Z' flat_name: process.parent.start level: extended - name: parent.start + name: start normalize: [] + original_fieldset: parent short: The time the process started. type: date parent.thread.id: @@ -5846,8 +5994,9 @@ process: flat_name: process.parent.thread.id format: string level: extended - name: parent.thread.id + name: thread.id normalize: [] + original_fieldset: parent short: Thread ID. type: long parent.thread.name: @@ -5857,8 +6006,9 @@ process: flat_name: process.parent.thread.name ignore_above: 1024 level: extended - name: parent.thread.name + name: thread.name normalize: [] + original_fieldset: parent short: Thread name. type: keyword parent.title: @@ -5875,8 +6025,9 @@ process: name: text norms: false type: text - name: parent.title + name: title normalize: [] + original_fieldset: parent short: Process title. type: keyword parent.uptime: @@ -5885,8 +6036,9 @@ process: example: 1325 flat_name: process.parent.uptime level: extended - name: parent.uptime + name: uptime normalize: [] + original_fieldset: parent short: Seconds the process has been up. type: long parent.working_directory: @@ -5901,8 +6053,9 @@ process: name: text norms: false type: text - name: parent.working_directory + name: working_directory normalize: [] + original_fieldset: parent short: The working directory of the process. type: keyword pe.architecture: @@ -6106,10 +6259,18 @@ process: nestings: - process.code_signature - process.hash + - process.parent - process.parent.code_signature - process.parent.hash + - process.parent.pe - process.pe prefix: process. + reusable: + expected: + - as: parent + at: process + full: process.parent + top_level: true short: These fields contain information about a process. title: Process type: group @@ -8219,11 +8380,21 @@ user: prefix: user. reusable: expected: - - client - - destination - - host - - server - - source + - as: user + at: client + full: client.user + - as: user + at: destination + full: destination.user + - as: user + at: host + full: host.user + - as: user + at: server + full: server.user + - as: user + at: source + full: source.user top_level: true short: Fields to describe the user relevant to the event. title: User @@ -8418,10 +8589,18 @@ vlan: prefix: vlan. reusable: expected: - - observer.ingress - - observer.egress - - network - - network.inner + - as: vlan + at: observer.ingress + full: observer.ingress.vlan + - as: vlan + at: observer.egress + full: observer.egress.vlan + - as: vlan + at: network + full: network.vlan + - as: vlan + at: network.inner + full: network.inner.vlan top_level: false short: Fields to describe observed VLAN information. title: VLAN diff --git a/generated/elasticsearch/6/template.json b/generated/elasticsearch/6/template.json index 01fd060de5..36cad2f3ba 100644 --- a/generated/elasticsearch/6/template.json +++ b/generated/elasticsearch/6/template.json @@ -1753,6 +1753,38 @@ "ignore_above": 1024, "type": "keyword" }, + "pe": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, + "company": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "file_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "imphash": { + "ignore_above": 1024, + "type": "keyword" + }, + "original_file_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "pgid": { "type": "long" }, diff --git a/generated/elasticsearch/7/template.json b/generated/elasticsearch/7/template.json index 5b94b60a32..b2a63213de 100644 --- a/generated/elasticsearch/7/template.json +++ b/generated/elasticsearch/7/template.json @@ -1752,6 +1752,38 @@ "ignore_above": 1024, "type": "keyword" }, + "pe": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, + "company": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "file_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "imphash": { + "ignore_above": 1024, + "type": "keyword" + }, + "original_file_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "pgid": { "type": "long" }, diff --git a/schemas/code_signature.yml b/schemas/code_signature.yml index a4be8cf758..1b22434eb1 100644 --- a/schemas/code_signature.yml +++ b/schemas/code_signature.yml @@ -9,7 +9,6 @@ expected: - file - process - - process.parent - dll # - driver fields: diff --git a/schemas/hash.yml b/schemas/hash.yml index 08de3e0d29..cc44dfcc8b 100644 --- a/schemas/hash.yml +++ b/schemas/hash.yml @@ -16,7 +16,6 @@ expected: - file - process - - process.parent - dll fields: diff --git a/schemas/process.yml b/schemas/process.yml index 8b48c0ea1a..e228444e16 100644 --- a/schemas/process.yml +++ b/schemas/process.yml @@ -24,6 +24,11 @@ from a log message. The `process.pid` often stays in the metric itself and is copied to the global field for correlation. type: group + reusable: + top_level: true + expected: + - at: process + as: parent fields: - name: pid @@ -34,14 +39,6 @@ Process id. example: 4242 - - name: parent.pid - format: string - level: core - type: long - description: > - Process id. - example: 4242 - - name: entity_id level: extended type: keyword @@ -59,23 +56,6 @@ monitored hosts. example: c2c455d9f99375d - - name: parent.entity_id - level: extended - type: keyword - short: Unique identifier for the process. - description: > - Unique identifier for the process. - - The implementation of this is specified by the data source, but some - examples of what could be used here are a process-generated UUID, - Sysmon Process GUIDs, or a hash of some uniquely identifying components - of a process. - - Constructing a globally unique identifier is a common practice to mitigate - PID reuse as well as to identify a specific process over time, across multiple - monitored hosts. - example: c2c455d9f99375d - - name: name level: extended type: keyword @@ -89,20 +69,6 @@ - type: text name: text - - name: parent.name - level: extended - type: keyword - short: Process name. - description: > - Process name. - - Sometimes called program name or similar. - example: ssh - multi_fields: - - type: text - name: text - - - name: ppid format: string level: extended @@ -111,15 +77,6 @@ Parent process' pid. example: 4241 - - name: parent.ppid - format: string - level: extended - type: long - description: > - Parent process' pid. - example: 4241 - - - name: pgid format: string level: extended @@ -127,14 +84,6 @@ description: > Identifier of the group of processes the process belongs to. - - name: parent.pgid - format: string - level: extended - type: long - description: > - Identifier of the group of processes the process belongs to. - - - name: command_line level: extended type: keyword @@ -149,21 +98,6 @@ - type: text name: text - - name: parent.command_line - level: extended - type: keyword - short: Full command line that started the process. - description: > - Full command line that started the process, including the absolute path - to the executable, and all arguments. - - Some arguments may be filtered to protect sensitive information. - example: "/usr/bin/ssh -l user 10.0.0.16" - multi_fields: - - type: text - name: text - - - name: args level: extended type: keyword @@ -176,18 +110,6 @@ normalize: - array - - name: parent.args - level: extended - type: keyword - short: Array of process arguments. - description: > - Array of process arguments. - - May be filtered to protect sensitive information. - example: ["ssh", "-l", "user", "10.0.0.16"] - normalize: - - array - - name: args_count level: extended type: long @@ -200,19 +122,6 @@ More arguments may be an indication of suspicious activity. example: 4 - - name: parent.args_count - level: extended - type: long - short: Length of the process.args array. - description: > - Length of the process.args array. - - This field can be useful for querying or performing bucket analysis on - how many arguments were provided to start a process. - More arguments may be an indication of suspicious activity. - example: 4 - - - name: executable level: extended type: keyword @@ -223,17 +132,6 @@ - type: text name: text - - name: parent.executable - level: extended - type: keyword - description: > - Absolute path to the process executable. - example: /usr/bin/ssh - multi_fields: - - type: text - name: text - - - name: title level: extended type: keyword @@ -247,20 +145,6 @@ - type: text name: text - - name: parent.title - level: extended - type: keyword - short: Process title. - description: > - Process title. - - The proctitle, some times the same as process name. Can also be different: - for example a browser setting its title to the web page currently opened. - multi_fields: - - type: text - name: text - - - name: thread.id format: string level: extended @@ -269,15 +153,6 @@ description: > Thread ID. - - name: parent.thread.id - format: string - level: extended - type: long - example: 4242 - description: > - Thread ID. - - - name: thread.name level: extended type: keyword @@ -285,14 +160,6 @@ description: > Thread name. - - name: parent.thread.name - level: extended - type: keyword - example: 'thread-0' - description: > - Thread name. - - - name: start level: extended type: date @@ -300,14 +167,6 @@ description: > The time the process started. - - name: parent.start - level: extended - type: date - example: "2016-05-23T08:05:34.853Z" - description: > - The time the process started. - - - name: uptime level: extended type: long @@ -315,14 +174,6 @@ description: > Seconds the process has been up. - - name: parent.uptime - level: extended - type: long - example: 1325 - description: > - Seconds the process has been up. - - - name: working_directory level: extended type: keyword @@ -333,17 +184,6 @@ - type: text name: text - - name: parent.working_directory - level: extended - type: keyword - example: /home/alice - description: > - The working directory of the process. - multi_fields: - - type: text - name: text - - - name: exit_code level: extended type: long @@ -354,14 +194,3 @@ The field should be absent if there is no exit code for the event (e.g. process start). - - - name: parent.exit_code - level: extended - type: long - example: 137 - short: The exit code of the process. - description: > - The exit code of the process, if this is a termination event. - - The field should be absent if there is no exit code for the event (e.g. - process start). diff --git a/scripts/generators/asciidoc_fields.py b/scripts/generators/asciidoc_fields.py index 84096b5332..3e8967efe5 100644 --- a/scripts/generators/asciidoc_fields.py +++ b/scripts/generators/asciidoc_fields.py @@ -132,11 +132,11 @@ def render_fieldset_reuse_section(fieldset, intermediate_nested): ) rows = [] for nested_fs_name in fieldset['nestings']: - ecs = ecs_helpers.get_nested_field(nested_fs_name, intermediate_nested) + nested_fs = ecs_helpers.get_nested_field(nested_fs_name, intermediate_nested) rows.append({ 'flat_nesting': "{}.*".format(nested_fs_name), - 'name': nested_fs_name.split('.')[-1], - 'short': ecs['short'] + 'name': nested_fs['name'], + 'short': nested_fs['short'] }) for row in sorted(rows, key=lambda x: x['flat_nesting']): text += render_nesting_row(row) @@ -150,16 +150,16 @@ def render_fieldset_reuses_text(fieldset): return '' section_name = fieldset['name'] - sorted_fields = sorted(fieldset['reusable']['expected']) - rendered_fields = map(lambda f: "`{}.{}`".format(f, section_name), sorted_fields) + sorted_fields = sorted(fieldset['reusable']['expected'], key=lambda k: k['full']) + rendered_fields = map(lambda f: "`{}`".format(f['full']), sorted_fields) text = "The `{}` fields are expected to be nested at: {}.\n\n".format( section_name, ', '.join(rendered_fields)) if 'top_level' in fieldset['reusable'] and fieldset['reusable']['top_level']: - template = "Note also that the `{}` fields may be used directly at the top level.\n\n" + template = "Note also that the `{}` fields may be used directly at the root of the events.\n\n" else: template = "Note also that the `{}` fields are not expected to " + \ - "be used directly at the top level.\n\n" + "be used directly at the root of the events.\n\n" text += template.format(section_name) return text diff --git a/scripts/generators/ecs_helpers.py b/scripts/generators/ecs_helpers.py index a55b67bf1b..9544898196 100644 --- a/scripts/generators/ecs_helpers.py +++ b/scripts/generators/ecs_helpers.py @@ -151,14 +151,6 @@ def list_extract_keys(lst, key_name): return acc -def list_split_by(lst, size): - '''Splits a list in smaller lists of a given size''' - acc = [] - for i in range(0, len(lst), size): - acc.append(lst[i:i + size]) - return acc - - def get_nested_field(fieldname, field_dict): """Takes a field name in dot notation and a dictionary of fields and finds the field in the dictionary""" fields = fieldname.split('.') diff --git a/scripts/schema_reader.py b/scripts/schema_reader.py index b2b46f1557..d851836aed 100644 --- a/scripts/schema_reader.py +++ b/scripts/schema_reader.py @@ -10,6 +10,8 @@ # yml file load (ECS or custom) + cleanup of field set attributes. # merge_schema_fields() # Merge ECS field sets with custom field sets +# assemble_reusables() +# TODO # generate_nested_flat() # Finalize the intermediate representation of all fields. Fills field defaults, # performs field nestings, and precalculates many values used by various generators. @@ -147,21 +149,64 @@ def duplicate_reusable_fieldsets(schema, fields_nested): # Here it simplifies the nesting of 'group' under 'user', # which is in turn reusable in a few places. if 'reusable' in schema: + self_nestings = [] + resolve_reusable_shorthands(schema) for new_nesting in schema['reusable']['expected']: - split_flat_name = new_nesting.split('.') + nest_at = new_nesting['at'] + nest_as = new_nesting['as'] + split_flat_name = nest_at.split('.') top_level = split_flat_name[0] - # List field set names expected under another field set. - # E.g. host.nestings = [ 'geo', 'os', 'user' ] - nested_schema = fields_nested[top_level]['fields'] - for level in split_flat_name[1:]: - nested_schema = nested_schema.get(level, None) - if not nested_schema: - raise ValueError('Field {} in path {} not found in schema'.format(level, new_nesting)) - if nested_schema.get('reusable', None): - raise ValueError( - 'Reusable fields cannot be put inside other reusable fields except when the destination reusable is at the top level') - nested_schema = nested_schema.setdefault('fields', {}) - nested_schema[schema['name']] = schema + if nest_at == schema['name']: + # If nesting schema within itself, we do so as a second step. + # We don't want self-nestings to be copied to all other destinations. + self_nestings.append(new_nesting) + else: + # List field set names expected under another field set. + # E.g. host.nestings = [ 'geo', 'os', 'user' ] + nesting_destination = fields_nested[top_level]['fields'] + for level in split_flat_name[1:]: + nesting_destination = nesting_destination.get(level, None) + if not nesting_destination: + raise ValueError('Field {} in path {} not found in schema'.format(level, nest_at)) + if nesting_destination.get('reusable', None): + raise ValueError( + 'Reusable fields cannot be put inside other reusable fields except when the destination reusable is at the top level') + nesting_destination = nesting_destination.setdefault('fields', {}) + nesting_destination[nest_as] = copy.deepcopy(schema) + for self_nesting in self_nestings: + fields_nested[self_nesting['at']]['fields'][self_nesting['as']] = copy.deepcopy(schema) + + +def resolve_reusable_shorthands(schema): + """ + Replace single word reuse shorthands with the explicit {at: , as:} notation. + + When marking "user" as reusable under "destination" with the shorthand entry + `- destination`, this is expanded to the complete entry + `- { "at": "destination", "as": "user" }`. + The field set is thus nested at `destination.user.*`, with fields such as `destination.user.name`. + + The dictionary notation enables nesting a field set as a different name. + An example is nesting "process" fields to capture parent process details + at `process.parent.*`. + The dictionary notation `- { "at": "process", "as": "parent" }` will yield + fields such as `process.parent.pid`. + """ + if 'reusable' in schema: + reuse_entries = [] + for reuse_entry in schema['reusable']['expected']: + if type(reuse_entry) is dict: + if 'at' in reuse_entry and 'as' in reuse_entry: + explicit_entry = reuse_entry + else: + raise ValueError("When specifying reusable expected locations " + + "with the dictionary notation, keys 'as' and 'at' are required. " + + "Got {}.".format(reuse_entry)) + else: + explicit_entry = {'at': reuse_entry, 'as': schema['name']} + explicit_entry['full'] = explicit_entry['at'] + '.' + explicit_entry['as'] + reuse_entries.append(explicit_entry) + schema['reusable']['expected'] = reuse_entries def cleanup_fields_recursive(fields, prefix, original_fieldset=None): diff --git a/scripts/tests/test_ecs_helpers.py b/scripts/tests/test_ecs_helpers.py index 4ad009f8a8..c092777e9e 100644 --- a/scripts/tests/test_ecs_helpers.py +++ b/scripts/tests/test_ecs_helpers.py @@ -84,11 +84,6 @@ def test_clean_string_values(self): ecs_helpers.dict_clean_string_values(dict) self.assertEqual(dict, {'dirty': 'space, the final frontier', 'clean': 'val', 'int': 1}) - def test_list_slit_by(self): - lst = ['ecs', 'has', 'a', 'meme', 'now'] - split_list = ecs_helpers.list_split_by(lst, 3) - self.assertEqual(split_list, [['ecs', 'has', 'a'], ['meme', 'now']]) - def test_recursive_subset_merge(self): subset_a = { 'field1': { diff --git a/scripts/tests/test_schema_reader.py b/scripts/tests/test_schema_reader.py index e9ec485ef3..fb16aa8db6 100644 --- a/scripts/tests/test_schema_reader.py +++ b/scripts/tests/test_schema_reader.py @@ -39,6 +39,33 @@ def test_set_default_values_no_overwrite(self): schema_reader.schema_set_default_values(schema) self.assertEqual(schema, {'group': 1, 'type': 'group', 'description': '...', 'short': '...'}) + def test_resolve_reusable_shorthands(self): + reusable_with_shorthand = [ + 'destination', + {'at': 'user', 'as': 'effective'} + ] + schema = { + 'name': 'user', + 'reusable': {'top_level': False, 'expected': reusable_with_shorthand} + } + schema_reader.resolve_reusable_shorthands(schema) + expected_reusable = [ + {'at': 'destination', 'as': 'user', 'full': 'destination.user'}, + {'at': 'user', 'as': 'effective', 'full': 'user.effective'} + ] + self.assertEqual(expected_reusable, schema['reusable']['expected']) + + def test_resolve_reusable_shorthands_raises_when_missing_keys_as_at(self): + reusable_with_key_errors = [ + {'hat': 'user', 'has': 'effective'} + ] + schema = { + 'name': 'user', + 'reusable': {'top_level': False, 'expected': reusable_with_key_errors} + } + with self.assertRaises(ValueError): + schema_reader.resolve_reusable_shorthands(schema) + # field definitions def test_field_set_defaults_no_short(self): @@ -359,7 +386,11 @@ def test_reusable_dot_notation(self): 'reusable': { 'top_level': False, 'expected': [ - 'test_fieldset.sub_field' + { + 'at': 'test_fieldset.sub_field', + 'as': 'reusable_fieldset1', + 'full': 'test_fieldset.sub_field.reusable_fieldset1' + } ] }, 'fields': {