diff --git a/packages/windows/changelog.yml b/packages/windows/changelog.yml index dd5b50624b5..929b94b614e 100644 --- a/packages/windows/changelog.yml +++ b/packages/windows/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.12.1" + changes: + - description: Drop unset fields in sysmon_operational data stream. + type: bugfix + link: https://github.com/elastic/integrations/pull/3283 - version: "1.12.0" changes: - description: Support for Sysmon Registry non-QWORD/DWORD events diff --git a/packages/windows/data_stream/sysmon_operational/_dev/test/pipeline/test-events.json b/packages/windows/data_stream/sysmon_operational/_dev/test/pipeline/test-events.json index 498a049c26f..577c9798a8f 100644 --- a/packages/windows/data_stream/sysmon_operational/_dev/test/pipeline/test-events.json +++ b/packages/windows/data_stream/sysmon_operational/_dev/test/pipeline/test-events.json @@ -8956,6 +8956,63 @@ "log": { "level": "information" } + }, + { + "@timestamp": "2021-05-05T15:30:52.731Z", + "winlog": { + "event_id": "1", + "level": "information", + "computer_name": "vagrant-2012-r2", + "opcode": "Info", + "provider_guid": "{9990385f-c22a-43e0-bf4c-06f5698ffbd9}", + "version": 5, + "time_created": "2022-01-24T05:12:52.04227Z", + "channel": "Microsoft-Windows-Sysmon/Operational", + "provider_name": "Microsoft-Windows-Sysmon", + "record_id": 3, + "user": { + "identifier": "S-1-5-18" + }, + "event_data": { + "RuleName": "-", + "UtcTime": "2022-04-27 18:13:46.501", + "ProcessGuid": "{34B92076-87DA-6269-16EE-240000002800}", + "ProcessId": "8232", + "Image": "C:\\Windows\\System32\\cmd.exe", + "FileVersion": "10.0.14393.0 (rs1_release.160715-1616)", + "Description": "Windows Command Processor", + "Product": "Microsoft® Windows® Operating System", + "Company": "Microsoft Corporation", + "OriginalFileName": "Cmd.Exe", + "CommandLine": "\u0026quot;C:\\Windows\\system32\\cmd.exe\u0026quot; /c SQLCMD -S \u0026quot;X\u0026quot; -h -1 -Q \u0026quot;EXEC DBA.dbo.AGSYNC_ALERT\u0026quot; \u0026gt; C:\\tmp\\SQL_ASYNCMonitor.txt", + "CurrentDirectory": "C:\\Windows\\system32\\", + "User": "DOMAIN\\user", + "LogonGuid": "{99999999-3A6F-6235-680C-030000000000}", + "LogonId": "0x30c68", + "TerminalSessionId": "0", + "IntegrityLevel": "High", + "Hashes": "MD5=999999996175B77E0C3A000549D2922C,SHA256=935C1861DF1F4018D698E8B65ABFA02D7E9037D8F68CA3C2065B6CA165D44AD2", + "ParentProcessGuid": "{00000000-0000-0000-0000-000000000000}", + "ParentProcessId": "5060", + "ParentImage": "-", + "ParentCommandLine": "-", + "ParentUser": "-" + }, + "process": { + "pid": 4860, + "thread": { + "id": 4516 + } + } + }, + "log": { + "level": "information" + }, + "event": { + "code": "1", + "kind": "event", + "provider": "Microsoft-Windows-Sysmon" + } } ] } \ No newline at end of file diff --git a/packages/windows/data_stream/sysmon_operational/_dev/test/pipeline/test-events.json-expected.json b/packages/windows/data_stream/sysmon_operational/_dev/test/pipeline/test-events.json-expected.json index 21cf15f4a9f..e2dc82e9654 100644 --- a/packages/windows/data_stream/sysmon_operational/_dev/test/pipeline/test-events.json-expected.json +++ b/packages/windows/data_stream/sysmon_operational/_dev/test/pipeline/test-events.json-expected.json @@ -22000,6 +22000,107 @@ }, "version": 2 } + }, + { + "@timestamp": "2022-04-27T18:13:46.501Z", + "ecs": { + "version": "8.0.0" + }, + "event": { + "category": [ + "process" + ], + "code": "1", + "created": "2022-01-24T05:12:52.042Z", + "kind": "event", + "provider": "Microsoft-Windows-Sysmon", + "type": [ + "start" + ] + }, + "log": { + "level": "information" + }, + "process": { + "args": [ + "\u0026quot;C:\\Windows\\system32\\cmd.exe\u0026quot;", + "/c", + "SQLCMD", + "-S", + "\u0026quot;X\u0026quot;", + "-h", + "-1", + "-Q", + "\u0026quot;EXEC", + "DBA.dbo.AGSYNC_ALERT\u0026quot;", + "\u0026gt;", + "C:\\tmp\\SQL_ASYNCMonitor.txt" + ], + "args_count": 12, + "command_line": "\u0026quot;C:\\Windows\\system32\\cmd.exe\u0026quot; /c SQLCMD -S \u0026quot;X\u0026quot; -h -1 -Q \u0026quot;EXEC DBA.dbo.AGSYNC_ALERT\u0026quot; \u0026gt; C:\\tmp\\SQL_ASYNCMonitor.txt", + "entity_id": "{34B92076-87DA-6269-16EE-240000002800}", + "executable": "C:\\Windows\\System32\\cmd.exe", + "hash": { + "md5": "999999996175b77e0c3a000549d2922c", + "sha256": "935c1861df1f4018d698e8b65abfa02d7e9037d8f68ca3c2065b6ca165d44ad2" + }, + "name": "cmd.exe", + "parent": { + "pid": 5060 + }, + "pe": { + "company": "Microsoft Corporation", + "description": "Windows Command Processor", + "file_version": "10.0.14393.0 (rs1_release.160715-1616)", + "original_file_name": "Cmd.Exe", + "product": "Microsoft® Windows® Operating System" + }, + "pid": 8232, + "working_directory": "C:\\Windows\\system32\\" + }, + "related": { + "hash": [ + "935c1861df1f4018d698e8b65abfa02d7e9037d8f68ca3c2065b6ca165d44ad2", + "999999996175b77e0c3a000549d2922c" + ], + "user": [ + "user" + ] + }, + "user": { + "domain": "DOMAIN", + "id": "S-1-5-18", + "name": "user" + }, + "winlog": { + "channel": "Microsoft-Windows-Sysmon/Operational", + "computer_name": "vagrant-2012-r2", + "event_data": { + "Company": "Microsoft Corporation", + "Description": "Windows Command Processor", + "FileVersion": "10.0.14393.0 (rs1_release.160715-1616)", + "IntegrityLevel": "High", + "LogonGuid": "{99999999-3A6F-6235-680C-030000000000}", + "LogonId": "0x30c68", + "Product": "Microsoft® Windows® Operating System", + "TerminalSessionId": "0" + }, + "event_id": "1", + "opcode": "Info", + "process": { + "pid": 4860, + "thread": { + "id": 4516 + } + }, + "provider_guid": "{9990385f-c22a-43e0-bf4c-06f5698ffbd9}", + "provider_name": "Microsoft-Windows-Sysmon", + "record_id": "3", + "user": { + "identifier": "S-1-5-18" + }, + "version": 5 + } } ] } \ No newline at end of file diff --git a/packages/windows/data_stream/sysmon_operational/elasticsearch/ingest_pipeline/default.yml b/packages/windows/data_stream/sysmon_operational/elasticsearch/ingest_pipeline/default.yml index c75458c77c3..843d73b827c 100644 --- a/packages/windows/data_stream/sysmon_operational/elasticsearch/ingest_pipeline/default.yml +++ b/packages/windows/data_stream/sysmon_operational/elasticsearch/ingest_pipeline/default.yml @@ -6,6 +6,10 @@ processors: - set: field: ecs.version value: '8.0.0' + - script: + description: Remove all empty values from event_data. + lang: painless + source: ctx.winlog?.event_data?.entrySet().removeIf(entry -> [null, "", "-", "{00000000-0000-0000-0000-000000000000}"].contains(entry.getValue())) - rename: field: winlog.level target_field: log.level @@ -166,29 +170,26 @@ processors: target_field: error.code ignore_failure: true ignore_missing: true - if: ctx.event.code == "255" && ctx.winlog?.event_data?.ID != null && ctx.winlog?.event_data?.ID != "" + if: ctx.event.code == "255" - rename: field: winlog.event_data.RuleName target_field: rule.name ignore_missing: true ignore_failure: true - if: ctx?.winlog?.event_data?.RuleName != null && ctx?.winlog?.event_data?.RuleName != "" && ctx?.winlog?.event_data?.RuleName != "-" - - rename: field: winlog.event_data.Type target_field: message ignore_missing: true ignore_failure: true - if: ctx.event.code == "25" && ctx?.winlog?.event_data?.Type != null && ctx?.winlog?.event_data?.Type != "" + if: ctx.event.code == "25" - rename: field: winlog.event_data.Hash target_field: winlog.event_data.Hashes ignore_missing: true ignore_failure: true - if: ctx?.winlog?.event_data?.Hash != null && ctx?.winlog?.event_data?.Hash != "" - kv: field: winlog.event_data.Hashes target_field: _temp.hashes @@ -250,101 +251,86 @@ processors: target_field: process.entity_id ignore_missing: true ignore_failure: true - if: ctx?.winlog?.event_data?.ProcessGuid != null && ctx?.winlog?.event_data?.ProcessGuid != "" - convert: field: winlog.event_data.ProcessId target_field: process.pid type: long ignore_failure: true ignore_missing: true - if: ctx?.winlog?.event_data?.ProcessId != null && ctx?.winlog?.event_data?.ProcessId != "" - rename: field: winlog.event_data.Image target_field: process.executable ignore_missing: true ignore_failure: true - if: ctx?.winlog?.event_data?.Image != null && ctx?.winlog?.event_data?.Image != "" - rename: field: winlog.event_data.SourceProcessGuid target_field: process.entity_id ignore_missing: true ignore_failure: true - if: ctx?.winlog?.event_data?.SourceProcessGuid != null && ctx?.winlog?.event_data?.SourceProcessGuid != "" - rename: field: winlog.event_data.SourceProcessGUID target_field: process.entity_id ignore_missing: true ignore_failure: true - if: ctx?.winlog?.event_data?.SourceProcessGUID != null && ctx?.winlog?.event_data?.SourceProcessGUID != "" - convert: field: winlog.event_data.SourceProcessId target_field: process.pid type: long ignore_failure: true ignore_missing: true - if: ctx?.winlog?.event_data?.SourceProcessId != null && ctx?.winlog?.event_data?.SourceProcessId != "" - convert: field: winlog.event_data.SourceThreadId target_field: process.thread.id type: long ignore_failure: true ignore_missing: true - if: ctx?.winlog?.event_data?.SourceThreadId != null && ctx?.winlog?.event_data?.SourceThreadId != "" - rename: field: winlog.event_data.SourceImage target_field: process.executable ignore_missing: true ignore_failure: true - if: ctx?.winlog?.event_data?.SourceImage != null && ctx?.winlog?.event_data?.SourceImage != "" - rename: field: winlog.event_data.Destination target_field: process.executable ignore_missing: true ignore_failure: true - if: ctx?.winlog?.event_data?.Destination != null && ctx?.winlog?.event_data?.Destination != "" - rename: field: winlog.event_data.CommandLine target_field: process.command_line ignore_missing: true ignore_failure: true - if: ctx?.winlog?.event_data?.CommandLine != null && ctx?.winlog?.event_data?.CommandLine != "" - rename: field: winlog.event_data.CurrentDirectory target_field: process.working_directory ignore_missing: true ignore_failure: true - if: ctx?.winlog?.event_data?.CurrentDirectory != null && ctx?.winlog?.event_data?.CurrentDirectory != "" - rename: field: winlog.event_data.ParentProcessGuid target_field: process.parent.entity_id ignore_missing: true ignore_failure: true - if: ctx?.winlog?.event_data?.ParentProcessGuid != null && ctx?.winlog?.event_data?.ParentProcessGuid != "" - convert: field: winlog.event_data.ParentProcessId target_field: process.parent.pid type: long ignore_failure: true ignore_missing: true - if: ctx?.winlog?.event_data?.ParentProcessId != null && ctx?.winlog?.event_data?.ParentProcessId != "" - rename: field: winlog.event_data.ParentImage target_field: process.parent.executable ignore_missing: true ignore_failure: true - if: ctx?.winlog?.event_data?.ParentImage != null && ctx?.winlog?.event_data?.ParentImage != "" - rename: field: winlog.event_data.ParentCommandLine target_field: process.parent.command_line ignore_missing: true ignore_failure: true - if: ctx?.winlog?.event_data?.ParentCommandLine != null && ctx?.winlog?.event_data?.ParentCommandLine != "" - rename: field: winlog.event_data.OriginalFileName target_field: process.pe.original_file_name ignore_missing: true ignore_failure: true - if: ctx.event.code != "7" && ctx?.winlog?.event_data?.OriginalFileName != null && ctx?.winlog?.event_data?.OriginalFileName != "" + if: ctx.event.code != "7" - set: field: process.pe.company copy_from: winlog.event_data.Company @@ -509,25 +495,21 @@ processors: target_field: file.path ignore_missing: true ignore_failure: true - if: ctx?.winlog?.event_data?.TargetFilename != null && ctx?.winlog?.event_data?.TargetFilename != "" - rename: field: winlog.event_data.Device target_field: file.path ignore_missing: true ignore_failure: true - if: ctx?.winlog?.event_data?.Device != null && ctx?.winlog?.event_data?.Device != "" - rename: field: winlog.event_data.PipeName target_field: file.name ignore_missing: true ignore_failure: true - if: ctx?.winlog?.event_data?.PipeName != null && ctx?.winlog?.event_data?.PipeName != "" - rename: field: winlog.event_data.ImageLoaded target_field: file.path ignore_missing: true ignore_failure: true - if: ctx?.winlog?.event_data?.ImageLoaded != null && ctx?.winlog?.event_data?.ImageLoaded != "" - set: field: file.code_signature.subject_name copy_from: winlog.event_data.Signature @@ -543,7 +525,7 @@ processors: target_field: file.pe.original_file_name ignore_missing: true ignore_failure: true - if: ctx.event.code == "7" && ctx?.winlog?.event_data?.OriginalFileName != null && ctx?.winlog?.event_data?.OriginalFileName != "" + if: ctx.event.code == "7" - set: field: file.pe.company copy_from: winlog.event_data.Company @@ -571,11 +553,11 @@ processors: - set: field: file.code_signature.signed value: true - if: ctx?.winlog?.event_data?.Signed != null && ctx.winlog.event_data.Signed == true + if: ctx?.winlog?.event_data?.Signed == true - set: field: file.code_signature.valid value: true - if: ctx?.winlog?.event_data?.SignatureStatus != null && ctx?.winlog?.event_data?.SignatureStatus == "Valid" + if: ctx?.winlog?.event_data?.SignatureStatus == "Valid" - script: description: Adds file information. @@ -604,19 +586,18 @@ processors: target_field: network.transport ignore_missing: true ignore_failure: true - if: ctx?.winlog?.event_data?.Protocol != null && ctx?.winlog?.event_data?.Protocol != "" - rename: field: winlog.event_data.DestinationPortName target_field: network.protocol ignore_missing: true ignore_failure: true - if: ctx.event.code != "22" && ctx?.winlog?.event_data?.DestinationPortName != null && ctx?.winlog?.event_data?.DestinationPortName != "" + if: ctx.event.code != "22" - rename: field: winlog.event_data.SourcePortName target_field: network.protocol ignore_missing: true ignore_failure: true - if: ctx.event.code != "22" && ctx?.winlog?.event_data?.SourcePortName != null && ctx?.winlog?.event_data?.SourcePortName != "" + if: ctx.event.code != "22" - set: field: network.protocol value: dns @@ -627,68 +608,61 @@ processors: type: ip ignore_failure: true ignore_missing: true - if: ctx?.winlog?.event_data?.SourceIp != null && ctx?.winlog?.event_data?.SourceIp != "" - rename: field: winlog.event_data.SourceHostname target_field: source.domain ignore_missing: true ignore_failure: true - if: ctx?.winlog?.event_data?.SourceHostname != null && ctx?.winlog?.event_data?.SourceHostname != "" - convert: field: winlog.event_data.SourcePort target_field: source.port type: long ignore_failure: true ignore_missing: true - if: ctx?.winlog?.event_data?.SourcePort != null && ctx?.winlog?.event_data?.SourcePort != "" - convert: field: winlog.event_data.DestinationIp target_field: destination.ip type: ip ignore_failure: true ignore_missing: true - if: ctx?.winlog?.event_data?.DestinationIp != null && ctx?.winlog?.event_data?.DestinationIp != "" - rename: field: winlog.event_data.DestinationHostname target_field: destination.domain ignore_missing: true ignore_failure: true - if: ctx?.winlog?.event_data?.DestinationHostname != null && ctx?.winlog?.event_data?.DestinationHostname != "" - convert: field: winlog.event_data.DestinationPort target_field: destination.port type: long ignore_failure: true ignore_missing: true - if: ctx?.winlog?.event_data?.DestinationPort != null && ctx?.winlog?.event_data?.DestinationPort != "" - rename: field: winlog.event_data.QueryName target_field: dns.question.name ignore_missing: true ignore_failure: true - if: ctx?.winlog?.event_data?.QueryName != null && ctx?.winlog?.event_data?.QueryName != "" - set: field: network.direction value: egress - if: ctx?.winlog?.event_data?.Initiated != null && ctx?.winlog?.event_data?.Initiated == "true" + if: ctx?.winlog?.event_data?.Initiated == "true" - set: field: network.direction value: ingress - if: ctx?.winlog?.event_data?.Initiated != null && ctx?.winlog?.event_data?.Initiated == "false" + if: ctx?.winlog?.event_data?.Initiated == "false" - set: field: network.type value: ipv4 - if: ctx?.winlog?.event_data?.SourceIsIpv6 != null && ctx?.winlog?.event_data?.SourceIsIpv6 == "false" + if: ctx?.winlog?.event_data?.SourceIsIpv6 == "false" - set: field: network.type value: ipv6 - if: ctx?.winlog?.event_data?.SourceIsIpv6 != null && ctx?.winlog?.event_data?.SourceIsIpv6 == "true" + if: ctx?.winlog?.event_data?.SourceIsIpv6 == "true" - script: description: | Splits the QueryResults field that contains the DNS responses. Example: "type: 5 f2.taboola.map.fastly.net;::ffff:151.101.66.2;::ffff:151.101.130.2;::ffff:151.101.194.2;::ffff:151.101.2.2;" lang: painless - if: ctx?.winlog?.event_data?.QueryResults != null && ctx?.winlog?.event_data?.QueryResults != "" + if: ctx?.winlog?.event_data?.QueryResults != null params: "1": "A" "2": "NS" @@ -885,7 +859,6 @@ processors: target_field: sysmon.dns.status ignore_missing: true ignore_failure: true - if: ctx?.winlog?.event_data?.QueryStatus != null && ctx?.winlog?.event_data?.QueryStatus != "" - script: description: Translate DNS Query status. lang: painless @@ -1099,14 +1072,12 @@ processors: type: boolean ignore_missing: true ignore_failure: true - if: ctx?.winlog?.event_data?.Archived != null && ctx?.winlog?.event_data?.Archived != "" - convert: field: winlog.event_data.IsExecutable target_field: sysmon.file.is_executable type: boolean ignore_missing: true ignore_failure: true - if: ctx?.winlog?.event_data?.IsExecutable != null && ctx?.winlog?.event_data?.IsExecutable != "" ## Related fields @@ -1135,8 +1106,7 @@ processors: description: Set registry fields. lang: painless if: |- - ctx?.winlog?.event_data?.TargetObject != null && ctx?.winlog?.event_data?.TargetObject != "" && - ["12", "13", "14"].contains(ctx.event.code) + ctx?.winlog?.event_data?.TargetObject != null && ["12", "13", "14"].contains(ctx.event.code) params: HKEY_CLASSES_ROOT: "HKCR" HKCR: "HKCR" @@ -1257,10 +1227,6 @@ processors: - winlog.level ignore_failure: true ignore_missing: true - - script: - description: Remove all empty values from event_data. - lang: painless - source: ctx?.winlog?.event_data?.entrySet().removeIf(entry -> entry.getValue() == null || entry.getValue().equals("") || entry.getValue().equals("-")); - remove: description: Remove empty event data. field: winlog.event_data diff --git a/packages/windows/manifest.yml b/packages/windows/manifest.yml index d5f148b2ff0..a3632b677b2 100644 --- a/packages/windows/manifest.yml +++ b/packages/windows/manifest.yml @@ -1,6 +1,6 @@ name: windows title: Windows -version: 1.12.0 +version: 1.12.1 description: Collect logs and metrics from Windows OS and services with Elastic Agent. type: integration categories: