diff --git a/packages/microsoft_sqlserver/changelog.yml b/packages/microsoft_sqlserver/changelog.yml index 3d674396cc9..73fd18b821a 100644 --- a/packages/microsoft_sqlserver/changelog.yml +++ b/packages/microsoft_sqlserver/changelog.yml @@ -1,4 +1,15 @@ # newer versions go on top +- version: "0.3.0" + changes: + - description: Expose winlog input ignore_older option. + type: enhancement + link: https://github.com/elastic/integrations/pull/2542 + - description: Fix preserve original event option + type: bugfix + link: https://github.com/elastic/integrations/pull/2542 + - description: Make order of options consistent with other winlog based integrations. + type: enhancement + link: https://github.com/elastic/integrations/pull/2542 - version: "0.2.0" changes: - description: Expose winlog input language option. diff --git a/packages/microsoft_sqlserver/data_stream/audit/_dev/test/pipeline/test-events.json-expected.json b/packages/microsoft_sqlserver/data_stream/audit/_dev/test/pipeline/test-events.json-expected.json index 37808fdce20..f784754d1eb 100644 --- a/packages/microsoft_sqlserver/data_stream/audit/_dev/test/pipeline/test-events.json-expected.json +++ b/packages/microsoft_sqlserver/data_stream/audit/_dev/test/pipeline/test-events.json-expected.json @@ -1,531 +1,532 @@ { "expected": [ { + "@timestamp": "2021-10-23T23:01:20.405Z", "agent": { - "name": "WinDev2108Eval", + "ephemeral_id": "64d8db00-4d07-452f-a9af-cd828d6b9d33", "hostname": "WinDev2108Eval", "id": "b53be7b1-9e86-49b0-ad0b-1464bceabc65", - "ephemeral_id": "64d8db00-4d07-452f-a9af-cd828d6b9d33", + "name": "WinDev2108Eval", "type": "filebeat", "version": "7.15.1" }, - "process": { - "pid": 648, - "thread": { - "id": 364 - } - }, - "winlog": { - "computer_name": "WinDev2108Eval", - "record_id": 17607, - "event_id": "33205", - "keywords": [ - "Audit Success", - "Classic" - ], - "activity_id": "{3d5039f7-c692-0000-653a-503d92c6d701}", - "channel": "Security", - "api": "wineventlog", - "provider_name": "MSSQLSERVER$AUDIT", - "user": { - "name": "MSSQLSERVER", - "identifier": "S-1-5-80-3880718306-3832830129-1677859214-2598158968-1052248003", - "type": "Well Known Group", - "domain": "NT SERVICE" - } - }, - "log": { - "level": "information" - }, - "elastic_agent": { - "version": "7.15.1", - "snapshot": false, - "id": "b53be7b1-9e86-49b0-ad0b-1464bceabc65" + "data_stream": { + "dataset": "microsoft_sqlserver.audit", + "namespace": "default", + "type": "logs" }, - "@timestamp": "2021-10-23T23:01:20.405Z", "ecs": { "version": "1.12.0" }, - "sqlserver": { - "audit": { - "target_database_principal_id": "0", - "is_column_permission": true, - "sequence_group_id": "8F6581B6-23C3-4800-AD85-A4780B5332BB", - "permission_bitmask": "00000000000000000000000000000001", - "class_type": "TABLE", - "application_name": "Microsoft SQL Server Management Studio - Query", - "session_server_principal_name": "WINDEV2108EVAL\\User", - "action_id": "SELECT", - "audit_schema_version": "1", - "object_name": "personal", - "statement": "SELECT * FROM dbo.personal", - "client_ip": "local machine", - "database_principal_id": "1", - "transaction_id": "39562", - "database_name": "TestDB", - "target_server_principal_id": "0", - "server_principal_id": "259", - "response_rows": 0, - "session_id": "55", - "database_principal_name": "dbo", - "affected_rows": 0, - "schema_name": "dbo", - "object_id": "885578193", - "server_instance_name": "WINDEV2108EVAL", - "sequence_number": 1, - "connection_id": "FBB762BA-B0C5-4D09-9349-4A408C909199", - "event_time": "2021-10-23 23:01:20.4050216", - "user_defined_event_id": "0", - "host_name": "WINDEV2108EVAL", - "succeeded": true - } + "elastic_agent": { + "id": "b53be7b1-9e86-49b0-ad0b-1464bceabc65", + "snapshot": false, + "version": "7.15.1" }, - "data_stream": { - "namespace": "default", - "type": "logs", - "dataset": "microsoft_sqlserver.audit" + "event": { + "action": "select", + "agent_id_status": "verified", + "category": [ + "database" + ], + "code": "33205", + "created": "2021-10-21T11:01:45.955Z", + "dataset": "microsoft_sqlserver.audit", + "duration": 0, + "ingested": "2021-10-21T11:01:47Z", + "kind": "event", + "outcome": "success", + "provider": "MSSQLSERVER$AUDIT", + "type": [ + "access" + ] }, "host": { + "architecture": "x86_64", "hostname": "WinDev2108Eval", - "os": { - "build": "19043.1288", - "kernel": "10.0.19041.1288 (WinBuild.160101.0800)", - "name": "Windows 10 Enterprise Evaluation", - "family": "windows", - "type": "windows", - "version": "10.0", - "platform": "windows" - }, + "id": "abe598f7-ff01-46dc-999b-d1751abbe398", "ip": [ "fe80::d567:3328:3f4c:113b", "10.0.2.15", "fe80::74fb:1698:31ac:f597", "192.168.56.103" ], - "name": "WinDev2108Eval", - "id": "abe598f7-ff01-46dc-999b-d1751abbe398", "mac": [ "08-00-27-DE-7B-66", "08-00-27-6A-C1-FB" ], - "architecture": "x86_64" - }, - "event": { - "agent_id_status": "verified", - "duration": 0, - "ingested": "2021-10-21T11:01:47Z", - "code": "33205", - "provider": "MSSQLSERVER$AUDIT", - "created": "2021-10-21T11:01:45.955Z", - "kind": "event", - "action": "select", - "category": [ - "database" - ], - "type": [ - "access" - ], - "dataset": "microsoft_sqlserver.audit", - "outcome": "success" - }, - "user": { - "name": "User", - "domain": "WINDEV2108EVAL", - "id": "010500000000000515000000b7baa3e36a4480a90b1df1eae9030000" - } - }, - { - "agent": { "name": "WinDev2108Eval", - "hostname": "WinDev2108Eval", - "id": "8b2d19ad-2ecf-40d9-ad3b-746991df9989", - "ephemeral_id": "0fa6e59e-1439-4bd3-910c-82b99d3f17cc", - "type": "filebeat", - "version": "7.15.1" + "os": { + "build": "19043.1288", + "family": "windows", + "kernel": "10.0.19041.1288 (WinBuild.160101.0800)", + "name": "Windows 10 Enterprise Evaluation", + "platform": "windows", + "type": "windows", + "version": "10.0" + } + }, + "log": { + "level": "information" }, "process": { "pid": 648, "thread": { - "id": 2744 + "id": 364 + } + }, + "sqlserver": { + "audit": { + "action_id": "SELECT", + "affected_rows": 0, + "application_name": "Microsoft SQL Server Management Studio - Query", + "audit_schema_version": "1", + "class_type": "TABLE", + "client_ip": "local machine", + "connection_id": "FBB762BA-B0C5-4D09-9349-4A408C909199", + "database_name": "TestDB", + "database_principal_id": "1", + "database_principal_name": "dbo", + "event_time": "2021-10-23 23:01:20.4050216", + "host_name": "WINDEV2108EVAL", + "is_column_permission": true, + "object_id": "885578193", + "object_name": "personal", + "permission_bitmask": "00000000000000000000000000000001", + "response_rows": 0, + "schema_name": "dbo", + "sequence_group_id": "8F6581B6-23C3-4800-AD85-A4780B5332BB", + "sequence_number": 1, + "server_instance_name": "WINDEV2108EVAL", + "server_principal_id": "259", + "session_id": "55", + "session_server_principal_name": "WINDEV2108EVAL\\User", + "statement": "SELECT * FROM dbo.personal", + "succeeded": true, + "target_database_principal_id": "0", + "target_server_principal_id": "0", + "transaction_id": "39562", + "user_defined_event_id": "0" } }, + "user": { + "domain": "WINDEV2108EVAL", + "id": "010500000000000515000000b7baa3e36a4480a90b1df1eae9030000", + "name": "User" + }, "winlog": { + "activity_id": "{3d5039f7-c692-0000-653a-503d92c6d701}", + "api": "wineventlog", + "channel": "Security", "computer_name": "WinDev2108Eval", - "record_id": 26134, "event_id": "33205", "keywords": [ "Audit Success", "Classic" ], - "activity_id": "{855644c9-ca0a-0001-4945-56850acad701}", - "channel": "Security", - "api": "wineventlog", "provider_name": "MSSQLSERVER$AUDIT", + "record_id": 17607, "user": { - "name": "MSSQLSERVER", + "domain": "NT SERVICE", "identifier": "S-1-5-80-3880718306-3832830129-1677859214-2598158968-1052248003", - "type": "Well Known Group", - "domain": "NT SERVICE" + "name": "MSSQLSERVER", + "type": "Well Known Group" } + } + }, + { + "@timestamp": "2021-10-28T08:44:33.974Z", + "agent": { + "ephemeral_id": "0fa6e59e-1439-4bd3-910c-82b99d3f17cc", + "hostname": "WinDev2108Eval", + "id": "8b2d19ad-2ecf-40d9-ad3b-746991df9989", + "name": "WinDev2108Eval", + "type": "filebeat", + "version": "7.15.1" }, - "log": { - "level": "information" - }, - "elastic_agent": { - "version": "7.15.1", - "snapshot": false, - "id": "8b2d19ad-2ecf-40d9-ad3b-746991df9989" + "data_stream": { + "dataset": "microsoft_sqlserver.audit", + "namespace": "default", + "type": "logs" }, - "@timestamp": "2021-10-28T08:44:33.974Z", "ecs": { "version": "1.12.0" }, - "sqlserver": { - "audit": { - "target_database_principal_id": "0", - "is_column_permission": false, - "sequence_group_id": "D883B193-67E7-4A29-8E00-AD24C7FF3ABC", - "permission_bitmask": "00000000000000000000000000000008", - "class_type": "TABLE", - "application_name": "Microsoft SQL Server Management Studio - Query", - "session_server_principal_name": "WINDEV2108EVAL\\User", - "action_id": "INSERT", - "audit_schema_version": "1", - "object_name": "Persons", - "statement": "INSERT INTO dbo.Persons (PersonID, LastName, FirstName, Address, City)\nVALUES (102, 'G', 'Sai', 'Australia', 'Melbourne')", - "client_ip": "local machine", - "database_principal_id": "1", - "transaction_id": "693425", - "database_name": "Auditable", - "target_server_principal_id": "0", - "server_principal_id": "259", - "response_rows": 0, - "session_id": "52", - "database_principal_name": "dbo", - "affected_rows": 0, - "schema_name": "dbo", - "object_id": "581577110", - "server_instance_name": "WINDEV2108EVAL", - "sequence_number": 1, - "connection_id": "D3194377-1174-4BA9-BAFC-CDD5EC69C962", - "event_time": "2021-10-28 08:44:33.9741294", - "user_defined_event_id": "0", - "host_name": "WINDEV2108EVAL", - "succeeded": true - } + "elastic_agent": { + "id": "8b2d19ad-2ecf-40d9-ad3b-746991df9989", + "snapshot": false, + "version": "7.15.1" }, - "data_stream": { - "namespace": "default", - "type": "logs", - "dataset": "microsoft_sqlserver.audit" + "event": { + "action": "insert", + "agent_id_status": "verified", + "category": [ + "database" + ], + "code": "33205", + "created": "2021-10-28T08:44:35.844Z", + "dataset": "microsoft_sqlserver.audit", + "duration": 0, + "ingested": "2021-10-28T08:44:36Z", + "kind": "event", + "original": "Audit event: audit_schema_version:1\nevent_time:2021-10-28 08:44:33.9741294\nsequence_number:1\naction_id:IN \nsucceeded:true\nis_column_permission:false\nsession_id:52\nserver_principal_id:259\ndatabase_principal_id:1\ntarget_server_principal_id:0\ntarget_database_principal_id:0\nobject_id:581577110\nuser_defined_event_id:0\ntransaction_id:693425\nclass_type:U \nduration_milliseconds:0\nresponse_rows:0\naffected_rows:0\nclient_ip:local machine\npermission_bitmask:00000000000000000000000000000008\nsequence_group_id:D883B193-67E7-4A29-8E00-AD24C7FF3ABC\nsession_server_principal_name:WINDEV2108EVAL\\User\nserver_principal_name:WINDEV2108EVAL\\User\nserver_principal_sid:010500000000000515000000b7baa3e36a4480a90b1df1eae9030000\ndatabase_principal_name:dbo\ntarget_server_principal_name:\ntarget_server_principal_sid:\ntarget_database_principal_name:\nserver_instance_name:WINDEV2108EVAL\ndatabase_name:Auditable\nschema_name:dbo\nobject_name:Persons\nstatement:INSERT INTO dbo.Persons (PersonID, LastName, FirstName, Address, City)\nVALUES (102, 'G', 'Sai', 'Australia', 'Melbourne')\nadditional_information:\nuser_defined_information:\napplication_name:Microsoft SQL Server Management Studio - Query\nconnection_id:D3194377-1174-4BA9-BAFC-CDD5EC69C962\ndata_sensitivity_information:\nhost_name:WINDEV2108EVAL\n.", + "outcome": "success", + "provider": "MSSQLSERVER$AUDIT", + "type": [ + "info" + ] }, "host": { + "architecture": "x86_64", "hostname": "WinDev2108Eval", - "os": { - "build": "19043.1288", - "kernel": "10.0.19041.1288 (WinBuild.160101.0800)", - "name": "Windows 10 Enterprise Evaluation", - "family": "windows", - "type": "windows", - "version": "10.0", - "platform": "windows" - }, + "id": "abe598f7-ff01-46dc-999b-d1751abbe398", "ip": [ "fe80::d567:3328:3f4c:113b", "10.0.2.15", "fe80::74fb:1698:31ac:f597", "192.168.56.103" ], - "name": "WinDev2108Eval", - "id": "abe598f7-ff01-46dc-999b-d1751abbe398", "mac": [ "08-00-27-DE-7B-66", "08-00-27-6A-C1-FB" ], - "architecture": "x86_64" - }, - "event": { - "agent_id_status": "verified", - "duration": 0, - "ingested": "2021-10-28T08:44:36Z", - "code": "33205", - "provider": "MSSQLSERVER$AUDIT", - "created": "2021-10-28T08:44:35.844Z", - "kind": "event", - "action": "insert", - "type": [ - "info" - ], - "category": [ - "database" - ], - "dataset": "microsoft_sqlserver.audit", - "outcome": "success" - }, - "user": { - "name": "User", - "domain": "WINDEV2108EVAL", - "id": "010500000000000515000000b7baa3e36a4480a90b1df1eae9030000" - } - }, - { - "agent": { "name": "WinDev2108Eval", - "hostname": "WinDev2108Eval", - "id": "df0dd5ff-cce7-4861-b49f-fd70f0b207b6", - "ephemeral_id": "968f4b33-dfd4-455e-af64-ef3d7c3e7daf", - "type": "filebeat", - "version": "7.15.1" + "os": { + "build": "19043.1288", + "family": "windows", + "kernel": "10.0.19041.1288 (WinBuild.160101.0800)", + "name": "Windows 10 Enterprise Evaluation", + "platform": "windows", + "type": "windows", + "version": "10.0" + } + }, + "log": { + "level": "information" }, "process": { - "pid": 656, + "pid": 648, "thread": { - "id": 1688 + "id": 2744 } }, + "sqlserver": { + "audit": { + "action_id": "INSERT", + "affected_rows": 0, + "application_name": "Microsoft SQL Server Management Studio - Query", + "audit_schema_version": "1", + "class_type": "TABLE", + "client_ip": "local machine", + "connection_id": "D3194377-1174-4BA9-BAFC-CDD5EC69C962", + "database_name": "Auditable", + "database_principal_id": "1", + "database_principal_name": "dbo", + "event_time": "2021-10-28 08:44:33.9741294", + "host_name": "WINDEV2108EVAL", + "is_column_permission": false, + "object_id": "581577110", + "object_name": "Persons", + "permission_bitmask": "00000000000000000000000000000008", + "response_rows": 0, + "schema_name": "dbo", + "sequence_group_id": "D883B193-67E7-4A29-8E00-AD24C7FF3ABC", + "sequence_number": 1, + "server_instance_name": "WINDEV2108EVAL", + "server_principal_id": "259", + "session_id": "52", + "session_server_principal_name": "WINDEV2108EVAL\\User", + "statement": "INSERT INTO dbo.Persons (PersonID, LastName, FirstName, Address, City)\nVALUES (102, 'G', 'Sai', 'Australia', 'Melbourne')", + "succeeded": true, + "target_database_principal_id": "0", + "target_server_principal_id": "0", + "transaction_id": "693425", + "user_defined_event_id": "0" + } + }, + "user": { + "domain": "WINDEV2108EVAL", + "id": "010500000000000515000000b7baa3e36a4480a90b1df1eae9030000", + "name": "User" + }, "winlog": { + "activity_id": "{855644c9-ca0a-0001-4945-56850acad701}", + "api": "wineventlog", + "channel": "Security", "computer_name": "WinDev2108Eval", - "record_id": 27810, "event_id": "33205", "keywords": [ "Audit Success", "Classic" ], - "activity_id": "{3622691f-ccfb-0000-9469-2236fbccd701}", - "channel": "Security", - "api": "wineventlog", "provider_name": "MSSQLSERVER$AUDIT", + "record_id": 26134, "user": { - "name": "MSSQLSERVER", + "domain": "NT SERVICE", "identifier": "S-1-5-80-3880718306-3832830129-1677859214-2598158968-1052248003", - "type": "Well Known Group", - "domain": "NT SERVICE" + "name": "MSSQLSERVER", + "type": "Well Known Group" } + } + }, + { + "@timestamp": "2021-10-29T02:09:40.498Z", + "agent": { + "ephemeral_id": "968f4b33-dfd4-455e-af64-ef3d7c3e7daf", + "hostname": "WinDev2108Eval", + "id": "df0dd5ff-cce7-4861-b49f-fd70f0b207b6", + "name": "WinDev2108Eval", + "type": "filebeat", + "version": "7.15.1" }, - "log": { - "level": "information" - }, - "elastic_agent": { - "version": "7.15.1", - "snapshot": false, - "id": "df0dd5ff-cce7-4861-b49f-fd70f0b207b6" + "data_stream": { + "dataset": "microsoft_sqlserver.audit", + "namespace": "default", + "type": "logs" }, - "@timestamp": "2021-10-29T02:09:40.498Z", "ecs": { "version": "1.12.0" }, - "sqlserver": { - "audit": { - "target_database_principal_id": "0", - "is_column_permission": false, - "sequence_group_id": "DBDB0CF9-276D-4291-9A39-6FDA3895176C", - "permission_bitmask": "00000000000000000000000000000000", - "class_type": "LOGIN", - "application_name": "SQLServerCEIP", - "session_server_principal_name": "NT Service\\SQLTELEMETRY", - "action_id": "LOGIN SUCCEEDED", - "audit_schema_version": "1", - "statement": "-- network protocol: LPC\nset quoted_identifier on\nset arithabort off\nset numeric_roundabort off\nset ansi_warnings on\nset ansi_padding on\nset ansi_nulls on\nset concat_null_yields_null on\nset cursor_close_on_commit off\nset implicit_transactions off\nset language us_english\nset dateformat mdy\nset datefirst 7\nset transaction isolation level read committed\n", - "client_ip": "local machine", - "database_principal_id": "0", - "transaction_id": "0", - "target_server_principal_id": "0", - "server_principal_id": "265", - "response_rows": 0, - "session_id": "64", - "affected_rows": 0, - "object_id": "0", - "server_instance_name": "WINDEV2108EVAL", - "sequence_number": 1, - "connection_id": "FADD287E-2D7D-4D1C-9C51-6A9AA7AF461E", - "event_time": "2021-10-29 02:09:40.4984112", - "user_defined_event_id": "0", - "host_name": "WINDEV2108EVAL", - "succeeded": true - } + "elastic_agent": { + "id": "df0dd5ff-cce7-4861-b49f-fd70f0b207b6", + "snapshot": false, + "version": "7.15.1" }, - "data_stream": { - "namespace": "default", - "type": "logs", - "dataset": "microsoft_sqlserver.audit" + "event": { + "action": "login-succeeded", + "agent_id_status": "verified", + "category": [ + "database", + "session" + ], + "code": "33205", + "created": "2021-10-29T02:09:41.540Z", + "dataset": "microsoft_sqlserver.audit", + "duration": 0, + "ingested": "2021-10-29T02:09:42Z", + "kind": "event", + "outcome": "success", + "provider": "MSSQLSERVER$AUDIT", + "type": [ + "info", + "start" + ] }, "host": { + "architecture": "x86_64", "hostname": "WinDev2108Eval", - "os": { - "build": "19043.1288", - "kernel": "10.0.19041.1288 (WinBuild.160101.0800)", - "name": "Windows 10 Enterprise Evaluation", - "family": "windows", - "type": "windows", - "version": "10.0", - "platform": "windows" - }, + "id": "abe598f7-ff01-46dc-999b-d1751abbe398", "ip": [ "fe80::d567:3328:3f4c:113b", "10.0.2.15", "fe80::74fb:1698:31ac:f597", "192.168.56.103" ], - "name": "WinDev2108Eval", - "id": "abe598f7-ff01-46dc-999b-d1751abbe398", "mac": [ "08-00-27-DE-7B-66", "08-00-27-6A-C1-FB" ], - "architecture": "x86_64" - }, - "event": { - "agent_id_status": "verified", - "duration": 0, - "ingested": "2021-10-29T02:09:42Z", - "code": "33205", - "provider": "MSSQLSERVER$AUDIT", - "created": "2021-10-29T02:09:41.540Z", - "kind": "event", - "action": "login-succeeded", - "category": [ - "database", - "session" - ], - "type": [ - "info", - "start" - ], - "dataset": "microsoft_sqlserver.audit", - "outcome": "success" - }, - "user": { - "name": "SQLTELEMETRY", - "domain": "NT SERVICE", - "id": "010600000000000550000000447a1a9ee0235381234a54aa9bd0549c4fcc0642" - } - }, - { - "agent": { "name": "WinDev2108Eval", - "hostname": "WinDev2108Eval", - "id": "df0dd5ff-cce7-4861-b49f-fd70f0b207b6", - "ephemeral_id": "968f4b33-dfd4-455e-af64-ef3d7c3e7daf", - "type": "filebeat", - "version": "7.15.1" + "os": { + "build": "19043.1288", + "family": "windows", + "kernel": "10.0.19041.1288 (WinBuild.160101.0800)", + "name": "Windows 10 Enterprise Evaluation", + "platform": "windows", + "type": "windows", + "version": "10.0" + } + }, + "log": { + "level": "information" }, "process": { "pid": 656, "thread": { - "id": 708 + "id": 1688 + } + }, + "sqlserver": { + "audit": { + "action_id": "LOGIN SUCCEEDED", + "affected_rows": 0, + "application_name": "SQLServerCEIP", + "audit_schema_version": "1", + "class_type": "LOGIN", + "client_ip": "local machine", + "connection_id": "FADD287E-2D7D-4D1C-9C51-6A9AA7AF461E", + "database_principal_id": "0", + "event_time": "2021-10-29 02:09:40.4984112", + "host_name": "WINDEV2108EVAL", + "is_column_permission": false, + "object_id": "0", + "permission_bitmask": "00000000000000000000000000000000", + "response_rows": 0, + "sequence_group_id": "DBDB0CF9-276D-4291-9A39-6FDA3895176C", + "sequence_number": 1, + "server_instance_name": "WINDEV2108EVAL", + "server_principal_id": "265", + "session_id": "64", + "session_server_principal_name": "NT Service\\SQLTELEMETRY", + "statement": "-- network protocol: LPC\nset quoted_identifier on\nset arithabort off\nset numeric_roundabort off\nset ansi_warnings on\nset ansi_padding on\nset ansi_nulls on\nset concat_null_yields_null on\nset cursor_close_on_commit off\nset implicit_transactions off\nset language us_english\nset dateformat mdy\nset datefirst 7\nset transaction isolation level read committed\n", + "succeeded": true, + "target_database_principal_id": "0", + "target_server_principal_id": "0", + "transaction_id": "0", + "user_defined_event_id": "0" } }, + "user": { + "domain": "NT SERVICE", + "id": "010600000000000550000000447a1a9ee0235381234a54aa9bd0549c4fcc0642", + "name": "SQLTELEMETRY" + }, "winlog": { + "activity_id": "{3622691f-ccfb-0000-9469-2236fbccd701}", + "api": "wineventlog", + "channel": "Security", "computer_name": "WinDev2108Eval", - "record_id": 28002, "event_id": "33205", "keywords": [ - "Audit Failure", + "Audit Success", "Classic" ], - "activity_id": "{3622691f-ccfb-0000-9469-2236fbccd701}", - "channel": "Security", - "api": "wineventlog", "provider_name": "MSSQLSERVER$AUDIT", + "record_id": 27810, "user": { - "name": "MSSQLSERVER", + "domain": "NT SERVICE", "identifier": "S-1-5-80-3880718306-3832830129-1677859214-2598158968-1052248003", - "type": "Well Known Group", - "domain": "NT SERVICE" + "name": "MSSQLSERVER", + "type": "Well Known Group" } + } + }, + { + "@timestamp": "2021-10-29T04:54:20.589Z", + "agent": { + "ephemeral_id": "968f4b33-dfd4-455e-af64-ef3d7c3e7daf", + "hostname": "WinDev2108Eval", + "id": "df0dd5ff-cce7-4861-b49f-fd70f0b207b6", + "name": "WinDev2108Eval", + "type": "filebeat", + "version": "7.15.1" }, - "log": { - "level": "information" - }, - "elastic_agent": { - "version": "7.15.1", - "snapshot": false, - "id": "df0dd5ff-cce7-4861-b49f-fd70f0b207b6" + "data_stream": { + "dataset": "microsoft_sqlserver.audit", + "namespace": "default", + "type": "logs" }, - "@timestamp": "2021-10-29T04:54:20.589Z", "ecs": { "version": "1.12.0" }, - "sqlserver": { - "audit": { - "target_database_principal_id": "0", - "is_column_permission": false, - "sequence_group_id": "04B96B8E-5013-4F30-9C51-CEEDC009CBAA", - "permission_bitmask": "00000000000000000000000000000000", - "class_type": "LOGIN", - "application_name": "SQLCMD", - "action_id": "LOGIN FAILED", - "audit_schema_version": "1", - "statement": "Login failed for user 'User'. Reason: Could not find a login matching the name provided. [CLIENT: \u003clocal machine\u003e]", - "client_ip": "local machine", - "database_principal_id": "0", - "transaction_id": "0", - "target_server_principal_id": "0", - "server_principal_id": "0", - "response_rows": 0, - "session_id": "0", - "affected_rows": 0, - "object_id": "0", - "server_instance_name": "WINDEV2108EVAL", - "sequence_number": 1, - "connection_id": "5BB7AFC4-A4B5-46CF-B961-55426EAE3968", - "event_time": "2021-10-29 04:54:20.5894484", - "user_defined_event_id": "0", - "host_name": "WINDEV2108EVAL", - "succeeded": false - } + "elastic_agent": { + "id": "df0dd5ff-cce7-4861-b49f-fd70f0b207b6", + "snapshot": false, + "version": "7.15.1" }, - "data_stream": { - "namespace": "default", - "type": "logs", - "dataset": "microsoft_sqlserver.audit" + "event": { + "action": "login-failed", + "agent_id_status": "verified", + "category": [ + "database", + "authentication" + ], + "code": "33205", + "created": "2021-10-29T04:54:22.763Z", + "dataset": "microsoft_sqlserver.audit", + "duration": 0, + "ingested": "2021-10-29T04:54:24Z", + "kind": "event", + "outcome": "failure", + "provider": "MSSQLSERVER$AUDIT", + "type": [ + "error" + ] }, "host": { + "architecture": "x86_64", "hostname": "WinDev2108Eval", - "os": { - "build": "19043.1288", - "kernel": "10.0.19041.1288 (WinBuild.160101.0800)", - "name": "Windows 10 Enterprise Evaluation", - "family": "windows", - "type": "windows", - "version": "10.0", - "platform": "windows" - }, + "id": "abe598f7-ff01-46dc-999b-d1751abbe398", "ip": [ "fe80::d567:3328:3f4c:113b", "10.0.2.15", "fe80::74fb:1698:31ac:f597", "192.168.56.103" ], - "name": "WinDev2108Eval", - "id": "abe598f7-ff01-46dc-999b-d1751abbe398", "mac": [ "08-00-27-DE-7B-66", "08-00-27-6A-C1-FB" ], - "architecture": "x86_64" + "name": "WinDev2108Eval", + "os": { + "build": "19043.1288", + "family": "windows", + "kernel": "10.0.19041.1288 (WinBuild.160101.0800)", + "name": "Windows 10 Enterprise Evaluation", + "platform": "windows", + "type": "windows", + "version": "10.0" + } }, - "event": { - "agent_id_status": "verified", - "duration": 0, - "ingested": "2021-10-29T04:54:24Z", - "code": "33205", - "provider": "MSSQLSERVER$AUDIT", - "created": "2021-10-29T04:54:22.763Z", - "kind": "event", - "action": "login-failed", - "category": [ - "database", - "authentication" - ], - "type": [ - "error" - ], - "dataset": "microsoft_sqlserver.audit", - "outcome": "failure" + "log": { + "level": "information" + }, + "process": { + "pid": 656, + "thread": { + "id": 708 + } + }, + "sqlserver": { + "audit": { + "action_id": "LOGIN FAILED", + "affected_rows": 0, + "application_name": "SQLCMD", + "audit_schema_version": "1", + "class_type": "LOGIN", + "client_ip": "local machine", + "connection_id": "5BB7AFC4-A4B5-46CF-B961-55426EAE3968", + "database_principal_id": "0", + "event_time": "2021-10-29 04:54:20.5894484", + "host_name": "WINDEV2108EVAL", + "is_column_permission": false, + "object_id": "0", + "permission_bitmask": "00000000000000000000000000000000", + "response_rows": 0, + "sequence_group_id": "04B96B8E-5013-4F30-9C51-CEEDC009CBAA", + "sequence_number": 1, + "server_instance_name": "WINDEV2108EVAL", + "server_principal_id": "0", + "session_id": "0", + "statement": "Login failed for user 'User'. Reason: Could not find a login matching the name provided. [CLIENT: \u003clocal machine\u003e]", + "succeeded": false, + "target_database_principal_id": "0", + "target_server_principal_id": "0", + "transaction_id": "0", + "user_defined_event_id": "0" + } }, "user": { "name": "User" + }, + "winlog": { + "activity_id": "{3622691f-ccfb-0000-9469-2236fbccd701}", + "api": "wineventlog", + "channel": "Security", + "computer_name": "WinDev2108Eval", + "event_id": "33205", + "keywords": [ + "Audit Failure", + "Classic" + ], + "provider_name": "MSSQLSERVER$AUDIT", + "record_id": 28002, + "user": { + "domain": "NT SERVICE", + "identifier": "S-1-5-80-3880718306-3832830129-1677859214-2598158968-1052248003", + "name": "MSSQLSERVER", + "type": "Well Known Group" + } } } ] diff --git a/packages/microsoft_sqlserver/data_stream/audit/agent/stream/winlog.yml.hbs b/packages/microsoft_sqlserver/data_stream/audit/agent/stream/winlog.yml.hbs index c495f232125..ed053196d19 100644 --- a/packages/microsoft_sqlserver/data_stream/audit/agent/stream/winlog.yml.hbs +++ b/packages/microsoft_sqlserver/data_stream/audit/agent/stream/winlog.yml.hbs @@ -1,21 +1,22 @@ name: {{channel}} condition: ${host.platform} == 'windows' event_id: {{event_id}} +{{#if ignore_older}} +ignore_older: {{ignore_older}} +{{/if}} {{#if language}} language: {{language}} {{/if}} {{#if tags.length}} tags: -{{else if preserve_original_event}} -tags: -{{/if}} {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{/if}} {{#if preserve_original_event}} - - preserve_original_event +include_xml: true {{/if}} -{{#if processors}} +{{#if processors.length}} processors: {{processors}} {{/if}} diff --git a/packages/microsoft_sqlserver/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/microsoft_sqlserver/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index 2f4c126988f..ae08b8d34b3 100644 --- a/packages/microsoft_sqlserver/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/microsoft_sqlserver/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -1243,11 +1243,6 @@ processors: - sqlserver.audit.target_server_principal_name - sqlserver.audit.target_server_principal_sid ignore_missing: true -- remove: - field: event.original - if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true - script: lang: painless description: This script processor iterates over the whole document to remove fields with null values. diff --git a/packages/microsoft_sqlserver/data_stream/audit/manifest.yml b/packages/microsoft_sqlserver/data_stream/audit/manifest.yml index e0bbb1624e1..da4c3838e16 100644 --- a/packages/microsoft_sqlserver/data_stream/audit/manifest.yml +++ b/packages/microsoft_sqlserver/data_stream/audit/manifest.yml @@ -6,6 +6,15 @@ streams: description: Collect SQL Server audit events from the Windows event logs template_path: winlog.yml.hbs vars: + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: >- + Preserves a raw copy of the original XML event, added to the field `event.original` + type: bool + multi: false + default: false - name: event_id type: text title: Event ID @@ -23,6 +32,14 @@ streams: required: true default: Security show_user: true + - name: ignore_older + type: text + title: Ignore events older than + default: 72h + required: false + show_user: false + description: >- + If this option is specified, events that are older than the specified amount of time are ignored. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". - name: language type: text title: Language ID @@ -31,14 +48,11 @@ streams: required: false show_user: false default: 0 - - name: preserve_original_event - required: true - show_user: true - title: Preserve original event - description: Preserves a raw copy of the original event, added to the field `event.original` - type: bool - multi: false - default: false + - name: tags + type: text + title: Tags + multi: true + show_user: false - name: processors type: yaml title: Processors @@ -46,8 +60,3 @@ streams: required: false show_user: false description: "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. \nThis executes in the agent before the logs are parsed. \nSee [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n" - - name: tags - type: text - title: Tags - multi: true - show_user: false diff --git a/packages/microsoft_sqlserver/manifest.yml b/packages/microsoft_sqlserver/manifest.yml index 9484bb02b65..2fb7c09657d 100644 --- a/packages/microsoft_sqlserver/manifest.yml +++ b/packages/microsoft_sqlserver/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: microsoft_sqlserver title: "Microsoft SQL Server" -version: 0.2.0 +version: 0.3.0 license: basic description: Collect audit events from Microsoft SQL Server with Elastic Agent. type: integration diff --git a/packages/system/changelog.yml b/packages/system/changelog.yml index e801d0100a5..7a6d117d231 100644 --- a/packages/system/changelog.yml +++ b/packages/system/changelog.yml @@ -1,4 +1,15 @@ # newer versions go on top +- version: "1.10.0" + changes: + - description: Expose winlog input ignore_older option. + type: enhancement + link: https://github.com/elastic/integrations/pull/2542 + - description: Fix preserve original event option + type: bugfix + link: https://github.com/elastic/integrations/pull/2542 + - description: Make order of Security, Application, System options consistent with other winlog based integrations. + type: enhancement + link: https://github.com/elastic/integrations/pull/2542 - version: "1.9.0" changes: - description: Update to ECS 8.0 diff --git a/packages/system/data_stream/application/agent/stream/winlog.yml.hbs b/packages/system/data_stream/application/agent/stream/winlog.yml.hbs index 80695ab6792..ca336f1198d 100644 --- a/packages/system/data_stream/application/agent/stream/winlog.yml.hbs +++ b/packages/system/data_stream/application/agent/stream/winlog.yml.hbs @@ -1,20 +1,24 @@ name: Application condition: ${host.platform} == 'windows' -ignore_older: 72h {{#if event_id}} event_id: {{event_id}} {{/if}} +{{#if ignore_older}} +ignore_older: {{ignore_older}} +{{/if}} {{#if language}} language: {{language}} {{/if}} -{{#if processors}} +{{#if preserve_original_event}} +include_xml: true +{{/if}} +{{#if processors.length}} processors: {{processors}} {{/if}} +{{#if tags.length}} tags: -{{#if preserve_original_event}} - - preserve_original_event -{{/if}} {{#each tags as |tag i|}} - {{tag}} -{{/each}} \ No newline at end of file +{{/each}} +{{/if}} diff --git a/packages/system/data_stream/application/elasticsearch/ingest_pipeline/default.yml b/packages/system/data_stream/application/elasticsearch/ingest_pipeline/default.yml index 2cc46718dcc..7d7aa4443ec 100644 --- a/packages/system/data_stream/application/elasticsearch/ingest_pipeline/default.yml +++ b/packages/system/data_stream/application/elasticsearch/ingest_pipeline/default.yml @@ -7,11 +7,6 @@ processors: - set: field: ecs.version value: 8.0.0 - - remove: - field: event.original - if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true on_failure: - set: field: "error.message" diff --git a/packages/system/data_stream/application/manifest.yml b/packages/system/data_stream/application/manifest.yml index 35e432df6eb..cafd7ca00dd 100644 --- a/packages/system/data_stream/application/manifest.yml +++ b/packages/system/data_stream/application/manifest.yml @@ -7,6 +7,15 @@ streams: title: Application description: 'Collect Windows application logs' vars: + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: >- + Preserves a raw copy of the original XML event, added to the field `event.original` + type: bool + multi: false + default: false - name: event_id type: text title: Event ID @@ -15,19 +24,14 @@ streams: show_user: false description: >- A list of included and excluded (blocked) event IDs. The value is a comma-separated list. The accepted values are single event IDs to include (e.g. 4624), a range of event IDs to include (e.g. 4700-4800), and single event IDs to exclude (e.g. -4735). Limit 22 IDs. - - name: processors - type: yaml - title: Processors - multi: false + - name: ignore_older + type: text + title: Ignore events older than + default: 72h required: false show_user: false description: >- - Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. - - name: tags - type: text - title: Tags - multi: true - show_user: false + If this option is specified, events that are older than the specified amount of time are ignored. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". - name: language type: text title: Language ID @@ -36,6 +40,19 @@ streams: required: false show_user: false default: 0 + - name: tags + type: text + title: Tags + multi: true + show_user: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: >- + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. - input: httpjson title: Windows Application Events via Splunk Enterprise REST API description: Collect Application Events via Splunk Enterprise REST API diff --git a/packages/system/data_stream/security/agent/stream/winlog.yml.hbs b/packages/system/data_stream/security/agent/stream/winlog.yml.hbs index 49434256ef0..0b2f99c8eba 100644 --- a/packages/system/data_stream/security/agent/stream/winlog.yml.hbs +++ b/packages/system/data_stream/security/agent/stream/winlog.yml.hbs @@ -3,17 +3,22 @@ condition: ${host.platform} == 'windows' {{#if event_id}} event_id: {{event_id}} {{/if}} +{{#if ignore_older}} +ignore_older: {{ignore_older}} +{{/if}} {{#if language}} language: {{language}} {{/if}} -{{#if processors}} -processors: -{{processors}} -{{/if}} +{{#if tags.length}} tags: -{{#if preserve_original_event}} - - preserve_original_event -{{/if}} {{#each tags as |tag i|}} - {{tag}} -{{/each}} \ No newline at end of file +{{/each}} +{{/if}} +{{#if preserve_original_event}} +include_xml: true +{{/if}} +{{#if processors.length}} +processors: +{{processors}} +{{/if}} diff --git a/packages/system/data_stream/security/elasticsearch/ingest_pipeline/default.yml b/packages/system/data_stream/security/elasticsearch/ingest_pipeline/default.yml index 5ded5d81c2a..fd88f3d3d58 100644 --- a/packages/system/data_stream/security/elasticsearch/ingest_pipeline/default.yml +++ b/packages/system/data_stream/security/elasticsearch/ingest_pipeline/default.yml @@ -46,11 +46,6 @@ processors: - ISO8601 ignore_failure: true if: ctx?.winlog?.time_created != null - - remove: - field: event.original - if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/system/data_stream/security/manifest.yml b/packages/system/data_stream/security/manifest.yml index 0fcb4463248..373ecfeffa3 100644 --- a/packages/system/data_stream/security/manifest.yml +++ b/packages/system/data_stream/security/manifest.yml @@ -7,6 +7,15 @@ streams: title: Security description: 'Security channel' vars: + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: >- + Preserves a raw copy of the original XML event, added to the field `event.original` + type: bool + multi: false + default: false - name: event_id type: text title: Event ID @@ -15,19 +24,14 @@ streams: show_user: false description: >- A list of included and excluded (blocked) event IDs. The value is a comma-separated list. The accepted values are single event IDs to include (e.g. 4624), a range of event IDs to include (e.g. 4700-4800), and single event IDs to exclude (e.g. -4735). Limit 22 IDs. - - name: processors - type: yaml - title: Processors - multi: false + - name: ignore_older + type: text + title: Ignore events older than + default: 72h required: false show_user: false description: >- - Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. - - name: tags - type: text - title: Tags - multi: true - show_user: false + If this option is specified, events that are older than the specified amount of time are ignored. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". - name: language type: text title: Language ID @@ -36,6 +40,19 @@ streams: required: false show_user: false default: 0 + - name: tags + type: text + title: Tags + multi: true + show_user: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: >- + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. - input: httpjson title: Windows Security Events via Splunk Enterprise REST API description: Collect Security Events via Splunk Enterprise REST API diff --git a/packages/system/data_stream/system/agent/stream/winlog.yml.hbs b/packages/system/data_stream/system/agent/stream/winlog.yml.hbs index b5db7808dc3..927c959680d 100644 --- a/packages/system/data_stream/system/agent/stream/winlog.yml.hbs +++ b/packages/system/data_stream/system/agent/stream/winlog.yml.hbs @@ -3,17 +3,22 @@ condition: ${host.platform} == 'windows' {{#if event_id}} event_id: {{event_id}} {{/if}} +{{#if ignore_older}} +ignore_older: {{ignore_older}} +{{/if}} {{#if language}} language: {{language}} {{/if}} -{{#if processors}} -processors: -{{processors}} -{{/if}} +{{#if tags.length}} tags: -{{#if preserve_original_event}} - - preserve_original_event -{{/if}} {{#each tags as |tag i|}} - {{tag}} -{{/each}} \ No newline at end of file +{{/each}} +{{/if}} +{{#if preserve_original_event}} +include_xml: true +{{/if}} +{{#if processors.length}} +processors: +{{processors}} +{{/if}} \ No newline at end of file diff --git a/packages/system/data_stream/system/elasticsearch/ingest_pipeline/default.yml b/packages/system/data_stream/system/elasticsearch/ingest_pipeline/default.yml index f25f3218099..9172454fd66 100644 --- a/packages/system/data_stream/system/elasticsearch/ingest_pipeline/default.yml +++ b/packages/system/data_stream/system/elasticsearch/ingest_pipeline/default.yml @@ -7,11 +7,6 @@ processors: - set: field: ecs.version value: 8.0.0 - - remove: - field: event.original - if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true on_failure: - set: field: "error.message" diff --git a/packages/system/data_stream/system/manifest.yml b/packages/system/data_stream/system/manifest.yml index 14e4948477b..106b961fbd9 100644 --- a/packages/system/data_stream/system/manifest.yml +++ b/packages/system/data_stream/system/manifest.yml @@ -7,6 +7,15 @@ streams: title: System description: 'Collect Windows system logs' vars: + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: >- + Preserves a raw copy of the original XML event, added to the field `event.original` + type: bool + multi: false + default: false - name: event_id type: text title: Event ID @@ -15,19 +24,14 @@ streams: show_user: false description: >- A list of included and excluded (blocked) event IDs. The value is a comma-separated list. The accepted values are single event IDs to include (e.g. 4624), a range of event IDs to include (e.g. 4700-4800), and single event IDs to exclude (e.g. -4735). Limit 22 IDs. - - name: processors - type: yaml - title: Processors - multi: false + - name: ignore_older + type: text + title: Ignore events older than + default: 72h required: false show_user: false description: >- - Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. - - name: tags - type: text - title: Tags - multi: true - show_user: false + If this option is specified, events that are older than the specified amount of time are ignored. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". - name: language type: text title: Language ID @@ -36,6 +40,19 @@ streams: required: false show_user: false default: 0 + - name: tags + type: text + title: Tags + multi: true + show_user: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: >- + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. - input: httpjson title: Windows System Events via Splunk Enterprise REST API description: Collect System Events via Splunk Enterprise REST API diff --git a/packages/system/manifest.yml b/packages/system/manifest.yml index 776eef4b994..868de640756 100644 --- a/packages/system/manifest.yml +++ b/packages/system/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: system title: System -version: 1.9.0 +version: 1.10.0 license: basic description: Collect system logs and metrics from your servers with Elastic Agent. type: integration @@ -36,15 +36,6 @@ policy_templates: - type: winlog title: 'Collect events from the Windows event log' description: 'Collecting events from Windows event log' - vars: - - name: preserve_original_event - required: true - show_user: true - title: Preserve original event - description: Preserves a raw copy of the original event, added to the field `event.original` - type: bool - multi: false - default: false - type: system/metrics title: Collect metrics from System instances description: Collecting System core, CPU, diskio, entropy, filesystem, fsstat, load, memory, network, Network Summary, process, Process Summary, raid, service, socket, Socket Summary, uptime and users metrics diff --git a/packages/windows/changelog.yml b/packages/windows/changelog.yml index bd4db3579ca..b9463cbdec5 100644 --- a/packages/windows/changelog.yml +++ b/packages/windows/changelog.yml @@ -1,9 +1,20 @@ +# newer versions go on top +- version: "1.9.0" + changes: + - description: Expose winlog input ignore_older option. + type: enhancement + link: https://github.com/elastic/integrations/pull/2542 + - description: Fix preserve original event option + type: bugfix + link: https://github.com/elastic/integrations/pull/2542 + - description: Make order of options consistent with other winlog based integrations. + type: enhancement + link: https://github.com/elastic/integrations/pull/2542 - version: "1.8.0" changes: - description: Update to ECS 8.0 type: enhancement link: https://github.com/elastic/integrations/pull/2515 -# newer versions go on top - version: "1.7.0" changes: - description: Add provider name check to forwarded/security conditional. diff --git a/packages/windows/data_stream/forwarded/agent/stream/winlog.yml.hbs b/packages/windows/data_stream/forwarded/agent/stream/winlog.yml.hbs index ee8b1ce5ed4..965be31d600 100644 --- a/packages/windows/data_stream/forwarded/agent/stream/winlog.yml.hbs +++ b/packages/windows/data_stream/forwarded/agent/stream/winlog.yml.hbs @@ -3,19 +3,20 @@ condition: ${host.platform} == 'windows' {{#if event_id}} event_id: {{event_id}} {{/if}} +{{#if ignore_older}} +ignore_older: {{ignore_older}} +{{/if}} {{#if language}} language: {{language}} {{/if}} {{#if tags.length}} tags: -{{else if preserve_original_event}} -tags: -{{/if}} {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{/if}} {{#if preserve_original_event}} - - preserve_original_event +include_xml: true {{/if}} {{#contains "forwarded" tags}} publisher_pipeline.disable_host: true diff --git a/packages/windows/data_stream/forwarded/elasticsearch/ingest_pipeline/security.yml b/packages/windows/data_stream/forwarded/elasticsearch/ingest_pipeline/security.yml index 7b5137c3245..a01dcde9ed6 100644 --- a/packages/windows/data_stream/forwarded/elasticsearch/ingest_pipeline/security.yml +++ b/packages/windows/data_stream/forwarded/elasticsearch/ingest_pipeline/security.yml @@ -3169,12 +3169,6 @@ processors: ignore_failure: true if: ctx?.winlog?.time_created != null - - remove: - field: event.original - if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true - on_failure: - set: field: error.message diff --git a/packages/windows/data_stream/forwarded/manifest.yml b/packages/windows/data_stream/forwarded/manifest.yml index 6ae30fde364..b5ebb051ea7 100644 --- a/packages/windows/data_stream/forwarded/manifest.yml +++ b/packages/windows/data_stream/forwarded/manifest.yml @@ -14,13 +14,30 @@ streams: title: Forwarded description: 'Collect ForwardedEvents channel logs' vars: + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: >- + Preserves a raw copy of the original XML event, added to the field `event.original` + type: bool + multi: false + default: false - name: event_id type: text title: Event ID description: >- A list of included and excluded (blocked) event IDs. The value is a comma-separated list. The accepted values are single event IDs to include (e.g. 4624), a range of event IDs to include (e.g. 4700-4800), and single event IDs to exclude (e.g. -4735). Limit 22 IDs. - required: true + required: false + show_user: false + - name: ignore_older + type: text + title: Ignore events older than + default: 72h + required: false show_user: false + description: >- + If this option is specified, events that are older than the specified amount of time are ignored. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". - name: language type: text title: Language ID @@ -36,14 +53,6 @@ streams: show_user: false default: - forwarded - - name: preserve_original_event - required: true - show_user: true - title: Preserve original event - description: Preserves a raw copy of the original event, added to the field `event.original` - type: bool - multi: false - default: false - name: processors type: yaml title: Processors diff --git a/packages/windows/data_stream/powershell/agent/stream/winlog.yml.hbs b/packages/windows/data_stream/powershell/agent/stream/winlog.yml.hbs index 48eb09600fb..8695fa23005 100644 --- a/packages/windows/data_stream/powershell/agent/stream/winlog.yml.hbs +++ b/packages/windows/data_stream/powershell/agent/stream/winlog.yml.hbs @@ -3,19 +3,20 @@ condition: ${host.platform} == 'windows' {{#if event_id}} event_id: {{event_id}} {{/if}} +{{#if ignore_older}} +ignore_older: {{ignore_older}} +{{/if}} {{#if language}} language: {{language}} {{/if}} {{#if tags.length}} tags: -{{else if preserve_original_event}} -tags: -{{/if}} {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{/if}} {{#if preserve_original_event}} - - preserve_original_event +include_xml: true {{/if}} {{#if processors.length}} processors: diff --git a/packages/windows/data_stream/powershell/manifest.yml b/packages/windows/data_stream/powershell/manifest.yml index 32d9f792c36..7b712964ee7 100644 --- a/packages/windows/data_stream/powershell/manifest.yml +++ b/packages/windows/data_stream/powershell/manifest.yml @@ -14,6 +14,15 @@ streams: title: Powershell description: 'Windows Powershell channel' vars: + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: >- + Preserves a raw copy of the original XML event, added to the field `event.original` + type: bool + multi: false + default: false - name: event_id type: text title: Event ID @@ -22,6 +31,14 @@ streams: required: true show_user: false default: 400, 403, 600, 800 + - name: ignore_older + type: text + title: Ignore events older than + default: 72h + required: false + show_user: false + description: >- + If this option is specified, events that are older than the specified amount of time are ignored. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". - name: language type: text title: Language ID @@ -35,14 +52,6 @@ streams: title: Tags multi: true show_user: false - - name: preserve_original_event - required: true - show_user: true - title: Preserve original event - description: Preserves a raw copy of the original event, added to the field `event.original` - type: bool - multi: false - default: false - name: processors type: yaml title: Processors diff --git a/packages/windows/data_stream/powershell_operational/agent/stream/winlog.yml.hbs b/packages/windows/data_stream/powershell_operational/agent/stream/winlog.yml.hbs index b09aa847ca5..55799473ece 100644 --- a/packages/windows/data_stream/powershell_operational/agent/stream/winlog.yml.hbs +++ b/packages/windows/data_stream/powershell_operational/agent/stream/winlog.yml.hbs @@ -3,19 +3,20 @@ condition: ${host.platform} == 'windows' {{#if event_id}} event_id: {{event_id}} {{/if}} +{{#if ignore_older}} +ignore_older: {{ignore_older}} +{{/if}} {{#if language}} language: {{language}} {{/if}} {{#if tags.length}} tags: -{{else if preserve_original_event}} -tags: -{{/if}} {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{/if}} {{#if preserve_original_event}} - - preserve_original_event +include_xml: true {{/if}} {{#if processors.length}} processors: diff --git a/packages/windows/data_stream/powershell_operational/manifest.yml b/packages/windows/data_stream/powershell_operational/manifest.yml index 249ed34242a..5b3a0c3ad6f 100644 --- a/packages/windows/data_stream/powershell_operational/manifest.yml +++ b/packages/windows/data_stream/powershell_operational/manifest.yml @@ -14,6 +14,15 @@ streams: title: Powershell Operational description: 'Microsoft-Windows-Powershell/Operational channel' vars: + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: >- + Preserves a raw copy of the original XML event, added to the field `event.original` + type: bool + multi: false + default: false - name: event_id type: text title: Event ID @@ -22,6 +31,14 @@ streams: required: true show_user: false default: 4103, 4104, 4105, 4106 + - name: ignore_older + type: text + title: Ignore events older than + default: 72h + required: false + show_user: false + description: >- + If this option is specified, events that are older than the specified amount of time are ignored. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". - name: language type: text title: Language ID @@ -35,14 +52,6 @@ streams: title: Tags multi: true show_user: false - - name: preserve_original_event - required: true - show_user: true - title: Preserve original event - description: Preserves a raw copy of the original event, added to the field `event.original` - type: bool - multi: false - default: false - name: processors type: yaml title: Processors diff --git a/packages/windows/data_stream/sysmon_operational/agent/stream/winlog.yml.hbs b/packages/windows/data_stream/sysmon_operational/agent/stream/winlog.yml.hbs index a4b06e1f178..7795afb123d 100644 --- a/packages/windows/data_stream/sysmon_operational/agent/stream/winlog.yml.hbs +++ b/packages/windows/data_stream/sysmon_operational/agent/stream/winlog.yml.hbs @@ -3,19 +3,20 @@ condition: ${host.platform} == 'windows' {{#if event_id}} event_id: {{event_id}} {{/if}} +{{#if ignore_older}} +ignore_older: {{ignore_older}} +{{/if}} {{#if language}} language: {{language}} {{/if}} {{#if tags.length}} tags: -{{else if preserve_original_event}} -tags: -{{/if}} {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{/if}} {{#if preserve_original_event}} - - preserve_original_event +include_xml: true {{/if}} {{#if processors.length}} processors: diff --git a/packages/windows/data_stream/sysmon_operational/manifest.yml b/packages/windows/data_stream/sysmon_operational/manifest.yml index 4c70cff0ec5..24eb2f3039c 100644 --- a/packages/windows/data_stream/sysmon_operational/manifest.yml +++ b/packages/windows/data_stream/sysmon_operational/manifest.yml @@ -6,13 +6,30 @@ streams: title: Sysmon Operational description: 'Collect Microsoft-Windows-Sysmon/Operational channel logs' vars: + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: >- + Preserves a raw copy of the original XML event, added to the field `event.original` + type: bool + multi: false + default: false - name: event_id type: text title: Event ID description: >- A list of included and excluded (blocked) event IDs. The value is a comma-separated list. The accepted values are single event IDs to include (e.g. 4624), a range of event IDs to include (e.g. 4700-4800), and single event IDs to exclude (e.g. -4735). Limit 22 IDs. - required: true + required: false + show_user: false + - name: ignore_older + type: text + title: Ignore events older than + default: 72h + required: false show_user: false + description: >- + If this option is specified, events that are older than the specified amount of time are ignored. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". - name: language type: text title: Language ID @@ -26,14 +43,6 @@ streams: title: Tags multi: true show_user: false - - name: preserve_original_event - required: true - show_user: true - title: Preserve original event - description: Preserves a raw copy of the original event, added to the field `event.original` - type: bool - multi: false - default: false - name: processors type: yaml title: Processors diff --git a/packages/windows/manifest.yml b/packages/windows/manifest.yml index efd2635da11..e5b9fe2cf65 100644 --- a/packages/windows/manifest.yml +++ b/packages/windows/manifest.yml @@ -1,6 +1,6 @@ name: windows title: Windows -version: 1.8.0 +version: 1.9.0 description: Collect logs and metrics from Windows OS and services with Elastic Agent. type: integration categories: diff --git a/packages/winlog/changelog.yml b/packages/winlog/changelog.yml index 91f0cb65f92..93fa85cff7b 100644 --- a/packages/winlog/changelog.yml +++ b/packages/winlog/changelog.yml @@ -1,4 +1,15 @@ # newer versions go on top +- version: "1.4.0" + changes: + - description: Expose winlog input ignore_older option. + type: enhancement + link: https://github.com/elastic/integrations/pull/2542 + - description: Fix preserve original event option + type: bugfix + link: https://github.com/elastic/integrations/pull/2542 + - description: Make order of options consistent with other winlog based integrations. + type: enhancement + link: https://github.com/elastic/integrations/pull/2542 - version: "1.3.0" changes: - description: Expose winlog input language option. diff --git a/packages/winlog/data_stream/winlog/agent/stream/winlog.yml.hbs b/packages/winlog/data_stream/winlog/agent/stream/winlog.yml.hbs index c0a23d6ae49..0284cea2d08 100644 --- a/packages/winlog/data_stream/winlog/agent/stream/winlog.yml.hbs +++ b/packages/winlog/data_stream/winlog/agent/stream/winlog.yml.hbs @@ -2,14 +2,22 @@ condition: ${host.platform} == 'windows' data_stream: dataset: {{data_stream.dataset}} name: {{channel}} +{{#if preserve_original_event}} +include_xml: true +{{/if}} {{#if event_id}} event_id: {{event_id}} {{/if}} +{{#if ignore_older}} +ignore_older: {{ignore_older}} +{{/if}} {{#if language}} language: {{language}} {{/if}} +{{#if tags.length}} tags: -{{#each tags}} - - {{this}} +{{#each tags as |tag i|}} + - {{tag}} {{/each}} +{{/if}} {{custom}} \ No newline at end of file diff --git a/packages/winlog/data_stream/winlog/manifest.yml b/packages/winlog/data_stream/winlog/manifest.yml index a2f53e03a2f..6b9b4669d7b 100644 --- a/packages/winlog/data_stream/winlog/manifest.yml +++ b/packages/winlog/data_stream/winlog/manifest.yml @@ -21,6 +21,38 @@ streams: default: winlog.winlog required: true show_user: true + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: >- + Preserves a raw copy of the original XML event, added to the field `event.original` + type: bool + multi: false + default: false + - name: event_id + type: text + title: Event ID + description: >- + A list of included and excluded (blocked) event IDs. The value is a comma-separated list. The accepted values are single event IDs to include (e.g. 4624), a range of event IDs to include (e.g. 4700-4800), and single event IDs to exclude (e.g. -4735). Limit 22 IDs. + required: false + show_user: false + - name: ignore_older + type: text + title: Ignore events older than + default: 72h + required: false + show_user: false + description: >- + If this option is specified, events that are older than the specified amount of time are ignored. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". + - name: language + type: text + title: Language ID + description: >- + The language ID the events will be rendered in. The language will be forced regardless of the system language. A complete list of language IDs can be found [here](https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-lcid/a9eac961-e77d-41a6-90a5-ce1a8b0cdb9c). It defaults to `0`, which indicates to use the system language. E.g.: `0x0409` for `en-US` + required: false + show_user: false + default: 0 - name: tags type: text title: Tags @@ -41,21 +73,6 @@ streams: # - drop_event.when.not.or: # - equals.winlog.event_id: '903' # - equals.winlog.event_id: '1024' - - name: event_id - type: text - title: Event ID - description: >- - A list of included and excluded (blocked) event IDs. The value is a comma-separated list. The accepted values are single event IDs to include (e.g. 4624), a range of event IDs to include (e.g. 4700-4800), and single event IDs to exclude (e.g. -4735). Limit 22 IDs. - required: true - show_user: false - - name: language - type: text - title: Language ID - description: >- - The language ID the events will be rendered in. The language will be forced regardless of the system language. A complete list of language IDs can be found [here](https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-lcid/a9eac961-e77d-41a6-90a5-ce1a8b0cdb9c). It defaults to `0`, which indicates to use the system language. E.g.: `0x0409` for `en-US` - required: false - show_user: false - default: 0 - input: httpjson title: Windows ForwardedEvents via Splunk Enterprise REST API description: Collect ForwardedEvents via Splunk Enterprise REST API diff --git a/packages/winlog/manifest.yml b/packages/winlog/manifest.yml index 395a83ebbac..32c6341ccc4 100644 --- a/packages/winlog/manifest.yml +++ b/packages/winlog/manifest.yml @@ -3,7 +3,7 @@ name: winlog title: Custom Windows Event Logs description: Collect and parse logs from any Windows event log channel with Elastic Agent. type: integration -version: 1.3.0 +version: 1.4.0 release: ga conditions: kibana.version: '^7.16.0 || ^8.0.0'