[ECS] Winlogbeat ecs 1.8 changes (elastic#23563)

* User enhancements for powershell module * User enhancements for security and sysmon module * Add registry category to events * Add session category to events * Set target group when possible
leweafan · Feb 3, 2021 · 18b1268 · 18b1268
1 parent 358941d
commit 18b1268
Show file tree

Hide file tree

Showing 46 changed files with 307 additions and 57 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -1003,6 +1003,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add dns.question.subdomain fields for sysmon DNS events. {pull}22999[22999]
 - Add dns.question.top_level_domain fields for sysmon DNS events. {pull}23046[23046]
 - Add Audit and Authentication Polixy Change Events and related.ip information {pull}20684[20684]
+- Add new ECS 1.8 improvements. {pull}23563[23563]
 
 *Elastic Log Driver*
 

diff --git a/x-pack/winlogbeat/module/powershell/config/winlogbeat-powershell.js b/x-pack/winlogbeat/module/powershell/config/winlogbeat-powershell.js
@@ -333,11 +333,9 @@ var powershell = (function () {
         var userParts = evt.Get("winlog.event_data.UserId").split("\\");
         evt.Delete("winlog.event_data.UserId");
         if (userParts.length === 2) {
-            evt.Delete("user");
             evt.Put("user.domain", userParts[0]);
             evt.Put("user.name", userParts[1]);
             evt.AppendTo("related.user", userParts[1]);
-            evt.Delete("winlog.event_data.UserId");
         }
     };
 
@@ -346,7 +344,18 @@ var powershell = (function () {
         evt.Delete("winlog.event_data.Connected User");
         if (userParts.length === 2) {
             evt.Put("powershell.connected_user.domain", userParts[0]);
+            if (evt.Get("user.domain")) {
+                evt.Put("destination.user.domain", evt.Get("user.domain"));
+            }
+            evt.Put("source.user.domain", userParts[0]);
+            evt.Put("user.domain", userParts[0]);
+
             evt.Put("powershell.connected_user.name", userParts[1]);
+            if (evt.Get("user.name")) {
+                evt.Put("destination.user.name", evt.Get("user.name"));
+            }
+            evt.Put("source.user.name", userParts[1]);
+            evt.Put("user.name", userParts[1]);
             evt.AppendTo("related.user", userParts[1]);
         }
     };
@@ -541,6 +550,18 @@ var powershell = (function () {
             ignore_missing: true,
             fail_on_error: false,
         })
+        .Convert({
+            fields: [
+                {
+                    from: "winlog.user.identifier",
+                    to: "user.id",
+                    type: "string",
+                },
+            ],
+            mode: "copy",
+            ignore_missing: true,
+            fail_on_error: false,
+        })
         .Add(normalizeCommonFieldNames)
         .Add(addEngineVersion)
         .Add(addPipelineID)
@@ -583,6 +604,18 @@ var powershell = (function () {
             ignore_missing: true,
             fail_on_error: false,
         })
+        .Convert({
+            fields: [
+                {
+                    from: "winlog.user.identifier",
+                    to: "user.id",
+                    type: "string",
+                },
+            ],
+            mode: "copy",
+            ignore_missing: true,
+            fail_on_error: false,
+        })
         .Add(normalizeCommonFieldNames)
         .Add(addFileInfo)
         .Add(addScriptBlockID)
@@ -594,6 +627,18 @@ var powershell = (function () {
         .Add(addRunspaceID)
         .Add(addScriptBlockID)
         .Add(removeEmptyEventData)
+        .Convert({
+            fields: [
+                {
+                    from: "winlog.user.identifier",
+                    to: "user.id",
+                    type: "string",
+                },
+            ],
+            mode: "copy",
+            ignore_missing: true,
+            fail_on_error: false,
+        })
         .Build();
 
     var event4105 = new processor.Chain()

diff --git a/x-pack/winlogbeat/module/powershell/test/testdata/4103.evtx.golden.json b/x-pack/winlogbeat/module/powershell/test/testdata/4103.evtx.golden.json
@@ -1,6 +1,12 @@
 [
   {
     "@timestamp": "2020-05-15T08:11:47.8979495Z",
+    "destination": {
+      "user": {
+        "domain": "VAGRANT",
+        "name": "vagrant"
+      }
+    },
     "event": {
       "action": "Executing Pipeline",
       "category": [
@@ -72,8 +78,15 @@
     "related": {
       "user": "vagrant"
     },
+    "source": {
+      "user": {
+        "domain": "VAGRANT",
+        "name": "vagrant"
+      }
+    },
     "user": {
       "domain": "VAGRANT",
+      "id": "S-1-5-21-1350058589-2282154016-2764056528-1000",
       "name": "vagrant"
     },
     "winlog": {
@@ -196,6 +209,7 @@
     },
     "user": {
       "domain": "VAGRANT",
+      "id": "S-1-5-21-1350058589-2282154016-2764056528-1000",
       "name": "vagrant"
     },
     "winlog": {

diff --git a/x-pack/winlogbeat/module/powershell/test/testdata/4104.evtx.golden.json b/x-pack/winlogbeat/module/powershell/test/testdata/4104.evtx.golden.json
@@ -28,6 +28,9 @@
       "sequence": 1,
       "total": 1
     },
+    "user": {
+      "id": "S-1-5-21-1350058589-2282154016-2764056528-1000"
+    },
     "winlog": {
       "activity_id": "{fb13c9de-29f7-0001-18e0-13fbf729d601}",
       "api": "wineventlog",
@@ -85,6 +88,9 @@
       "sequence": 1,
       "total": 1
     },
+    "user": {
+      "id": "S-1-5-21-1350058589-2282154016-2764056528-1000"
+    },
     "winlog": {
       "activity_id": "{fb13c9de-29f7-0000-79db-13fbf729d601}",
       "api": "wineventlog",

diff --git a/x-pack/winlogbeat/module/powershell/test/testdata/4105.evtx.golden.json b/x-pack/winlogbeat/module/powershell/test/testdata/4105.evtx.golden.json
@@ -26,6 +26,9 @@
       },
       "runspace_id": "9c031e5c-8d5a-4b91-a12e-b3624970b623"
     },
+    "user": {
+      "id": "S-1-5-21-1350058589-2282154016-2764056528-1000"
+    },
     "winlog": {
       "activity_id": "{dd68516a-2930-0000-5962-68dd3029d601}",
       "api": "wineventlog",

diff --git a/x-pack/winlogbeat/module/powershell/test/testdata/4106.evtx.golden.json b/x-pack/winlogbeat/module/powershell/test/testdata/4106.evtx.golden.json
@@ -26,6 +26,9 @@
       },
       "runspace_id": "3f1a9181-0523-4645-a42c-2c1868c39332"
     },
+    "user": {
+      "id": "S-1-5-21-1350058589-2282154016-2764056528-1000"
+    },
     "winlog": {
       "activity_id": "{e3200b8a-290e-0002-332a-20e30e29d601}",
       "api": "wineventlog",

diff --git a/x-pack/winlogbeat/module/security/config/winlogbeat-security.js b/x-pack/winlogbeat/module/security/config/winlogbeat-security.js
@@ -179,7 +179,7 @@ var security = (function () {
         "4634": [["authentication"], ["end"], "logged-out"],
         "4647": [["authentication"], ["end"], "logged-out"],
         "4648": [["authentication"], ["start"], "logged-in-explicit"],
-        "4657": [["configuration"], ["change"], "registry-value-modified"],
+        "4657": [["registry", "configuration"], ["change"], "registry-value-modified"],
         "4670": [["iam", "configuration"],["admin", "change"],"permissions-changed"],
         "4672": [["iam"], ["admin"], "logged-in-special"],
         "4673": [["iam"], ["admin"], "privileged-service-called"],
@@ -250,8 +250,8 @@ var security = (function () {
         "4770": [["authentication"], ["start"], "kerberos-service-ticket-renewed"],
         "4771": [["authentication"], ["start"], "kerberos-preauth-failed"],
         "4776": [["authentication"], ["start"], "credential-validated"],
-        "4778": [["authentication"], ["start"], "session-reconnected"],
-        "4779": [["authentication"], ["end"], "session-disconnected"],
+        "4778": [["authentication", "session"], ["start"], "session-reconnected"],
+        "4779": [["authentication", "session"], ["end"], "session-disconnected"],
         "4781": [["iam"], ["user", "change"], "renamed-user-account"],
         "4798": [["iam"], ["user", "info"], "group-membership-enumerated"], // process enumerates the local groups to which the specified user belongs
         "4799": [["iam"], ["group", "info"], "user-member-enumerated"], // a process enumerates the members of the specified local group
@@ -1351,7 +1351,7 @@ var security = (function () {
         "16903": "Publish",
     };
 
-    // Trust Types 
+    // Trust Types
     // https://docs.microsoft.com/en-us/windows/security/threat-protection/auditing/event-4706
     var trustTypes = {
         "1": "TRUST_TYPE_DOWNLEVEL",
@@ -1360,7 +1360,7 @@ var security = (function () {
         "4": "TRUST_TYPE_DCE"
     }
 
-    // Trust Direction 
+    // Trust Direction
     // https://docs.microsoft.com/en-us/windows/security/threat-protection/auditing/event-4706
     var trustDirection = {
         "0": "TRUST_DIRECTION_DISABLED",
@@ -1369,7 +1369,7 @@ var security = (function () {
         "3": "TRUST_DIRECTION_BIDIRECTIONAL"
     }
 
-    // Trust Attributes 
+    // Trust Attributes
     // https://docs.microsoft.com/en-us/windows/security/threat-protection/auditing/event-4706
     var trustAttributes = {
         "0": "UNDEFINED",
@@ -1899,35 +1899,58 @@ var security = (function () {
 
         })
         .Build();
-    var copyTargetUser = new processor.Chain()
-        .Convert({
-            fields: [
-                {from: "winlog.event_data.TargetUserSid", to: "user.id"},
-                {from: "winlog.event_data.TargetUserName", to: "user.name"},
-                {from: "winlog.event_data.TargetDomainName", to: "user.domain"},
-            ],
-            ignore_missing: true,
-        })
-        .Add(function(evt) {
-            var user = evt.Get("winlog.event_data.TargetUserName");
-            if (user) {
-                if (/.@*/.test(user)) {
-                    user = user.split('@')[0];
-                    evt.Put('user.name', user);
-                }
-                evt.AppendTo('related.user', user);
+
+    var copyTargetUser = function(evt) {
+        var targetUserId = evt.Get("winlog.event_data.TargetUserSid");
+        if (targetUserId) {
+            if (evt.Get("user.id")) evt.Put("user.target.id", targetUserId);
+            else evt.Put("user.id", targetUserId);
+        }
+
+        var targetUserName = evt.Get("winlog.event_data.TargetUserName");
+        if (targetUserName) {
+            if (/.@*/.test(targetUserName)) {
+                targetUserName = targetUserName.split('@')[0];
             }
-        })
-        .Build();
+
+            evt.AppendTo("related.user", targetUserName);
+            if (evt.Get("user.name")) evt.Put("user.target.name", targetUserName);
+            else evt.Put("user.name", targetUserName);
+        }
+
+        var targetUserDomain = evt.Get("winlog.event_data.TargetDomainName");
+        if (targetUserDomain) {
+            if (evt.Get("user.domain")) evt.Put("user.target.domain", targetUserDomain);
+            else evt.Put("user.domain", targetUserDomain);
+        }
+    }
+
+    var copyMemberToUser = function(evt) {
+        var member = evt.Get("winlog.event_data.MemberName");
+        if (!member) {
+            return;
+        }
+
+        var userName = member.split(',')[0].replace('CN=', '').replace('cn=', '');
+
+        evt.AppendTo("related.user", userName);
+        evt.Put("user.target.name", userName);
+    }
 
     var copyTargetUserToGroup = new processor.Chain()
         .Convert({
             fields: [
                 {from: "winlog.event_data.TargetUserSid", to: "group.id"},
+                {from: "winlog.event_data.TargetSid", to: "group.id"},
                 {from: "winlog.event_data.TargetUserName", to: "group.name"},
                 {from: "winlog.event_data.TargetDomainName", to: "group.domain"},
             ],
             ignore_missing: true,
+        }).Add(function(evt) {
+            if (!evt.Get("user.target")) return;
+            evt.Put("user.target.group.id", evt.Get("group.id"));
+            evt.Put("user.target.group.name", evt.Get("group.name"));
+            evt.Put("user.target.group.domain", evt.Get("group.domain"));
         })
         .Build();
 
@@ -2194,16 +2217,10 @@ var security = (function () {
     var groupMgmtEvts = new processor.Chain()
         .Add(copySubjectUser)
         .Add(copySubjectUserLogonId)
+        .Add(copyMemberToUser)
         .Add(copyTargetUserToGroup)
         .Add(renameCommonAuthFields)
         .Add(addEventFields)
-        .Add(function(evt) {
-            var member = evt.Get("winlog.event_data.MemberName");
-            if (!member) {
-                return;
-            }
-            evt.AppendTo("related.user", member.split(',')[0].replace('CN=', '').replace('cn=', ''));
-        })
         .Build();
 
     var auditLogCleared = new processor.Chain()

diff --git a/x-pack/winlogbeat/module/security/test/testdata/4744.evtx.golden.json b/x-pack/winlogbeat/module/security/test/testdata/4744.evtx.golden.json
@@ -18,6 +18,7 @@
     },
     "group": {
       "domain": "TEST",
+      "id": "S-1-5-21-1717121054-434620538-60925301-2903",
       "name": "testdistlocal"
     },
     "host": {

diff --git a/x-pack/winlogbeat/module/security/test/testdata/4745.evtx.golden.json b/x-pack/winlogbeat/module/security/test/testdata/4745.evtx.golden.json
@@ -18,6 +18,7 @@
     },
     "group": {
       "domain": "TEST",
+      "id": "S-1-5-21-1717121054-434620538-60925301-2903",
       "name": "testdistlocal1"
     },
     "host": {

diff --git a/x-pack/winlogbeat/module/security/test/testdata/4746.evtx.golden.json b/x-pack/winlogbeat/module/security/test/testdata/4746.evtx.golden.json
@@ -18,6 +18,7 @@
     },
     "group": {
       "domain": "TEST",
+      "id": "S-1-5-21-1717121054-434620538-60925301-2903",
       "name": "testdistlocal1"
     },
     "host": {
@@ -35,7 +36,15 @@
     "user": {
       "domain": "TEST",
       "id": "S-1-5-21-1717121054-434620538-60925301-2794",
-      "name": "at_adm"
+      "name": "at_adm",
+      "target": {
+        "group": {
+          "domain": "TEST",
+          "id": "S-1-5-21-1717121054-434620538-60925301-2903",
+          "name": "testdistlocal1"
+        },
+        "name": "Administrator"
+      }
     },
     "winlog": {
       "api": "wineventlog",

diff --git a/x-pack/winlogbeat/module/security/test/testdata/4747.evtx.golden.json b/x-pack/winlogbeat/module/security/test/testdata/4747.evtx.golden.json
@@ -18,6 +18,7 @@
     },
     "group": {
       "domain": "TEST",
+      "id": "S-1-5-21-1717121054-434620538-60925301-2903",
       "name": "testdistlocal1"
     },
     "host": {
@@ -35,7 +36,15 @@
     "user": {
       "domain": "TEST",
       "id": "S-1-5-21-1717121054-434620538-60925301-2794",
-      "name": "at_adm"
+      "name": "at_adm",
+      "target": {
+        "group": {
+          "domain": "TEST",
+          "id": "S-1-5-21-1717121054-434620538-60925301-2903",
+          "name": "testdistlocal1"
+        },
+        "name": "Administrator"
+      }
     },
     "winlog": {
       "api": "wineventlog",

diff --git a/x-pack/winlogbeat/module/security/test/testdata/4748.evtx.golden.json b/x-pack/winlogbeat/module/security/test/testdata/4748.evtx.golden.json
@@ -18,6 +18,7 @@
     },
     "group": {
       "domain": "TEST",
+      "id": "S-1-5-21-1717121054-434620538-60925301-2903",
       "name": "testdistlocal1"
     },
     "host": {

diff --git a/x-pack/winlogbeat/module/security/test/testdata/4749.evtx.golden.json b/x-pack/winlogbeat/module/security/test/testdata/4749.evtx.golden.json
@@ -18,6 +18,7 @@
     },
     "group": {
       "domain": "TEST",
+      "id": "S-1-5-21-1717121054-434620538-60925301-2904",
       "name": "testglobal"
     },
     "host": {