Cherry-pick #15731 to 7.x: Set event.outcome based on googlecloud audit log output

CHANGELOG.next.asciidoc

@@ -153,6 +153,10 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - New fileset googlecloud/firewall for ingesting Google Cloud Firewall logs. {pull}14553[14553]
 - google-pubsub input: ACK pub/sub message when acknowledged by publisher. {issue}13346[13346] {pull}14715[14715]
 - Remove Beta label from google-pubsub input. {issue}13346[13346] {pull}14715[14715]
+- Set event.outcome field based on googlecloud audit log output. {pull}15731[15731]
+
+*Heartbeat*
+
 - Allow a list of status codes for HTTP checks. {pull}15587[15587]
 
 

x-pack/filebeat/module/googlecloud/audit/config/pipeline.js

@@ -118,8 +118,30 @@ function Audit(keep_original_message) {
     var RenameNestedFields = function(evt) {
         var arr = evt.Get("googlecloud.audit.authorization_info");
         for (var i = 0; i < arr.length; i++) {
-          arr[i].resource_attributes = arr[i].resourceAttributes;
-          delete arr[i].resourceAttributes;
+            arr[i].resource_attributes = arr[i].resourceAttributes;
+            delete arr[i].resourceAttributes;
+        }
+    };
+
+    // Set event.outcome based on authentication_info and status.
+    var setEventOutcome = function(evt) {
+        if (evt.Get("googlecloud.audit.status.code") == null) {
+            var authorization_info = evt.Get("googlecloud.audit.authorization_info");
+            if (authorization_info.length === 1) {
+                if (authorization_info[0].granted == null) {
+                    evt.Put("event.outcome", "unknown");
+                } else if (authorization_info[0].granted === true) {
+                    evt.Put("event.outcome", "success");
+                } else {
+                    evt.Put("event.outcome", "failure");
+                }
+            } else {
+                evt.Put("event.outcome", "unknown");
+            } 
+        } else if (evt.Get("googlecloud.audit.status.code") === 0) {
+            evt.Put("event.outcome", "success");
+        } else {
+            evt.Put("event.outcome", "failure");
         }
     };
 
@@ -135,6 +157,7 @@ function Audit(keep_original_message) {
         .Add(copyFields)
         .Add(dropExtraFields)
         .Add(RenameNestedFields)
+        .Add(setEventOutcome)
         .Build();
 
     return {

x-pack/filebeat/module/googlecloud/audit/test/audit-log-entries.json.log

@@ -1,3 +1,4 @@
 {"insertId":"-uihnmjctwo","logName":"projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access","protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"xxx@xxx.xxx"},"authorizationInfo":[{"granted":true,"permission":"resourcemanager.projects.get","resource":"projects/elastic-beats","resourceAttributes":{}}],"methodName":"GetResourceBillingInfo","request":{"@type":"type.googleapis.com/google.internal.cloudbilling.billingaccount.v1.GetResourceBillingInfoRequest","resourceName":"projects/189716325846"},"requestMetadata":{"callerIp":"192.168.1.1","destinationAttributes":{},"requestAttributes":{}},"resourceName":"projects/elastic-beats","serviceName":"cloudbilling.googleapis.com","status":{}},"receiveTimestamp":"2019-12-19T00:49:36.313482371Z","resource":{"labels":{"project_id":"elastic-beats"},"type":"project"},"severity":"INFO","timestamp":"2019-12-19T00:49:36.086Z"}
-{"insertId":"-h6onuze1h7dg","logName":"projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access","protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"xxx@xxx.xxx"},"authorizationInfo":[{"granted":true,"permission":"compute.machineTypes.list","resourceAttributes":{"name":"projects/elastic-beats","service":"resourcemanager","type":"resourcemanager.projects"}}],"methodName":"beta.compute.machineTypes.aggregatedList","numResponseItems":"71","request":{"@type":"type.googleapis.com/compute.machineTypes.aggregatedList"},"requestMetadata":{"callerIp":"192.168.1.1","callerSuppliedUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:71.0) Gecko/20100101 Firefox/71.0,gzip(gfe),gzip(gfe)","destinationAttributes":{},"requestAttributes":{"auth":{},"time":"2019-12-19T00:45:51.711Z"}},"resourceLocation":{"currentLocations":["global"]},"resourceName":"projects/elastic-beats/global/machineTypes","serviceName":"compute.googleapis.com"},"receiveTimestamp":"2019-12-19T00:45:52.367887078Z","resource":{"labels":{"location":"global","method":"compute.machineTypes.aggregatedList","project_id":"elastic-beats","service":"compute.googleapis.com","version":"beta"},"type":"api"},"severity":"INFO","timestamp":"2019-12-19T00:45:51.228Z"}
+{"insertId":"-h6onuze1h7dg","logName":"projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access","protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"xxx@xxx.xxx"},"authorizationInfo":[{"granted":false,"permission":"compute.machineTypes.list","resourceAttributes":{"name":"projects/elastic-beats","service":"resourcemanager","type":"resourcemanager.projects"}}],"methodName":"beta.compute.machineTypes.aggregatedList","numResponseItems":"71","request":{"@type":"type.googleapis.com/compute.machineTypes.aggregatedList"},"requestMetadata":{"callerIp":"192.168.1.1","callerSuppliedUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:71.0) Gecko/20100101 Firefox/71.0,gzip(gfe),gzip(gfe)","destinationAttributes":{},"requestAttributes":{"auth":{},"time":"2019-12-19T00:45:51.711Z"}},"resourceLocation":{"currentLocations":["global"]},"resourceName":"projects/elastic-beats/global/machineTypes","serviceName":"compute.googleapis.com"},"receiveTimestamp":"2019-12-19T00:45:52.367887078Z","resource":{"labels":{"location":"global","method":"compute.machineTypes.aggregatedList","project_id":"elastic-beats","service":"compute.googleapis.com","version":"beta"},"type":"api"},"severity":"INFO","timestamp":"2019-12-19T00:45:51.228Z"}
 {"insertId":"yonau2dg2zi","logName":"projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access","protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"xxx@xxx.xxx"},"authorizationInfo":[{"granted":true,"permission":"compute.instances.list","resourceAttributes":{"name":"projects/elastic-beats","service":"resourcemanager","type":"resourcemanager.projects"}}],"methodName":"beta.compute.instances.aggregatedList","numResponseItems":"61","request":{"@type":"type.googleapis.com/compute.instances.aggregatedList"},"requestMetadata":{"callerIp":"192.168.1.1","callerSuppliedUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:71.0) Gecko/20100101 Firefox/71.0,gzip(gfe),gzip(gfe)","destinationAttributes":{},"requestAttributes":{"auth":{},"time":"2019-12-19T00:44:25.198Z"}},"resourceLocation":{"currentLocations":["global"]},"resourceName":"projects/elastic-beats/global/instances","serviceName":"compute.googleapis.com"},"receiveTimestamp":"2019-12-19T00:44:25.262379373Z","resource":{"labels":{"location":"global","method":"compute.instances.aggregatedList","project_id":"elastic-beats","service":"compute.googleapis.com","version":"beta"},"type":"api"},"severity":"INFO","timestamp":"2019-12-19T00:44:25.051Z"}
+{"insertId":"yonau3dc2zi","logName":"projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access","protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"xxx@xxx.xxx"},"authorizationInfo":[{"permission":"compute.instances.list","resourceAttributes":{"name":"projects/elastic-beats","service":"resourcemanager","type":"resourcemanager.projects"}}],"methodName":"beta.compute.instances.aggregatedList","numResponseItems":"61","request":{"@type":"type.googleapis.com/compute.instances.aggregatedList"},"requestMetadata":{"callerIp":"192.168.1.1","callerSuppliedUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:71.0) Gecko/20100101 Firefox/71.0,gzip(gfe),gzip(gfe)","destinationAttributes":{},"requestAttributes":{"auth":{},"time":"2019-12-19T00:44:25.198Z"}},"resourceLocation":{"currentLocations":["global"]},"resourceName":"projects/elastic-beats/global/instances","serviceName":"compute.googleapis.com","status":{"code":7,"message":"PERMISSION_DENIED"}},"receiveTimestamp":"2019-12-19T00:44:25.262379373Z","resource":{"labels":{"location":"global","method":"compute.instances.aggregatedList","project_id":"elastic-beats","service":"compute.googleapis.com","version":"beta"},"type":"api"},"severity":"INFO","timestamp":"2019-12-19T00:44:25.051Z"}

x-pack/filebeat/module/googlecloud/audit/test/audit-log-entries.json.log-expected.json

@@ -4,6 +4,7 @@
         "cloud.project.id": "elastic-beats",
         "event.dataset": "googlecloud.audit",
         "event.module": "googlecloud",
+        "event.outcome": "success",
         "fileset.name": "audit",
         "googlecloud.audit.authentication_info.principal_email": "xxx@xxx.xxx",
         "googlecloud.audit.authorization_info": [
@@ -34,11 +35,12 @@
         "cloud.project.id": "elastic-beats",
         "event.dataset": "googlecloud.audit",
         "event.module": "googlecloud",
+        "event.outcome": "failure",
         "fileset.name": "audit",
         "googlecloud.audit.authentication_info.principal_email": "xxx@xxx.xxx",
         "googlecloud.audit.authorization_info": [
             {
-                "granted": true,
+                "granted": false,
                 "permission": "compute.machineTypes.list",
                 "resource_attributes": {
                     "name": "projects/elastic-beats",
@@ -78,6 +80,7 @@
         "cloud.project.id": "elastic-beats",
         "event.dataset": "googlecloud.audit",
         "event.module": "googlecloud",
+        "event.outcome": "success",
         "fileset.name": "audit",
         "googlecloud.audit.authentication_info.principal_email": "xxx@xxx.xxx",
         "googlecloud.audit.authorization_info": [
@@ -104,7 +107,53 @@
         "googlecloud.audit.type": "type.googleapis.com/google.cloud.audit.AuditLog",
         "input.type": "log",
         "log.logger": "projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access",
-        "log.offset": 2251,
+        "log.offset": 2252,
+        "service.name": "compute.googleapis.com",
+        "service.type": "googlecloud",
+        "source.ip": "192.168.1.1",
+        "user.email": "xxx@xxx.xxx",
+        "user_agent.device.name": "Other",
+        "user_agent.name": "Firefox",
+        "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:71.0) Gecko/20100101 Firefox/71.0,gzip(gfe),gzip(gfe)",
+        "user_agent.os.full": "Mac OS X 10.15",
+        "user_agent.os.name": "Mac OS X",
+        "user_agent.os.version": "10.15",
+        "user_agent.version": "71.0."
+    },
+    {
+        "@timestamp": "2019-12-19T00:44:25.051Z",
+        "cloud.project.id": "elastic-beats",
+        "event.dataset": "googlecloud.audit",
+        "event.module": "googlecloud",
+        "event.outcome": "failure",
+        "fileset.name": "audit",
+        "googlecloud.audit.authentication_info.principal_email": "xxx@xxx.xxx",
+        "googlecloud.audit.authorization_info": [
+            {
+                "permission": "compute.instances.list",
+                "resource_attributes": {
+                    "name": "projects/elastic-beats",
+                    "service": "resourcemanager",
+                    "type": "resourcemanager.projects"
+                }
+            }
+        ],
+        "googlecloud.audit.method_name": "beta.compute.instances.aggregatedList",
+        "googlecloud.audit.num_response_items": 61,
+        "googlecloud.audit.request.proto_name": "type.googleapis.com/compute.instances.aggregatedList",
+        "googlecloud.audit.request_metadata.caller_ip": "192.168.1.1",
+        "googlecloud.audit.request_metadata.caller_supplied_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:71.0) Gecko/20100101 Firefox/71.0,gzip(gfe),gzip(gfe)",
+        "googlecloud.audit.resource_location.current_locations": [
+            "global"
+        ],
+        "googlecloud.audit.resource_name": "projects/elastic-beats/global/instances",
+        "googlecloud.audit.service_name": "compute.googleapis.com",
+        "googlecloud.audit.status.code": 7,
+        "googlecloud.audit.status.message": "PERMISSION_DENIED",
+        "googlecloud.audit.type": "type.googleapis.com/google.cloud.audit.AuditLog",
+        "input.type": "log",
+        "log.logger": "projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access",
+        "log.offset": 3541,
         "service.name": "compute.googleapis.com",
         "service.type": "googlecloud",
         "source.ip": "192.168.1.1",