diff --git a/packages/gcp/changelog.yml b/packages/gcp/changelog.yml index f66bbc3ae1f..eacc3050fa0 100644 --- a/packages/gcp/changelog.yml +++ b/packages/gcp/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.9.0" + changes: + - description: Preserve request and response in flattened fields. + type: enhancement + link: https://github.com/elastic/integrations/pull/3390 - version: "1.8.0" changes: - description: Add missing `cloud.provider` field. diff --git a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-audit.log b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-audit.log index 11fb77acd22..9d5155ba651 100644 --- a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-audit.log +++ b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-audit.log @@ -10,4 +10,5 @@ {"insertId":"e973134d-b4d5-4e2f-92b8-82bba13fdb92","labels":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"system:public-info-viewer\" of ClusterRole \"system:public-info-viewer\" to Group \"system:unauthenticated\""},"logName":"projects/elastic-siem/logs/cloudaudit.googleapis.com%2Fdata_access","operation":{"first":true,"id":"e973134d-b4d5-4e2f-92b8-82bba13fdb92","last":true,"producer":"k8s.io"},"protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"system:anonymous"},"authorizationInfo":[{"granted":true,"permission":"io.k8s.get","resource":"readyz"}],"methodName":"io.k8s.get","requestMetadata":{"callerIp":"127.0.0.1","callerSuppliedUserAgent":"kube-probe/1.19+"},"resourceName":"readyz","serviceName":"k8s.io","status":{}},"receiveTimestamp":"2021-04-29T08:19:21.606980385Z","resource":{"labels":{"cluster_name":"analysis-cluster","location":"us-central1-a","project_id":"elastic-siem"},"type":"k8s_cluster"},"timestamp":"2021-04-29T08:19:20.80581Z"} {"insertId":"03adfb9f-71a3-4f41-9701-29b5542f4d22","labels":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"system:discovery\" of ClusterRole \"system:discovery\" to Group \"system:authenticated\""},"logName":"projects/elastic-siem/logs/cloudaudit.googleapis.com%2Fdata_access","operation":{"first":true,"id":"03adfb9f-71a3-4f41-9701-29b5542f4d22","last":true,"producer":"k8s.io"},"protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"system:serviceaccount:kube-system:generic-garbage-collector"},"authorizationInfo":[{"granted":true,"permission":"io.k8s.get","resource":"api/v1"}],"methodName":"io.k8s.get","requestMetadata":{"callerIp":"::1","callerSuppliedUserAgent":"kube-controller-manager/v1.19.8 (linux/amd64) kubernetes/4f6f69f/system:serviceaccount:kube-system:generic-garbage-collector"},"resourceName":"api/v1","serviceName":"k8s.io","status":{}},"receiveTimestamp":"2021-04-29T08:23:19.71757101Z","resource":{"labels":{"cluster_name":"analysis-cluster","location":"us-central1-a","project_id":"elastic-siem"},"type":"k8s_cluster"},"timestamp":"2021-04-29T08:23:18.899153Z"} {"insertId":"03adfb9f-71a3-4f41-9701-29b5542f4d23","logName":"projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access","protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"xxx@xxx.xxx","principalSubject":"sub","serviceAccountKeyName":"//xxx@xxx"},"authorizationInfo":[{"granted":true,"permission":"iam.serviceAccounts.list","resource":"projects/project","resourceAttributes":{}}],"methodName":"google.iam.admin.v1.ListServiceAccounts","request":{"@type":"type.googleapis.com/google.iam.admin.v1.ListServiceAccountsRequest","name":"projects/project","page_token":"cg:FFFFFF"},"requestMetadata":{"callerIp":"gce-internal-ip","callerSuppliedUserAgent":"google-api-go-client/0.5,gzip(gfe)","destinationAttributes":{},"requestAttributes":{"auth":{},"time":"2022-02-21T13:57:39.178418578Z"}},"resourceName":"projects/project","serviceName":"iam.googleapis.com","status":{}},"receiveTimestamp":"2022-02-21T13:57:39.341344991Z","resource":{"labels":{"location":"global","method":"google.iam.admin.v1.ListServiceAccounts","project_id":"project","service":"iam.googleapis.com","version":"v1"},"type":"api"},"severity":"INFO","timestamp":"2022-02-21T13:57:39.174555198Z"} -{"insertId":"03adfb9f-71a3-4f41-9701-29b5542f4d24","labels":{"authentication.k8s.io/legacy-token":"system:serviceaccount:kube-system:metrics-server","authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"metrics-server:system:auth-delegator\" of ClusterRole \"system:auth-delegator\" to ServiceAccount \"metrics-server/kube-system\"","k8s.io/deprecated":"true","k8s.io/removed-release":"1.22"},"logName":"projects/project","operation":{"first":true,"id":"924fbbf6-1982-4173-9355-3fca0ab7b0ee","last":true,"producer":"k8s.io"},"protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"xxx@xxx.xxx"},"authorizationInfo":[{"granted":true,"permission":"io.k8s.authorization.v1beta1.subjectaccessreviews.create","resource":"authorization.k8s.io/v1beta1/subjectaccessreviews"}],"methodName":"io.k8s.authorization.v1beta1.subjectaccessreviews.create","request":{"@type":"authorization.k8s.io/v1beta1.SubjectAccessReview","apiVersion":"authorization.k8s.io/v1beta1","kind":"SubjectAccessReview","metadata":{"creationTimestamp":null},"spec":{"group":["system:serviceaccounts","system:serviceaccounts:kube-system","system:authenticated"],"nonResourceAttributes":{"path":"/apis/metrics.k8s.io/v1beta1","verb":"get"},"user":"system:serviceaccount:kube-system:resourcequota-controller"},"status":{"allowed":false}},"requestMetadata":{"callerIp":"67.43.156.13","callerSuppliedUserAgent":"metrics-server/v0.0.0 (linux/amd64) kubernetes/$Format"},"resourceName":"authorization.k8s.io/v1beta1/subjectaccessreviews","response":{"@type":"authorization.k8s.io/v1beta1.SubjectAccessReview","apiVersion":"authorization.k8s.io/v1beta1","kind":"SubjectAccessReview","metadata":{"creationTimestamp":null,"managedFields":[{"apiVersion":"authorization.k8s.io/v1beta1","fieldsType":"FieldsV1","fieldsV1":{"f:spec":{"f:group":{},"f:nonResourceAttributes":{".":{},"f:path":{},"f:verb":{}},"f:user":{}}},"manager":"metrics-server","operation":"Update","time":"2022-02-21T14:00:40Z"}]},"spec":{"group":["system:serviceaccounts","system:serviceaccounts:kube-system","system:authenticated"],"nonResourceAttributes":{"path":"/apis/metrics.k8s.io/v1beta1","verb":"get"},"user":"system:serviceaccount:kube-system:resourcequota-controller"},"status":{"allowed":true,"reason":"RBAC: allowed by ClusterRoleBinding \"system:discovery\" of ClusterRole \"system:discovery\" to Group \"system:authenticated\""}},"serviceName":"k8s.io","status":{}},"receiveTimestamp":"2022-02-21T14:00:42.030209174Z","resource":{"labels":{"cluster_name":"elastic","location":"europe-west1","project_id":"project"},"type":"k8s_cluster"},"timestamp":"2022-02-21T14:00:40.802327Z"} \ No newline at end of file +{"insertId":"03adfb9f-71a3-4f41-9701-29b5542f4d24","labels":{"authentication.k8s.io/legacy-token":"system:serviceaccount:kube-system:metrics-server","authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"metrics-server:system:auth-delegator\" of ClusterRole \"system:auth-delegator\" to ServiceAccount \"metrics-server/kube-system\"","k8s.io/deprecated":"true","k8s.io/removed-release":"1.22"},"logName":"projects/project","operation":{"first":true,"id":"924fbbf6-1982-4173-9355-3fca0ab7b0ee","last":true,"producer":"k8s.io"},"protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"xxx@xxx.xxx"},"authorizationInfo":[{"granted":true,"permission":"io.k8s.authorization.v1beta1.subjectaccessreviews.create","resource":"authorization.k8s.io/v1beta1/subjectaccessreviews"}],"methodName":"io.k8s.authorization.v1beta1.subjectaccessreviews.create","request":{"@type":"authorization.k8s.io/v1beta1.SubjectAccessReview","apiVersion":"authorization.k8s.io/v1beta1","kind":"SubjectAccessReview","metadata":{"creationTimestamp":null},"spec":{"group":["system:serviceaccounts","system:serviceaccounts:kube-system","system:authenticated"],"nonResourceAttributes":{"path":"/apis/metrics.k8s.io/v1beta1","verb":"get"},"user":"system:serviceaccount:kube-system:resourcequota-controller"},"status":{"allowed":false}},"requestMetadata":{"callerIp":"67.43.156.13","callerSuppliedUserAgent":"metrics-server/v0.0.0 (linux/amd64) kubernetes/$Format"},"resourceName":"authorization.k8s.io/v1beta1/subjectaccessreviews","response":{"@type":"authorization.k8s.io/v1beta1.SubjectAccessReview","apiVersion":"authorization.k8s.io/v1beta1","kind":"SubjectAccessReview","metadata":{"creationTimestamp":null,"managedFields":[{"apiVersion":"authorization.k8s.io/v1beta1","fieldsType":"FieldsV1","fieldsV1":{"f:spec":{"f:group":{},"f:nonResourceAttributes":{".":{},"f:path":{},"f:verb":{}},"f:user":{}}},"manager":"metrics-server","operation":"Update","time":"2022-02-21T14:00:40Z"}]},"spec":{"group":["system:serviceaccounts","system:serviceaccounts:kube-system","system:authenticated"],"nonResourceAttributes":{"path":"/apis/metrics.k8s.io/v1beta1","verb":"get"},"user":"system:serviceaccount:kube-system:resourcequota-controller"},"status":{"allowed":true,"reason":"RBAC: allowed by ClusterRoleBinding \"system:discovery\" of ClusterRole \"system:discovery\" to Group \"system:authenticated\""}},"serviceName":"k8s.io","status":{}},"receiveTimestamp":"2022-02-21T14:00:42.030209174Z","resource":{"labels":{"cluster_name":"elastic","location":"europe-west1","project_id":"project"},"type":"k8s_cluster"},"timestamp":"2022-02-21T14:00:40.802327Z"} +{"insertId": "e5132c86-462b-41b3-9b6a-47966addbb0b","labels": {"authorization.k8s.io/decision": "allow","authorization.k8s.io/reason": ""},"logName": "projects/iammai-340819/logs/cloudaudit.googleapis.com%2Factivity","operation": {"first": true,"id": "e5132c86-462b-41b3-9b6a-47966addbb0b","last": true,"producer": "k8s.io"},"protoPayload": {"@type": "type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo": {"principalEmail": "system:addon-manager"},"authorizationInfo": [ { "granted": true, "permission": "io.k8s.apps.v1.deployments.patch", "resource": "apps/v1/namespaces/kube-system/deployments/konnectivity-agent" } ], "methodName": "io.k8s.apps.v1.deployments.patch", "request": { "@type": "k8s.io/Patch", "spec": { "strategy": { "$retainKeys": [ "type" ] }, "template": { "spec": { "$setElementOrder/volumes": [ { "name": "konnectivity-agent-token" } ], "volumes": [ { "$retainKeys": [ "name", "projected" ], "name": "konnectivity-agent-token", "projected": { "sources": [ { "serviceAccountToken": { "audience": "system:konnectivity-server", "path": "konnectivity-agent-token" } } ] } } ] } } } }, "requestMetadata": { "callerIp": "10.142.0.152", "callerSuppliedUserAgent": "kubectl/v1.20.2 (linux/amd64) kubernetes/faecb19" }, "resourceName": "apps/v1/namespaces/kube-system/deployments/konnectivity-agent", "response": { "@type": "apps.k8s.io/v1.Deployment", "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { "annotations": { "components.gke.io/layer": "addon", "deployment.kubernetes.io/revision": "1", "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"apps/v1\",\"kind\":\"Deployment\",\"metadata\":{\"annotations\":{\"components.gke.io/layer\":\"addon\"},\"labels\":{\"addonmanager.kubernetes.io/mode\":\"Reconcile\",\"k8s-app\":\"konnectivity-agent\"},\"name\":\"konnectivity-agent\",\"namespace\":\"kube-system\"},\"spec\":{\"selector\":{\"matchLabels\":{\"k8s-app\":\"konnectivity-agent\"}},\"strategy\":{\"type\":\"RollingUpdate\"},\"template\":{\"metadata\":{\"annotations\":{\"cluster-autoscaler.kubernetes.io/safe-to-evict\":\"true\",\"components.gke.io/component-name\":\"konnectivitynetworkproxy-combined\",\"components.gke.io/component-version\":\"1.3.3\"},\"labels\":{\"k8s-app\":\"konnectivity-agent\"}},\"spec\":{\"containers\":[{\"args\":[\"--logtostderr=true\",\"--ca-cert=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt\",\"--proxy-server-host=34.75.195.103\",\"--proxy-server-port=8132\",\"--health-server-port=8093\",\"--admin-server-port=8094\",\"--sync-interval=5s\",\"--probe-interval=5s\",\"--service-account-token-path=/var/run/secrets/tokens/konnectivity-agent-token\",\"--v=3\"],\"command\":[\"/proxy-agent\"],\"env\":[{\"name\":\"POD_NAME\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.name\"}}},{\"name\":\"POD_NAMESPACE\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.namespace\"}}}],\"image\":\"gke.gcr.io/proxy-agent:v0.0.24-gke.0\",\"livenessProbe\":{\"httpGet\":{\"path\":\"/healthz\",\"port\":8093},\"initialDelaySeconds\":15,\"timeoutSeconds\":15},\"name\":\"konnectivity-agent\",\"ports\":[{\"containerPort\":8093,\"name\":\"metrics\",\"protocol\":\"TCP\"}],\"resources\":{\"limits\":{\"memory\":\"125Mi\"},\"requests\":{\"cpu\":\"10m\",\"memory\":\"30Mi\"}},\"securityContext\":{\"allowPrivilegeEscalation\":false,\"capabilities\":{\"drop\":[\"all\"]}},\"volumeMounts\":[{\"mountPath\":\"/var/run/secrets/tokens\",\"name\":\"konnectivity-agent-token\"}]}],\"nodeSelector\":{\"beta.kubernetes.io/os\":\"linux\"},\"priorityClassName\":\"system-cluster-critical\",\"securityContext\":{\"fsGroup\":1000,\"runAsGroup\":1000,\"runAsUser\":1000},\"serviceAccountName\":\"konnectivity-agent\",\"tolerations\":[{\"key\":\"CriticalAddonsOnly\",\"operator\":\"Exists\"},{\"effect\":\"NoSchedule\",\"key\":\"sandbox.gke.io/runtime\",\"operator\":\"Equal\",\"value\":\"gvisor\"},{\"key\":\"components.gke.io/gke-managed-components\",\"operator\":\"Exists\"}],\"topologySpreadConstraints\":[{\"labelSelector\":{\"matchLabels\":{\"k8s-app\":\"konnectivity-agent\"}},\"maxSkew\":1,\"topologyKey\":\"topology.kubernetes.io/zone\",\"whenUnsatisfiable\":\"ScheduleAnyway\"},{\"labelSelector\":{\"matchLabels\":{\"k8s-app\":\"konnectivity-agent\"}},\"maxSkew\":1,\"topologyKey\":\"kubernetes.io/hostname\",\"whenUnsatisfiable\":\"ScheduleAnyway\"}],\"volumes\":[{\"name\":\"konnectivity-agent-token\",\"projected\":{\"sources\":[{\"serviceAccountToken\":{\"audience\":\"system:konnectivity-server\",\"path\":\"konnectivity-agent-token\"}}]}}]}}}}" }, "creationTimestamp": "2022-03-16T21:29:13Z", "generation": 2, "labels": { "addonmanager.kubernetes.io/mode": "Reconcile", "k8s-app": "konnectivity-agent" }, "managedFields": [ { "apiVersion": "apps/v1", "fieldsType": "FieldsV1", "fieldsV1": { "f:metadata": { "f:annotations": { ".": {}, "f:components.gke.io/layer": {}, "f:kubectl.kubernetes.io/last-applied-configuration": {} }, "f:labels": { ".": {}, "f:addonmanager.kubernetes.io/mode": {}, "f:k8s-app": {} } }, "f:spec": { "f:progressDeadlineSeconds": {}, "f:replicas": {}, "f:revisionHistoryLimit": {}, "f:selector": {}, "f:strategy": { "f:rollingUpdate": { ".": {}, "f:maxSurge": {}, "f:maxUnavailable": {} }, "f:type": {} }, "f:template": { "f:metadata": { "f:annotations": { ".": {}, "f:cluster-autoscaler.kubernetes.io/safe-to-evict": {}, "f:components.gke.io/component-name": {}, "f:components.gke.io/component-version": {} }, "f:labels": { ".": {}, "f:k8s-app": {} } }, "f:spec": { "f:containers": { "k:{\"name\":\"konnectivity-agent\"}": { ".": {}, "f:args": {}, "f:command": {}, "f:env": { ".": {}, "k:{\"name\":\"POD_NAME\"}": { ".": {}, "f:name": {}, "f:valueFrom": { ".": {}, "f:fieldRef": { ".": {}, "f:apiVersion": {}, "f:fieldPath": {} } } }, "k:{\"name\":\"POD_NAMESPACE\"}": { ".": {}, "f:name": {}, "f:valueFrom": { ".": {}, "f:fieldRef": { ".": {}, "f:apiVersion": {}, "f:fieldPath": {} } } } }, "f:image": {}, "f:imagePullPolicy": {}, "f:livenessProbe": { ".": {}, "f:failureThreshold": {}, "f:httpGet": { ".": {}, "f:path": {}, "f:port": {}, "f:scheme": {} }, "f:initialDelaySeconds": {}, "f:periodSeconds": {}, "f:successThreshold": {}, "f:timeoutSeconds": {} }, "f:name": {}, "f:ports": { ".": {}, "k:{\"containerPort\":8093,\"protocol\":\"TCP\"}": { ".": {}, "f:containerPort": {}, "f:name": {}, "f:protocol": {} } }, "f:resources": { ".": {}, "f:limits": { ".": {}, "f:memory": {} }, "f:requests": { ".": {}, "f:cpu": {}, "f:memory": {} } }, "f:securityContext": { ".": {}, "f:allowPrivilegeEscalation": {}, "f:capabilities": { ".": {}, "f:drop": {} } }, "f:terminationMessagePath": {}, "f:terminationMessagePolicy": {}, "f:volumeMounts": { ".": {}, "k:{\"mountPath\":\"/var/run/secrets/tokens\"}": { ".": {}, "f:mountPath": {}, "f:name": {} } } } }, "f:dnsPolicy": {}, "f:nodeSelector": { ".": {}, "f:beta.kubernetes.io/os": {} }, "f:priorityClassName": {}, "f:restartPolicy": {}, "f:schedulerName": {}, "f:securityContext": { ".": {}, "f:fsGroup": {}, "f:runAsGroup": {}, "f:runAsUser": {} }, "f:serviceAccount": {}, "f:serviceAccountName": {}, "f:terminationGracePeriodSeconds": {}, "f:tolerations": {}, "f:topologySpreadConstraints": { ".": {}, "k:{\"topologyKey\":\"kubernetes.io/hostname\",\"whenUnsatisfiable\":\"ScheduleAnyway\"}": { ".": {}, "f:labelSelector": {}, "f:maxSkew": {}, "f:topologyKey": {}, "f:whenUnsatisfiable": {} }, "k:{\"topologyKey\":\"topology.kubernetes.io/zone\",\"whenUnsatisfiable\":\"ScheduleAnyway\"}": { ".": {}, "f:labelSelector": {}, "f:maxSkew": {}, "f:topologyKey": {}, "f:whenUnsatisfiable": {} } }, "f:volumes": { ".": {}, "k:{\"name\":\"konnectivity-agent-token\"}": { ".": {}, "f:name": {}, "f:projected": { ".": {}, "f:defaultMode": {}, "f:sources": {} } } } } } } }, "manager": "kubectl-client-side-apply", "operation": "Update", "time": "2022-03-16T21:29:13Z" }, { "apiVersion": "apps/v1", "fieldsType": "FieldsV1", "fieldsV1": { "f:metadata": { "f:annotations": { "f:deployment.kubernetes.io/revision": {} } }, "f:status": { "f:availableReplicas": {}, "f:conditions": { ".": {}, "k:{\"type\":\"Available\"}": { ".": {}, "f:lastTransitionTime": {}, "f:lastUpdateTime": {}, "f:message": {}, "f:reason": {}, "f:status": {}, "f:type": {} }, "k:{\"type\":\"Progressing\"}": { ".": {}, "f:lastTransitionTime": {}, "f:lastUpdateTime": {}, "f:message": {}, "f:reason": {}, "f:status": {}, "f:type": {} } }, "f:observedGeneration": {}, "f:readyReplicas": {}, "f:replicas": {}, "f:updatedReplicas": {} } }, "manager": "kube-controller-manager", "operation": "Update", "time": "2022-03-17T08:55:52Z" } ], "name": "konnectivity-agent", "namespace": "kube-system", "resourceVersion": "280105", "uid": "d3b49e97-7bac-435e-bfc6-19a25fe494fe" }, "spec": { "progressDeadlineSeconds": 600, "replicas": 6, "revisionHistoryLimit": 10, "selector": { "matchLabels": { "k8s-app": "konnectivity-agent" } }, "strategy": { "rollingUpdate": { "maxSurge": "25%", "maxUnavailable": "25%" }, "type": "RollingUpdate" }, "template": { "metadata": { "annotations": { "cluster-autoscaler.kubernetes.io/safe-to-evict": "true", "components.gke.io/component-name": "konnectivitynetworkproxy-combined", "components.gke.io/component-version": "1.3.3" }, "creationTimestamp": null, "labels": { "k8s-app": "konnectivity-agent" } }, "spec": { "containers": [ { "args": [ "--logtostderr=true", "--ca-cert=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", "--proxy-server-host=34.75.195.103", "--proxy-server-port=8132", "--health-server-port=8093", "--admin-server-port=8094", "--sync-interval=5s", "--probe-interval=5s", "--service-account-token-path=/var/run/secrets/tokens/konnectivity-agent-token", "--v=3" ], "command": [ "/proxy-agent" ], "env": [ { "name": "POD_NAME", "valueFrom": { "fieldRef": { "apiVersion": "v1", "fieldPath": "metadata.name" } } }, { "name": "POD_NAMESPACE", "valueFrom": { "fieldRef": { "apiVersion": "v1", "fieldPath": "metadata.namespace" } } } ], "image": "gke.gcr.io/proxy-agent:v0.0.24-gke.0", "imagePullPolicy": "IfNotPresent", "livenessProbe": { "failureThreshold": 3, "httpGet": { "path": "/healthz", "port": 8093, "scheme": "HTTP" }, "initialDelaySeconds": 15, "periodSeconds": 10, "successThreshold": 1, "timeoutSeconds": 15 }, "name": "konnectivity-agent", "ports": [ { "containerPort": 8093, "name": "metrics", "protocol": "TCP" } ], "resources": { "limits": { "memory": "125Mi" }, "requests": { "cpu": "10m", "memory": "30Mi" } }, "securityContext": { "allowPrivilegeEscalation": false, "capabilities": { "drop": [ "all" ] } }, "terminationMessagePath": "/dev/termination-log", "terminationMessagePolicy": "File", "volumeMounts": [ { "mountPath": "/var/run/secrets/tokens", "name": "konnectivity-agent-token" } ] } ], "dnsPolicy": "ClusterFirst", "nodeSelector": { "beta.kubernetes.io/os": "linux" }, "priorityClassName": "system-cluster-critical", "restartPolicy": "Always", "schedulerName": "default-scheduler", "securityContext": { "fsGroup": 1000, "runAsGroup": 1000, "runAsUser": 1000 }, "serviceAccount": "konnectivity-agent", "serviceAccountName": "konnectivity-agent", "terminationGracePeriodSeconds": 30, "tolerations": [ { "key": "CriticalAddonsOnly", "operator": "Exists" }, { "effect": "NoSchedule", "key": "sandbox.gke.io/runtime", "operator": "Equal", "value": "gvisor" }, { "key": "components.gke.io/gke-managed-components", "operator": "Exists" } ], "topologySpreadConstraints": [ { "labelSelector": { "matchLabels": { "k8s-app": "konnectivity-agent" } }, "maxSkew": 1, "topologyKey": "topology.kubernetes.io/zone", "whenUnsatisfiable": "ScheduleAnyway" }, { "labelSelector": { "matchLabels": { "k8s-app": "konnectivity-agent" } }, "maxSkew": 1, "topologyKey": "kubernetes.io/hostname", "whenUnsatisfiable": "ScheduleAnyway" } ], "volumes": [ { "name": "konnectivity-agent-token", "projected": { "defaultMode": 420, "sources": [ { "serviceAccountToken": { "audience": "system:konnectivity-server", "expirationSeconds": 3600, "path": "konnectivity-agent-token" } } ] } } ] } } }, "status": { "availableReplicas": 6, "conditions": [ { "lastTransitionTime": "2022-03-17T08:55:41Z", "lastUpdateTime": "2022-03-17T08:55:41Z", "message": "ReplicaSet \"konnectivity-agent-56c9b8cf8\" has successfully progressed.", "reason": "NewReplicaSetAvailable", "status": "True", "type": "Progressing" }, { "lastTransitionTime": "2022-03-17T08:55:52Z", "lastUpdateTime": "2022-03-17T08:55:52Z", "message": "Deployment has minimum availability.", "reason": "MinimumReplicasAvailable", "status": "True", "type": "Available" } ], "observedGeneration": 2, "readyReplicas": 6, "replicas": 6, "updatedReplicas": 6 } }, "serviceName": "k8s.io", "status": {} }, "receiveTimestamp": "2022-03-21T19:46:38.090036928Z", "resource": { "labels": { "cluster_name": "iammai-340819-gke-cluster", "location": "us-east1", "project_id": "iammai-340819" }, "type": "k8s_cluster" }, "timestamp": "2022-03-21T19:46:36.090498Z" } diff --git a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-audit.log-expected.json b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-audit.log-expected.json index af8f7c01543..b121d0a1a3f 100644 --- a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-audit.log-expected.json +++ b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-audit.log-expected.json @@ -2,6 +2,11 @@ "expected": [ { "@timestamp": "2019-12-19T00:49:36.086Z", + "client": { + "user": { + "email": "xxx@xxx.xxx" + } + }, "cloud": { "project": { "id": "elastic-beats" @@ -13,10 +18,19 @@ }, "event": { "action": "GetResourceBillingInfo", + "category": [ + "network", + "configuration" + ], "id": "-uihnmjctwo", "kind": "event", "original": "{\"insertId\":\"-uihnmjctwo\",\"logName\":\"projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access\",\"protoPayload\":{\"@type\":\"type.googleapis.com/google.cloud.audit.AuditLog\",\"authenticationInfo\":{\"principalEmail\":\"xxx@xxx.xxx\"},\"authorizationInfo\":[{\"granted\":true,\"permission\":\"resourcemanager.projects.get\",\"resource\":\"projects/elastic-beats\",\"resourceAttributes\":{}}],\"methodName\":\"GetResourceBillingInfo\",\"request\":{\"@type\":\"type.googleapis.com/google.internal.cloudbilling.billingaccount.v1.GetResourceBillingInfoRequest\",\"resourceName\":\"projects/189716325846\"},\"requestMetadata\":{\"callerIp\":\"192.168.1.1\",\"destinationAttributes\":{},\"requestAttributes\":{}},\"resourceName\":\"projects/elastic-beats\",\"serviceName\":\"cloudbilling.googleapis.com\",\"status\":{}},\"receiveTimestamp\":\"2019-12-19T00:49:36.313482371Z\",\"resource\":{\"labels\":{\"project_id\":\"elastic-beats\"},\"type\":\"project\"},\"severity\":\"INFO\",\"timestamp\":\"2019-12-19T00:49:36.086Z\"}", - "outcome": "success" + "outcome": "success", + "provider": "data_access", + "type": [ + "access", + "allowed" + ] }, "gcp": { "audit": { @@ -32,11 +46,8 @@ ], "method_name": "GetResourceBillingInfo", "request": { - "proto_name": "type.googleapis.com/google.internal.cloudbilling.billingaccount.v1.GetResourceBillingInfoRequest", - "resource_name": "projects/189716325846" - }, - "request_metadata": { - "caller_ip": "192.168.1.1" + "@type": "type.googleapis.com/google.internal.cloudbilling.billingaccount.v1.GetResourceBillingInfoRequest", + "resourceName": "projects/189716325846" }, "resource_name": "projects/elastic-beats", "service_name": "cloudbilling.googleapis.com", @@ -44,6 +55,7 @@ } }, "log": { + "level": "INFO", "logger": "projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access" }, "service": { @@ -54,13 +66,15 @@ }, "tags": [ "preserve_original_event" - ], - "user": { - "email": "xxx@xxx.xxx" - } + ] }, { "@timestamp": "2019-12-19T00:45:51.228Z", + "client": { + "user": { + "email": "xxx@xxx.xxx" + } + }, "cloud": { "project": { "id": "elastic-beats" @@ -72,10 +86,19 @@ }, "event": { "action": "beta.compute.machineTypes.aggregatedList", + "category": [ + "network", + "configuration" + ], "id": "-h6onuze1h7dg", "kind": "event", "original": "{\"insertId\":\"-h6onuze1h7dg\",\"logName\":\"projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access\",\"protoPayload\":{\"@type\":\"type.googleapis.com/google.cloud.audit.AuditLog\",\"authenticationInfo\":{\"principalEmail\":\"xxx@xxx.xxx\"},\"authorizationInfo\":[{\"granted\":false,\"permission\":\"compute.machineTypes.list\",\"resourceAttributes\":{\"name\":\"projects/elastic-beats\",\"service\":\"resourcemanager\",\"type\":\"resourcemanager.projects\"}}],\"methodName\":\"beta.compute.machineTypes.aggregatedList\",\"numResponseItems\":\"71\",\"request\":{\"@type\":\"type.googleapis.com/compute.machineTypes.aggregatedList\"},\"requestMetadata\":{\"callerIp\":\"192.168.1.1\",\"callerSuppliedUserAgent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:71.0) Gecko/20100101 Firefox/71.0,gzip(gfe),gzip(gfe)\",\"destinationAttributes\":{},\"requestAttributes\":{\"auth\":{},\"time\":\"2019-12-19T00:45:51.711Z\"}},\"resourceLocation\":{\"currentLocations\":[\"global\"]},\"resourceName\":\"projects/elastic-beats/global/machineTypes\",\"serviceName\":\"compute.googleapis.com\"},\"receiveTimestamp\":\"2019-12-19T00:45:52.367887078Z\",\"resource\":{\"labels\":{\"location\":\"global\",\"method\":\"compute.machineTypes.aggregatedList\",\"project_id\":\"elastic-beats\",\"service\":\"compute.googleapis.com\",\"version\":\"beta\"},\"type\":\"api\"},\"severity\":\"INFO\",\"timestamp\":\"2019-12-19T00:45:51.228Z\"}", - "outcome": "failure" + "outcome": "failure", + "provider": "data_access", + "type": [ + "access", + "denied" + ] }, "gcp": { "audit": { @@ -96,10 +119,9 @@ "method_name": "beta.compute.machineTypes.aggregatedList", "num_response_items": 71, "request": { - "proto_name": "type.googleapis.com/compute.machineTypes.aggregatedList" + "@type": "type.googleapis.com/compute.machineTypes.aggregatedList" }, "request_metadata": { - "caller_ip": "192.168.1.1", "caller_supplied_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:71.0) Gecko/20100101 Firefox/71.0,gzip(gfe),gzip(gfe)" }, "resource_location": { @@ -113,6 +135,7 @@ } }, "log": { + "level": "INFO", "logger": "projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access" }, "service": { @@ -124,9 +147,6 @@ "tags": [ "preserve_original_event" ], - "user": { - "email": "xxx@xxx.xxx" - }, "user_agent": { "device": { "name": "Mac" @@ -143,6 +163,11 @@ }, { "@timestamp": "2019-12-19T00:44:25.051Z", + "client": { + "user": { + "email": "xxx@xxx.xxx" + } + }, "cloud": { "project": { "id": "elastic-beats" @@ -154,10 +179,19 @@ }, "event": { "action": "beta.compute.instances.aggregatedList", + "category": [ + "network", + "configuration" + ], "id": "yonau2dg2zi", "kind": "event", "original": "{\"insertId\":\"yonau2dg2zi\",\"logName\":\"projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access\",\"protoPayload\":{\"@type\":\"type.googleapis.com/google.cloud.audit.AuditLog\",\"authenticationInfo\":{\"principalEmail\":\"xxx@xxx.xxx\"},\"authorizationInfo\":[{\"granted\":true,\"permission\":\"compute.instances.list\",\"resourceAttributes\":{\"name\":\"projects/elastic-beats\",\"service\":\"resourcemanager\",\"type\":\"resourcemanager.projects\"}}],\"methodName\":\"beta.compute.instances.aggregatedList\",\"numResponseItems\":\"61\",\"request\":{\"@type\":\"type.googleapis.com/compute.instances.aggregatedList\"},\"requestMetadata\":{\"callerIp\":\"192.168.1.1\",\"callerSuppliedUserAgent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:71.0) Gecko/20100101 Firefox/71.0,gzip(gfe),gzip(gfe)\",\"destinationAttributes\":{},\"requestAttributes\":{\"auth\":{},\"time\":\"2019-12-19T00:44:25.198Z\"}},\"response\":{\"@type\":\"core.k8s.io/v1.Status\",\"apiVersion\":\"v1\",\"details\":{\"group\":\"batch\",\"kind\":\"jobs\",\"name\":\"gsuite-exporter-1589294700\",\"uid\":\"2beff34a-945f-11ea-bacf-42010a80007f\"},\"kind\":\"Status\",\"metadata\":{},\"status\":\"Success\"},\"resourceLocation\":{\"currentLocations\":[\"global\"]},\"resourceName\":\"projects/elastic-beats/global/instances\",\"serviceName\":\"compute.googleapis.com\"},\"receiveTimestamp\":\"2019-12-19T00:44:25.262379373Z\",\"resource\":{\"labels\":{\"location\":\"global\",\"method\":\"compute.instances.aggregatedList\",\"project_id\":\"elastic-beats\",\"service\":\"compute.googleapis.com\",\"version\":\"beta\"},\"type\":\"api\"},\"severity\":\"INFO\",\"timestamp\":\"2019-12-19T00:44:25.051Z\"}", - "outcome": "success" + "outcome": "success", + "provider": "data_access", + "type": [ + "access", + "allowed" + ] }, "gcp": { "audit": { @@ -178,10 +212,9 @@ "method_name": "beta.compute.instances.aggregatedList", "num_response_items": 61, "request": { - "proto_name": "type.googleapis.com/compute.instances.aggregatedList" + "@type": "type.googleapis.com/compute.instances.aggregatedList" }, "request_metadata": { - "caller_ip": "192.168.1.1", "caller_supplied_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:71.0) Gecko/20100101 Firefox/71.0,gzip(gfe),gzip(gfe)" }, "resource_location": { @@ -191,22 +224,23 @@ }, "resource_name": "projects/elastic-beats/global/instances", "response": { + "@type": "core.k8s.io/v1.Status", + "apiVersion": "v1", "details": { "group": "batch", "kind": "jobs", "name": "gsuite-exporter-1589294700", "uid": "2beff34a-945f-11ea-bacf-42010a80007f" }, - "proto_name": "core.k8s.io/v1.Status", - "status": { - "value": "Success" - } + "kind": "Status", + "status": "Success" }, "service_name": "compute.googleapis.com", "type": "type.googleapis.com/google.cloud.audit.AuditLog" } }, "log": { + "level": "INFO", "logger": "projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access" }, "service": { @@ -218,9 +252,6 @@ "tags": [ "preserve_original_event" ], - "user": { - "email": "xxx@xxx.xxx" - }, "user_agent": { "device": { "name": "Mac" @@ -237,6 +268,11 @@ }, { "@timestamp": "2019-12-19T00:44:25.051Z", + "client": { + "user": { + "email": "xxx@xxx.xxx" + } + }, "cloud": { "project": { "id": "elastic-beats" @@ -248,10 +284,15 @@ }, "event": { "action": "beta.compute.instances.aggregatedList", + "category": [ + "network", + "configuration" + ], "id": "yonau3dc2zi", "kind": "event", "original": "{\"insertId\":\"yonau3dc2zi\",\"logName\":\"projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access\",\"protoPayload\":{\"@type\":\"type.googleapis.com/google.cloud.audit.AuditLog\",\"authenticationInfo\":{\"principalEmail\":\"xxx@xxx.xxx\"},\"authorizationInfo\":[{\"permission\":\"compute.instances.list\",\"resourceAttributes\":{\"name\":\"projects/elastic-beats\",\"service\":\"resourcemanager\",\"type\":\"resourcemanager.projects\"}}],\"methodName\":\"beta.compute.instances.aggregatedList\",\"numResponseItems\":\"61\",\"request\":{\"@type\":\"type.googleapis.com/compute.instances.aggregatedList\"},\"requestMetadata\":{\"callerIp\":\"192.168.1.1\",\"callerSuppliedUserAgent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:71.0) Gecko/20100101 Firefox/71.0,gzip(gfe),gzip(gfe)\",\"destinationAttributes\":{},\"requestAttributes\":{\"auth\":{},\"time\":\"2019-12-19T00:44:25.198Z\"}},\"resourceLocation\":{\"currentLocations\":[\"global\"]},\"resourceName\":\"projects/elastic-beats/global/instances\",\"serviceName\":\"compute.googleapis.com\",\"status\":{\"code\":7,\"message\":\"PERMISSION_DENIED\"}},\"receiveTimestamp\":\"2019-12-19T00:44:25.262379373Z\",\"resource\":{\"labels\":{\"location\":\"global\",\"method\":\"compute.instances.aggregatedList\",\"project_id\":\"elastic-beats\",\"service\":\"compute.googleapis.com\",\"version\":\"beta\"},\"type\":\"api\"},\"severity\":\"INFO\",\"timestamp\":\"2019-12-19T00:44:25.051Z\"}", - "outcome": "failure" + "outcome": "failure", + "provider": "data_access" }, "gcp": { "audit": { @@ -271,10 +312,9 @@ "method_name": "beta.compute.instances.aggregatedList", "num_response_items": 61, "request": { - "proto_name": "type.googleapis.com/compute.instances.aggregatedList" + "@type": "type.googleapis.com/compute.instances.aggregatedList" }, "request_metadata": { - "caller_ip": "192.168.1.1", "caller_supplied_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:71.0) Gecko/20100101 Firefox/71.0,gzip(gfe),gzip(gfe)" }, "resource_location": { @@ -292,6 +332,7 @@ } }, "log": { + "level": "INFO", "logger": "projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access" }, "service": { @@ -303,9 +344,6 @@ "tags": [ "preserve_original_event" ], - "user": { - "email": "xxx@xxx.xxx" - }, "user_agent": { "device": { "name": "Mac" @@ -322,6 +360,11 @@ }, { "@timestamp": "2020-08-05T21:07:30.974Z", + "client": { + "user": { + "email": "system:serviceaccount:cert-manager:cert-manager-webhook" + } + }, "cloud": { "project": { "id": "elastic-siem" @@ -333,10 +376,19 @@ }, "event": { "action": "io.k8s.authorization.v1beta1.subjectaccessreviews.create", + "category": [ + "network", + "configuration" + ], "id": "87efd529-6349-45d2-b905-fc607e6c5d3b", "kind": "event", "original": "{\"insertId\":\"87efd529-6349-45d2-b905-fc607e6c5d3b\",\"labels\":{\"authorization.k8s.io/decision\":\"allow\",\"authorization.k8s.io/reason\":\"RBAC: allowed by ClusterRoleBinding \\\"cert-manager-webhook:auth-delegator\\\" of ClusterRole \\\"system:auth-delegator\\\" to ServiceAccount \\\"cert-manager-webhook/cert-manager\\\"\"},\"logName\":\"projects/foo/logs/cloudaudit.googleapis.com%2Fdata_access\",\"operation\":{\"first\":true,\"id\":\"5555555-6349-45d2-b905-fc607e6c5d3b\",\"last\":true,\"producer\":\"k8s.io\"},\"protoPayload\":{\"@type\":\"type.googleapis.com/google.cloud.audit.AuditLog\",\"authenticationInfo\":{\"principalEmail\":\"system:serviceaccount:cert-manager:cert-manager-webhook\"},\"authorizationInfo\":[{\"granted\":true,\"permission\":\"io.k8s.authorization.v1beta1.subjectaccessreviews.create\",\"resource\":\"authorization.k8s.io/v1beta1/subjectaccessreviews\"}],\"methodName\":\"io.k8s.authorization.v1beta1.subjectaccessreviews.create\",\"request\":{\"@type\":\"authorization.k8s.io/v1beta1.SubjectAccessReview\",\"apiVersion\":\"authorization.k8s.io/v1beta1\",\"kind\":\"SubjectAccessReview\",\"metadata\":{\"creationTimestamp\":null},\"spec\":{\"group\":[\"system:serviceaccounts\",\"system:serviceaccounts:kube-system\",\"system:authenticated\"],\"nonResourceAttributes\":{\"path\":\"/apis/webhook.cert-manager.io/v1beta1\",\"verb\":\"get\"},\"user\":\"system:serviceaccount:kube-system:resourcequota-controller\"},\"status\":{\"allowed\":false}},\"requestMetadata\":{\"callerIp\":\"10.11.12.13\",\"callerSuppliedUserAgent\":\"webhook/v0.0.0 (linux/amd64) kubernetes/$Format\"},\"resourceName\":\"authorization.k8s.io/v1beta1/subjectaccessreviews\",\"response\":{\"@type\":\"authorization.k8s.io/v1beta1.SubjectAccessReview\",\"apiVersion\":\"authorization.k8s.io/v1beta1\",\"kind\":\"SubjectAccessReview\",\"metadata\":{\"creationTimestamp\":null},\"spec\":{\"group\":[\"system:serviceaccounts\",\"system:serviceaccounts:kube-system\",\"system:authenticated\"],\"nonResourceAttributes\":{\"path\":\"/apis/webhook.cert-manager.io/v1beta1\",\"verb\":\"get\"},\"user\":\"system:serviceaccount:kube-system:resourcequota-controller\"},\"status\":{\"allowed\":true,\"reason\":\"RBAC: allowed by ClusterRoleBinding \\\"system:discovery\\\" of ClusterRole \\\"system:discovery\\\" to Group \\\"system:authenticated\\\"\"}},\"serviceName\":\"k8s.io\",\"status\":{\"code\":0}},\"receiveTimestamp\":\"2020-08-05T21:07:32.157698684Z\",\"resource\":{\"labels\":{\"cluster_name\":\"analysis-cluster\",\"location\":\"us-central1-a\",\"project_id\":\"elastic-siem\"},\"type\":\"k8s_cluster\"},\"timestamp\":\"2020-08-05T21:07:30.974750Z\"}", - "outcome": "success" + "outcome": "success", + "provider": "data_access", + "type": [ + "access", + "allowed" + ] }, "gcp": { "audit": { @@ -350,17 +402,53 @@ "resource": "authorization.k8s.io/v1beta1/subjectaccessreviews" } ], + "labels": { + "authorization.k8s.io/decision": "allow", + "authorization.k8s.io/reason": "RBAC: allowed by ClusterRoleBinding \"cert-manager-webhook:auth-delegator\" of ClusterRole \"system:auth-delegator\" to ServiceAccount \"cert-manager-webhook/cert-manager\"" + }, + "logentry_operation": { + "id": "5555555-6349-45d2-b905-fc607e6c5d3b" + }, "method_name": "io.k8s.authorization.v1beta1.subjectaccessreviews.create", "request": { - "proto_name": "authorization.k8s.io/v1beta1.SubjectAccessReview" + "@type": "authorization.k8s.io/v1beta1.SubjectAccessReview", + "apiVersion": "authorization.k8s.io/v1beta1", + "kind": "SubjectAccessReview", + "spec": { + "group": [ + "system:serviceaccounts", + "system:serviceaccounts:kube-system", + "system:authenticated" + ], + "nonResourceAttributes": { + "path": "/apis/webhook.cert-manager.io/v1beta1", + "verb": "get" + }, + "user": "system:serviceaccount:kube-system:resourcequota-controller" + }, + "status": { + "allowed": false + } }, "request_metadata": { - "caller_ip": "10.11.12.13", "caller_supplied_user_agent": "webhook/v0.0.0 (linux/amd64) kubernetes/$Format" }, - "resource_name": "authorization.k8s.io/v1beta1/subjectaccessreviews", "response": { - "proto_name": "authorization.k8s.io/v1beta1.SubjectAccessReview", + "@type": "authorization.k8s.io/v1beta1.SubjectAccessReview", + "apiVersion": "authorization.k8s.io/v1beta1", + "kind": "SubjectAccessReview", + "spec": { + "group": [ + "system:serviceaccounts", + "system:serviceaccounts:kube-system", + "system:authenticated" + ], + "nonResourceAttributes": { + "path": "/apis/webhook.cert-manager.io/v1beta1", + "verb": "get" + }, + "user": "system:serviceaccount:kube-system:resourcequota-controller" + }, "status": { "allowed": true, "reason": "RBAC: allowed by ClusterRoleBinding \"system:discovery\" of ClusterRole \"system:discovery\" to Group \"system:authenticated\"" @@ -377,9 +465,13 @@ "logger": "projects/foo/logs/cloudaudit.googleapis.com%2Fdata_access" }, "orchestrator": { + "api_version": "v1beta1", "cluster": { "name": "analysis-cluster" }, + "resource": { + "type": "subjectaccessreviews" + }, "type": "kubernetes" }, "service": { @@ -391,9 +483,6 @@ "tags": [ "preserve_original_event" ], - "user": { - "email": "system:serviceaccount:cert-manager:cert-manager-webhook" - }, "user_agent": { "device": { "name": "Other" @@ -407,6 +496,11 @@ }, { "@timestamp": "2020-08-05T21:59:26.456Z", + "client": { + "user": { + "email": "user@mycompany.com" + } + }, "cloud": { "project": { "id": "foo" @@ -418,10 +512,21 @@ }, "event": { "action": "v1.compute.images.insert", + "category": [ + "session", + "network", + "configuration" + ], "id": "v2spcwdzmc2", "kind": "event", "original": "{\"insertId\":\"v2spcwdzmc2\",\"logName\":\"projects/foo/logs/cloudaudit.googleapis.com%2Factivity\",\"operation\":{\"first\":true,\"id\":\"operation-1596664766354-5ac287c395484-fa3923bd-543e018e\",\"producer\":\"compute.googleapis.com\"},\"protoPayload\":{\"@type\":\"type.googleapis.com/google.cloud.audit.AuditLog\",\"authenticationInfo\":{\"principalEmail\":\"user@mycompany.com\"},\"authorizationInfo\":[{\"granted\":true,\"permission\":\"compute.images.create\",\"resourceAttributes\":{\"name\":\"projects/foo/global/images/windows-server-2016-v20200805\",\"service\":\"compute\",\"type\":\"compute.images\"}}],\"methodName\":\"v1.compute.images.insert\",\"request\":{\"@type\":\"type.googleapis.com/compute.images.insert\",\"family\":\"windows-server-2016\",\"guestOsFeatures\":[{\"type\":\"VIRTIO_SCSI_MULTIQUEUE\"},{\"type\":\"WINDOWS\"}],\"name\":\"windows-server-2016-v20200805\",\"rawDisk\":{\"source\":\"https://storage.googleapis.com/storage/v1/b/foo/o/windows-server-2016-v20200805.tar.gz\"},\"sourceType\":\"RAW\"},\"requestMetadata\":{\"callerIp\":\"67.43.156.13\",\"callerSuppliedUserAgent\":\"google-cloud-sdk gcloud/290.0.1 command/gcloud.compute.images.create invocation-id/032752ad0fa44b4ea951951d2deef6a3 environment/None environment-version/None interactive/True from-script/False python/2.7.17 term/xterm-256color (Macintosh; Intel Mac OS X 19.6.0),gzip(gfe)\",\"destinationAttributes\":{},\"requestAttributes\":{\"auth\":{},\"time\":\"2020-08-05T21:59:27.515Z\"}},\"resourceLocation\":{\"currentLocations\":[\"eu\"]},\"resourceName\":\"projects/foo/global/images/windows-server-2016-v20200805\",\"response\":{\"@type\":\"type.googleapis.com/operation\",\"id\":\"44919313\",\"insertTime\":\"2020-08-05T14:59:27.259-07:00\",\"name\":\"operation-1596664766354-5ac287c395484-fa3923bd-543e018e\",\"operationType\":\"insert\",\"progress\":\"0\",\"selfLink\":\"https://www.googleapis.com/compute/v1/projects/foo/global/operations/operation-1596664766354-5ac287c395484-fa3923bd-543e018e\",\"selfLinkWithId\":\"https://www.googleapis.com/compute/v1/projects/foo/global/operations/4491931805423146320\",\"startTime\":\"2020-08-05T14:59:27.274-07:00\",\"status\":\"RUNNING\",\"targetId\":\"12345\",\"targetLink\":\"https://www.googleapis.com/compute/v1/projects/foo/global/images/windows-server-2016-v20200805\",\"user\":\"user@mycompany.com\"},\"serviceName\":\"compute.googleapis.com\"},\"receiveTimestamp\":\"2020-08-05T21:59:27.822546978Z\",\"resource\":{\"labels\":{\"image_id\":\"771879043\",\"project_id\":\"foo\"},\"type\":\"gce_image\"},\"severity\":\"NOTICE\",\"timestamp\":\"2020-08-05T21:59:26.456Z\"}", - "outcome": "success" + "outcome": "success", + "provider": "activity", + "type": [ + "start", + "access", + "allowed" + ] }, "gcp": { "audit": { @@ -439,13 +544,28 @@ } } ], + "logentry_operation": { + "id": "operation-1596664766354-5ac287c395484-fa3923bd-543e018e" + }, "method_name": "v1.compute.images.insert", "request": { + "@type": "type.googleapis.com/compute.images.insert", + "family": "windows-server-2016", + "guestOsFeatures": [ + { + "type": "VIRTIO_SCSI_MULTIQUEUE" + }, + { + "type": "WINDOWS" + } + ], "name": "windows-server-2016-v20200805", - "proto_name": "type.googleapis.com/compute.images.insert" + "rawDisk": { + "source": "https://storage.googleapis.com/storage/v1/b/foo/o/windows-server-2016-v20200805.tar.gz" + }, + "sourceType": "RAW" }, "request_metadata": { - "caller_ip": "67.43.156.13", "caller_supplied_user_agent": "google-cloud-sdk gcloud/290.0.1 command/gcloud.compute.images.create invocation-id/032752ad0fa44b4ea951951d2deef6a3 environment/None environment-version/None interactive/True from-script/False python/2.7.17 term/xterm-256color (Macintosh; Intel Mac OS X 19.6.0),gzip(gfe)" }, "resource_location": { @@ -455,16 +575,26 @@ }, "resource_name": "projects/foo/global/images/windows-server-2016-v20200805", "response": { - "proto_name": "type.googleapis.com/operation", - "status": { - "value": "RUNNING" - } + "@type": "type.googleapis.com/operation", + "id": "44919313", + "insertTime": "2020-08-05T14:59:27.259-07:00", + "name": "operation-1596664766354-5ac287c395484-fa3923bd-543e018e", + "operationType": "insert", + "progress": "0", + "selfLink": "https://www.googleapis.com/compute/v1/projects/foo/global/operations/operation-1596664766354-5ac287c395484-fa3923bd-543e018e", + "selfLinkWithId": "https://www.googleapis.com/compute/v1/projects/foo/global/operations/4491931805423146320", + "startTime": "2020-08-05T14:59:27.274-07:00", + "status": "RUNNING", + "targetId": "12345", + "targetLink": "https://www.googleapis.com/compute/v1/projects/foo/global/images/windows-server-2016-v20200805", + "user": "user@mycompany.com" }, "service_name": "compute.googleapis.com", "type": "type.googleapis.com/google.cloud.audit.AuditLog" } }, "log": { + "level": "NOTICE", "logger": "projects/foo/logs/cloudaudit.googleapis.com%2Factivity" }, "service": { @@ -488,9 +618,6 @@ "tags": [ "preserve_original_event" ], - "user": { - "email": "user@mycompany.com" - }, "user_agent": { "device": { "name": "Mac" @@ -506,6 +633,11 @@ }, { "@timestamp": "2020-08-05T16:56:40.428Z", + "client": { + "user": { + "email": "user@mycompany.com" + } + }, "cloud": { "instance": { "id": "590261181" @@ -520,22 +652,31 @@ }, "event": { "action": "beta.compute.instances.stop", + "category": [ + "session" + ], "id": "-c7ctxmd2zab", "kind": "event", "original": "{\"insertId\":\"-c7ctxmd2zab\",\"logName\":\"projects/foo/logs/cloudaudit.googleapis.com%2Factivity\",\"operation\":{\"id\":\"operation-1596646123456-5ac2438b775f6-f8ca1382-e70b6831\",\"last\":true,\"producer\":\"compute.googleapis.com\"},\"protoPayload\":{\"@type\":\"type.googleapis.com/google.cloud.audit.AuditLog\",\"authenticationInfo\":{\"principalEmail\":\"user@mycompany.com\"},\"methodName\":\"beta.compute.instances.stop\",\"request\":{\"@type\":\"type.googleapis.com/compute.instances.stop\"},\"requestMetadata\":{\"callerIp\":\"67.43.156.13\",\"callerSuppliedUserAgent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:79.0) Gecko/20100101 Firefox/79.0,gzip(gfe),gzip(gfe)\"},\"resourceName\":\"projects/foo/zones/us-central1-a/instances/win10-test\",\"serviceName\":\"compute.googleapis.com\"},\"receiveTimestamp\":\"2020-08-05T16:56:41.315135528Z\",\"resource\":{\"labels\":{\"instance_id\":\"590261181\",\"project_id\":\"foo\",\"zone\":\"us-central1-a\"},\"type\":\"gce_instance\"},\"severity\":\"NOTICE\",\"timestamp\":\"2020-08-05T16:56:40.428Z\"}", - "outcome": "unknown" + "outcome": "unknown", + "provider": "activity", + "type": [ + "end" + ] }, "gcp": { "audit": { "authentication_info": { "principal_email": "user@mycompany.com" }, + "logentry_operation": { + "id": "operation-1596646123456-5ac2438b775f6-f8ca1382-e70b6831" + }, "method_name": "beta.compute.instances.stop", "request": { - "proto_name": "type.googleapis.com/compute.instances.stop" + "@type": "type.googleapis.com/compute.instances.stop" }, "request_metadata": { - "caller_ip": "67.43.156.13", "caller_supplied_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:79.0) Gecko/20100101 Firefox/79.0,gzip(gfe),gzip(gfe)" }, "resource_name": "projects/foo/zones/us-central1-a/instances/win10-test", @@ -544,6 +685,7 @@ } }, "log": { + "level": "NOTICE", "logger": "projects/foo/logs/cloudaudit.googleapis.com%2Factivity" }, "service": { @@ -567,9 +709,6 @@ "tags": [ "preserve_original_event" ], - "user": { - "email": "user@mycompany.com" - }, "user_agent": { "device": { "name": "Mac" @@ -586,6 +725,11 @@ }, { "@timestamp": "2021-04-23T14:47:07.535Z", + "client": { + "user": { + "email": "xxx@xxx.xxx" + } + }, "cloud": { "project": { "id": "elastic-siem" @@ -597,10 +741,19 @@ }, "event": { "action": "io.k8s.core.v1.nodes.list", + "category": [ + "network", + "configuration" + ], "id": "94170ac4-6e82-4345-98ad-3c780222d19d", "kind": "event", "original": "{\"insertId\":\"94170ac4-6e82-4345-98ad-3c780222d19d\",\"labels\":{\"authorization.k8s.io/decision\":\"allow\",\"authorization.k8s.io/reason\":\"\"},\"logName\":\"projects/elastic-siem/logs/cloudaudit.googleapis.com%2Fdata_access\",\"operation\":{\"first\":true,\"id\":\"94170ac4-6e82-4345-98ad-3c780222d19d\",\"last\":true,\"producer\":\"k8s.io\"},\"protoPayload\":{\"@type\":\"type.googleapis.com/google.cloud.audit.AuditLog\",\"authenticationInfo\":{\"principalEmail\":\"xxx@xxx.xxx\"},\"authorizationInfo\":[{\"granted\":true,\"permission\":\"io.k8s.core.v1.nodes.list\",\"resource\":\"core/v1/nodes\"}],\"methodName\":\"io.k8s.core.v1.nodes.list\",\"requestMetadata\":{\"callerIp\":\"192.168.1.1\",\"callerSuppliedUserAgent\":\"GoogleCloudConsole\"},\"resourceName\":\"core/v1/nodes\",\"serviceName\":\"k8s.io\",\"status\":{}},\"receiveTimestamp\":\"2021-04-23T14:47:31.94822935Z\",\"resource\":{\"labels\":{\"cluster_name\":\"analysis-cluster\",\"location\":\"us-central1-a\",\"project_id\":\"elastic-siem\"},\"type\":\"k8s_cluster\"},\"timestamp\":\"2021-04-23T14:47:07.535383Z\"}", - "outcome": "success" + "outcome": "success", + "provider": "data_access", + "type": [ + "access", + "allowed" + ] }, "gcp": { "audit": { @@ -614,12 +767,13 @@ "resource": "core/v1/nodes" } ], + "labels": { + "authorization.k8s.io/decision": "allow" + }, "method_name": "io.k8s.core.v1.nodes.list", "request_metadata": { - "caller_ip": "192.168.1.1", "caller_supplied_user_agent": "GoogleCloudConsole" }, - "resource_name": "core/v1/nodes", "service_name": "k8s.io", "type": "type.googleapis.com/google.cloud.audit.AuditLog" } @@ -628,9 +782,13 @@ "logger": "projects/elastic-siem/logs/cloudaudit.googleapis.com%2Fdata_access" }, "orchestrator": { + "api_version": "v1", "cluster": { "name": "analysis-cluster" }, + "resource": { + "type": "nodes" + }, "type": "kubernetes" }, "service": { @@ -642,9 +800,6 @@ "tags": [ "preserve_original_event" ], - "user": { - "email": "xxx@xxx.xxx" - }, "user_agent": { "device": { "name": "Other" @@ -655,6 +810,11 @@ }, { "@timestamp": "2021-04-23T14:16:07.574Z", + "client": { + "user": { + "email": "xxx@xxx.xxx" + } + }, "cloud": { "project": { "id": "elastic-siem" @@ -666,10 +826,19 @@ }, "event": { "action": "io.k8s.extensions.v1beta1.ingresses.list", + "category": [ + "network", + "configuration" + ], "id": "b10a904a-faa4-4e0d-9ec3-7bc6a180196a", "kind": "event", "original": "{\"insertId\":\"b10a904a-faa4-4e0d-9ec3-7bc6a180196a\",\"labels\":{\"authorization.k8s.io/decision\":\"allow\",\"authorization.k8s.io/reason\":\"\",\"k8s.io/deprecated\":\"true\",\"k8s.io/removed-release\":\"1.22\"},\"logName\":\"projects/elastic-siem/logs/cloudaudit.googleapis.com%2Fdata_access\",\"operation\":{\"first\":true,\"id\":\"b10a904a-faa4-4e0d-9ec3-7bc6a180196a\",\"last\":true,\"producer\":\"k8s.io\"},\"protoPayload\":{\"@type\":\"type.googleapis.com/google.cloud.audit.AuditLog\",\"authenticationInfo\":{\"principalEmail\":\"xxx@xxx.xxx\"},\"authorizationInfo\":[{\"granted\":true,\"permission\":\"io.k8s.extensions.v1beta1.ingresses.list\",\"resource\":\"extensions/v1beta1/namespaces/cos-auditd/ingresses\"}],\"methodName\":\"io.k8s.extensions.v1beta1.ingresses.list\",\"requestMetadata\":{\"callerIp\":\"192.168.1.1\",\"callerSuppliedUserAgent\":\"GoogleCloudConsole\"},\"resourceName\":\"extensions/v1beta1/namespaces/cos-auditd/ingresses\",\"serviceName\":\"k8s.io\",\"status\":{}},\"receiveTimestamp\":\"2021-04-23T14:16:36.37362467Z\",\"resource\":{\"labels\":{\"cluster_name\":\"analysis-cluster\",\"location\":\"us-central1-a\",\"project_id\":\"elastic-siem\"},\"type\":\"k8s_cluster\"},\"timestamp\":\"2021-04-23T14:16:07.574776Z\"}", - "outcome": "success" + "outcome": "success", + "provider": "data_access", + "type": [ + "access", + "allowed" + ] }, "gcp": { "audit": { @@ -683,12 +852,15 @@ "resource": "extensions/v1beta1/namespaces/cos-auditd/ingresses" } ], + "labels": { + "authorization.k8s.io/decision": "allow", + "k8s.io/deprecated": "true", + "k8s.io/removed-release": "1.22" + }, "method_name": "io.k8s.extensions.v1beta1.ingresses.list", "request_metadata": { - "caller_ip": "192.168.1.1", "caller_supplied_user_agent": "GoogleCloudConsole" }, - "resource_name": "extensions/v1beta1/namespaces/cos-auditd/ingresses", "service_name": "k8s.io", "type": "type.googleapis.com/google.cloud.audit.AuditLog" } @@ -697,9 +869,14 @@ "logger": "projects/elastic-siem/logs/cloudaudit.googleapis.com%2Fdata_access" }, "orchestrator": { + "api_version": "v1beta1", "cluster": { "name": "analysis-cluster" }, + "namespace": "cos-auditd", + "resource": { + "type": "ingresses" + }, "type": "kubernetes" }, "service": { @@ -711,9 +888,6 @@ "tags": [ "preserve_original_event" ], - "user": { - "email": "xxx@xxx.xxx" - }, "user_agent": { "device": { "name": "Other" @@ -724,6 +898,11 @@ }, { "@timestamp": "2021-04-29T08:19:20.805Z", + "client": { + "user": { + "email": "system:anonymous" + } + }, "cloud": { "project": { "id": "elastic-siem" @@ -735,10 +914,19 @@ }, "event": { "action": "io.k8s.get", + "category": [ + "network", + "configuration" + ], "id": "e973134d-b4d5-4e2f-92b8-82bba13fdb92", "kind": "event", "original": "{\"insertId\":\"e973134d-b4d5-4e2f-92b8-82bba13fdb92\",\"labels\":{\"authorization.k8s.io/decision\":\"allow\",\"authorization.k8s.io/reason\":\"RBAC: allowed by ClusterRoleBinding \\\"system:public-info-viewer\\\" of ClusterRole \\\"system:public-info-viewer\\\" to Group \\\"system:unauthenticated\\\"\"},\"logName\":\"projects/elastic-siem/logs/cloudaudit.googleapis.com%2Fdata_access\",\"operation\":{\"first\":true,\"id\":\"e973134d-b4d5-4e2f-92b8-82bba13fdb92\",\"last\":true,\"producer\":\"k8s.io\"},\"protoPayload\":{\"@type\":\"type.googleapis.com/google.cloud.audit.AuditLog\",\"authenticationInfo\":{\"principalEmail\":\"system:anonymous\"},\"authorizationInfo\":[{\"granted\":true,\"permission\":\"io.k8s.get\",\"resource\":\"readyz\"}],\"methodName\":\"io.k8s.get\",\"requestMetadata\":{\"callerIp\":\"127.0.0.1\",\"callerSuppliedUserAgent\":\"kube-probe/1.19+\"},\"resourceName\":\"readyz\",\"serviceName\":\"k8s.io\",\"status\":{}},\"receiveTimestamp\":\"2021-04-29T08:19:21.606980385Z\",\"resource\":{\"labels\":{\"cluster_name\":\"analysis-cluster\",\"location\":\"us-central1-a\",\"project_id\":\"elastic-siem\"},\"type\":\"k8s_cluster\"},\"timestamp\":\"2021-04-29T08:19:20.80581Z\"}", - "outcome": "success" + "outcome": "success", + "provider": "data_access", + "type": [ + "access", + "allowed" + ] }, "gcp": { "audit": { @@ -752,12 +940,14 @@ "resource": "readyz" } ], + "labels": { + "authorization.k8s.io/decision": "allow", + "authorization.k8s.io/reason": "RBAC: allowed by ClusterRoleBinding \"system:public-info-viewer\" of ClusterRole \"system:public-info-viewer\" to Group \"system:unauthenticated\"" + }, "method_name": "io.k8s.get", "request_metadata": { - "caller_ip": "127.0.0.1", "caller_supplied_user_agent": "kube-probe/1.19+" }, - "resource_name": "readyz", "service_name": "k8s.io", "type": "type.googleapis.com/google.cloud.audit.AuditLog" } @@ -769,6 +959,9 @@ "cluster": { "name": "analysis-cluster" }, + "resource": { + "type": "readyz" + }, "type": "kubernetes" }, "service": { @@ -780,9 +973,6 @@ "tags": [ "preserve_original_event" ], - "user": { - "email": "system:anonymous" - }, "user_agent": { "device": { "name": "Other" @@ -793,6 +983,11 @@ }, { "@timestamp": "2021-04-29T08:23:18.899Z", + "client": { + "user": { + "email": "system:serviceaccount:kube-system:generic-garbage-collector" + } + }, "cloud": { "project": { "id": "elastic-siem" @@ -804,10 +999,19 @@ }, "event": { "action": "io.k8s.get", + "category": [ + "network", + "configuration" + ], "id": "03adfb9f-71a3-4f41-9701-29b5542f4d22", "kind": "event", "original": "{\"insertId\":\"03adfb9f-71a3-4f41-9701-29b5542f4d22\",\"labels\":{\"authorization.k8s.io/decision\":\"allow\",\"authorization.k8s.io/reason\":\"RBAC: allowed by ClusterRoleBinding \\\"system:discovery\\\" of ClusterRole \\\"system:discovery\\\" to Group \\\"system:authenticated\\\"\"},\"logName\":\"projects/elastic-siem/logs/cloudaudit.googleapis.com%2Fdata_access\",\"operation\":{\"first\":true,\"id\":\"03adfb9f-71a3-4f41-9701-29b5542f4d22\",\"last\":true,\"producer\":\"k8s.io\"},\"protoPayload\":{\"@type\":\"type.googleapis.com/google.cloud.audit.AuditLog\",\"authenticationInfo\":{\"principalEmail\":\"system:serviceaccount:kube-system:generic-garbage-collector\"},\"authorizationInfo\":[{\"granted\":true,\"permission\":\"io.k8s.get\",\"resource\":\"api/v1\"}],\"methodName\":\"io.k8s.get\",\"requestMetadata\":{\"callerIp\":\"::1\",\"callerSuppliedUserAgent\":\"kube-controller-manager/v1.19.8 (linux/amd64) kubernetes/4f6f69f/system:serviceaccount:kube-system:generic-garbage-collector\"},\"resourceName\":\"api/v1\",\"serviceName\":\"k8s.io\",\"status\":{}},\"receiveTimestamp\":\"2021-04-29T08:23:19.71757101Z\",\"resource\":{\"labels\":{\"cluster_name\":\"analysis-cluster\",\"location\":\"us-central1-a\",\"project_id\":\"elastic-siem\"},\"type\":\"k8s_cluster\"},\"timestamp\":\"2021-04-29T08:23:18.899153Z\"}", - "outcome": "success" + "outcome": "success", + "provider": "data_access", + "type": [ + "access", + "allowed" + ] }, "gcp": { "audit": { @@ -821,12 +1025,14 @@ "resource": "api/v1" } ], + "labels": { + "authorization.k8s.io/decision": "allow", + "authorization.k8s.io/reason": "RBAC: allowed by ClusterRoleBinding \"system:discovery\" of ClusterRole \"system:discovery\" to Group \"system:authenticated\"" + }, "method_name": "io.k8s.get", "request_metadata": { - "caller_ip": "::1", "caller_supplied_user_agent": "kube-controller-manager/v1.19.8 (linux/amd64) kubernetes/4f6f69f/system:serviceaccount:kube-system:generic-garbage-collector" }, - "resource_name": "api/v1", "service_name": "k8s.io", "type": "type.googleapis.com/google.cloud.audit.AuditLog" } @@ -835,6 +1041,7 @@ "logger": "projects/elastic-siem/logs/cloudaudit.googleapis.com%2Fdata_access" }, "orchestrator": { + "api_version": "v1", "cluster": { "name": "analysis-cluster" }, @@ -849,9 +1056,6 @@ "tags": [ "preserve_original_event" ], - "user": { - "email": "system:serviceaccount:kube-system:generic-garbage-collector" - }, "user_agent": { "device": { "name": "Other" @@ -865,6 +1069,12 @@ }, { "@timestamp": "2022-02-21T13:57:39.174Z", + "client": { + "user": { + "email": "xxx@xxx.xxx", + "id": "sub" + } + }, "cloud": { "project": { "id": "project" @@ -876,15 +1086,25 @@ }, "event": { "action": "google.iam.admin.v1.ListServiceAccounts", + "category": [ + "network", + "configuration" + ], "id": "03adfb9f-71a3-4f41-9701-29b5542f4d23", "kind": "event", "original": "{\"insertId\":\"03adfb9f-71a3-4f41-9701-29b5542f4d23\",\"logName\":\"projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access\",\"protoPayload\":{\"@type\":\"type.googleapis.com/google.cloud.audit.AuditLog\",\"authenticationInfo\":{\"principalEmail\":\"xxx@xxx.xxx\",\"principalSubject\":\"sub\",\"serviceAccountKeyName\":\"//xxx@xxx\"},\"authorizationInfo\":[{\"granted\":true,\"permission\":\"iam.serviceAccounts.list\",\"resource\":\"projects/project\",\"resourceAttributes\":{}}],\"methodName\":\"google.iam.admin.v1.ListServiceAccounts\",\"request\":{\"@type\":\"type.googleapis.com/google.iam.admin.v1.ListServiceAccountsRequest\",\"name\":\"projects/project\",\"page_token\":\"cg:FFFFFF\"},\"requestMetadata\":{\"callerIp\":\"gce-internal-ip\",\"callerSuppliedUserAgent\":\"google-api-go-client/0.5,gzip(gfe)\",\"destinationAttributes\":{},\"requestAttributes\":{\"auth\":{},\"time\":\"2022-02-21T13:57:39.178418578Z\"}},\"resourceName\":\"projects/project\",\"serviceName\":\"iam.googleapis.com\",\"status\":{}},\"receiveTimestamp\":\"2022-02-21T13:57:39.341344991Z\",\"resource\":{\"labels\":{\"location\":\"global\",\"method\":\"google.iam.admin.v1.ListServiceAccounts\",\"project_id\":\"project\",\"service\":\"iam.googleapis.com\",\"version\":\"v1\"},\"type\":\"api\"},\"severity\":\"INFO\",\"timestamp\":\"2022-02-21T13:57:39.174555198Z\"}", - "outcome": "success" + "outcome": "success", + "provider": "data_access", + "type": [ + "access", + "allowed" + ] }, "gcp": { "audit": { "authentication_info": { - "principal_email": "xxx@xxx.xxx" + "principal_email": "xxx@xxx.xxx", + "principal_subject": "sub" }, "authorization_info": [ { @@ -895,14 +1115,12 @@ ], "method_name": "google.iam.admin.v1.ListServiceAccounts", "request": { + "@type": "type.googleapis.com/google.iam.admin.v1.ListServiceAccountsRequest", "name": "projects/project", - "proto_name": "type.googleapis.com/google.iam.admin.v1.ListServiceAccountsRequest" + "page_token": "cg:FFFFFF" }, "request_metadata": { - "caller_supplied_user_agent": "google-api-go-client/0.5,gzip(gfe)", - "raw": { - "caller_ip": "gce-internal-ip" - } + "caller_supplied_user_agent": "google-api-go-client/0.5,gzip(gfe)" }, "resource_name": "projects/project", "service_name": "iam.googleapis.com", @@ -910,6 +1128,7 @@ } }, "log": { + "level": "INFO", "logger": "projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access" }, "service": { @@ -918,9 +1137,6 @@ "tags": [ "preserve_original_event" ], - "user": { - "email": "xxx@xxx.xxx" - }, "user_agent": { "device": { "name": "Other" @@ -931,6 +1147,11 @@ }, { "@timestamp": "2022-02-21T14:00:40.802Z", + "client": { + "user": { + "email": "xxx@xxx.xxx" + } + }, "cloud": { "project": { "id": "project" @@ -942,10 +1163,18 @@ }, "event": { "action": "io.k8s.authorization.v1beta1.subjectaccessreviews.create", + "category": [ + "network", + "configuration" + ], "id": "03adfb9f-71a3-4f41-9701-29b5542f4d24", "kind": "event", "original": "{\"insertId\":\"03adfb9f-71a3-4f41-9701-29b5542f4d24\",\"labels\":{\"authentication.k8s.io/legacy-token\":\"system:serviceaccount:kube-system:metrics-server\",\"authorization.k8s.io/decision\":\"allow\",\"authorization.k8s.io/reason\":\"RBAC: allowed by ClusterRoleBinding \\\"metrics-server:system:auth-delegator\\\" of ClusterRole \\\"system:auth-delegator\\\" to ServiceAccount \\\"metrics-server/kube-system\\\"\",\"k8s.io/deprecated\":\"true\",\"k8s.io/removed-release\":\"1.22\"},\"logName\":\"projects/project\",\"operation\":{\"first\":true,\"id\":\"924fbbf6-1982-4173-9355-3fca0ab7b0ee\",\"last\":true,\"producer\":\"k8s.io\"},\"protoPayload\":{\"@type\":\"type.googleapis.com/google.cloud.audit.AuditLog\",\"authenticationInfo\":{\"principalEmail\":\"xxx@xxx.xxx\"},\"authorizationInfo\":[{\"granted\":true,\"permission\":\"io.k8s.authorization.v1beta1.subjectaccessreviews.create\",\"resource\":\"authorization.k8s.io/v1beta1/subjectaccessreviews\"}],\"methodName\":\"io.k8s.authorization.v1beta1.subjectaccessreviews.create\",\"request\":{\"@type\":\"authorization.k8s.io/v1beta1.SubjectAccessReview\",\"apiVersion\":\"authorization.k8s.io/v1beta1\",\"kind\":\"SubjectAccessReview\",\"metadata\":{\"creationTimestamp\":null},\"spec\":{\"group\":[\"system:serviceaccounts\",\"system:serviceaccounts:kube-system\",\"system:authenticated\"],\"nonResourceAttributes\":{\"path\":\"/apis/metrics.k8s.io/v1beta1\",\"verb\":\"get\"},\"user\":\"system:serviceaccount:kube-system:resourcequota-controller\"},\"status\":{\"allowed\":false}},\"requestMetadata\":{\"callerIp\":\"67.43.156.13\",\"callerSuppliedUserAgent\":\"metrics-server/v0.0.0 (linux/amd64) kubernetes/$Format\"},\"resourceName\":\"authorization.k8s.io/v1beta1/subjectaccessreviews\",\"response\":{\"@type\":\"authorization.k8s.io/v1beta1.SubjectAccessReview\",\"apiVersion\":\"authorization.k8s.io/v1beta1\",\"kind\":\"SubjectAccessReview\",\"metadata\":{\"creationTimestamp\":null,\"managedFields\":[{\"apiVersion\":\"authorization.k8s.io/v1beta1\",\"fieldsType\":\"FieldsV1\",\"fieldsV1\":{\"f:spec\":{\"f:group\":{},\"f:nonResourceAttributes\":{\".\":{},\"f:path\":{},\"f:verb\":{}},\"f:user\":{}}},\"manager\":\"metrics-server\",\"operation\":\"Update\",\"time\":\"2022-02-21T14:00:40Z\"}]},\"spec\":{\"group\":[\"system:serviceaccounts\",\"system:serviceaccounts:kube-system\",\"system:authenticated\"],\"nonResourceAttributes\":{\"path\":\"/apis/metrics.k8s.io/v1beta1\",\"verb\":\"get\"},\"user\":\"system:serviceaccount:kube-system:resourcequota-controller\"},\"status\":{\"allowed\":true,\"reason\":\"RBAC: allowed by ClusterRoleBinding \\\"system:discovery\\\" of ClusterRole \\\"system:discovery\\\" to Group \\\"system:authenticated\\\"\"}},\"serviceName\":\"k8s.io\",\"status\":{}},\"receiveTimestamp\":\"2022-02-21T14:00:42.030209174Z\",\"resource\":{\"labels\":{\"cluster_name\":\"elastic\",\"location\":\"europe-west1\",\"project_id\":\"project\"},\"type\":\"k8s_cluster\"},\"timestamp\":\"2022-02-21T14:00:40.802327Z\"}", - "outcome": "success" + "outcome": "success", + "type": [ + "access", + "allowed" + ] }, "gcp": { "audit": { @@ -959,17 +1188,67 @@ "resource": "authorization.k8s.io/v1beta1/subjectaccessreviews" } ], + "labels": { + "authentication.k8s.io/legacy-token": "system:serviceaccount:kube-system:metrics-server", + "authorization.k8s.io/decision": "allow", + "authorization.k8s.io/reason": "RBAC: allowed by ClusterRoleBinding \"metrics-server:system:auth-delegator\" of ClusterRole \"system:auth-delegator\" to ServiceAccount \"metrics-server/kube-system\"", + "k8s.io/deprecated": "true", + "k8s.io/removed-release": "1.22" + }, + "logentry_operation": { + "id": "924fbbf6-1982-4173-9355-3fca0ab7b0ee" + }, "method_name": "io.k8s.authorization.v1beta1.subjectaccessreviews.create", "request": { - "proto_name": "authorization.k8s.io/v1beta1.SubjectAccessReview" + "@type": "authorization.k8s.io/v1beta1.SubjectAccessReview", + "apiVersion": "authorization.k8s.io/v1beta1", + "kind": "SubjectAccessReview", + "spec": { + "group": [ + "system:serviceaccounts", + "system:serviceaccounts:kube-system", + "system:authenticated" + ], + "nonResourceAttributes": { + "path": "/apis/metrics.k8s.io/v1beta1", + "verb": "get" + }, + "user": "system:serviceaccount:kube-system:resourcequota-controller" + }, + "status": { + "allowed": false + } }, "request_metadata": { - "caller_ip": "67.43.156.13", "caller_supplied_user_agent": "metrics-server/v0.0.0 (linux/amd64) kubernetes/$Format" }, - "resource_name": "authorization.k8s.io/v1beta1/subjectaccessreviews", "response": { - "proto_name": "authorization.k8s.io/v1beta1.SubjectAccessReview", + "@type": "authorization.k8s.io/v1beta1.SubjectAccessReview", + "apiVersion": "authorization.k8s.io/v1beta1", + "kind": "SubjectAccessReview", + "metadata": { + "managedFields": [ + { + "apiVersion": "authorization.k8s.io/v1beta1", + "fieldsType": "FieldsV1", + "manager": "metrics-server", + "operation": "Update", + "time": "2022-02-21T14:00:40Z" + } + ] + }, + "spec": { + "group": [ + "system:serviceaccounts", + "system:serviceaccounts:kube-system", + "system:authenticated" + ], + "nonResourceAttributes": { + "path": "/apis/metrics.k8s.io/v1beta1", + "verb": "get" + }, + "user": "system:serviceaccount:kube-system:resourcequota-controller" + }, "status": { "allowed": true, "reason": "RBAC: allowed by ClusterRoleBinding \"system:discovery\" of ClusterRole \"system:discovery\" to Group \"system:authenticated\"" @@ -983,9 +1262,13 @@ "logger": "projects/project" }, "orchestrator": { + "api_version": "v1beta1", "cluster": { "name": "elastic" }, + "resource": { + "type": "subjectaccessreviews" + }, "type": "kubernetes" }, "service": { @@ -1009,9 +1292,6 @@ "tags": [ "preserve_original_event" ], - "user": { - "email": "xxx@xxx.xxx" - }, "user_agent": { "device": { "name": "Other" @@ -1022,6 +1302,384 @@ "name": "Linux" } } + }, + { + "@timestamp": "2022-03-21T19:46:36.090Z", + "client": { + "user": { + "email": "system:addon-manager" + } + }, + "cloud": { + "project": { + "id": "iammai-340819" + }, + "provider": "gcp" + }, + "ecs": { + "version": "8.2.0" + }, + "event": { + "action": "io.k8s.apps.v1.deployments.patch", + "category": [ + "network", + "configuration" + ], + "id": "e5132c86-462b-41b3-9b6a-47966addbb0b", + "kind": "event", + "original": "{\"insertId\": \"e5132c86-462b-41b3-9b6a-47966addbb0b\",\"labels\": {\"authorization.k8s.io/decision\": \"allow\",\"authorization.k8s.io/reason\": \"\"},\"logName\": \"projects/iammai-340819/logs/cloudaudit.googleapis.com%2Factivity\",\"operation\": {\"first\": true,\"id\": \"e5132c86-462b-41b3-9b6a-47966addbb0b\",\"last\": true,\"producer\": \"k8s.io\"},\"protoPayload\": {\"@type\": \"type.googleapis.com/google.cloud.audit.AuditLog\",\"authenticationInfo\": {\"principalEmail\": \"system:addon-manager\"},\"authorizationInfo\": [ { \"granted\": true, \"permission\": \"io.k8s.apps.v1.deployments.patch\", \"resource\": \"apps/v1/namespaces/kube-system/deployments/konnectivity-agent\" } ], \"methodName\": \"io.k8s.apps.v1.deployments.patch\", \"request\": { \"@type\": \"k8s.io/Patch\", \"spec\": { \"strategy\": { \"$retainKeys\": [ \"type\" ] }, \"template\": { \"spec\": { \"$setElementOrder/volumes\": [ { \"name\": \"konnectivity-agent-token\" } ], \"volumes\": [ { \"$retainKeys\": [ \"name\", \"projected\" ], \"name\": \"konnectivity-agent-token\", \"projected\": { \"sources\": [ { \"serviceAccountToken\": { \"audience\": \"system:konnectivity-server\", \"path\": \"konnectivity-agent-token\" } } ] } } ] } } } }, \"requestMetadata\": { \"callerIp\": \"10.142.0.152\", \"callerSuppliedUserAgent\": \"kubectl/v1.20.2 (linux/amd64) kubernetes/faecb19\" }, \"resourceName\": \"apps/v1/namespaces/kube-system/deployments/konnectivity-agent\", \"response\": { \"@type\": \"apps.k8s.io/v1.Deployment\", \"apiVersion\": \"apps/v1\", \"kind\": \"Deployment\", \"metadata\": { \"annotations\": { \"components.gke.io/layer\": \"addon\", \"deployment.kubernetes.io/revision\": \"1\", \"kubectl.kubernetes.io/last-applied-configuration\": \"{\\\"apiVersion\\\":\\\"apps/v1\\\",\\\"kind\\\":\\\"Deployment\\\",\\\"metadata\\\":{\\\"annotations\\\":{\\\"components.gke.io/layer\\\":\\\"addon\\\"},\\\"labels\\\":{\\\"addonmanager.kubernetes.io/mode\\\":\\\"Reconcile\\\",\\\"k8s-app\\\":\\\"konnectivity-agent\\\"},\\\"name\\\":\\\"konnectivity-agent\\\",\\\"namespace\\\":\\\"kube-system\\\"},\\\"spec\\\":{\\\"selector\\\":{\\\"matchLabels\\\":{\\\"k8s-app\\\":\\\"konnectivity-agent\\\"}},\\\"strategy\\\":{\\\"type\\\":\\\"RollingUpdate\\\"},\\\"template\\\":{\\\"metadata\\\":{\\\"annotations\\\":{\\\"cluster-autoscaler.kubernetes.io/safe-to-evict\\\":\\\"true\\\",\\\"components.gke.io/component-name\\\":\\\"konnectivitynetworkproxy-combined\\\",\\\"components.gke.io/component-version\\\":\\\"1.3.3\\\"},\\\"labels\\\":{\\\"k8s-app\\\":\\\"konnectivity-agent\\\"}},\\\"spec\\\":{\\\"containers\\\":[{\\\"args\\\":[\\\"--logtostderr=true\\\",\\\"--ca-cert=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt\\\",\\\"--proxy-server-host=34.75.195.103\\\",\\\"--proxy-server-port=8132\\\",\\\"--health-server-port=8093\\\",\\\"--admin-server-port=8094\\\",\\\"--sync-interval=5s\\\",\\\"--probe-interval=5s\\\",\\\"--service-account-token-path=/var/run/secrets/tokens/konnectivity-agent-token\\\",\\\"--v=3\\\"],\\\"command\\\":[\\\"/proxy-agent\\\"],\\\"env\\\":[{\\\"name\\\":\\\"POD_NAME\\\",\\\"valueFrom\\\":{\\\"fieldRef\\\":{\\\"fieldPath\\\":\\\"metadata.name\\\"}}},{\\\"name\\\":\\\"POD_NAMESPACE\\\",\\\"valueFrom\\\":{\\\"fieldRef\\\":{\\\"fieldPath\\\":\\\"metadata.namespace\\\"}}}],\\\"image\\\":\\\"gke.gcr.io/proxy-agent:v0.0.24-gke.0\\\",\\\"livenessProbe\\\":{\\\"httpGet\\\":{\\\"path\\\":\\\"/healthz\\\",\\\"port\\\":8093},\\\"initialDelaySeconds\\\":15,\\\"timeoutSeconds\\\":15},\\\"name\\\":\\\"konnectivity-agent\\\",\\\"ports\\\":[{\\\"containerPort\\\":8093,\\\"name\\\":\\\"metrics\\\",\\\"protocol\\\":\\\"TCP\\\"}],\\\"resources\\\":{\\\"limits\\\":{\\\"memory\\\":\\\"125Mi\\\"},\\\"requests\\\":{\\\"cpu\\\":\\\"10m\\\",\\\"memory\\\":\\\"30Mi\\\"}},\\\"securityContext\\\":{\\\"allowPrivilegeEscalation\\\":false,\\\"capabilities\\\":{\\\"drop\\\":[\\\"all\\\"]}},\\\"volumeMounts\\\":[{\\\"mountPath\\\":\\\"/var/run/secrets/tokens\\\",\\\"name\\\":\\\"konnectivity-agent-token\\\"}]}],\\\"nodeSelector\\\":{\\\"beta.kubernetes.io/os\\\":\\\"linux\\\"},\\\"priorityClassName\\\":\\\"system-cluster-critical\\\",\\\"securityContext\\\":{\\\"fsGroup\\\":1000,\\\"runAsGroup\\\":1000,\\\"runAsUser\\\":1000},\\\"serviceAccountName\\\":\\\"konnectivity-agent\\\",\\\"tolerations\\\":[{\\\"key\\\":\\\"CriticalAddonsOnly\\\",\\\"operator\\\":\\\"Exists\\\"},{\\\"effect\\\":\\\"NoSchedule\\\",\\\"key\\\":\\\"sandbox.gke.io/runtime\\\",\\\"operator\\\":\\\"Equal\\\",\\\"value\\\":\\\"gvisor\\\"},{\\\"key\\\":\\\"components.gke.io/gke-managed-components\\\",\\\"operator\\\":\\\"Exists\\\"}],\\\"topologySpreadConstraints\\\":[{\\\"labelSelector\\\":{\\\"matchLabels\\\":{\\\"k8s-app\\\":\\\"konnectivity-agent\\\"}},\\\"maxSkew\\\":1,\\\"topologyKey\\\":\\\"topology.kubernetes.io/zone\\\",\\\"whenUnsatisfiable\\\":\\\"ScheduleAnyway\\\"},{\\\"labelSelector\\\":{\\\"matchLabels\\\":{\\\"k8s-app\\\":\\\"konnectivity-agent\\\"}},\\\"maxSkew\\\":1,\\\"topologyKey\\\":\\\"kubernetes.io/hostname\\\",\\\"whenUnsatisfiable\\\":\\\"ScheduleAnyway\\\"}],\\\"volumes\\\":[{\\\"name\\\":\\\"konnectivity-agent-token\\\",\\\"projected\\\":{\\\"sources\\\":[{\\\"serviceAccountToken\\\":{\\\"audience\\\":\\\"system:konnectivity-server\\\",\\\"path\\\":\\\"konnectivity-agent-token\\\"}}]}}]}}}}\" }, \"creationTimestamp\": \"2022-03-16T21:29:13Z\", \"generation\": 2, \"labels\": { \"addonmanager.kubernetes.io/mode\": \"Reconcile\", \"k8s-app\": \"konnectivity-agent\" }, \"managedFields\": [ { \"apiVersion\": \"apps/v1\", \"fieldsType\": \"FieldsV1\", \"fieldsV1\": { \"f:metadata\": { \"f:annotations\": { \".\": {}, \"f:components.gke.io/layer\": {}, \"f:kubectl.kubernetes.io/last-applied-configuration\": {} }, \"f:labels\": { \".\": {}, \"f:addonmanager.kubernetes.io/mode\": {}, \"f:k8s-app\": {} } }, \"f:spec\": { \"f:progressDeadlineSeconds\": {}, \"f:replicas\": {}, \"f:revisionHistoryLimit\": {}, \"f:selector\": {}, \"f:strategy\": { \"f:rollingUpdate\": { \".\": {}, \"f:maxSurge\": {}, \"f:maxUnavailable\": {} }, \"f:type\": {} }, \"f:template\": { \"f:metadata\": { \"f:annotations\": { \".\": {}, \"f:cluster-autoscaler.kubernetes.io/safe-to-evict\": {}, \"f:components.gke.io/component-name\": {}, \"f:components.gke.io/component-version\": {} }, \"f:labels\": { \".\": {}, \"f:k8s-app\": {} } }, \"f:spec\": { \"f:containers\": { \"k:{\\\"name\\\":\\\"konnectivity-agent\\\"}\": { \".\": {}, \"f:args\": {}, \"f:command\": {}, \"f:env\": { \".\": {}, \"k:{\\\"name\\\":\\\"POD_NAME\\\"}\": { \".\": {}, \"f:name\": {}, \"f:valueFrom\": { \".\": {}, \"f:fieldRef\": { \".\": {}, \"f:apiVersion\": {}, \"f:fieldPath\": {} } } }, \"k:{\\\"name\\\":\\\"POD_NAMESPACE\\\"}\": { \".\": {}, \"f:name\": {}, \"f:valueFrom\": { \".\": {}, \"f:fieldRef\": { \".\": {}, \"f:apiVersion\": {}, \"f:fieldPath\": {} } } } }, \"f:image\": {}, \"f:imagePullPolicy\": {}, \"f:livenessProbe\": { \".\": {}, \"f:failureThreshold\": {}, \"f:httpGet\": { \".\": {}, \"f:path\": {}, \"f:port\": {}, \"f:scheme\": {} }, \"f:initialDelaySeconds\": {}, \"f:periodSeconds\": {}, \"f:successThreshold\": {}, \"f:timeoutSeconds\": {} }, \"f:name\": {}, \"f:ports\": { \".\": {}, \"k:{\\\"containerPort\\\":8093,\\\"protocol\\\":\\\"TCP\\\"}\": { \".\": {}, \"f:containerPort\": {}, \"f:name\": {}, \"f:protocol\": {} } }, \"f:resources\": { \".\": {}, \"f:limits\": { \".\": {}, \"f:memory\": {} }, \"f:requests\": { \".\": {}, \"f:cpu\": {}, \"f:memory\": {} } }, \"f:securityContext\": { \".\": {}, \"f:allowPrivilegeEscalation\": {}, \"f:capabilities\": { \".\": {}, \"f:drop\": {} } }, \"f:terminationMessagePath\": {}, \"f:terminationMessagePolicy\": {}, \"f:volumeMounts\": { \".\": {}, \"k:{\\\"mountPath\\\":\\\"/var/run/secrets/tokens\\\"}\": { \".\": {}, \"f:mountPath\": {}, \"f:name\": {} } } } }, \"f:dnsPolicy\": {}, \"f:nodeSelector\": { \".\": {}, \"f:beta.kubernetes.io/os\": {} }, \"f:priorityClassName\": {}, \"f:restartPolicy\": {}, \"f:schedulerName\": {}, \"f:securityContext\": { \".\": {}, \"f:fsGroup\": {}, \"f:runAsGroup\": {}, \"f:runAsUser\": {} }, \"f:serviceAccount\": {}, \"f:serviceAccountName\": {}, \"f:terminationGracePeriodSeconds\": {}, \"f:tolerations\": {}, \"f:topologySpreadConstraints\": { \".\": {}, \"k:{\\\"topologyKey\\\":\\\"kubernetes.io/hostname\\\",\\\"whenUnsatisfiable\\\":\\\"ScheduleAnyway\\\"}\": { \".\": {}, \"f:labelSelector\": {}, \"f:maxSkew\": {}, \"f:topologyKey\": {}, \"f:whenUnsatisfiable\": {} }, \"k:{\\\"topologyKey\\\":\\\"topology.kubernetes.io/zone\\\",\\\"whenUnsatisfiable\\\":\\\"ScheduleAnyway\\\"}\": { \".\": {}, \"f:labelSelector\": {}, \"f:maxSkew\": {}, \"f:topologyKey\": {}, \"f:whenUnsatisfiable\": {} } }, \"f:volumes\": { \".\": {}, \"k:{\\\"name\\\":\\\"konnectivity-agent-token\\\"}\": { \".\": {}, \"f:name\": {}, \"f:projected\": { \".\": {}, \"f:defaultMode\": {}, \"f:sources\": {} } } } } } } }, \"manager\": \"kubectl-client-side-apply\", \"operation\": \"Update\", \"time\": \"2022-03-16T21:29:13Z\" }, { \"apiVersion\": \"apps/v1\", \"fieldsType\": \"FieldsV1\", \"fieldsV1\": { \"f:metadata\": { \"f:annotations\": { \"f:deployment.kubernetes.io/revision\": {} } }, \"f:status\": { \"f:availableReplicas\": {}, \"f:conditions\": { \".\": {}, \"k:{\\\"type\\\":\\\"Available\\\"}\": { \".\": {}, \"f:lastTransitionTime\": {}, \"f:lastUpdateTime\": {}, \"f:message\": {}, \"f:reason\": {}, \"f:status\": {}, \"f:type\": {} }, \"k:{\\\"type\\\":\\\"Progressing\\\"}\": { \".\": {}, \"f:lastTransitionTime\": {}, \"f:lastUpdateTime\": {}, \"f:message\": {}, \"f:reason\": {}, \"f:status\": {}, \"f:type\": {} } }, \"f:observedGeneration\": {}, \"f:readyReplicas\": {}, \"f:replicas\": {}, \"f:updatedReplicas\": {} } }, \"manager\": \"kube-controller-manager\", \"operation\": \"Update\", \"time\": \"2022-03-17T08:55:52Z\" } ], \"name\": \"konnectivity-agent\", \"namespace\": \"kube-system\", \"resourceVersion\": \"280105\", \"uid\": \"d3b49e97-7bac-435e-bfc6-19a25fe494fe\" }, \"spec\": { \"progressDeadlineSeconds\": 600, \"replicas\": 6, \"revisionHistoryLimit\": 10, \"selector\": { \"matchLabels\": { \"k8s-app\": \"konnectivity-agent\" } }, \"strategy\": { \"rollingUpdate\": { \"maxSurge\": \"25%\", \"maxUnavailable\": \"25%\" }, \"type\": \"RollingUpdate\" }, \"template\": { \"metadata\": { \"annotations\": { \"cluster-autoscaler.kubernetes.io/safe-to-evict\": \"true\", \"components.gke.io/component-name\": \"konnectivitynetworkproxy-combined\", \"components.gke.io/component-version\": \"1.3.3\" }, \"creationTimestamp\": null, \"labels\": { \"k8s-app\": \"konnectivity-agent\" } }, \"spec\": { \"containers\": [ { \"args\": [ \"--logtostderr=true\", \"--ca-cert=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt\", \"--proxy-server-host=34.75.195.103\", \"--proxy-server-port=8132\", \"--health-server-port=8093\", \"--admin-server-port=8094\", \"--sync-interval=5s\", \"--probe-interval=5s\", \"--service-account-token-path=/var/run/secrets/tokens/konnectivity-agent-token\", \"--v=3\" ], \"command\": [ \"/proxy-agent\" ], \"env\": [ { \"name\": \"POD_NAME\", \"valueFrom\": { \"fieldRef\": { \"apiVersion\": \"v1\", \"fieldPath\": \"metadata.name\" } } }, { \"name\": \"POD_NAMESPACE\", \"valueFrom\": { \"fieldRef\": { \"apiVersion\": \"v1\", \"fieldPath\": \"metadata.namespace\" } } } ], \"image\": \"gke.gcr.io/proxy-agent:v0.0.24-gke.0\", \"imagePullPolicy\": \"IfNotPresent\", \"livenessProbe\": { \"failureThreshold\": 3, \"httpGet\": { \"path\": \"/healthz\", \"port\": 8093, \"scheme\": \"HTTP\" }, \"initialDelaySeconds\": 15, \"periodSeconds\": 10, \"successThreshold\": 1, \"timeoutSeconds\": 15 }, \"name\": \"konnectivity-agent\", \"ports\": [ { \"containerPort\": 8093, \"name\": \"metrics\", \"protocol\": \"TCP\" } ], \"resources\": { \"limits\": { \"memory\": \"125Mi\" }, \"requests\": { \"cpu\": \"10m\", \"memory\": \"30Mi\" } }, \"securityContext\": { \"allowPrivilegeEscalation\": false, \"capabilities\": { \"drop\": [ \"all\" ] } }, \"terminationMessagePath\": \"/dev/termination-log\", \"terminationMessagePolicy\": \"File\", \"volumeMounts\": [ { \"mountPath\": \"/var/run/secrets/tokens\", \"name\": \"konnectivity-agent-token\" } ] } ], \"dnsPolicy\": \"ClusterFirst\", \"nodeSelector\": { \"beta.kubernetes.io/os\": \"linux\" }, \"priorityClassName\": \"system-cluster-critical\", \"restartPolicy\": \"Always\", \"schedulerName\": \"default-scheduler\", \"securityContext\": { \"fsGroup\": 1000, \"runAsGroup\": 1000, \"runAsUser\": 1000 }, \"serviceAccount\": \"konnectivity-agent\", \"serviceAccountName\": \"konnectivity-agent\", \"terminationGracePeriodSeconds\": 30, \"tolerations\": [ { \"key\": \"CriticalAddonsOnly\", \"operator\": \"Exists\" }, { \"effect\": \"NoSchedule\", \"key\": \"sandbox.gke.io/runtime\", \"operator\": \"Equal\", \"value\": \"gvisor\" }, { \"key\": \"components.gke.io/gke-managed-components\", \"operator\": \"Exists\" } ], \"topologySpreadConstraints\": [ { \"labelSelector\": { \"matchLabels\": { \"k8s-app\": \"konnectivity-agent\" } }, \"maxSkew\": 1, \"topologyKey\": \"topology.kubernetes.io/zone\", \"whenUnsatisfiable\": \"ScheduleAnyway\" }, { \"labelSelector\": { \"matchLabels\": { \"k8s-app\": \"konnectivity-agent\" } }, \"maxSkew\": 1, \"topologyKey\": \"kubernetes.io/hostname\", \"whenUnsatisfiable\": \"ScheduleAnyway\" } ], \"volumes\": [ { \"name\": \"konnectivity-agent-token\", \"projected\": { \"defaultMode\": 420, \"sources\": [ { \"serviceAccountToken\": { \"audience\": \"system:konnectivity-server\", \"expirationSeconds\": 3600, \"path\": \"konnectivity-agent-token\" } } ] } } ] } } }, \"status\": { \"availableReplicas\": 6, \"conditions\": [ { \"lastTransitionTime\": \"2022-03-17T08:55:41Z\", \"lastUpdateTime\": \"2022-03-17T08:55:41Z\", \"message\": \"ReplicaSet \\\"konnectivity-agent-56c9b8cf8\\\" has successfully progressed.\", \"reason\": \"NewReplicaSetAvailable\", \"status\": \"True\", \"type\": \"Progressing\" }, { \"lastTransitionTime\": \"2022-03-17T08:55:52Z\", \"lastUpdateTime\": \"2022-03-17T08:55:52Z\", \"message\": \"Deployment has minimum availability.\", \"reason\": \"MinimumReplicasAvailable\", \"status\": \"True\", \"type\": \"Available\" } ], \"observedGeneration\": 2, \"readyReplicas\": 6, \"replicas\": 6, \"updatedReplicas\": 6 } }, \"serviceName\": \"k8s.io\", \"status\": {} }, \"receiveTimestamp\": \"2022-03-21T19:46:38.090036928Z\", \"resource\": { \"labels\": { \"cluster_name\": \"iammai-340819-gke-cluster\", \"location\": \"us-east1\", \"project_id\": \"iammai-340819\" }, \"type\": \"k8s_cluster\" }, \"timestamp\": \"2022-03-21T19:46:36.090498Z\" }", + "outcome": "success", + "provider": "activity", + "type": [ + "access", + "allowed" + ] + }, + "gcp": { + "audit": { + "authentication_info": { + "principal_email": "system:addon-manager" + }, + "authorization_info": [ + { + "granted": true, + "permission": "io.k8s.apps.v1.deployments.patch", + "resource": "apps/v1/namespaces/kube-system/deployments/konnectivity-agent" + } + ], + "labels": { + "authorization.k8s.io/decision": "allow" + }, + "method_name": "io.k8s.apps.v1.deployments.patch", + "request": { + "@type": "k8s.io/Patch", + "spec": { + "strategy": { + "$retainKeys": [ + "type" + ] + }, + "template": { + "spec": { + "$setElementOrder/volumes": [ + { + "name": "konnectivity-agent-token" + } + ], + "volumes": [ + { + "$retainKeys": [ + "name", + "projected" + ], + "name": "konnectivity-agent-token", + "projected": { + "sources": [ + { + "serviceAccountToken": { + "audience": "system:konnectivity-server", + "path": "konnectivity-agent-token" + } + } + ] + } + } + ] + } + } + } + }, + "request_metadata": { + "caller_supplied_user_agent": "kubectl/v1.20.2 (linux/amd64) kubernetes/faecb19" + }, + "response": { + "@type": "apps.k8s.io/v1.Deployment", + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "annotations": { + "components.gke.io/layer": "addon", + "deployment.kubernetes.io/revision": "1", + "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"apps/v1\",\"kind\":\"Deployment\",\"metadata\":{\"annotations\":{\"components.gke.io/layer\":\"addon\"},\"labels\":{\"addonmanager.kubernetes.io/mode\":\"Reconcile\",\"k8s-app\":\"konnectivity-agent\"},\"name\":\"konnectivity-agent\",\"namespace\":\"kube-system\"},\"spec\":{\"selector\":{\"matchLabels\":{\"k8s-app\":\"konnectivity-agent\"}},\"strategy\":{\"type\":\"RollingUpdate\"},\"template\":{\"metadata\":{\"annotations\":{\"cluster-autoscaler.kubernetes.io/safe-to-evict\":\"true\",\"components.gke.io/component-name\":\"konnectivitynetworkproxy-combined\",\"components.gke.io/component-version\":\"1.3.3\"},\"labels\":{\"k8s-app\":\"konnectivity-agent\"}},\"spec\":{\"containers\":[{\"args\":[\"--logtostderr=true\",\"--ca-cert=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt\",\"--proxy-server-host=34.75.195.103\",\"--proxy-server-port=8132\",\"--health-server-port=8093\",\"--admin-server-port=8094\",\"--sync-interval=5s\",\"--probe-interval=5s\",\"--service-account-token-path=/var/run/secrets/tokens/konnectivity-agent-token\",\"--v=3\"],\"command\":[\"/proxy-agent\"],\"env\":[{\"name\":\"POD_NAME\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.name\"}}},{\"name\":\"POD_NAMESPACE\",\"valueFrom\":{\"fieldRef\":{\"fieldPath\":\"metadata.namespace\"}}}],\"image\":\"gke.gcr.io/proxy-agent:v0.0.24-gke.0\",\"livenessProbe\":{\"httpGet\":{\"path\":\"/healthz\",\"port\":8093},\"initialDelaySeconds\":15,\"timeoutSeconds\":15},\"name\":\"konnectivity-agent\",\"ports\":[{\"containerPort\":8093,\"name\":\"metrics\",\"protocol\":\"TCP\"}],\"resources\":{\"limits\":{\"memory\":\"125Mi\"},\"requests\":{\"cpu\":\"10m\",\"memory\":\"30Mi\"}},\"securityContext\":{\"allowPrivilegeEscalation\":false,\"capabilities\":{\"drop\":[\"all\"]}},\"volumeMounts\":[{\"mountPath\":\"/var/run/secrets/tokens\",\"name\":\"konnectivity-agent-token\"}]}],\"nodeSelector\":{\"beta.kubernetes.io/os\":\"linux\"},\"priorityClassName\":\"system-cluster-critical\",\"securityContext\":{\"fsGroup\":1000,\"runAsGroup\":1000,\"runAsUser\":1000},\"serviceAccountName\":\"konnectivity-agent\",\"tolerations\":[{\"key\":\"CriticalAddonsOnly\",\"operator\":\"Exists\"},{\"effect\":\"NoSchedule\",\"key\":\"sandbox.gke.io/runtime\",\"operator\":\"Equal\",\"value\":\"gvisor\"},{\"key\":\"components.gke.io/gke-managed-components\",\"operator\":\"Exists\"}],\"topologySpreadConstraints\":[{\"labelSelector\":{\"matchLabels\":{\"k8s-app\":\"konnectivity-agent\"}},\"maxSkew\":1,\"topologyKey\":\"topology.kubernetes.io/zone\",\"whenUnsatisfiable\":\"ScheduleAnyway\"},{\"labelSelector\":{\"matchLabels\":{\"k8s-app\":\"konnectivity-agent\"}},\"maxSkew\":1,\"topologyKey\":\"kubernetes.io/hostname\",\"whenUnsatisfiable\":\"ScheduleAnyway\"}],\"volumes\":[{\"name\":\"konnectivity-agent-token\",\"projected\":{\"sources\":[{\"serviceAccountToken\":{\"audience\":\"system:konnectivity-server\",\"path\":\"konnectivity-agent-token\"}}]}}]}}}}" + }, + "creationTimestamp": "2022-03-16T21:29:13Z", + "generation": 2, + "labels": { + "addonmanager.kubernetes.io/mode": "Reconcile", + "k8s-app": "konnectivity-agent" + }, + "managedFields": [ + { + "apiVersion": "apps/v1", + "fieldsType": "FieldsV1", + "manager": "kubectl-client-side-apply", + "operation": "Update", + "time": "2022-03-16T21:29:13Z" + }, + { + "apiVersion": "apps/v1", + "fieldsType": "FieldsV1", + "manager": "kube-controller-manager", + "operation": "Update", + "time": "2022-03-17T08:55:52Z" + } + ], + "name": "konnectivity-agent", + "namespace": "kube-system", + "resourceVersion": "280105", + "uid": "d3b49e97-7bac-435e-bfc6-19a25fe494fe" + }, + "spec": { + "progressDeadlineSeconds": 600, + "replicas": 6, + "revisionHistoryLimit": 10, + "selector": { + "matchLabels": { + "k8s-app": "konnectivity-agent" + } + }, + "strategy": { + "rollingUpdate": { + "maxSurge": "25%", + "maxUnavailable": "25%" + }, + "type": "RollingUpdate" + }, + "template": { + "metadata": { + "annotations": { + "cluster-autoscaler.kubernetes.io/safe-to-evict": "true", + "components.gke.io/component-name": "konnectivitynetworkproxy-combined", + "components.gke.io/component-version": "1.3.3" + }, + "labels": { + "k8s-app": "konnectivity-agent" + } + }, + "spec": { + "containers": [ + { + "args": [ + "--logtostderr=true", + "--ca-cert=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", + "--proxy-server-host=34.75.195.103", + "--proxy-server-port=8132", + "--health-server-port=8093", + "--admin-server-port=8094", + "--sync-interval=5s", + "--probe-interval=5s", + "--service-account-token-path=/var/run/secrets/tokens/konnectivity-agent-token", + "--v=3" + ], + "command": [ + "/proxy-agent" + ], + "env": [ + { + "name": "POD_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.name" + } + } + }, + { + "name": "POD_NAMESPACE", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + } + } + } + ], + "image": "gke.gcr.io/proxy-agent:v0.0.24-gke.0", + "imagePullPolicy": "IfNotPresent", + "livenessProbe": { + "failureThreshold": 3, + "httpGet": { + "path": "/healthz", + "port": 8093, + "scheme": "HTTP" + }, + "initialDelaySeconds": 15, + "periodSeconds": 10, + "successThreshold": 1, + "timeoutSeconds": 15 + }, + "name": "konnectivity-agent", + "ports": [ + { + "containerPort": 8093, + "name": "metrics", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "125Mi" + }, + "requests": { + "cpu": "10m", + "memory": "30Mi" + } + }, + "securityContext": { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "all" + ] + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/tokens", + "name": "konnectivity-agent-token" + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "nodeSelector": { + "beta.kubernetes.io/os": "linux" + }, + "priorityClassName": "system-cluster-critical", + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": { + "fsGroup": 1000, + "runAsGroup": 1000, + "runAsUser": 1000 + }, + "serviceAccount": "konnectivity-agent", + "serviceAccountName": "konnectivity-agent", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "key": "CriticalAddonsOnly", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "sandbox.gke.io/runtime", + "operator": "Equal", + "value": "gvisor" + }, + { + "key": "components.gke.io/gke-managed-components", + "operator": "Exists" + } + ], + "topologySpreadConstraints": [ + { + "labelSelector": { + "matchLabels": { + "k8s-app": "konnectivity-agent" + } + }, + "maxSkew": 1, + "topologyKey": "topology.kubernetes.io/zone", + "whenUnsatisfiable": "ScheduleAnyway" + }, + { + "labelSelector": { + "matchLabels": { + "k8s-app": "konnectivity-agent" + } + }, + "maxSkew": 1, + "topologyKey": "kubernetes.io/hostname", + "whenUnsatisfiable": "ScheduleAnyway" + } + ], + "volumes": [ + { + "name": "konnectivity-agent-token", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "audience": "system:konnectivity-server", + "expirationSeconds": 3600, + "path": "konnectivity-agent-token" + } + } + ] + } + } + ] + } + } + }, + "status": { + "availableReplicas": 6, + "conditions": [ + { + "lastTransitionTime": "2022-03-17T08:55:41Z", + "lastUpdateTime": "2022-03-17T08:55:41Z", + "message": "ReplicaSet \"konnectivity-agent-56c9b8cf8\" has successfully progressed.", + "reason": "NewReplicaSetAvailable", + "status": "True", + "type": "Progressing" + }, + { + "lastTransitionTime": "2022-03-17T08:55:52Z", + "lastUpdateTime": "2022-03-17T08:55:52Z", + "message": "Deployment has minimum availability.", + "reason": "MinimumReplicasAvailable", + "status": "True", + "type": "Available" + } + ], + "observedGeneration": 2, + "readyReplicas": 6, + "replicas": 6, + "updatedReplicas": 6 + } + }, + "service_name": "k8s.io", + "type": "type.googleapis.com/google.cloud.audit.AuditLog" + } + }, + "log": { + "logger": "projects/iammai-340819/logs/cloudaudit.googleapis.com%2Factivity" + }, + "orchestrator": { + "api_version": "v1", + "cluster": { + "name": "iammai-340819-gke-cluster" + }, + "namespace": "kube-system", + "resource": { + "name": "konnectivity-agent", + "type": "deployments" + }, + "type": "kubernetes" + }, + "service": { + "name": "k8s.io" + }, + "source": { + "ip": "10.142.0.152" + }, + "tags": [ + "preserve_original_event" + ], + "user_agent": { + "device": { + "name": "Other" + }, + "name": "Other", + "original": "kubectl/v1.20.2 (linux/amd64) kubernetes/faecb19", + "os": { + "name": "Linux" + } + } } ] } \ No newline at end of file diff --git a/packages/gcp/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/gcp/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index 2bd5c9dec31..fc0ff88942c 100644 --- a/packages/gcp/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/gcp/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -12,6 +12,43 @@ processors: - json: field: event.original target_field: json + - set: + field: gcp.audit.type + copy_from: "json.protoPayload.@type" + ignore_failure: true +## +# https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry +# protoPayload @type must be type.googleapis.com/google.cloud.audit.AuditLog +## + - drop: + description: Drop the document if it is not of AuditLog type + if: ctx.gcp?.audit?.type != null && ctx.gcp?.audit?.type != 'type.googleapis.com/google.cloud.audit.AuditLog' +# .insertId + - set: + field: event.id + copy_from: json.insertId + if: ctx.json?.insertId != null +# .logName + - rename: + field: json.logName + target_field: log.logger + ignore_missing: true +# .severity + - rename: + field: json.severity + target_field: log.level + ignore_missing: true +## +# Extract the type of audit logging data from logName to event.provider +# https://cloud.google.com/pubsub/docs/audit-logging#log_name +## + - dissect: + field: log.logger + pattern: "%{}%2F%{event.provider}" + ignore_missing: true + # NOTE test data fails the spec + ignore_failure: true + - set: field: event.kind value: event @@ -23,92 +60,121 @@ processors: timezone: UTC formats: - ISO8601 - - rename: - field: json.logName - target_field: log.logger - ignore_missing: true - - set: - field: event.id - copy_from: json.insertId +## +# MonitoredResource +# .resource +# MonitoredResource https://cloud.google.com/logging/docs/reference/v2/rest/v2/MonitoredResource +## + - set: + field: cloud.project.id + copy_from: json.resource.labels.project_id + if: ctx.json?.resource?.labels?.project_id != null + - set: + field: cloud.instance.id + copy_from: json.resource.labels.instance_id + if: ctx.json?.resource?.labels?.instance_id != null +## +# MonitoredResourceDescriptor type +# https://cloud.google.com/logging/docs/reference/v2/rest/v2/monitoredResourceDescriptors/list#MonitoredResourceDescriptor +# resource list values https://cloud.google.com/logging/docs/api/v2/resource-list +## + - set: + field: orchestrator.type + value: kubernetes + if: ctx.json?.resource?.type != null && (ctx.json?.resource?.type == 'k8s_cluster' || ctx.json?.resource?.type == 'gke_cluster') + - set: + field: orchestrator.cluster.name + copy_from: json.resource.labels.cluster_name ignore_empty_value: true - ignore_failure: true - - convert: - field: json.resource.labels.project_id - target_field: cloud.project.id - type: string - ignore_missing: true - ignore_failure: true - - convert: - field: json.resource.labels.instance_id - target_field: cloud.instance.id - type: string - ignore_missing: true - ignore_failure: true - - rename: - field: "json.protoPayload.@type" - target_field: gcp.audit.type + if: ctx.json?.resource?.type != null && (ctx.json?.resource?.type == 'k8s_cluster' || ctx.json?.resource?.type == 'gke_cluster') + - set: + field: _temp.type + copy_from: json.protoPayload.resourceName + ignore_empty_value: true + if: ctx.json?.resource?.type != null && ctx.json?.resource?.type == 'k8s_cluster' + - grok: + field: _temp.type + patterns: + - '%{DATA}/%{API_VERSION:orchestrator.api_version}/namespaces/%{DATA:orchestrator.namespace}/%{RESOURCE_TYPE:orchestrator.resource.type}(/%{HOSTNAME:orchestrator.resource.name})?' + - '%{DATA}/%{API_VERSION:orchestrator.api_version}/%{RESOURCE_TYPE:orchestrator.resource.type}' + - 'apis/%{RESOURCE_TYPE:orchestrator.resource.type}/%{API_VERSION:orchestrator.api_version}' + - 'api/%{API_VERSION:orchestrator.api_version}' + - '%{RESOURCE_TYPE:orchestrator.resource.type}' + pattern_definitions: + API_VERSION: (v\d+([a-z]+)?(\d+)?) + RESOURCE_TYPE: ([a-z]+((\.[a-z0-9]+)+)?) ignore_missing: true + +## +# AuthenticationInfo +# .protoPayload.authenticationInfo +# https://cloud.google.com/logging/docs/reference/audit/auditlog/rest/Shared.Types/AuditLog#AuthenticationInfo +## +# email address of authenticated user (redacted) or service account +# principalEmail -> client.user.email - rename: field: json.protoPayload.authenticationInfo.principalEmail - target_field: gcp.audit.authentication_info.principal_email + target_field: client.user.email + ignore_missing: true +# identity of requesting first or third party +# principalSubject -> client.user.id + - rename: + field: json.protoPayload.authenticationInfo.principalSubject + target_field: client.user.id ignore_missing: true - - set: - field: user.email - value: "{{gcp.audit.authentication_info.principal_email}}" - if: ctx?.gcp?.audit?.authentication_info?.principal_email != null - rename: field: json.protoPayload.authenticationInfo.authoritySelector target_field: gcp.audit.authentication_info.authority_selector ignore_missing: true + +# TODO remove - duplicated in client.user.email and client.user.id + - set: + field: gcp.audit.authentication_info.principal_email + copy_from: client.user.email + if: ctx.client?.user?.email != null + - set: + field: gcp.audit.authentication_info.principal_subject + copy_from: client.user.id + if: ctx.client?.user?.id != null +## +# AuthorizationInfo +# .protoPayload.authorizationInfo +# https://cloud.google.com/logging/docs/reference/audit/auditlog/rest/Shared.Types/AuditLog#authorizationinfo +## - rename: field: json.protoPayload.authorizationInfo target_field: gcp.audit.authorization_info ignore_missing: true - - rename: - field: json.protoPayload.methodName - target_field: gcp.audit.method_name + - foreach: + field: gcp.audit.authorization_info ignore_missing: true + ignore_failure: true + processor: + rename: + field: _ingest._value.resourceAttributes + target_field: _ingest._value.resource_attributes + if: ctx?.gcp?.audit?.authorization_info != null && ctx?.gcp?.audit?.authorization_info instanceof List + +## +# Labels +# .labels +## - set: - field: event.action - value: "{{gcp.audit.method_name}}" - if: ctx?.gcp?.audit?.method_name != null - - convert: - field: json.protoPayload.numResponseItems - target_field: gcp.audit.num_response_items - type: long - ignore_missing: true - - rename: - field: "json.protoPayload.request.@type" - target_field: gcp.audit.request.proto_name - ignore_missing: true - - rename: - field: json.protoPayload.request.filter - target_field: gcp.audit.request.filter - ignore_missing: true - - rename: - field: json.protoPayload.request.name - target_field: gcp.audit.request.name - ignore_missing: true - - rename: - field: json.protoPayload.request.resourceName - target_field: gcp.audit.request.resource_name - ignore_missing: true + field: gcp.audit.labels + copy_from: json.labels + if: ctx.json?.labels != null +## +# RequestMetadata +# .protoPayload.requestMetadata +# https://cloud.google.com/logging/docs/reference/audit/auditlog/rest/Shared.Types/AuditLog#requestmetadata +## - convert: + if: ctx.json?.protoPayload?.requestMetadata?.callerIp != null && ctx.json?.protoPayload?.requestMetadata?.callerIp != "gce-internal-ip" type: ip field: json.protoPayload.requestMetadata.callerIp + target_field: source.ip ignore_missing: true - on_failure: - - rename: - field: json.protoPayload.requestMetadata.callerIp - target_field: gcp.audit.request_metadata.raw.caller_ip - - rename: - field: json.protoPayload.requestMetadata.callerIp - target_field: gcp.audit.request_metadata.caller_ip - ignore_missing: true - - set: - field: source.ip - value: "{{gcp.audit.request_metadata.caller_ip}}" - if: ctx?.gcp?.audit?.request_metadata?.caller_ip != null + # TODO remove - duplicated in useragent - rename: field: json.protoPayload.requestMetadata.callerSuppliedUserAgent target_field: gcp.audit.request_metadata.caller_supplied_user_agent @@ -117,43 +183,76 @@ processors: field: user_agent.original value: "{{gcp.audit.request_metadata.caller_supplied_user_agent}}" if: ctx?.gcp?.audit?.request_metadata?.caller_supplied_user_agent != null - - rename: - field: "json.protoPayload.response.@type" - target_field: gcp.audit.response.proto_name - ignore_missing: true - - rename: - field: json.protoPayload.response.status - target_field: gcp.audit.response.status - ignore_missing: true - - rename: - field: gcp.audit.response.status - target_field: gcp.audit.response.status.value - if: ctx?.gcp?.audit?.response?.status instanceof String - ignore_missing: true - - rename: - field: json.protoPayload.response.details.group - target_field: gcp.audit.response.details.group - ignore_missing: true - - rename: - field: json.protoPayload.response.details.kind - target_field: gcp.audit.response.details.kind + - user_agent: + field: user_agent.original ignore_missing: true +## +# LogEntryOperation +# .operation +# https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry#logentryoperation +## +# set only if it is not the same as insertId + - set: + field: gcp.audit.logentry_operation.id + copy_from: json.operation.id + if: ctx.json?.operation?.id != null && ctx.event?.id != null && ctx.event?.id != ctx.json?.operation?.id + - script: + lang: painless + description: set event.category and type for long running operation + tag: set-event-type-for-long-operations + if: ctx.json?.operation != null + source: | + def first = (ctx.json.operation.first == null) ? false : ctx.json.operation.first; + def last = (ctx.json.operation.last == null) ? false : ctx.json.operation.last; + if (first && last) { + return; + } + if (ctx.event.category == null) { + ctx.event.category = new ArrayList(); + } + if (ctx.event.type == null) { + ctx.event.type = new ArrayList(); + } + ctx.event.category.add('session'); + if (first == true && last == false) { + ctx.event.type.add('start'); + } + if (first == false && last == true) { + ctx.event.type.add('end'); + } + +# TODO remove duplicate protoPayload.methodName - rename: - field: json.protoPayload.response.details.name - target_field: gcp.audit.response.details.name + field: json.protoPayload.methodName + target_field: gcp.audit.method_name ignore_missing: true - - rename: - field: json.protoPayload.response.details.uid - target_field: gcp.audit.response.details.uid + - set: + field: event.action + value: "{{gcp.audit.method_name}}" + if: ctx?.gcp?.audit?.method_name != null + - convert: + field: json.protoPayload.numResponseItems + target_field: gcp.audit.num_response_items + type: long ignore_missing: true + - set: + field: gcp.audit.request + copy_from: json.protoPayload.request + if: ctx.json?.protoPayload?.request != null + - set: + field: gcp.audit.response + copy_from: json.protoPayload.response + if: ctx.json?.protoPayload?.response != null - rename: field: json.protoPayload.resourceName target_field: gcp.audit.resource_name ignore_missing: true + if: ctx.orchestrator?.type != 'kubernetes' - rename: field: json.protoPayload.resourceLocation.currentLocations target_field: gcp.audit.resource_location.current_locations ignore_missing: true +# TODO remove duplicate json.protoPayload.serviceName - rename: field: json.protoPayload.serviceName target_field: gcp.audit.service_name @@ -162,25 +261,26 @@ processors: field: service.name value: "{{gcp.audit.service_name}}" if: ctx?.gcp?.audit?.service_name != null +## +# .protoPayload.Status +# https://cloud.google.com/logging/docs/reference/audit/auditlog/rest/Shared.Types/AuditLog#Status +# google.rpc.Code referred in Status can have the following values +# https://github.com/googleapis/googleapis/blob/master/google/rpc/code.proto +## - convert: field: json.protoPayload.status.code target_field: gcp.audit.status.code type: long ignore_missing: true - - foreach: - field: gcp.audit.authorization_info + - rename: + field: json.protoPayload.status.message + target_field: gcp.audit.status.message ignore_missing: true - ignore_failure: true - processor: - rename: - field: _ingest._value.resourceAttributes - target_field: _ingest._value.resource_attributes - if: ctx?.gcp?.audit?.authorization_info != null && ctx?.gcp?.audit?.authorization_info instanceof List - - set: + - set: field: event.outcome value: success - if: ctx?.gcp?.audit?.status?.code != null && ctx?.gcp?.audit?.status?.code == 0 - - set: + if: ctx.gcp?.audit?.status?.code != null && ctx.gcp?.audit?.status?.code == 0 + - set: field: event.outcome value: failure if: ctx?.gcp?.audit?.status?.code != null && ctx?.gcp?.audit?.status?.code != 0 @@ -196,46 +296,28 @@ processors: field: event.outcome value: unknown if: ctx?.event?.outcome == null - - rename: - field: json.protoPayload.status.message - target_field: gcp.audit.status.message - ignore_missing: true - - user_agent: - field: user_agent.original - ignore_missing: true - # Orchestrator fields - - set: - field: orchestrator.type - value: kubernetes - if: ctx.json?.resource?.type == 'k8s_cluster' - - set: - field: orchestrator.cluster.name - copy_from: json.resource.labels.cluster_name - ignore_empty_value: true - if: ctx.json?.resource?.type == 'k8s_cluster' - - set: - field: orchestrator.resource.type_temp - copy_from: json.protoPayload.resourceName - ignore_empty_value: true - if: ctx.json?.resource?.type == 'k8s_cluster' - - grok: - field: orchestrator.resource.type_temp - patterns: - - '%{DATA}/%{API_VERSION:orchestrator.api_version}/namespaces/%{DATA:orchestrator.namespace}/%{RESOURCE_TYPE:orchestrator.resource.type}(/%{HOSTNAME:orchestrator.resource.name})?' - - '%{DATA}/%{API_VERSION:orchestrator.api_version}/%{RESOURCE_TYPE:orchestrator.resource.type}' - - 'apis/%{RESOURCE_TYPE:orchestrator.resource.type}/%{API_VERSION:orchestrator.api_version}' - - 'api/%{API_VERSION:orchestrator.api_version}' - - '%{RESOURCE_TYPE:orchestrator.resource.type}' - pattern_definitions: - API_VERSION: (v\d+([a-z]+)?(\d+)?) - RESOURCE_TYPE: ([a-z]+((\.[a-z0-9]+)+)?) - ignore_missing: true - - remove: - field: orchestrator.resource.type_temp - ignore_missing: true - - remove: - field: json - ignore_missing: true + +## +# if gcp.audit.authorization_info.[0].granted is true then +# set event.category [network, configuration] and event.type to [access, allowed]; +# Caveat +# 1. protoPayload.resourceName is a single value while authorization_info[].resource +# is a list. +# 2. as per test data authorization_info may not be as per spec. +## + - append: + field: event.category + value: ['network', 'configuration'] + if: ctx?.gcp?.audit?.authorization_info != null && ctx?.gcp?.audit?.authorization_info instanceof List && ctx?.gcp?.audit?.authorization_info.size() == 1 + - append: + field: event.type + value: ['access', 'allowed'] + if: ctx?.gcp?.audit?.authorization_info != null && ctx?.gcp?.audit?.authorization_info instanceof List && ctx?.gcp?.audit?.authorization_info.size() == 1 && ctx?.gcp?.audit?.authorization_info[0]?.granted != null && ctx?.gcp?.audit?.authorization_info[0]?.granted + - append: + field: event.type + value: ['access', 'denied'] + if: ctx?.gcp?.audit?.authorization_info != null && ctx?.gcp?.audit?.authorization_info instanceof List && ctx?.gcp?.audit?.authorization_info.size() == 1 && ctx?.gcp?.audit?.authorization_info[0]?.granted != null && !ctx?.gcp?.audit?.authorization_info[0]?.granted + # IP Geolocation Lookup - geoip: field: source.ip @@ -258,6 +340,15 @@ processors: field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true + +## +# clean-up +## + - remove: + field: + - _temp + - json + ignore_missing: true - remove: field: event.original if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" diff --git a/packages/gcp/data_stream/audit/fields/ecs.yml b/packages/gcp/data_stream/audit/fields/ecs.yml index c95072b50f5..4456cea03e1 100644 --- a/packages/gcp/data_stream/audit/fields/ecs.yml +++ b/packages/gcp/data_stream/audit/fields/ecs.yml @@ -22,6 +22,8 @@ name: log.file.path - external: ecs name: log.logger +- external: ecs + name: log.level - external: ecs name: message - external: ecs @@ -66,6 +68,8 @@ name: source.geo.region_name - external: ecs name: source.ip +- external: ecs + name: source.address - external: ecs name: tags - external: ecs @@ -90,3 +94,11 @@ name: user_agent.os.version - external: ecs name: user_agent.version +- external: ecs + name: client.user.email +- external: ecs + name: client.user.id +- external: ecs + name: error.code +- external: ecs + name: error.message diff --git a/packages/gcp/data_stream/audit/fields/fields.yml b/packages/gcp/data_stream/audit/fields/fields.yml index 80c217b22ba..12064f765e5 100644 --- a/packages/gcp/data_stream/audit/fields/fields.yml +++ b/packages/gcp/data_stream/audit/fields/fields.yml @@ -14,6 +14,9 @@ - name: authority_selector type: keyword description: "The authority selector specified by the requestor, if any. It is not guaranteed that the principal was allowed to use this authority." + - name: principal_subject + type: keyword + description: "String representation of identity of requesting party. Populated for both first and third party identities. Only present for APIs that support third-party identities." - name: authorization_info type: array description: | @@ -25,6 +28,9 @@ - name: granted type: boolean description: "Whether or not authorization for resource and permission was granted." + - name: resource + type: keyword + description: "The resource being accessed, as a REST-style string." - name: resource_attributes type: group fields: @@ -40,6 +46,24 @@ type: keyword description: | The type of the resource. + - name: labels + type: flattened + description: "A map of key, value pairs that provides additional information about the log entry. The labels can be user-defined or system-defined." + - name: logentry_operation + type: group + fields: + - name: id + type: keyword + description: "Optional. An arbitrary operation identifier. Log entries with the same identifier are assumed to be part of the same operation." + - name: producer + type: keyword + description: "Optional. An arbitrary producer identifier. The combination of id and producer must be globally unique." + - name: first + type: boolean + description: "Optional. Set this to True if this is the first log entry in the operation." + - name: last + type: boolean + description: "Optional. Set this to True if this is the last log entry in the operation." - name: method_name type: keyword description: | @@ -49,22 +73,7 @@ description: | The number of items returned from a List or Query API method, if applicable. - name: request - type: group - fields: - - name: proto_name - type: keyword - description: | - Type property of the request. - - name: filter - type: keyword - description: | - Filter of the request. - - name: name - type: keyword - description: "Name of the request." - - name: resource_name - type: keyword - description: "Name of the request resource." + type: flattened - name: request_metadata type: group fields: @@ -79,37 +88,7 @@ description: | The user agent of the caller. This information is not authenticated and should be treated accordingly. - name: response - type: group - fields: - - name: proto_name - type: keyword - description: | - Type property of the response. - - name: details - type: group - fields: - - name: group - type: keyword - description: | - The name of the group. - - name: kind - type: keyword - description: | - The kind of the response details. - - name: name - type: keyword - description: | - The name of the response details. - - name: uid - type: keyword - description: | - The uid of the response details. - - name: status.allowed - type: boolean - - name: status.reason - type: keyword - - name: status.value - type: keyword + type: flattened - name: resource_name type: keyword description: | @@ -118,7 +97,7 @@ type: group fields: - name: current_locations - type: keyword + type: array description: | Current locations of the resource. - name: service_name diff --git a/packages/gcp/data_stream/audit/sample_event.json b/packages/gcp/data_stream/audit/sample_event.json index 224c98c8d3b..b5881e272fd 100644 --- a/packages/gcp/data_stream/audit/sample_event.json +++ b/packages/gcp/data_stream/audit/sample_event.json @@ -1,16 +1,23 @@ { "@timestamp": "2019-12-19T00:44:25.051Z", "agent": { - "ephemeral_id": "0365945c-c25a-4f02-b62c-a94a0b661f02", - "id": "c53ddea2-61ac-4643-8676-0c70ebf51c91", + "ephemeral_id": "15ffa48e-049a-4ead-9716-cea0236748c4", + "hostname": "docker-fleet-agent", + "id": "df142714-8028-4ef0-a80c-4eb03051c084", "name": "docker-fleet-agent", "type": "filebeat", - "version": "8.0.0-beta1" + "version": "7.17.0" + }, + "client": { + "user": { + "email": "xxx@xxx.xxx" + } }, "cloud": { "project": { "id": "elastic-beats" - } + }, + "provider": "gcp" }, "data_stream": { "dataset": "gcp.audit", @@ -21,19 +28,28 @@ "version": "8.2.0" }, "elastic_agent": { - "id": "c53ddea2-61ac-4643-8676-0c70ebf51c91", + "id": "df142714-8028-4ef0-a80c-4eb03051c084", "snapshot": false, - "version": "8.0.0-beta1" + "version": "7.17.0" }, "event": { "action": "beta.compute.instances.aggregatedList", "agent_id_status": "verified", - "created": "2021-12-31T03:10:44.655Z", + "category": [ + "network", + "configuration" + ], + "created": "2022-05-20T07:25:00.534Z", "dataset": "gcp.audit", "id": "yonau2dg2zi", - "ingested": "2021-12-31T03:10:45Z", + "ingested": "2022-05-20T07:25:01Z", "kind": "event", - "outcome": "success" + "outcome": "success", + "provider": "data_access", + "type": [ + "access", + "allowed" + ] }, "gcp": { "audit": { @@ -54,10 +70,9 @@ "method_name": "beta.compute.instances.aggregatedList", "num_response_items": 61, "request": { - "proto_name": "type.googleapis.com/compute.instances.aggregatedList" + "@type": "type.googleapis.com/compute.instances.aggregatedList" }, "request_metadata": { - "caller_ip": "192.168.1.1", "caller_supplied_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:71.0) Gecko/20100101 Firefox/71.0,gzip(gfe),gzip(gfe)" }, "resource_location": { @@ -67,16 +82,16 @@ }, "resource_name": "projects/elastic-beats/global/instances", "response": { + "@type": "core.k8s.io/v1.Status", + "apiVersion": "v1", "details": { "group": "batch", "kind": "jobs", "name": "gsuite-exporter-1589294700", "uid": "2beff34a-945f-11ea-bacf-42010a80007f" }, - "proto_name": "core.k8s.io/v1.Status", - "status": { - "value": "Success" - } + "kind": "Status", + "status": "Success" }, "service_name": "compute.googleapis.com", "type": "type.googleapis.com/google.cloud.audit.AuditLog" @@ -86,6 +101,7 @@ "type": "gcp-pubsub" }, "log": { + "level": "INFO", "logger": "projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access" }, "service": { @@ -98,9 +114,6 @@ "forwarded", "gcp-audit" ], - "user": { - "email": "xxx@xxx.xxx" - }, "user_agent": { "device": { "name": "Mac" diff --git a/packages/gcp/data_stream/dns/sample_event.json b/packages/gcp/data_stream/dns/sample_event.json index 7bf4d443265..10349b6d73b 100644 --- a/packages/gcp/data_stream/dns/sample_event.json +++ b/packages/gcp/data_stream/dns/sample_event.json @@ -1,5 +1,13 @@ { "@timestamp": "2022-01-23T09:16:05.341Z", + "agent": { + "ephemeral_id": "0d2f83ac-67e6-454f-84eb-859aa503167a", + "hostname": "docker-fleet-agent", + "id": "df142714-8028-4ef0-a80c-4eb03051c084", + "name": "docker-fleet-agent", + "type": "filebeat", + "version": "7.17.0" + }, "cloud": { "availability_zone": "europe-west2-a", "instance": { @@ -9,8 +17,14 @@ "project": { "id": "project" }, + "provider": "gcp", "region": "europe-west2" }, + "data_stream": { + "dataset": "gcp.dns", + "namespace": "ep", + "type": "logs" + }, "dns": { "answers": [ { @@ -35,11 +49,19 @@ "ecs": { "version": "8.2.0" }, + "elastic_agent": { + "id": "df142714-8028-4ef0-a80c-4eb03051c084", + "snapshot": false, + "version": "7.17.0" + }, "event": { + "agent_id_status": "verified", + "created": "2022-05-20T07:25:43.755Z", + "dataset": "gcp.dns", "id": "vwroyze8pg7y", + "ingested": "2022-05-20T07:25:44Z", "kind": "event", - "outcome": "success", - "original": "{\"insertId\":\"vwroyze8pg7y\",\"jsonPayload\":{\"authAnswer\":true,\"protocol\":\"UDP\",\"queryName\":\"elastic.co.\",\"queryType\":\"A\",\"rdata\":\"elastic.co.\\t300\\tIN\\ta\\t127.0.0.1\",\"responseCode\":\"NOERROR\",\"serverLatency\":14,\"sourceIP\":\"10.154.0.3\",\"sourceNetwork\":\"default\",\"vmInstanceId\":8340998530665147,\"vmInstanceIdString\":\"8340998530665147\",\"vmInstanceName\":\"694119234537.instance\",\"vmProjectId\":\"project\",\"vmZoneName\":\"europe-west2-a\"},\"logName\":\"projects/project/logs/dns.googleapis.com%2Fdns_queries\",\"receiveTimestamp\":\"2022-01-23T09:16:05.502805637Z\",\"resource\":{\"labels\":{\"location\":\"europe-west2\",\"project_id\":\"project\",\"source_type\":\"gce-vm\",\"target_name\":\"\",\"target_type\":\"external\"},\"type\":\"dns_query\"},\"severity\":\"INFO\",\"timestamp\":\"2022-01-23T09:16:05.341873447Z\"}" + "outcome": "success" }, "gcp": { "dns": { @@ -58,6 +80,9 @@ "vm_zone_name": "europe-west2-a" } }, + "input": { + "type": "gcp-pubsub" + }, "log": { "logger": "projects/project/logs/dns.googleapis.com%2Fdns_queries" }, @@ -69,6 +94,7 @@ "ip": "10.154.0.3" }, "tags": [ - "preserve_original_event" + "forwarded", + "gcp-dns" ] } \ No newline at end of file diff --git a/packages/gcp/data_stream/firewall/sample_event.json b/packages/gcp/data_stream/firewall/sample_event.json index 13f6dbd6fb7..feeb2644cfb 100644 --- a/packages/gcp/data_stream/firewall/sample_event.json +++ b/packages/gcp/data_stream/firewall/sample_event.json @@ -1,17 +1,19 @@ { "@timestamp": "2019-10-30T13:52:42.191Z", "agent": { - "ephemeral_id": "4fed48b9-0848-4ceb-88b1-30fb7da99604", - "id": "c53ddea2-61ac-4643-8676-0c70ebf51c91", + "ephemeral_id": "1f7633a7-3410-4684-bb55-14b0bd0e2bd4", + "hostname": "docker-fleet-agent", + "id": "df142714-8028-4ef0-a80c-4eb03051c084", "name": "docker-fleet-agent", "type": "filebeat", - "version": "8.0.0-beta1" + "version": "7.17.0" }, "cloud": { "availability_zone": "us-east1-b", "project": { "id": "test-beats" }, + "provider": "gcp", "region": "us-east1" }, "data_stream": { @@ -29,18 +31,18 @@ "version": "8.2.0" }, "elastic_agent": { - "id": "c53ddea2-61ac-4643-8676-0c70ebf51c91", + "id": "df142714-8028-4ef0-a80c-4eb03051c084", "snapshot": false, - "version": "8.0.0-beta1" + "version": "7.17.0" }, "event": { "action": "firewall-rule", "agent_id_status": "verified", "category": "network", - "created": "2021-12-31T03:11:30.136Z", + "created": "2022-05-20T07:26:27.445Z", "dataset": "gcp.firewall", "id": "1f21ciqfpfssuo", - "ingested": "2021-12-31T03:11:31Z", + "ingested": "2022-05-20T07:26:28Z", "kind": "event", "type": "connection" }, diff --git a/packages/gcp/data_stream/vpcflow/sample_event.json b/packages/gcp/data_stream/vpcflow/sample_event.json index 98ae3ebcede..3d743d26e53 100644 --- a/packages/gcp/data_stream/vpcflow/sample_event.json +++ b/packages/gcp/data_stream/vpcflow/sample_event.json @@ -1,11 +1,20 @@ { "@timestamp": "2019-06-14T03:50:10.845Z", "agent": { - "ephemeral_id": "e58d02a0-e7a0-45c0-aba6-a8c983782744", - "id": "c53ddea2-61ac-4643-8676-0c70ebf51c91", + "ephemeral_id": "10bb82a5-c0e6-4aed-8589-003f734a7183", + "hostname": "docker-fleet-agent", + "id": "df142714-8028-4ef0-a80c-4eb03051c084", "name": "docker-fleet-agent", "type": "filebeat", - "version": "8.0.0-beta1" + "version": "7.17.0" + }, + "cloud": { + "availability_zone": "us-east1-b", + "project": { + "id": "my-sample-project" + }, + "provider": "gcp", + "region": "us-east1" }, "data_stream": { "dataset": "gcp.vpcflow", @@ -13,10 +22,11 @@ "type": "logs" }, "destination": { - "address": "67.43.156.13", + "address": "67.43.156.14", "as": { "number": 35908 }, + "domain": "elasticsearch", "geo": { "continent_name": "Asia", "country_iso_code": "BT", @@ -26,30 +36,42 @@ "lon": 90.5 } }, - "ip": "67.43.156.13", - "port": 33478 + "ip": "67.43.156.14", + "port": 9200 }, "ecs": { "version": "8.2.0" }, "elastic_agent": { - "id": "c53ddea2-61ac-4643-8676-0c70ebf51c91", + "id": "df142714-8028-4ef0-a80c-4eb03051c084", "snapshot": false, - "version": "8.0.0-beta1" + "version": "7.17.0" }, "event": { "agent_id_status": "verified", "category": "network", - "created": "2021-12-31T03:12:25.823Z", + "created": "2022-05-20T07:27:09.739Z", "dataset": "gcp.vpcflow", - "end": "2019-06-14T03:45:37.301953198Z", - "id": "ut8lbrffooxyw", - "ingested": "2021-12-31T03:12:26Z", + "end": "2019-06-14T03:49:51.821308944Z", + "id": "ut8lbrffooxyp", + "ingested": "2022-05-20T07:27:10Z", "kind": "event", - "start": "2019-06-14T03:45:37.186193305Z", + "start": "2019-06-14T03:40:08.469099728Z", "type": "connection" }, "gcp": { + "destination": { + "instance": { + "project_id": "my-sample-project", + "region": "us-east1", + "zone": "us-east1-b" + }, + "vpc": { + "project_id": "my-sample-project", + "subnetwork_name": "default", + "vpc_name": "default" + } + }, "source": { "instance": { "project_id": "my-sample-project", @@ -65,7 +87,7 @@ "vpcflow": { "reporter": "SRC", "rtt": { - "ms": 36 + "ms": 3 } } }, @@ -76,27 +98,28 @@ "logger": "projects/my-sample-project/logs/compute.googleapis.com%2Fvpc_flows" }, "network": { - "bytes": 1776, - "community_id": "1:Wa+aonxAQZ59AWtNdQD0CH6FnsM=", - "direction": "outbound", + "bytes": 15169, + "community_id": "1:NAY9D1IuyJAG+Hm34t3LIlP6/4c=", + "direction": "internal", "iana_number": "6", - "packets": 7, + "name": "default", + "packets": 92, "transport": "tcp", "type": "ipv4" }, "related": { "ip": [ "10.87.40.76", - "67.43.156.13" + "67.43.156.14" ] }, "source": { "address": "10.87.40.76", - "bytes": 1776, + "bytes": 15169, "domain": "kibana", "ip": "10.87.40.76", - "packets": 7, - "port": 5601 + "packets": 92, + "port": 33880 }, "tags": [ "forwarded", diff --git a/packages/gcp/docs/README.md b/packages/gcp/docs/README.md index c070e2f3f6b..9e6a1643d4d 100644 --- a/packages/gcp/docs/README.md +++ b/packages/gcp/docs/README.md @@ -187,6 +187,8 @@ The `audit` dataset collects audit logs of administrative activities and accesse | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | +| client.user.email | User email address. | keyword | +| client.user.id | Unique identifier of the user. | keyword | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | @@ -205,6 +207,8 @@ The `audit` dataset collects audit logs of administrative activities and accesse | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| error.code | Error code describing the error. | keyword | +| error.message | Error message. | match_only_text | | event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | | event.created | event.created contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from @timestamp in that @timestamp typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, @timestamp should be used. | date | | event.dataset | Event dataset | constant_keyword | @@ -216,30 +220,27 @@ The `audit` dataset collects audit logs of administrative activities and accesse | event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | | gcp.audit.authentication_info.authority_selector | The authority selector specified by the requestor, if any. It is not guaranteed that the principal was allowed to use this authority. | keyword | | gcp.audit.authentication_info.principal_email | The email address of the authenticated user making the request. | keyword | +| gcp.audit.authentication_info.principal_subject | String representation of identity of requesting party. Populated for both first and third party identities. Only present for APIs that support third-party identities. | keyword | | gcp.audit.authorization_info.granted | Whether or not authorization for resource and permission was granted. | boolean | | gcp.audit.authorization_info.permission | The required IAM permission. | keyword | +| gcp.audit.authorization_info.resource | The resource being accessed, as a REST-style string. | keyword | | gcp.audit.authorization_info.resource_attributes.name | The name of the resource. | keyword | | gcp.audit.authorization_info.resource_attributes.service | The name of the service. | keyword | | gcp.audit.authorization_info.resource_attributes.type | The type of the resource. | keyword | +| gcp.audit.labels | A map of key, value pairs that provides additional information about the log entry. The labels can be user-defined or system-defined. | flattened | +| gcp.audit.logentry_operation.first | Optional. Set this to True if this is the first log entry in the operation. | boolean | +| gcp.audit.logentry_operation.id | Optional. An arbitrary operation identifier. Log entries with the same identifier are assumed to be part of the same operation. | keyword | +| gcp.audit.logentry_operation.last | Optional. Set this to True if this is the last log entry in the operation. | boolean | +| gcp.audit.logentry_operation.producer | Optional. An arbitrary producer identifier. The combination of id and producer must be globally unique. | keyword | | gcp.audit.method_name | The name of the service method or operation. For API calls, this should be the name of the API method. For example, 'google.datastore.v1.Datastore.RunQuery'. | keyword | | gcp.audit.num_response_items | The number of items returned from a List or Query API method, if applicable. | long | -| gcp.audit.request.filter | Filter of the request. | keyword | -| gcp.audit.request.name | Name of the request. | keyword | -| gcp.audit.request.proto_name | Type property of the request. | keyword | -| gcp.audit.request.resource_name | Name of the request resource. | keyword | +| gcp.audit.request | | flattened | | gcp.audit.request_metadata.caller_ip | The IP address of the caller. | ip | | gcp.audit.request_metadata.caller_supplied_user_agent | The user agent of the caller. This information is not authenticated and should be treated accordingly. | keyword | | gcp.audit.request_metadata.raw.caller_ip | The raw IP address of the caller. | keyword | -| gcp.audit.resource_location.current_locations | Current locations of the resource. | keyword | +| gcp.audit.resource_location.current_locations | Current locations of the resource. | array | | gcp.audit.resource_name | The resource or collection that is the target of the operation. The name is a scheme-less URI, not including the API service name. For example, 'shelves/SHELF_ID/books'. | keyword | -| gcp.audit.response.details.group | The name of the group. | keyword | -| gcp.audit.response.details.kind | The kind of the response details. | keyword | -| gcp.audit.response.details.name | The name of the response details. | keyword | -| gcp.audit.response.details.uid | The uid of the response details. | keyword | -| gcp.audit.response.proto_name | Type property of the response. | keyword | -| gcp.audit.response.status.allowed | | boolean | -| gcp.audit.response.status.reason | | keyword | -| gcp.audit.response.status.value | | keyword | +| gcp.audit.response | | flattened | | gcp.audit.service_name | The name of the API service performing the operation. For example, datastore.googleapis.com. | keyword | | gcp.audit.status.code | The status code, which should be an enum value of google.rpc.Code. | integer | | gcp.audit.status.message | A developer-facing error message, which should be in English. Any user-facing error message should be localized and sent in the google.rpc.Status.details field, or localized by the client. | keyword | @@ -275,6 +276,7 @@ The `audit` dataset collects audit logs of administrative activities and accesse | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | +| log.level | Original log level of the log event. If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). Some examples are `warn`, `err`, `i`, `informational`. | keyword | | log.logger | The name of the logger inside an application. This is usually the name of the class which initialized the logger, or can be a custom name. | keyword | | log.offset | Log offset | long | | message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | @@ -288,6 +290,7 @@ The `audit` dataset collects audit logs of administrative activities and accesse | orchestrator.resource.type | Type of resource being acted upon. | keyword | | orchestrator.type | Orchestrator cluster type (e.g. kubernetes, nomad or cloudfoundry). | keyword | | service.name | Name of the service data is collected from. The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. | keyword | +| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | | source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | | source.as.organization.name | Organization name. | keyword | | source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | @@ -322,16 +325,23 @@ An example event for `audit` looks as following: { "@timestamp": "2019-12-19T00:44:25.051Z", "agent": { - "ephemeral_id": "0365945c-c25a-4f02-b62c-a94a0b661f02", - "id": "c53ddea2-61ac-4643-8676-0c70ebf51c91", + "ephemeral_id": "15ffa48e-049a-4ead-9716-cea0236748c4", + "hostname": "docker-fleet-agent", + "id": "df142714-8028-4ef0-a80c-4eb03051c084", "name": "docker-fleet-agent", "type": "filebeat", - "version": "8.0.0-beta1" + "version": "7.17.0" + }, + "client": { + "user": { + "email": "xxx@xxx.xxx" + } }, "cloud": { "project": { "id": "elastic-beats" - } + }, + "provider": "gcp" }, "data_stream": { "dataset": "gcp.audit", @@ -342,19 +352,28 @@ An example event for `audit` looks as following: "version": "8.2.0" }, "elastic_agent": { - "id": "c53ddea2-61ac-4643-8676-0c70ebf51c91", + "id": "df142714-8028-4ef0-a80c-4eb03051c084", "snapshot": false, - "version": "8.0.0-beta1" + "version": "7.17.0" }, "event": { "action": "beta.compute.instances.aggregatedList", "agent_id_status": "verified", - "created": "2021-12-31T03:10:44.655Z", + "category": [ + "network", + "configuration" + ], + "created": "2022-05-20T07:25:00.534Z", "dataset": "gcp.audit", "id": "yonau2dg2zi", - "ingested": "2021-12-31T03:10:45Z", + "ingested": "2022-05-20T07:25:01Z", "kind": "event", - "outcome": "success" + "outcome": "success", + "provider": "data_access", + "type": [ + "access", + "allowed" + ] }, "gcp": { "audit": { @@ -375,10 +394,9 @@ An example event for `audit` looks as following: "method_name": "beta.compute.instances.aggregatedList", "num_response_items": 61, "request": { - "proto_name": "type.googleapis.com/compute.instances.aggregatedList" + "@type": "type.googleapis.com/compute.instances.aggregatedList" }, "request_metadata": { - "caller_ip": "192.168.1.1", "caller_supplied_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:71.0) Gecko/20100101 Firefox/71.0,gzip(gfe),gzip(gfe)" }, "resource_location": { @@ -388,16 +406,16 @@ An example event for `audit` looks as following: }, "resource_name": "projects/elastic-beats/global/instances", "response": { + "@type": "core.k8s.io/v1.Status", + "apiVersion": "v1", "details": { "group": "batch", "kind": "jobs", "name": "gsuite-exporter-1589294700", "uid": "2beff34a-945f-11ea-bacf-42010a80007f" }, - "proto_name": "core.k8s.io/v1.Status", - "status": { - "value": "Success" - } + "kind": "Status", + "status": "Success" }, "service_name": "compute.googleapis.com", "type": "type.googleapis.com/google.cloud.audit.AuditLog" @@ -407,6 +425,7 @@ An example event for `audit` looks as following: "type": "gcp-pubsub" }, "log": { + "level": "INFO", "logger": "projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access" }, "service": { @@ -419,9 +438,6 @@ An example event for `audit` looks as following: "forwarded", "gcp-audit" ], - "user": { - "email": "xxx@xxx.xxx" - }, "user_agent": { "device": { "name": "Mac" @@ -571,17 +587,19 @@ An example event for `firewall` looks as following: { "@timestamp": "2019-10-30T13:52:42.191Z", "agent": { - "ephemeral_id": "4fed48b9-0848-4ceb-88b1-30fb7da99604", - "id": "c53ddea2-61ac-4643-8676-0c70ebf51c91", + "ephemeral_id": "1f7633a7-3410-4684-bb55-14b0bd0e2bd4", + "hostname": "docker-fleet-agent", + "id": "df142714-8028-4ef0-a80c-4eb03051c084", "name": "docker-fleet-agent", "type": "filebeat", - "version": "8.0.0-beta1" + "version": "7.17.0" }, "cloud": { "availability_zone": "us-east1-b", "project": { "id": "test-beats" }, + "provider": "gcp", "region": "us-east1" }, "data_stream": { @@ -599,18 +617,18 @@ An example event for `firewall` looks as following: "version": "8.2.0" }, "elastic_agent": { - "id": "c53ddea2-61ac-4643-8676-0c70ebf51c91", + "id": "df142714-8028-4ef0-a80c-4eb03051c084", "snapshot": false, - "version": "8.0.0-beta1" + "version": "7.17.0" }, "event": { "action": "firewall-rule", "agent_id_status": "verified", "category": "network", - "created": "2021-12-31T03:11:30.136Z", + "created": "2022-05-20T07:26:27.445Z", "dataset": "gcp.firewall", "id": "1f21ciqfpfssuo", - "ingested": "2021-12-31T03:11:31Z", + "ingested": "2022-05-20T07:26:28Z", "kind": "event", "type": "connection" }, @@ -818,11 +836,20 @@ An example event for `vpcflow` looks as following: { "@timestamp": "2019-06-14T03:50:10.845Z", "agent": { - "ephemeral_id": "e58d02a0-e7a0-45c0-aba6-a8c983782744", - "id": "c53ddea2-61ac-4643-8676-0c70ebf51c91", + "ephemeral_id": "10bb82a5-c0e6-4aed-8589-003f734a7183", + "hostname": "docker-fleet-agent", + "id": "df142714-8028-4ef0-a80c-4eb03051c084", "name": "docker-fleet-agent", "type": "filebeat", - "version": "8.0.0-beta1" + "version": "7.17.0" + }, + "cloud": { + "availability_zone": "us-east1-b", + "project": { + "id": "my-sample-project" + }, + "provider": "gcp", + "region": "us-east1" }, "data_stream": { "dataset": "gcp.vpcflow", @@ -830,10 +857,11 @@ An example event for `vpcflow` looks as following: "type": "logs" }, "destination": { - "address": "67.43.156.13", + "address": "67.43.156.14", "as": { "number": 35908 }, + "domain": "elasticsearch", "geo": { "continent_name": "Asia", "country_iso_code": "BT", @@ -843,30 +871,42 @@ An example event for `vpcflow` looks as following: "lon": 90.5 } }, - "ip": "67.43.156.13", - "port": 33478 + "ip": "67.43.156.14", + "port": 9200 }, "ecs": { "version": "8.2.0" }, "elastic_agent": { - "id": "c53ddea2-61ac-4643-8676-0c70ebf51c91", + "id": "df142714-8028-4ef0-a80c-4eb03051c084", "snapshot": false, - "version": "8.0.0-beta1" + "version": "7.17.0" }, "event": { "agent_id_status": "verified", "category": "network", - "created": "2021-12-31T03:12:25.823Z", + "created": "2022-05-20T07:27:09.739Z", "dataset": "gcp.vpcflow", - "end": "2019-06-14T03:45:37.301953198Z", - "id": "ut8lbrffooxyw", - "ingested": "2021-12-31T03:12:26Z", + "end": "2019-06-14T03:49:51.821308944Z", + "id": "ut8lbrffooxyp", + "ingested": "2022-05-20T07:27:10Z", "kind": "event", - "start": "2019-06-14T03:45:37.186193305Z", + "start": "2019-06-14T03:40:08.469099728Z", "type": "connection" }, "gcp": { + "destination": { + "instance": { + "project_id": "my-sample-project", + "region": "us-east1", + "zone": "us-east1-b" + }, + "vpc": { + "project_id": "my-sample-project", + "subnetwork_name": "default", + "vpc_name": "default" + } + }, "source": { "instance": { "project_id": "my-sample-project", @@ -882,7 +922,7 @@ An example event for `vpcflow` looks as following: "vpcflow": { "reporter": "SRC", "rtt": { - "ms": 36 + "ms": 3 } } }, @@ -893,27 +933,28 @@ An example event for `vpcflow` looks as following: "logger": "projects/my-sample-project/logs/compute.googleapis.com%2Fvpc_flows" }, "network": { - "bytes": 1776, - "community_id": "1:Wa+aonxAQZ59AWtNdQD0CH6FnsM=", - "direction": "outbound", + "bytes": 15169, + "community_id": "1:NAY9D1IuyJAG+Hm34t3LIlP6/4c=", + "direction": "internal", "iana_number": "6", - "packets": 7, + "name": "default", + "packets": 92, "transport": "tcp", "type": "ipv4" }, "related": { "ip": [ "10.87.40.76", - "67.43.156.13" + "67.43.156.14" ] }, "source": { "address": "10.87.40.76", - "bytes": 1776, + "bytes": 15169, "domain": "kibana", "ip": "10.87.40.76", - "packets": 7, - "port": 5601 + "packets": 92, + "port": 33880 }, "tags": [ "forwarded", @@ -1017,6 +1058,14 @@ An example event for `dns` looks as following: ```json { "@timestamp": "2022-01-23T09:16:05.341Z", + "agent": { + "ephemeral_id": "0d2f83ac-67e6-454f-84eb-859aa503167a", + "hostname": "docker-fleet-agent", + "id": "df142714-8028-4ef0-a80c-4eb03051c084", + "name": "docker-fleet-agent", + "type": "filebeat", + "version": "7.17.0" + }, "cloud": { "availability_zone": "europe-west2-a", "instance": { @@ -1026,8 +1075,14 @@ An example event for `dns` looks as following: "project": { "id": "project" }, + "provider": "gcp", "region": "europe-west2" }, + "data_stream": { + "dataset": "gcp.dns", + "namespace": "ep", + "type": "logs" + }, "dns": { "answers": [ { @@ -1052,11 +1107,19 @@ An example event for `dns` looks as following: "ecs": { "version": "8.2.0" }, + "elastic_agent": { + "id": "df142714-8028-4ef0-a80c-4eb03051c084", + "snapshot": false, + "version": "7.17.0" + }, "event": { + "agent_id_status": "verified", + "created": "2022-05-20T07:25:43.755Z", + "dataset": "gcp.dns", "id": "vwroyze8pg7y", + "ingested": "2022-05-20T07:25:44Z", "kind": "event", - "outcome": "success", - "original": "{\"insertId\":\"vwroyze8pg7y\",\"jsonPayload\":{\"authAnswer\":true,\"protocol\":\"UDP\",\"queryName\":\"elastic.co.\",\"queryType\":\"A\",\"rdata\":\"elastic.co.\\t300\\tIN\\ta\\t127.0.0.1\",\"responseCode\":\"NOERROR\",\"serverLatency\":14,\"sourceIP\":\"10.154.0.3\",\"sourceNetwork\":\"default\",\"vmInstanceId\":8340998530665147,\"vmInstanceIdString\":\"8340998530665147\",\"vmInstanceName\":\"694119234537.instance\",\"vmProjectId\":\"project\",\"vmZoneName\":\"europe-west2-a\"},\"logName\":\"projects/project/logs/dns.googleapis.com%2Fdns_queries\",\"receiveTimestamp\":\"2022-01-23T09:16:05.502805637Z\",\"resource\":{\"labels\":{\"location\":\"europe-west2\",\"project_id\":\"project\",\"source_type\":\"gce-vm\",\"target_name\":\"\",\"target_type\":\"external\"},\"type\":\"dns_query\"},\"severity\":\"INFO\",\"timestamp\":\"2022-01-23T09:16:05.341873447Z\"}" + "outcome": "success" }, "gcp": { "dns": { @@ -1075,6 +1138,9 @@ An example event for `dns` looks as following: "vm_zone_name": "europe-west2-a" } }, + "input": { + "type": "gcp-pubsub" + }, "log": { "logger": "projects/project/logs/dns.googleapis.com%2Fdns_queries" }, @@ -1086,7 +1152,8 @@ An example event for `dns` looks as following: "ip": "10.154.0.3" }, "tags": [ - "preserve_original_event" + "forwarded", + "gcp-dns" ] } ``` diff --git a/packages/gcp/docs/audit.md b/packages/gcp/docs/audit.md index 6b844dec9f0..2e8c6995f7b 100644 --- a/packages/gcp/docs/audit.md +++ b/packages/gcp/docs/audit.md @@ -9,6 +9,8 @@ The `audit` dataset collects audit logs of administrative activities and accesse | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | +| client.user.email | User email address. | keyword | +| client.user.id | Unique identifier of the user. | keyword | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | @@ -27,6 +29,8 @@ The `audit` dataset collects audit logs of administrative activities and accesse | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| error.code | Error code describing the error. | keyword | +| error.message | Error message. | match_only_text | | event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | | event.created | event.created contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from @timestamp in that @timestamp typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, @timestamp should be used. | date | | event.dataset | Event dataset | constant_keyword | @@ -38,30 +42,27 @@ The `audit` dataset collects audit logs of administrative activities and accesse | event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | | gcp.audit.authentication_info.authority_selector | The authority selector specified by the requestor, if any. It is not guaranteed that the principal was allowed to use this authority. | keyword | | gcp.audit.authentication_info.principal_email | The email address of the authenticated user making the request. | keyword | +| gcp.audit.authentication_info.principal_subject | String representation of identity of requesting party. Populated for both first and third party identities. Only present for APIs that support third-party identities. | keyword | | gcp.audit.authorization_info.granted | Whether or not authorization for resource and permission was granted. | boolean | | gcp.audit.authorization_info.permission | The required IAM permission. | keyword | +| gcp.audit.authorization_info.resource | The resource being accessed, as a REST-style string. | keyword | | gcp.audit.authorization_info.resource_attributes.name | The name of the resource. | keyword | | gcp.audit.authorization_info.resource_attributes.service | The name of the service. | keyword | | gcp.audit.authorization_info.resource_attributes.type | The type of the resource. | keyword | +| gcp.audit.labels | A map of key, value pairs that provides additional information about the log entry. The labels can be user-defined or system-defined. | flattened | +| gcp.audit.logentry_operation.first | Optional. Set this to True if this is the first log entry in the operation. | boolean | +| gcp.audit.logentry_operation.id | Optional. An arbitrary operation identifier. Log entries with the same identifier are assumed to be part of the same operation. | keyword | +| gcp.audit.logentry_operation.last | Optional. Set this to True if this is the last log entry in the operation. | boolean | +| gcp.audit.logentry_operation.producer | Optional. An arbitrary producer identifier. The combination of id and producer must be globally unique. | keyword | | gcp.audit.method_name | The name of the service method or operation. For API calls, this should be the name of the API method. For example, 'google.datastore.v1.Datastore.RunQuery'. | keyword | | gcp.audit.num_response_items | The number of items returned from a List or Query API method, if applicable. | long | -| gcp.audit.request.filter | Filter of the request. | keyword | -| gcp.audit.request.name | Name of the request. | keyword | -| gcp.audit.request.proto_name | Type property of the request. | keyword | -| gcp.audit.request.resource_name | Name of the request resource. | keyword | +| gcp.audit.request | | flattened | | gcp.audit.request_metadata.caller_ip | The IP address of the caller. | ip | | gcp.audit.request_metadata.caller_supplied_user_agent | The user agent of the caller. This information is not authenticated and should be treated accordingly. | keyword | | gcp.audit.request_metadata.raw.caller_ip | The raw IP address of the caller. | keyword | -| gcp.audit.resource_location.current_locations | Current locations of the resource. | keyword | +| gcp.audit.resource_location.current_locations | Current locations of the resource. | array | | gcp.audit.resource_name | The resource or collection that is the target of the operation. The name is a scheme-less URI, not including the API service name. For example, 'shelves/SHELF_ID/books'. | keyword | -| gcp.audit.response.details.group | The name of the group. | keyword | -| gcp.audit.response.details.kind | The kind of the response details. | keyword | -| gcp.audit.response.details.name | The name of the response details. | keyword | -| gcp.audit.response.details.uid | The uid of the response details. | keyword | -| gcp.audit.response.proto_name | Type property of the response. | keyword | -| gcp.audit.response.status.allowed | | boolean | -| gcp.audit.response.status.reason | | keyword | -| gcp.audit.response.status.value | | keyword | +| gcp.audit.response | | flattened | | gcp.audit.service_name | The name of the API service performing the operation. For example, datastore.googleapis.com. | keyword | | gcp.audit.status.code | The status code, which should be an enum value of google.rpc.Code. | integer | | gcp.audit.status.message | A developer-facing error message, which should be in English. Any user-facing error message should be localized and sent in the google.rpc.Status.details field, or localized by the client. | keyword | @@ -97,6 +98,7 @@ The `audit` dataset collects audit logs of administrative activities and accesse | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | +| log.level | Original log level of the log event. If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). Some examples are `warn`, `err`, `i`, `informational`. | keyword | | log.logger | The name of the logger inside an application. This is usually the name of the class which initialized the logger, or can be a custom name. | keyword | | log.offset | Log offset | long | | message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | @@ -110,6 +112,7 @@ The `audit` dataset collects audit logs of administrative activities and accesse | orchestrator.resource.type | Type of resource being acted upon. | keyword | | orchestrator.type | Orchestrator cluster type (e.g. kubernetes, nomad or cloudfoundry). | keyword | | service.name | Name of the service data is collected from. The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. | keyword | +| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | | source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | | source.as.organization.name | Organization name. | keyword | | source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | @@ -144,16 +147,23 @@ An example event for `audit` looks as following: { "@timestamp": "2019-12-19T00:44:25.051Z", "agent": { - "ephemeral_id": "0365945c-c25a-4f02-b62c-a94a0b661f02", - "id": "c53ddea2-61ac-4643-8676-0c70ebf51c91", + "ephemeral_id": "15ffa48e-049a-4ead-9716-cea0236748c4", + "hostname": "docker-fleet-agent", + "id": "df142714-8028-4ef0-a80c-4eb03051c084", "name": "docker-fleet-agent", "type": "filebeat", - "version": "8.0.0-beta1" + "version": "7.17.0" + }, + "client": { + "user": { + "email": "xxx@xxx.xxx" + } }, "cloud": { "project": { "id": "elastic-beats" - } + }, + "provider": "gcp" }, "data_stream": { "dataset": "gcp.audit", @@ -164,19 +174,28 @@ An example event for `audit` looks as following: "version": "8.2.0" }, "elastic_agent": { - "id": "c53ddea2-61ac-4643-8676-0c70ebf51c91", + "id": "df142714-8028-4ef0-a80c-4eb03051c084", "snapshot": false, - "version": "8.0.0-beta1" + "version": "7.17.0" }, "event": { "action": "beta.compute.instances.aggregatedList", "agent_id_status": "verified", - "created": "2021-12-31T03:10:44.655Z", + "category": [ + "network", + "configuration" + ], + "created": "2022-05-20T07:25:00.534Z", "dataset": "gcp.audit", "id": "yonau2dg2zi", - "ingested": "2021-12-31T03:10:45Z", + "ingested": "2022-05-20T07:25:01Z", "kind": "event", - "outcome": "success" + "outcome": "success", + "provider": "data_access", + "type": [ + "access", + "allowed" + ] }, "gcp": { "audit": { @@ -197,10 +216,9 @@ An example event for `audit` looks as following: "method_name": "beta.compute.instances.aggregatedList", "num_response_items": 61, "request": { - "proto_name": "type.googleapis.com/compute.instances.aggregatedList" + "@type": "type.googleapis.com/compute.instances.aggregatedList" }, "request_metadata": { - "caller_ip": "192.168.1.1", "caller_supplied_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:71.0) Gecko/20100101 Firefox/71.0,gzip(gfe),gzip(gfe)" }, "resource_location": { @@ -210,16 +228,16 @@ An example event for `audit` looks as following: }, "resource_name": "projects/elastic-beats/global/instances", "response": { + "@type": "core.k8s.io/v1.Status", + "apiVersion": "v1", "details": { "group": "batch", "kind": "jobs", "name": "gsuite-exporter-1589294700", "uid": "2beff34a-945f-11ea-bacf-42010a80007f" }, - "proto_name": "core.k8s.io/v1.Status", - "status": { - "value": "Success" - } + "kind": "Status", + "status": "Success" }, "service_name": "compute.googleapis.com", "type": "type.googleapis.com/google.cloud.audit.AuditLog" @@ -229,6 +247,7 @@ An example event for `audit` looks as following: "type": "gcp-pubsub" }, "log": { + "level": "INFO", "logger": "projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access" }, "service": { @@ -241,9 +260,6 @@ An example event for `audit` looks as following: "forwarded", "gcp-audit" ], - "user": { - "email": "xxx@xxx.xxx" - }, "user_agent": { "device": { "name": "Mac" diff --git a/packages/gcp/docs/dns.md b/packages/gcp/docs/dns.md index 18f0001dae4..1d7e31a9003 100644 --- a/packages/gcp/docs/dns.md +++ b/packages/gcp/docs/dns.md @@ -95,6 +95,14 @@ An example event for `dns` looks as following: ```json { "@timestamp": "2022-01-23T09:16:05.341Z", + "agent": { + "ephemeral_id": "0d2f83ac-67e6-454f-84eb-859aa503167a", + "hostname": "docker-fleet-agent", + "id": "df142714-8028-4ef0-a80c-4eb03051c084", + "name": "docker-fleet-agent", + "type": "filebeat", + "version": "7.17.0" + }, "cloud": { "availability_zone": "europe-west2-a", "instance": { @@ -104,8 +112,14 @@ An example event for `dns` looks as following: "project": { "id": "project" }, + "provider": "gcp", "region": "europe-west2" }, + "data_stream": { + "dataset": "gcp.dns", + "namespace": "ep", + "type": "logs" + }, "dns": { "answers": [ { @@ -130,11 +144,19 @@ An example event for `dns` looks as following: "ecs": { "version": "8.2.0" }, + "elastic_agent": { + "id": "df142714-8028-4ef0-a80c-4eb03051c084", + "snapshot": false, + "version": "7.17.0" + }, "event": { + "agent_id_status": "verified", + "created": "2022-05-20T07:25:43.755Z", + "dataset": "gcp.dns", "id": "vwroyze8pg7y", + "ingested": "2022-05-20T07:25:44Z", "kind": "event", - "outcome": "success", - "original": "{\"insertId\":\"vwroyze8pg7y\",\"jsonPayload\":{\"authAnswer\":true,\"protocol\":\"UDP\",\"queryName\":\"elastic.co.\",\"queryType\":\"A\",\"rdata\":\"elastic.co.\\t300\\tIN\\ta\\t127.0.0.1\",\"responseCode\":\"NOERROR\",\"serverLatency\":14,\"sourceIP\":\"10.154.0.3\",\"sourceNetwork\":\"default\",\"vmInstanceId\":8340998530665147,\"vmInstanceIdString\":\"8340998530665147\",\"vmInstanceName\":\"694119234537.instance\",\"vmProjectId\":\"project\",\"vmZoneName\":\"europe-west2-a\"},\"logName\":\"projects/project/logs/dns.googleapis.com%2Fdns_queries\",\"receiveTimestamp\":\"2022-01-23T09:16:05.502805637Z\",\"resource\":{\"labels\":{\"location\":\"europe-west2\",\"project_id\":\"project\",\"source_type\":\"gce-vm\",\"target_name\":\"\",\"target_type\":\"external\"},\"type\":\"dns_query\"},\"severity\":\"INFO\",\"timestamp\":\"2022-01-23T09:16:05.341873447Z\"}" + "outcome": "success" }, "gcp": { "dns": { @@ -153,6 +175,9 @@ An example event for `dns` looks as following: "vm_zone_name": "europe-west2-a" } }, + "input": { + "type": "gcp-pubsub" + }, "log": { "logger": "projects/project/logs/dns.googleapis.com%2Fdns_queries" }, @@ -164,7 +189,8 @@ An example event for `dns` looks as following: "ip": "10.154.0.3" }, "tags": [ - "preserve_original_event" + "forwarded", + "gcp-dns" ] } ``` diff --git a/packages/gcp/docs/firewall.md b/packages/gcp/docs/firewall.md index 2e200e7b879..bfdbb35812c 100644 --- a/packages/gcp/docs/firewall.md +++ b/packages/gcp/docs/firewall.md @@ -133,17 +133,19 @@ An example event for `firewall` looks as following: { "@timestamp": "2019-10-30T13:52:42.191Z", "agent": { - "ephemeral_id": "4fed48b9-0848-4ceb-88b1-30fb7da99604", - "id": "c53ddea2-61ac-4643-8676-0c70ebf51c91", + "ephemeral_id": "1f7633a7-3410-4684-bb55-14b0bd0e2bd4", + "hostname": "docker-fleet-agent", + "id": "df142714-8028-4ef0-a80c-4eb03051c084", "name": "docker-fleet-agent", "type": "filebeat", - "version": "8.0.0-beta1" + "version": "7.17.0" }, "cloud": { "availability_zone": "us-east1-b", "project": { "id": "test-beats" }, + "provider": "gcp", "region": "us-east1" }, "data_stream": { @@ -161,18 +163,18 @@ An example event for `firewall` looks as following: "version": "8.2.0" }, "elastic_agent": { - "id": "c53ddea2-61ac-4643-8676-0c70ebf51c91", + "id": "df142714-8028-4ef0-a80c-4eb03051c084", "snapshot": false, - "version": "8.0.0-beta1" + "version": "7.17.0" }, "event": { "action": "firewall-rule", "agent_id_status": "verified", "category": "network", - "created": "2021-12-31T03:11:30.136Z", + "created": "2022-05-20T07:26:27.445Z", "dataset": "gcp.firewall", "id": "1f21ciqfpfssuo", - "ingested": "2021-12-31T03:11:31Z", + "ingested": "2022-05-20T07:26:28Z", "kind": "event", "type": "connection" }, diff --git a/packages/gcp/docs/vpcflow.md b/packages/gcp/docs/vpcflow.md index ad81bab4a6e..05973abc335 100644 --- a/packages/gcp/docs/vpcflow.md +++ b/packages/gcp/docs/vpcflow.md @@ -130,11 +130,20 @@ An example event for `vpcflow` looks as following: { "@timestamp": "2019-06-14T03:50:10.845Z", "agent": { - "ephemeral_id": "e58d02a0-e7a0-45c0-aba6-a8c983782744", - "id": "c53ddea2-61ac-4643-8676-0c70ebf51c91", + "ephemeral_id": "10bb82a5-c0e6-4aed-8589-003f734a7183", + "hostname": "docker-fleet-agent", + "id": "df142714-8028-4ef0-a80c-4eb03051c084", "name": "docker-fleet-agent", "type": "filebeat", - "version": "8.0.0-beta1" + "version": "7.17.0" + }, + "cloud": { + "availability_zone": "us-east1-b", + "project": { + "id": "my-sample-project" + }, + "provider": "gcp", + "region": "us-east1" }, "data_stream": { "dataset": "gcp.vpcflow", @@ -142,10 +151,11 @@ An example event for `vpcflow` looks as following: "type": "logs" }, "destination": { - "address": "67.43.156.13", + "address": "67.43.156.14", "as": { "number": 35908 }, + "domain": "elasticsearch", "geo": { "continent_name": "Asia", "country_iso_code": "BT", @@ -155,30 +165,42 @@ An example event for `vpcflow` looks as following: "lon": 90.5 } }, - "ip": "67.43.156.13", - "port": 33478 + "ip": "67.43.156.14", + "port": 9200 }, "ecs": { "version": "8.2.0" }, "elastic_agent": { - "id": "c53ddea2-61ac-4643-8676-0c70ebf51c91", + "id": "df142714-8028-4ef0-a80c-4eb03051c084", "snapshot": false, - "version": "8.0.0-beta1" + "version": "7.17.0" }, "event": { "agent_id_status": "verified", "category": "network", - "created": "2021-12-31T03:12:25.823Z", + "created": "2022-05-20T07:27:09.739Z", "dataset": "gcp.vpcflow", - "end": "2019-06-14T03:45:37.301953198Z", - "id": "ut8lbrffooxyw", - "ingested": "2021-12-31T03:12:26Z", + "end": "2019-06-14T03:49:51.821308944Z", + "id": "ut8lbrffooxyp", + "ingested": "2022-05-20T07:27:10Z", "kind": "event", - "start": "2019-06-14T03:45:37.186193305Z", + "start": "2019-06-14T03:40:08.469099728Z", "type": "connection" }, "gcp": { + "destination": { + "instance": { + "project_id": "my-sample-project", + "region": "us-east1", + "zone": "us-east1-b" + }, + "vpc": { + "project_id": "my-sample-project", + "subnetwork_name": "default", + "vpc_name": "default" + } + }, "source": { "instance": { "project_id": "my-sample-project", @@ -194,7 +216,7 @@ An example event for `vpcflow` looks as following: "vpcflow": { "reporter": "SRC", "rtt": { - "ms": 36 + "ms": 3 } } }, @@ -205,27 +227,28 @@ An example event for `vpcflow` looks as following: "logger": "projects/my-sample-project/logs/compute.googleapis.com%2Fvpc_flows" }, "network": { - "bytes": 1776, - "community_id": "1:Wa+aonxAQZ59AWtNdQD0CH6FnsM=", - "direction": "outbound", + "bytes": 15169, + "community_id": "1:NAY9D1IuyJAG+Hm34t3LIlP6/4c=", + "direction": "internal", "iana_number": "6", - "packets": 7, + "name": "default", + "packets": 92, "transport": "tcp", "type": "ipv4" }, "related": { "ip": [ "10.87.40.76", - "67.43.156.13" + "67.43.156.14" ] }, "source": { "address": "10.87.40.76", - "bytes": 1776, + "bytes": 15169, "domain": "kibana", "ip": "10.87.40.76", - "packets": 7, - "port": 5601 + "packets": 92, + "port": 33880 }, "tags": [ "forwarded", diff --git a/packages/gcp/manifest.yml b/packages/gcp/manifest.yml index b5587716cf7..2fb47f814aa 100644 --- a/packages/gcp/manifest.yml +++ b/packages/gcp/manifest.yml @@ -1,6 +1,6 @@ name: gcp title: Google Cloud Platform -version: "1.8.0" +version: "1.9.0" release: ga description: Collect logs from Google Cloud Platform with Elastic Agent. type: integration