diff --git a/packages/santa/changelog.yml b/packages/santa/changelog.yml index 5028e1cd08f..55978f55f42 100644 --- a/packages/santa/changelog.yml +++ b/packages/santa/changelog.yml @@ -1,8 +1,13 @@ # newer versions go on top +- version: "3.1.0" + changes: + - description: Add `process.entity_id` field. + type: enhancement + link: https://github.com/elastic/integrations/pull/3373 - version: "3.0.0" changes: - description: Update log format to support the GA releases of Santa. The pre-GA Santa log format (circa 2017) is no longer accepted. - type: breaking-change + type: enhancement link: https://github.com/elastic/integrations/pull/3347 - version: "2.1.0" changes: diff --git a/packages/santa/data_stream/log/_dev/test/pipeline/test-santa-raw.log-expected.json b/packages/santa/data_stream/log/_dev/test/pipeline/test-santa-raw.log-expected.json index aca0ba1701f..a61ef2c6684 100644 --- a/packages/santa/data_stream/log/_dev/test/pipeline/test-santa-raw.log-expected.json +++ b/packages/santa/data_stream/log/_dev/test/pipeline/test-santa-raw.log-expected.json @@ -37,6 +37,7 @@ "xpcproxy", "com.apple.CoreAuthentication.agent" ], + "entity_id": "71993-1097732", "executable": "/usr/libexec/xpcproxy", "hash": { "sha256": "43158bf397bf52001067319c591249307e2862daf8690828c15cdcc1ddf6166d" @@ -112,6 +113,7 @@ "/System/Library/PrivateFrameworks/TrustEvaluationAgent.framework/Versions/A/Resources/trustevaluationagent", "trustevaluationagent" ], + "entity_id": "72012-1097765", "executable": "/System/Library/PrivateFrameworks/TrustEvaluationAgent.framework/Versions/A/Resources/trustevaluationagent", "hash": { "sha256": "7207307ca09d2707368ec394e67c6ccff6e48a2d1d86225a3115fe3535a8237c" @@ -174,6 +176,7 @@ "args": [ "/usr/libexec/syspolicyd" ], + "entity_id": "377-833", "executable": "/usr/libexec/syspolicyd", "name": "syspolicyd", "parent": { @@ -224,6 +227,7 @@ "args": [ "/usr/sbin/newsyslog" ], + "entity_id": "71559-1096716", "executable": "/usr/sbin/newsyslog", "name": "newsyslog", "parent": { @@ -274,6 +278,7 @@ "args": [ "/System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/Metadata.framework/Versions/A/Support/mds_stores" ], + "entity_id": "546-1285", "executable": "/System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/Metadata.framework/Versions/A/Support/mds_stores", "name": "mds_stores", "parent": { @@ -323,6 +328,7 @@ "args": [ "/sbin/launchd" ], + "entity_id": "1-521", "executable": "/sbin/launchd", "name": "launchd", "parent": { diff --git a/packages/santa/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/santa/data_stream/log/elasticsearch/ingest_pipeline/default.yml index fab9d306a75..54491ba6095 100644 --- a/packages/santa/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/santa/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -16,6 +16,14 @@ processors: - '\[%{TIMESTAMP_ISO8601:timestamp}\] %{NOT_SEPARATOR:log.level} santad: action=%{NOT_SEPARATOR:santa.action}\|mount=%{NOT_SEPARATOR:santa.disk.mount}?\|volume=%{NOT_SEPARATOR:santa.disk.volume}\|bsdname=%{NOT_SEPARATOR:santa.disk.bsdname}?(\|fs=%{NOT_SEPARATOR:santa.disk.fs})?(\|model=%{NOT_SEPARATOR:santa.disk.model}?)?(\|serial=%{NOT_SEPARATOR:santa.disk.serial}?)?(\|bus=%{NOT_SEPARATOR:santa.disk.bus}?)?(\|dmgpath=%{NOT_SEPARATOR:santa.disk.dmgpath}?)?(\|appearance=%{TIMESTAMP_ISO8601:santa.disk.appearance})?' pattern_definitions: NOT_SEPARATOR: '[^\|]+' + - set: + field: process.entity_id + value: "{{{process.pid}}}-{{{santa.pidversion}}}" + if: "ctx.process?.pid != null && ctx.santa?.pidversion != null" + - set: + field: process.entity_id + value: "{{{agent.id}}}-{{{process.entity_id}}}" + if: "ctx.agent?.id != null && ctx.process?.entity_id != null" - date: field: process.start target_field: process.start diff --git a/packages/santa/data_stream/log/fields/ecs.yml b/packages/santa/data_stream/log/fields/ecs.yml index 64d94d01105..ca4a4858ec6 100644 --- a/packages/santa/data_stream/log/fields/ecs.yml +++ b/packages/santa/data_stream/log/fields/ecs.yml @@ -2,6 +2,8 @@ name: ecs.version - external: ecs name: event.ingested +- external: ecs + name: agent.id - external: ecs name: file.path - external: ecs @@ -24,6 +26,8 @@ name: process.hash.sha256 - external: ecs name: process.pid +- external: ecs + name: process.entity_id - external: ecs name: process.parent.pid - external: ecs diff --git a/packages/santa/data_stream/log/sample_event.json b/packages/santa/data_stream/log/sample_event.json index 66e962dc5f6..b3dcbaecd43 100644 --- a/packages/santa/data_stream/log/sample_event.json +++ b/packages/santa/data_stream/log/sample_event.json @@ -1,73 +1,95 @@ { - "@timestamp": "2022-05-12T11:38:03.923Z", + "@timestamp": "2022-05-12T11:30:05.248Z", + "agent": { + "ephemeral_id": "ea9b3ab9-896a-456a-8e87-7a6452edad19", + "id": "2c596a05-d358-406e-924c-bf221088f43c", + "name": "docker-fleet-agent", + "type": "filebeat", + "version": "8.2.1" + }, + "data_stream": { + "dataset": "santa.log", + "namespace": "ep", + "type": "logs" + }, "ecs": { "version": "8.2.0" }, + "elastic_agent": { + "id": "2c596a05-d358-406e-924c-bf221088f43c", + "snapshot": true, + "version": "8.2.1" + }, "event": { - "action": "exec", - "category": [ - "process" - ], - "kind": "event", - "original": "[2022-05-12T11:38:03.923Z] I santad: action=EXEC|decision=ALLOW|reason=BINARY|explain=critical system binary|sha256=43158bf397bf52001067319c591249307e2862daf8690828c15cdcc1ddf6166d|cert_sha256=d84db96af8c2e60ac4c851a21ec460f6f84e0235beb17d24a78712b9b021ed57|cert_cn=Software Signing|pid=71993|pidversion=1097732|ppid=1|uid=0|user=root|gid=0|group=wheel|mode=M|path=/usr/libexec/xpcproxy|args=xpcproxy com.apple.CoreAuthentication.agent", - "outcome": "success", - "type": [ - "start" - ] + "action": "link", + "agent_id_status": "verified", + "dataset": "santa.log", + "ingested": "2022-05-18T03:34:40Z", + "kind": "event" }, "file": { - "x509": { - "issuer": { - "common_name": "Software Signing" - } - } + "path": "/private/var/db/santa/santa.log", + "target_path": "/private/var/db/santa/santa.log.0" }, "group": { "id": "0", "name": "wheel" }, + "host": { + "architecture": "x86_64", + "containerized": false, + "hostname": "docker-fleet-agent", + "ip": [ + "192.168.160.7" + ], + "mac": [ + "02:42:c0:a8:a0:07" + ], + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.10.104-linuxkit", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.4 LTS (Focal Fossa)" + } + }, + "input": { + "type": "log" + }, "log": { - "level": "I" + "file": { + "path": "/tmp/service_logs/santa.log" + }, + "level": "I", + "offset": 1150 }, "process": { "args": [ - "/usr/libexec/xpcproxy", - "xpcproxy", - "com.apple.CoreAuthentication.agent" + "/usr/sbin/newsyslog" ], - "executable": "/usr/libexec/xpcproxy", - "hash": { - "sha256": "43158bf397bf52001067319c591249307e2862daf8690828c15cdcc1ddf6166d" - }, + "entity_id": "2c596a05-d358-406e-924c-bf221088f43c-71559-1096716", + "executable": "/usr/sbin/newsyslog", + "name": "newsyslog", "parent": { "pid": 1 }, - "pid": 71993, - "start": "2022-05-12T11:38:03.923Z" + "pid": 71559, + "start": "2022-05-12T11:30:05.248Z" }, "related": { - "hash": [ - "d84db96af8c2e60ac4c851a21ec460f6f84e0235beb17d24a78712b9b021ed57", - "43158bf397bf52001067319c591249307e2862daf8690828c15cdcc1ddf6166d" - ], "user": [ "root" ] }, "santa": { - "action": "EXEC", - "certificate": { - "common_name": "Software Signing", - "sha256": "d84db96af8c2e60ac4c851a21ec460f6f84e0235beb17d24a78712b9b021ed57" - }, - "decision": "ALLOW", - "explain": "critical system binary", - "mode": "M", - "pidversion": 1097732, - "reason": "BINARY" + "action": "LINK", + "pidversion": 1096716 }, "tags": [ - "preserve_original_event" + "santa-log" ], "user": { "id": "0", diff --git a/packages/santa/docs/README.md b/packages/santa/docs/README.md index eace619b35d..2e9a94e4b25 100644 --- a/packages/santa/docs/README.md +++ b/packages/santa/docs/README.md @@ -21,75 +21,97 @@ An example event for `log` looks as following: ```json { - "@timestamp": "2022-05-12T11:38:03.923Z", + "@timestamp": "2022-05-12T11:30:05.248Z", + "agent": { + "ephemeral_id": "ea9b3ab9-896a-456a-8e87-7a6452edad19", + "id": "2c596a05-d358-406e-924c-bf221088f43c", + "name": "docker-fleet-agent", + "type": "filebeat", + "version": "8.2.1" + }, + "data_stream": { + "dataset": "santa.log", + "namespace": "ep", + "type": "logs" + }, "ecs": { "version": "8.2.0" }, + "elastic_agent": { + "id": "2c596a05-d358-406e-924c-bf221088f43c", + "snapshot": true, + "version": "8.2.1" + }, "event": { - "action": "exec", - "category": [ - "process" - ], - "kind": "event", - "original": "[2022-05-12T11:38:03.923Z] I santad: action=EXEC|decision=ALLOW|reason=BINARY|explain=critical system binary|sha256=43158bf397bf52001067319c591249307e2862daf8690828c15cdcc1ddf6166d|cert_sha256=d84db96af8c2e60ac4c851a21ec460f6f84e0235beb17d24a78712b9b021ed57|cert_cn=Software Signing|pid=71993|pidversion=1097732|ppid=1|uid=0|user=root|gid=0|group=wheel|mode=M|path=/usr/libexec/xpcproxy|args=xpcproxy com.apple.CoreAuthentication.agent", - "outcome": "success", - "type": [ - "start" - ] + "action": "link", + "agent_id_status": "verified", + "dataset": "santa.log", + "ingested": "2022-05-18T03:34:40Z", + "kind": "event" }, "file": { - "x509": { - "issuer": { - "common_name": "Software Signing" - } - } + "path": "/private/var/db/santa/santa.log", + "target_path": "/private/var/db/santa/santa.log.0" }, "group": { "id": "0", "name": "wheel" }, + "host": { + "architecture": "x86_64", + "containerized": false, + "hostname": "docker-fleet-agent", + "ip": [ + "192.168.160.7" + ], + "mac": [ + "02:42:c0:a8:a0:07" + ], + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.10.104-linuxkit", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.4 LTS (Focal Fossa)" + } + }, + "input": { + "type": "log" + }, "log": { - "level": "I" + "file": { + "path": "/tmp/service_logs/santa.log" + }, + "level": "I", + "offset": 1150 }, "process": { "args": [ - "/usr/libexec/xpcproxy", - "xpcproxy", - "com.apple.CoreAuthentication.agent" + "/usr/sbin/newsyslog" ], - "executable": "/usr/libexec/xpcproxy", - "hash": { - "sha256": "43158bf397bf52001067319c591249307e2862daf8690828c15cdcc1ddf6166d" - }, + "entity_id": "2c596a05-d358-406e-924c-bf221088f43c-71559-1096716", + "executable": "/usr/sbin/newsyslog", + "name": "newsyslog", "parent": { "pid": 1 }, - "pid": 71993, - "start": "2022-05-12T11:38:03.923Z" + "pid": 71559, + "start": "2022-05-12T11:30:05.248Z" }, "related": { - "hash": [ - "d84db96af8c2e60ac4c851a21ec460f6f84e0235beb17d24a78712b9b021ed57", - "43158bf397bf52001067319c591249307e2862daf8690828c15cdcc1ddf6166d" - ], "user": [ "root" ] }, "santa": { - "action": "EXEC", - "certificate": { - "common_name": "Software Signing", - "sha256": "d84db96af8c2e60ac4c851a21ec460f6f84e0235beb17d24a78712b9b021ed57" - }, - "decision": "ALLOW", - "explain": "critical system binary", - "mode": "M", - "pidversion": 1097732, - "reason": "BINARY" + "action": "LINK", + "pidversion": 1096716 }, "tags": [ - "preserve_original_event" + "santa-log" ], "user": { "id": "0", @@ -103,6 +125,7 @@ An example event for `log` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | +| agent.id | Unique identifier of this agent (if one exists). Example: For Beats this would be beat.id. | keyword | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | @@ -152,6 +175,7 @@ An example event for `log` looks as following: | log.level | Original log level of the log event. If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). Some examples are `warn`, `err`, `i`, `informational`. | keyword | | log.offset | Log offset | long | | process.args | Array of process arguments, starting with the absolute path to the executable. May be filtered to protect sensitive information. | keyword | +| process.entity_id | Unique identifier for the process. The implementation of this is specified by the data source, but some examples of what could be used here are a process-generated UUID, Sysmon Process GUIDs, or a hash of some uniquely identifying components of a process. Constructing a globally unique identifier is a common practice to mitigate PID reuse as well as to identify a specific process over time, across multiple monitored hosts. | keyword | | process.executable | Absolute path to the process executable. | keyword | | process.executable.text | Multi-field of `process.executable`. | match_only_text | | process.hash.sha256 | SHA256 hash. | keyword | diff --git a/packages/santa/manifest.yml b/packages/santa/manifest.yml index 9fcd71b70fe..94e2bdd37df 100644 --- a/packages/santa/manifest.yml +++ b/packages/santa/manifest.yml @@ -1,6 +1,6 @@ name: santa title: Google Santa Logs -version: 3.0.0 +version: 3.1.0 release: ga description: Collect and parse logs from Google Santa instances with Elastic Agent. type: integration