From 7328c826f74a7c6e83ba13c4c87ad82d017b49cf Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 9 May 2022 11:46:21 -0400 Subject: [PATCH 01/10] Sort ecs.yml [git-generate] cd packages/snort yq -i '. | sort_by(.name) | sort_keys(..)' data_stream/log/fields/ecs.yml --- packages/snort/data_stream/log/fields/ecs.yml | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/packages/snort/data_stream/log/fields/ecs.yml b/packages/snort/data_stream/log/fields/ecs.yml index df23393a900..ce4abae6d58 100644 --- a/packages/snort/data_stream/log/fields/ecs.yml +++ b/packages/snort/data_stream/log/fields/ecs.yml @@ -42,10 +42,14 @@ name: event.severity - external: ecs name: host.ip +- external: ecs + name: log.file.path - external: ecs name: message - external: ecs name: network.community_id +- external: ecs + name: network.direction - external: ecs name: network.protocol - external: ecs @@ -53,21 +57,37 @@ - external: ecs name: network.type - external: ecs - name: network.direction + name: observer.ip +- external: ecs + name: observer.name +- external: ecs + name: observer.product +- external: ecs + name: observer.type +- external: ecs + name: observer.vendor +- external: ecs + name: process.name +- external: ecs + name: process.pid - external: ecs name: related.ip - external: ecs name: rule.category +- external: ecs + name: rule.description - external: ecs name: rule.id - external: ecs name: rule.name -- external: ecs - name: rule.description - external: ecs name: rule.version - external: ecs name: source.address +- external: ecs + name: source.as.number +- external: ecs + name: source.as.organization.name - external: ecs name: source.bytes - external: ecs @@ -94,25 +114,5 @@ name: source.packets - external: ecs name: source.port -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name - external: ecs name: tags -- external: ecs - name: observer.name -- external: ecs - name: observer.ip -- external: ecs - name: observer.vendor -- external: ecs - name: observer.type -- external: ecs - name: observer.product -- external: ecs - name: process.name -- external: ecs - name: process.pid -- external: ecs - name: log.file.path From 79b566157ab4bd9664ed20b6ccb05e4b3f70e26e Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 9 May 2022 11:50:13 -0400 Subject: [PATCH 02/10] Add missing event field mappings event.category event.created event.kind event.original event.timezone event.type --- packages/snort/data_stream/log/fields/ecs.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/snort/data_stream/log/fields/ecs.yml b/packages/snort/data_stream/log/fields/ecs.yml index ce4abae6d58..2fca1b8bb55 100644 --- a/packages/snort/data_stream/log/fields/ecs.yml +++ b/packages/snort/data_stream/log/fields/ecs.yml @@ -36,10 +36,22 @@ name: destination.port - external: ecs name: ecs.version +- external: ecs + name: event.category +- external: ecs + name: event.created +- external: ecs + name: event.kind +- external: ecs + name: event.original - external: ecs name: event.outcome - external: ecs name: event.severity +- external: ecs + name: event.timezone +- external: ecs + name: event.type - external: ecs name: host.ip - external: ecs From 6ed3d10cd3322c872d632ead16bc61585e0f935c Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 9 May 2022 12:16:22 -0400 Subject: [PATCH 03/10] Fix lint error for @timestamp The error was: expected type "date" for required field "@timestamp" --- packages/snort/data_stream/log/fields/ecs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/snort/data_stream/log/fields/ecs.yml b/packages/snort/data_stream/log/fields/ecs.yml index 2fca1b8bb55..b4463ca5f5d 100644 --- a/packages/snort/data_stream/log/fields/ecs.yml +++ b/packages/snort/data_stream/log/fields/ecs.yml @@ -1,5 +1,6 @@ - external: ecs name: '@timestamp' + type: date # TODO: Remove after https://github.com/elastic/elastic-package/issues/749 is fixed. - external: ecs name: destination.address - external: ecs From 7380a44a895e9bf53916735b4a29c71aab670623 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 9 May 2022 12:19:29 -0400 Subject: [PATCH 04/10] Remove duplicate host.ip definition --- packages/snort/data_stream/log/fields/agent.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/snort/data_stream/log/fields/agent.yml b/packages/snort/data_stream/log/fields/agent.yml index 79a7a39864b..befedc3a20a 100644 --- a/packages/snort/data_stream/log/fields/agent.yml +++ b/packages/snort/data_stream/log/fields/agent.yml @@ -107,10 +107,6 @@ type: keyword ignore_above: 1024 description: "Unique host id.\nAs hostname is not always unique, use values that are meaningful in your environment.\nExample: The current usage of `beat.name`." - - name: ip - level: core - type: ip - description: Host ip addresses. - name: mac level: core type: keyword From 67bf9008b7a7e856a3951e1e3957ff13fceb926a Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 9 May 2022 12:20:05 -0400 Subject: [PATCH 05/10] Use ECS for {source,destination}.geo.location --- packages/snort/data_stream/log/fields/ecs.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/packages/snort/data_stream/log/fields/ecs.yml b/packages/snort/data_stream/log/fields/ecs.yml index b4463ca5f5d..eb12ef4375d 100644 --- a/packages/snort/data_stream/log/fields/ecs.yml +++ b/packages/snort/data_stream/log/fields/ecs.yml @@ -19,10 +19,8 @@ name: destination.geo.country_iso_code - external: ecs name: destination.geo.country_name -- description: Longitude and latitude. - level: core +- external: ecs name: destination.geo.location - type: geo_point - external: ecs name: destination.geo.region_iso_code - external: ecs @@ -111,10 +109,8 @@ name: source.geo.country_iso_code - external: ecs name: source.geo.country_name -- description: Longitude and latitude. - level: core +- external: ecs name: source.geo.location - type: geo_point - external: ecs name: source.geo.region_iso_code - external: ecs From d6c3923be293ae7744d0fbb76cc94f28f588e1bb Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 9 May 2022 11:51:50 -0400 Subject: [PATCH 06/10] Format source.mac, destination.mac as per ECS --- .../elasticsearch/ingest_pipeline/default.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/default.yml index c772455d0f2..b7afa7bdf6c 100644 --- a/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -80,6 +80,22 @@ processors: target_field: destination.ip type: ip ignore_missing: true + - uppercase: + field: destination.mac + ignore_missing: true + - uppercase: + field: source.mac + ignore_missing: true + - gsub: + field: destination.mac + pattern: '[.:]' + replacement: '-' + ignore_missing: true + - gsub: + field: source.mac + pattern: '[.:]' + replacement: '-' + ignore_missing: true - lowercase: field: network.transport ignore_missing: true From 421270f98f3b2b0cacb050dcb3bf30445a5b6842 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 9 May 2022 12:06:18 -0400 Subject: [PATCH 07/10] Update elastic/stream --- packages/snort/_dev/deploy/docker/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/snort/_dev/deploy/docker/docker-compose.yml b/packages/snort/_dev/deploy/docker/docker-compose.yml index 45c37613e85..d2e6a12dc79 100644 --- a/packages/snort/_dev/deploy/docker/docker-compose.yml +++ b/packages/snort/_dev/deploy/docker/docker-compose.yml @@ -7,7 +7,7 @@ services: - ${SERVICE_LOGS_DIR}:/var/log command: /bin/sh -c "cp /sample_logs/*.log /var/log/" snort-log-udp: - image: akroh/stream:v0.3.0 + image: docker.elastic.co/observability/stream:v0.7.0 volumes: - ./sample_logs:/sample_logs:ro command: log --start-signal=SIGHUP --delay=5s --addr elastic-agent:9514 -p=udp /sample_logs/test-syslog.log From b2d7d55d37305fa771fd0760f85943441dbc9105 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 9 May 2022 12:06:36 -0400 Subject: [PATCH 08/10] Update sample event --- .../snort/data_stream/log/sample_event.json | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/snort/data_stream/log/sample_event.json b/packages/snort/data_stream/log/sample_event.json index d3cb5c9ad28..e2add115dab 100644 --- a/packages/snort/data_stream/log/sample_event.json +++ b/packages/snort/data_stream/log/sample_event.json @@ -1,11 +1,11 @@ { "@timestamp": "2022-09-05T16:02:55.000-05:00", "agent": { - "ephemeral_id": "d1ca036e-57c0-4c4a-9b92-ddc5f4cdb3a2", - "id": "584f3aea-648c-4e58-aba4-32b8f88d4396", + "ephemeral_id": "3ada3cc1-9563-4aa5-880e-585d87fc6adf", + "id": "ca0beb8d-9522-4450-8af7-3cb7f3d8c478", "name": "docker-fleet-agent", "type": "filebeat", - "version": "8.0.0-beta1" + "version": "8.2.0" }, "data_stream": { "dataset": "snort.log", @@ -29,12 +29,12 @@ "ip": "175.16.199.1" }, "ecs": { - "version": "8.2.0" + "version": "8.3.0" }, "elastic_agent": { - "id": "584f3aea-648c-4e58-aba4-32b8f88d4396", + "id": "ca0beb8d-9522-4450-8af7-3cb7f3d8c478", "snapshot": false, - "version": "8.0.0-beta1" + "version": "8.2.0" }, "event": { "agent_id_status": "verified", @@ -43,9 +43,9 @@ ], "created": "2022-09-05T16:02:55.000-05:00", "dataset": "snort.log", - "ingested": "2022-02-03T09:26:00Z", + "ingested": "2022-05-09T16:00:09Z", "kind": "alert", - "original": "Sep 5 16:02:55 dev snort: [1:1000015:0] Pinging... [Classification: Misc activity] [Priority: 3] {ICMP} 10.50.10.88 -\u003e 175.16.199.1\n", + "original": "Sep 5 16:02:55 dev snort: [1:1000015:0] Pinging... [Classification: Misc activity] [Priority: 3] {ICMP} 10.50.10.88 -\u003e 175.16.199.1", "severity": 3, "timezone": "-05:00" }, @@ -54,7 +54,7 @@ }, "log": { "source": { - "address": "172.19.0.7:38583" + "address": "172.18.0.4:54924" } }, "network": { From 0ca08c71e9e9f1479ad01837745ed198c7bee873 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 9 May 2022 12:07:10 -0400 Subject: [PATCH 09/10] Update readme and generate pipeline test files [git-generate] cd packages/snort elastic-package build elastic-package test pipeline -g --- .../pipeline/test-log-csv.log-expected.json | 32 +++++++++---------- packages/snort/docs/README.md | 24 ++++++++------ 2 files changed, 31 insertions(+), 25 deletions(-) diff --git a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-csv.log-expected.json b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-csv.log-expected.json index fcc059c37c3..3c594dba257 100644 --- a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-csv.log-expected.json +++ b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-csv.log-expected.json @@ -5,7 +5,7 @@ "destination": { "address": "10.100.10.190", "ip": "10.100.10.190", - "mac": "00:50:56:9D:A5:BE", + "mac": "00-50-56-9D-A5-BE", "port": 22 }, "ecs": { @@ -66,7 +66,7 @@ "source": { "address": "10.100.20.59", "ip": "10.100.20.59", - "mac": "00:25:90:3A:05:13", + "mac": "00-25-90-3A-05-13", "port": 57263 }, "tags": [ @@ -78,7 +78,7 @@ "destination": { "address": "10.100.10.190", "ip": "10.100.10.190", - "mac": "00:50:56:9D:A5:BE", + "mac": "00-50-56-9D-A5-BE", "port": 22 }, "ecs": { @@ -139,7 +139,7 @@ "source": { "address": "10.100.20.59", "ip": "10.100.20.59", - "mac": "00:25:90:3A:05:13", + "mac": "00-25-90-3A-05-13", "port": 57263 }, "tags": [ @@ -151,7 +151,7 @@ "destination": { "address": "10.100.10.190", "ip": "10.100.10.190", - "mac": "00:50:56:9D:A5:BE", + "mac": "00-50-56-9D-A5-BE", "port": 55475 }, "ecs": { @@ -209,7 +209,7 @@ "source": { "address": "10.100.10.1", "ip": "10.100.10.1", - "mac": "00:25:90:3A:05:13", + "mac": "00-25-90-3A-05-13", "port": 53 }, "tags": [ @@ -221,7 +221,7 @@ "destination": { "address": "10.100.10.190", "ip": "10.100.10.190", - "mac": "00:50:56:9D:A5:BE", + "mac": "00-50-56-9D-A5-BE", "port": 55333 }, "ecs": { @@ -279,7 +279,7 @@ "source": { "address": "10.100.10.1", "ip": "10.100.10.1", - "mac": "00:25:90:3A:05:13", + "mac": "00-25-90-3A-05-13", "port": 53 }, "tags": [ @@ -291,7 +291,7 @@ "destination": { "address": "10.100.10.255", "ip": "10.100.10.255", - "mac": "FF:FF:FF:FF:FF:FF", + "mac": "FF-FF-FF-FF-FF-FF", "port": 32414 }, "ecs": { @@ -349,7 +349,7 @@ "source": { "address": "10.100.10.75", "ip": "10.100.10.75", - "mac": "00:0C:29:B8:43:CE", + "mac": "00-0C-29-B8-43-CE", "port": 55776 }, "tags": [ @@ -373,7 +373,7 @@ "region_name": "Jilin Sheng" }, "ip": "175.16.199.1", - "mac": "00:25:90:3A:05:13" + "mac": "00-25-90-3A-05-13" }, "ecs": { "version": "8.2.0" @@ -436,7 +436,7 @@ "source": { "address": "10.100.10.190", "ip": "10.100.10.190", - "mac": "00:50:56:9D:A5:BE" + "mac": "00-50-56-9D-A5-BE" }, "tags": [ "preserve_original_event" @@ -447,7 +447,7 @@ "destination": { "address": "10.100.10.190", "ip": "10.100.10.190", - "mac": "00:50:56:9D:A5:BE" + "mac": "00-50-56-9D-A5-BE" }, "ecs": { "version": "8.2.0" @@ -522,7 +522,7 @@ "region_name": "Jilin Sheng" }, "ip": "175.16.199.1", - "mac": "00:25:90:3A:05:13" + "mac": "00-25-90-3A-05-13" }, "tags": [ "preserve_original_event" @@ -545,7 +545,7 @@ "region_name": "Jilin Sheng" }, "ip": "175.16.199.1", - "mac": "00:25:90:3A:05:13" + "mac": "00-25-90-3A-05-13" }, "ecs": { "version": "8.2.0" @@ -608,7 +608,7 @@ "source": { "address": "10.100.10.190", "ip": "10.100.10.190", - "mac": "00:50:56:9D:A5:BE" + "mac": "00-50-56-9D-A5-BE" }, "tags": [ "preserve_original_event" diff --git a/packages/snort/docs/README.md b/packages/snort/docs/README.md index daa71c3d7d5..b72798fbb80 100644 --- a/packages/snort/docs/README.md +++ b/packages/snort/docs/README.md @@ -16,11 +16,11 @@ An example event for `log` looks as following: { "@timestamp": "2022-09-05T16:02:55.000-05:00", "agent": { - "ephemeral_id": "d1ca036e-57c0-4c4a-9b92-ddc5f4cdb3a2", - "id": "584f3aea-648c-4e58-aba4-32b8f88d4396", + "ephemeral_id": "3ada3cc1-9563-4aa5-880e-585d87fc6adf", + "id": "ca0beb8d-9522-4450-8af7-3cb7f3d8c478", "name": "docker-fleet-agent", "type": "filebeat", - "version": "8.0.0-beta1" + "version": "8.2.0" }, "data_stream": { "dataset": "snort.log", @@ -44,12 +44,12 @@ An example event for `log` looks as following: "ip": "175.16.199.1" }, "ecs": { - "version": "8.2.0" + "version": "8.3.0" }, "elastic_agent": { - "id": "584f3aea-648c-4e58-aba4-32b8f88d4396", + "id": "ca0beb8d-9522-4450-8af7-3cb7f3d8c478", "snapshot": false, - "version": "8.0.0-beta1" + "version": "8.2.0" }, "event": { "agent_id_status": "verified", @@ -58,9 +58,9 @@ An example event for `log` looks as following: ], "created": "2022-09-05T16:02:55.000-05:00", "dataset": "snort.log", - "ingested": "2022-02-03T09:26:00Z", + "ingested": "2022-05-09T16:00:09Z", "kind": "alert", - "original": "Sep 5 16:02:55 dev snort: [1:1000015:0] Pinging... [Classification: Misc activity] [Priority: 3] {ICMP} 10.50.10.88 -\u003e 175.16.199.1\n", + "original": "Sep 5 16:02:55 dev snort: [1:1000015:0] Pinging... [Classification: Misc activity] [Priority: 3] {ICMP} 10.50.10.88 -\u003e 175.16.199.1", "severity": 3, "timezone": "-05:00" }, @@ -69,7 +69,7 @@ An example event for `log` looks as following: }, "log": { "source": { - "address": "172.19.0.7:38583" + "address": "172.18.0.4:54924" } }, "network": { @@ -153,10 +153,16 @@ An example event for `log` looks as following: | destination.packets | Packets sent from the destination to the source. | long | | destination.port | Port of the destination. | long | | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | +| event.created | event.created contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from @timestamp in that @timestamp typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, @timestamp should be used. | date | | event.dataset | Event dataset | constant_keyword | +| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | +| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | | event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | | event.severity | The numeric severity of the event according to your event source. What the different severity values mean can be different between sources and use cases. It's up to the implementer to make sure severities are consistent across events from the same source. The Syslog severity belongs in `log.syslog.severity.code`. `event.severity` is meant to represent the severity according to the event source (e.g. firewall, IDS). If the event source does not publish its own severity, you may optionally copy the `log.syslog.severity.code` to `event.severity`. | long | +| event.timezone | This field should be populated when the event's timestamp does not include timezone information already (e.g. default Syslog timestamps). It's optional otherwise. Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00"). | keyword | +| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | From 25c17f38626a3aecf05074c2b7b36cdb8322a07a Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 9 May 2022 12:31:17 -0400 Subject: [PATCH 10/10] Update changelog [git-generate] elastic-package-changelog add-next --pr 3301 --type bug --description "Format source.mac and destination.mac as per ECS and add missing mappings for various event.* fields." --- packages/snort/changelog.yml | 5 +++++ packages/snort/manifest.yml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/snort/changelog.yml b/packages/snort/changelog.yml index f4c786b4c49..7501529198d 100644 --- a/packages/snort/changelog.yml +++ b/packages/snort/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.3.1" + changes: + - description: Format source.mac and destination.mac as per ECS and add missing mappings for various event.* fields. + type: bugfix + link: https://github.com/elastic/integrations/pull/3301 - version: "0.3.0" changes: - description: Update to ECS 8.2 diff --git a/packages/snort/manifest.yml b/packages/snort/manifest.yml index 7418522fa76..d1e61c02cfd 100644 --- a/packages/snort/manifest.yml +++ b/packages/snort/manifest.yml @@ -1,6 +1,6 @@ name: snort title: Snort -version: 0.3.0 +version: "0.3.1" release: experimental description: Collect logs from Snort with Elastic Agent. type: integration