diff --git a/packages/snort/_dev/deploy/docker/docker-compose.yml b/packages/snort/_dev/deploy/docker/docker-compose.yml index 45c37613e85..d2e6a12dc79 100644 --- a/packages/snort/_dev/deploy/docker/docker-compose.yml +++ b/packages/snort/_dev/deploy/docker/docker-compose.yml @@ -7,7 +7,7 @@ services: - ${SERVICE_LOGS_DIR}:/var/log command: /bin/sh -c "cp /sample_logs/*.log /var/log/" snort-log-udp: - image: akroh/stream:v0.3.0 + image: docker.elastic.co/observability/stream:v0.7.0 volumes: - ./sample_logs:/sample_logs:ro command: log --start-signal=SIGHUP --delay=5s --addr elastic-agent:9514 -p=udp /sample_logs/test-syslog.log diff --git a/packages/snort/changelog.yml b/packages/snort/changelog.yml index f4c786b4c49..7501529198d 100644 --- a/packages/snort/changelog.yml +++ b/packages/snort/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.3.1" + changes: + - description: Format source.mac and destination.mac as per ECS and add missing mappings for various event.* fields. + type: bugfix + link: https://github.com/elastic/integrations/pull/3301 - version: "0.3.0" changes: - description: Update to ECS 8.2 diff --git a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-csv.log-expected.json b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-csv.log-expected.json index fcc059c37c3..3c594dba257 100644 --- a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-csv.log-expected.json +++ b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-csv.log-expected.json @@ -5,7 +5,7 @@ "destination": { "address": "10.100.10.190", "ip": "10.100.10.190", - "mac": "00:50:56:9D:A5:BE", + "mac": "00-50-56-9D-A5-BE", "port": 22 }, "ecs": { @@ -66,7 +66,7 @@ "source": { "address": "10.100.20.59", "ip": "10.100.20.59", - "mac": "00:25:90:3A:05:13", + "mac": "00-25-90-3A-05-13", "port": 57263 }, "tags": [ @@ -78,7 +78,7 @@ "destination": { "address": "10.100.10.190", "ip": "10.100.10.190", - "mac": "00:50:56:9D:A5:BE", + "mac": "00-50-56-9D-A5-BE", "port": 22 }, "ecs": { @@ -139,7 +139,7 @@ "source": { "address": "10.100.20.59", "ip": "10.100.20.59", - "mac": "00:25:90:3A:05:13", + "mac": "00-25-90-3A-05-13", "port": 57263 }, "tags": [ @@ -151,7 +151,7 @@ "destination": { "address": "10.100.10.190", "ip": "10.100.10.190", - "mac": "00:50:56:9D:A5:BE", + "mac": "00-50-56-9D-A5-BE", "port": 55475 }, "ecs": { @@ -209,7 +209,7 @@ "source": { "address": "10.100.10.1", "ip": "10.100.10.1", - "mac": "00:25:90:3A:05:13", + "mac": "00-25-90-3A-05-13", "port": 53 }, "tags": [ @@ -221,7 +221,7 @@ "destination": { "address": "10.100.10.190", "ip": "10.100.10.190", - "mac": "00:50:56:9D:A5:BE", + "mac": "00-50-56-9D-A5-BE", "port": 55333 }, "ecs": { @@ -279,7 +279,7 @@ "source": { "address": "10.100.10.1", "ip": "10.100.10.1", - "mac": "00:25:90:3A:05:13", + "mac": "00-25-90-3A-05-13", "port": 53 }, "tags": [ @@ -291,7 +291,7 @@ "destination": { "address": "10.100.10.255", "ip": "10.100.10.255", - "mac": "FF:FF:FF:FF:FF:FF", + "mac": "FF-FF-FF-FF-FF-FF", "port": 32414 }, "ecs": { @@ -349,7 +349,7 @@ "source": { "address": "10.100.10.75", "ip": "10.100.10.75", - "mac": "00:0C:29:B8:43:CE", + "mac": "00-0C-29-B8-43-CE", "port": 55776 }, "tags": [ @@ -373,7 +373,7 @@ "region_name": "Jilin Sheng" }, "ip": "175.16.199.1", - "mac": "00:25:90:3A:05:13" + "mac": "00-25-90-3A-05-13" }, "ecs": { "version": "8.2.0" @@ -436,7 +436,7 @@ "source": { "address": "10.100.10.190", "ip": "10.100.10.190", - "mac": "00:50:56:9D:A5:BE" + "mac": "00-50-56-9D-A5-BE" }, "tags": [ "preserve_original_event" @@ -447,7 +447,7 @@ "destination": { "address": "10.100.10.190", "ip": "10.100.10.190", - "mac": "00:50:56:9D:A5:BE" + "mac": "00-50-56-9D-A5-BE" }, "ecs": { "version": "8.2.0" @@ -522,7 +522,7 @@ "region_name": "Jilin Sheng" }, "ip": "175.16.199.1", - "mac": "00:25:90:3A:05:13" + "mac": "00-25-90-3A-05-13" }, "tags": [ "preserve_original_event" @@ -545,7 +545,7 @@ "region_name": "Jilin Sheng" }, "ip": "175.16.199.1", - "mac": "00:25:90:3A:05:13" + "mac": "00-25-90-3A-05-13" }, "ecs": { "version": "8.2.0" @@ -608,7 +608,7 @@ "source": { "address": "10.100.10.190", "ip": "10.100.10.190", - "mac": "00:50:56:9D:A5:BE" + "mac": "00-50-56-9D-A5-BE" }, "tags": [ "preserve_original_event" diff --git a/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/default.yml index c772455d0f2..b7afa7bdf6c 100644 --- a/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -80,6 +80,22 @@ processors: target_field: destination.ip type: ip ignore_missing: true + - uppercase: + field: destination.mac + ignore_missing: true + - uppercase: + field: source.mac + ignore_missing: true + - gsub: + field: destination.mac + pattern: '[.:]' + replacement: '-' + ignore_missing: true + - gsub: + field: source.mac + pattern: '[.:]' + replacement: '-' + ignore_missing: true - lowercase: field: network.transport ignore_missing: true diff --git a/packages/snort/data_stream/log/fields/agent.yml b/packages/snort/data_stream/log/fields/agent.yml index 79a7a39864b..befedc3a20a 100644 --- a/packages/snort/data_stream/log/fields/agent.yml +++ b/packages/snort/data_stream/log/fields/agent.yml @@ -107,10 +107,6 @@ type: keyword ignore_above: 1024 description: "Unique host id.\nAs hostname is not always unique, use values that are meaningful in your environment.\nExample: The current usage of `beat.name`." - - name: ip - level: core - type: ip - description: Host ip addresses. - name: mac level: core type: keyword diff --git a/packages/snort/data_stream/log/fields/ecs.yml b/packages/snort/data_stream/log/fields/ecs.yml index df23393a900..eb12ef4375d 100644 --- a/packages/snort/data_stream/log/fields/ecs.yml +++ b/packages/snort/data_stream/log/fields/ecs.yml @@ -1,5 +1,6 @@ - external: ecs name: '@timestamp' + type: date # TODO: Remove after https://github.com/elastic/elastic-package/issues/749 is fixed. - external: ecs name: destination.address - external: ecs @@ -18,10 +19,8 @@ name: destination.geo.country_iso_code - external: ecs name: destination.geo.country_name -- description: Longitude and latitude. - level: core +- external: ecs name: destination.geo.location - type: geo_point - external: ecs name: destination.geo.region_iso_code - external: ecs @@ -36,16 +35,32 @@ name: destination.port - external: ecs name: ecs.version +- external: ecs + name: event.category +- external: ecs + name: event.created +- external: ecs + name: event.kind +- external: ecs + name: event.original - external: ecs name: event.outcome - external: ecs name: event.severity +- external: ecs + name: event.timezone +- external: ecs + name: event.type - external: ecs name: host.ip +- external: ecs + name: log.file.path - external: ecs name: message - external: ecs name: network.community_id +- external: ecs + name: network.direction - external: ecs name: network.protocol - external: ecs @@ -53,21 +68,37 @@ - external: ecs name: network.type - external: ecs - name: network.direction + name: observer.ip +- external: ecs + name: observer.name +- external: ecs + name: observer.product +- external: ecs + name: observer.type +- external: ecs + name: observer.vendor +- external: ecs + name: process.name +- external: ecs + name: process.pid - external: ecs name: related.ip - external: ecs name: rule.category +- external: ecs + name: rule.description - external: ecs name: rule.id - external: ecs name: rule.name -- external: ecs - name: rule.description - external: ecs name: rule.version - external: ecs name: source.address +- external: ecs + name: source.as.number +- external: ecs + name: source.as.organization.name - external: ecs name: source.bytes - external: ecs @@ -78,10 +109,8 @@ name: source.geo.country_iso_code - external: ecs name: source.geo.country_name -- description: Longitude and latitude. - level: core +- external: ecs name: source.geo.location - type: geo_point - external: ecs name: source.geo.region_iso_code - external: ecs @@ -94,25 +123,5 @@ name: source.packets - external: ecs name: source.port -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name - external: ecs name: tags -- external: ecs - name: observer.name -- external: ecs - name: observer.ip -- external: ecs - name: observer.vendor -- external: ecs - name: observer.type -- external: ecs - name: observer.product -- external: ecs - name: process.name -- external: ecs - name: process.pid -- external: ecs - name: log.file.path diff --git a/packages/snort/data_stream/log/sample_event.json b/packages/snort/data_stream/log/sample_event.json index d3cb5c9ad28..e2add115dab 100644 --- a/packages/snort/data_stream/log/sample_event.json +++ b/packages/snort/data_stream/log/sample_event.json @@ -1,11 +1,11 @@ { "@timestamp": "2022-09-05T16:02:55.000-05:00", "agent": { - "ephemeral_id": "d1ca036e-57c0-4c4a-9b92-ddc5f4cdb3a2", - "id": "584f3aea-648c-4e58-aba4-32b8f88d4396", + "ephemeral_id": "3ada3cc1-9563-4aa5-880e-585d87fc6adf", + "id": "ca0beb8d-9522-4450-8af7-3cb7f3d8c478", "name": "docker-fleet-agent", "type": "filebeat", - "version": "8.0.0-beta1" + "version": "8.2.0" }, "data_stream": { "dataset": "snort.log", @@ -29,12 +29,12 @@ "ip": "175.16.199.1" }, "ecs": { - "version": "8.2.0" + "version": "8.3.0" }, "elastic_agent": { - "id": "584f3aea-648c-4e58-aba4-32b8f88d4396", + "id": "ca0beb8d-9522-4450-8af7-3cb7f3d8c478", "snapshot": false, - "version": "8.0.0-beta1" + "version": "8.2.0" }, "event": { "agent_id_status": "verified", @@ -43,9 +43,9 @@ ], "created": "2022-09-05T16:02:55.000-05:00", "dataset": "snort.log", - "ingested": "2022-02-03T09:26:00Z", + "ingested": "2022-05-09T16:00:09Z", "kind": "alert", - "original": "Sep 5 16:02:55 dev snort: [1:1000015:0] Pinging... [Classification: Misc activity] [Priority: 3] {ICMP} 10.50.10.88 -\u003e 175.16.199.1\n", + "original": "Sep 5 16:02:55 dev snort: [1:1000015:0] Pinging... [Classification: Misc activity] [Priority: 3] {ICMP} 10.50.10.88 -\u003e 175.16.199.1", "severity": 3, "timezone": "-05:00" }, @@ -54,7 +54,7 @@ }, "log": { "source": { - "address": "172.19.0.7:38583" + "address": "172.18.0.4:54924" } }, "network": { diff --git a/packages/snort/docs/README.md b/packages/snort/docs/README.md index daa71c3d7d5..b72798fbb80 100644 --- a/packages/snort/docs/README.md +++ b/packages/snort/docs/README.md @@ -16,11 +16,11 @@ An example event for `log` looks as following: { "@timestamp": "2022-09-05T16:02:55.000-05:00", "agent": { - "ephemeral_id": "d1ca036e-57c0-4c4a-9b92-ddc5f4cdb3a2", - "id": "584f3aea-648c-4e58-aba4-32b8f88d4396", + "ephemeral_id": "3ada3cc1-9563-4aa5-880e-585d87fc6adf", + "id": "ca0beb8d-9522-4450-8af7-3cb7f3d8c478", "name": "docker-fleet-agent", "type": "filebeat", - "version": "8.0.0-beta1" + "version": "8.2.0" }, "data_stream": { "dataset": "snort.log", @@ -44,12 +44,12 @@ An example event for `log` looks as following: "ip": "175.16.199.1" }, "ecs": { - "version": "8.2.0" + "version": "8.3.0" }, "elastic_agent": { - "id": "584f3aea-648c-4e58-aba4-32b8f88d4396", + "id": "ca0beb8d-9522-4450-8af7-3cb7f3d8c478", "snapshot": false, - "version": "8.0.0-beta1" + "version": "8.2.0" }, "event": { "agent_id_status": "verified", @@ -58,9 +58,9 @@ An example event for `log` looks as following: ], "created": "2022-09-05T16:02:55.000-05:00", "dataset": "snort.log", - "ingested": "2022-02-03T09:26:00Z", + "ingested": "2022-05-09T16:00:09Z", "kind": "alert", - "original": "Sep 5 16:02:55 dev snort: [1:1000015:0] Pinging... [Classification: Misc activity] [Priority: 3] {ICMP} 10.50.10.88 -\u003e 175.16.199.1\n", + "original": "Sep 5 16:02:55 dev snort: [1:1000015:0] Pinging... [Classification: Misc activity] [Priority: 3] {ICMP} 10.50.10.88 -\u003e 175.16.199.1", "severity": 3, "timezone": "-05:00" }, @@ -69,7 +69,7 @@ An example event for `log` looks as following: }, "log": { "source": { - "address": "172.19.0.7:38583" + "address": "172.18.0.4:54924" } }, "network": { @@ -153,10 +153,16 @@ An example event for `log` looks as following: | destination.packets | Packets sent from the destination to the source. | long | | destination.port | Port of the destination. | long | | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | +| event.created | event.created contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from @timestamp in that @timestamp typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, @timestamp should be used. | date | | event.dataset | Event dataset | constant_keyword | +| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | +| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | | event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | | event.severity | The numeric severity of the event according to your event source. What the different severity values mean can be different between sources and use cases. It's up to the implementer to make sure severities are consistent across events from the same source. The Syslog severity belongs in `log.syslog.severity.code`. `event.severity` is meant to represent the severity according to the event source (e.g. firewall, IDS). If the event source does not publish its own severity, you may optionally copy the `log.syslog.severity.code` to `event.severity`. | long | +| event.timezone | This field should be populated when the event's timestamp does not include timezone information already (e.g. default Syslog timestamps). It's optional otherwise. Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00"). | keyword | +| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | diff --git a/packages/snort/manifest.yml b/packages/snort/manifest.yml index 7418522fa76..d1e61c02cfd 100644 --- a/packages/snort/manifest.yml +++ b/packages/snort/manifest.yml @@ -1,6 +1,6 @@ name: snort title: Snort -version: 0.3.0 +version: "0.3.1" release: experimental description: Collect logs from Snort with Elastic Agent. type: integration