Skip to content

[snort] Format {source,destination}.mac per ECS #3301

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
andrewkroh merged 10 commits into from
May 10, 2022
Merged

[snort] Format {source,destination}.mac per ECS #3301

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
7328c82
Sort ecs.yml
andrewkroh May 9, 2022
79b5661
Add missing event field mappings
andrewkroh May 9, 2022
6ed3d10
Fix lint error for @timestamp
andrewkroh May 9, 2022
7380a44
Remove duplicate host.ip definition
andrewkroh May 9, 2022
67bf900
Use ECS for {source,destination}.geo.location
andrewkroh May 9, 2022
d6c3923
Format source.mac, destination.mac as per ECS
andrewkroh May 9, 2022
421270f
Update elastic/stream
andrewkroh May 9, 2022
b2d7d55
Update sample event
andrewkroh May 9, 2022
0ca08c7
Update readme and generate pipeline test files
andrewkroh May 9, 2022
25c17f3
Update changelog
andrewkroh May 9, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion packages/snort/_dev/deploy/docker/docker-compose.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ services:
- ${SERVICE_LOGS_DIR}:/var/log
command: /bin/sh -c "cp /sample_logs/*.log /var/log/"
snort-log-udp:
image: akroh/stream:v0.3.0
image: docker.elastic.co/observability/stream:v0.7.0
volumes:
- ./sample_logs:/sample_logs:ro
command: log --start-signal=SIGHUP --delay=5s --addr elastic-agent:9514 -p=udp /sample_logs/test-syslog.log
5 changes: 5 additions & 0 deletions packages/snort/changelog.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
# newer versions go on top
- version: "0.3.1"
changes:
- description: Format source.mac and destination.mac as per ECS and add missing mappings for various event.* fields.
type: bugfix
link: https://github.com/elastic/integrations/pull/3301
- version: "0.3.0"
changes:
- description: Update to ECS 8.2
Expand Down
32 changes: 16 additions & 16 deletions packages/snort/data_stream/log/_dev/test/pipeline/test-log-csv.log-expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
"destination": {
"address": "10.100.10.190",
"ip": "10.100.10.190",
"mac": "00:50:56:9D:A5:BE",
"mac": "00-50-56-9D-A5-BE",
"port": 22
},
"ecs": {
Expand Down Expand Up @@ -66,7 +66,7 @@
"source": {
"address": "10.100.20.59",
"ip": "10.100.20.59",
"mac": "00:25:90:3A:05:13",
"mac": "00-25-90-3A-05-13",
"port": 57263
},
"tags": [
Expand All @@ -78,7 +78,7 @@
"destination": {
"address": "10.100.10.190",
"ip": "10.100.10.190",
"mac": "00:50:56:9D:A5:BE",
"mac": "00-50-56-9D-A5-BE",
"port": 22
},
"ecs": {
Expand Down Expand Up @@ -139,7 +139,7 @@
"source": {
"address": "10.100.20.59",
"ip": "10.100.20.59",
"mac": "00:25:90:3A:05:13",
"mac": "00-25-90-3A-05-13",
"port": 57263
},
"tags": [
Expand All @@ -151,7 +151,7 @@
"destination": {
"address": "10.100.10.190",
"ip": "10.100.10.190",
"mac": "00:50:56:9D:A5:BE",
"mac": "00-50-56-9D-A5-BE",
"port": 55475
},
"ecs": {
Expand Down Expand Up @@ -209,7 +209,7 @@
"source": {
"address": "10.100.10.1",
"ip": "10.100.10.1",
"mac": "00:25:90:3A:05:13",
"mac": "00-25-90-3A-05-13",
"port": 53
},
"tags": [
Expand All @@ -221,7 +221,7 @@
"destination": {
"address": "10.100.10.190",
"ip": "10.100.10.190",
"mac": "00:50:56:9D:A5:BE",
"mac": "00-50-56-9D-A5-BE",
"port": 55333
},
"ecs": {
Expand Down Expand Up @@ -279,7 +279,7 @@
"source": {
"address": "10.100.10.1",
"ip": "10.100.10.1",
"mac": "00:25:90:3A:05:13",
"mac": "00-25-90-3A-05-13",
"port": 53
},
"tags": [
Expand All @@ -291,7 +291,7 @@
"destination": {
"address": "10.100.10.255",
"ip": "10.100.10.255",
"mac": "FF:FF:FF:FF:FF:FF",
"mac": "FF-FF-FF-FF-FF-FF",
"port": 32414
},
"ecs": {
Expand Down Expand Up @@ -349,7 +349,7 @@
"source": {
"address": "10.100.10.75",
"ip": "10.100.10.75",
"mac": "00:0C:29:B8:43:CE",
"mac": "00-0C-29-B8-43-CE",
"port": 55776
},
"tags": [
Expand All @@ -373,7 +373,7 @@
"region_name": "Jilin Sheng"
},
"ip": "175.16.199.1",
"mac": "00:25:90:3A:05:13"
"mac": "00-25-90-3A-05-13"
},
"ecs": {
"version": "8.2.0"
Expand Down Expand Up @@ -436,7 +436,7 @@
"source": {
"address": "10.100.10.190",
"ip": "10.100.10.190",
"mac": "00:50:56:9D:A5:BE"
"mac": "00-50-56-9D-A5-BE"
},
"tags": [
"preserve_original_event"
Expand All @@ -447,7 +447,7 @@
"destination": {
"address": "10.100.10.190",
"ip": "10.100.10.190",
"mac": "00:50:56:9D:A5:BE"
"mac": "00-50-56-9D-A5-BE"
},
"ecs": {
"version": "8.2.0"
Expand Down Expand Up @@ -522,7 +522,7 @@
"region_name": "Jilin Sheng"
},
"ip": "175.16.199.1",
"mac": "00:25:90:3A:05:13"
"mac": "00-25-90-3A-05-13"
},
"tags": [
"preserve_original_event"
Expand All @@ -545,7 +545,7 @@
"region_name": "Jilin Sheng"
},
"ip": "175.16.199.1",
"mac": "00:25:90:3A:05:13"
"mac": "00-25-90-3A-05-13"
},
"ecs": {
"version": "8.2.0"
Expand Down Expand Up @@ -608,7 +608,7 @@
"source": {
"address": "10.100.10.190",
"ip": "10.100.10.190",
"mac": "00:50:56:9D:A5:BE"
"mac": "00-50-56-9D-A5-BE"
},
"tags": [
"preserve_original_event"
Expand Down
16 changes: 16 additions & 0 deletions packages/snort/data_stream/log/elasticsearch/ingest_pipeline/default.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,22 @@ processors:
target_field: destination.ip
type: ip
ignore_missing: true
- uppercase:
field: destination.mac
ignore_missing: true
- uppercase:
field: source.mac
ignore_missing: true
- gsub:
field: destination.mac
pattern: '[.:]'
replacement: '-'
ignore_missing: true
- gsub:
field: source.mac
pattern: '[.:]'
replacement: '-'
ignore_missing: true
- lowercase:
field: network.transport
ignore_missing: true
Expand Down
4 changes: 0 additions & 4 deletions packages/snort/data_stream/log/fields/agent.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,10 +107,6 @@
type: keyword
ignore_above: 1024
description: "Unique host id.\nAs hostname is not always unique, use values that are meaningful in your environment.\nExample: The current usage of `beat.name`."
- name: ip
level: core
type: ip
description: Host ip addresses.
- name: mac
level: core
type: keyword
Expand Down
67 changes: 38 additions & 29 deletions packages/snort/data_stream/log/fields/ecs.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
- external: ecs
name: '@timestamp'
type: date # TODO: Remove after https://github.com/elastic/elastic-package/issues/749 is fixed.
- external: ecs
name: destination.address
- external: ecs
Expand All @@ -18,10 +19,8 @@
name: destination.geo.country_iso_code
- external: ecs
name: destination.geo.country_name
- description: Longitude and latitude.
level: core
- external: ecs
name: destination.geo.location
type: geo_point
- external: ecs
name: destination.geo.region_iso_code
- external: ecs
Expand All @@ -36,38 +35,70 @@
name: destination.port
- external: ecs
name: ecs.version
- external: ecs
name: event.category
- external: ecs
name: event.created
- external: ecs
name: event.kind
- external: ecs
name: event.original
- external: ecs
name: event.outcome
- external: ecs
name: event.severity
- external: ecs
name: event.timezone
- external: ecs
name: event.type
- external: ecs
name: host.ip
- external: ecs
name: log.file.path
- external: ecs
name: message
- external: ecs
name: network.community_id
- external: ecs
name: network.direction
- external: ecs
name: network.protocol
- external: ecs
name: network.transport
- external: ecs
name: network.type
- external: ecs
name: network.direction
name: observer.ip
- external: ecs
name: observer.name
- external: ecs
name: observer.product
- external: ecs
name: observer.type
- external: ecs
name: observer.vendor
- external: ecs
name: process.name
- external: ecs
name: process.pid
- external: ecs
name: related.ip
- external: ecs
name: rule.category
- external: ecs
name: rule.description
- external: ecs
name: rule.id
- external: ecs
name: rule.name
- external: ecs
name: rule.description
- external: ecs
name: rule.version
- external: ecs
name: source.address
- external: ecs
name: source.as.number
- external: ecs
name: source.as.organization.name
- external: ecs
name: source.bytes
- external: ecs
Expand All @@ -78,10 +109,8 @@
name: source.geo.country_iso_code
- external: ecs
name: source.geo.country_name
- description: Longitude and latitude.
level: core
- external: ecs
name: source.geo.location
type: geo_point
- external: ecs
name: source.geo.region_iso_code
- external: ecs
Expand All @@ -94,25 +123,5 @@
name: source.packets
- external: ecs
name: source.port
- external: ecs
name: source.as.number
- external: ecs
name: source.as.organization.name
- external: ecs
name: tags
- external: ecs
name: observer.name
- external: ecs
name: observer.ip
- external: ecs
name: observer.vendor
- external: ecs
name: observer.type
- external: ecs
name: observer.product
- external: ecs
name: process.name
- external: ecs
name: process.pid
- external: ecs
name: log.file.path
18 changes: 9 additions & 9 deletions packages/snort/data_stream/log/sample_event.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
{
"@timestamp": "2022-09-05T16:02:55.000-05:00",
"agent": {
"ephemeral_id": "d1ca036e-57c0-4c4a-9b92-ddc5f4cdb3a2",
"id": "584f3aea-648c-4e58-aba4-32b8f88d4396",
"ephemeral_id": "3ada3cc1-9563-4aa5-880e-585d87fc6adf",
"id": "ca0beb8d-9522-4450-8af7-3cb7f3d8c478",
"name": "docker-fleet-agent",
"type": "filebeat",
"version": "8.0.0-beta1"
"version": "8.2.0"
},
"data_stream": {
"dataset": "snort.log",
Expand All @@ -29,12 +29,12 @@
"ip": "175.16.199.1"
},
"ecs": {
"version": "8.2.0"
"version": "8.3.0"
},
"elastic_agent": {
"id": "584f3aea-648c-4e58-aba4-32b8f88d4396",
"id": "ca0beb8d-9522-4450-8af7-3cb7f3d8c478",
"snapshot": false,
"version": "8.0.0-beta1"
"version": "8.2.0"
},
"event": {
"agent_id_status": "verified",
Expand All @@ -43,9 +43,9 @@
],
"created": "2022-09-05T16:02:55.000-05:00",
"dataset": "snort.log",
"ingested": "2022-02-03T09:26:00Z",
"ingested": "2022-05-09T16:00:09Z",
"kind": "alert",
"original": "Sep 5 16:02:55 dev snort: [1:1000015:0] Pinging... [Classification: Misc activity] [Priority: 3] {ICMP} 10.50.10.88 -\u003e 175.16.199.1\n",
"original": "Sep 5 16:02:55 dev snort: [1:1000015:0] Pinging... [Classification: Misc activity] [Priority: 3] {ICMP} 10.50.10.88 -\u003e 175.16.199.1",
"severity": 3,
"timezone": "-05:00"
},
Expand All @@ -54,7 +54,7 @@
},
"log": {
"source": {
"address": "172.19.0.7:38583"
"address": "172.18.0.4:54924"
}
},
"network": {
Expand Down
Loading