From 4bd97150f484d175246e27835f15836c4c291df2 Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Thu, 28 Jul 2022 00:19:45 +0000 Subject: [PATCH 1/8] Add SNort 3 JSON alert file compatibility --- packages/snort/changelog.yml | 5 + .../_dev/test/pipeline/test-common-config.yml | 2 - .../pipeline/test-log-csv.log-expected.json | 28 ++-- .../pipeline/test-log-fast.log-expected.json | 30 ++-- .../pipeline/test-log-full.log-config.yml | 2 - .../pipeline/test-log-full.log-expected.json | 21 ++- .../log/_dev/test/pipeline/test-log-json.log | 2 + .../pipeline/test-log-json.log-expected.json | 151 +++++++++++++++++ .../test-log-pfsense.log-expected.json | 9 +- .../test-log-syslog.log-expected.json | 9 +- .../elasticsearch/ingest_pipeline/default.yml | 81 ++++------ .../elasticsearch/ingest_pipeline/json.yml | 153 ++++++++++++++++++ .../ingest_pipeline/plaintext.yml | 53 ++++++ packages/snort/data_stream/log/fields/ecs.yml | 8 + packages/snort/manifest.yml | 2 +- 15 files changed, 465 insertions(+), 91 deletions(-) create mode 100644 packages/snort/data_stream/log/_dev/test/pipeline/test-log-json.log create mode 100644 packages/snort/data_stream/log/_dev/test/pipeline/test-log-json.log-expected.json create mode 100644 packages/snort/data_stream/log/elasticsearch/ingest_pipeline/json.yml create mode 100644 packages/snort/data_stream/log/elasticsearch/ingest_pipeline/plaintext.yml diff --git a/packages/snort/changelog.yml b/packages/snort/changelog.yml index 14839fe95f5..ef39c733fb1 100644 --- a/packages/snort/changelog.yml +++ b/packages/snort/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.1.0" + changes: + - description: Add Snort 3 JSON support. + type: enhancement + link: https://github.com/elastic/integrations/pull/1 - version: "1.0.0" changes: - description: Make GA diff --git a/packages/snort/data_stream/log/_dev/test/pipeline/test-common-config.yml b/packages/snort/data_stream/log/_dev/test/pipeline/test-common-config.yml index cddc2869dc5..b34d8ab8c20 100644 --- a/packages/snort/data_stream/log/_dev/test/pipeline/test-common-config.yml +++ b/packages/snort/data_stream/log/_dev/test/pipeline/test-common-config.yml @@ -1,6 +1,4 @@ dynamic_fields: - event.created: "^[0-9]{4}(-[0-9]{2}){2}T[0-9]{2}(:[0-9]{2}){2}\\.[0-9]{3}-[0-9]{2}:[0-9]{2}$" - event.ingested: ".*" "@timestamp": "^[0-9]{4}(-[0-9]{2}){2}T[0-9]{2}(:[0-9]{2}){2}\\.[0-9]{3}-[0-9]{2}:[0-9]{2}$" fields: "@timestamp": "2020-04-28T11:07:58.223Z" diff --git a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-csv.log-expected.json b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-csv.log-expected.json index 302f2aa5656..2dface04931 100644 --- a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-csv.log-expected.json +++ b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-csv.log-expected.json @@ -15,7 +15,7 @@ "category": [ "network" ], - "created": "2022-09-04T21:45:37.536-05:00", + "created": "2020-04-28T11:07:58.223Z", "kind": "alert", "original": "09/04-21:45:37.536335 ,1,1000006,0,\"TCP connection\",TCP,10.100.20.59,57263,10.100.10.190,22,00:25:90:3A:05:13,00:50:56:9D:A5:BE,0x72,***AP***,0x688F0DFC,0xBC763516,,0x80C,127,0,55665,100,102400,,,,", "timezone": "America/Chicago" @@ -23,6 +23,7 @@ "network": { "community_id": "1:mnJdrIaujYJdP8lXFem/hodYAt0=", "direction": "internal", + "iana_number": "6", "transport": "tcp", "type": "ipv4" }, @@ -58,7 +59,7 @@ }, "tcp": { "ack": 3161863446, - "flags": "AP", + "flags": "***AP***", "seq": 1754205692, "window": 2060 } @@ -88,7 +89,7 @@ "category": [ "network" ], - "created": "2022-09-04T21:45:37.553-05:00", + "created": "2020-04-28T11:07:58.223Z", "kind": "alert", "original": "09/04-21:45:37.553882 ,1,1000006,0,\"TCP connection\",TCP,10.100.20.59,57263,10.100.10.190,22,00:25:90:3A:05:13,00:50:56:9D:A5:BE,0x72,***AP***,0x688F0E38,0xBC763552,,0x80C,127,0,55666,100,102400,,,,", "timezone": "America/Chicago" @@ -96,6 +97,7 @@ "network": { "community_id": "1:mnJdrIaujYJdP8lXFem/hodYAt0=", "direction": "internal", + "iana_number": "6", "transport": "tcp", "type": "ipv4" }, @@ -131,7 +133,7 @@ }, "tcp": { "ack": 3161863506, - "flags": "AP", + "flags": "***AP***", "seq": 1754205752, "window": 2060 } @@ -161,7 +163,7 @@ "category": [ "network" ], - "created": "2022-09-04T21:50:40.017-05:00", + "created": "2020-04-28T11:07:58.223Z", "kind": "alert", "original": "09/04-21:50:40.017935 ,1,1000005,0,\"UDP Connection\",UDP,10.100.10.1,53,10.100.10.190,55475,00:25:90:3A:05:13,00:50:56:9D:A5:BE,0xC1,,,,,,64,0,56094,179,183296,,,,", "timezone": "America/Chicago" @@ -169,6 +171,7 @@ "network": { "community_id": "1:wvunc3EtDmKBjBft1PFlQ2pSLzw=", "direction": "internal", + "iana_number": "17", "transport": "udp", "type": "ipv4" }, @@ -231,7 +234,7 @@ "category": [ "network" ], - "created": "2022-09-04T21:50:39.947-05:00", + "created": "2020-04-28T11:07:58.223Z", "kind": "alert", "original": "09/04-21:50:39.947383 ,1,1000005,0,\"UDP Connection\",UDP,10.100.10.1,53,10.100.10.190,55333,00:25:90:3A:05:13,00:50:56:9D:A5:BE,0xB1,,,,,,64,0,26112,163,166912,,,,", "timezone": "America/Chicago" @@ -239,6 +242,7 @@ "network": { "community_id": "1:IcqpMEB/fJpNhZgyJVhx8VHROwY=", "direction": "internal", + "iana_number": "17", "transport": "udp", "type": "ipv4" }, @@ -301,7 +305,7 @@ "category": [ "network" ], - "created": "2022-09-04T21:50:40.666-05:00", + "created": "2020-04-28T11:07:58.223Z", "kind": "alert", "original": "09/04-21:50:40.666095 ,1,1000005,0,\"UDP Connection\",UDP,10.100.10.75,55776,10.100.10.255,32414,00:0C:29:B8:43:CE,FF:FF:FF:FF:FF:FF,0x3F,,,,,,64,0,37712,49,50176,,,,", "timezone": "America/Chicago" @@ -309,6 +313,7 @@ "network": { "community_id": "1:NW0wNEOLThLuO4EsoJXFbyp6zII=", "direction": "internal", + "iana_number": "17", "transport": "udp", "type": "ipv4" }, @@ -382,7 +387,7 @@ "category": [ "network" ], - "created": "2022-09-04T21:49:55.900-05:00", + "created": "2020-04-28T11:07:58.223Z", "kind": "alert", "original": "09/04-21:49:55.900215 ,1,1000004,0,\"Pinging...\",ICMP,10.100.10.190,,175.16.199.1,,00:50:56:9D:A5:BE,00:25:90:3A:05:13,0x62,,,,,,64,0,37607,84,86016,8,0,83,1", "timezone": "America/Chicago" @@ -390,6 +395,7 @@ "network": { "community_id": "1:jEaJsIOzda45Q8FjN0LeZEATihA=", "direction": "outbound", + "iana_number": "1", "transport": "icmp", "type": "ipv4" }, @@ -456,7 +462,7 @@ "category": [ "network" ], - "created": "2022-09-04T21:49:55.911-05:00", + "created": "2020-04-28T11:07:58.223Z", "kind": "alert", "original": "09/04-21:49:55.911592 ,1,1000004,0,\"Pinging...\",ICMP,175.16.199.1,,10.100.10.190,,00:25:90:3A:05:13,00:50:56:9D:A5:BE,0x62,,,,,,54,0,23522,84,86016,0,0,83,1", "timezone": "America/Chicago" @@ -464,6 +470,7 @@ "network": { "community_id": "1:PsO7nB0G1KDZ1IDLocfXBmcxiaA=", "direction": "inbound", + "iana_number": "1", "transport": "icmp", "type": "ipv4" }, @@ -554,7 +561,7 @@ "category": [ "network" ], - "created": "2022-09-04T21:49:56.900-05:00", + "created": "2020-04-28T11:07:58.223Z", "kind": "alert", "original": "09/04-21:49:56.900997 ,1,1000004,0,\"Pinging...\",ICMP,10.100.10.190,,175.16.199.1,,00:50:56:9D:A5:BE,00:25:90:3A:05:13,0x62,,,,,,64,0,37636,84,86016,8,0,83,2", "timezone": "America/Chicago" @@ -562,6 +569,7 @@ "network": { "community_id": "1:jEaJsIOzda45Q8FjN0LeZEATihA=", "direction": "outbound", + "iana_number": "1", "transport": "icmp", "type": "ipv4" }, diff --git a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-fast.log-expected.json b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-fast.log-expected.json index 0b3c5dab677..62fdf9decbb 100644 --- a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-fast.log-expected.json +++ b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-fast.log-expected.json @@ -14,7 +14,7 @@ "category": [ "network" ], - "created": "2022-05-30T19:09:10.917-05:00", + "created": "2020-04-28T11:07:58.223Z", "kind": "alert", "original": "05/30-19:09:10.917356 [**] [1:527:8] BAD-TRAFFIC same SRC/DST [**] [Classification: Potentially Bad Traffic] [Priority: 2] {UDP} 0.0.0.0:68 -\u003e 255.255.255.255:67", "severity": 2, @@ -23,6 +23,7 @@ "network": { "community_id": "1:t9O1j0qj71O4wJM7gnaHtgmfev8=", "direction": "external", + "iana_number": "17", "transport": "udp", "type": "ipv4" }, @@ -81,7 +82,7 @@ "category": [ "network" ], - "created": "2022-05-30T19:09:28.472-05:00", + "created": "2020-04-28T11:07:58.223Z", "kind": "alert", "original": "05/30-19:09:28.472094 [**] [1:2012811:2] ET DNS DNS Query to a .tk domain - Likely Hostile [**] [Classification: Potentially Bad Traffic] [Priority: 2] {UDP} 192.168.88.10:1029 -\u003e 175.16.199.1:53", "severity": 2, @@ -90,6 +91,7 @@ "network": { "community_id": "1:RZ4iVwBzp5juqzQJiu5WebaF9J4=", "direction": "outbound", + "iana_number": "17", "transport": "udp", "type": "ipv4" }, @@ -147,7 +149,7 @@ "category": [ "network" ], - "created": "2022-05-30T19:09:10.917-05:00", + "created": "2020-04-28T11:07:58.223Z", "kind": "alert", "original": "05/30-19:09:10.917356 [**] [1:477:3] ICMP Packet [**] [Priority: 0] {ICMP} 175.16.199.1 -\u003e 175.16.199.1", "severity": 0, @@ -156,6 +158,7 @@ "network": { "community_id": "1:ae//KI+huidgn9Nxeaibd8SUiVA=", "direction": "external", + "iana_number": "1", "transport": "icmp", "type": "ipv4" }, @@ -223,7 +226,7 @@ "category": [ "network" ], - "created": "2022-12-30T14:09:21.116-06:00", + "created": "2020-04-28T11:07:58.223Z", "kind": "alert", "original": "12/30-14:09:21.116402 [**] [1:1917:6] SCAN UPnP service discover attempt [**] [Classification: Detection of a Network Scan] [Priority: 3] {TCP} 192.168.15.10:1035 -\u003e 175.16.199.1:1900", "severity": 3, @@ -232,6 +235,7 @@ "network": { "community_id": "1:lTRw3g8ZdxItqss80+SSa07uVWc=", "direction": "outbound", + "iana_number": "6", "transport": "tcp", "type": "ipv4" }, @@ -278,7 +282,7 @@ "category": [ "network" ], - "created": "2022-01-21T02:23:42.327-06:00", + "created": "2020-04-28T11:07:58.223Z", "kind": "alert", "original": "01/21-02:23:42.327730 [**] [1:2014520:2] ET INFO EXE - Served Attached HTTP [**] [Classification: Misc activity] [Priority: 3] {TCP} 175.16.199.1:80 -\u003e 192.168.115.10:1051", "severity": 3, @@ -287,6 +291,7 @@ "network": { "community_id": "1:qSaSgRpopkbN/a7ST5y66ztJl8U=", "direction": "inbound", + "iana_number": "6", "transport": "tcp", "type": "ipv4" }, @@ -344,7 +349,7 @@ "category": [ "network" ], - "created": "2022-01-21T02:23:42.208-06:00", + "created": "2020-04-28T11:07:58.223Z", "kind": "alert", "original": "01/21-02:23:42.208605 [**] [1:408:5] ICMP Echo Reply [**] [Classification: Misc activity] [Priority: 3] {ICMP} 175.16.199.1 -\u003e 192.168.115.10", "severity": 3, @@ -353,6 +358,7 @@ "network": { "community_id": "1:EtB/zlC1JmfdF0An9MzN1EDqn7o=", "direction": "inbound", + "iana_number": "1", "transport": "icmp", "type": "ipv4" }, @@ -410,7 +416,7 @@ "category": [ "network" ], - "created": "2022-09-04T21:55:02.041-05:00", + "created": "2020-04-28T11:07:58.223Z", "kind": "alert", "original": "09/04-21:55:02.041364 [**] [1:1000005:0] UDP Connection [**] [Classification: A Network Trojan was Detected] [Priority: 1] {UDP} 175.16.199.1:53 -\u003e 10.100.10.190:54757", "severity": 1, @@ -419,6 +425,7 @@ "network": { "community_id": "1:Rj/XwIFirLCUpBLJSDip5ZzpVZY=", "direction": "inbound", + "iana_number": "17", "transport": "udp", "type": "ipv4" }, @@ -477,7 +484,7 @@ "category": [ "network" ], - "created": "2022-09-04T21:55:02.118-05:00", + "created": "2020-04-28T11:07:58.223Z", "kind": "alert", "original": "09/04-21:55:02.118427 [**] [1:1000005:0] UDP Connection [**] [Classification: A Network Trojan was Detected] [Priority: 1] {UDP} 175.16.199.1:53 -\u003e 10.100.10.190:36312", "severity": 1, @@ -486,6 +493,7 @@ "network": { "community_id": "1:lFRQEVyjqFCLDyAOzC3sRuoFLkI=", "direction": "inbound", + "iana_number": "17", "transport": "udp", "type": "ipv4" }, @@ -555,7 +563,7 @@ "category": [ "network" ], - "created": "2022-09-04T21:54:43.216-05:00", + "created": "2020-04-28T11:07:58.223Z", "kind": "alert", "original": "09/04-21:54:43.216486 [**] [1:1000004:0] Pinging... [**] [Classification: Attempted Information Leak] [Priority: 2] {ICMP} 10.100.10.190 -\u003e 175.16.199.1", "severity": 2, @@ -564,6 +572,7 @@ "network": { "community_id": "1:jEaJsIOzda45Q8FjN0LeZEATihA=", "direction": "outbound", + "iana_number": "1", "transport": "icmp", "type": "ipv4" }, @@ -608,7 +617,7 @@ "category": [ "network" ], - "created": "2022-09-04T21:54:43.227-05:00", + "created": "2020-04-28T11:07:58.223Z", "kind": "alert", "original": "09/04-21:54:43.227117 [**] [1:1000004:0] Pinging... [**] [Classification: Attempted Information Leak] [Priority: 2] {ICMP} 175.16.199.1 -\u003e 10.100.10.190", "severity": 2, @@ -617,6 +626,7 @@ "network": { "community_id": "1:PsO7nB0G1KDZ1IDLocfXBmcxiaA=", "direction": "inbound", + "iana_number": "1", "transport": "icmp", "type": "ipv4" }, diff --git a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-full.log-config.yml b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-full.log-config.yml index e93d9f49eb5..d6f314c62a7 100644 --- a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-full.log-config.yml +++ b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-full.log-config.yml @@ -1,6 +1,4 @@ dynamic_fields: - event.created: "^[0-9]{4}(-[0-9]{2}){2}T[0-9]{2}(:[0-9]{2}){2}\\.[0-9]{3}-[0-9]{2}:[0-9]{2}$" - event.ingested: ".*" "@timestamp": "^[0-9]{4}(-[0-9]{2}){2}T[0-9]{2}(:[0-9]{2}){2}\\.[0-9]{3}-[0-9]{2}:[0-9]{2}$" fields: "@timestamp": "2020-04-28T11:07:58.223Z" diff --git a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-full.log-expected.json b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-full.log-expected.json index 1998e37074d..6fed563769f 100644 --- a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-full.log-expected.json +++ b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-full.log-expected.json @@ -14,7 +14,7 @@ "category": [ "network" ], - "created": "2022-09-04T21:42:42.860-05:00", + "created": "2020-04-28T11:07:58.223Z", "kind": "alert", "original": "[**] [1:1000006:0] TCP connection [**]\n[Priority: 0] \n09/04-21:42:42.860730 10.100.20.59:57263 -\u003e 10.100.10.190:22\nTCP TTL:127 TOS:0x0 ID:53730 IpLen:20 DgmLen:108 DF\n***AP*** Seq: 0x688E00E4 Ack: 0xBC730BB6 Win: 0x80B TcpLen: 20\n", "severity": 0, @@ -23,6 +23,7 @@ "network": { "community_id": "1:mnJdrIaujYJdP8lXFem/hodYAt0=", "direction": "internal", + "iana_number": "6", "transport": "tcp", "type": "ipv4" }, @@ -78,7 +79,7 @@ "category": [ "network" ], - "created": "2022-09-04T21:42:42.903-05:00", + "created": "2020-04-28T11:07:58.223Z", "kind": "alert", "original": "[**] [1:1000006:0] TCP connection [**]\n[Priority: 0] \n09/04-21:42:42.903092 10.100.20.59:57263 -\u003e 10.100.10.190:22\nTCP TTL:127 TOS:0x0 ID:53731 IpLen:20 DgmLen:40 DF\n***A**** Seq: 0x688E0128 Ack: 0xBC730C02 Win: 0x80B TcpLen: 20\n", "severity": 0, @@ -87,6 +88,7 @@ "network": { "community_id": "1:mnJdrIaujYJdP8lXFem/hodYAt0=", "direction": "internal", + "iana_number": "6", "transport": "tcp", "type": "ipv4" }, @@ -142,7 +144,7 @@ "category": [ "network" ], - "created": "2022-09-04T21:53:15.299-05:00", + "created": "2020-04-28T11:07:58.223Z", "kind": "alert", "original": "[**] [1:1000005:0] UDP Connection [**]\n[Classification: A Network Trojan was Detected] [Priority: 1] \n09/04-21:53:15.299702 10.100.10.1:53 -\u003e 10.100.10.190:36635\nUDP TTL:64 TOS:0x0 ID:58363 IpLen:20 DgmLen:83\nLen: 55\n", "severity": 1, @@ -151,6 +153,7 @@ "network": { "community_id": "1:M7q1/qKDOLyHIWtG7LwCmcINfXQ=", "direction": "internal", + "iana_number": "17", "transport": "udp", "type": "ipv4" }, @@ -220,7 +223,7 @@ "category": [ "network" ], - "created": "2022-09-04T21:53:15.299-05:00", + "created": "2020-04-28T11:07:58.223Z", "kind": "alert", "original": "[**] [1:1000004:0] Pinging... [**]\n[Classification: Attempted Information Leak] [Priority: 2] \n09/04-21:53:15.299988 10.100.10.190 -\u003e 175.16.199.1\nICMP TTL:64 TOS:0x0 ID:6922 IpLen:20 DgmLen:84 DF\nType:8 Code:0 ID:101 Seq:1 ECHO\n", "severity": 2, @@ -229,6 +232,7 @@ "network": { "community_id": "1:jEaJsIOzda45Q8FjN0LeZEATihA=", "direction": "outbound", + "iana_number": "1", "transport": "icmp", "type": "ipv4" }, @@ -290,7 +294,7 @@ "category": [ "network" ], - "created": "2022-09-04T21:53:15.301-05:00", + "created": "2020-04-28T11:07:58.223Z", "kind": "alert", "original": "[**] [1:1000006:0] TCP connection [**]\n[Classification: Potentially Bad Traffic] [Priority: 2] \n09/04-21:53:15.301504 10.100.20.59:57263 -\u003e 10.100.10.190:22\nTCP TTL:127 TOS:0x0 ID:61472 IpLen:20 DgmLen:40 DF\n***A**** Seq: 0x68940D74 Ack: 0xBC811F16 Win: 0x80E TcpLen: 20\n", "severity": 2, @@ -299,6 +303,7 @@ "network": { "community_id": "1:mnJdrIaujYJdP8lXFem/hodYAt0=", "direction": "internal", + "iana_number": "6", "transport": "tcp", "type": "ipv4" }, @@ -354,7 +359,7 @@ "category": [ "network" ], - "created": "2022-09-04T21:53:15.309-05:00", + "created": "2020-04-28T11:07:58.223Z", "kind": "alert", "original": "[**] [1:1000004:0] Pinging... [**]\n[Classification: Attempted Information Leak] [Priority: 2] \n09/04-21:53:15.309468 175.16.199.1 -\u003e 10.100.10.190\nICMP TTL:114 TOS:0x0 ID:0 IpLen:20 DgmLen:84\nType:0 Code:0 ID:101 Seq:1 ECHO REPLY\n", "severity": 2, @@ -363,6 +368,7 @@ "network": { "community_id": "1:PsO7nB0G1KDZ1IDLocfXBmcxiaA=", "direction": "inbound", + "iana_number": "1", "transport": "icmp", "type": "ipv4" }, @@ -435,7 +441,7 @@ "category": [ "network" ], - "created": "2022-09-04T21:53:15.358-05:00", + "created": "2020-04-28T11:07:58.223Z", "kind": "alert", "original": "[**] [1:1000005:0] UDP Connection [**]\n[Classification: A Network Trojan was Detected] [Priority: 1] \n09/04-21:53:15.358155 10.100.10.1:53 -\u003e 10.100.10.190:56012\nUDP TTL:64 TOS:0x0 ID:33955 IpLen:20 DgmLen:153\nLen: 125", "severity": 1, @@ -444,6 +450,7 @@ "network": { "community_id": "1:+L8vYWrVdJH2UDDD4Z31DIDLk6E=", "direction": "internal", + "iana_number": "17", "transport": "udp", "type": "ipv4" }, diff --git a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-json.log b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-json.log new file mode 100644 index 00000000000..1ac918b86c0 --- /dev/null +++ b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-json.log @@ -0,0 +1,2 @@ +{ "seconds" : 1608147213, "action" : "allow", "class" : "none", "b64_data" : "DWHaXwAAAADO0wgAAAAAABAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc=", "dir" : "S2C", "dst_addr" : "10.10.10.1", "dst_ap" : "10.10.10.1:0", "eth_dst" : "52:54:00:1F:8A:1C", "eth_len" :98, "eth_src" : "52:54:00:70:78:9F", "eth_type" : "0x800", "gid" : 1, "icmp_code" : 0, "icmp_id" :5203, "icmp_seq" : 3, "icmp_type" : 0, "iface" : "ens3", "ip_id" : 3006, "ip_len" : 64, "msg" : "ICMP Traffic Detected", "mpls" : 0, "pkt_gen" : "raw", "pkt_len" : 84, "pkt_num" : 8, "priority" :0, "proto" : "ICMP", "rev" : 0, "rule" : "1:10000001:0", "service" : "unknown", "sid" : 10000001, "src_addr" : "10.10.10.88", "src_ap" : "10.10.10.88:0", "tos" : 0, "ttl" : 64, "vlan" : 0, "timestamp" : "12/16-20:33:33.603502" } +{ "seconds" : 1574352110, "action" : "allow", "class" : "Attempted Administrator Privilege Gain", "b64_data" : "AAAQPP9TTUIvAAAAABgFSAAAyIAtOUGjAkUAAAII6AYACGILDP8AAAACQABAAwD/AAAABAAAEAAAABA8AAEQAMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzIuNTP7//+kvw/z/i41M/v//g8EE6ZbB/P+LVCQIjUIMi4pU/v//M8jobFb9/4tK/DPI6GJW/f+4/JpIAOmqUv3/zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzItN8OnSwvz/i03wg8EE6TzB/P+LVCQIjUIMi0r4M8joFVb9/7hgm0gA6V1S/f/MzMzMzMzMzMzMzMzMzMzMzMyLRbRQ6CdX/f9Zw4tUJAiNQgyLSrQzyOjdVf3/i0rwM8jo01X9/7iUm0gA6RtS/f/MzMzMzMzMzMzMzMzMzMzMi41M/v//g8EE6UzC/P+LjUz+//+DwQjpMcD8/4tUJAiNQgyLilT+//8zyOiJVf3/i0r8M8jof1X9/7jMm0gA6cdR/f/MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMi41A/v//g8EE6ezB/P+LjUD+//+DwQjp0b/8/4tUJAiNQgyLikj+//8zyOgpVf3/i0r8M8joH1X9/7gInEgA6WdR/f/MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMi03wg8EE6Y/B/P+LTfCDwQjpd7/8/4tUJAiNQgyLSvgzyOjSVP3/uEScSADpGlH9/8zMzMzMzMzMzMzMzMzMzItF7FDo51X9/1nDi0XkUOjcVf3/WcOLVCQIjUIMi0rgM8joklT9/7iAnEgA6dpQ/f/MzMzMzMzMzMzMzMzMzMyLTeyDwQjpD8H8/4tUJAiNQgyLSvQzyOhdVP3/uLScSADppVD9/8zMzMzMzMzMzMyLRZxQ6HdV/f9Zw4tUJAiNQgyLSogzyOgtVP3/i0r8M8joI1T9/7jknEgA6WtQ/f/MzMzMzMzMzMzMzMzMzMzMjU3k6YW//P+NTeTpw7/8/4tUJAiNQgyLirD+//8zyOjlU/3/uFCdSADpLVD9/8zMzMzMzMzMzMzMzMzMzMzMzI1NsOnCv/z/jU3M6XZe/f+NjWz////pr7/8/41NiOljXv3/jY0o////6Zy//P+LVCQIjUIMi4qw/v//M8joh1P9/7iUnUgA6c9P/f/MzMzMzMzMzMzMzMzMzMzMzMzMzItNhOkeXv3/i1QkCI1CDItKjDPI6FBT/f+4ZJ5IAOmYT/3/zMzMzMzMzMzMzMzMzItNhOnuXf3/i1QkCI1CDItKjDPI6CBT/f+4lJ5IAOloT/3/zMzMzMzMzMzMzMzMzIuN6P7//+nIvvz/i1QkCI1CDIuK8P7//zPI6OpS/f+4xJ5IAOkyT/3/zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMyLTejpi778/4tUJAiNQgyLSvAzyOiwUv3/uPSeSADp+E79/8zMzMzMzMzMzMzMzMyLVCQIjUIMi4rM/v//M8johVL9/7hQn0gA6c1O/f/MzMzMzMzMzMzMzMzMzMzMzMyLVCQIjUIMi4pk////M8joVVL9/7i8n0gA6Z1O/f/MzMzMzMzMzMzMzMzMzMzMzMyNTdzptb38/41N3Onzvfz/i1QkCI1CDIuKYP7//zPI6BVS/f+4XKBIAOldTv3/zMzMzMzMzMzMzMzMzMzMzMzMjU3Y6fK9/P+LRdSD4AEPhAwAAACDZdT+i00I6dq9/P/Di1QkCI1CDIuKXP///zPI6MRR/f+4nKBIAOkMTv3/zMzMzMzMzMzMzMzMzMzMi1QkCI1CDItK7DPI6JpR/f+4kKJIAOniTf3/i03wg8EE6VM+//+LVCQIjUIMi0rsM8jodFH9/7gEqkgA6bxN/f+LTfDpDEH//4tN8IPBBOlYvfz/i03wg8Eg6U29/P+LTfCDwTzpQr38/4tN8IPBWOk3vfz/i1QkCI1CDItK7DPI6CVR/f+4KKpIAOltTf3/i03w6b1A//+LTfCDwQTpCb38/4tN8IPBIOn+vPz/i03wg8E86fO8/P+LTfCDwVjp6Lz8/4tUJAiNQgyLSuwzyOjWUP3/uHSqSADpHk39/41N1OnFvPz/i1QkCI1CDItKmDPI6LNQ/f+LSvwzyOipUP3/uMiqSADp8Uz9/41N1OmYvPz/i1QkCI1CDItKqDPI6IZQ/f+LSvwzyOh8UP3/uPSqSADpxEz9//91COihUf3/WcOLRfCD4AEPhAwAAACDZfD+jU3U6VG8/P/Di1QkCI1CDIuKXP///zPI6DtQ/f+4MKtIAOmDTP3/jU3U6Sq8/P+LVCQIjUIMi0qcM8joGFD9/4tK/DPI6A5Q/f+4XKtIAOlWTP3/jU3s6aY///+LVCQIjUIMi0rcM8jo60/9/7iYq0gA6TNM/f//dQjoEFH9/1nDi0Xwg+ABD4QMAAAAg2Xw/o1N1OnAu/z/w4tUJAiNQgyLilz///8zyOiqT/3/uPyrSADp8kv9/4tN8OnWJP//i1QkCI1CDItK7DPI6IdP/f+4KKxIAOnPS/3/jU3w6XQf//+LVCQIjUIMi0rsM8joZE/9/7hUrEgA6axL/f+NTezp/D7//4tUJAiNQgyLStwzyOhBT/3/uICsSADpiUv9/4tF8IPgAQ+EDwAAAINl8P6LTeyDwQTpfiD//8OLVCQIjUIMi0roM8joCk/9/7isrEgA6VJL/f+LTfDptVn9/4tUJAiNQgyLSuwzyOjnTv3/uNisSADpL0v9/41N2OnWuvz/i1QkCI1CDItKrDPI6MRO/f+4FK1IAOkMS/3/jU3Y6bO6/P+LVCQIjUIMi0qsM8jooU79/7iYrUgA6elK/f+NTdjpkLr8/4tUJAiNQgyLSqwzyOh+Tv3/uACuSADpxkr9/4tN8OkpWf3/i1QkCI1CDItK7DPI6FtO/f+4WK5IAOmjSv3/jU3w6fM9//+LVCQIjUIMi0rsM8joOE79/7iErkgA6YBK/f+LTfDp0B3//4tUJAiNQgyLSuwzyOgVTv3/uLCuSADpXUr9/4tN8OmtHf//i03wg8EY6fm5/P+LVCQIjUIMi0rsM8jo5039/7jkrkgA6S9K/f+NTfDpfz3///917OgET/3/WcOLVCQIjUIMi0roM8jouk39/7gYr0gA6QJK/f//dfDo3079/1nDi1QkCI1CDItK7DPI6JVN/f+4RK9IAOndSf3/zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMw=", "dir" : "C2S", "dst_addr" : "10.11.21.11", "dst_ap" : "10.11.21.11:445", "dst_port" : 445, "gid" : 1, "iface" : "/home/noah/pcaps/pcaps from malware-traffica-analysis.net/2019-11-21-Emotet-epoch-3-with-Trickbot-gtag-mor49-and-spambot-traffic.pcap", "msg" : "OS-WINDOWS Microsoft Windows raw WriteAndX InData pointer adjustment attempt", "mpls" : 0, "pkt_gen" : "stream_tcp", "pkt_len" : 4160, "pkt_num" : 61571, "priority" : 1, "proto" : "TCP", "rev" : 1, "rule" : "1:50626:1", "service" : "netbios-ssn", "sid" : 50626, "src_addr" : "10.11.21.101", "src_ap" : "10.11.21.101:50084", "src_port" : 50084, "vlan" : 0, "timestamp" : "11/21-18:01:50.061909" } \ No newline at end of file diff --git a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-json.log-expected.json b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-json.log-expected.json new file mode 100644 index 00000000000..ad975c09c06 --- /dev/null +++ b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-json.log-expected.json @@ -0,0 +1,151 @@ +{ + "expected": [ + { + "@timestamp": "2022-12-16T20:33:33.603-06:00", + "destination": { + "address": "10.10.10.1", + "ip": "10.10.10.1", + "mac": "52-54-00-1F-8A-1C" + }, + "ecs": { + "version": "8.3.0" + }, + "event": { + "category": [ + "network" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "alert", + "original": "{ \"seconds\" : 1608147213, \"action\" : \"allow\", \"class\" : \"none\", \"b64_data\" : \"DWHaXwAAAADO0wgAAAAAABAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc=\", \"dir\" : \"S2C\", \"dst_addr\" : \"10.10.10.1\", \"dst_ap\" : \"10.10.10.1:0\", \"eth_dst\" : \"52:54:00:1F:8A:1C\", \"eth_len\" :98, \"eth_src\" : \"52:54:00:70:78:9F\", \"eth_type\" : \"0x800\", \"gid\" : 1, \"icmp_code\" : 0, \"icmp_id\" :5203, \"icmp_seq\" : 3, \"icmp_type\" : 0, \"iface\" : \"ens3\", \"ip_id\" : 3006, \"ip_len\" : 64, \"msg\" : \"ICMP Traffic Detected\", \"mpls\" : 0, \"pkt_gen\" : \"raw\", \"pkt_len\" : 84, \"pkt_num\" : 8, \"priority\" :0, \"proto\" : \"ICMP\", \"rev\" : 0, \"rule\" : \"1:10000001:0\", \"service\" : \"unknown\", \"sid\" : 10000001, \"src_addr\" : \"10.10.10.88\", \"src_ap\" : \"10.10.10.88:0\", \"tos\" : 0, \"ttl\" : 64, \"vlan\" : 0, \"timestamp\" : \"12/16-20:33:33.603502\" }", + "severity": 0, + "timezone": "America/Chicago", + "type": [ + "allowed" + ] + }, + "network": { + "bytes": 84, + "community_id": "1:NOMQYgbhDm3hmIIfQYchm6UBEaY=", + "direction": "internal", + "iana_number": "1", + "packets": 8, + "transport": "icmp", + "type": "ipv4" + }, + "observer": { + "ingress": { + "interface": { + "name": "ens3" + } + }, + "product": "ids", + "type": "ids", + "vendor": "snort" + }, + "related": { + "ip": [ + "10.10.10.88", + "10.10.10.1" + ] + }, + "rule": { + "category": "none", + "description": "ICMP Traffic Detected", + "id": "10000001", + "version": "0" + }, + "snort": { + "eth": { + "length": 98 + }, + "gid": 1, + "icmp": { + "code": 0, + "id": 5203, + "seq": 3, + "type": 0 + }, + "ip": { + "id": 3006, + "tos": 0, + "ttl": 64 + } + }, + "source": { + "address": "10.10.10.88", + "ip": "10.10.10.88", + "mac": "52-54-00-70-78-9F" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2022-11-21T18:01:50.061-06:00", + "destination": { + "address": "10.11.21.11", + "ip": "10.11.21.11", + "port": 445 + }, + "ecs": { + "version": "8.3.0" + }, + "event": { + "category": [ + "network" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "alert", + "original": "{ \"seconds\" : 1574352110, \"action\" : \"allow\", \"class\" : \"Attempted Administrator Privilege Gain\", \"b64_data\" : \"AAAQPP9TTUIvAAAAABgFSAAAyIAtOUGjAkUAAAII6AYACGILDP8AAAACQABAAwD/AAAABAAAEAAAABA8AAEQAMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzIuNTP7//+kvw/z/i41M/v//g8EE6ZbB/P+LVCQIjUIMi4pU/v//M8jobFb9/4tK/DPI6GJW/f+4/JpIAOmqUv3/zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzItN8OnSwvz/i03wg8EE6TzB/P+LVCQIjUIMi0r4M8joFVb9/7hgm0gA6V1S/f/MzMzMzMzMzMzMzMzMzMzMzMyLRbRQ6CdX/f9Zw4tUJAiNQgyLSrQzyOjdVf3/i0rwM8jo01X9/7iUm0gA6RtS/f/MzMzMzMzMzMzMzMzMzMzMi41M/v//g8EE6UzC/P+LjUz+//+DwQjpMcD8/4tUJAiNQgyLilT+//8zyOiJVf3/i0r8M8jof1X9/7jMm0gA6cdR/f/MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMi41A/v//g8EE6ezB/P+LjUD+//+DwQjp0b/8/4tUJAiNQgyLikj+//8zyOgpVf3/i0r8M8joH1X9/7gInEgA6WdR/f/MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMi03wg8EE6Y/B/P+LTfCDwQjpd7/8/4tUJAiNQgyLSvgzyOjSVP3/uEScSADpGlH9/8zMzMzMzMzMzMzMzMzMzItF7FDo51X9/1nDi0XkUOjcVf3/WcOLVCQIjUIMi0rgM8joklT9/7iAnEgA6dpQ/f/MzMzMzMzMzMzMzMzMzMyLTeyDwQjpD8H8/4tUJAiNQgyLSvQzyOhdVP3/uLScSADppVD9/8zMzMzMzMzMzMyLRZxQ6HdV/f9Zw4tUJAiNQgyLSogzyOgtVP3/i0r8M8joI1T9/7jknEgA6WtQ/f/MzMzMzMzMzMzMzMzMzMzMjU3k6YW//P+NTeTpw7/8/4tUJAiNQgyLirD+//8zyOjlU/3/uFCdSADpLVD9/8zMzMzMzMzMzMzMzMzMzMzMzI1NsOnCv/z/jU3M6XZe/f+NjWz////pr7/8/41NiOljXv3/jY0o////6Zy//P+LVCQIjUIMi4qw/v//M8joh1P9/7iUnUgA6c9P/f/MzMzMzMzMzMzMzMzMzMzMzMzMzItNhOkeXv3/i1QkCI1CDItKjDPI6FBT/f+4ZJ5IAOmYT/3/zMzMzMzMzMzMzMzMzItNhOnuXf3/i1QkCI1CDItKjDPI6CBT/f+4lJ5IAOloT/3/zMzMzMzMzMzMzMzMzIuN6P7//+nIvvz/i1QkCI1CDIuK8P7//zPI6OpS/f+4xJ5IAOkyT/3/zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMyLTejpi778/4tUJAiNQgyLSvAzyOiwUv3/uPSeSADp+E79/8zMzMzMzMzMzMzMzMyLVCQIjUIMi4rM/v//M8johVL9/7hQn0gA6c1O/f/MzMzMzMzMzMzMzMzMzMzMzMyLVCQIjUIMi4pk////M8joVVL9/7i8n0gA6Z1O/f/MzMzMzMzMzMzMzMzMzMzMzMyNTdzptb38/41N3Onzvfz/i1QkCI1CDIuKYP7//zPI6BVS/f+4XKBIAOldTv3/zMzMzMzMzMzMzMzMzMzMzMzMjU3Y6fK9/P+LRdSD4AEPhAwAAACDZdT+i00I6dq9/P/Di1QkCI1CDIuKXP///zPI6MRR/f+4nKBIAOkMTv3/zMzMzMzMzMzMzMzMzMzMi1QkCI1CDItK7DPI6JpR/f+4kKJIAOniTf3/i03wg8EE6VM+//+LVCQIjUIMi0rsM8jodFH9/7gEqkgA6bxN/f+LTfDpDEH//4tN8IPBBOlYvfz/i03wg8Eg6U29/P+LTfCDwTzpQr38/4tN8IPBWOk3vfz/i1QkCI1CDItK7DPI6CVR/f+4KKpIAOltTf3/i03w6b1A//+LTfCDwQTpCb38/4tN8IPBIOn+vPz/i03wg8E86fO8/P+LTfCDwVjp6Lz8/4tUJAiNQgyLSuwzyOjWUP3/uHSqSADpHk39/41N1OnFvPz/i1QkCI1CDItKmDPI6LNQ/f+LSvwzyOipUP3/uMiqSADp8Uz9/41N1OmYvPz/i1QkCI1CDItKqDPI6IZQ/f+LSvwzyOh8UP3/uPSqSADpxEz9//91COihUf3/WcOLRfCD4AEPhAwAAACDZfD+jU3U6VG8/P/Di1QkCI1CDIuKXP///zPI6DtQ/f+4MKtIAOmDTP3/jU3U6Sq8/P+LVCQIjUIMi0qcM8joGFD9/4tK/DPI6A5Q/f+4XKtIAOlWTP3/jU3s6aY///+LVCQIjUIMi0rcM8jo60/9/7iYq0gA6TNM/f//dQjoEFH9/1nDi0Xwg+ABD4QMAAAAg2Xw/o1N1OnAu/z/w4tUJAiNQgyLilz///8zyOiqT/3/uPyrSADp8kv9/4tN8OnWJP//i1QkCI1CDItK7DPI6IdP/f+4KKxIAOnPS/3/jU3w6XQf//+LVCQIjUIMi0rsM8joZE/9/7hUrEgA6axL/f+NTezp/D7//4tUJAiNQgyLStwzyOhBT/3/uICsSADpiUv9/4tF8IPgAQ+EDwAAAINl8P6LTeyDwQTpfiD//8OLVCQIjUIMi0roM8joCk/9/7isrEgA6VJL/f+LTfDptVn9/4tUJAiNQgyLSuwzyOjnTv3/uNisSADpL0v9/41N2OnWuvz/i1QkCI1CDItKrDPI6MRO/f+4FK1IAOkMS/3/jU3Y6bO6/P+LVCQIjUIMi0qsM8jooU79/7iYrUgA6elK/f+NTdjpkLr8/4tUJAiNQgyLSqwzyOh+Tv3/uACuSADpxkr9/4tN8OkpWf3/i1QkCI1CDItK7DPI6FtO/f+4WK5IAOmjSv3/jU3w6fM9//+LVCQIjUIMi0rsM8joOE79/7iErkgA6YBK/f+LTfDp0B3//4tUJAiNQgyLSuwzyOgVTv3/uLCuSADpXUr9/4tN8OmtHf//i03wg8EY6fm5/P+LVCQIjUIMi0rsM8jo5039/7jkrkgA6S9K/f+NTfDpfz3///917OgET/3/WcOLVCQIjUIMi0roM8jouk39/7gYr0gA6QJK/f//dfDo3079/1nDi1QkCI1CDItK7DPI6JVN/f+4RK9IAOndSf3/zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMw=\", \"dir\" : \"C2S\", \"dst_addr\" : \"10.11.21.11\", \"dst_ap\" : \"10.11.21.11:445\", \"dst_port\" : 445, \"gid\" : 1, \"iface\" : \"/home/noah/pcaps/pcaps from malware-traffica-analysis.net/2019-11-21-Emotet-epoch-3-with-Trickbot-gtag-mor49-and-spambot-traffic.pcap\", \"msg\" : \"OS-WINDOWS Microsoft Windows raw WriteAndX InData pointer adjustment attempt\", \"mpls\" : 0, \"pkt_gen\" : \"stream_tcp\", \"pkt_len\" : 4160, \"pkt_num\" : 61571, \"priority\" : 1, \"proto\" : \"TCP\", \"rev\" : 1, \"rule\" : \"1:50626:1\", \"service\" : \"netbios-ssn\", \"sid\" : 50626, \"src_addr\" : \"10.11.21.101\", \"src_ap\" : \"10.11.21.101:50084\", \"src_port\" : 50084, \"vlan\" : 0, \"timestamp\" : \"11/21-18:01:50.061909\" }", + "severity": 1, + "timezone": "America/Chicago", + "type": [ + "allowed" + ] + }, + "network": { + "bytes": 4160, + "community_id": "1:S5lsROZyWDa9wtuxT4CyNDzjmGM=", + "direction": "internal", + "iana_number": "6", + "packets": 61571, + "protocol": "netbios-ssn", + "transport": "tcp", + "type": "ipv4" + }, + "observer": { + "ingress": { + "interface": { + "name": "/home/noah/pcaps/pcaps from malware-traffica-analysis.net/2019-11-21-Emotet-epoch-3-with-Trickbot-gtag-mor49-and-spambot-traffic.pcap" + } + }, + "product": "ids", + "type": "ids", + "vendor": "snort" + }, + "related": { + "ip": [ + "10.11.21.101", + "10.11.21.11" + ] + }, + "rule": { + "category": "Attempted Administrator Privilege Gain", + "description": "OS-WINDOWS Microsoft Windows raw WriteAndX InData pointer adjustment attempt", + "id": "50626", + "version": "1" + }, + "snort": { + "gid": 1 + }, + "source": { + "address": "10.11.21.101", + "ip": "10.11.21.101", + "port": 50084 + }, + "tags": [ + "preserve_original_event" + ] + } + ] +} \ No newline at end of file diff --git a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-pfsense.log-expected.json b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-pfsense.log-expected.json index 84b59dd4fe4..464513b18d4 100644 --- a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-pfsense.log-expected.json +++ b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-pfsense.log-expected.json @@ -26,7 +26,7 @@ "category": [ "network" ], - "created": "2021-01-04T12:37:16.428-06:00", + "created": "2020-04-28T11:07:58.223Z", "kind": "alert", "original": "09/03/21-12:37:16.428952 ,1,2403488,68499,\"ET CINS Active Threat Intelligence Poor Reputation IP TCP group 95\",TCP,175.16.199.1,36847,175.16.199.1,91,54321,Misc Attack,2,alert,Allow", "severity": 2, @@ -38,6 +38,7 @@ "network": { "community_id": "1:QZjg2eWEv0AR1/Sfa6zE1x0jQIg=", "direction": "external", + "iana_number": "6", "transport": "tcp", "type": "ipv4" }, @@ -110,7 +111,7 @@ "category": [ "network" ], - "created": "2021-01-04T12:56:44.310-06:00", + "created": "2020-04-28T11:07:58.223Z", "kind": "alert", "original": "09/03/21-12:56:44.310212 ,1,2011716,4,\"ET SCAN Sipvicious User-Agent Detected (friendly-scanner)\",UDP,175.16.199.1,5103,175.16.199.1,5060,54925,Attempted Information Leak,2,alert,Allow", "severity": 2, @@ -122,6 +123,7 @@ "network": { "community_id": "1:dHh+jdcD2h6T0VDqCQgahOokJmk=", "direction": "external", + "iana_number": "17", "transport": "udp", "type": "ipv4" }, @@ -193,7 +195,7 @@ "category": [ "network" ], - "created": "2021-01-04T16:29:03.494-06:00", + "created": "2020-04-28T11:07:58.223Z", "kind": "alert", "original": "09/03/21-16:29:03.494387 ,1,477,3,\"ICMP Packet\",ICMP,175.16.199.1,,175.16.199.1,,40546,,0,alert,Allow", "severity": 0, @@ -205,6 +207,7 @@ "network": { "community_id": "1:ae//KI+huidgn9Nxeaibd8SUiVA=", "direction": "external", + "iana_number": "1", "transport": "icmp", "type": "ipv4" }, diff --git a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-syslog.log-expected.json b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-syslog.log-expected.json index 60778ac9e9b..2c9504fffc3 100644 --- a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-syslog.log-expected.json +++ b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-syslog.log-expected.json @@ -14,7 +14,7 @@ "category": [ "network" ], - "created": "2022-09-05T16:05:26.000-05:00", + "created": "2020-04-28T11:07:58.223Z", "kind": "alert", "original": "Sep 5 16:05:26 dev snort: [1:1000017:0] UDP Connection [Classification: Misc activity] [Priority: 3] {UDP} 10.150.10.44:55776 -\u003e 10.25.10.22:32414", "severity": 3, @@ -23,6 +23,7 @@ "network": { "community_id": "1:xdk4oWoq+8Q2+Iaf1JdosfY7OOc=", "direction": "internal", + "iana_number": "17", "transport": "udp", "type": "ipv4" }, @@ -73,7 +74,7 @@ "category": [ "network" ], - "created": "2022-09-05T16:05:26.000-05:00", + "created": "2020-04-28T11:07:58.223Z", "kind": "alert", "original": "Sep 5 16:05:26 dev snort: [1:1000016:0] TCP Connection [Priority: 3] {TCP} 10.50.20.59:58720 -\u003e 10.50.10.190:22", "severity": 3, @@ -82,6 +83,7 @@ "network": { "community_id": "1:7dT4p40n4wpXC4y0CEDB0vejj6k=", "direction": "internal", + "iana_number": "6", "transport": "tcp", "type": "ipv4" }, @@ -142,7 +144,7 @@ "category": [ "network" ], - "created": "2022-09-05T16:02:55.000-05:00", + "created": "2020-04-28T11:07:58.223Z", "kind": "alert", "original": "Sep 5 16:02:55 dev snort: [1:1000015:0] Pinging... [Classification: Misc activity] [Priority: 3] {ICMP} 10.50.10.88 -\u003e 175.16.199.1", "severity": 3, @@ -151,6 +153,7 @@ "network": { "community_id": "1:AwywM3uuS+luH6U/hUKtj2x2LWU=", "direction": "outbound", + "iana_number": "1", "transport": "icmp", "type": "ipv4" }, diff --git a/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 99d3e0d1e54..9b7c34c640c 100644 --- a/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -16,34 +16,21 @@ processors: - set: field: observer.type value: ids + - set: + field: event.created + copy_from: '@timestamp' - grok: field: event.original patterns: - # Syslog - - '^(%{ECS_SYSLOG_PRI})?%{SYSLOGTIMESTAMP:_tmp.timestamp} (?:%{SYSLOGFACILITY} )?%{OBSERVER} %{SYSLOGPROG}:%{HEADER}%{FAST_BODY}' - # PFsense CSV - - '%{CSV_START},%{NONNEGINT:snort.ip.id:long},(%{DATA:rule.category}|),%{NONNEGINT:event.severity:long},%{WORD},%{WORD:_tmp.action}' - # Alert CSV (Default) - - '%{CSV_START},(%{MAC:source.mac}|),(%{MAC:destination.mac}|),(%{DATA:snort.eth.length}|),(%{DATA:snort.tcp.flags}|),(%{BASE16NUM:snort.tcp.seq}|),(%{BASE16NUM:snort.tcp.ack}|),(|%{DATA:snort.tcp.length}),(%{BASE16NUM:snort.tcp.window}|),(%{NONNEGINT:snort.ip.ttl:long}|),(%{NONNEGINT:snort.ip.tos:long}|),(%{NONNEGINT:snort.ip.id:long}|),(%{NONNEGINT:snort.dgm.length:long}|),(%{NONNEGINT:snort.ip.length:long}|),(%{NONNEGINT:snort.icmp.type:long}|),(%{NONNEGINT:snort.icmp.code:long}|),(%{NONNEGINT:snort.icmp.id:long}|),(%{NONNEGINT:snort.icmp.seq:long}|)' - # Snort Alert Fast - - '%{SNORT_DATE:_tmp.timestamp}%{SPACE}%{FAST_HEADER}%{FAST_BODY}' - # Snort Alert Full (Multiline) - - '%{FAST_HEADER}\n(%{CLASSIFICATION} )?%{PRIORITY} \n%{SNORT_DATE:_tmp.timestamp} %{IP:source.address}(:%{POSINT:source.port:long}|) -> %{IP:destination.address}(:%{POSINT:destination.port:long}|)\n%{WORD:network.transport} (TTL:%{NONNEGINT:snort.ip.ttl:long}|) (TOS:%{BASE16NUM:snort.ip.tos}|) (ID:%{NONNEGINT:snort.ip.id:long}|) (IpLen:%{NONNEGINT:snort.ip.length:long}|) (DgmLen:%{NONNEGINT:snort.dgm.length:long}|)(%{SPACE}%{NOTSPACE:snort.ip.flags})?\n(%{UDP_DATA}|%{ICMP_DATA}|%{TCP_DATA})' + - ^%{CHAR:_tmp.first_char} pattern_definitions: - SNORT_DATE: '%{MONTHNUM}/%{MONTHDAY}(/%{YEAR})?-%{TIME}' - SEP: '(\[\*\*\])' - CSV_START: '%{SNORT_DATE:_tmp.timestamp}(%{SPACE})?,%{NONNEGINT:snort.gid:long},%{NONNEGINT:rule.id},%{NONNEGINT:rule.version},("?%{DATA:rule.description}"?|),%{WORD:network.transport},%{IP:source.address},(%{POSINT:source.port:long}|),%{IP:destination.address},(%{POSINT:destination.port:long}|)' - HEADER: '%{SPACE}\[%{NONNEGINT:snort.gid:long}:%{NONNEGINT:rule.id}:%{NONNEGINT:rule.version}\]%{SPACE}%{DATA:rule.description}%{SPACE}' - FAST_HEADER: '%{SEP}%{HEADER}%{SEP}' - FAST_BODY: '%{SPACE}%{CLASSIFICATION} %{PRIORITY} \{%{WORD:network.transport}\} %{IP:source.address}(:%{POSINT:source.port:long}|) -> %{IP:destination.address}(:%{POSINT:destination.port:long}|)' - TCP_DATA: '(%{NOTSPACE:snort.tcp.flags}|)%{SPACE}(Seq: %{BASE16NUM:snort.tcp.seq}|)%{SPACE}(Ack: %{BASE16NUM:snort.tcp.ack}|)%{SPACE}(Win: %{BASE16NUM:snort.tcp.window}|)%{SPACE}(TcpLen: %{NONNEGINT:snort.tcp.length:long}|)' - UDP_DATA: '(Len: %{NONNEGINT:snort.udp.length:long})' - ICMP_DATA: '(Type:%{NONNEGINT:snort.icmp.type:long}|)%{SPACE}(Code:%{NONNEGINT:snort.icmp.code:long}|)%{SPACE}(ID:%{NONNEGINT:snort.icmp.id:long}|)%{SPACE}(Seq:%{NONNEGINT:snort.icmp.seq:long}|)%{GREEDYDATA}' - CLASSIFICATION: '(\[Classification: %{DATA:rule.category}\])?' - PRIORITY: '\[Priority: %{NONNEGINT:event.severity:long}\]' - OBSERVER: '(?:%{IP:observer.ip}|%{HOSTNAME:observer.name})' - ECS_SYSLOG_PRI: '<%{NONNEGINT:log.syslog.priority:long}>' - SYSLOGPROG: '%{PROG:process.name}(?:\[%{POSINT:process.pid:int}\])?' + CHAR: . + - pipeline: + if: ctx._tmp.first_char != '{' + name: '{{ IngestPipeline "plaintext" }}' + - pipeline: + if: ctx._tmp.first_char == '{' + name: '{{ IngestPipeline "json" }}' - set: field: event.timezone value: "{{_tmp.tz_offset}}" @@ -66,10 +53,7 @@ processors: - MM/dd/YY-HH:mm:ss.SSSSSS - MMM d HH:mm:ss - MMM dd HH:mm:ss - if: ctx.event?.timezone == null - - set: - field: event.created - copy_from: '@timestamp' + if: ctx.event?.timezone == null - convert: field: source.address target_field: source.ip @@ -118,42 +102,32 @@ processors: field: network.type value: ipv6 if: ctx.source?.ip != null && !ctx.source?.ip.contains(".") + - script: + lang: painless + ignore_failure: true + if: ctx?.network?.transport != null + source: | + def transport = ctx.network.transport; + if (transport == 'udp') { + ctx.network.iana_number = '17'; + } else if (transport == 'tcp') { + ctx.network.iana_number = '6'; + } else if (transport == 'icmp') { + ctx.network.iana_number = '1'; + } - network_direction: internal_networks_field: _tmp.internal_networks - community_id: ignore_failure: true ignore_missing: true - - script: - lang: painless - source: >- - if (ctx.snort?.ip?.tos != null && ctx.snort?.ip?.tos instanceof String) { - ctx.snort.ip.tos = Long.decode(ctx.snort.ip.tos); - } - if (ctx.snort?.eth?.length != null && ctx.snort?.eth?.length instanceof String) { - ctx.snort.eth.length = Long.decode(ctx.snort.eth.length); - } - if (ctx.snort?.tcp?.ack != null && ctx.snort?.tcp?.ack instanceof String) { - ctx.snort.tcp.ack = Long.decode(ctx.snort.tcp.ack); - } - if (ctx.snort?.tcp?.seq != null && ctx.snort?.tcp?.seq instanceof String) { - ctx.snort.tcp.seq = Long.decode(ctx.snort.tcp.seq); - } - if (ctx.snort?.tcp?.window != null && ctx.snort?.tcp?.window instanceof String) { - ctx.snort.tcp.window = Long.decode(ctx.snort.tcp.window); - } - - gsub: - field: snort.tcp.flags - pattern: \* - replacement: '' - ignore_missing: true - append: field: event.type value: allowed - if: ctx._tmp?.action == 'Allow' + if: ctx._tmp?.action?.toLowerCase() == 'allow' - append: field: event.type value: denied - if: ctx._tmp?.action == 'Block' + if: ctx._tmp?.action?.toLowerCase() == 'block' # IP Geolocation Lookup - geoip: field: source.ip @@ -210,6 +184,7 @@ processors: - remove: field: - _tmp + - json ignore_missing: true - remove: field: event.original diff --git a/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/json.yml b/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/json.yml new file mode 100644 index 00000000000..511636be3a9 --- /dev/null +++ b/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/json.yml @@ -0,0 +1,153 @@ +--- +description: Pipeline for parsing Snort JSON logs +processors: + - json: + field: event.original + target_field: json + - remove: + field: json.b64_data + ignore_missing: true + - rename: + field: json.timestamp + target_field: _tmp.timestamp + ignore_missing: true + - rename: + field: json.timestamp + target_field: _tmp.timestamp + ignore_missing: true + - convert: + field: json.src_port + target_field: source.port + type: long + ignore_missing: true + - convert: + field: json.dst_port + target_field: destination.port + type: long + ignore_missing: true + - rename: + field: json.dst_addr + target_field: destination.address + ignore_missing: true + - rename: + field: json.src_addr + target_field: source.address + ignore_missing: true + - rename: + field: json.eth_dst + target_field: destination.mac + ignore_missing: true + - rename: + field: json.eth_src + target_field: source.mac + ignore_missing: true + - convert: + field: json.eth_len + target_field: snort.eth.length + type: long + ignore_missing: true +## Rule + - rename: + field: json.class + target_field: rule.category + ignore_missing: true + - rename: + field: json.msg + target_field: rule.description + ignore_missing: true + - convert: + field: json.rev + target_field: rule.version + type: string + ignore_missing: true + - convert: + field: json.sid + target_field: rule.id + type: string + ignore_missing: true + - convert: + field: json.gid + target_field: snort.gid + type: long + ignore_missing: true +## ICMP + - convert: + field: json.icmp_type + target_field: snort.icmp.type + type: long + ignore_missing: true + - convert: + field: json.icmp_code + target_field: snort.icmp.code + type: long + ignore_missing: true + - convert: + field: json.icmp_id + target_field: snort.icmp.id + type: long + ignore_missing: true + - convert: + field: json.icmp_seq + target_field: snort.icmp.seq + type: long + ignore_missing: true +## IP + - convert: + field: json.ip_id + target_field: snort.ip.id + type: long + ignore_missing: true + - convert: + field: json.ip_id + target_field: snort.ip.id + type: long + ignore_missing: true + - convert: + field: json.tos + target_field: snort.ip.tos + type: long + ignore_missing: true + - convert: + field: json.ttl + target_field: snort.ip.ttl + type: long + ignore_missing: true +## Network + - convert: + field: json.pkt_num + target_field: network.packets + type: long + ignore_missing: true + - convert: + field: json.pkt_len + target_field: network.bytes + type: long + ignore_missing: true + - rename: + field: json.proto + target_field: network.transport + ignore_missing: true + - rename: + field: json.service + target_field: network.protocol + ignore_missing: true + if: ctx.json?.service != 'unknown' +## Other + - convert: + field: json.priority + target_field: event.severity + type: long + ignore_missing: true + - rename: + field: json.action + target_field: _tmp.action + ignore_missing: true + - rename: + field: json.iface + target_field: observer.ingress.interface.name + ignore_missing: true + +on_failure: + - set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/plaintext.yml b/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/plaintext.yml new file mode 100644 index 00000000000..1f0f9caa2d3 --- /dev/null +++ b/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/plaintext.yml @@ -0,0 +1,53 @@ +--- +description: Pipeline for parsing Snort Plaintext logs +processors: + - grok: + field: event.original + patterns: + # Syslog + - '^(%{ECS_SYSLOG_PRI})?%{SYSLOGTIMESTAMP:_tmp.timestamp} (?:%{SYSLOGFACILITY} )?%{OBSERVER} %{SYSLOGPROG}:%{HEADER}%{FAST_BODY}' + # PFsense CSV + - '%{CSV_START},%{NONNEGINT:snort.ip.id:long},(%{DATA:rule.category}|),%{NONNEGINT:event.severity:long},%{WORD},%{WORD:_tmp.action}' + # Alert CSV (Default) + - '%{CSV_START},(%{MAC:source.mac}|),(%{MAC:destination.mac}|),(%{DATA:snort.eth.length}|),(%{DATA:snort.tcp.flags}|),(%{BASE16NUM:snort.tcp.seq}|),(%{BASE16NUM:snort.tcp.ack}|),(|%{DATA:snort.tcp.length}),(%{BASE16NUM:snort.tcp.window}|),(%{NONNEGINT:snort.ip.ttl:long}|),(%{NONNEGINT:snort.ip.tos:long}|),(%{NONNEGINT:snort.ip.id:long}|),(%{NONNEGINT:snort.dgm.length:long}|),(%{NONNEGINT:snort.ip.length:long}|),(%{NONNEGINT:snort.icmp.type:long}|),(%{NONNEGINT:snort.icmp.code:long}|),(%{NONNEGINT:snort.icmp.id:long}|),(%{NONNEGINT:snort.icmp.seq:long}|)' + # Snort Alert Fast + - '%{SNORT_DATE:_tmp.timestamp}%{SPACE}%{FAST_HEADER}%{FAST_BODY}' + # Snort Alert Full (Multiline) + - '%{FAST_HEADER}\n(%{CLASSIFICATION} )?%{PRIORITY} \n%{SNORT_DATE:_tmp.timestamp} %{IP:source.address}(:%{POSINT:source.port:long}|) -> %{IP:destination.address}(:%{POSINT:destination.port:long}|)\n%{WORD:network.transport} (TTL:%{NONNEGINT:snort.ip.ttl:long}|) (TOS:%{BASE16NUM:snort.ip.tos}|) (ID:%{NONNEGINT:snort.ip.id:long}|) (IpLen:%{NONNEGINT:snort.ip.length:long}|) (DgmLen:%{NONNEGINT:snort.dgm.length:long}|)(%{SPACE}%{NOTSPACE:snort.ip.flags})?\n(%{UDP_DATA}|%{ICMP_DATA}|%{TCP_DATA})' + pattern_definitions: + SNORT_DATE: '%{MONTHNUM}/%{MONTHDAY}(/%{YEAR})?-%{TIME}' + SEP: '(\[\*\*\])' + CSV_START: '%{SNORT_DATE:_tmp.timestamp}(%{SPACE})?,%{NONNEGINT:snort.gid:long},%{NONNEGINT:rule.id},%{NONNEGINT:rule.version},("?%{DATA:rule.description}"?|),%{WORD:network.transport},%{IP:source.address},(%{POSINT:source.port:long}|),%{IP:destination.address},(%{POSINT:destination.port:long}|)' + HEADER: '%{SPACE}\[%{NONNEGINT:snort.gid:long}:%{NONNEGINT:rule.id}:%{NONNEGINT:rule.version}\]%{SPACE}%{DATA:rule.description}%{SPACE}' + FAST_HEADER: '%{SEP}%{HEADER}%{SEP}' + FAST_BODY: '%{SPACE}%{CLASSIFICATION} %{PRIORITY} \{%{WORD:network.transport}\} %{IP:source.address}(:%{POSINT:source.port:long}|) -> %{IP:destination.address}(:%{POSINT:destination.port:long}|)' + TCP_DATA: '(%{NOTSPACE:snort.tcp.flags}|)%{SPACE}(Seq: %{BASE16NUM:snort.tcp.seq}|)%{SPACE}(Ack: %{BASE16NUM:snort.tcp.ack}|)%{SPACE}(Win: %{BASE16NUM:snort.tcp.window}|)%{SPACE}(TcpLen: %{NONNEGINT:snort.tcp.length:long}|)' + UDP_DATA: '(Len: %{NONNEGINT:snort.udp.length:long})' + ICMP_DATA: '(Type:%{NONNEGINT:snort.icmp.type:long}|)%{SPACE}(Code:%{NONNEGINT:snort.icmp.code:long}|)%{SPACE}(ID:%{NONNEGINT:snort.icmp.id:long}|)%{SPACE}(Seq:%{NONNEGINT:snort.icmp.seq:long}|)%{GREEDYDATA}' + CLASSIFICATION: '(\[Classification: %{DATA:rule.category}\])?' + PRIORITY: '\[Priority: %{NONNEGINT:event.severity:long}\]' + OBSERVER: '(?:%{IP:observer.ip}|%{HOSTNAME:observer.name})' + ECS_SYSLOG_PRI: '<%{NONNEGINT:log.syslog.priority:long}>' + SYSLOGPROG: '%{PROG:process.name}(?:\[%{POSINT:process.pid:int}\])?' + - script: + lang: painless + source: >- + if (ctx.snort?.ip?.tos != null && ctx.snort?.ip?.tos instanceof String) { + ctx.snort.ip.tos = Long.decode(ctx.snort.ip.tos); + } + if (ctx.snort?.eth?.length != null && ctx.snort?.eth?.length instanceof String) { + ctx.snort.eth.length = Long.decode(ctx.snort.eth.length); + } + if (ctx.snort?.tcp?.ack != null && ctx.snort?.tcp?.ack instanceof String) { + ctx.snort.tcp.ack = Long.decode(ctx.snort.tcp.ack); + } + if (ctx.snort?.tcp?.seq != null && ctx.snort?.tcp?.seq instanceof String) { + ctx.snort.tcp.seq = Long.decode(ctx.snort.tcp.seq); + } + if (ctx.snort?.tcp?.window != null && ctx.snort?.tcp?.window instanceof String) { + ctx.snort.tcp.window = Long.decode(ctx.snort.tcp.window); + } +on_failure: + - set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/packages/snort/data_stream/log/fields/ecs.yml b/packages/snort/data_stream/log/fields/ecs.yml index eb12ef4375d..8c75b61e929 100644 --- a/packages/snort/data_stream/log/fields/ecs.yml +++ b/packages/snort/data_stream/log/fields/ecs.yml @@ -65,8 +65,14 @@ name: network.protocol - external: ecs name: network.transport +- external: ecs + name: network.iana_number - external: ecs name: network.type +- external: ecs + name: network.packets +- external: ecs + name: network.bytes - external: ecs name: observer.ip - external: ecs @@ -77,6 +83,8 @@ name: observer.type - external: ecs name: observer.vendor +- external: ecs + name: observer.ingress.interface.name - external: ecs name: process.name - external: ecs diff --git a/packages/snort/manifest.yml b/packages/snort/manifest.yml index 01e853c8cfe..7e22f3a79ea 100644 --- a/packages/snort/manifest.yml +++ b/packages/snort/manifest.yml @@ -1,6 +1,6 @@ name: snort title: Snort -version: "1.0.0" +version: "1.1.0" release: ga description: Collect logs from Snort with Elastic Agent. type: integration From 3a0418df065362ad847726e0627e45a8513bc4f4 Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Wed, 27 Jul 2022 19:22:17 -0500 Subject: [PATCH 2/8] Update changelog.yml --- packages/snort/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/snort/changelog.yml b/packages/snort/changelog.yml index ef39c733fb1..56533c9db33 100644 --- a/packages/snort/changelog.yml +++ b/packages/snort/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add Snort 3 JSON support. type: enhancement - link: https://github.com/elastic/integrations/pull/1 + link: https://github.com/elastic/integrations/pull/3876 - version: "1.0.0" changes: - description: Make GA From fe10e6ced4b70d82f51267d049704ac1acb7abeb Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Thu, 28 Jul 2022 00:24:47 +0000 Subject: [PATCH 3/8] update readme --- packages/snort/docs/README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/snort/docs/README.md b/packages/snort/docs/README.md index fb6133cff7f..4a070eec6b3 100644 --- a/packages/snort/docs/README.md +++ b/packages/snort/docs/README.md @@ -186,11 +186,15 @@ An example event for `log` looks as following: | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | | message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | +| network.bytes | Total bytes transferred in both directions. If `source.bytes` and `destination.bytes` are known, `network.bytes` is their sum. | long | | network.community_id | A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows. Learn more at https://github.com/corelight/community-id-spec. | keyword | -| network.direction | Direction of the network traffic. When mapping events from a host-based monitoring context, populate this field from the host's point of view, using the values "ingress" or "egress". When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of the network perimeter, using the values "inbound", "outbound", "internal" or "external". Note that "internal" is not crossing perimeter boundaries, and is meant to describe communication between two hosts within the perimeter. Note also that "external" is meant to describe traffic between two hosts that are external to the perimeter. This could for example be useful for ISPs or VPN service providers. | keyword | +| network.direction | Direction of the network traffic. Recommended values are: \* ingress \* egress \* inbound \* outbound \* internal \* external \* unknown When mapping events from a host-based monitoring context, populate this field from the host's point of view, using the values "ingress" or "egress". When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of the network perimeter, using the values "inbound", "outbound", "internal" or "external". Note that "internal" is not crossing perimeter boundaries, and is meant to describe communication between two hosts within the perimeter. Note also that "external" is meant to describe traffic between two hosts that are external to the perimeter. This could for example be useful for ISPs or VPN service providers. | keyword | +| network.iana_number | IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). Standardized list of protocols. This aligns well with NetFlow and sFlow related logs which use the IANA Protocol Number. | keyword | +| network.packets | Total packets transferred in both directions. If `source.packets` and `destination.packets` are known, `network.packets` is their sum. | long | | network.protocol | In the OSI Model this would be the Application Layer protocol. For example, `http`, `dns`, or `ssh`. The field value must be normalized to lowercase for querying. | keyword | | network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | | network.type | In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc The field value must be normalized to lowercase for querying. | keyword | +| observer.ingress.interface.name | Interface name as reported by the system. | keyword | | observer.ip | IP addresses of the observer. | ip | | observer.name | Custom name of the observer. This is a name that can be given to an observer. This can be helpful for example if multiple firewalls of the same model are used in an organization. If no custom name is needed, the field can be left empty. | keyword | | observer.product | The product name of the observer. | keyword | From a5c83b916fc13142117aa3ef20c8e80005323e47 Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Thu, 28 Jul 2022 01:31:03 +0000 Subject: [PATCH 4/8] update per comments --- .../test/pipeline/test-log-csv.log-expected.json | 4 ++-- .../log/elasticsearch/ingest_pipeline/default.yml | 2 +- .../elasticsearch/ingest_pipeline/plaintext.yml | 15 ++++++++++----- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-csv.log-expected.json b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-csv.log-expected.json index 2dface04931..6f7ad646f36 100644 --- a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-csv.log-expected.json +++ b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-csv.log-expected.json @@ -59,7 +59,7 @@ }, "tcp": { "ack": 3161863446, - "flags": "***AP***", + "flags": "AP", "seq": 1754205692, "window": 2060 } @@ -133,7 +133,7 @@ }, "tcp": { "ack": 3161863506, - "flags": "***AP***", + "flags": "AP", "seq": 1754205752, "window": 2060 } diff --git a/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 9b7c34c640c..f12bc4d6ec8 100644 --- a/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -105,7 +105,7 @@ processors: - script: lang: painless ignore_failure: true - if: ctx?.network?.transport != null + if: ctx.network?.transport != null source: | def transport = ctx.network.transport; if (transport == 'udp') { diff --git a/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/plaintext.yml b/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/plaintext.yml index 1f0f9caa2d3..67e558173ed 100644 --- a/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/plaintext.yml +++ b/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/plaintext.yml @@ -29,22 +29,27 @@ processors: OBSERVER: '(?:%{IP:observer.ip}|%{HOSTNAME:observer.name})' ECS_SYSLOG_PRI: '<%{NONNEGINT:log.syslog.priority:long}>' SYSLOGPROG: '%{PROG:process.name}(?:\[%{POSINT:process.pid:int}\])?' + - gsub: + field: snort.tcp.flags + pattern: \* + replacement: '' + ignore_missing: true - script: lang: painless source: >- - if (ctx.snort?.ip?.tos != null && ctx.snort?.ip?.tos instanceof String) { + if (ctx.snort?.ip?.tos != null && ctx.snort.ip.tos instanceof String) { ctx.snort.ip.tos = Long.decode(ctx.snort.ip.tos); } - if (ctx.snort?.eth?.length != null && ctx.snort?.eth?.length instanceof String) { + if (ctx.snort?.eth?.length != null && ctx.snort.eth.length instanceof String) { ctx.snort.eth.length = Long.decode(ctx.snort.eth.length); } - if (ctx.snort?.tcp?.ack != null && ctx.snort?.tcp?.ack instanceof String) { + if (ctx.snort?.tcp?.ack != null && ctx.snort.tcp.ack instanceof String) { ctx.snort.tcp.ack = Long.decode(ctx.snort.tcp.ack); } - if (ctx.snort?.tcp?.seq != null && ctx.snort?.tcp?.seq instanceof String) { + if (ctx.snort?.tcp?.seq != null && ctx.snort.tcp.seq instanceof String) { ctx.snort.tcp.seq = Long.decode(ctx.snort.tcp.seq); } - if (ctx.snort?.tcp?.window != null && ctx.snort?.tcp?.window instanceof String) { + if (ctx.snort?.tcp?.window != null && ctx.snort.tcp.window instanceof String) { ctx.snort.tcp.window = Long.decode(ctx.snort.tcp.window); } on_failure: From 31caf0d9cef0e2d22909d4936c905634cf6f50e5 Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Thu, 28 Jul 2022 03:04:34 +0000 Subject: [PATCH 5/8] Fix TCP grok matching, add Additional JSON parsing --- .../pipeline/test-log-csv.log-expected.json | 16 +- .../pipeline/test-log-fast.log-expected.json | 20 +- .../pipeline/test-log-full.log-expected.json | 35 +++- .../log/_dev/test/pipeline/test-log-json.log | 6 +- .../pipeline/test-log-json.log-expected.json | 189 +++++++++++++++++- .../test-log-pfsense.log-expected.json | 6 +- .../test-log-syslog.log-expected.json | 6 +- .../elasticsearch/ingest_pipeline/default.yml | 5 + .../elasticsearch/ingest_pipeline/json.yml | 33 ++- .../ingest_pipeline/plaintext.yml | 13 +- .../snort/data_stream/log/fields/fields.yml | 2 +- 11 files changed, 278 insertions(+), 53 deletions(-) diff --git a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-csv.log-expected.json b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-csv.log-expected.json index 6f7ad646f36..fa8549f1417 100644 --- a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-csv.log-expected.json +++ b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-csv.log-expected.json @@ -50,7 +50,7 @@ "eth": { "length": 114 }, - "gid": 1, + "gid": "1", "ip": { "id": 55665, "length": 102400, @@ -124,7 +124,7 @@ "eth": { "length": 114 }, - "gid": 1, + "gid": "1", "ip": { "id": 55666, "length": 102400, @@ -198,7 +198,7 @@ "eth": { "length": 193 }, - "gid": 1, + "gid": "1", "ip": { "id": 56094, "length": 183296, @@ -269,7 +269,7 @@ "eth": { "length": 177 }, - "gid": 1, + "gid": "1", "ip": { "id": 26112, "length": 166912, @@ -340,7 +340,7 @@ "eth": { "length": 63 }, - "gid": 1, + "gid": "1", "ip": { "id": 37712, "length": 50176, @@ -422,7 +422,7 @@ "eth": { "length": 98 }, - "gid": 1, + "gid": "1", "icmp": { "code": 0, "id": 83, @@ -497,7 +497,7 @@ "eth": { "length": 98 }, - "gid": 1, + "gid": "1", "icmp": { "code": 0, "id": 83, @@ -596,7 +596,7 @@ "eth": { "length": 98 }, - "gid": 1, + "gid": "1", "icmp": { "code": 0, "id": 83, diff --git a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-fast.log-expected.json b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-fast.log-expected.json index 62fdf9decbb..d545ef9ff41 100644 --- a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-fast.log-expected.json +++ b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-fast.log-expected.json @@ -45,7 +45,7 @@ "version": "8" }, "snort": { - "gid": 1 + "gid": "1" }, "source": { "address": "0.0.0.0", @@ -113,7 +113,7 @@ "version": "2" }, "snort": { - "gid": 1 + "gid": "1" }, "source": { "address": "192.168.88.10", @@ -178,7 +178,7 @@ "version": "3" }, "snort": { - "gid": 1 + "gid": "1" }, "source": { "address": "175.16.199.1", @@ -257,7 +257,7 @@ "version": "6" }, "snort": { - "gid": 1 + "gid": "1" }, "source": { "address": "192.168.15.10", @@ -313,7 +313,7 @@ "version": "2" }, "snort": { - "gid": 1 + "gid": "1" }, "source": { "address": "175.16.199.1", @@ -380,7 +380,7 @@ "version": "5" }, "snort": { - "gid": 1 + "gid": "1" }, "source": { "address": "175.16.199.1", @@ -447,7 +447,7 @@ "version": "0" }, "snort": { - "gid": 1 + "gid": "1" }, "source": { "address": "175.16.199.1", @@ -515,7 +515,7 @@ "version": "0" }, "snort": { - "gid": 1 + "gid": "1" }, "source": { "address": "175.16.199.1", @@ -594,7 +594,7 @@ "version": "0" }, "snort": { - "gid": 1 + "gid": "1" }, "source": { "address": "10.100.10.190", @@ -648,7 +648,7 @@ "version": "0" }, "snort": { - "gid": 1 + "gid": "1" }, "source": { "address": "175.16.199.1", diff --git a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-full.log-expected.json b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-full.log-expected.json index 6fed563769f..9028a9aa2e6 100644 --- a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-full.log-expected.json +++ b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-full.log-expected.json @@ -47,13 +47,20 @@ "dgm": { "length": 108 }, - "gid": 1, + "gid": "1", "ip": { "flags": "DF", "id": 53730, "length": 20, "tos": 0, "ttl": 127 + }, + "tcp": { + "ack": 3161656246, + "flags": "AP", + "length": 20, + "seq": 1754136804, + "window": 2059 } }, "source": { @@ -112,13 +119,20 @@ "dgm": { "length": 40 }, - "gid": 1, + "gid": "1", "ip": { "flags": "DF", "id": 53731, "length": 20, "tos": 0, "ttl": 127 + }, + "tcp": { + "ack": 3161656322, + "flags": "A", + "length": 20, + "seq": 1754136872, + "window": 2059 } }, "source": { @@ -178,7 +192,7 @@ "dgm": { "length": 83 }, - "gid": 1, + "gid": "1", "ip": { "id": 58363, "length": 20, @@ -257,7 +271,7 @@ "dgm": { "length": 84 }, - "gid": 1, + "gid": "1", "icmp": { "code": 0, "id": 101, @@ -328,13 +342,20 @@ "dgm": { "length": 40 }, - "gid": 1, + "gid": "1", "ip": { "flags": "DF", "id": 61472, "length": 20, "tos": 0, "ttl": 127 + }, + "tcp": { + "ack": 3162578710, + "flags": "A", + "length": 20, + "seq": 1754533236, + "window": 2062 } }, "source": { @@ -393,7 +414,7 @@ "dgm": { "length": 84 }, - "gid": 1, + "gid": "1", "icmp": { "code": 0, "id": 101, @@ -475,7 +496,7 @@ "dgm": { "length": 153 }, - "gid": 1, + "gid": "1", "ip": { "id": 33955, "length": 20, diff --git a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-json.log b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-json.log index 1ac918b86c0..6a5c521134f 100644 --- a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-json.log +++ b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-json.log @@ -1,2 +1,4 @@ -{ "seconds" : 1608147213, "action" : "allow", "class" : "none", "b64_data" : "DWHaXwAAAADO0wgAAAAAABAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc=", "dir" : "S2C", "dst_addr" : "10.10.10.1", "dst_ap" : "10.10.10.1:0", "eth_dst" : "52:54:00:1F:8A:1C", "eth_len" :98, "eth_src" : "52:54:00:70:78:9F", "eth_type" : "0x800", "gid" : 1, "icmp_code" : 0, "icmp_id" :5203, "icmp_seq" : 3, "icmp_type" : 0, "iface" : "ens3", "ip_id" : 3006, "ip_len" : 64, "msg" : "ICMP Traffic Detected", "mpls" : 0, "pkt_gen" : "raw", "pkt_len" : 84, "pkt_num" : 8, "priority" :0, "proto" : "ICMP", "rev" : 0, "rule" : "1:10000001:0", "service" : "unknown", "sid" : 10000001, "src_addr" : "10.10.10.88", "src_ap" : "10.10.10.88:0", "tos" : 0, "ttl" : 64, "vlan" : 0, "timestamp" : "12/16-20:33:33.603502" } -{ "seconds" : 1574352110, "action" : "allow", "class" : "Attempted Administrator Privilege Gain", "b64_data" : "AAAQPP9TTUIvAAAAABgFSAAAyIAtOUGjAkUAAAII6AYACGILDP8AAAACQABAAwD/AAAABAAAEAAAABA8AAEQAMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzIuNTP7//+kvw/z/i41M/v//g8EE6ZbB/P+LVCQIjUIMi4pU/v//M8jobFb9/4tK/DPI6GJW/f+4/JpIAOmqUv3/zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzItN8OnSwvz/i03wg8EE6TzB/P+LVCQIjUIMi0r4M8joFVb9/7hgm0gA6V1S/f/MzMzMzMzMzMzMzMzMzMzMzMyLRbRQ6CdX/f9Zw4tUJAiNQgyLSrQzyOjdVf3/i0rwM8jo01X9/7iUm0gA6RtS/f/MzMzMzMzMzMzMzMzMzMzMi41M/v//g8EE6UzC/P+LjUz+//+DwQjpMcD8/4tUJAiNQgyLilT+//8zyOiJVf3/i0r8M8jof1X9/7jMm0gA6cdR/f/MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMi41A/v//g8EE6ezB/P+LjUD+//+DwQjp0b/8/4tUJAiNQgyLikj+//8zyOgpVf3/i0r8M8joH1X9/7gInEgA6WdR/f/MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMi03wg8EE6Y/B/P+LTfCDwQjpd7/8/4tUJAiNQgyLSvgzyOjSVP3/uEScSADpGlH9/8zMzMzMzMzMzMzMzMzMzItF7FDo51X9/1nDi0XkUOjcVf3/WcOLVCQIjUIMi0rgM8joklT9/7iAnEgA6dpQ/f/MzMzMzMzMzMzMzMzMzMyLTeyDwQjpD8H8/4tUJAiNQgyLSvQzyOhdVP3/uLScSADppVD9/8zMzMzMzMzMzMyLRZxQ6HdV/f9Zw4tUJAiNQgyLSogzyOgtVP3/i0r8M8joI1T9/7jknEgA6WtQ/f/MzMzMzMzMzMzMzMzMzMzMjU3k6YW//P+NTeTpw7/8/4tUJAiNQgyLirD+//8zyOjlU/3/uFCdSADpLVD9/8zMzMzMzMzMzMzMzMzMzMzMzI1NsOnCv/z/jU3M6XZe/f+NjWz////pr7/8/41NiOljXv3/jY0o////6Zy//P+LVCQIjUIMi4qw/v//M8joh1P9/7iUnUgA6c9P/f/MzMzMzMzMzMzMzMzMzMzMzMzMzItNhOkeXv3/i1QkCI1CDItKjDPI6FBT/f+4ZJ5IAOmYT/3/zMzMzMzMzMzMzMzMzItNhOnuXf3/i1QkCI1CDItKjDPI6CBT/f+4lJ5IAOloT/3/zMzMzMzMzMzMzMzMzIuN6P7//+nIvvz/i1QkCI1CDIuK8P7//zPI6OpS/f+4xJ5IAOkyT/3/zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMyLTejpi778/4tUJAiNQgyLSvAzyOiwUv3/uPSeSADp+E79/8zMzMzMzMzMzMzMzMyLVCQIjUIMi4rM/v//M8johVL9/7hQn0gA6c1O/f/MzMzMzMzMzMzMzMzMzMzMzMyLVCQIjUIMi4pk////M8joVVL9/7i8n0gA6Z1O/f/MzMzMzMzMzMzMzMzMzMzMzMyNTdzptb38/41N3Onzvfz/i1QkCI1CDIuKYP7//zPI6BVS/f+4XKBIAOldTv3/zMzMzMzMzMzMzMzMzMzMzMzMjU3Y6fK9/P+LRdSD4AEPhAwAAACDZdT+i00I6dq9/P/Di1QkCI1CDIuKXP///zPI6MRR/f+4nKBIAOkMTv3/zMzMzMzMzMzMzMzMzMzMi1QkCI1CDItK7DPI6JpR/f+4kKJIAOniTf3/i03wg8EE6VM+//+LVCQIjUIMi0rsM8jodFH9/7gEqkgA6bxN/f+LTfDpDEH//4tN8IPBBOlYvfz/i03wg8Eg6U29/P+LTfCDwTzpQr38/4tN8IPBWOk3vfz/i1QkCI1CDItK7DPI6CVR/f+4KKpIAOltTf3/i03w6b1A//+LTfCDwQTpCb38/4tN8IPBIOn+vPz/i03wg8E86fO8/P+LTfCDwVjp6Lz8/4tUJAiNQgyLSuwzyOjWUP3/uHSqSADpHk39/41N1OnFvPz/i1QkCI1CDItKmDPI6LNQ/f+LSvwzyOipUP3/uMiqSADp8Uz9/41N1OmYvPz/i1QkCI1CDItKqDPI6IZQ/f+LSvwzyOh8UP3/uPSqSADpxEz9//91COihUf3/WcOLRfCD4AEPhAwAAACDZfD+jU3U6VG8/P/Di1QkCI1CDIuKXP///zPI6DtQ/f+4MKtIAOmDTP3/jU3U6Sq8/P+LVCQIjUIMi0qcM8joGFD9/4tK/DPI6A5Q/f+4XKtIAOlWTP3/jU3s6aY///+LVCQIjUIMi0rcM8jo60/9/7iYq0gA6TNM/f//dQjoEFH9/1nDi0Xwg+ABD4QMAAAAg2Xw/o1N1OnAu/z/w4tUJAiNQgyLilz///8zyOiqT/3/uPyrSADp8kv9/4tN8OnWJP//i1QkCI1CDItK7DPI6IdP/f+4KKxIAOnPS/3/jU3w6XQf//+LVCQIjUIMi0rsM8joZE/9/7hUrEgA6axL/f+NTezp/D7//4tUJAiNQgyLStwzyOhBT/3/uICsSADpiUv9/4tF8IPgAQ+EDwAAAINl8P6LTeyDwQTpfiD//8OLVCQIjUIMi0roM8joCk/9/7isrEgA6VJL/f+LTfDptVn9/4tUJAiNQgyLSuwzyOjnTv3/uNisSADpL0v9/41N2OnWuvz/i1QkCI1CDItKrDPI6MRO/f+4FK1IAOkMS/3/jU3Y6bO6/P+LVCQIjUIMi0qsM8jooU79/7iYrUgA6elK/f+NTdjpkLr8/4tUJAiNQgyLSqwzyOh+Tv3/uACuSADpxkr9/4tN8OkpWf3/i1QkCI1CDItK7DPI6FtO/f+4WK5IAOmjSv3/jU3w6fM9//+LVCQIjUIMi0rsM8joOE79/7iErkgA6YBK/f+LTfDp0B3//4tUJAiNQgyLSuwzyOgVTv3/uLCuSADpXUr9/4tN8OmtHf//i03wg8EY6fm5/P+LVCQIjUIMi0rsM8jo5039/7jkrkgA6S9K/f+NTfDpfz3///917OgET/3/WcOLVCQIjUIMi0roM8jouk39/7gYr0gA6QJK/f//dfDo3079/1nDi1QkCI1CDItK7DPI6JVN/f+4RK9IAOndSf3/zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMw=", "dir" : "C2S", "dst_addr" : "10.11.21.11", "dst_ap" : "10.11.21.11:445", "dst_port" : 445, "gid" : 1, "iface" : "/home/noah/pcaps/pcaps from malware-traffica-analysis.net/2019-11-21-Emotet-epoch-3-with-Trickbot-gtag-mor49-and-spambot-traffic.pcap", "msg" : "OS-WINDOWS Microsoft Windows raw WriteAndX InData pointer adjustment attempt", "mpls" : 0, "pkt_gen" : "stream_tcp", "pkt_len" : 4160, "pkt_num" : 61571, "priority" : 1, "proto" : "TCP", "rev" : 1, "rule" : "1:50626:1", "service" : "netbios-ssn", "sid" : 50626, "src_addr" : "10.11.21.101", "src_ap" : "10.11.21.101:50084", "src_port" : 50084, "vlan" : 0, "timestamp" : "11/21-18:01:50.061909" } \ No newline at end of file +{"seconds":1608147213,"action":"allow","class":"none","b64_data":"DWHaXwAAAADO0wgAAAAAABAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc=","dir":"S2C","dst_addr":"10.10.10.1","dst_ap":"10.10.10.1:0","eth_dst":"52:54:00:1F:8A:1C","eth_len":98,"eth_src":"52:54:00:70:78:9F","eth_type":"0x800","gid":1,"icmp_code":0,"icmp_id":5203,"icmp_seq":3,"icmp_type":0,"iface":"ens3","ip_id":3006,"ip_len":64,"msg":"ICMP Traffic Detected","mpls":0,"pkt_gen":"raw","pkt_len":84,"pkt_num":8,"priority":0,"proto":"ICMP","rev":0,"rule":"1:10000001:0","service":"unknown","sid":10000001,"src_addr":"10.10.10.88","src_ap":"10.10.10.88:0","tos":0,"ttl":64,"vlan":0,"timestamp":"12/16-20:33:33.603502"} +{"seconds":1574352110,"action":"allow","class":"Attempted Administrator Privilege Gain","b64_data":"AAAQPP9TTUIvAAAAABgFSAAAyIAtOUGjAkUAAAII6AYACGILDP8AAAACQABAAwD/AAAABAAAEAAAABA8AAEQAMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzIuNTP7//+kvw/z/i41M/v//g8EE6ZbB/P+LVCQIjUIMi4pU/v//M8jobFb9/4tK/DPI6GJW/f+4/JpIAOmqUv3/zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzItN8OnSwvz/i03wg8EE6TzB/P+LVCQIjUIMi0r4M8joFVb9/7hgm0gA6V1S/f/MzMzMzMzMzMzMzMzMzMzMzMyLRbRQ6CdX/f9Zw4tUJAiNQgyLSrQzyOjdVf3/i0rwM8jo01X9/7iUm0gA6RtS/f/MzMzMzMzMzMzMzMzMzMzMi41M/v//g8EE6UzC/P+LjUz+//+DwQjpMcD8/4tUJAiNQgyLilT+//8zyOiJVf3/i0r8M8jof1X9/7jMm0gA6cdR/f/MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMi41A/v//g8EE6ezB/P+LjUD+//+DwQjp0b/8/4tUJAiNQgyLikj+//8zyOgpVf3/i0r8M8joH1X9/7gInEgA6WdR/f/MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMi03wg8EE6Y/B/P+LTfCDwQjpd7/8/4tUJAiNQgyLSvgzyOjSVP3/uEScSADpGlH9/8zMzMzMzMzMzMzMzMzMzItF7FDo51X9/1nDi0XkUOjcVf3/WcOLVCQIjUIMi0rgM8joklT9/7iAnEgA6dpQ/f/MzMzMzMzMzMzMzMzMzMyLTeyDwQjpD8H8/4tUJAiNQgyLSvQzyOhdVP3/uLScSADppVD9/8zMzMzMzMzMzMyLRZxQ6HdV/f9Zw4tUJAiNQgyLSogzyOgtVP3/i0r8M8joI1T9/7jknEgA6WtQ/f/MzMzMzMzMzMzMzMzMzMzMjU3k6YW//P+NTeTpw7/8/4tUJAiNQgyLirD+//8zyOjlU/3/uFCdSADpLVD9/8zMzMzMzMzMzMzMzMzMzMzMzI1NsOnCv/z/jU3M6XZe/f+NjWz////pr7/8/41NiOljXv3/jY0o////6Zy//P+LVCQIjUIMi4qw/v//M8joh1P9/7iUnUgA6c9P/f/MzMzMzMzMzMzMzMzMzMzMzMzMzItNhOkeXv3/i1QkCI1CDItKjDPI6FBT/f+4ZJ5IAOmYT/3/zMzMzMzMzMzMzMzMzItNhOnuXf3/i1QkCI1CDItKjDPI6CBT/f+4lJ5IAOloT/3/zMzMzMzMzMzMzMzMzIuN6P7//+nIvvz/i1QkCI1CDIuK8P7//zPI6OpS/f+4xJ5IAOkyT/3/zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMyLTejpi778/4tUJAiNQgyLSvAzyOiwUv3/uPSeSADp+E79/8zMzMzMzMzMzMzMzMyLVCQIjUIMi4rM/v//M8johVL9/7hQn0gA6c1O/f/MzMzMzMzMzMzMzMzMzMzMzMyLVCQIjUIMi4pk////M8joVVL9/7i8n0gA6Z1O/f/MzMzMzMzMzMzMzMzMzMzMzMyNTdzptb38/41N3Onzvfz/i1QkCI1CDIuKYP7//zPI6BVS/f+4XKBIAOldTv3/zMzMzMzMzMzMzMzMzMzMzMzMjU3Y6fK9/P+LRdSD4AEPhAwAAACDZdT+i00I6dq9/P/Di1QkCI1CDIuKXP///zPI6MRR/f+4nKBIAOkMTv3/zMzMzMzMzMzMzMzMzMzMi1QkCI1CDItK7DPI6JpR/f+4kKJIAOniTf3/i03wg8EE6VM+//+LVCQIjUIMi0rsM8jodFH9/7gEqkgA6bxN/f+LTfDpDEH//4tN8IPBBOlYvfz/i03wg8Eg6U29/P+LTfCDwTzpQr38/4tN8IPBWOk3vfz/i1QkCI1CDItK7DPI6CVR/f+4KKpIAOltTf3/i03w6b1A//+LTfCDwQTpCb38/4tN8IPBIOn+vPz/i03wg8E86fO8/P+LTfCDwVjp6Lz8/4tUJAiNQgyLSuwzyOjWUP3/uHSqSADpHk39/41N1OnFvPz/i1QkCI1CDItKmDPI6LNQ/f+LSvwzyOipUP3/uMiqSADp8Uz9/41N1OmYvPz/i1QkCI1CDItKqDPI6IZQ/f+LSvwzyOh8UP3/uPSqSADpxEz9//91COihUf3/WcOLRfCD4AEPhAwAAACDZfD+jU3U6VG8/P/Di1QkCI1CDIuKXP///zPI6DtQ/f+4MKtIAOmDTP3/jU3U6Sq8/P+LVCQIjUIMi0qcM8joGFD9/4tK/DPI6A5Q/f+4XKtIAOlWTP3/jU3s6aY///+LVCQIjUIMi0rcM8jo60/9/7iYq0gA6TNM/f//dQjoEFH9/1nDi0Xwg+ABD4QMAAAAg2Xw/o1N1OnAu/z/w4tUJAiNQgyLilz///8zyOiqT/3/uPyrSADp8kv9/4tN8OnWJP//i1QkCI1CDItK7DPI6IdP/f+4KKxIAOnPS/3/jU3w6XQf//+LVCQIjUIMi0rsM8joZE/9/7hUrEgA6axL/f+NTezp/D7//4tUJAiNQgyLStwzyOhBT/3/uICsSADpiUv9/4tF8IPgAQ+EDwAAAINl8P6LTeyDwQTpfiD//8OLVCQIjUIMi0roM8joCk/9/7isrEgA6VJL/f+LTfDptVn9/4tUJAiNQgyLSuwzyOjnTv3/uNisSADpL0v9/41N2OnWuvz/i1QkCI1CDItKrDPI6MRO/f+4FK1IAOkMS/3/jU3Y6bO6/P+LVCQIjUIMi0qsM8jooU79/7iYrUgA6elK/f+NTdjpkLr8/4tUJAiNQgyLSqwzyOh+Tv3/uACuSADpxkr9/4tN8OkpWf3/i1QkCI1CDItK7DPI6FtO/f+4WK5IAOmjSv3/jU3w6fM9//+LVCQIjUIMi0rsM8joOE79/7iErkgA6YBK/f+LTfDp0B3//4tUJAiNQgyLSuwzyOgVTv3/uLCuSADpXUr9/4tN8OmtHf//i03wg8EY6fm5/P+LVCQIjUIMi0rsM8jo5039/7jkrkgA6S9K/f+NTfDpfz3///917OgET/3/WcOLVCQIjUIMi0roM8jouk39/7gYr0gA6QJK/f//dfDo3079/1nDi1QkCI1CDItK7DPI6JVN/f+4RK9IAOndSf3/zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMw=","dir":"C2S","dst_addr":"10.11.21.11","dst_ap":"10.11.21.11:445","dst_port":445,"gid":1,"iface":"/home/noah/pcaps/pcaps from malware-traffica-analysis.net/2019-11-21-Emotet-epoch-3-with-Trickbot-gtag-mor49-and-spambot-traffic.pcap","msg":"OS-WINDOWS Microsoft Windows raw WriteAndX InData pointer adjustment attempt","mpls":0,"pkt_gen":"stream_tcp","pkt_len":4160,"pkt_num":61571,"priority":1,"proto":"TCP","rev":1,"rule":"1:50626:1","service":"netbios-ssn","sid":50626,"src_addr":"10.11.21.101","src_ap":"10.11.21.101:50084","src_port":50084,"vlan":0,"timestamp":"11/21-18:01:50.061909"} +{"seconds":1263690203,"action":"allow","class":"none","b64_data":"SFRUUC8xLjEgMjAwIE9LDQpEYXRlOiBTdW4sIDE3IEphbiAyMDEwIDAxOjAzOjIzIEdNVA0KQ29udGVudC1MZW5ndGg6IDkzDQpDb25uZWN0aW9uOiBLZWVwLUFsaXZlDQpDb250ZW50LVR5cGU6IGFwcGxpY2F0aW9uL29jdGV0LXN0cmVhbQ0KDQqIUD9i2EiKmsaSRMwz+KykbZuhBHl51gb+ncb3E+k56qaeXYeonZrAin2EYqRNqg40EDHjyzBj2rR+H6RvSHD4CznUGYujs3yybbSPm5ijXksL6/4xUrH2izVXw74=","dir":"S2C","dst_addr":"192.168.3.35","dst_ap":"192.168.3.35:1047","dst_port":1047,"eth_dst":"00:0C:29:92:E9:86","eth_len":287,"eth_src":"00:0C:29:B9:39:C3","eth_type":"0x800","gid":119,"iface":"/home/noah/samples//bredolab-sample.pcap","ip_id":10856,"ip_len":253,"msg":"(http_inspect) gzip decompression failed","mpls":0,"pkt_gen":"raw","pkt_len":273,"pkt_num":1612,"priority":3,"proto":"TCP","rev":1,"rule":"119:217:1","service":"http","sid":217,"src_addr":"89.160.20.114","src_ap":"89.160.20.114:80","src_port":80,"tcp_ack":3174915760,"tcp_flags":"***AP***","tcp_len":20,"tcp_seq":4204308887,"tcp_win":64903,"tos":0,"ttl":115,"vlan":0,"timestamp":"01/17-03:03:23.476194"} +{"seconds":1238569343,"action":"allow","class":"none","b64_data":"AgEGADPfV5sAAAAAAAAAAMCoAWcAAAAAAAAAAAAX8kPA0wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABjglNjNQECNgTAqAEBAQT///8AMwQAdqcAAwTAqAEBBghBIAVvQSAFcA8KY2ZsLnJyLmNvbRoCBdT/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=","dir":"C2S","dst_addr":"255.255.255.255","dst_ap":"255.255.255.255:68","dst_port":68,"eth_dst":"FF:FF:FF:FF:FF:FF","eth_len":590,"eth_src":"00:0F:66:80:D2:2A","eth_type":"0x800","gid":116,"iface":"/home/noah/samples//ConfickerB9hrs.pcap","ip_id":3913,"ip_len":556,"msg":"(ipv4) IPv4 packet to broadcast dest address","mpls":0,"pkt_gen":"raw","pkt_len":576,"pkt_num":689,"priority":3,"proto":"UDP","rev":1,"rule":"116:414:1","service":"unknown","sid":414,"src_addr":"192.168.1.1","src_ap":"192.168.1.1:67","src_port":67,"tos":0,"ttl":64,"udp_len":556,"vlan":0,"timestamp":"04/01-09:02:23.126173"} \ No newline at end of file diff --git a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-json.log-expected.json b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-json.log-expected.json index ad975c09c06..5975368b4e0 100644 --- a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-json.log-expected.json +++ b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-json.log-expected.json @@ -16,7 +16,7 @@ ], "created": "2020-04-28T11:07:58.223Z", "kind": "alert", - "original": "{ \"seconds\" : 1608147213, \"action\" : \"allow\", \"class\" : \"none\", \"b64_data\" : \"DWHaXwAAAADO0wgAAAAAABAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc=\", \"dir\" : \"S2C\", \"dst_addr\" : \"10.10.10.1\", \"dst_ap\" : \"10.10.10.1:0\", \"eth_dst\" : \"52:54:00:1F:8A:1C\", \"eth_len\" :98, \"eth_src\" : \"52:54:00:70:78:9F\", \"eth_type\" : \"0x800\", \"gid\" : 1, \"icmp_code\" : 0, \"icmp_id\" :5203, \"icmp_seq\" : 3, \"icmp_type\" : 0, \"iface\" : \"ens3\", \"ip_id\" : 3006, \"ip_len\" : 64, \"msg\" : \"ICMP Traffic Detected\", \"mpls\" : 0, \"pkt_gen\" : \"raw\", \"pkt_len\" : 84, \"pkt_num\" : 8, \"priority\" :0, \"proto\" : \"ICMP\", \"rev\" : 0, \"rule\" : \"1:10000001:0\", \"service\" : \"unknown\", \"sid\" : 10000001, \"src_addr\" : \"10.10.10.88\", \"src_ap\" : \"10.10.10.88:0\", \"tos\" : 0, \"ttl\" : 64, \"vlan\" : 0, \"timestamp\" : \"12/16-20:33:33.603502\" }", + "original": "{\"seconds\":1608147213,\"action\":\"allow\",\"class\":\"none\",\"b64_data\":\"DWHaXwAAAADO0wgAAAAAABAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc=\",\"dir\":\"S2C\",\"dst_addr\":\"10.10.10.1\",\"dst_ap\":\"10.10.10.1:0\",\"eth_dst\":\"52:54:00:1F:8A:1C\",\"eth_len\":98,\"eth_src\":\"52:54:00:70:78:9F\",\"eth_type\":\"0x800\",\"gid\":1,\"icmp_code\":0,\"icmp_id\":5203,\"icmp_seq\":3,\"icmp_type\":0,\"iface\":\"ens3\",\"ip_id\":3006,\"ip_len\":64,\"msg\":\"ICMP Traffic Detected\",\"mpls\":0,\"pkt_gen\":\"raw\",\"pkt_len\":84,\"pkt_num\":8,\"priority\":0,\"proto\":\"ICMP\",\"rev\":0,\"rule\":\"1:10000001:0\",\"service\":\"unknown\",\"sid\":10000001,\"src_addr\":\"10.10.10.88\",\"src_ap\":\"10.10.10.88:0\",\"tos\":0,\"ttl\":64,\"vlan\":0,\"timestamp\":\"12/16-20:33:33.603502\"}", "severity": 0, "timezone": "America/Chicago", "type": [ @@ -58,7 +58,7 @@ "eth": { "length": 98 }, - "gid": 1, + "gid": "1", "icmp": { "code": 0, "id": 5203, @@ -96,7 +96,7 @@ ], "created": "2020-04-28T11:07:58.223Z", "kind": "alert", - "original": "{ \"seconds\" : 1574352110, \"action\" : \"allow\", \"class\" : \"Attempted Administrator Privilege Gain\", \"b64_data\" : \"AAAQPP9TTUIvAAAAABgFSAAAyIAtOUGjAkUAAAII6AYACGILDP8AAAACQABAAwD/AAAABAAAEAAAABA8AAEQAMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzIuNTP7//+kvw/z/i41M/v//g8EE6ZbB/P+LVCQIjUIMi4pU/v//M8jobFb9/4tK/DPI6GJW/f+4/JpIAOmqUv3/zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzItN8OnSwvz/i03wg8EE6TzB/P+LVCQIjUIMi0r4M8joFVb9/7hgm0gA6V1S/f/MzMzMzMzMzMzMzMzMzMzMzMyLRbRQ6CdX/f9Zw4tUJAiNQgyLSrQzyOjdVf3/i0rwM8jo01X9/7iUm0gA6RtS/f/MzMzMzMzMzMzMzMzMzMzMi41M/v//g8EE6UzC/P+LjUz+//+DwQjpMcD8/4tUJAiNQgyLilT+//8zyOiJVf3/i0r8M8jof1X9/7jMm0gA6cdR/f/MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMi41A/v//g8EE6ezB/P+LjUD+//+DwQjp0b/8/4tUJAiNQgyLikj+//8zyOgpVf3/i0r8M8joH1X9/7gInEgA6WdR/f/MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMi03wg8EE6Y/B/P+LTfCDwQjpd7/8/4tUJAiNQgyLSvgzyOjSVP3/uEScSADpGlH9/8zMzMzMzMzMzMzMzMzMzItF7FDo51X9/1nDi0XkUOjcVf3/WcOLVCQIjUIMi0rgM8joklT9/7iAnEgA6dpQ/f/MzMzMzMzMzMzMzMzMzMyLTeyDwQjpD8H8/4tUJAiNQgyLSvQzyOhdVP3/uLScSADppVD9/8zMzMzMzMzMzMyLRZxQ6HdV/f9Zw4tUJAiNQgyLSogzyOgtVP3/i0r8M8joI1T9/7jknEgA6WtQ/f/MzMzMzMzMzMzMzMzMzMzMjU3k6YW//P+NTeTpw7/8/4tUJAiNQgyLirD+//8zyOjlU/3/uFCdSADpLVD9/8zMzMzMzMzMzMzMzMzMzMzMzI1NsOnCv/z/jU3M6XZe/f+NjWz////pr7/8/41NiOljXv3/jY0o////6Zy//P+LVCQIjUIMi4qw/v//M8joh1P9/7iUnUgA6c9P/f/MzMzMzMzMzMzMzMzMzMzMzMzMzItNhOkeXv3/i1QkCI1CDItKjDPI6FBT/f+4ZJ5IAOmYT/3/zMzMzMzMzMzMzMzMzItNhOnuXf3/i1QkCI1CDItKjDPI6CBT/f+4lJ5IAOloT/3/zMzMzMzMzMzMzMzMzIuN6P7//+nIvvz/i1QkCI1CDIuK8P7//zPI6OpS/f+4xJ5IAOkyT/3/zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMyLTejpi778/4tUJAiNQgyLSvAzyOiwUv3/uPSeSADp+E79/8zMzMzMzMzMzMzMzMyLVCQIjUIMi4rM/v//M8johVL9/7hQn0gA6c1O/f/MzMzMzMzMzMzMzMzMzMzMzMyLVCQIjUIMi4pk////M8joVVL9/7i8n0gA6Z1O/f/MzMzMzMzMzMzMzMzMzMzMzMyNTdzptb38/41N3Onzvfz/i1QkCI1CDIuKYP7//zPI6BVS/f+4XKBIAOldTv3/zMzMzMzMzMzMzMzMzMzMzMzMjU3Y6fK9/P+LRdSD4AEPhAwAAACDZdT+i00I6dq9/P/Di1QkCI1CDIuKXP///zPI6MRR/f+4nKBIAOkMTv3/zMzMzMzMzMzMzMzMzMzMi1QkCI1CDItK7DPI6JpR/f+4kKJIAOniTf3/i03wg8EE6VM+//+LVCQIjUIMi0rsM8jodFH9/7gEqkgA6bxN/f+LTfDpDEH//4tN8IPBBOlYvfz/i03wg8Eg6U29/P+LTfCDwTzpQr38/4tN8IPBWOk3vfz/i1QkCI1CDItK7DPI6CVR/f+4KKpIAOltTf3/i03w6b1A//+LTfCDwQTpCb38/4tN8IPBIOn+vPz/i03wg8E86fO8/P+LTfCDwVjp6Lz8/4tUJAiNQgyLSuwzyOjWUP3/uHSqSADpHk39/41N1OnFvPz/i1QkCI1CDItKmDPI6LNQ/f+LSvwzyOipUP3/uMiqSADp8Uz9/41N1OmYvPz/i1QkCI1CDItKqDPI6IZQ/f+LSvwzyOh8UP3/uPSqSADpxEz9//91COihUf3/WcOLRfCD4AEPhAwAAACDZfD+jU3U6VG8/P/Di1QkCI1CDIuKXP///zPI6DtQ/f+4MKtIAOmDTP3/jU3U6Sq8/P+LVCQIjUIMi0qcM8joGFD9/4tK/DPI6A5Q/f+4XKtIAOlWTP3/jU3s6aY///+LVCQIjUIMi0rcM8jo60/9/7iYq0gA6TNM/f//dQjoEFH9/1nDi0Xwg+ABD4QMAAAAg2Xw/o1N1OnAu/z/w4tUJAiNQgyLilz///8zyOiqT/3/uPyrSADp8kv9/4tN8OnWJP//i1QkCI1CDItK7DPI6IdP/f+4KKxIAOnPS/3/jU3w6XQf//+LVCQIjUIMi0rsM8joZE/9/7hUrEgA6axL/f+NTezp/D7//4tUJAiNQgyLStwzyOhBT/3/uICsSADpiUv9/4tF8IPgAQ+EDwAAAINl8P6LTeyDwQTpfiD//8OLVCQIjUIMi0roM8joCk/9/7isrEgA6VJL/f+LTfDptVn9/4tUJAiNQgyLSuwzyOjnTv3/uNisSADpL0v9/41N2OnWuvz/i1QkCI1CDItKrDPI6MRO/f+4FK1IAOkMS/3/jU3Y6bO6/P+LVCQIjUIMi0qsM8jooU79/7iYrUgA6elK/f+NTdjpkLr8/4tUJAiNQgyLSqwzyOh+Tv3/uACuSADpxkr9/4tN8OkpWf3/i1QkCI1CDItK7DPI6FtO/f+4WK5IAOmjSv3/jU3w6fM9//+LVCQIjUIMi0rsM8joOE79/7iErkgA6YBK/f+LTfDp0B3//4tUJAiNQgyLSuwzyOgVTv3/uLCuSADpXUr9/4tN8OmtHf//i03wg8EY6fm5/P+LVCQIjUIMi0rsM8jo5039/7jkrkgA6S9K/f+NTfDpfz3///917OgET/3/WcOLVCQIjUIMi0roM8jouk39/7gYr0gA6QJK/f//dfDo3079/1nDi1QkCI1CDItK7DPI6JVN/f+4RK9IAOndSf3/zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMw=\", \"dir\" : \"C2S\", \"dst_addr\" : \"10.11.21.11\", \"dst_ap\" : \"10.11.21.11:445\", \"dst_port\" : 445, \"gid\" : 1, \"iface\" : \"/home/noah/pcaps/pcaps from malware-traffica-analysis.net/2019-11-21-Emotet-epoch-3-with-Trickbot-gtag-mor49-and-spambot-traffic.pcap\", \"msg\" : \"OS-WINDOWS Microsoft Windows raw WriteAndX InData pointer adjustment attempt\", \"mpls\" : 0, \"pkt_gen\" : \"stream_tcp\", \"pkt_len\" : 4160, \"pkt_num\" : 61571, \"priority\" : 1, \"proto\" : \"TCP\", \"rev\" : 1, \"rule\" : \"1:50626:1\", \"service\" : \"netbios-ssn\", \"sid\" : 50626, \"src_addr\" : \"10.11.21.101\", \"src_ap\" : \"10.11.21.101:50084\", \"src_port\" : 50084, \"vlan\" : 0, \"timestamp\" : \"11/21-18:01:50.061909\" }", + "original": "{\"seconds\":1574352110,\"action\":\"allow\",\"class\":\"Attempted Administrator Privilege Gain\",\"b64_data\":\"AAAQPP9TTUIvAAAAABgFSAAAyIAtOUGjAkUAAAII6AYACGILDP8AAAACQABAAwD/AAAABAAAEAAAABA8AAEQAMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzIuNTP7//+kvw/z/i41M/v//g8EE6ZbB/P+LVCQIjUIMi4pU/v//M8jobFb9/4tK/DPI6GJW/f+4/JpIAOmqUv3/zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzItN8OnSwvz/i03wg8EE6TzB/P+LVCQIjUIMi0r4M8joFVb9/7hgm0gA6V1S/f/MzMzMzMzMzMzMzMzMzMzMzMyLRbRQ6CdX/f9Zw4tUJAiNQgyLSrQzyOjdVf3/i0rwM8jo01X9/7iUm0gA6RtS/f/MzMzMzMzMzMzMzMzMzMzMi41M/v//g8EE6UzC/P+LjUz+//+DwQjpMcD8/4tUJAiNQgyLilT+//8zyOiJVf3/i0r8M8jof1X9/7jMm0gA6cdR/f/MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMi41A/v//g8EE6ezB/P+LjUD+//+DwQjp0b/8/4tUJAiNQgyLikj+//8zyOgpVf3/i0r8M8joH1X9/7gInEgA6WdR/f/MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMi03wg8EE6Y/B/P+LTfCDwQjpd7/8/4tUJAiNQgyLSvgzyOjSVP3/uEScSADpGlH9/8zMzMzMzMzMzMzMzMzMzItF7FDo51X9/1nDi0XkUOjcVf3/WcOLVCQIjUIMi0rgM8joklT9/7iAnEgA6dpQ/f/MzMzMzMzMzMzMzMzMzMyLTeyDwQjpD8H8/4tUJAiNQgyLSvQzyOhdVP3/uLScSADppVD9/8zMzMzMzMzMzMyLRZxQ6HdV/f9Zw4tUJAiNQgyLSogzyOgtVP3/i0r8M8joI1T9/7jknEgA6WtQ/f/MzMzMzMzMzMzMzMzMzMzMjU3k6YW//P+NTeTpw7/8/4tUJAiNQgyLirD+//8zyOjlU/3/uFCdSADpLVD9/8zMzMzMzMzMzMzMzMzMzMzMzI1NsOnCv/z/jU3M6XZe/f+NjWz////pr7/8/41NiOljXv3/jY0o////6Zy//P+LVCQIjUIMi4qw/v//M8joh1P9/7iUnUgA6c9P/f/MzMzMzMzMzMzMzMzMzMzMzMzMzItNhOkeXv3/i1QkCI1CDItKjDPI6FBT/f+4ZJ5IAOmYT/3/zMzMzMzMzMzMzMzMzItNhOnuXf3/i1QkCI1CDItKjDPI6CBT/f+4lJ5IAOloT/3/zMzMzMzMzMzMzMzMzIuN6P7//+nIvvz/i1QkCI1CDIuK8P7//zPI6OpS/f+4xJ5IAOkyT/3/zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMyLTejpi778/4tUJAiNQgyLSvAzyOiwUv3/uPSeSADp+E79/8zMzMzMzMzMzMzMzMyLVCQIjUIMi4rM/v//M8johVL9/7hQn0gA6c1O/f/MzMzMzMzMzMzMzMzMzMzMzMyLVCQIjUIMi4pk////M8joVVL9/7i8n0gA6Z1O/f/MzMzMzMzMzMzMzMzMzMzMzMyNTdzptb38/41N3Onzvfz/i1QkCI1CDIuKYP7//zPI6BVS/f+4XKBIAOldTv3/zMzMzMzMzMzMzMzMzMzMzMzMjU3Y6fK9/P+LRdSD4AEPhAwAAACDZdT+i00I6dq9/P/Di1QkCI1CDIuKXP///zPI6MRR/f+4nKBIAOkMTv3/zMzMzMzMzMzMzMzMzMzMi1QkCI1CDItK7DPI6JpR/f+4kKJIAOniTf3/i03wg8EE6VM+//+LVCQIjUIMi0rsM8jodFH9/7gEqkgA6bxN/f+LTfDpDEH//4tN8IPBBOlYvfz/i03wg8Eg6U29/P+LTfCDwTzpQr38/4tN8IPBWOk3vfz/i1QkCI1CDItK7DPI6CVR/f+4KKpIAOltTf3/i03w6b1A//+LTfCDwQTpCb38/4tN8IPBIOn+vPz/i03wg8E86fO8/P+LTfCDwVjp6Lz8/4tUJAiNQgyLSuwzyOjWUP3/uHSqSADpHk39/41N1OnFvPz/i1QkCI1CDItKmDPI6LNQ/f+LSvwzyOipUP3/uMiqSADp8Uz9/41N1OmYvPz/i1QkCI1CDItKqDPI6IZQ/f+LSvwzyOh8UP3/uPSqSADpxEz9//91COihUf3/WcOLRfCD4AEPhAwAAACDZfD+jU3U6VG8/P/Di1QkCI1CDIuKXP///zPI6DtQ/f+4MKtIAOmDTP3/jU3U6Sq8/P+LVCQIjUIMi0qcM8joGFD9/4tK/DPI6A5Q/f+4XKtIAOlWTP3/jU3s6aY///+LVCQIjUIMi0rcM8jo60/9/7iYq0gA6TNM/f//dQjoEFH9/1nDi0Xwg+ABD4QMAAAAg2Xw/o1N1OnAu/z/w4tUJAiNQgyLilz///8zyOiqT/3/uPyrSADp8kv9/4tN8OnWJP//i1QkCI1CDItK7DPI6IdP/f+4KKxIAOnPS/3/jU3w6XQf//+LVCQIjUIMi0rsM8joZE/9/7hUrEgA6axL/f+NTezp/D7//4tUJAiNQgyLStwzyOhBT/3/uICsSADpiUv9/4tF8IPgAQ+EDwAAAINl8P6LTeyDwQTpfiD//8OLVCQIjUIMi0roM8joCk/9/7isrEgA6VJL/f+LTfDptVn9/4tUJAiNQgyLSuwzyOjnTv3/uNisSADpL0v9/41N2OnWuvz/i1QkCI1CDItKrDPI6MRO/f+4FK1IAOkMS/3/jU3Y6bO6/P+LVCQIjUIMi0qsM8jooU79/7iYrUgA6elK/f+NTdjpkLr8/4tUJAiNQgyLSqwzyOh+Tv3/uACuSADpxkr9/4tN8OkpWf3/i1QkCI1CDItK7DPI6FtO/f+4WK5IAOmjSv3/jU3w6fM9//+LVCQIjUIMi0rsM8joOE79/7iErkgA6YBK/f+LTfDp0B3//4tUJAiNQgyLSuwzyOgVTv3/uLCuSADpXUr9/4tN8OmtHf//i03wg8EY6fm5/P+LVCQIjUIMi0rsM8jo5039/7jkrkgA6S9K/f+NTfDpfz3///917OgET/3/WcOLVCQIjUIMi0roM8jouk39/7gYr0gA6QJK/f//dfDo3079/1nDi1QkCI1CDItK7DPI6JVN/f+4RK9IAOndSf3/zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMw=\",\"dir\":\"C2S\",\"dst_addr\":\"10.11.21.11\",\"dst_ap\":\"10.11.21.11:445\",\"dst_port\":445,\"gid\":1,\"iface\":\"/home/noah/pcaps/pcaps from malware-traffica-analysis.net/2019-11-21-Emotet-epoch-3-with-Trickbot-gtag-mor49-and-spambot-traffic.pcap\",\"msg\":\"OS-WINDOWS Microsoft Windows raw WriteAndX InData pointer adjustment attempt\",\"mpls\":0,\"pkt_gen\":\"stream_tcp\",\"pkt_len\":4160,\"pkt_num\":61571,\"priority\":1,\"proto\":\"TCP\",\"rev\":1,\"rule\":\"1:50626:1\",\"service\":\"netbios-ssn\",\"sid\":50626,\"src_addr\":\"10.11.21.101\",\"src_ap\":\"10.11.21.101:50084\",\"src_port\":50084,\"vlan\":0,\"timestamp\":\"11/21-18:01:50.061909\"}", "severity": 1, "timezone": "America/Chicago", "type": [ @@ -136,7 +136,7 @@ "version": "1" }, "snort": { - "gid": 1 + "gid": "1" }, "source": { "address": "10.11.21.101", @@ -146,6 +146,187 @@ "tags": [ "preserve_original_event" ] + }, + { + "@timestamp": "2022-01-17T03:03:23.476-06:00", + "destination": { + "address": "192.168.3.35", + "ip": "192.168.3.35", + "mac": "00-0C-29-92-E9-86", + "port": 1047 + }, + "ecs": { + "version": "8.3.0" + }, + "event": { + "category": [ + "network" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "alert", + "original": "{\"seconds\":1263690203,\"action\":\"allow\",\"class\":\"none\",\"b64_data\":\"SFRUUC8xLjEgMjAwIE9LDQpEYXRlOiBTdW4sIDE3IEphbiAyMDEwIDAxOjAzOjIzIEdNVA0KQ29udGVudC1MZW5ndGg6IDkzDQpDb25uZWN0aW9uOiBLZWVwLUFsaXZlDQpDb250ZW50LVR5cGU6IGFwcGxpY2F0aW9uL29jdGV0LXN0cmVhbQ0KDQqIUD9i2EiKmsaSRMwz+KykbZuhBHl51gb+ncb3E+k56qaeXYeonZrAin2EYqRNqg40EDHjyzBj2rR+H6RvSHD4CznUGYujs3yybbSPm5ijXksL6/4xUrH2izVXw74=\",\"dir\":\"S2C\",\"dst_addr\":\"192.168.3.35\",\"dst_ap\":\"192.168.3.35:1047\",\"dst_port\":1047,\"eth_dst\":\"00:0C:29:92:E9:86\",\"eth_len\":287,\"eth_src\":\"00:0C:29:B9:39:C3\",\"eth_type\":\"0x800\",\"gid\":119,\"iface\":\"/home/noah/samples//bredolab-sample.pcap\",\"ip_id\":10856,\"ip_len\":253,\"msg\":\"(http_inspect) gzip decompression failed\",\"mpls\":0,\"pkt_gen\":\"raw\",\"pkt_len\":273,\"pkt_num\":1612,\"priority\":3,\"proto\":\"TCP\",\"rev\":1,\"rule\":\"119:217:1\",\"service\":\"http\",\"sid\":217,\"src_addr\":\"89.160.20.114\",\"src_ap\":\"89.160.20.114:80\",\"src_port\":80,\"tcp_ack\":3174915760,\"tcp_flags\":\"***AP***\",\"tcp_len\":20,\"tcp_seq\":4204308887,\"tcp_win\":64903,\"tos\":0,\"ttl\":115,\"vlan\":0,\"timestamp\":\"01/17-03:03:23.476194\"}", + "severity": 3, + "timezone": "America/Chicago", + "type": [ + "allowed" + ] + }, + "network": { + "bytes": 273, + "community_id": "1:De/02XSMKB1hAkZFS02R05gxe3E=", + "direction": "inbound", + "iana_number": "6", + "packets": 1612, + "protocol": "http", + "transport": "tcp", + "type": "ipv4" + }, + "observer": { + "ingress": { + "interface": { + "name": "/home/noah/samples//bredolab-sample.pcap" + } + }, + "product": "ids", + "type": "ids", + "vendor": "snort" + }, + "related": { + "ip": [ + "89.160.20.114", + "192.168.3.35" + ] + }, + "rule": { + "category": "none", + "description": "(http_inspect) gzip decompression failed", + "id": "217", + "version": "1" + }, + "snort": { + "eth": { + "length": 287 + }, + "gid": "119", + "ip": { + "id": 10856, + "tos": 0, + "ttl": 115 + }, + "tcp": { + "ack": 3174915760, + "flags": "AP", + "length": 20, + "seq": 4204308887, + "window": 64903 + } + }, + "source": { + "address": "89.160.20.114", + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "geo": { + "city_name": "Linköping", + "continent_name": "Europe", + "country_iso_code": "SE", + "country_name": "Sweden", + "location": { + "lat": 58.4167, + "lon": 15.6167 + }, + "region_iso_code": "SE-E", + "region_name": "Östergötland County" + }, + "ip": "89.160.20.114", + "mac": "00-0C-29-B9-39-C3", + "port": 80 + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2022-04-01T09:02:23.126-05:00", + "destination": { + "address": "255.255.255.255", + "ip": "255.255.255.255", + "mac": "FF-FF-FF-FF-FF-FF", + "port": 68 + }, + "ecs": { + "version": "8.3.0" + }, + "event": { + "category": [ + "network" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "alert", + "original": "{\"seconds\":1238569343,\"action\":\"allow\",\"class\":\"none\",\"b64_data\":\"AgEGADPfV5sAAAAAAAAAAMCoAWcAAAAAAAAAAAAX8kPA0wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABjglNjNQECNgTAqAEBAQT///8AMwQAdqcAAwTAqAEBBghBIAVvQSAFcA8KY2ZsLnJyLmNvbRoCBdT/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\",\"dir\":\"C2S\",\"dst_addr\":\"255.255.255.255\",\"dst_ap\":\"255.255.255.255:68\",\"dst_port\":68,\"eth_dst\":\"FF:FF:FF:FF:FF:FF\",\"eth_len\":590,\"eth_src\":\"00:0F:66:80:D2:2A\",\"eth_type\":\"0x800\",\"gid\":116,\"iface\":\"/home/noah/samples//ConfickerB9hrs.pcap\",\"ip_id\":3913,\"ip_len\":556,\"msg\":\"(ipv4) IPv4 packet to broadcast dest address\",\"mpls\":0,\"pkt_gen\":\"raw\",\"pkt_len\":576,\"pkt_num\":689,\"priority\":3,\"proto\":\"UDP\",\"rev\":1,\"rule\":\"116:414:1\",\"service\":\"unknown\",\"sid\":414,\"src_addr\":\"192.168.1.1\",\"src_ap\":\"192.168.1.1:67\",\"src_port\":67,\"tos\":0,\"ttl\":64,\"udp_len\":556,\"vlan\":0,\"timestamp\":\"04/01-09:02:23.126173\"}", + "severity": 3, + "timezone": "America/Chicago", + "type": [ + "allowed" + ] + }, + "network": { + "bytes": 576, + "community_id": "1:RTu96ufLudxAGu4dGf2hzG8lF7w=", + "direction": "outbound", + "iana_number": "17", + "packets": 689, + "transport": "udp", + "type": "ipv4" + }, + "observer": { + "ingress": { + "interface": { + "name": "/home/noah/samples//ConfickerB9hrs.pcap" + } + }, + "product": "ids", + "type": "ids", + "vendor": "snort" + }, + "related": { + "ip": [ + "192.168.1.1", + "255.255.255.255" + ] + }, + "rule": { + "category": "none", + "description": "(ipv4) IPv4 packet to broadcast dest address", + "id": "414", + "version": "1" + }, + "snort": { + "eth": { + "length": 590 + }, + "gid": "116", + "ip": { + "id": 3913, + "tos": 0, + "ttl": 64 + }, + "udp": { + "length": 556 + } + }, + "source": { + "address": "192.168.1.1", + "ip": "192.168.1.1", + "mac": "00-0F-66-80-D2-2A", + "port": 67 + }, + "tags": [ + "preserve_original_event" + ] } ] } \ No newline at end of file diff --git a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-pfsense.log-expected.json b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-pfsense.log-expected.json index 464513b18d4..a41878a3f3a 100644 --- a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-pfsense.log-expected.json +++ b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-pfsense.log-expected.json @@ -59,7 +59,7 @@ "version": "68499" }, "snort": { - "gid": 1, + "gid": "1", "ip": { "id": 54321 } @@ -144,7 +144,7 @@ "version": "4" }, "snort": { - "gid": 1, + "gid": "1", "ip": { "id": 54925 } @@ -228,7 +228,7 @@ "version": "3" }, "snort": { - "gid": 1, + "gid": "1", "ip": { "id": 40546 } diff --git a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-syslog.log-expected.json b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-syslog.log-expected.json index 2c9504fffc3..e792296de7d 100644 --- a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-syslog.log-expected.json +++ b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-syslog.log-expected.json @@ -49,7 +49,7 @@ "version": "0" }, "snort": { - "gid": 1 + "gid": "1" }, "source": { "address": "10.150.10.44", @@ -108,7 +108,7 @@ "version": "0" }, "snort": { - "gid": 1 + "gid": "1" }, "source": { "address": "10.50.20.59", @@ -179,7 +179,7 @@ "version": "0" }, "snort": { - "gid": 1 + "gid": "1" }, "source": { "address": "10.50.10.88", diff --git a/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/default.yml index f12bc4d6ec8..024c7016f89 100644 --- a/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -80,6 +80,11 @@ processors: pattern: '[.:]' replacement: '-' ignore_missing: true + - gsub: + field: snort.tcp.flags + pattern: \* + replacement: '' + ignore_missing: true - lowercase: field: network.transport ignore_missing: true diff --git a/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/json.yml b/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/json.yml index 511636be3a9..a251fe0f725 100644 --- a/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/json.yml +++ b/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/json.yml @@ -68,7 +68,7 @@ processors: - convert: field: json.gid target_field: snort.gid - type: long + type: string ignore_missing: true ## ICMP - convert: @@ -91,12 +91,33 @@ processors: target_field: snort.icmp.seq type: long ignore_missing: true -## IP - - convert: - field: json.ip_id - target_field: snort.ip.id - type: long +## TCP + - rename: + field: json.tcp_flags + target_field: snort.tcp.flags + ignore_missing: true + - rename: + field: json.tcp_len + target_field: snort.tcp.length ignore_missing: true + - rename: + field: json.tcp_seq + target_field: snort.tcp.seq + ignore_missing: true + - rename: + field: json.tcp_ack + target_field: snort.tcp.ack + ignore_missing: true + - rename: + field: json.tcp_win + target_field: snort.tcp.window + ignore_missing: true +## UDP + - rename: + field: json.udp_len + target_field: snort.udp.length + ignore_missing: true +## IP - convert: field: json.ip_id target_field: snort.ip.id diff --git a/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/plaintext.yml b/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/plaintext.yml index 67e558173ed..2dc453ab57c 100644 --- a/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/plaintext.yml +++ b/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/plaintext.yml @@ -17,23 +17,18 @@ processors: pattern_definitions: SNORT_DATE: '%{MONTHNUM}/%{MONTHDAY}(/%{YEAR})?-%{TIME}' SEP: '(\[\*\*\])' - CSV_START: '%{SNORT_DATE:_tmp.timestamp}(%{SPACE})?,%{NONNEGINT:snort.gid:long},%{NONNEGINT:rule.id},%{NONNEGINT:rule.version},("?%{DATA:rule.description}"?|),%{WORD:network.transport},%{IP:source.address},(%{POSINT:source.port:long}|),%{IP:destination.address},(%{POSINT:destination.port:long}|)' - HEADER: '%{SPACE}\[%{NONNEGINT:snort.gid:long}:%{NONNEGINT:rule.id}:%{NONNEGINT:rule.version}\]%{SPACE}%{DATA:rule.description}%{SPACE}' + CSV_START: '%{SNORT_DATE:_tmp.timestamp}(%{SPACE})?,%{NONNEGINT:snort.gid},%{NONNEGINT:rule.id},%{NONNEGINT:rule.version},("?%{DATA:rule.description}"?|),%{WORD:network.transport},%{IP:source.address},(%{POSINT:source.port:long}|),%{IP:destination.address},(%{POSINT:destination.port:long}|)' + HEADER: '%{SPACE}\[%{NONNEGINT:snort.gid}:%{NONNEGINT:rule.id}:%{NONNEGINT:rule.version}\]%{SPACE}%{DATA:rule.description}%{SPACE}' FAST_HEADER: '%{SEP}%{HEADER}%{SEP}' FAST_BODY: '%{SPACE}%{CLASSIFICATION} %{PRIORITY} \{%{WORD:network.transport}\} %{IP:source.address}(:%{POSINT:source.port:long}|) -> %{IP:destination.address}(:%{POSINT:destination.port:long}|)' - TCP_DATA: '(%{NOTSPACE:snort.tcp.flags}|)%{SPACE}(Seq: %{BASE16NUM:snort.tcp.seq}|)%{SPACE}(Ack: %{BASE16NUM:snort.tcp.ack}|)%{SPACE}(Win: %{BASE16NUM:snort.tcp.window}|)%{SPACE}(TcpLen: %{NONNEGINT:snort.tcp.length:long}|)' + TCP_DATA: '(%{NOTSPACE:snort.tcp.flags})%{SPACE}(Seq: %{BASE16NUM:snort.tcp.seq})%{SPACE}(Ack: %{BASE16NUM:snort.tcp.ack})%{SPACE}(Win: %{BASE16NUM:snort.tcp.window})%{SPACE}(TcpLen: %{NONNEGINT:snort.tcp.length:long})' UDP_DATA: '(Len: %{NONNEGINT:snort.udp.length:long})' - ICMP_DATA: '(Type:%{NONNEGINT:snort.icmp.type:long}|)%{SPACE}(Code:%{NONNEGINT:snort.icmp.code:long}|)%{SPACE}(ID:%{NONNEGINT:snort.icmp.id:long}|)%{SPACE}(Seq:%{NONNEGINT:snort.icmp.seq:long}|)%{GREEDYDATA}' + ICMP_DATA: '(Type:%{NONNEGINT:snort.icmp.type:long})%{SPACE}(Code:%{NONNEGINT:snort.icmp.code:long})%{SPACE}(ID:%{NONNEGINT:snort.icmp.id:long})%{SPACE}(Seq:%{NONNEGINT:snort.icmp.seq:long})%{GREEDYDATA}' CLASSIFICATION: '(\[Classification: %{DATA:rule.category}\])?' PRIORITY: '\[Priority: %{NONNEGINT:event.severity:long}\]' OBSERVER: '(?:%{IP:observer.ip}|%{HOSTNAME:observer.name})' ECS_SYSLOG_PRI: '<%{NONNEGINT:log.syslog.priority:long}>' SYSLOGPROG: '%{PROG:process.name}(?:\[%{POSINT:process.pid:int}\])?' - - gsub: - field: snort.tcp.flags - pattern: \* - replacement: '' - ignore_missing: true - script: lang: painless source: >- diff --git a/packages/snort/data_stream/log/fields/fields.yml b/packages/snort/data_stream/log/fields/fields.yml index f06cbb5c5f1..02a43e8eb95 100644 --- a/packages/snort/data_stream/log/fields/fields.yml +++ b/packages/snort/data_stream/log/fields/fields.yml @@ -1,5 +1,5 @@ - name: snort.gid - type: long + type: keyword description: > The gid keyword (generator id) is used to identify what part of Snort generates the event when a particular rule fires.dd From 2cb5088aabd762dbdab9ecbb724e99c6a21c1afc Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Thu, 28 Jul 2022 03:16:24 +0000 Subject: [PATCH 6/8] update docs --- packages/snort/_dev/build/docs/README.md | 6 +- .../log/_dev/test/pipeline/test-log-json.log | 3 +- .../pipeline/test-log-json.log-expected.json | 83 +++++++++++++++++++ .../elasticsearch/ingest_pipeline/json.yml | 6 ++ packages/snort/data_stream/log/fields/ecs.yml | 2 + packages/snort/docs/README.md | 9 +- 6 files changed, 101 insertions(+), 8 deletions(-) diff --git a/packages/snort/_dev/build/docs/README.md b/packages/snort/_dev/build/docs/README.md index 21f23083330..7f66c73b760 100644 --- a/packages/snort/_dev/build/docs/README.md +++ b/packages/snort/_dev/build/docs/README.md @@ -4,9 +4,9 @@ This integration is for [Snort](https://www.snort.org/). ## Compatibility -This module has been developed against Snort v2.9, but is expected to work -with other versions of Snort. This package is designed to read from the PFsense CSV output -and the Alert Fast output either via reading a local logfile or receiving messages via syslog +This module has been developed against Snort v2.9 and v3, but is expected to work +with other versions of Snort. This package is designed to read from the PFsense CSV output, +the Alert Fast output either via reading a local logfile or receiving messages via syslog and the Snort 3 JSON log file. ## Log diff --git a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-json.log b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-json.log index 6a5c521134f..7cd9bf06931 100644 --- a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-json.log +++ b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-json.log @@ -1,4 +1,5 @@ {"seconds":1608147213,"action":"allow","class":"none","b64_data":"DWHaXwAAAADO0wgAAAAAABAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc=","dir":"S2C","dst_addr":"10.10.10.1","dst_ap":"10.10.10.1:0","eth_dst":"52:54:00:1F:8A:1C","eth_len":98,"eth_src":"52:54:00:70:78:9F","eth_type":"0x800","gid":1,"icmp_code":0,"icmp_id":5203,"icmp_seq":3,"icmp_type":0,"iface":"ens3","ip_id":3006,"ip_len":64,"msg":"ICMP Traffic Detected","mpls":0,"pkt_gen":"raw","pkt_len":84,"pkt_num":8,"priority":0,"proto":"ICMP","rev":0,"rule":"1:10000001:0","service":"unknown","sid":10000001,"src_addr":"10.10.10.88","src_ap":"10.10.10.88:0","tos":0,"ttl":64,"vlan":0,"timestamp":"12/16-20:33:33.603502"} {"seconds":1574352110,"action":"allow","class":"Attempted Administrator Privilege Gain","b64_data":"AAAQPP9TTUIvAAAAABgFSAAAyIAtOUGjAkUAAAII6AYACGILDP8AAAACQABAAwD/AAAABAAAEAAAABA8AAEQAMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzIuNTP7//+kvw/z/i41M/v//g8EE6ZbB/P+LVCQIjUIMi4pU/v//M8jobFb9/4tK/DPI6GJW/f+4/JpIAOmqUv3/zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzItN8OnSwvz/i03wg8EE6TzB/P+LVCQIjUIMi0r4M8joFVb9/7hgm0gA6V1S/f/MzMzMzMzMzMzMzMzMzMzMzMyLRbRQ6CdX/f9Zw4tUJAiNQgyLSrQzyOjdVf3/i0rwM8jo01X9/7iUm0gA6RtS/f/MzMzMzMzMzMzMzMzMzMzMi41M/v//g8EE6UzC/P+LjUz+//+DwQjpMcD8/4tUJAiNQgyLilT+//8zyOiJVf3/i0r8M8jof1X9/7jMm0gA6cdR/f/MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMi41A/v//g8EE6ezB/P+LjUD+//+DwQjp0b/8/4tUJAiNQgyLikj+//8zyOgpVf3/i0r8M8joH1X9/7gInEgA6WdR/f/MzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMi03wg8EE6Y/B/P+LTfCDwQjpd7/8/4tUJAiNQgyLSvgzyOjSVP3/uEScSADpGlH9/8zMzMzMzMzMzMzMzMzMzItF7FDo51X9/1nDi0XkUOjcVf3/WcOLVCQIjUIMi0rgM8joklT9/7iAnEgA6dpQ/f/MzMzMzMzMzMzMzMzMzMyLTeyDwQjpD8H8/4tUJAiNQgyLSvQzyOhdVP3/uLScSADppVD9/8zMzMzMzMzMzMyLRZxQ6HdV/f9Zw4tUJAiNQgyLSogzyOgtVP3/i0r8M8joI1T9/7jknEgA6WtQ/f/MzMzMzMzMzMzMzMzMzMzMjU3k6YW//P+NTeTpw7/8/4tUJAiNQgyLirD+//8zyOjlU/3/uFCdSADpLVD9/8zMzMzMzMzMzMzMzMzMzMzMzI1NsOnCv/z/jU3M6XZe/f+NjWz////pr7/8/41NiOljXv3/jY0o////6Zy//P+LVCQIjUIMi4qw/v//M8joh1P9/7iUnUgA6c9P/f/MzMzMzMzMzMzMzMzMzMzMzMzMzItNhOkeXv3/i1QkCI1CDItKjDPI6FBT/f+4ZJ5IAOmYT/3/zMzMzMzMzMzMzMzMzItNhOnuXf3/i1QkCI1CDItKjDPI6CBT/f+4lJ5IAOloT/3/zMzMzMzMzMzMzMzMzIuN6P7//+nIvvz/i1QkCI1CDIuK8P7//zPI6OpS/f+4xJ5IAOkyT/3/zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMyLTejpi778/4tUJAiNQgyLSvAzyOiwUv3/uPSeSADp+E79/8zMzMzMzMzMzMzMzMyLVCQIjUIMi4rM/v//M8johVL9/7hQn0gA6c1O/f/MzMzMzMzMzMzMzMzMzMzMzMyLVCQIjUIMi4pk////M8joVVL9/7i8n0gA6Z1O/f/MzMzMzMzMzMzMzMzMzMzMzMyNTdzptb38/41N3Onzvfz/i1QkCI1CDIuKYP7//zPI6BVS/f+4XKBIAOldTv3/zMzMzMzMzMzMzMzMzMzMzMzMjU3Y6fK9/P+LRdSD4AEPhAwAAACDZdT+i00I6dq9/P/Di1QkCI1CDIuKXP///zPI6MRR/f+4nKBIAOkMTv3/zMzMzMzMzMzMzMzMzMzMi1QkCI1CDItK7DPI6JpR/f+4kKJIAOniTf3/i03wg8EE6VM+//+LVCQIjUIMi0rsM8jodFH9/7gEqkgA6bxN/f+LTfDpDEH//4tN8IPBBOlYvfz/i03wg8Eg6U29/P+LTfCDwTzpQr38/4tN8IPBWOk3vfz/i1QkCI1CDItK7DPI6CVR/f+4KKpIAOltTf3/i03w6b1A//+LTfCDwQTpCb38/4tN8IPBIOn+vPz/i03wg8E86fO8/P+LTfCDwVjp6Lz8/4tUJAiNQgyLSuwzyOjWUP3/uHSqSADpHk39/41N1OnFvPz/i1QkCI1CDItKmDPI6LNQ/f+LSvwzyOipUP3/uMiqSADp8Uz9/41N1OmYvPz/i1QkCI1CDItKqDPI6IZQ/f+LSvwzyOh8UP3/uPSqSADpxEz9//91COihUf3/WcOLRfCD4AEPhAwAAACDZfD+jU3U6VG8/P/Di1QkCI1CDIuKXP///zPI6DtQ/f+4MKtIAOmDTP3/jU3U6Sq8/P+LVCQIjUIMi0qcM8joGFD9/4tK/DPI6A5Q/f+4XKtIAOlWTP3/jU3s6aY///+LVCQIjUIMi0rcM8jo60/9/7iYq0gA6TNM/f//dQjoEFH9/1nDi0Xwg+ABD4QMAAAAg2Xw/o1N1OnAu/z/w4tUJAiNQgyLilz///8zyOiqT/3/uPyrSADp8kv9/4tN8OnWJP//i1QkCI1CDItK7DPI6IdP/f+4KKxIAOnPS/3/jU3w6XQf//+LVCQIjUIMi0rsM8joZE/9/7hUrEgA6axL/f+NTezp/D7//4tUJAiNQgyLStwzyOhBT/3/uICsSADpiUv9/4tF8IPgAQ+EDwAAAINl8P6LTeyDwQTpfiD//8OLVCQIjUIMi0roM8joCk/9/7isrEgA6VJL/f+LTfDptVn9/4tUJAiNQgyLSuwzyOjnTv3/uNisSADpL0v9/41N2OnWuvz/i1QkCI1CDItKrDPI6MRO/f+4FK1IAOkMS/3/jU3Y6bO6/P+LVCQIjUIMi0qsM8jooU79/7iYrUgA6elK/f+NTdjpkLr8/4tUJAiNQgyLSqwzyOh+Tv3/uACuSADpxkr9/4tN8OkpWf3/i1QkCI1CDItK7DPI6FtO/f+4WK5IAOmjSv3/jU3w6fM9//+LVCQIjUIMi0rsM8joOE79/7iErkgA6YBK/f+LTfDp0B3//4tUJAiNQgyLSuwzyOgVTv3/uLCuSADpXUr9/4tN8OmtHf//i03wg8EY6fm5/P+LVCQIjUIMi0rsM8jo5039/7jkrkgA6S9K/f+NTfDpfz3///917OgET/3/WcOLVCQIjUIMi0roM8jouk39/7gYr0gA6QJK/f//dfDo3079/1nDi1QkCI1CDItK7DPI6JVN/f+4RK9IAOndSf3/zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMw=","dir":"C2S","dst_addr":"10.11.21.11","dst_ap":"10.11.21.11:445","dst_port":445,"gid":1,"iface":"/home/noah/pcaps/pcaps from malware-traffica-analysis.net/2019-11-21-Emotet-epoch-3-with-Trickbot-gtag-mor49-and-spambot-traffic.pcap","msg":"OS-WINDOWS Microsoft Windows raw WriteAndX InData pointer adjustment attempt","mpls":0,"pkt_gen":"stream_tcp","pkt_len":4160,"pkt_num":61571,"priority":1,"proto":"TCP","rev":1,"rule":"1:50626:1","service":"netbios-ssn","sid":50626,"src_addr":"10.11.21.101","src_ap":"10.11.21.101:50084","src_port":50084,"vlan":0,"timestamp":"11/21-18:01:50.061909"} {"seconds":1263690203,"action":"allow","class":"none","b64_data":"SFRUUC8xLjEgMjAwIE9LDQpEYXRlOiBTdW4sIDE3IEphbiAyMDEwIDAxOjAzOjIzIEdNVA0KQ29udGVudC1MZW5ndGg6IDkzDQpDb25uZWN0aW9uOiBLZWVwLUFsaXZlDQpDb250ZW50LVR5cGU6IGFwcGxpY2F0aW9uL29jdGV0LXN0cmVhbQ0KDQqIUD9i2EiKmsaSRMwz+KykbZuhBHl51gb+ncb3E+k56qaeXYeonZrAin2EYqRNqg40EDHjyzBj2rR+H6RvSHD4CznUGYujs3yybbSPm5ijXksL6/4xUrH2izVXw74=","dir":"S2C","dst_addr":"192.168.3.35","dst_ap":"192.168.3.35:1047","dst_port":1047,"eth_dst":"00:0C:29:92:E9:86","eth_len":287,"eth_src":"00:0C:29:B9:39:C3","eth_type":"0x800","gid":119,"iface":"/home/noah/samples//bredolab-sample.pcap","ip_id":10856,"ip_len":253,"msg":"(http_inspect) gzip decompression failed","mpls":0,"pkt_gen":"raw","pkt_len":273,"pkt_num":1612,"priority":3,"proto":"TCP","rev":1,"rule":"119:217:1","service":"http","sid":217,"src_addr":"89.160.20.114","src_ap":"89.160.20.114:80","src_port":80,"tcp_ack":3174915760,"tcp_flags":"***AP***","tcp_len":20,"tcp_seq":4204308887,"tcp_win":64903,"tos":0,"ttl":115,"vlan":0,"timestamp":"01/17-03:03:23.476194"} -{"seconds":1238569343,"action":"allow","class":"none","b64_data":"AgEGADPfV5sAAAAAAAAAAMCoAWcAAAAAAAAAAAAX8kPA0wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABjglNjNQECNgTAqAEBAQT///8AMwQAdqcAAwTAqAEBBghBIAVvQSAFcA8KY2ZsLnJyLmNvbRoCBdT/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=","dir":"C2S","dst_addr":"255.255.255.255","dst_ap":"255.255.255.255:68","dst_port":68,"eth_dst":"FF:FF:FF:FF:FF:FF","eth_len":590,"eth_src":"00:0F:66:80:D2:2A","eth_type":"0x800","gid":116,"iface":"/home/noah/samples//ConfickerB9hrs.pcap","ip_id":3913,"ip_len":556,"msg":"(ipv4) IPv4 packet to broadcast dest address","mpls":0,"pkt_gen":"raw","pkt_len":576,"pkt_num":689,"priority":3,"proto":"UDP","rev":1,"rule":"116:414:1","service":"unknown","sid":414,"src_addr":"192.168.1.1","src_ap":"192.168.1.1:67","src_port":67,"tos":0,"ttl":64,"udp_len":556,"vlan":0,"timestamp":"04/01-09:02:23.126173"} \ No newline at end of file +{"seconds":1238569343,"action":"allow","class":"none","b64_data":"AgEGADPfV5sAAAAAAAAAAMCoAWcAAAAAAAAAAAAX8kPA0wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABjglNjNQECNgTAqAEBAQT///8AMwQAdqcAAwTAqAEBBghBIAVvQSAFcA8KY2ZsLnJyLmNvbRoCBdT/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=","dir":"C2S","dst_addr":"255.255.255.255","dst_ap":"255.255.255.255:68","dst_port":68,"eth_dst":"FF:FF:FF:FF:FF:FF","eth_len":590,"eth_src":"00:0F:66:80:D2:2A","eth_type":"0x800","gid":116,"iface":"/home/noah/samples//ConfickerB9hrs.pcap","ip_id":3913,"ip_len":556,"msg":"(ipv4) IPv4 packet to broadcast dest address","mpls":0,"pkt_gen":"raw","pkt_len":576,"pkt_num":689,"priority":3,"proto":"UDP","rev":1,"rule":"116:414:1","service":"unknown","sid":414,"src_addr":"192.168.1.1","src_ap":"192.168.1.1:67","src_port":67,"tos":0,"ttl":64,"udp_len":556,"vlan":0,"timestamp":"04/01-09:02:23.126173"} +{"seconds":1331903482,"action":"allow","class":"none","dir":"C2S","dst_addr":"192.168.27.27","dst_ap":"192.168.27.27:0","eth_dst":"00:16:47:9D:F2:C2","eth_len":64,"eth_src":"BC:AE:C5:9E:F3:B6","eth_type":"0x8100","gid":116,"icmp_code":0,"icmp_id":17807,"icmp_seq":0,"icmp_type":8,"iface":"/home/noah/pcaps/maccdc2012_00000.pcap","ip_id":17433,"ip_len":8,"msg":"(icmp4) ICMP ping Nmap","mpls":0,"pkt_gen":"raw","pkt_len":28,"pkt_num":3217551,"priority":3,"proto":"ICMP","rev":1,"rule":"116:434:1","service":"unknown","sid":434,"src_addr":"192.168.202.110","src_ap":"192.168.202.110:0","tos":0,"ttl":40,"vlan":120,"timestamp":"03/16-15:11:22.800000"} \ No newline at end of file diff --git a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-json.log-expected.json b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-json.log-expected.json index 5975368b4e0..a2b105b7aea 100644 --- a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-json.log-expected.json +++ b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-json.log-expected.json @@ -327,6 +327,89 @@ "tags": [ "preserve_original_event" ] + }, + { + "@timestamp": "2022-03-16T15:11:22.800-05:00", + "destination": { + "address": "192.168.27.27", + "ip": "192.168.27.27", + "mac": "00-16-47-9D-F2-C2" + }, + "ecs": { + "version": "8.3.0" + }, + "event": { + "category": [ + "network" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "alert", + "original": "{\"seconds\":1331903482,\"action\":\"allow\",\"class\":\"none\",\"dir\":\"C2S\",\"dst_addr\":\"192.168.27.27\",\"dst_ap\":\"192.168.27.27:0\",\"eth_dst\":\"00:16:47:9D:F2:C2\",\"eth_len\":64,\"eth_src\":\"BC:AE:C5:9E:F3:B6\",\"eth_type\":\"0x8100\",\"gid\":116,\"icmp_code\":0,\"icmp_id\":17807,\"icmp_seq\":0,\"icmp_type\":8,\"iface\":\"/home/noah/pcaps/maccdc2012_00000.pcap\",\"ip_id\":17433,\"ip_len\":8,\"msg\":\"(icmp4) ICMP ping Nmap\",\"mpls\":0,\"pkt_gen\":\"raw\",\"pkt_len\":28,\"pkt_num\":3217551,\"priority\":3,\"proto\":\"ICMP\",\"rev\":1,\"rule\":\"116:434:1\",\"service\":\"unknown\",\"sid\":434,\"src_addr\":\"192.168.202.110\",\"src_ap\":\"192.168.202.110:0\",\"tos\":0,\"ttl\":40,\"vlan\":120,\"timestamp\":\"03/16-15:11:22.800000\"}", + "severity": 3, + "timezone": "America/Chicago", + "type": [ + "allowed" + ] + }, + "network": { + "bytes": 28, + "community_id": "1:SFVNlWDkKZ3WV2RMphV7s0dUMr0=", + "direction": "internal", + "iana_number": "1", + "packets": 3217551, + "transport": "icmp", + "type": "ipv4", + "vlan": { + "id": "120" + } + }, + "observer": { + "ingress": { + "interface": { + "name": "/home/noah/pcaps/maccdc2012_00000.pcap" + } + }, + "product": "ids", + "type": "ids", + "vendor": "snort" + }, + "related": { + "ip": [ + "192.168.202.110", + "192.168.27.27" + ] + }, + "rule": { + "category": "none", + "description": "(icmp4) ICMP ping Nmap", + "id": "434", + "version": "1" + }, + "snort": { + "eth": { + "length": 64 + }, + "gid": "116", + "icmp": { + "code": 0, + "id": 17807, + "seq": 0, + "type": 8 + }, + "ip": { + "id": 17433, + "tos": 0, + "ttl": 40 + } + }, + "source": { + "address": "192.168.202.110", + "ip": "192.168.202.110", + "mac": "BC-AE-C5-9E-F3-B6" + }, + "tags": [ + "preserve_original_event" + ] } ] } \ No newline at end of file diff --git a/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/json.yml b/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/json.yml index a251fe0f725..cf944010d5b 100644 --- a/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/json.yml +++ b/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/json.yml @@ -153,6 +153,12 @@ processors: target_field: network.protocol ignore_missing: true if: ctx.json?.service != 'unknown' + - convert: + field: json.vlan + target_field: network.vlan.id + type: string + ignore_missing: true + if: ctx.json?.vlan != 0 ## Other - convert: field: json.priority diff --git a/packages/snort/data_stream/log/fields/ecs.yml b/packages/snort/data_stream/log/fields/ecs.yml index 8c75b61e929..7b6f4a0b150 100644 --- a/packages/snort/data_stream/log/fields/ecs.yml +++ b/packages/snort/data_stream/log/fields/ecs.yml @@ -73,6 +73,8 @@ name: network.packets - external: ecs name: network.bytes +- external: ecs + name: network.vlan.id - external: ecs name: observer.ip - external: ecs diff --git a/packages/snort/docs/README.md b/packages/snort/docs/README.md index 4a070eec6b3..58b61ada02e 100644 --- a/packages/snort/docs/README.md +++ b/packages/snort/docs/README.md @@ -4,9 +4,9 @@ This integration is for [Snort](https://www.snort.org/). ## Compatibility -This module has been developed against Snort v2.9, but is expected to work -with other versions of Snort. This package is designed to read from the PFsense CSV output -and the Alert Fast output either via reading a local logfile or receiving messages via syslog +This module has been developed against Snort v2.9 and v3, but is expected to work +with other versions of Snort. This package is designed to read from the PFsense CSV output, +the Alert Fast output either via reading a local logfile or receiving messages via syslog and the Snort 3 JSON log file. ## Log @@ -194,6 +194,7 @@ An example event for `log` looks as following: | network.protocol | In the OSI Model this would be the Application Layer protocol. For example, `http`, `dns`, or `ssh`. The field value must be normalized to lowercase for querying. | keyword | | network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | | network.type | In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc The field value must be normalized to lowercase for querying. | keyword | +| network.vlan.id | VLAN ID as reported by the observer. | keyword | | observer.ingress.interface.name | Interface name as reported by the system. | keyword | | observer.ip | IP addresses of the observer. | ip | | observer.name | Custom name of the observer. This is a name that can be given to an observer. This can be helpful for example if multiple firewalls of the same model are used in an organization. If no custom name is needed, the field can be left empty. | keyword | @@ -211,7 +212,7 @@ An example event for `log` looks as following: | rule.version | The version / revision of the rule being used for analysis. | keyword | | snort.dgm.length | Length of | long | | snort.eth.length | Length of the Ethernet header and payload. | long | -| snort.gid | The gid keyword (generator id) is used to identify what part of Snort generates the event when a particular rule fires.dd | long | +| snort.gid | The gid keyword (generator id) is used to identify what part of Snort generates the event when a particular rule fires.dd | keyword | | snort.icmp.code | ICMP code. | long | | snort.icmp.id | ID of the echo request/reply | long | | snort.icmp.seq | ICMP sequence number. | long | From fc2545390aa93053bee93652eaa7a8d4a463d42e Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Tue, 2 Aug 2022 23:44:44 +0000 Subject: [PATCH 7/8] update generated files --- .../_dev/test/pipeline/test-log-json.log-expected.json | 10 +++++----- packages/snort/docs/README.md | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-json.log-expected.json b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-json.log-expected.json index a2b105b7aea..41365e50fe5 100644 --- a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-json.log-expected.json +++ b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-json.log-expected.json @@ -8,7 +8,7 @@ "mac": "52-54-00-1F-8A-1C" }, "ecs": { - "version": "8.3.0" + "version": "8.4.0" }, "event": { "category": [ @@ -88,7 +88,7 @@ "port": 445 }, "ecs": { - "version": "8.3.0" + "version": "8.4.0" }, "event": { "category": [ @@ -156,7 +156,7 @@ "port": 1047 }, "ecs": { - "version": "8.3.0" + "version": "8.4.0" }, "event": { "category": [ @@ -258,7 +258,7 @@ "port": 68 }, "ecs": { - "version": "8.3.0" + "version": "8.4.0" }, "event": { "category": [ @@ -336,7 +336,7 @@ "mac": "00-16-47-9D-F2-C2" }, "ecs": { - "version": "8.3.0" + "version": "8.4.0" }, "event": { "category": [ diff --git a/packages/snort/docs/README.md b/packages/snort/docs/README.md index 58b61ada02e..96fcb40cb90 100644 --- a/packages/snort/docs/README.md +++ b/packages/snort/docs/README.md @@ -188,7 +188,7 @@ An example event for `log` looks as following: | message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | | network.bytes | Total bytes transferred in both directions. If `source.bytes` and `destination.bytes` are known, `network.bytes` is their sum. | long | | network.community_id | A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows. Learn more at https://github.com/corelight/community-id-spec. | keyword | -| network.direction | Direction of the network traffic. Recommended values are: \* ingress \* egress \* inbound \* outbound \* internal \* external \* unknown When mapping events from a host-based monitoring context, populate this field from the host's point of view, using the values "ingress" or "egress". When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of the network perimeter, using the values "inbound", "outbound", "internal" or "external". Note that "internal" is not crossing perimeter boundaries, and is meant to describe communication between two hosts within the perimeter. Note also that "external" is meant to describe traffic between two hosts that are external to the perimeter. This could for example be useful for ISPs or VPN service providers. | keyword | +| network.direction | Direction of the network traffic. When mapping events from a host-based monitoring context, populate this field from the host's point of view, using the values "ingress" or "egress". When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of the network perimeter, using the values "inbound", "outbound", "internal" or "external". Note that "internal" is not crossing perimeter boundaries, and is meant to describe communication between two hosts within the perimeter. Note also that "external" is meant to describe traffic between two hosts that are external to the perimeter. This could for example be useful for ISPs or VPN service providers. | keyword | | network.iana_number | IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). Standardized list of protocols. This aligns well with NetFlow and sFlow related logs which use the IANA Protocol Number. | keyword | | network.packets | Total packets transferred in both directions. If `source.packets` and `destination.packets` are known, `network.packets` is their sum. | long | | network.protocol | In the OSI Model this would be the Application Layer protocol. For example, `http`, `dns`, or `ssh`. The field value must be normalized to lowercase for querying. | keyword | From 63c12d0892a19c39100315d621449948c95c93cc Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Wed, 3 Aug 2022 22:29:58 +0000 Subject: [PATCH 8/8] revert snort.gid back to long --- .../pipeline/test-log-csv.log-expected.json | 16 +++++++-------- .../pipeline/test-log-fast.log-expected.json | 20 +++++++++---------- .../pipeline/test-log-full.log-expected.json | 14 ++++++------- .../pipeline/test-log-json.log-expected.json | 10 +++++----- .../test-log-pfsense.log-expected.json | 6 +++--- .../test-log-syslog.log-expected.json | 6 +++--- .../elasticsearch/ingest_pipeline/json.yml | 2 +- .../ingest_pipeline/plaintext.yml | 4 ++-- .../snort/data_stream/log/fields/fields.yml | 2 +- packages/snort/docs/README.md | 2 +- 10 files changed, 41 insertions(+), 41 deletions(-) diff --git a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-csv.log-expected.json b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-csv.log-expected.json index fa8549f1417..6f7ad646f36 100644 --- a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-csv.log-expected.json +++ b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-csv.log-expected.json @@ -50,7 +50,7 @@ "eth": { "length": 114 }, - "gid": "1", + "gid": 1, "ip": { "id": 55665, "length": 102400, @@ -124,7 +124,7 @@ "eth": { "length": 114 }, - "gid": "1", + "gid": 1, "ip": { "id": 55666, "length": 102400, @@ -198,7 +198,7 @@ "eth": { "length": 193 }, - "gid": "1", + "gid": 1, "ip": { "id": 56094, "length": 183296, @@ -269,7 +269,7 @@ "eth": { "length": 177 }, - "gid": "1", + "gid": 1, "ip": { "id": 26112, "length": 166912, @@ -340,7 +340,7 @@ "eth": { "length": 63 }, - "gid": "1", + "gid": 1, "ip": { "id": 37712, "length": 50176, @@ -422,7 +422,7 @@ "eth": { "length": 98 }, - "gid": "1", + "gid": 1, "icmp": { "code": 0, "id": 83, @@ -497,7 +497,7 @@ "eth": { "length": 98 }, - "gid": "1", + "gid": 1, "icmp": { "code": 0, "id": 83, @@ -596,7 +596,7 @@ "eth": { "length": 98 }, - "gid": "1", + "gid": 1, "icmp": { "code": 0, "id": 83, diff --git a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-fast.log-expected.json b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-fast.log-expected.json index d545ef9ff41..62fdf9decbb 100644 --- a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-fast.log-expected.json +++ b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-fast.log-expected.json @@ -45,7 +45,7 @@ "version": "8" }, "snort": { - "gid": "1" + "gid": 1 }, "source": { "address": "0.0.0.0", @@ -113,7 +113,7 @@ "version": "2" }, "snort": { - "gid": "1" + "gid": 1 }, "source": { "address": "192.168.88.10", @@ -178,7 +178,7 @@ "version": "3" }, "snort": { - "gid": "1" + "gid": 1 }, "source": { "address": "175.16.199.1", @@ -257,7 +257,7 @@ "version": "6" }, "snort": { - "gid": "1" + "gid": 1 }, "source": { "address": "192.168.15.10", @@ -313,7 +313,7 @@ "version": "2" }, "snort": { - "gid": "1" + "gid": 1 }, "source": { "address": "175.16.199.1", @@ -380,7 +380,7 @@ "version": "5" }, "snort": { - "gid": "1" + "gid": 1 }, "source": { "address": "175.16.199.1", @@ -447,7 +447,7 @@ "version": "0" }, "snort": { - "gid": "1" + "gid": 1 }, "source": { "address": "175.16.199.1", @@ -515,7 +515,7 @@ "version": "0" }, "snort": { - "gid": "1" + "gid": 1 }, "source": { "address": "175.16.199.1", @@ -594,7 +594,7 @@ "version": "0" }, "snort": { - "gid": "1" + "gid": 1 }, "source": { "address": "10.100.10.190", @@ -648,7 +648,7 @@ "version": "0" }, "snort": { - "gid": "1" + "gid": 1 }, "source": { "address": "175.16.199.1", diff --git a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-full.log-expected.json b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-full.log-expected.json index 9028a9aa2e6..d06aa6e0155 100644 --- a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-full.log-expected.json +++ b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-full.log-expected.json @@ -47,7 +47,7 @@ "dgm": { "length": 108 }, - "gid": "1", + "gid": 1, "ip": { "flags": "DF", "id": 53730, @@ -119,7 +119,7 @@ "dgm": { "length": 40 }, - "gid": "1", + "gid": 1, "ip": { "flags": "DF", "id": 53731, @@ -192,7 +192,7 @@ "dgm": { "length": 83 }, - "gid": "1", + "gid": 1, "ip": { "id": 58363, "length": 20, @@ -271,7 +271,7 @@ "dgm": { "length": 84 }, - "gid": "1", + "gid": 1, "icmp": { "code": 0, "id": 101, @@ -342,7 +342,7 @@ "dgm": { "length": 40 }, - "gid": "1", + "gid": 1, "ip": { "flags": "DF", "id": 61472, @@ -414,7 +414,7 @@ "dgm": { "length": 84 }, - "gid": "1", + "gid": 1, "icmp": { "code": 0, "id": 101, @@ -496,7 +496,7 @@ "dgm": { "length": 153 }, - "gid": "1", + "gid": 1, "ip": { "id": 33955, "length": 20, diff --git a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-json.log-expected.json b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-json.log-expected.json index 41365e50fe5..40e2f71287c 100644 --- a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-json.log-expected.json +++ b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-json.log-expected.json @@ -58,7 +58,7 @@ "eth": { "length": 98 }, - "gid": "1", + "gid": 1, "icmp": { "code": 0, "id": 5203, @@ -136,7 +136,7 @@ "version": "1" }, "snort": { - "gid": "1" + "gid": 1 }, "source": { "address": "10.11.21.101", @@ -207,7 +207,7 @@ "eth": { "length": 287 }, - "gid": "119", + "gid": 119, "ip": { "id": 10856, "tos": 0, @@ -308,7 +308,7 @@ "eth": { "length": 590 }, - "gid": "116", + "gid": 116, "ip": { "id": 3913, "tos": 0, @@ -389,7 +389,7 @@ "eth": { "length": 64 }, - "gid": "116", + "gid": 116, "icmp": { "code": 0, "id": 17807, diff --git a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-pfsense.log-expected.json b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-pfsense.log-expected.json index a41878a3f3a..464513b18d4 100644 --- a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-pfsense.log-expected.json +++ b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-pfsense.log-expected.json @@ -59,7 +59,7 @@ "version": "68499" }, "snort": { - "gid": "1", + "gid": 1, "ip": { "id": 54321 } @@ -144,7 +144,7 @@ "version": "4" }, "snort": { - "gid": "1", + "gid": 1, "ip": { "id": 54925 } @@ -228,7 +228,7 @@ "version": "3" }, "snort": { - "gid": "1", + "gid": 1, "ip": { "id": 40546 } diff --git a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-syslog.log-expected.json b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-syslog.log-expected.json index e792296de7d..2c9504fffc3 100644 --- a/packages/snort/data_stream/log/_dev/test/pipeline/test-log-syslog.log-expected.json +++ b/packages/snort/data_stream/log/_dev/test/pipeline/test-log-syslog.log-expected.json @@ -49,7 +49,7 @@ "version": "0" }, "snort": { - "gid": "1" + "gid": 1 }, "source": { "address": "10.150.10.44", @@ -108,7 +108,7 @@ "version": "0" }, "snort": { - "gid": "1" + "gid": 1 }, "source": { "address": "10.50.20.59", @@ -179,7 +179,7 @@ "version": "0" }, "snort": { - "gid": "1" + "gid": 1 }, "source": { "address": "10.50.10.88", diff --git a/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/json.yml b/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/json.yml index cf944010d5b..73b5ff08334 100644 --- a/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/json.yml +++ b/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/json.yml @@ -68,7 +68,7 @@ processors: - convert: field: json.gid target_field: snort.gid - type: string + type: long ignore_missing: true ## ICMP - convert: diff --git a/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/plaintext.yml b/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/plaintext.yml index 2dc453ab57c..eaa2bb40c0b 100644 --- a/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/plaintext.yml +++ b/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/plaintext.yml @@ -17,8 +17,8 @@ processors: pattern_definitions: SNORT_DATE: '%{MONTHNUM}/%{MONTHDAY}(/%{YEAR})?-%{TIME}' SEP: '(\[\*\*\])' - CSV_START: '%{SNORT_DATE:_tmp.timestamp}(%{SPACE})?,%{NONNEGINT:snort.gid},%{NONNEGINT:rule.id},%{NONNEGINT:rule.version},("?%{DATA:rule.description}"?|),%{WORD:network.transport},%{IP:source.address},(%{POSINT:source.port:long}|),%{IP:destination.address},(%{POSINT:destination.port:long}|)' - HEADER: '%{SPACE}\[%{NONNEGINT:snort.gid}:%{NONNEGINT:rule.id}:%{NONNEGINT:rule.version}\]%{SPACE}%{DATA:rule.description}%{SPACE}' + CSV_START: '%{SNORT_DATE:_tmp.timestamp}(%{SPACE})?,%{NONNEGINT:snort.gid:long},%{NONNEGINT:rule.id},%{NONNEGINT:rule.version},("?%{DATA:rule.description}"?|),%{WORD:network.transport},%{IP:source.address},(%{POSINT:source.port:long}|),%{IP:destination.address},(%{POSINT:destination.port:long}|)' + HEADER: '%{SPACE}\[%{NONNEGINT:snort.gid:long}:%{NONNEGINT:rule.id}:%{NONNEGINT:rule.version}\]%{SPACE}%{DATA:rule.description}%{SPACE}' FAST_HEADER: '%{SEP}%{HEADER}%{SEP}' FAST_BODY: '%{SPACE}%{CLASSIFICATION} %{PRIORITY} \{%{WORD:network.transport}\} %{IP:source.address}(:%{POSINT:source.port:long}|) -> %{IP:destination.address}(:%{POSINT:destination.port:long}|)' TCP_DATA: '(%{NOTSPACE:snort.tcp.flags})%{SPACE}(Seq: %{BASE16NUM:snort.tcp.seq})%{SPACE}(Ack: %{BASE16NUM:snort.tcp.ack})%{SPACE}(Win: %{BASE16NUM:snort.tcp.window})%{SPACE}(TcpLen: %{NONNEGINT:snort.tcp.length:long})' diff --git a/packages/snort/data_stream/log/fields/fields.yml b/packages/snort/data_stream/log/fields/fields.yml index 02a43e8eb95..f06cbb5c5f1 100644 --- a/packages/snort/data_stream/log/fields/fields.yml +++ b/packages/snort/data_stream/log/fields/fields.yml @@ -1,5 +1,5 @@ - name: snort.gid - type: keyword + type: long description: > The gid keyword (generator id) is used to identify what part of Snort generates the event when a particular rule fires.dd diff --git a/packages/snort/docs/README.md b/packages/snort/docs/README.md index 96fcb40cb90..d39736c888f 100644 --- a/packages/snort/docs/README.md +++ b/packages/snort/docs/README.md @@ -212,7 +212,7 @@ An example event for `log` looks as following: | rule.version | The version / revision of the rule being used for analysis. | keyword | | snort.dgm.length | Length of | long | | snort.eth.length | Length of the Ethernet header and payload. | long | -| snort.gid | The gid keyword (generator id) is used to identify what part of Snort generates the event when a particular rule fires.dd | keyword | +| snort.gid | The gid keyword (generator id) is used to identify what part of Snort generates the event when a particular rule fires.dd | long | | snort.icmp.code | ICMP code. | long | | snort.icmp.id | ID of the echo request/reply | long | | snort.icmp.seq | ICMP sequence number. | long |