diff --git a/packages/pfsense/changelog.yml b/packages/pfsense/changelog.yml index 3e57e61f6de..4d0036f42c5 100644 --- a/packages/pfsense/changelog.yml +++ b/packages/pfsense/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.3.0" + changes: + - description: Add DHCPv6 support + type: enhancement + link: https://github.com/elastic/integrations/pull/3815 - version: "1.2.0" changes: - description: Update package to ECS 8.4.0 diff --git a/packages/pfsense/data_stream/log/_dev/test/pipeline/test-opensense.log-expected.json b/packages/pfsense/data_stream/log/_dev/test/pipeline/test-opensense.log-expected.json index a2ca7a56643..ceae1ac5bb8 100644 --- a/packages/pfsense/data_stream/log/_dev/test/pipeline/test-opensense.log-expected.json +++ b/packages/pfsense/data_stream/log/_dev/test/pipeline/test-opensense.log-expected.json @@ -49,7 +49,10 @@ "direction": "inbound", "iana_number": "6", "transport": "tcp", - "type": "ipv4" + "type": "ipv4", + "vlan": { + "id": "12" + } }, "observer": { "ingress": { @@ -144,7 +147,10 @@ "direction": "inbound", "iana_number": "17", "transport": "udp", - "type": "ipv6" + "type": "ipv6", + "vlan": { + "id": "27" + } }, "observer": { "ingress": { @@ -194,11 +200,7 @@ { "@timestamp": "2022-12-31T22:06:16.000-04:00", "client": { - "mac": "4C-55-41-A0-FA-99", - "port": 68 - }, - "destination": { - "port": 67 + "mac": "4C-55-41-A0-FA-99" }, "ecs": { "version": "8.4.0" @@ -226,7 +228,10 @@ "message": "DHCPDISCOVER from 4c:55:41:a0:fa:99 via eth0.60", "network": { "protocol": "dhcp", - "transport": "udp" + "transport": "udp", + "vlan": { + "id": "60" + } }, "observer": { "ingress": { @@ -245,12 +250,8 @@ "name": "dhcpd", "pid": 40152 }, - "server": { - "port": 67 - }, "source": { - "mac": "4C-55-41-A0-FA-99", - "port": 68 + "mac": "4C-55-41-A0-FA-99" }, "tags": [ "preserve_original_event" @@ -296,7 +297,8 @@ }, "message": "[26931:0] info: 192.168.1.1 api.opensubtitles.org. A IN", "network": { - "protocol": "dns" + "protocol": "dns", + "type": "ipv4" }, "observer": { "name": "firewall.opnsense.net", diff --git a/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-bsd.log-expected.json b/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-bsd.log-expected.json index 2bf6192fb0d..9fa3166825f 100644 --- a/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-bsd.log-expected.json +++ b/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-bsd.log-expected.json @@ -49,7 +49,10 @@ "direction": "inbound", "iana_number": "6", "transport": "tcp", - "type": "ipv4" + "type": "ipv4", + "vlan": { + "id": "12" + } }, "observer": { "ingress": { @@ -155,7 +158,10 @@ "direction": "inbound", "iana_number": "6", "transport": "tcp", - "type": "ipv4" + "type": "ipv4", + "vlan": { + "id": "12" + } }, "observer": { "ingress": { @@ -261,7 +267,10 @@ "direction": "inbound", "iana_number": "17", "transport": "udp", - "type": "ipv4" + "type": "ipv4", + "vlan": { + "id": "27" + } }, "observer": { "ingress": { @@ -358,7 +367,10 @@ "direction": "inbound", "iana_number": "6", "transport": "tcp", - "type": "ipv4" + "type": "ipv4", + "vlan": { + "id": "12" + } }, "observer": { "ingress": { @@ -464,7 +476,10 @@ "direction": "inbound", "iana_number": "6", "transport": "tcp", - "type": "ipv4" + "type": "ipv4", + "vlan": { + "id": "12" + } }, "observer": { "ingress": { @@ -570,7 +585,10 @@ "direction": "inbound", "iana_number": "6", "transport": "tcp", - "type": "ipv4" + "type": "ipv4", + "vlan": { + "id": "12" + } }, "observer": { "ingress": { @@ -664,7 +682,10 @@ "direction": "inbound", "iana_number": "17", "transport": "udp", - "type": "ipv6" + "type": "ipv6", + "vlan": { + "id": "27" + } }, "observer": { "ingress": { @@ -759,7 +780,10 @@ "direction": "inbound", "iana_number": "6", "transport": "tcp", - "type": "ipv4" + "type": "ipv4", + "vlan": { + "id": "12" + } }, "observer": { "ingress": { @@ -865,7 +889,10 @@ "direction": "inbound", "iana_number": "6", "transport": "tcp", - "type": "ipv4" + "type": "ipv4", + "vlan": { + "id": "12" + } }, "observer": { "ingress": { @@ -971,7 +998,10 @@ "direction": "inbound", "iana_number": "6", "transport": "tcp", - "type": "ipv4" + "type": "ipv4", + "vlan": { + "id": "12" + } }, "observer": { "ingress": { @@ -1077,7 +1107,10 @@ "direction": "inbound", "iana_number": "6", "transport": "tcp", - "type": "ipv4" + "type": "ipv4", + "vlan": { + "id": "12" + } }, "observer": { "ingress": { @@ -1183,7 +1216,10 @@ "direction": "inbound", "iana_number": "6", "transport": "tcp", - "type": "ipv4" + "type": "ipv4", + "vlan": { + "id": "12" + } }, "observer": { "ingress": { @@ -1289,7 +1325,10 @@ "direction": "inbound", "iana_number": "6", "transport": "tcp", - "type": "ipv4" + "type": "ipv4", + "vlan": { + "id": "12" + } }, "observer": { "ingress": { @@ -1395,7 +1434,10 @@ "direction": "inbound", "iana_number": "6", "transport": "tcp", - "type": "ipv4" + "type": "ipv4", + "vlan": { + "id": "12" + } }, "observer": { "ingress": { @@ -1489,7 +1531,10 @@ "direction": "inbound", "iana_number": "17", "transport": "udp", - "type": "ipv4" + "type": "ipv4", + "vlan": { + "id": "27" + } }, "observer": { "ingress": { @@ -1586,7 +1631,10 @@ "direction": "inbound", "iana_number": "6", "transport": "tcp", - "type": "ipv4" + "type": "ipv4", + "vlan": { + "id": "12" + } }, "observer": { "ingress": { @@ -1692,7 +1740,10 @@ "iana_number": "2", "packets": 8, "transport": "igmp", - "type": "ipv4" + "type": "ipv4", + "vlan": { + "id": "12" + } }, "observer": { "ingress": { @@ -1784,7 +1835,10 @@ "direction": "inbound", "iana_number": "1", "transport": "icmp", - "type": "ipv4" + "type": "ipv4", + "vlan": { + "id": "10" + } }, "observer": { "ingress": { @@ -1869,7 +1923,10 @@ "direction": "inbound", "iana_number": "1", "transport": "icmp", - "type": "ipv4" + "type": "ipv4", + "vlan": { + "id": "15" + } }, "observer": { "ingress": { diff --git a/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-dhcp.log b/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-dhcp.log index efe6247e1b7..3552b239f07 100644 --- a/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-dhcp.log +++ b/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-dhcp.log @@ -2,6 +2,28 @@ <190>Jul 4 09:39:41 dhcpd[64305]: DHCPOFFER on 10.150.60.56 to 4c:55:41:a0:fa:99 (computer-name) via eth0.60 <190>Jul 4 09:39:41 dhcpd[64305]: DHCPREQUEST for 10.150.60.56 (10.150.60.1) from 4c:55:41:a0:fa:99 (computer-name) via eth0.60 <190>Jul 4 09:39:41 dhcpd[64305]: DHCPACK on 10.150.60.56 to 4c:55:41:a0:fa:99 (computer-name) via eth0.60 -<14>Jul 4 09:39:41 dhcpleases[18688]: Sending HUP signal to dns daemon(17883) <13>Jul 4 09:40:40 dhclient[89531]: RENEW -<13>Jul 4 09:40:40 dhclient[89547]: Creating resolv.conf \ No newline at end of file +<13>Jul 4 09:40:40 dhclient[89547]: Creating resolv.conf +<190>Jul 4 09:39:41 dhcpd[64305]: Listening on BPF/igb1.15/5f:a5:54:63:cc:1f/10.50.11.0/25 +<190>Jul 4 09:39:41 dhcpd[64305]: Sending on BPF/igb1.15/5f:a5:54:63:cc:1f/10.50.11.0/25 +<190>Jul 23 18:07:11 dhcp6c[10256]: restarting +<190>Jul 23 18:07:11 dhcp6c[10256]: Start address release +<190>Jul 23 18:07:11 dhcp6c[10256]: Sending Release +<190>Jul 23 18:07:11 dhcp6c[10256]: remove an address 2a02:cf40:72dc:dd12:7378:913c:b42e:099c/128 on igb0 +<190>Jul 23 18:07:11 dhcp6c[10256]: Start address release +<190>Jul 23 18:07:11 dhcp6c[10256]: Sending Release +<190>Jul 23 18:07:11 dhcp6c[10256]: dhcp6c Received RELEASE +<190>Jul 23 18:07:11 dhcp6c[10256]: status code: success +<190>Jul 23 18:07:21 dhcp6c[10256]: add an address 2a02:cf40:72dc:dd12:7378:913c:b42e:099c/128 on igb0 +<190>Jul 23 18:11:57 dhcpd[6555]: Listening on Socket/6/igb1.10/9f21:c09b:6837:e2f::/64 +<190>Jul 23 18:11:57 dhcpd[6555]: Sending on Socket/6/igb1.10/9f21:c09b:6837:e2f::/64 +<190>Jul 23 18:11:57 dhcpd[6555]: Server starting service. +<190>Jul 23 18:11:58 dhcpd[6555]: Solicit message from fe80::e6c9:2b22:f9db:bfad port 546, transaction ID 0x3C21A200 +<190>Jul 23 18:11:58 dhcpd[6555]: Picking pool address 2a02:cf40:38d6:c4db:cafb:917b:44ec:c873 +<190>Jul 23 18:11:58 dhcpd[6555]: Advertise NA: address 2a02:cf40:38d6:c4db:cafb:917b:44ec:c873 to client with duid 00:02:00:00:0d:e9:30:30:44:30:39:45:2d:31 iaid = -1620146908 valid for 7200 seconds +<190>Jul 23 18:11:58 dhcpd[6555]: Sending Advertise to fe80::e6c9:2b22:f9db:bfad port 546 +<190>Jul 23 18:11:58 dhcpd[6555]: Request message from fe80::e6c9:2b22:f9db:bfad port 546, transaction ID 0x36D30200 +<190>Jul 23 18:11:58 dhcpd[6555]: Reply NA: address 2a02:cf40:38d6:c4db:cafb:917b:44ec:c873 to client with duid 00:02:00:00:0d:e9:30:30:44:30:39:45:2d:31 iaid = -1620146908 valid for 7200 seconds +<190>Jul 23 18:11:58 dhcpd[6555]: Sending Reply to fe80::e6c9:2b22:f9db:bfad port 546 +<190>Jul 23 18:12:00 dhcpd[6555]: Information-request message from fe80::208:0138:95bb:a400 port 546, transaction ID 0x9A75EE00 +<190>Jul 23 18:12:00 dhcpd[6555]: Reusing lease for: 2a02:cf40:38d6:c4db:cafb:917b:44ec:c873, age 265 secs < 25%, sending shortened lifetimes - preferred: 4235, valid 6935 \ No newline at end of file diff --git a/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-dhcp.log-expected.json b/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-dhcp.log-expected.json index 6fa8cb11d75..2c842c7dbf8 100644 --- a/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-dhcp.log-expected.json +++ b/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-dhcp.log-expected.json @@ -3,11 +3,7 @@ { "@timestamp": "2022-07-04T09:39:40.000-04:00", "client": { - "mac": "4C-55-41-A0-FA-99", - "port": 68 - }, - "destination": { - "port": 67 + "mac": "4C-55-41-A0-FA-99" }, "ecs": { "version": "8.4.0" @@ -35,7 +31,10 @@ "message": "DHCPDISCOVER from 4c:55:41:a0:fa:99 via eth0.60", "network": { "protocol": "dhcp", - "transport": "udp" + "transport": "udp", + "vlan": { + "id": "60" + } }, "observer": { "ingress": { @@ -53,12 +52,8 @@ "name": "dhcpd", "pid": 64305 }, - "server": { - "port": 67 - }, "source": { - "mac": "4C-55-41-A0-FA-99", - "port": 68 + "mac": "4C-55-41-A0-FA-99" }, "tags": [ "preserve_original_event" @@ -69,11 +64,7 @@ "client": { "address": "10.150.60.56", "ip": "10.150.60.56", - "mac": "4C-55-41-A0-FA-99", - "port": 68 - }, - "destination": { - "port": 67 + "mac": "4C-55-41-A0-FA-99" }, "ecs": { "version": "8.4.0" @@ -101,7 +92,11 @@ "message": "DHCPOFFER on 10.150.60.56 to 4c:55:41:a0:fa:99 (computer-name) via eth0.60", "network": { "protocol": "dhcp", - "transport": "udp" + "transport": "udp", + "type": "ipv4", + "vlan": { + "id": "60" + } }, "observer": { "ingress": { @@ -129,14 +124,10 @@ "10.150.60.56" ] }, - "server": { - "port": 67 - }, "source": { "address": "10.150.60.56", "ip": "10.150.60.56", - "mac": "4C-55-41-A0-FA-99", - "port": 68 + "mac": "4C-55-41-A0-FA-99" }, "tags": [ "preserve_original_event" @@ -147,13 +138,11 @@ "client": { "address": "10.150.60.56", "ip": "10.150.60.56", - "mac": "4C-55-41-A0-FA-99", - "port": 68 + "mac": "4C-55-41-A0-FA-99" }, "destination": { "address": "10.150.60.1", - "ip": "10.150.60.1", - "port": 67 + "ip": "10.150.60.1" }, "ecs": { "version": "8.4.0" @@ -180,9 +169,12 @@ }, "message": "DHCPREQUEST for 10.150.60.56 (10.150.60.1) from 4c:55:41:a0:fa:99 (computer-name) via eth0.60", "network": { - "community_id": "1:0xgDC3HDYkAlzU6VWJp5W5LrTPE=", "protocol": "dhcp", - "transport": "udp" + "transport": "udp", + "type": "ipv4", + "vlan": { + "id": "60" + } }, "observer": { "ingress": { @@ -207,20 +199,18 @@ }, "related": { "ip": [ - "10.150.60.56", - "10.150.60.1" + "10.150.60.1", + "10.150.60.56" ] }, "server": { "address": "10.150.60.1", - "ip": "10.150.60.1", - "port": 67 + "ip": "10.150.60.1" }, "source": { "address": "10.150.60.56", "ip": "10.150.60.56", - "mac": "4C-55-41-A0-FA-99", - "port": 68 + "mac": "4C-55-41-A0-FA-99" }, "tags": [ "preserve_original_event" @@ -231,11 +221,7 @@ "client": { "address": "10.150.60.56", "ip": "10.150.60.56", - "mac": "4C-55-41-A0-FA-99", - "port": 68 - }, - "destination": { - "port": 67 + "mac": "4C-55-41-A0-FA-99" }, "ecs": { "version": "8.4.0" @@ -263,7 +249,11 @@ "message": "DHCPACK on 10.150.60.56 to 4c:55:41:a0:fa:99 (computer-name) via eth0.60", "network": { "protocol": "dhcp", - "transport": "udp" + "transport": "udp", + "type": "ipv4", + "vlan": { + "id": "60" + } }, "observer": { "ingress": { @@ -291,21 +281,1358 @@ "10.150.60.56" ] }, - "server": { - "port": 67 - }, "source": { "address": "10.150.60.56", "ip": "10.150.60.56", - "mac": "4C-55-41-A0-FA-99", - "port": 68 + "mac": "4C-55-41-A0-FA-99" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2022-07-04T09:40:40.000-04:00", + "ecs": { + "version": "8.4.0" + }, + "event": { + "category": [ + "network" + ], + "kind": "event", + "original": "\u003c13\u003eJul 4 09:40:40 dhclient[89531]: RENEW", + "provider": "dhclient", + "timezone": "-04:00", + "type": [ + "connection", + "protocol", + "info" + ] + }, + "log": { + "syslog": { + "priority": 13 + } + }, + "message": "RENEW", + "network": { + "protocol": "dhcp", + "transport": "udp" + }, + "observer": { + "type": "firewall", + "vendor": "netgate" + }, + "process": { + "name": "dhclient", + "pid": 89531 + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2022-07-04T09:40:40.000-04:00", + "ecs": { + "version": "8.4.0" + }, + "event": { + "category": [ + "network" + ], + "kind": "event", + "original": "\u003c13\u003eJul 4 09:40:40 dhclient[89547]: Creating resolv.conf", + "provider": "dhclient", + "timezone": "-04:00", + "type": [ + "connection", + "protocol", + "info" + ] + }, + "log": { + "syslog": { + "priority": 13 + } + }, + "message": "Creating resolv.conf", + "network": { + "protocol": "dhcp", + "transport": "udp" + }, + "observer": { + "type": "firewall", + "vendor": "netgate" + }, + "process": { + "name": "dhclient", + "pid": 89547 + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2022-07-04T09:39:41.000-04:00", + "destination": { + "mac": "5F-A5-54-63-CC-1F" + }, + "ecs": { + "version": "8.4.0" + }, + "event": { + "action": "listening-on-bpf", + "category": [ + "network" + ], + "kind": "event", + "original": "\u003c190\u003eJul 4 09:39:41 dhcpd[64305]: Listening on BPF/igb1.15/5f:a5:54:63:cc:1f/10.50.11.0/25", + "provider": "dhcpd", + "timezone": "-04:00", + "type": [ + "connection", + "protocol", + "info" + ] + }, + "log": { + "syslog": { + "priority": 190 + } + }, + "message": "Listening on BPF/igb1.15/5f:a5:54:63:cc:1f/10.50.11.0/25", + "network": { + "protocol": "dhcp", + "transport": "udp", + "vlan": { + "id": "15" + } + }, + "observer": { + "ingress": { + "interface": { + "name": "igb1.15" + }, + "vlan": { + "id": "15" + } + }, + "type": "firewall", + "vendor": "netgate" + }, + "pfsense": { + "dhcp": { + "subnet": "10.50.11.0/25" + } + }, + "process": { + "name": "dhcpd", + "pid": 64305 + }, + "server": { + "mac": "5F-A5-54-63-CC-1F" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2022-07-04T09:39:41.000-04:00", + "destination": { + "mac": "5F-A5-54-63-CC-1F" + }, + "ecs": { + "version": "8.4.0" + }, + "event": { + "action": "sending-on-bpf", + "category": [ + "network" + ], + "kind": "event", + "original": "\u003c190\u003eJul 4 09:39:41 dhcpd[64305]: Sending on BPF/igb1.15/5f:a5:54:63:cc:1f/10.50.11.0/25", + "provider": "dhcpd", + "timezone": "-04:00", + "type": [ + "connection", + "protocol", + "info" + ] + }, + "log": { + "syslog": { + "priority": 190 + } + }, + "message": "Sending on BPF/igb1.15/5f:a5:54:63:cc:1f/10.50.11.0/25", + "network": { + "protocol": "dhcp", + "transport": "udp", + "vlan": { + "id": "15" + } + }, + "observer": { + "ingress": { + "interface": { + "name": "igb1.15" + }, + "vlan": { + "id": "15" + } + }, + "type": "firewall", + "vendor": "netgate" + }, + "pfsense": { + "dhcp": { + "subnet": "10.50.11.0/25" + } + }, + "process": { + "name": "dhcpd", + "pid": 64305 + }, + "server": { + "mac": "5F-A5-54-63-CC-1F" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2022-07-23T18:07:11.000-04:00", + "ecs": { + "version": "8.4.0" + }, + "event": { + "category": [ + "network" + ], + "kind": "event", + "original": "\u003c190\u003eJul 23 18:07:11 dhcp6c[10256]: restarting", + "provider": "dhcp6c", + "timezone": "-04:00", + "type": [ + "connection", + "protocol", + "info" + ] + }, + "log": { + "syslog": { + "priority": 190 + } + }, + "message": "restarting", + "network": { + "protocol": "dhcpv6", + "transport": "udp" + }, + "observer": { + "type": "firewall", + "vendor": "netgate" + }, + "process": { + "name": "dhcp6c", + "pid": 10256 }, "tags": [ "preserve_original_event" ] }, - null, - null, - null + { + "@timestamp": "2022-07-23T18:07:11.000-04:00", + "ecs": { + "version": "8.4.0" + }, + "event": { + "category": [ + "network" + ], + "kind": "event", + "original": "\u003c190\u003eJul 23 18:07:11 dhcp6c[10256]: Start address release", + "provider": "dhcp6c", + "timezone": "-04:00", + "type": [ + "connection", + "protocol", + "info" + ] + }, + "log": { + "syslog": { + "priority": 190 + } + }, + "message": "Start address release", + "network": { + "protocol": "dhcpv6", + "transport": "udp" + }, + "observer": { + "type": "firewall", + "vendor": "netgate" + }, + "process": { + "name": "dhcp6c", + "pid": 10256 + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2022-07-23T18:07:11.000-04:00", + "ecs": { + "version": "8.4.0" + }, + "event": { + "category": [ + "network" + ], + "kind": "event", + "original": "\u003c190\u003eJul 23 18:07:11 dhcp6c[10256]: Sending Release", + "provider": "dhcp6c", + "timezone": "-04:00", + "type": [ + "connection", + "protocol", + "info" + ] + }, + "log": { + "syslog": { + "priority": 190 + } + }, + "message": "Sending Release", + "network": { + "protocol": "dhcpv6", + "transport": "udp" + }, + "observer": { + "type": "firewall", + "vendor": "netgate" + }, + "process": { + "name": "dhcp6c", + "pid": 10256 + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2022-07-23T18:07:11.000-04:00", + "client": { + "address": "2a02:cf40:72dc:dd12:7378:913c:b42e:099c", + "ip": "2a02:cf40:72dc:dd12:7378:913c:b42e:099c" + }, + "ecs": { + "version": "8.4.0" + }, + "event": { + "action": "remove-an-address", + "category": [ + "network" + ], + "kind": "event", + "original": "\u003c190\u003eJul 23 18:07:11 dhcp6c[10256]: remove an address 2a02:cf40:72dc:dd12:7378:913c:b42e:099c/128 on igb0", + "provider": "dhcp6c", + "timezone": "-04:00", + "type": [ + "connection", + "protocol", + "info" + ] + }, + "log": { + "syslog": { + "priority": 190 + } + }, + "message": "remove an address 2a02:cf40:72dc:dd12:7378:913c:b42e:099c/128 on igb0", + "network": { + "protocol": "dhcpv6", + "transport": "udp", + "type": "ipv6" + }, + "observer": { + "ingress": { + "interface": { + "name": "igb0" + } + }, + "type": "firewall", + "vendor": "netgate" + }, + "process": { + "name": "dhcp6c", + "pid": 10256 + }, + "related": { + "ip": [ + "2a02:cf40:72dc:dd12:7378:913c:b42e:099c" + ] + }, + "source": { + "address": "2a02:cf40:72dc:dd12:7378:913c:b42e:099c", + "geo": { + "continent_name": "Europe", + "country_iso_code": "NO", + "country_name": "Norway", + "location": { + "lat": 62.0, + "lon": 10.0 + } + }, + "ip": "2a02:cf40:72dc:dd12:7378:913c:b42e:099c" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2022-07-23T18:07:11.000-04:00", + "ecs": { + "version": "8.4.0" + }, + "event": { + "category": [ + "network" + ], + "kind": "event", + "original": "\u003c190\u003eJul 23 18:07:11 dhcp6c[10256]: Start address release", + "provider": "dhcp6c", + "timezone": "-04:00", + "type": [ + "connection", + "protocol", + "info" + ] + }, + "log": { + "syslog": { + "priority": 190 + } + }, + "message": "Start address release", + "network": { + "protocol": "dhcpv6", + "transport": "udp" + }, + "observer": { + "type": "firewall", + "vendor": "netgate" + }, + "process": { + "name": "dhcp6c", + "pid": 10256 + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2022-07-23T18:07:11.000-04:00", + "ecs": { + "version": "8.4.0" + }, + "event": { + "category": [ + "network" + ], + "kind": "event", + "original": "\u003c190\u003eJul 23 18:07:11 dhcp6c[10256]: Sending Release", + "provider": "dhcp6c", + "timezone": "-04:00", + "type": [ + "connection", + "protocol", + "info" + ] + }, + "log": { + "syslog": { + "priority": 190 + } + }, + "message": "Sending Release", + "network": { + "protocol": "dhcpv6", + "transport": "udp" + }, + "observer": { + "type": "firewall", + "vendor": "netgate" + }, + "process": { + "name": "dhcp6c", + "pid": 10256 + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2022-07-23T18:07:11.000-04:00", + "ecs": { + "version": "8.4.0" + }, + "event": { + "category": [ + "network" + ], + "kind": "event", + "original": "\u003c190\u003eJul 23 18:07:11 dhcp6c[10256]: dhcp6c Received RELEASE", + "provider": "dhcp6c", + "timezone": "-04:00", + "type": [ + "connection", + "protocol", + "info" + ] + }, + "log": { + "syslog": { + "priority": 190 + } + }, + "message": "dhcp6c Received RELEASE", + "network": { + "protocol": "dhcpv6", + "transport": "udp" + }, + "observer": { + "type": "firewall", + "vendor": "netgate" + }, + "process": { + "name": "dhcp6c", + "pid": 10256 + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2022-07-23T18:07:11.000-04:00", + "ecs": { + "version": "8.4.0" + }, + "event": { + "category": [ + "network" + ], + "kind": "event", + "original": "\u003c190\u003eJul 23 18:07:11 dhcp6c[10256]: status code: success", + "provider": "dhcp6c", + "timezone": "-04:00", + "type": [ + "connection", + "protocol", + "info" + ] + }, + "log": { + "syslog": { + "priority": 190 + } + }, + "message": "status code: success", + "network": { + "protocol": "dhcpv6", + "transport": "udp" + }, + "observer": { + "type": "firewall", + "vendor": "netgate" + }, + "process": { + "name": "dhcp6c", + "pid": 10256 + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2022-07-23T18:07:21.000-04:00", + "client": { + "address": "2a02:cf40:72dc:dd12:7378:913c:b42e:099c", + "ip": "2a02:cf40:72dc:dd12:7378:913c:b42e:099c" + }, + "ecs": { + "version": "8.4.0" + }, + "event": { + "action": "add-an-address", + "category": [ + "network" + ], + "kind": "event", + "original": "\u003c190\u003eJul 23 18:07:21 dhcp6c[10256]: add an address 2a02:cf40:72dc:dd12:7378:913c:b42e:099c/128 on igb0", + "provider": "dhcp6c", + "timezone": "-04:00", + "type": [ + "connection", + "protocol", + "info" + ] + }, + "log": { + "syslog": { + "priority": 190 + } + }, + "message": "add an address 2a02:cf40:72dc:dd12:7378:913c:b42e:099c/128 on igb0", + "network": { + "protocol": "dhcpv6", + "transport": "udp", + "type": "ipv6" + }, + "observer": { + "ingress": { + "interface": { + "name": "igb0" + } + }, + "type": "firewall", + "vendor": "netgate" + }, + "process": { + "name": "dhcp6c", + "pid": 10256 + }, + "related": { + "ip": [ + "2a02:cf40:72dc:dd12:7378:913c:b42e:099c" + ] + }, + "source": { + "address": "2a02:cf40:72dc:dd12:7378:913c:b42e:099c", + "geo": { + "continent_name": "Europe", + "country_iso_code": "NO", + "country_name": "Norway", + "location": { + "lat": 62.0, + "lon": 10.0 + } + }, + "ip": "2a02:cf40:72dc:dd12:7378:913c:b42e:099c" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2022-07-23T18:11:57.000-04:00", + "ecs": { + "version": "8.4.0" + }, + "event": { + "category": [ + "network" + ], + "kind": "event", + "original": "\u003c190\u003eJul 23 18:11:57 dhcpd[6555]: Listening on Socket/6/igb1.10/9f21:c09b:6837:e2f::/64", + "provider": "dhcpd", + "timezone": "-04:00", + "type": [ + "connection", + "protocol", + "info" + ] + }, + "log": { + "syslog": { + "priority": 190 + } + }, + "message": "Listening on Socket/6/igb1.10/9f21:c09b:6837:e2f::/64", + "network": { + "protocol": "dhcp", + "transport": "udp" + }, + "observer": { + "type": "firewall", + "vendor": "netgate" + }, + "process": { + "name": "dhcpd", + "pid": 6555 + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2022-07-23T18:11:57.000-04:00", + "ecs": { + "version": "8.4.0" + }, + "event": { + "category": [ + "network" + ], + "kind": "event", + "original": "\u003c190\u003eJul 23 18:11:57 dhcpd[6555]: Sending on Socket/6/igb1.10/9f21:c09b:6837:e2f::/64", + "provider": "dhcpd", + "timezone": "-04:00", + "type": [ + "connection", + "protocol", + "info" + ] + }, + "log": { + "syslog": { + "priority": 190 + } + }, + "message": "Sending on Socket/6/igb1.10/9f21:c09b:6837:e2f::/64", + "network": { + "protocol": "dhcp", + "transport": "udp" + }, + "observer": { + "type": "firewall", + "vendor": "netgate" + }, + "process": { + "name": "dhcpd", + "pid": 6555 + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2022-07-23T18:11:57.000-04:00", + "ecs": { + "version": "8.4.0" + }, + "event": { + "category": [ + "network" + ], + "kind": "event", + "original": "\u003c190\u003eJul 23 18:11:57 dhcpd[6555]: Server starting service.", + "provider": "dhcpd", + "timezone": "-04:00", + "type": [ + "connection", + "protocol", + "info" + ] + }, + "log": { + "syslog": { + "priority": 190 + } + }, + "message": "Server starting service.", + "network": { + "protocol": "dhcp", + "transport": "udp" + }, + "observer": { + "type": "firewall", + "vendor": "netgate" + }, + "process": { + "name": "dhcpd", + "pid": 6555 + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2022-07-23T18:11:58.000-04:00", + "client": { + "address": "fe80::e6c9:2b22:f9db:bfad", + "ip": "fe80::e6c9:2b22:f9db:bfad", + "port": 546 + }, + "ecs": { + "version": "8.4.0" + }, + "event": { + "action": "solicit-message", + "category": [ + "network" + ], + "kind": "event", + "original": "\u003c190\u003eJul 23 18:11:58 dhcpd[6555]: Solicit message from fe80::e6c9:2b22:f9db:bfad port 546, transaction ID 0x3C21A200", + "provider": "dhcpd", + "timezone": "-04:00", + "type": [ + "connection", + "protocol", + "info" + ] + }, + "log": { + "syslog": { + "priority": 190 + } + }, + "message": "Solicit message from fe80::e6c9:2b22:f9db:bfad port 546, transaction ID 0x3C21A200", + "network": { + "protocol": "dhcpv6", + "transport": "udp", + "type": "ipv6" + }, + "observer": { + "type": "firewall", + "vendor": "netgate" + }, + "pfsense": { + "dhcp": { + "transaction_id": "0x3C21A200" + } + }, + "process": { + "name": "dhcpd", + "pid": 6555 + }, + "related": { + "ip": [ + "fe80::e6c9:2b22:f9db:bfad" + ] + }, + "source": { + "address": "fe80::e6c9:2b22:f9db:bfad", + "ip": "fe80::e6c9:2b22:f9db:bfad", + "port": 546 + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2022-07-23T18:11:58.000-04:00", + "client": { + "address": "2a02:cf40:38d6:c4db:cafb:917b:44ec:c873", + "ip": "2a02:cf40:38d6:c4db:cafb:917b:44ec:c873" + }, + "ecs": { + "version": "8.4.0" + }, + "event": { + "action": "picking-pool-address", + "category": [ + "network" + ], + "kind": "event", + "original": "\u003c190\u003eJul 23 18:11:58 dhcpd[6555]: Picking pool address 2a02:cf40:38d6:c4db:cafb:917b:44ec:c873", + "provider": "dhcpd", + "timezone": "-04:00", + "type": [ + "connection", + "protocol", + "info" + ] + }, + "log": { + "syslog": { + "priority": 190 + } + }, + "message": "Picking pool address 2a02:cf40:38d6:c4db:cafb:917b:44ec:c873", + "network": { + "protocol": "dhcpv6", + "transport": "udp", + "type": "ipv6" + }, + "observer": { + "type": "firewall", + "vendor": "netgate" + }, + "process": { + "name": "dhcpd", + "pid": 6555 + }, + "related": { + "ip": [ + "2a02:cf40:38d6:c4db:cafb:917b:44ec:c873" + ] + }, + "source": { + "address": "2a02:cf40:38d6:c4db:cafb:917b:44ec:c873", + "geo": { + "continent_name": "Europe", + "country_iso_code": "NO", + "country_name": "Norway", + "location": { + "lat": 62.0, + "lon": 10.0 + } + }, + "ip": "2a02:cf40:38d6:c4db:cafb:917b:44ec:c873" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2022-07-23T18:11:58.000-04:00", + "client": { + "address": "2a02:cf40:38d6:c4db:cafb:917b:44ec:c873", + "ip": "2a02:cf40:38d6:c4db:cafb:917b:44ec:c873" + }, + "ecs": { + "version": "8.4.0" + }, + "event": { + "action": "advertise-na", + "category": [ + "network" + ], + "kind": "event", + "original": "\u003c190\u003eJul 23 18:11:58 dhcpd[6555]: Advertise NA: address 2a02:cf40:38d6:c4db:cafb:917b:44ec:c873 to client with duid 00:02:00:00:0d:e9:30:30:44:30:39:45:2d:31 iaid = -1620146908 valid for 7200 seconds", + "provider": "dhcpd", + "timezone": "-04:00", + "type": [ + "connection", + "protocol", + "info" + ] + }, + "log": { + "syslog": { + "priority": 190 + } + }, + "message": "Advertise NA: address 2a02:cf40:38d6:c4db:cafb:917b:44ec:c873 to client with duid 00:02:00:00:0d:e9:30:30:44:30:39:45:2d:31 iaid = -1620146908 valid for 7200 seconds", + "network": { + "protocol": "dhcpv6", + "transport": "udp", + "type": "ipv6" + }, + "observer": { + "type": "firewall", + "vendor": "netgate" + }, + "pfsense": { + "dhcp": { + "duid": "00:02:00:00:0d:e9:30:30:44:30:39:45:2d:31", + "iaid": "1620146908", + "lease_time": 7200 + } + }, + "process": { + "name": "dhcpd", + "pid": 6555 + }, + "related": { + "ip": [ + "2a02:cf40:38d6:c4db:cafb:917b:44ec:c873" + ] + }, + "source": { + "address": "2a02:cf40:38d6:c4db:cafb:917b:44ec:c873", + "geo": { + "continent_name": "Europe", + "country_iso_code": "NO", + "country_name": "Norway", + "location": { + "lat": 62.0, + "lon": 10.0 + } + }, + "ip": "2a02:cf40:38d6:c4db:cafb:917b:44ec:c873" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2022-07-23T18:11:58.000-04:00", + "client": { + "address": "fe80::e6c9:2b22:f9db:bfad", + "ip": "fe80::e6c9:2b22:f9db:bfad", + "port": 546 + }, + "ecs": { + "version": "8.4.0" + }, + "event": { + "action": "sending-advertise", + "category": [ + "network" + ], + "kind": "event", + "original": "\u003c190\u003eJul 23 18:11:58 dhcpd[6555]: Sending Advertise to fe80::e6c9:2b22:f9db:bfad port 546", + "provider": "dhcpd", + "timezone": "-04:00", + "type": [ + "connection", + "protocol", + "info" + ] + }, + "log": { + "syslog": { + "priority": 190 + } + }, + "message": "Sending Advertise to fe80::e6c9:2b22:f9db:bfad port 546", + "network": { + "protocol": "dhcpv6", + "transport": "udp", + "type": "ipv6" + }, + "observer": { + "type": "firewall", + "vendor": "netgate" + }, + "process": { + "name": "dhcpd", + "pid": 6555 + }, + "related": { + "ip": [ + "fe80::e6c9:2b22:f9db:bfad" + ] + }, + "source": { + "address": "fe80::e6c9:2b22:f9db:bfad", + "ip": "fe80::e6c9:2b22:f9db:bfad", + "port": 546 + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2022-07-23T18:11:58.000-04:00", + "client": { + "address": "fe80::e6c9:2b22:f9db:bfad", + "ip": "fe80::e6c9:2b22:f9db:bfad", + "port": 546 + }, + "ecs": { + "version": "8.4.0" + }, + "event": { + "action": "request-message", + "category": [ + "network" + ], + "kind": "event", + "original": "\u003c190\u003eJul 23 18:11:58 dhcpd[6555]: Request message from fe80::e6c9:2b22:f9db:bfad port 546, transaction ID 0x36D30200", + "provider": "dhcpd", + "timezone": "-04:00", + "type": [ + "connection", + "protocol", + "info" + ] + }, + "log": { + "syslog": { + "priority": 190 + } + }, + "message": "Request message from fe80::e6c9:2b22:f9db:bfad port 546, transaction ID 0x36D30200", + "network": { + "protocol": "dhcpv6", + "transport": "udp", + "type": "ipv6" + }, + "observer": { + "type": "firewall", + "vendor": "netgate" + }, + "pfsense": { + "dhcp": { + "transaction_id": "0x36D30200" + } + }, + "process": { + "name": "dhcpd", + "pid": 6555 + }, + "related": { + "ip": [ + "fe80::e6c9:2b22:f9db:bfad" + ] + }, + "source": { + "address": "fe80::e6c9:2b22:f9db:bfad", + "ip": "fe80::e6c9:2b22:f9db:bfad", + "port": 546 + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2022-07-23T18:11:58.000-04:00", + "client": { + "address": "2a02:cf40:38d6:c4db:cafb:917b:44ec:c873", + "ip": "2a02:cf40:38d6:c4db:cafb:917b:44ec:c873" + }, + "ecs": { + "version": "8.4.0" + }, + "event": { + "action": "reply-na", + "category": [ + "network" + ], + "kind": "event", + "original": "\u003c190\u003eJul 23 18:11:58 dhcpd[6555]: Reply NA: address 2a02:cf40:38d6:c4db:cafb:917b:44ec:c873 to client with duid 00:02:00:00:0d:e9:30:30:44:30:39:45:2d:31 iaid = -1620146908 valid for 7200 seconds", + "provider": "dhcpd", + "timezone": "-04:00", + "type": [ + "connection", + "protocol", + "info" + ] + }, + "log": { + "syslog": { + "priority": 190 + } + }, + "message": "Reply NA: address 2a02:cf40:38d6:c4db:cafb:917b:44ec:c873 to client with duid 00:02:00:00:0d:e9:30:30:44:30:39:45:2d:31 iaid = -1620146908 valid for 7200 seconds", + "network": { + "protocol": "dhcpv6", + "transport": "udp", + "type": "ipv6" + }, + "observer": { + "type": "firewall", + "vendor": "netgate" + }, + "pfsense": { + "dhcp": { + "duid": "00:02:00:00:0d:e9:30:30:44:30:39:45:2d:31", + "iaid": "1620146908", + "lease_time": 7200 + } + }, + "process": { + "name": "dhcpd", + "pid": 6555 + }, + "related": { + "ip": [ + "2a02:cf40:38d6:c4db:cafb:917b:44ec:c873" + ] + }, + "source": { + "address": "2a02:cf40:38d6:c4db:cafb:917b:44ec:c873", + "geo": { + "continent_name": "Europe", + "country_iso_code": "NO", + "country_name": "Norway", + "location": { + "lat": 62.0, + "lon": 10.0 + } + }, + "ip": "2a02:cf40:38d6:c4db:cafb:917b:44ec:c873" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2022-07-23T18:11:58.000-04:00", + "client": { + "address": "fe80::e6c9:2b22:f9db:bfad", + "ip": "fe80::e6c9:2b22:f9db:bfad", + "port": 546 + }, + "ecs": { + "version": "8.4.0" + }, + "event": { + "action": "sending-reply", + "category": [ + "network" + ], + "kind": "event", + "original": "\u003c190\u003eJul 23 18:11:58 dhcpd[6555]: Sending Reply to fe80::e6c9:2b22:f9db:bfad port 546", + "provider": "dhcpd", + "timezone": "-04:00", + "type": [ + "connection", + "protocol", + "info" + ] + }, + "log": { + "syslog": { + "priority": 190 + } + }, + "message": "Sending Reply to fe80::e6c9:2b22:f9db:bfad port 546", + "network": { + "protocol": "dhcpv6", + "transport": "udp", + "type": "ipv6" + }, + "observer": { + "type": "firewall", + "vendor": "netgate" + }, + "process": { + "name": "dhcpd", + "pid": 6555 + }, + "related": { + "ip": [ + "fe80::e6c9:2b22:f9db:bfad" + ] + }, + "source": { + "address": "fe80::e6c9:2b22:f9db:bfad", + "ip": "fe80::e6c9:2b22:f9db:bfad", + "port": 546 + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2022-07-23T18:12:00.000-04:00", + "client": { + "address": "fe80::208:0138:95bb:a400", + "ip": "fe80::208:0138:95bb:a400", + "port": 546 + }, + "ecs": { + "version": "8.4.0" + }, + "event": { + "action": "information-request-message", + "category": [ + "network" + ], + "kind": "event", + "original": "\u003c190\u003eJul 23 18:12:00 dhcpd[6555]: Information-request message from fe80::208:0138:95bb:a400 port 546, transaction ID 0x9A75EE00", + "provider": "dhcpd", + "timezone": "-04:00", + "type": [ + "connection", + "protocol", + "info" + ] + }, + "log": { + "syslog": { + "priority": 190 + } + }, + "message": "Information-request message from fe80::208:0138:95bb:a400 port 546, transaction ID 0x9A75EE00", + "network": { + "protocol": "dhcpv6", + "transport": "udp", + "type": "ipv6" + }, + "observer": { + "type": "firewall", + "vendor": "netgate" + }, + "pfsense": { + "dhcp": { + "transaction_id": "0x9A75EE00" + } + }, + "process": { + "name": "dhcpd", + "pid": 6555 + }, + "related": { + "ip": [ + "fe80::208:0138:95bb:a400" + ] + }, + "source": { + "address": "fe80::208:0138:95bb:a400", + "ip": "fe80::208:0138:95bb:a400", + "port": 546 + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2022-07-23T18:12:00.000-04:00", + "client": { + "address": "2a02:cf40:38d6:c4db:cafb:917b:44ec:c873", + "ip": "2a02:cf40:38d6:c4db:cafb:917b:44ec:c873" + }, + "ecs": { + "version": "8.4.0" + }, + "event": { + "action": "reusing-lease", + "category": [ + "network" + ], + "kind": "event", + "original": "\u003c190\u003eJul 23 18:12:00 dhcpd[6555]: Reusing lease for: 2a02:cf40:38d6:c4db:cafb:917b:44ec:c873, age 265 secs \u003c 25%, sending shortened lifetimes - preferred: 4235, valid 6935", + "provider": "dhcpd", + "timezone": "-04:00", + "type": [ + "connection", + "protocol", + "info" + ] + }, + "log": { + "syslog": { + "priority": 190 + } + }, + "message": "Reusing lease for: 2a02:cf40:38d6:c4db:cafb:917b:44ec:c873, age 265 secs \u003c 25%, sending shortened lifetimes - preferred: 4235, valid 6935", + "network": { + "protocol": "dhcpv6", + "transport": "udp", + "type": "ipv6" + }, + "observer": { + "type": "firewall", + "vendor": "netgate" + }, + "pfsense": { + "dhcp": { + "age": 265 + } + }, + "process": { + "name": "dhcpd", + "pid": 6555 + }, + "related": { + "ip": [ + "2a02:cf40:38d6:c4db:cafb:917b:44ec:c873" + ] + }, + "source": { + "address": "2a02:cf40:38d6:c4db:cafb:917b:44ec:c873", + "geo": { + "continent_name": "Europe", + "country_iso_code": "NO", + "country_name": "Norway", + "location": { + "lat": 62.0, + "lon": 10.0 + } + }, + "ip": "2a02:cf40:38d6:c4db:cafb:917b:44ec:c873" + }, + "tags": [ + "preserve_original_event" + ] + } ] } \ No newline at end of file diff --git a/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-haproxy.log-expected.json b/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-haproxy.log-expected.json index ce21f623661..9dcea52a625 100644 --- a/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-haproxy.log-expected.json +++ b/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-haproxy.log-expected.json @@ -58,6 +58,9 @@ } }, "message": "10.87.93.55:59607 [15/Aug/2021:16:15:18.502] TestFrontend~ TestBackend/TestServer 0/0/0/2/2 400 182 - - ---- 2/2/0/1/0 0/0 \"GET /favicon.ico HTTP/1.1\" ", + "network": { + "type": "ipv4" + }, "observer": { "type": "firewall", "vendor": "netgate" @@ -143,6 +146,9 @@ } }, "message": "10.87.93.55:59607 [15/Aug/2021:16:15:18.407] TestFrontend~ TestBackend/TestServer 0/0/0/3/3 400 182 - - ---- 2/2/0/1/0 0/0 \"GET /login HTTP/1.1\" ", + "network": { + "type": "ipv4" + }, "observer": { "type": "firewall", "vendor": "netgate" @@ -223,6 +229,9 @@ } }, "message": "10.87.93.55:58722 [15/Aug/2021:16:15:10.549] TestFrontend~ TestBackend/\u003cNOSRV\u003e -1/-1/-1/-1/30014 408 212 - - cR-- 2/2/0/0/0 0/0 \"\u003cBADREQ\u003e\" ", + "network": { + "type": "ipv4" + }, "observer": { "type": "firewall", "vendor": "netgate" @@ -303,6 +312,9 @@ } }, "message": "10.0.200.110:50578 [13/Jun/2022:20:53:10.208] TestFrontend TestBackend_ipvANY/TestServer 0/0/0/433/537 200 65018 - - ---- 5/5/0/1/0 0/0 \"GET /static/fonts/roboto/Roboto-Bold.woff2 HTTP/1.1\"", + "network": { + "type": "ipv4" + }, "observer": { "name": "pfSense", "type": "firewall", @@ -367,6 +379,9 @@ } }, "message": "10.0.200.110:50611 [13/Jun/2022:20:56:55.187] TestFrontend-copy TestBackend_ipvANY/TestServer 0/0/204 366 -- 2/2/1/1/0 0/0", + "network": { + "type": "ipv4" + }, "observer": { "name": "pfSense", "type": "firewall", @@ -402,8 +417,7 @@ }, "event": { "category": [ - "web", - "network" + "web" ], "kind": "event", "original": "\u003c134\u003eJun 13 20:53:49 pfSense haproxy[80917]: Connect from 10.50.11.19:50583 to 192.168.75.211:80 (ACME/HTTP)", @@ -423,6 +437,9 @@ } }, "message": "Connect from 10.50.11.19:50583 to 192.168.75.211:80 (ACME/HTTP)", + "network": { + "type": "ipv4" + }, "observer": { "name": "pfSense", "type": "firewall", @@ -470,9 +487,6 @@ "version": "8.4.0" }, "event": { - "category": [ - "network" - ], "kind": "event", "original": "\u003c134\u003e1 2022-06-14T13:01:42.000000+02:00 haproxy.local haproxy 48723 - - 175.16.199.10:63605 [14/Jun/2022:12:41:42.440] URLS-merged/67.43.156.15:443: Timeout during SSL handshake", "provider": "haproxy", @@ -492,6 +506,9 @@ } }, "message": "175.16.199.10:63605 [14/Jun/2022:12:41:42.440] URLS-merged/67.43.156.15:443: Timeout during SSL handshake", + "network": { + "type": "ipv4" + }, "observer": { "name": "haproxy.local", "type": "firewall", diff --git a/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-ipsec.log-expected.json b/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-ipsec.log-expected.json index 997981b8351..1d79dfdd5ab 100644 --- a/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-ipsec.log-expected.json +++ b/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-ipsec.log-expected.json @@ -906,7 +906,8 @@ "message": "14[NET] \u003ccon1000|11\u003e sending packet: from 175.16.199.1[500] to 175.16.199.1[500] (464 bytes)", "network": { "bytes": 464, - "protocol": "ipsec" + "protocol": "ipsec", + "type": "ipv4" }, "observer": { "name": "pfSense.example.com", diff --git a/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-openvpn.log-expected.json b/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-openvpn.log-expected.json index 7eda33382b1..ca87c7a5c4f 100644 --- a/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-openvpn.log-expected.json +++ b/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-openvpn.log-expected.json @@ -24,7 +24,8 @@ }, "message": "10.170.120.149:37849 peer info: IV_VER=3.git:released:662eae9a:Release", "network": { - "protocol": "openvpn" + "protocol": "openvpn", + "type": "ipv4" }, "observer": { "type": "firewall", @@ -77,7 +78,8 @@ }, "message": "10.170.120.149:37849 peer info: IV_PLAT=android", "network": { - "protocol": "openvpn" + "protocol": "openvpn", + "type": "ipv4" }, "observer": { "type": "firewall", @@ -130,7 +132,8 @@ }, "message": "10.170.120.149:37849 peer info: IV_NCP=2", "network": { - "protocol": "openvpn" + "protocol": "openvpn", + "type": "ipv4" }, "observer": { "type": "firewall", @@ -183,7 +186,8 @@ }, "message": "10.170.120.149:37849 peer info: IV_TCPNL=1", "network": { - "protocol": "openvpn" + "protocol": "openvpn", + "type": "ipv4" }, "observer": { "type": "firewall", @@ -236,7 +240,8 @@ }, "message": "10.170.120.149:37849 peer info: IV_GUI_VER=net.openvpn.connect.android_3.2.4-5891", "network": { - "protocol": "openvpn" + "protocol": "openvpn", + "type": "ipv4" }, "observer": { "type": "firewall", @@ -289,7 +294,8 @@ }, "message": "10.170.120.149:37849 peer info: IV_PROTO=2", "network": { - "protocol": "openvpn" + "protocol": "openvpn", + "type": "ipv4" }, "observer": { "type": "firewall", @@ -342,7 +348,8 @@ }, "message": "10.170.120.149:37849 peer info: IV_SSO=openurl", "network": { - "protocol": "openvpn" + "protocol": "openvpn", + "type": "ipv4" }, "observer": { "type": "firewall", @@ -396,7 +403,8 @@ }, "message": "10.170.120.149:37849 [bob] Peer Connection Initiated with [AF_INET]10.170.120.149:37849", "network": { - "protocol": "openvpn" + "protocol": "openvpn", + "type": "ipv4" }, "observer": { "type": "firewall", @@ -433,8 +441,8 @@ }, "event": { "category": [ - "network", - "authentication" + "authentication", + "network" ], "kind": "event", "original": "\u003c37\u003eJul 3 21:42:57 openvpn[19830]: user 'bob' authenticated", @@ -494,7 +502,8 @@ }, "message": "bob/10.170.120.149:37849 MULTI_sva: pool returned IPv4=10.170.170.2, IPv6=(Not enabled)", "network": { - "protocol": "openvpn" + "protocol": "openvpn", + "type": "ipv4" }, "observer": { "type": "firewall", @@ -553,7 +562,8 @@ }, "message": "TLS Error: cannot locate HMAC in incoming packet from [AF_INET]175.16.199.1:34745", "network": { - "protocol": "openvpn" + "protocol": "openvpn", + "type": "ipv4" }, "observer": { "name": "pfSense.example.com", @@ -597,8 +607,8 @@ }, "event": { "category": [ - "network", - "authentication" + "authentication", + "network" ], "kind": "event", "original": "\u003c36\u003e1 2021-07-03T22:40:38.477134-05:00 pfSense.example.com openvpn 68813 - - user 'bob' could not authenticate.", diff --git a/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-squid.log-expected.json b/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-squid.log-expected.json index 7e0509842f3..b1a2703dc7d 100644 --- a/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-squid.log-expected.json +++ b/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-squid.log-expected.json @@ -23,8 +23,7 @@ }, "event": { "category": [ - "web", - "network" + "web" ], "kind": "event", "original": "\u003c166\u003eAug 21 22:08:13 myfirewall.my-domain.tld (squid-1)[6802]: [1598040493.253 325](tel:1598040493.253 325) 175.16.199.1 TCP_MISS/304 2912 GET https://github.com/3ilson/pfelk/file-list/master - HIER_DIRECT/81.2.69.145 -", @@ -47,6 +46,9 @@ } }, "message": "[1598040493.253 325](tel:1598040493.253 325) 175.16.199.1 TCP_MISS/304 2912 GET https://github.com/3ilson/pfelk/file-list/master - HIER_DIRECT/81.2.69.145 -", + "network": { + "type": "ipv4" + }, "observer": { "name": "myfirewall.my-domain.tld", "type": "firewall", diff --git a/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-syslog.log-expected.json b/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-syslog.log-expected.json index 1f35c917606..e3d344ca1d8 100644 --- a/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-syslog.log-expected.json +++ b/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-syslog.log-expected.json @@ -49,7 +49,10 @@ "direction": "inbound", "iana_number": "6", "transport": "tcp", - "type": "ipv4" + "type": "ipv4", + "vlan": { + "id": "12" + } }, "observer": { "ingress": { @@ -156,7 +159,10 @@ "direction": "inbound", "iana_number": "6", "transport": "tcp", - "type": "ipv4" + "type": "ipv4", + "vlan": { + "id": "12" + } }, "observer": { "ingress": { @@ -251,7 +257,10 @@ "direction": "inbound", "iana_number": "17", "transport": "udp", - "type": "ipv4" + "type": "ipv4", + "vlan": { + "id": "27" + } }, "observer": { "ingress": { @@ -349,7 +358,10 @@ "direction": "inbound", "iana_number": "6", "transport": "tcp", - "type": "ipv4" + "type": "ipv4", + "vlan": { + "id": "12" + } }, "observer": { "ingress": { @@ -456,7 +468,10 @@ "direction": "inbound", "iana_number": "6", "transport": "tcp", - "type": "ipv4" + "type": "ipv4", + "vlan": { + "id": "12" + } }, "observer": { "ingress": { @@ -551,7 +566,10 @@ "direction": "inbound", "iana_number": "6", "transport": "tcp", - "type": "ipv4" + "type": "ipv4", + "vlan": { + "id": "12" + } }, "observer": { "ingress": { @@ -662,7 +680,10 @@ "direction": "inbound", "iana_number": "6", "transport": "tcp", - "type": "ipv4" + "type": "ipv4", + "vlan": { + "id": "12" + } }, "observer": { "ingress": { @@ -769,7 +790,10 @@ "direction": "inbound", "iana_number": "6", "transport": "tcp", - "type": "ipv4" + "type": "ipv4", + "vlan": { + "id": "12" + } }, "observer": { "ingress": { @@ -864,7 +888,10 @@ "direction": "inbound", "iana_number": "17", "transport": "udp", - "type": "ipv4" + "type": "ipv4", + "vlan": { + "id": "27" + } }, "observer": { "ingress": { @@ -950,7 +977,10 @@ "direction": "inbound", "iana_number": "17", "transport": "udp", - "type": "ipv4" + "type": "ipv4", + "vlan": { + "id": "27" + } }, "observer": { "ingress": { @@ -1048,7 +1078,10 @@ "direction": "inbound", "iana_number": "6", "transport": "tcp", - "type": "ipv4" + "type": "ipv4", + "vlan": { + "id": "12" + } }, "observer": { "ingress": { @@ -1155,7 +1188,10 @@ "direction": "inbound", "iana_number": "6", "transport": "tcp", - "type": "ipv4" + "type": "ipv4", + "vlan": { + "id": "12" + } }, "observer": { "ingress": { @@ -1262,7 +1298,10 @@ "direction": "inbound", "iana_number": "17", "transport": "udp", - "type": "ipv4" + "type": "ipv4", + "vlan": { + "id": "40" + } }, "observer": { "ingress": { @@ -1348,7 +1387,10 @@ "iana_number": "2", "packets": 8, "transport": "igmp", - "type": "ipv4" + "type": "ipv4", + "vlan": { + "id": "50" + } }, "observer": { "ingress": { diff --git a/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-unbound.log-expected.json b/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-unbound.log-expected.json index d509705ea8c..504a163358c 100644 --- a/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-unbound.log-expected.json +++ b/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-unbound.log-expected.json @@ -40,7 +40,8 @@ }, "message": "[26931:0] info: 192.168.1.1 api.opensubtitles.org. A IN", "network": { - "protocol": "dns" + "protocol": "dns", + "type": "ipv4" }, "observer": { "type": "firewall", @@ -103,7 +104,8 @@ }, "message": "[26931:2] info: 172.16.33.2 clients4.google.com. A IN", "network": { - "protocol": "dns" + "protocol": "dns", + "type": "ipv4" }, "observer": { "type": "firewall", diff --git a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 8c2056dd5f8..297430504e4 100644 --- a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -13,10 +13,13 @@ processors: - rename: field: message target_field: event.original + - set: + field: event.kind + value: event - set: field: event.timezone value: "{{_tmp.tz_offset}}" - if: ctx?._tmp?.tz_offset != null && ctx?._tmp?.tz_offset != 'local' + if: ctx._tmp?.tz_offset != null && ctx._tmp?.tz_offset != 'local' - grok: description: Parse syslog header field: event.original @@ -31,13 +34,13 @@ processors: OBSERVER: '(?:%{IP:observer.ip}|%{HOSTNAME:observer.name})' PROCESS: '(\(%{DATA:process.name}\)|(%{UNIXPATH}/)?%{WORD:process.name})' - date: - if: ctx?._tmp.timestamp8601 != null + if: ctx._tmp.timestamp8601 != null field: _tmp.timestamp8601 target_field: '@timestamp' formats: - ISO8601 - date: - if: ctx?.event?.timezone != null && ctx?._tmp?.timestamp != null + if: ctx.event?.timezone != null && ctx._tmp?.timestamp != null field: _tmp.timestamp target_field: '@timestamp' formats: @@ -61,7 +64,7 @@ processors: if: ctx.event.provider == 'charon' - pipeline: name: '{{ IngestPipeline "dhcp" }}' - if: ctx.event.provider == 'dhcpd' + if: '["dhcpd", "dhclient", "dhcp6c"].contains(ctx.event.provider)' - pipeline: name: '{{ IngestPipeline "unbound" }}' if: ctx.event.provider == 'unbound' @@ -75,7 +78,31 @@ processors: name: '{{ IngestPipeline "squid" }}' if: ctx.event.provider == 'squid' - drop: - if: '!["filterlog", "openvpn", "charon", "dhcpd", "unbound", "haproxy", "php-fpm", "squid"].contains(ctx?.event?.provider)' + if: '!["filterlog", "openvpn", "charon", "dhcpd", "dhclient", "dhcp6c", "unbound", "haproxy", "php-fpm", "squid"].contains(ctx.event?.provider)' + - append: + field: event.category + value: network + if: "ctx.network != null" + - convert: + field: source.address + target_field: source.ip + type: ip + ignore_failure: true + ignore_missing: true + - convert: + field: destination.address + target_field: destination.ip + type: ip + ignore_failure: true + ignore_missing: true + - set: + field: network.type + value: ipv6 + if: 'ctx.source?.ip != null && ctx.source.ip.contains(":")' + - set: + field: network.type + value: ipv4 + if: 'ctx.source?.ip != null && ctx.source.ip.contains(".")' - geoip: field: source.ip target_field: source.geo @@ -116,33 +143,46 @@ processors: field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true + - community_id: + target_field: network.community_id + ignore_failure: true + - grok: + field: observer.ingress.interface.name + patterns: + - "%{DATA}.%{NONNEGINT:observer.ingress.vlan.id}" + ignore_missing: true + ignore_failure: true + - set: + field: network.vlan.id + copy_from: observer.ingress.vlan.id + ignore_empty_value: true - append: field: related.ip value: "{{destination.ip}}" allow_duplicates: false - if: ctx?.destination?.ip != null + if: ctx.destination?.ip != null - append: field: related.ip value: "{{source.ip}}" allow_duplicates: false - if: ctx?.source?.ip != null + if: ctx.source?.ip != null - append: field: related.ip value: "{{source.nat.ip}}" allow_duplicates: false - if: ctx?.source?.nat?.ip != null + if: ctx.source?.nat?.ip != null - append: field: related.hosts value: "{{destination.domain}}" - if: "ctx?.destination?.domain != null" + if: "ctx.destination?.domain != null" - append: field: related.user value: "{{user.name}}" - if: "ctx?.user?.name != null" + if: "ctx.user?.name != null" - set: field: network.direction value: "{{network.direction}}bound" - if: ctx?.network?.direction != null && ctx?.network?.direction =~ /^(in|out)$/ + if: ctx.network?.direction != null && ctx.network?.direction =~ /^(in|out)$/ - remove: field: - _tmp @@ -173,7 +213,7 @@ processors: handleMap(ctx); - remove: field: event.original - if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))" ignore_failure: true ignore_missing: true on_failure: diff --git a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/dhcp.yml b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/dhcp.yml index f366964fba1..085ac89752c 100644 --- a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/dhcp.yml +++ b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/dhcp.yml @@ -4,7 +4,13 @@ processors: - grok: field: message patterns: - - '%{WORD:event.action} %{MIDDLE} via %{INTERFACE:observer.ingress.interface.name}' + - '%{DATA:_tmp.action}/%{INTERFACE:observer.ingress.interface.name}/%{MAC_ADDRESS:server.mac}/%{NOTSPACE:pfsense.dhcp.subnet}' + - '%{DATA:_tmp.action} %{IPV6:client.address}(/%{NUMBER})? on %{INTERFACE:observer.ingress.interface.name}' + - '%{DATA:_tmp.action} (from|to) %{IPV6:client.address} port %{POSINT:client.port:long}(, transaction ID %{NOTSPACE:pfsense.dhcp.transaction_id})?' + - '%{DATA:_tmp.action} for: %{IPV6:client.address}(, age %{POSINT:pfsense.dhcp.age:long} secs %{DATA})?%{GREEDYDATA}?' + - '%{DATA:_tmp.action}: address %{IPV6:client.address} to client with duid %{DUID:pfsense.dhcp.duid} iaid = -%{NOTSPACE:pfsense.dhcp.iaid} valid for %{POSINT:pfsense.dhcp.lease_time:long} seconds' + - '%{WORD:event.action} %{MIDDLE} via %{INTERFACE:observer.ingress.interface.name}' + - '%{DATA:_tmp.action} %{IPV6:client.address}' - '%{GREEDYDATA}' pattern_definitions: INTERFACE: '[a-z0-9\.]+' @@ -13,47 +19,36 @@ processors: ON: 'on %{IP:client.address} to %{MAC_ADDRESS:client.mac} \(%{HOSTNAME:pfsense.dhcp.hostname}\)' FOR: 'for %{IP:client.address} \(%{IP:server.address}\)? from %{MAC_ADDRESS:client.mac} \(%{HOSTNAME:pfsense.dhcp.hostname}\)' MIDDLE: '(?:%{FROM}|%{ON}|%{FOR})' - - set: - field: event.kind - value: event - - append: - field: event.category - value: network - allow_duplicates: false - - append: - field: event.type - value: connection - allow_duplicates: false + DUID: '(?i)[0-9a-f]{2}(:[0-9a-f]{2})+' - append: field: event.type - value: protocol - allow_duplicates: false - - append: - field: event.type - value: info + value: + - connection + - protocol + - info allow_duplicates: false - set: field: network.protocol value: dhcp + - set: + field: network.protocol + value: dhcpv6 + if: ctx.event.provider == 'dhcp6c' || (ctx.server?.address != null && ctx.server.address.contains(':')) || (ctx.client?.address != null && ctx.client.address.contains(':')) - set: field: network.transport value: udp - - set: - field: client.port - value: 68 - if: ctx?.client?.port == null - - set: - field: server.port - value: 67 - if: ctx?.server?.port == null - - set: - field: client.ip - value: "{{client.address}}" - ignore_empty_value: true - - set: - field: server.ip - value: "{{server.address}}" - ignore_empty_value: true + - convert: + field: client.address + target_field: client.ip + type: ip + ignore_failure: true + ignore_missing: true + - convert: + field: server.address + target_field: server.ip + type: ip + ignore_failure: true + ignore_missing: true - uppercase: field: client.mac ignore_missing: true @@ -62,6 +57,23 @@ processors: pattern: '[:]' replacement: '-' ignore_missing: true + - uppercase: + field: server.mac + ignore_missing: true + - gsub: + field: server.mac + pattern: '[:]' + replacement: '-' + ignore_missing: true + - lowercase: + field: _tmp.action + ignore_missing: true + - gsub: + field: _tmp.action + target_field: event.action + pattern: ' ' + replacement: '-' + ignore_missing: true - set: field: source copy_from: client @@ -70,30 +82,11 @@ processors: field: destination copy_from: server ignore_empty_value: true - - community_id: - target_field: network.community_id - ignore_failure: true - - grok: - field: observer.ingress.interface.name - patterns: - - "%{DATA}.%{NONNEGINT:observer.ingress.vlan.id}" - ignore_missing: true - ignore_failure: true - - append: - field: related.ip - value: "{{source.ip}}" - allow_duplicates: false - if: "ctx?.source?.ip != null" - - append: - field: related.ip - value: "{{destination.ip}}" - allow_duplicates: false - if: "ctx?.destination?.ip != null" - append: field: related.hosts value: "{{pfsense.dhcp.hostname}}" allow_duplicates: false - if: "ctx?.pfsense?.log?.dhcp?.hostname != null" + if: "ctx.pfsense?.log?.dhcp?.hostname != null" on_failure: - append: field: error.message diff --git a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/firewall.yml b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/firewall.yml index 2e28538ca13..eeaa3e01b39 100644 --- a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/firewall.yml +++ b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/firewall.yml @@ -33,55 +33,30 @@ processors: - set: field: event.kind value: event - - append: - field: event.category - value: network - allow_duplicates: false - - set: - field: source.ip - value: "{{source.address}}" - ignore_empty_value: true - - set: - field: destination.ip - value: "{{destination.address}}" - ignore_empty_value: true - append: field: event.type value: connection allow_duplicates: false - if: ctx?.source?.ip != null && ctx?.destination?.ip != null + if: ctx.source?.address != null && ctx.destination?.address != null - append: field: event.type value: denied allow_duplicates: false - if: ctx?.event.action == 'block' + if: ctx.event.action == 'block' - append: field: event.type value: allowed allow_duplicates: false - if: ctx?.event.action == 'pass' - - set: - field: network.type - value: ipv{{network.type}} - if: ctx?.network?.type == '4' || ctx?.network?.type == '6' + if: ctx.event.action == 'pass' - lowercase: field: network.transport ignore_missing: true - remove: field: ack_number ignore_missing: true - if: ctx?.ack_number == null || ctx?.ack_number == '' + if: ctx.ack_number == null || ctx.ack_number == '' - network_direction: internal_networks_field: _tmp.internal_networks - - community_id: - target_field: network.community_id - ignore_failure: true - - grok: - field: observer.ingress.interface.name - patterns: - - "%{DATA}.%{NONNEGINT:observer.ingress.vlan.id}" - ignore_missing: true - ignore_failure: true - split: field: pfsense.tcp.options separator: ';' diff --git a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/haproxy.yml b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/haproxy.yml index 3ae2fb4c69d..d7bb578afbe 100644 --- a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/haproxy.yml +++ b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/haproxy.yml @@ -27,14 +27,14 @@ processors: - drop: description: Drop if not a connection log - date: - if: ctx?.haproxy?.request_date != null && ctx?.event?.timezone == null + if: ctx.haproxy?.request_date != null && ctx.event?.timezone == null field: haproxy.request_date target_field: '@timestamp' formats: - dd/MMM/yyyy:HH:mm:ss.SSS - MMM dd HH:mm:ss - date: - if: ctx?.haproxy?.request_date != null && ctx?.event?.timezone != null + if: ctx.haproxy?.request_date != null && ctx.event?.timezone != null field: haproxy.request_date target_field: '@timestamp' formats: @@ -50,39 +50,7 @@ processors: - uri_parts: field: url.original ignore_failure: true - if: ctx?.url?.original != null - - convert: - field: source.address - target_field: source.ip - type: ip - ignore_failure: true - ignore_missing: true - - convert: - field: destination.address - target_field: destination.ip - type: ip - ignore_failure: true - ignore_missing: true - - geoip: - field: source.ip - target_field: source.geo - ignore_missing: true - - geoip: - database_file: GeoLite2-ASN.mmdb - field: source.ip - target_field: source.as - properties: - - asn - - organization_name - ignore_missing: true - - rename: - field: source.as.asn - target_field: source.as.number - ignore_missing: true - - rename: - field: source.as.organization_name - target_field: source.as.organization.name - ignore_missing: true + if: ctx.url?.original != null - split: field: haproxy.http.request.captured_headers separator: \| @@ -105,29 +73,22 @@ processors: type: long ignore_missing: true if: ctx.containsKey('http') - - set: - field: event.kind - value: event - append: field: event.category value: web - if: "ctx?.haproxy?.mode == 'HTTP' || ctx?.haproxy?.http != null" - - append: - field: event.category - value: network - if: "ctx?.source.ip != null && ctx?.destination?.ip != null" + if: "ctx.haproxy?.mode == 'HTTP' || ctx.haproxy?.http != null" - append: field: event.type value: connection - if: "ctx?.source.ip != null && ctx?.destination?.ip != null" + if: "ctx.source?.address != null && ctx.destination?.address != null" - set: field: event.outcome value: success - if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400" + if: "ctx.http?.response?.status_code != null && ctx.http.response.status_code < 400" - set: field: event.outcome value: failure - if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400" + if: "ctx.http?.response?.status_code != null && ctx.http.response.status_code >= 400" - remove: field: - _temp diff --git a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/ipsec.yml b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/ipsec.yml index 0ee5fae5cea..251b346de2d 100644 --- a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/ipsec.yml +++ b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/ipsec.yml @@ -10,23 +10,16 @@ processors: PREFIX: '\d+\[%{WORD}\]' SOURCE: '%{IP:source.address}\[%{NONNEGINT:source.port:long}\]' DEST: '%{IP:destination.address}\[%{NONNEGINT:destination.port:long}\]' - - set: - field: event.kind - value: event - - append: - field: event.category - value: network - allow_duplicates: false - append: field: event.type value: connection allow_duplicates: false - if: ctx?.source?.address != null + if: ctx.source?.address != null - append: field: event.type value: end allow_duplicates: false - if: ctx?.message.toLowerCase().contains('disconnected') + if: ctx.message.toLowerCase().contains('disconnected') - set: field: source.ip value: "{{source.address}}" diff --git a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/openvpn.yml b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/openvpn.yml index 9afef3ba5de..68e9eec151d 100644 --- a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/openvpn.yml +++ b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/openvpn.yml @@ -13,33 +13,26 @@ processors: pattern_definitions: SOURCE: '%{IP:source.address}:%{NONNEGINT:source.port:long}' USERNAME: '[a-zA-Z0-9._-]+' - - set: - field: event.kind - value: event - - append: - field: event.category - value: network - allow_duplicates: false - append: field: event.category value: authentication allow_duplicates: false - if: ctx?.message.contains('auth') + if: ctx.message.contains('auth') - append: field: event.type value: connection allow_duplicates: false - if: ctx?.source?.address != null + if: ctx.source?.address != null - append: field: event.type value: error allow_duplicates: false - if: ctx?.message.toLowerCase().contains('error') || ctx?.message.toLowerCase().contains('not auth') + if: ctx.message.toLowerCase().contains('error') || ctx.message.toLowerCase().contains('not auth') - append: field: event.type value: start allow_duplicates: false - if: ctx?.message.toLowerCase().contains('initiat') + if: ctx.message.toLowerCase().contains('initiat') - set: field: source.ip value: "{{source.address}}" diff --git a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/php-fpm.yml b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/php-fpm.yml index 7570a1da685..f634f148f19 100644 --- a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/php-fpm.yml +++ b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/php-fpm.yml @@ -11,9 +11,6 @@ processors: PF_APP_LOGIN: "(%{DATA:_tmp.action}) for user '%{USER:user.name}' from: %{IP:source.address} \\(%{DATA}\\)" PF_APP_LOGOUT: "User (%{DATA:_tmp.action}) for user '%{USER:user.name}' from: %{IP:source.address}" PF_APP_ERROR: "webConfigurator %{DATA:_tmp.action} for user '%{DATA:user.name}' from: %{IP:source.address}" - - set: - field: event.kind - value: event - append: field: event.category value: authentication diff --git a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/squid.yml b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/squid.yml index 29e8bbee973..051af25eb30 100644 --- a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/squid.yml +++ b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/squid.yml @@ -9,81 +9,18 @@ processors: - uri_parts: field: url.original ignore_failure: true - if: ctx?.url?.original != null - - convert: - field: source.address - target_field: source.ip - type: ip - ignore_failure: true - ignore_missing: true - - convert: - field: destination.address - target_field: destination.ip - type: ip - ignore_failure: true - ignore_missing: true - - geoip: - field: source.ip - target_field: source.geo - ignore_missing: true - - geoip: - database_file: GeoLite2-ASN.mmdb - field: source.ip - target_field: source.as - properties: - - asn - - organization_name - ignore_missing: true - - rename: - field: source.as.asn - target_field: source.as.number - ignore_missing: true - - rename: - field: source.as.organization_name - target_field: source.as.organization.name - ignore_missing: true -# - split: -# field: haproxy.http.request.captured_headers -# separator: \| -# ignore_failure: true -# ignore_missing: true -# - split: -# field: haproxy.http.response.captured_headers -# separator: \| -# ignore_failure: true -# ignore_missing: true -# - script: -# lang: painless -# source: ctx.event.duration = Math.round(ctx.temp.duration * params.scale) -# params: -# scale: 1000000 -# if: ctx.temp?.duration != null -# - remove: -# field: temp.duration -# ignore_missing: true -# - convert: -# field: haproxy.bytes_read -# target_field: http.response.bytes -# type: long -# ignore_missing: true -# if: ctx.containsKey('http') - - set: - field: event.kind - value: event + if: ctx.url?.original != null - append: field: event.category value: web - - append: - field: event.category - value: network - set: field: event.outcome value: success - if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400" + if: "ctx.http?.response?.status_code != null && ctx.http.response.status_code < 400" - set: field: event.outcome value: failure - if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400" + if: "ctx.http?.response?.status_code != null && ctx.http.response.status_code >= 400" on_failure: - set: field: error.message diff --git a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/unbound.yml b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/unbound.yml index 5bdbdb6d2e6..c534f17bb36 100644 --- a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/unbound.yml +++ b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/unbound.yml @@ -8,30 +8,23 @@ processors: on_failure: - drop: description: Drop if not a query log - - set: - field: event.kind - value: event - - append: - field: event.category - value: network - allow_duplicates: false - append: field: event.type value: connection allow_duplicates: false - if: ctx?.source?.address != null + if: ctx.source?.address != null - append: field: event.type value: end allow_duplicates: false - if: ctx?.message.toLowerCase().contains('disconnected') + if: ctx.message.toLowerCase().contains('disconnected') - set: field: network.protocol value: dns - set: field: dns.type value: question - if: ctx?._tmp?.question?.name != null + if: ctx._tmp?.question?.name != null - registered_domain: field: _tmp.question.name target_field: dns.question @@ -58,12 +51,6 @@ processors: field: client copy_from: source ignore_empty_value: true - - convert: - field: destination.address - target_field: destination.ip - type: ip - ignore_failure: true - ignore_missing: true on_failure: - append: field: error.message diff --git a/packages/pfsense/data_stream/log/fields/ecs.yml b/packages/pfsense/data_stream/log/fields/ecs.yml index 799829902ce..fedc84a2e63 100644 --- a/packages/pfsense/data_stream/log/fields/ecs.yml +++ b/packages/pfsense/data_stream/log/fields/ecs.yml @@ -61,6 +61,8 @@ name: destination.geo.region_name - external: ecs name: destination.ip +- external: ecs + name: destination.mac - external: ecs name: destination.port - external: ecs @@ -146,6 +148,8 @@ name: network.transport - external: ecs name: network.type +- external: ecs + name: network.vlan.id - external: ecs name: observer.ingress.interface.name - external: ecs @@ -176,6 +180,8 @@ name: server.ip - external: ecs name: server.port +- external: ecs + name: server.mac - external: ecs name: source.address - external: ecs diff --git a/packages/pfsense/data_stream/log/fields/fields.yml b/packages/pfsense/data_stream/log/fields/fields.yml index 7a350eba173..4c3e8557856 100644 --- a/packages/pfsense/data_stream/log/fields/fields.yml +++ b/packages/pfsense/data_stream/log/fields/fields.yml @@ -124,10 +124,37 @@ type: long description: | ICMP sequence number. -- name: pfsense.dhcp.hostname - type: keyword - description: | - Hostname of DHCP client +- name: pfsense.dhcp + type: group + fields: + - name: hostname + type: keyword + description: | + Hostname of DHCP client + - name: age + type: long + description: | + Age of DHCP lease in seconds + - name: duid + type: keyword + description: | + The DHCP unique identifier (DUID) is used by a client to get an IP address from a DHCPv6 server. + - name: iaid + type: keyword + description: | + Identity Association Identifier used alongside the DUID to uniquely identify a DHCP client + - name: transaction_id + type: keyword + description: | + The DHCP transaction ID + - name: lease_time + type: long + description: | + The DHCP lease time in seconds + - name: subnet + type: keyword + description: | + The subnet for which the DHCP server is issuing IPs - name: pfsense.openvpn.peer_info type: keyword description: |- diff --git a/packages/pfsense/docs/README.md b/packages/pfsense/docs/README.md index 09c2a1393e1..e54d342ce99 100644 --- a/packages/pfsense/docs/README.md +++ b/packages/pfsense/docs/README.md @@ -233,6 +233,7 @@ An example event for `log` looks as following: | destination.geo.region_iso_code | Region ISO code. | keyword | | destination.geo.region_name | Region name. | keyword | | destination.ip | IP address of the destination (IPv4 or IPv6). | ip | +| destination.mac | MAC address of the destination. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | destination.port | Port of the destination. | long | | dns.question.class | The class of records being queried. | keyword | | dns.question.name | The name being queried. If the name field contains non-printable characters (below 32 or above 126), those characters should be represented as escaped base 10 integers (\DDD). Back slashes and quotes should be escaped. Tabs, carriage returns, and line feeds should be converted to \t, \r, and \n respectively. | keyword | @@ -324,13 +325,20 @@ An example event for `log` looks as following: | network.protocol | In the OSI Model this would be the Application Layer protocol. For example, `http`, `dns`, or `ssh`. The field value must be normalized to lowercase for querying. | keyword | | network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | | network.type | In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc The field value must be normalized to lowercase for querying. | keyword | +| network.vlan.id | VLAN ID as reported by the observer. | keyword | | observer.ingress.interface.name | Interface name as reported by the system. | keyword | | observer.ingress.vlan.id | VLAN ID as reported by the observer. | keyword | | observer.ip | IP addresses of the observer. | ip | | observer.name | Custom name of the observer. This is a name that can be given to an observer. This can be helpful for example if multiple firewalls of the same model are used in an organization. If no custom name is needed, the field can be left empty. | keyword | | observer.type | The type of the observer the data is coming from. There is no predefined list of observer types. Some examples are `forwarder`, `firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`. | keyword | | observer.vendor | Vendor name of the observer. | keyword | +| pfsense.dhcp.age | Age of DHCP lease in seconds | long | +| pfsense.dhcp.duid | The DHCP unique identifier (DUID) is used by a client to get an IP address from a DHCPv6 server. | keyword | | pfsense.dhcp.hostname | Hostname of DHCP client | keyword | +| pfsense.dhcp.iaid | Identity Association Identifier used alongside the DUID to uniquely identify a DHCP client | keyword | +| pfsense.dhcp.lease_time | The DHCP lease time in seconds | long | +| pfsense.dhcp.subnet | The subnet for which the DHCP server is issuing IPs | keyword | +| pfsense.dhcp.transaction_id | The DHCP transaction ID | keyword | | pfsense.icmp.code | ICMP code. | long | | pfsense.icmp.destination.ip | Original destination address of the connection that caused this notification | ip | | pfsense.icmp.id | ID of the echo request/reply | long | @@ -371,6 +379,7 @@ An example event for `log` looks as following: | server.address | Some event server addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | | server.bytes | Bytes sent from the server to the client. | long | | server.ip | IP address of the server (IPv4 or IPv6). | ip | +| server.mac | MAC address of the server. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | server.port | Port of the server. | long | | source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | | source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | diff --git a/packages/pfsense/manifest.yml b/packages/pfsense/manifest.yml index 50e94a223c4..66c62602b3c 100644 --- a/packages/pfsense/manifest.yml +++ b/packages/pfsense/manifest.yml @@ -1,6 +1,6 @@ name: pfsense title: pfSense -version: "1.2.0" +version: "1.3.0" release: ga description: Collect logs from pfSense and OPNsense with Elastic Agent. type: integration