Fix pfsense grok to support new opensense log format

packages/pfsense/_dev/deploy/docker/sample_logs/pfsense.log

@@ -40,4 +40,5 @@
 <134>Jan 1 02:21:38 firewall.opnsense.net filterlog[97530]: 146,,,1535324496,igb1.12,match,block,in,4,0x0,,63,12617,0,DF,6,tcp,60,10.170.12.50,175.16.199.1,49724,853,0,S,1891286705,,64240,,mss;sackOK;TS;nop;wscale
 <134>Dec 31 22:02:55 firewall.opnsense.net filterlog[15307]: 6,,,1000000105,igb1.27,match,block,in,6,0x00,0xf6279,1,UDP,17,32,fe80::208:9bff:fef3:652b,ff02::1:2,546,547,32
 <190>Dec 31 22:06:16 firewall.opnsense.net dhcpd[40152]: DHCPDISCOVER from 4c:55:41:a0:fa:99 via eth0.60
-<30>Dec 31 22:06:33 firewall.opnsense.net unbound[75499]: [26931:0] info: 192.168.1.1 api.opensubtitles.org. A IN
+<30>Dec 31 22:06:33 firewall.opnsense.net unbound[75499]: [26931:0] info: 192.168.1.1 api.opensubtitles.org. A IN
+<134>1 2022-06-09T14:44:11-06:00 OPNsense.example.com filterlog 76404 - [meta sequenceId="1"] 124,,,fae559338f65e11c53669fc3642c93c2,ixl1_vlan70,match,pass,out,4,0x0,,63,5687,0,DF,6,tcp,60,192.168.1.1,10.10.10.10,40370,80,0,S,3364871769,,65535,,mss;sackOK;TS;nop;wscale

packages/pfsense/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.1.1"
+  changes:
+    - description: Fix grok to support new opensense log format
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/3612
 - version: "1.1.0"
   changes:
     - description: Update package to ECS 8.3.0.

packages/pfsense/data_stream/log/_dev/test/pipeline/test-opensense.log

@@ -1,4 +1,5 @@
 <134>Jan 1 02:21:38 firewall.opnsense.net filterlog[97530]: 146,,,1535324496,igb1.12,match,block,in,4,0x0,,63,12617,0,DF,6,tcp,60,10.170.12.50,175.16.199.1,49724,853,0,S,1891286705,,64240,,mss;sackOK;TS;nop;wscale
 <134>Dec 31 22:02:55 firewall.opnsense.net filterlog[15307]: 6,,,1000000105,igb1.27,match,block,in,6,0x00,0xf6279,1,UDP,17,32,fe80::208:9bff:fef3:652b,ff02::1:2,546,547,32
 <190>Dec 31 22:06:16 firewall.opnsense.net dhcpd[40152]: DHCPDISCOVER from 4c:55:41:a0:fa:99 via eth0.60
-<30>Dec 31 22:06:33 firewall.opnsense.net unbound[75499]: [26931:0] info: 192.168.1.1 api.opensubtitles.org. A IN
+<30>Dec 31 22:06:33 firewall.opnsense.net unbound[75499]: [26931:0] info: 192.168.1.1 api.opensubtitles.org. A IN
+<134>1 2022-06-09T14:44:11-06:00 OPNsense.example.com filterlog 76404 - [meta sequenceId="1"] 124,,,fae559338f65e11c53669fc3642c93c2,ixl1_vlan70,match,pass,out,4,0x0,,63,5687,0,DF,6,tcp,60,192.168.1.1,10.10.10.10,40370,80,0,S,3364871769,,65535,,mss;sackOK;TS;nop;wscale

packages/pfsense/data_stream/log/_dev/test/pipeline/test-opensense.log-expected.json

@@ -319,6 +319,98 @@
             "tags": [
                 "preserve_original_event"
             ]
+        },
+        {
+            "@timestamp": "2022-06-09T20:44:11.000Z",
+            "destination": {
+                "address": "10.10.10.10",
+                "ip": "10.10.10.10",
+                "port": 80
+            },
+            "ecs": {
+                "version": "8.3.0"
+            },
+            "event": {
+                "action": "pass",
+                "category": [
+                    "network"
+                ],
+                "kind": "event",
+                "original": "\u003c134\u003e1 2022-06-09T14:44:11-06:00 OPNsense.example.com filterlog 76404 - [meta sequenceId=\"1\"] 124,,,fae559338f65e11c53669fc3642c93c2,ixl1_vlan70,match,pass,out,4,0x0,,63,5687,0,DF,6,tcp,60,192.168.1.1,10.10.10.10,40370,80,0,S,3364871769,,65535,,mss;sackOK;TS;nop;wscale",
+                "provider": "filterlog",
+                "reason": "match",
+                "timezone": "-06:00",
+                "type": [
+                    "connection",
+                    "allowed"
+                ]
+            },
+            "log": {
+                "syslog": {
+                    "priority": 134
+                }
+            },
+            "message": "124,,,fae559338f65e11c53669fc3642c93c2,ixl1_vlan70,match,pass,out,4,0x0,,63,5687,0,DF,6,tcp,60,192.168.1.1,10.10.10.10,40370,80,0,S,3364871769,,65535,,mss;sackOK;TS;nop;wscale",
+            "network": {
+                "bytes": 60,
+                "community_id": "1:DAGv22V0JdwDZ4VIyfmfWGCGjco=",
+                "direction": "out",
+                "iana_number": "6",
+                "transport": "tcp",
+                "type": "ipv4"
+            },
+            "observer": {
+                "ingress": {
+                    "interface": {
+                        "name": "ixl1_vlan70"
+                    }
+                },
+                "name": "OPNsense.example.com",
+                "type": "firewall",
+                "vendor": "netgate"
+            },
+            "pfsense": {
+                "ip": {
+                    "flags": "DF",
+                    "id": 5687,
+                    "offset": 0,
+                    "tos": "0x0",
+                    "ttl": 63
+                },
+                "tcp": {
+                    "flags": "S",
+                    "length": 0,
+                    "options": [
+                        "mss",
+                        "sackOK",
+                        "TS",
+                        "nop",
+                        "wscale"
+                    ],
+                    "window": 65535
+                }
+            },
+            "process": {
+                "name": "filterlog",
+                "pid": 76404
+            },
+            "related": {
+                "ip": [
+                    "10.10.10.10",
+                    "192.168.1.1"
+                ]
+            },
+            "rule": {
+                "id": "fae559338f65e11c53669fc3642c93c2"
+            },
+            "source": {
+                "address": "192.168.1.1",
+                "ip": "192.168.1.1",
+                "port": 40370
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
         }
     ]
 }

packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -25,7 +25,7 @@ processors:
       pattern_definitions:
         ECS_SYSLOG_PRI: '<%{NONNEGINT:log.syslog.priority:long}>(\d )?'
         BSD_TIMESTAMP_FORMAT: '%{SYSLOGTIMESTAMP:_tmp.timestamp}(%{SPACE}%{OBSERVER})?%{SPACE}%{PROCESS}(\[%{POSINT:process.pid:long}\])?:'
-        SYSLOG_TIMESTAMP_FORMAT: '%{TIMESTAMP_ISO8601:_tmp.timestamp8601}%{SPACE}%{OBSERVER}%{SPACE}%{PROCESS}%{SPACE}(%{POSINT:process.pid:long}|-) - -'
+        SYSLOG_TIMESTAMP_FORMAT: '%{TIMESTAMP_ISO8601:_tmp.timestamp8601}%{SPACE}%{OBSERVER}%{SPACE}%{PROCESS}%{SPACE}(%{POSINT:process.pid:long}|-) - (-|\[%{DATA}\])?'
         TIMESTAMP_ISO8601: '%{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE:event.timezone}?'
         TIMESTAMP: '(?:%{BSD_TIMESTAMP_FORMAT}|%{SYSLOG_TIMESTAMP_FORMAT})'
         OBSERVER: '(?:%{IP:observer.ip}|%{HOSTNAME:observer.name})'

packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/firewall.yml

@@ -7,7 +7,7 @@ processors:
         - "%{PF_LOG_ENTRY}%{GREEDYDATA}"
         pattern_definitions:
             PF_LOG_ENTRY: "%{PF_LOG_DATA}%{PF_IP_SPECIFIC_DATA}%{PF_IP_DATA}%{PF_PROTOCOL_DATA}?"
-            PF_LOG_DATA: "%{INT},%{INT}?,,%{INT:rule.id},%{DATA:observer.ingress.interface.name},%{PF_REASON:event.reason},%{WORD:event.action},%{WORD:network.direction},"
+            PF_LOG_DATA: "%{INT},%{INT}?,,%{WORD:rule.id},%{DATA:observer.ingress.interface.name},%{PF_REASON:event.reason},%{WORD:event.action},%{WORD:network.direction},"
             PF_REASON: '[a-zA-Z-]+'
             PF_IP_DATA: "%{NONNEGINT:network.bytes:long},%{IP:source.address},%{IP:destination.address},"
             PF_IP_SPECIFIC_DATA: "%{PF_IPv4_SPECIFIC_DATA}|%{PF_IPv6_SPECIFIC_DATA}"

packages/pfsense/data_stream/log/sample_event.json

@@ -1,8 +1,8 @@
 {
     "@timestamp": "2021-07-04T00:10:14.578Z",
     "agent": {
-        "ephemeral_id": "238d98ab-083f-4ff7-990f-1651450ce860",
-        "id": "584f3aea-648c-4e58-aba4-32b8f88d4396",
+        "ephemeral_id": "54ce1a5f-64b9-4475-9d01-4d9fb46c22ba",
+        "id": "1db51880-bfd3-4297-9dd1-f3def809da25",
         "name": "docker-fleet-agent",
         "type": "filebeat",
         "version": "8.2.0"
@@ -33,7 +33,7 @@
         "version": "8.3.0"
     },
     "elastic_agent": {
-        "id": "584f3aea-648c-4e58-aba4-32b8f88d4396",
+        "id": "1db51880-bfd3-4297-9dd1-f3def809da25",
         "snapshot": false,
         "version": "8.2.0"
     },
@@ -44,9 +44,9 @@
             "network"
         ],
         "dataset": "pfsense.log",
-        "ingested": "2022-05-09T17:35:12Z",
+        "ingested": "2022-06-29T13:24:24Z",
         "kind": "event",
-        "original": "\u003c134\u003e1 2021-07-03T19:10:14.578288-05:00 pfSense.example.com filterlog 72237 - - 146,,,1535324496,igb1.12,match,block,in,4,0x0,,63,32989,0,DF,6,tcp,60,10.170.12.50,175.16.199.1,49652,853,0,S,1818117648,,64240,,mss;sackOK;TS;nop;wscale\n",
+        "original": "\u003c134\u003e1 2021-07-03T19:10:14.578288-05:00 pfSense.example.com filterlog 72237 - - 146,,,1535324496,igb1.12,match,block,in,4,0x0,,63,32989,0,DF,6,tcp,60,10.170.12.50,175.16.199.1,49652,853,0,S,1818117648,,64240,,mss;sackOK;TS;nop;wscale",
         "provider": "filterlog",
         "reason": "match",
         "timezone": "-05:00",
@@ -56,11 +56,11 @@
         ]
     },
     "input": {
-        "type": "udp"
+        "type": "tcp"
     },
     "log": {
         "source": {
-            "address": "172.18.0.4:58663"
+            "address": "192.168.128.4:52326"
         },
         "syslog": {
             "priority": 134
@@ -70,7 +70,7 @@
     "network": {
         "bytes": 60,
         "community_id": "1:pOXVyPJTFJI5seusI/UD6SwvBjg=",
-        "direction": "outbound",
+        "direction": "in",
         "iana_number": "6",
         "transport": "tcp",
         "type": "ipv4"

packages/pfsense/manifest.yml

@@ -1,6 +1,6 @@
 name: pfsense
 title: pfSense Logs
-version: "1.1.0"
+version: "1.1.1"
 release: ga
 description: Collect and parse logs from pfSense and OPNsense devices with Elastic Agent.
 type: integration