diff --git a/packages/pfsense/changelog.yml b/packages/pfsense/changelog.yml index a82207b850e..c8157b28cd8 100644 --- a/packages/pfsense/changelog.yml +++ b/packages/pfsense/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.0.2" + changes: + - description: Update HAProxy log parsing to handle non HTTPS and TCP logs + type: bugfix + link: https://github.com/elastic/integrations/pull/3504 - version: "1.0.1" changes: - description: Format client.mac as per ECS. diff --git a/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-haproxy.log b/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-haproxy.log index 54fcddbfb0d..15ef9f4e0e2 100644 --- a/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-haproxy.log +++ b/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-haproxy.log @@ -1,3 +1,7 @@ <134>Aug 15 16:15:18 haproxy[41476]: 10.87.93.55:59607 [15/Aug/2021:16:15:18.502] TestFrontend~ TestBackend/TestServer 0/0/0/2/2 400 182 - - ---- 2/2/0/1/0 0/0 "GET /favicon.ico HTTP/1.1" <134>Aug 15 16:17:18 haproxy[41476]: 10.87.93.55:59607 [15/Aug/2021:16:15:18.407] TestFrontend~ TestBackend/TestServer 0/0/0/3/3 400 182 - - ---- 2/2/0/1/0 0/0 "GET /login HTTP/1.1" -<134>Aug 15 16:18:40 haproxy[41476]: 10.87.93.55:58722 [15/Aug/2021:16:15:10.549] TestFrontend~ TestBackend/ -1/-1/-1/-1/30014 408 212 - - cR-- 2/2/0/0/0 0/0 "" \ No newline at end of file +<134>Aug 15 16:18:40 haproxy[41476]: 10.87.93.55:58722 [15/Aug/2021:16:15:10.549] TestFrontend~ TestBackend/ -1/-1/-1/-1/30014 408 212 - - cR-- 2/2/0/0/0 0/0 "" +<134>Jun 13 20:53:10 pfSense haproxy[25571]: 10.0.200.110:50578 [13/Jun/2022:20:53:10.208] TestFrontend TestBackend_ipvANY/TestServer 0/0/0/433/537 200 65018 - - ---- 5/5/0/1/0 0/0 "GET /static/fonts/roboto/Roboto-Bold.woff2 HTTP/1.1" +<134>Jun 13 20:56:55 pfSense haproxy[31709]: 10.0.200.110:50611 [13/Jun/2022:20:56:55.187] TestFrontend-copy TestBackend_ipvANY/TestServer 0/0/204 366 -- 2/2/1/1/0 0/0 +<134>Jun 13 20:53:49 pfSense haproxy[80917]: Connect from 10.50.11.19:50583 to 192.168.75.211:80 (ACME/HTTP) +<134>1 2022-06-14T13:01:42.000000+02:00 haproxy.local haproxy 48723 - - 175.16.199.10:63605 [14/Jun/2022:12:41:42.440] URLS-merged/67.43.156.15:443: Timeout during SSL handshake \ No newline at end of file diff --git a/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-haproxy.log-expected.json b/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-haproxy.log-expected.json index 656f4e508f0..22a058f9e8f 100644 --- a/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-haproxy.log-expected.json +++ b/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-haproxy.log-expected.json @@ -28,7 +28,7 @@ "retries": 0, "server": 1 }, - "frontend_name": "TestFrontend", + "frontend_name": "TestFrontend~", "http": { "request": { "raw_request_line": "GET /favicon.ico HTTP/1.1", @@ -79,7 +79,6 @@ "tags": [ "preserve_original_event" ], - "temp": {}, "url": { "extension": "ico", "original": "/favicon.ico", @@ -114,7 +113,7 @@ "retries": 0, "server": 1 }, - "frontend_name": "TestFrontend", + "frontend_name": "TestFrontend~", "http": { "request": { "raw_request_line": "GET /login HTTP/1.1", @@ -165,7 +164,6 @@ "tags": [ "preserve_original_event" ], - "temp": {}, "url": { "original": "/login", "path": "/login" @@ -199,7 +197,7 @@ "retries": 0, "server": 0 }, - "frontend_name": "TestFrontend", + "frontend_name": "TestFrontend~", "http": { "request": { "raw_request_line": "\u003cBADREQ\u003e", @@ -243,10 +241,292 @@ "ip": "10.87.93.55", "port": 58722 }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2022-06-13T20:53:10.208-04:00", + "ecs": { + "version": "8.2.0" + }, + "event": { + "category": [ + "web" + ], + "duration": 537000000, + "kind": "event", + "original": "\u003c134\u003eJun 13 20:53:10 pfSense haproxy[25571]: 10.0.200.110:50578 [13/Jun/2022:20:53:10.208] TestFrontend TestBackend_ipvANY/TestServer 0/0/0/433/537 200 65018 - - ---- 5/5/0/1/0 0/0 \"GET /static/fonts/roboto/Roboto-Bold.woff2 HTTP/1.1\"", + "outcome": "success", + "provider": "haproxy", + "timezone": "-04:00" + }, + "haproxy": { + "backend_name": "TestBackend_ipvANY", + "backend_queue": 0, + "bytes_read": 65018, + "connection_wait_time_ms": 0, + "connections": { + "active": 5, + "backend": 0, + "frontend": 5, + "retries": 0, + "server": 1 + }, + "frontend_name": "TestFrontend", + "http": { + "request": { + "raw_request_line": "GET /static/fonts/roboto/Roboto-Bold.woff2 HTTP/1.1", + "time_wait_ms": 0, + "time_wait_without_data_ms": 433 + }, + "response": {} + }, + "server_name": "TestServer", + "server_queue": 0, + "termination_state": "----", + "total_waiting_time_ms": 0 + }, + "http": { + "request": { + "method": "GET" + }, + "response": { + "bytes": 65018, + "status_code": 200 + }, + "version": "1.1" + }, + "log": { + "syslog": { + "priority": 134 + } + }, + "message": "10.0.200.110:50578 [13/Jun/2022:20:53:10.208] TestFrontend TestBackend_ipvANY/TestServer 0/0/0/433/537 200 65018 - - ---- 5/5/0/1/0 0/0 \"GET /static/fonts/roboto/Roboto-Bold.woff2 HTTP/1.1\"", + "observer": { + "name": "pfSense", + "type": "firewall", + "vendor": "netgate" + }, + "process": { + "name": "haproxy", + "pid": 25571 + }, + "related": { + "ip": [ + "10.0.200.110" + ] + }, + "source": { + "address": "10.0.200.110", + "ip": "10.0.200.110", + "port": 50578 + }, "tags": [ "preserve_original_event" ], - "temp": {} + "url": { + "extension": "woff2", + "original": "/static/fonts/roboto/Roboto-Bold.woff2", + "path": "/static/fonts/roboto/Roboto-Bold.woff2" + } + }, + { + "@timestamp": "2022-06-13T20:56:55.187-04:00", + "ecs": { + "version": "8.2.0" + }, + "event": { + "duration": 204000000, + "kind": "event", + "original": "\u003c134\u003eJun 13 20:56:55 pfSense haproxy[31709]: 10.0.200.110:50611 [13/Jun/2022:20:56:55.187] TestFrontend-copy TestBackend_ipvANY/TestServer 0/0/204 366 -- 2/2/1/1/0 0/0", + "provider": "haproxy", + "timezone": "-04:00" + }, + "haproxy": { + "backend_name": "TestBackend_ipvANY", + "backend_queue": 0, + "bytes_read": 366, + "connection_wait_time_ms": 0, + "connections": { + "active": 2, + "backend": 1, + "frontend": 2, + "retries": 0, + "server": 1 + }, + "frontend_name": "TestFrontend-copy", + "server_name": "TestServer", + "server_queue": 0, + "termination_state": "--", + "total_waiting_time_ms": 0 + }, + "log": { + "syslog": { + "priority": 134 + } + }, + "message": "10.0.200.110:50611 [13/Jun/2022:20:56:55.187] TestFrontend-copy TestBackend_ipvANY/TestServer 0/0/204 366 -- 2/2/1/1/0 0/0", + "observer": { + "name": "pfSense", + "type": "firewall", + "vendor": "netgate" + }, + "process": { + "name": "haproxy", + "pid": 31709 + }, + "related": { + "ip": [ + "10.0.200.110" + ] + }, + "source": { + "address": "10.0.200.110", + "ip": "10.0.200.110", + "port": 50611 + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2022-06-13T20:53:49.000-04:00", + "destination": { + "address": "192.168.75.211", + "ip": "192.168.75.211", + "port": 80 + }, + "ecs": { + "version": "8.2.0" + }, + "event": { + "category": [ + "web", + "network" + ], + "kind": "event", + "original": "\u003c134\u003eJun 13 20:53:49 pfSense haproxy[80917]: Connect from 10.50.11.19:50583 to 192.168.75.211:80 (ACME/HTTP)", + "provider": "haproxy", + "timezone": "-04:00", + "type": [ + "connection" + ] + }, + "haproxy": { + "frontend_name": "ACME", + "mode": "HTTP" + }, + "log": { + "syslog": { + "priority": 134 + } + }, + "message": "Connect from 10.50.11.19:50583 to 192.168.75.211:80 (ACME/HTTP)", + "observer": { + "name": "pfSense", + "type": "firewall", + "vendor": "netgate" + }, + "process": { + "name": "haproxy", + "pid": 80917 + }, + "related": { + "ip": [ + "192.168.75.211", + "10.50.11.19" + ] + }, + "source": { + "address": "10.50.11.19", + "ip": "10.50.11.19", + "port": 50583 + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2022-06-14T12:41:42.440+02:00", + "destination": { + "address": "67.43.156.15", + "as": { + "number": 35908 + }, + "geo": { + "continent_name": "Asia", + "country_iso_code": "BT", + "country_name": "Bhutan", + "location": { + "lat": 27.5, + "lon": 90.5 + } + }, + "ip": "67.43.156.15", + "port": 443 + }, + "ecs": { + "version": "8.2.0" + }, + "event": { + "category": [ + "network" + ], + "kind": "event", + "original": "\u003c134\u003e1 2022-06-14T13:01:42.000000+02:00 haproxy.local haproxy 48723 - - 175.16.199.10:63605 [14/Jun/2022:12:41:42.440] URLS-merged/67.43.156.15:443: Timeout during SSL handshake", + "provider": "haproxy", + "timezone": "+02:00", + "type": [ + "connection" + ] + }, + "haproxy": { + "bind_name": "67.43.156.15:443", + "error_message": "Timeout during SSL handshake", + "frontend_name": "URLS-merged" + }, + "log": { + "syslog": { + "priority": 134 + } + }, + "message": "175.16.199.10:63605 [14/Jun/2022:12:41:42.440] URLS-merged/67.43.156.15:443: Timeout during SSL handshake", + "observer": { + "name": "haproxy.local", + "type": "firewall", + "vendor": "netgate" + }, + "process": { + "name": "haproxy", + "pid": 48723 + }, + "related": { + "ip": [ + "67.43.156.15", + "175.16.199.10" + ] + }, + "source": { + "address": "175.16.199.10", + "geo": { + "city_name": "Changchun", + "continent_name": "Asia", + "country_iso_code": "CN", + "country_name": "China", + "location": { + "lat": 43.88, + "lon": 125.3228 + }, + "region_iso_code": "CN-22", + "region_name": "Jilin Sheng" + }, + "ip": "175.16.199.10", + "port": 63605 + }, + "tags": [ + "preserve_original_event" + ] } ] } \ No newline at end of file diff --git a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/haproxy.yml b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/haproxy.yml index d499152373f..3ae2fb4c69d 100644 --- a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/haproxy.yml +++ b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/haproxy.yml @@ -4,15 +4,25 @@ processors: - grok: field: message patterns: - - 'Connect from (%{IPORHOST:source.address}|-):%{POSINT:source.port:long} %{WORD} %{IPORHOST:destination.ip}:%{POSINT:destination.port:long} \(%{WORD:haproxy.frontend_name}/%{WORD:haproxy.mode}\)' - - '(%{IPORHOST:source.address}|-):%{POSINT:source.port:long} \[%{NOTSPACE:haproxy.request_date}\] %{WORD:haproxy.frontend_name}~ %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} - %{NUMBER:haproxy.http.request.time_wait_ms:long}/%{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:haproxy.http.request.time_wait_without_data_ms:long}/%{NUMBER:temp.duration:long} + - 'Connect from (%{IPORHOST:source.address}|-):%{POSINT:source.port:long} %{WORD} %{IPORHOST:destination.address}:%{POSINT:destination.port:long} \(%{NOTSPACE:haproxy.frontend_name}/%{WORD:haproxy.mode}\)' + # HTTP(S) + - '(%{IPORHOST:source.address}|-):%{POSINT:source.port:long} \[%{NOTSPACE:haproxy.request_date}\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} + %{NUMBER:haproxy.http.request.time_wait_ms:long}/%{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:haproxy.http.request.time_wait_without_data_ms:long}/%{NUMBER:_temp.duration:long} %{NUMBER:http.response.status_code:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.http.request.captured_cookie} %{NOTSPACE:haproxy.http.response.captured_cookie} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long} (\{%{DATA:haproxy.http.request.captured_headers}\} \{%{DATA:haproxy.http.response.captured_headers}\} |\{%{DATA}\} )?"%{GREEDYDATA:haproxy.http.request.raw_request_line}"' + # TCP + - '(%{IP:source.address}|-):%{POSINT:source.port:long} \[%{NOTSPACE:haproxy.request_date}\] + %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} + %{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:_temp.duration:long} + %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} + %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long}' + # Error + - '(%{IP:source.address}|-):%{POSINT:source.port:long} \[%{NOTSPACE:haproxy.request_date}\] %{NOTSPACE:haproxy.frontend_name}/%{BIND_NAME:haproxy.bind_name}:? %{GREEDYDATA:haproxy.error_message}' ignore_missing: false pattern_definitions: HAPROXY_DATE: (%{MONTHDAY}[/-]%{MONTH}[/-]%{YEAR}:%{HOUR}:%{MINUTE}:%{SECOND})|%{SYSLOGTIMESTAMP} + BIND_NAME: ((%{IP:destination.address})?(:%{POSINT:destination.port:long})?|%{NOTSPACE}) on_failure: - drop: description: Drop if not a connection log @@ -31,8 +41,6 @@ processors: - dd/MMM/yyyy:HH:mm:ss.SSS - MMM dd HH:mm:ss timezone: '{{ event.timezone }}' - - remove: - field: haproxy.request_date - grok: field: haproxy.http.request.raw_request_line patterns: @@ -87,13 +95,10 @@ processors: ignore_missing: true - script: lang: painless - source: ctx.event.duration = Math.round(ctx.temp.duration * params.scale) + source: ctx.event.duration = Math.round(ctx._temp.duration * params.scale) params: scale: 1000000 - if: ctx.temp?.duration != null - - remove: - field: temp.duration - ignore_missing: true + if: ctx._temp?.duration != null - convert: field: haproxy.bytes_read target_field: http.response.bytes @@ -123,6 +128,11 @@ processors: field: event.outcome value: failure if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400" + - remove: + field: + - _temp + - haproxy.request_date + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/pfsense/manifest.yml b/packages/pfsense/manifest.yml index 69b14b9e330..f738f802565 100644 --- a/packages/pfsense/manifest.yml +++ b/packages/pfsense/manifest.yml @@ -1,6 +1,6 @@ name: pfsense title: pfSense Logs -version: "1.0.1" +version: "1.0.2" release: ga description: Collect and parse logs from pfSense and OPNsense devices with Elastic Agent. type: integration